@opengeni/runtime 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-DYBF7ZEB.js → chunk-RYDCVTMW.js} +1958 -207
- package/dist/chunk-RYDCVTMW.js.map +1 -0
- package/dist/{index-BTo_bZvN.d.ts → index-DIiiwQY2.d.ts} +689 -52
- package/dist/index.d.ts +296 -123
- package/dist/index.js +1907 -809
- package/dist/index.js.map +1 -1
- package/dist/sandbox/index.d.ts +1 -1
- package/dist/sandbox/index.js +79 -3
- package/package.json +21 -20
- package/src/codex-tool-search.ts +185 -31
- package/src/context-compaction.ts +78 -243
- package/src/history-sanitizer.ts +172 -20
- package/src/index.ts +2687 -920
- package/src/metrics.ts +28 -2
- package/src/sandbox/capabilities.ts +12 -3
- package/src/sandbox/channel-a.ts +420 -72
- package/src/sandbox/display-stack.ts +18 -4
- package/src/sandbox/errors.ts +13 -0
- package/src/sandbox/index.ts +398 -112
- package/src/sandbox/op-correlation.ts +70 -0
- package/src/sandbox/providers/index.ts +9 -3
- package/src/sandbox/providers/modal.ts +84 -14
- package/src/sandbox/providers/selfhosted.ts +1 -4
- package/src/sandbox/recording.ts +146 -90
- package/src/sandbox/routing/backend-resolver.ts +144 -10
- package/src/sandbox/routing/routing-session.ts +129 -25
- package/src/sandbox/select.ts +60 -8
- package/src/sandbox/selfhosted/capabilities.ts +59 -25
- package/src/sandbox/selfhosted/control-rpc.ts +160 -28
- package/src/sandbox/selfhosted/fault-rendering.ts +226 -0
- package/src/sandbox/selfhosted/op-observer.ts +54 -0
- package/src/sandbox/selfhosted/op-stream.ts +801 -0
- package/src/sandbox/selfhosted/op-testing.ts +468 -0
- package/src/sandbox/selfhosted/op-transport.ts +151 -0
- package/src/sandbox/selfhosted/retry-policy.ts +177 -0
- package/src/sandbox/selfhosted/session.ts +498 -55
- package/src/sandbox/selfhosted/testing.ts +60 -19
- package/src/sandbox/stream-port.ts +17 -5
- package/src/sandbox/stream-token.ts +8 -2
- package/src/sandbox/terminal-server.ts +18 -4
- package/src/sandbox-computer.ts +332 -70
- package/src/usage-telemetry.ts +73 -0
- package/dist/chunk-DYBF7ZEB.js.map +0 -1
- package/src/image-history.ts +0 -149
- package/src/screenshot-omitted-card.ts +0 -15
|
@@ -2,7 +2,7 @@ import { Settings } from '@opengeni/config';
|
|
|
2
2
|
import { SandboxBackend, CapabilityDescriptor, SandboxOs, SessionCapabilities, StreamTokenPayload, SessionEventType, SessionStructuredCapabilities, FsListRequest, FsListResponse, FsReadRequest, FsReadResponse, FsWriteRequest, FsWriteResponse, FsDeleteRequest, FsDeleteResponse, FsMoveRequest, FsMoveResponse, FsMkdirRequest, FsMkdirResponse, GitStatusRequest, GitStatusResponse, GitDiffRequest, GitDiffResponse, GitLogRequest, GitLogResponse, GitShowRequest, GitShowResponse, TerminalExecRequest, TerminalExecResponse, PtyOpenRequest, PtyOpenResponse, PtyWriteRequest, PtyResizeRequest, PtyCloseRequest, GitChangedPayload, GitDiffHunk, GitFileStatusCode, CapabilityUnavailableReason } from '@opengeni/contracts';
|
|
3
3
|
import { Manifest, SandboxClient, SandboxSessionState } from '@openai/agents/sandbox';
|
|
4
4
|
import * as modal from 'modal';
|
|
5
|
-
import { ControlRequest, ControlResponse, ErrorCode, AgentError, DesktopInputRequest, ExecRequest
|
|
5
|
+
import { ControlRequest, ControlResponse, ErrorCode, AgentError, ExecResponse, DesktopInputRequest, ExecRequest } from '@opengeni/agent-proto';
|
|
6
6
|
|
|
7
7
|
type RuntimeMetricsHooks = {
|
|
8
8
|
onModelCall?: (input: {
|
|
@@ -18,6 +18,24 @@ type RuntimeMetricsHooks = {
|
|
|
18
18
|
onSandboxWarmingTimeout?: (input: {
|
|
19
19
|
backend: string;
|
|
20
20
|
}) => void;
|
|
21
|
+
/**
|
|
22
|
+
* One completed Connected Machine (selfhosted) control op — the out-of-band
|
|
23
|
+
* telemetry twin of the in-band fault rendering. `code` is the typed wire-code
|
|
24
|
+
* NAME on a failure (bounded label cardinality); `healed` marks a success that
|
|
25
|
+
* only landed after ≥1 retry (the leading indicator of the next unhealed fault);
|
|
26
|
+
* `replyBytes` is set only on a payload-wall fault. Wired from the runtime's
|
|
27
|
+
* `SelfhostedOpObserver` seam.
|
|
28
|
+
*/
|
|
29
|
+
onSandboxOp?: (input: {
|
|
30
|
+
backend: string;
|
|
31
|
+
op: string;
|
|
32
|
+
outcome: "ok" | "failed";
|
|
33
|
+
code?: string;
|
|
34
|
+
healed: boolean;
|
|
35
|
+
retries: number;
|
|
36
|
+
durationSeconds: number;
|
|
37
|
+
replyBytes?: number;
|
|
38
|
+
}) => void;
|
|
21
39
|
};
|
|
22
40
|
|
|
23
41
|
/**
|
|
@@ -33,6 +51,13 @@ declare class SandboxConfigError extends Error {
|
|
|
33
51
|
readonly backend: SandboxBackend | string;
|
|
34
52
|
constructor(backend: SandboxBackend | string, message: string);
|
|
35
53
|
}
|
|
54
|
+
/** A resume-only caller was handed a warm lease without a provider identity it
|
|
55
|
+
* can resume. The lease-aware caller must retire that exact epoch and re-enter
|
|
56
|
+
* admission; silently creating here would bypass the single-spawner guard. */
|
|
57
|
+
declare class SandboxResumeStateUnavailableError extends Error {
|
|
58
|
+
readonly backend: SandboxBackend | string;
|
|
59
|
+
constructor(backend: SandboxBackend | string);
|
|
60
|
+
}
|
|
36
61
|
declare class SandboxProviderUnavailableError extends Error {
|
|
37
62
|
readonly backend: SandboxBackend | string;
|
|
38
63
|
constructor(backend: SandboxBackend | string);
|
|
@@ -499,7 +524,7 @@ declare function extForCodec(codec: RecordingCodec): string;
|
|
|
499
524
|
declare class RecordingUnavailableError extends Error {
|
|
500
525
|
constructor(message: string);
|
|
501
526
|
}
|
|
502
|
-
/** ffmpeg failed, the file is missing, or the
|
|
527
|
+
/** ffmpeg failed, the file is missing, or the artifact is over its size limit. */
|
|
503
528
|
declare class RecordingError extends Error {
|
|
504
529
|
readonly reason: "ffmpeg-error" | "box-death" | "max-bytes-exceeded" | "display-unavailable";
|
|
505
530
|
constructor(message: string, reason: "ffmpeg-error" | "box-death" | "max-bytes-exceeded" | "display-unavailable");
|
|
@@ -539,37 +564,29 @@ declare function startRecording(session: unknown, input: StartRecordingInput): P
|
|
|
539
564
|
* pid file is a no-op.
|
|
540
565
|
*/
|
|
541
566
|
declare function stopRecording(session: unknown, proc: RecordingProcess): Promise<void>;
|
|
542
|
-
type
|
|
543
|
-
bytes: Uint8Array;
|
|
567
|
+
type RecordingArtifactMetadata = {
|
|
544
568
|
contentType: RecordingContentType;
|
|
545
569
|
sizeBytes: number;
|
|
546
570
|
durationSeconds: number;
|
|
547
571
|
};
|
|
548
572
|
/**
|
|
549
|
-
*
|
|
550
|
-
*
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
*
|
|
555
|
-
*
|
|
556
|
-
*
|
|
557
|
-
*
|
|
558
|
-
*
|
|
559
|
-
* truncates a large recording's base64.
|
|
560
|
-
*
|
|
561
|
-
* F8: we DO NOT assume any over-limit behavior. First `stat` the file size on the
|
|
562
|
-
* box; if it exceeds maxBytes, fail `max-bytes-exceeded` (never upload a truncated
|
|
563
|
-
* video). Otherwise read the raw bytes.
|
|
564
|
-
*
|
|
565
|
-
* F9: this does NOT delete the box file. The caller deletes it (deleteRecordingArtifacts)
|
|
566
|
-
* ONLY after the storage PUT + `available` commit — so a failed upload leaves the
|
|
567
|
-
* bytes recoverable on the box for a retry.
|
|
568
|
-
*
|
|
569
|
-
* F14: duration is wall-clock (now − startedAt), a close approximation of the
|
|
570
|
-
* SIGINT-flushed video length.
|
|
573
|
+
* Inspect and size-gate the finalized recording while it is still on the box.
|
|
574
|
+
* No recording bytes cross into the worker process.
|
|
575
|
+
*/
|
|
576
|
+
declare function inspectRecordingArtifact(session: unknown, proc: RecordingProcess, maxBytes?: number): Promise<RecordingArtifactMetadata>;
|
|
577
|
+
/**
|
|
578
|
+
* Upload the finalized artifact directly from the sandbox to a scoped object-
|
|
579
|
+
* storage URL. The worker carries only the short URL/headers and metadata; it
|
|
580
|
+
* never materializes a base64 string or Uint8Array for a recording that may be
|
|
581
|
+
* hundreds of megabytes. curl's success marker makes an exec transport that
|
|
582
|
+
* returns non-zero command output instead of throwing still fail closed.
|
|
571
583
|
*/
|
|
572
|
-
declare function
|
|
584
|
+
declare function uploadRecordingArtifact(session: unknown, proc: RecordingProcess, input: {
|
|
585
|
+
url: string;
|
|
586
|
+
requiredHeaders: Record<string, string>;
|
|
587
|
+
maxBytes?: number;
|
|
588
|
+
maxSeconds?: number;
|
|
589
|
+
}): Promise<RecordingArtifactMetadata>;
|
|
573
590
|
/**
|
|
574
591
|
* Delete the box artifacts. F9: call this ONLY after the storage PUT confirmed
|
|
575
592
|
* and the `available` row committed — never before. Best-effort; never throws.
|
|
@@ -642,6 +659,13 @@ type SandboxChannelAServiceOptions = {
|
|
|
642
659
|
emit?: ChannelAEmitter;
|
|
643
660
|
runAs?: string;
|
|
644
661
|
};
|
|
662
|
+
declare const REPOSITORY_DISCOVERY_LIMIT = 256;
|
|
663
|
+
type RepositoryDiscoveryDegradedReason = "command_failed" | "command_timed_out" | "result_limit_exceeded";
|
|
664
|
+
type RepositoryDiscoveryResult = {
|
|
665
|
+
repos: string[];
|
|
666
|
+
complete: boolean;
|
|
667
|
+
degradedReason: RepositoryDiscoveryDegradedReason | null;
|
|
668
|
+
};
|
|
645
669
|
declare class SandboxChannelAService {
|
|
646
670
|
private readonly session;
|
|
647
671
|
private readonly workspaceRoot;
|
|
@@ -670,6 +694,22 @@ declare class SandboxChannelAService {
|
|
|
670
694
|
gitDiff(req: GitDiffRequest): Promise<GitDiffResponse>;
|
|
671
695
|
gitLog(req: GitLogRequest): Promise<GitLogResponse>;
|
|
672
696
|
gitShow(req: GitShowRequest): Promise<GitShowResponse>;
|
|
697
|
+
/**
|
|
698
|
+
* Detect repo roots within every supported workspace layout.
|
|
699
|
+
*
|
|
700
|
+
* The platform seeds GitHub repositories at
|
|
701
|
+
* `repos/<owner>/<repo>`, which puts the `.git` marker at depth four from the
|
|
702
|
+
* workspace root. Do not reintroduce a fixed maxdepth here. Traversal is
|
|
703
|
+
* bounded instead by pruning known machine/build residue, a wall-clock
|
|
704
|
+
* timeout, and a result limit. `.git` may be either a directory (ordinary
|
|
705
|
+
* clone/submodule) or a file (linked worktree).
|
|
706
|
+
*
|
|
707
|
+
* Callers that persist an authoritative snapshot must use this detailed
|
|
708
|
+
* result and surface an incomplete discovery. The compact `detectRepos()`
|
|
709
|
+
* wrapper remains for capability negotiation, where the best available list
|
|
710
|
+
* is preferable to failing the whole capabilities document.
|
|
711
|
+
*/
|
|
712
|
+
detectReposDetailed(): Promise<RepositoryDiscoveryResult>;
|
|
673
713
|
/** Detect repo roots within the workspace (for the Git.repos capability). */
|
|
674
714
|
detectRepos(): Promise<string[]>;
|
|
675
715
|
/** Run a bounded command, return buffered stdout/stderr + exit code inline. The
|
|
@@ -702,6 +742,7 @@ declare class SandboxChannelAService {
|
|
|
702
742
|
currentRevision(): number;
|
|
703
743
|
private joinRoot;
|
|
704
744
|
private repoWorkdir;
|
|
745
|
+
private terminalWorkdir;
|
|
705
746
|
private emitEvents;
|
|
706
747
|
private emitFsChanged;
|
|
707
748
|
/** Re-probe git after a mutation and emit git.changed (best-effort, used by the
|
|
@@ -776,6 +817,15 @@ declare class SelfhostedControlError extends Error {
|
|
|
776
817
|
readonly draining: boolean;
|
|
777
818
|
readonly agentOffline: boolean;
|
|
778
819
|
readonly osNotFound: boolean;
|
|
820
|
+
/** The reply exceeded the transport's max payload (a distinguishing flag so the
|
|
821
|
+
* fault renderer can name the size wall + recovery instead of a generic OS error). */
|
|
822
|
+
readonly payloadTooLarge: boolean;
|
|
823
|
+
/** The transport synthesized this fault PRE-SEND — it KNOWS the request never
|
|
824
|
+
* reached the machine (no connection, or no responder on the subject), so the op
|
|
825
|
+
* provably did not execute. Only such a fault is safe to re-issue for ANY op kind
|
|
826
|
+
* (a post-send timeout is ambiguous — at-least-once — and must not be re-sent
|
|
827
|
+
* unless the op is read-only). Never set for an agent-returned error. */
|
|
828
|
+
readonly neverSent: boolean;
|
|
779
829
|
readonly detail: Record<string, string>;
|
|
780
830
|
constructor(input: {
|
|
781
831
|
message: string;
|
|
@@ -786,6 +836,8 @@ declare class SelfhostedControlError extends Error {
|
|
|
786
836
|
draining?: boolean;
|
|
787
837
|
agentOffline?: boolean;
|
|
788
838
|
osNotFound?: boolean;
|
|
839
|
+
payloadTooLarge?: boolean;
|
|
840
|
+
neverSent?: boolean;
|
|
789
841
|
detail?: Record<string, string>;
|
|
790
842
|
});
|
|
791
843
|
}
|
|
@@ -804,13 +856,45 @@ declare class SelfhostedControlError extends Error {
|
|
|
804
856
|
* real "file does not exist"), no machine-liveness
|
|
805
857
|
* reason. (This is the ONLY NotFound; it is NOT the
|
|
806
858
|
* box-gone NotFound that licenses a cold restore.)
|
|
859
|
+
* - PAYLOAD_TOO_LARGE → the reply exceeded the transport's max payload;
|
|
860
|
+
* non-retryable, with actionable copy (the sizes + a
|
|
861
|
+
* redirect-to-file workaround).
|
|
807
862
|
* - OS / UNSUPPORTED / STREAM / PROTOCOL / UNSPECIFIED → op-level error, no
|
|
808
863
|
* reason, non-retryable.
|
|
809
864
|
*/
|
|
810
865
|
declare function agentErrorToControlError(err: AgentError): SelfhostedControlError;
|
|
866
|
+
/**
|
|
867
|
+
* PAYLOAD_TOO_LARGE copy: state the actual sizes (from the agent's `detail` map —
|
|
868
|
+
* `encoded_bytes` / `max_payload`) and tell the caller how to get the data anyway:
|
|
869
|
+
* redirect the output to a file and read it back in ranges/chunks.
|
|
870
|
+
*/
|
|
871
|
+
declare function payloadTooLargeMessage(detail: Record<string, string>): string;
|
|
872
|
+
/**
|
|
873
|
+
* DRAINING copy: the machine is at its concurrent-work capacity. `retries` is the
|
|
874
|
+
* number of times `SelfhostedSession.call` already re-tried before giving up (0 at
|
|
875
|
+
* the mapping layer, the final count when it surfaces after exhausting retries).
|
|
876
|
+
*/
|
|
877
|
+
declare function drainingMessage(retries: number): string;
|
|
878
|
+
/** Rebuild a DRAINING error with the final retry count folded into its message —
|
|
879
|
+
* used by `SelfhostedSession.call` when the bounded DRAINING retries are exhausted. */
|
|
880
|
+
declare function drainingExhaustedError(base: SelfhostedControlError, retries: number): SelfhostedControlError;
|
|
881
|
+
/**
|
|
882
|
+
* exec-deadline copy: the command was terminated at the exec time limit. Advise
|
|
883
|
+
* running long jobs in the background and polling their output rather than blocking
|
|
884
|
+
* the turn on one long command. Surfaced on a timed-out exec result's stderr.
|
|
885
|
+
*/
|
|
886
|
+
declare function execDeadlineHint(seconds: number): string;
|
|
887
|
+
/** The reserved `detail` key the transport sets to "1" when it KNOWS a synthesized
|
|
888
|
+
* AGENT_OFFLINE fault occurred pre-send (no connection / no responder) — so the op
|
|
889
|
+
* provably never reached the machine. `agentErrorToControlError` reads it into
|
|
890
|
+
* `SelfhostedControlError.neverSent`. Reserved (never sent by the agent). */
|
|
891
|
+
declare const NEVER_SENT_DETAIL_KEY = "_never_sent";
|
|
811
892
|
/** Build a synthesized AGENT_OFFLINE `AgentError` — the control plane uses this
|
|
812
|
-
* when no agent responds on the subject at all.
|
|
813
|
-
|
|
893
|
+
* when no agent responds on the subject at all. `neverSent` marks that the request
|
|
894
|
+
* provably never reached the machine (no connection / no responder), which makes it
|
|
895
|
+
* safe to re-issue for ANY op kind; leave it false for an AMBIGUOUS transport error
|
|
896
|
+
* (a connection torn mid-request may have delivered the op). */
|
|
897
|
+
declare function offlineAgentError(message?: string, neverSent?: boolean): AgentError;
|
|
814
898
|
/** Build a synthesized TIMEOUT `AgentError` — the control plane uses this when a
|
|
815
899
|
* responder existed but the request timed out (a transient blip → reconnecting). */
|
|
816
900
|
declare function timeoutAgentError(message?: string): AgentError;
|
|
@@ -844,17 +928,308 @@ interface NatsRequestConnection {
|
|
|
844
928
|
declare class NatsControlRpc implements ControlRpc {
|
|
845
929
|
private readonly connect;
|
|
846
930
|
private connection;
|
|
931
|
+
private connecting;
|
|
847
932
|
constructor(connect: () => Promise<NatsRequestConnection | null>);
|
|
848
933
|
private resolveConnection;
|
|
849
934
|
request(subject: string, req: ControlRequest, opts: {
|
|
850
935
|
timeoutMs: number;
|
|
851
936
|
}): Promise<ControlResponse>;
|
|
852
937
|
}
|
|
853
|
-
/** A `ControlResponse` carrying a synthesized AGENT_OFFLINE error.
|
|
854
|
-
|
|
938
|
+
/** A `ControlResponse` carrying a synthesized AGENT_OFFLINE error. `neverSent` marks
|
|
939
|
+
* a pre-send synthesis (no connection / no responder) — the op provably never
|
|
940
|
+
* reached the machine, so it is safe to re-issue for any op kind. */
|
|
941
|
+
declare function offlineControlResponse(requestId: string, neverSent?: boolean): ControlResponse;
|
|
855
942
|
/** A `ControlResponse` carrying a synthesized TIMEOUT error (→ reconnecting). */
|
|
856
943
|
declare function timeoutControlResponse(requestId: string): ControlResponse;
|
|
857
944
|
|
|
945
|
+
/**
|
|
946
|
+
* One completed control-op observation. `outcome` is the terminal result AFTER any
|
|
947
|
+
* in-call retries: `ok` (the op succeeded) or `failed` (a terminal fault surfaced).
|
|
948
|
+
* `healed` marks a success that only landed after ≥1 retry — the doctrine's
|
|
949
|
+
* "healed and invisible in-band, always out-of-band" breadcrumb (drives
|
|
950
|
+
* `machine.op.recovered`). Every field is derivable at the `SelfhostedSession.call`
|
|
951
|
+
* exit points with no extra round-trip.
|
|
952
|
+
*/
|
|
953
|
+
interface SelfhostedOpObservation {
|
|
954
|
+
/** The control-op kind (`ControlRequest["op"].$case`): exec / fsRead / ping / … */
|
|
955
|
+
op: string;
|
|
956
|
+
outcome: "ok" | "failed";
|
|
957
|
+
/** True when `outcome==="ok"` only after ≥1 in-call retry (draining/timeout/never-sent). */
|
|
958
|
+
healed: boolean;
|
|
959
|
+
/** Total in-call retries before this outcome (summed across the retry classes). */
|
|
960
|
+
retries: number;
|
|
961
|
+
/** Wall time from the first attempt to the terminal outcome, in ms. */
|
|
962
|
+
durationMs: number;
|
|
963
|
+
/** The typed wire code — present on `failed`. */
|
|
964
|
+
code?: ErrorCode;
|
|
965
|
+
/** The negotiated capability reason (agent_offline / agent_reconnecting / …), if any. */
|
|
966
|
+
reason?: SelfhostedUnavailableReason | null;
|
|
967
|
+
/** Whether a failed fault provably never reached the machine (a pre-send synthesis). */
|
|
968
|
+
neverSent?: boolean;
|
|
969
|
+
/** The enrolled machine (agent) id the op addressed — for machine-scoped fan-out. */
|
|
970
|
+
machineId?: string;
|
|
971
|
+
/** A stable fault-class string (`selfhostedFaultClass`): the fault class on a
|
|
972
|
+
* `failed` op, or the class of retries that a `healed` op recovered from. Absent
|
|
973
|
+
* on a clean success. Drives the `machine.op.*` event mapping + its infra filter. */
|
|
974
|
+
faultClass?: string;
|
|
975
|
+
/** The reply size in bytes when known — set on a PAYLOAD_TOO_LARGE fault (from the
|
|
976
|
+
* agent's `encoded_bytes` detail). The op-stream client fills the success-reply
|
|
977
|
+
* size once it owns the framed transport; today's request/reply does not surface it. */
|
|
978
|
+
replyBytes?: number;
|
|
979
|
+
}
|
|
980
|
+
/** A fire-and-forget observer of completed control ops. It MUST NOT throw (the
|
|
981
|
+
* session guards the call) and MUST NOT block the op — it is a telemetry tap. */
|
|
982
|
+
type SelfhostedOpObserver = (observation: SelfhostedOpObservation) => void;
|
|
983
|
+
|
|
984
|
+
/**
|
|
985
|
+
* A stable, low-cardinality fault-class string for a `SelfhostedControlError` — the
|
|
986
|
+
* out-of-band twin of the in-band renderer's class taxonomy (single-sourced here).
|
|
987
|
+
* Used for the `machine.op.*` event class + the metrics label + the infra filter.
|
|
988
|
+
*/
|
|
989
|
+
declare function selfhostedFaultClass(error: SelfhostedControlError): string;
|
|
990
|
+
/**
|
|
991
|
+
* The fault classes that count as an INFRASTRUCTURE fault (a `machine.op.failed`
|
|
992
|
+
* event). A semantic miss the model asked about — a missing path (`not_found`), a
|
|
993
|
+
* consent gate (`consent`), a nonzero exit (never a fault at all) — is an OUTCOME,
|
|
994
|
+
* not an infra fault, and is DELIBERATELY excluded; `fenced` is a routing/session
|
|
995
|
+
* condition, not a machine fault. (Doctrine two-planes / the team ruling.)
|
|
996
|
+
*/
|
|
997
|
+
declare const SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES: ReadonlySet<string>;
|
|
998
|
+
/** The four mandatory field labels (exported so tests can assert their presence). */
|
|
999
|
+
declare const FAULT_FIELD_WHAT_HAPPENED = "What happened:";
|
|
1000
|
+
declare const FAULT_FIELD_WHICH_LAYER = "Which layer:";
|
|
1001
|
+
declare const FAULT_FIELD_WHAT_PRESERVED = "What was preserved:";
|
|
1002
|
+
declare const FAULT_FIELD_WHAT_TO_TRY = "What to try:";
|
|
1003
|
+
/**
|
|
1004
|
+
* Render a `SelfhostedControlError` into the doctrine's four in-band fields with a
|
|
1005
|
+
* correct retry verdict. The fault CLASS is read from the typed flags (never the
|
|
1006
|
+
* message prose), so each class gets the right "which layer" and "what to try":
|
|
1007
|
+
*
|
|
1008
|
+
* - PAYLOAD_TOO_LARGE → the command ran but its reply was too big to return; the
|
|
1009
|
+
* fix is to bound the output, not to retry.
|
|
1010
|
+
* - DRAINING → pre-admission backpressure; nothing ran; safe to retry shortly.
|
|
1011
|
+
* - CONSENT_REQUIRED → a machine-side consent gate; retrying cannot help.
|
|
1012
|
+
* - AGENT_OFFLINE → the machine link is down. If the transport KNOWS the request
|
|
1013
|
+
* never left (`neverSent`), nothing ran; otherwise the effect is ambiguous and
|
|
1014
|
+
* the model must check before re-running. Either way "try again" is wrong until
|
|
1015
|
+
* the machine reconnects.
|
|
1016
|
+
* - agent_reconnecting (TIMEOUT) → a transient link blip AFTER send: ambiguous, so
|
|
1017
|
+
* a state-changing command must be checked before re-running (at-least-once).
|
|
1018
|
+
* - NOT_FOUND → a command/filesystem miss, not a link fault.
|
|
1019
|
+
* - FENCED → the active machine session changed (a swap); re-run.
|
|
1020
|
+
* - default (OS/UNSUPPORTED/STREAM/PROTOCOL) → the command/op failed on the machine.
|
|
1021
|
+
*/
|
|
1022
|
+
declare function renderSelfhostedFault(error: SelfhostedControlError): string;
|
|
1023
|
+
|
|
1024
|
+
/**
|
|
1025
|
+
* The read-only, idempotent-safe control ops that MAY be retried after a TIMEOUT
|
|
1026
|
+
* / `agent_reconnecting` blip.
|
|
1027
|
+
*
|
|
1028
|
+
* A timed-out request is AT-LEAST-ONCE on the wire: the agent may have RECEIVED
|
|
1029
|
+
* and fully EXECUTED the op before its reply was lost, so re-issuing it is only
|
|
1030
|
+
* safe when a second execution is observably identical to one — i.e. a pure read.
|
|
1031
|
+
* Every MUTATING op (`exec`, `git`, `fsWrite`/`fsMove`/`fsMkdir`/`fsRemove`, and
|
|
1032
|
+
* the pty/desktop input ops) is DELIBERATELY EXCLUDED: re-running a mutation that
|
|
1033
|
+
* already ran on the machine is a real correctness hazard (a duplicated write, a
|
|
1034
|
+
* second `git push`, a re-applied command). Only DRAINING (a pre-admission
|
|
1035
|
+
* rejection — see `decideSelfhostedRetry`) is safe to retry for those.
|
|
1036
|
+
*/
|
|
1037
|
+
declare const SELFHOSTED_IDEMPOTENT_READONLY_OPS: ReadonlySet<string>;
|
|
1038
|
+
/**
|
|
1039
|
+
* DRAINING is a pre-admission backpressure rejection (the op never started), so it
|
|
1040
|
+
* is safe to retry for ANY op kind. The budget is split by op class:
|
|
1041
|
+
*
|
|
1042
|
+
* - NON-EXEC control ops (fs/git/desktop/pty) keep a SHORT budget — 3 retries,
|
|
1043
|
+
* worst-case backoff 500+1000+2000 = 3.5s (≤5s). Their start latency matters
|
|
1044
|
+
* (fs/git ops gate the model's next step), so a saturated machine should surface
|
|
1045
|
+
* quickly rather than stall.
|
|
1046
|
+
* - EXEC gets a LONG, patient budget — 10 retries, worst-case ≈55.5s (<60s). exec
|
|
1047
|
+
* start is not latency-critical, and the agent-side admission pool is (until a
|
|
1048
|
+
* later agent release) a FLAT 8 permits with no per-class split: a burst of slow
|
|
1049
|
+
* execs can hold every permit and blanket-DRAIN all fs/git ops. A patient exec
|
|
1050
|
+
* retry converts those DRAINING storms into mild queueing instead of a wall of
|
|
1051
|
+
* failed tool calls, and pairs with the deliberately modest exec deadline (which,
|
|
1052
|
+
* for the same 8-permit reason, defaults to 2min, not 5).
|
|
1053
|
+
*/
|
|
1054
|
+
declare const SELFHOSTED_DRAINING_MAX_RETRIES = 3;
|
|
1055
|
+
declare const SELFHOSTED_EXEC_DRAINING_MAX_RETRIES = 10;
|
|
1056
|
+
/** A read-only op that timed out is retried at most once (see the at-least-once
|
|
1057
|
+
* note above — even a read gets a single re-issue, not an unbounded loop). */
|
|
1058
|
+
declare const SELFHOSTED_TIMEOUT_MAX_RETRIES = 1;
|
|
1059
|
+
/** A fault the transport synthesized PRE-SEND (no connection / no responder) — the
|
|
1060
|
+
* op provably never reached the machine, so it is safe to re-issue for ANY op kind.
|
|
1061
|
+
* Short bounded budget (≈3.5s summed): enough to ride out a NATS reconnect blip
|
|
1062
|
+
* quietly, not so long it stalls the model on a machine that is genuinely down. */
|
|
1063
|
+
declare const SELFHOSTED_NEVER_SENT_MAX_RETRIES = 3;
|
|
1064
|
+
/** Full-jitter backoff base (ms). Delay N is sampled in `[0, base * factor^N)`. */
|
|
1065
|
+
declare const SELFHOSTED_RETRY_BACKOFF_BASE_MS = 500;
|
|
1066
|
+
declare const SELFHOSTED_RETRY_BACKOFF_FACTOR = 2;
|
|
1067
|
+
/** Per-delay ceiling: 500 → 1s → 2s → 4s → 8s, then flat at 8s. Bounds the summed
|
|
1068
|
+
* backoff — non-exec (3 retries) ≈3.5s ≤5s; exec (10 retries) ≈55.5s <60s. */
|
|
1069
|
+
declare const SELFHOSTED_RETRY_BACKOFF_CAP_MS = 8000;
|
|
1070
|
+
type SelfhostedRetryDecision = {
|
|
1071
|
+
readonly action: "fail";
|
|
1072
|
+
} | {
|
|
1073
|
+
readonly action: "retry";
|
|
1074
|
+
readonly delayMs: number;
|
|
1075
|
+
};
|
|
1076
|
+
/**
|
|
1077
|
+
* Full-jitter exponential backoff. Returns a delay in
|
|
1078
|
+
* `[0, min(cap, base * factor^attempt))`. `jitter` is an injected sample in
|
|
1079
|
+
* `[0, 1)` so the delay is fully deterministic under test.
|
|
1080
|
+
*/
|
|
1081
|
+
declare function selfhostedRetryBackoffMs(attempt: number, jitter: number): number;
|
|
1082
|
+
/**
|
|
1083
|
+
* PURE retry policy for one selfhosted control-op reply.
|
|
1084
|
+
*
|
|
1085
|
+
* - `error.draining` (DRAINING — pre-admission host-work backpressure; the op was
|
|
1086
|
+
* rejected at the pool gate and NEVER started) → retry for ANY op kind, since
|
|
1087
|
+
* nothing executed. exec gets the long `SELFHOSTED_EXEC_DRAINING_MAX_RETRIES`
|
|
1088
|
+
* budget (≈60s of patient queueing under the flat 8-permit pool); every other op
|
|
1089
|
+
* keeps the short `SELFHOSTED_DRAINING_MAX_RETRIES` budget (≤5s).
|
|
1090
|
+
* - `error.neverSent` (a pre-send AGENT_OFFLINE synthesis — no connection / no
|
|
1091
|
+
* responder; the op provably never reached the machine) → retry for ANY op kind
|
|
1092
|
+
* up to `SELFHOSTED_NEVER_SENT_MAX_RETRIES`. At-least-once does NOT apply (nothing
|
|
1093
|
+
* executed), so even a mutation is safe to re-issue; this heals a NATS reconnect
|
|
1094
|
+
* blip quietly. A truly-offline machine surfaces honestly after the short budget.
|
|
1095
|
+
* - `error.reason === "agent_reconnecting"` (a TIMEOUT / transient blip AFTER send)
|
|
1096
|
+
* → retry at most `SELFHOSTED_TIMEOUT_MAX_RETRIES`, and ONLY for a read-only
|
|
1097
|
+
* idempotent-safe op (`SELFHOSTED_IDEMPOTENT_READONLY_OPS`). A timed-out
|
|
1098
|
+
* MUTATION is never retried here — it may already have run (at-least-once).
|
|
1099
|
+
* - Everything else → no retry at this layer:
|
|
1100
|
+
* · FENCED is retried by the routing proxy against a RE-RESOLVED backend
|
|
1101
|
+
* (retrying here under the same stale epoch would just re-fence);
|
|
1102
|
+
* · AGENT_OFFLINE / OS / PROTOCOL / PAYLOAD_TOO_LARGE / CONSENT_REQUIRED are
|
|
1103
|
+
* not transient and must surface to the caller.
|
|
1104
|
+
*
|
|
1105
|
+
* The draining and timeout budgets are SEPARATE counters so a read that first hit
|
|
1106
|
+
* backpressure still keeps its single timeout re-issue (and vice versa); both are
|
|
1107
|
+
* bounded, so the total number of attempts — and the summed backoff — is bounded.
|
|
1108
|
+
*/
|
|
1109
|
+
declare function decideSelfhostedRetry(input: {
|
|
1110
|
+
opKind: string;
|
|
1111
|
+
error: SelfhostedControlError;
|
|
1112
|
+
drainingRetries: number;
|
|
1113
|
+
timeoutRetries: number;
|
|
1114
|
+
/** Prior never-sent retries. Optional (default 0) — only the pre-send AGENT_OFFLINE
|
|
1115
|
+
* branch reads it, so callers that cannot hit that class may omit it. */
|
|
1116
|
+
neverSentRetries?: number;
|
|
1117
|
+
jitter: number;
|
|
1118
|
+
}): SelfhostedRetryDecision;
|
|
1119
|
+
/**
|
|
1120
|
+
* The injected clock the retry loop drives — real in production, a deterministic
|
|
1121
|
+
* fake in tests. `jitter()` returns a full-jitter sample in `[0, 1)`.
|
|
1122
|
+
*/
|
|
1123
|
+
interface SelfhostedRetryClock {
|
|
1124
|
+
sleep(ms: number): Promise<void>;
|
|
1125
|
+
jitter(): number;
|
|
1126
|
+
}
|
|
1127
|
+
/** The production clock: a real timer + `Math.random()` jitter. */
|
|
1128
|
+
declare const defaultSelfhostedRetryClock: SelfhostedRetryClock;
|
|
1129
|
+
|
|
1130
|
+
/** The per-op frame subject (runner→server, fire-and-forget). Mirrors the
|
|
1131
|
+
* runner's wire constant: `agent.<ws>.<id>.op.<op_id>`. */
|
|
1132
|
+
declare function opFrameSubject(workspaceId: string, agentId: string, opId: string): string;
|
|
1133
|
+
/** The per-agent ack subject (server→runner, fire-and-forget). Mirrors the
|
|
1134
|
+
* runner's wire constant: `agent.<ws>.<id>.ack`. */
|
|
1135
|
+
declare function opAckSubject(workspaceId: string, agentId: string): string;
|
|
1136
|
+
/** A live subscription handle. `unsubscribe` is idempotent and never throws. */
|
|
1137
|
+
interface OpStreamSubscription {
|
|
1138
|
+
unsubscribe(): void;
|
|
1139
|
+
}
|
|
1140
|
+
/**
|
|
1141
|
+
* The op-stream transport seam. `subscribe` MUST be established before the
|
|
1142
|
+
* OpStart that makes frames flow (subscription-before-start is a protocol
|
|
1143
|
+
* invariant: no frame is published before the consumer exists on a healthy
|
|
1144
|
+
* path; a missed frame is healed via OpAttach replay regardless). `onMessage`
|
|
1145
|
+
* is invoked in arrival order; a decode failure inside the handler must be
|
|
1146
|
+
* contained by the caller (a torn frame must never kill the subscription).
|
|
1147
|
+
*/
|
|
1148
|
+
interface OpStreamTransport {
|
|
1149
|
+
subscribe(subject: string, onMessage: (payload: Uint8Array) => void): Promise<OpStreamSubscription>;
|
|
1150
|
+
publish(subject: string, payload: Uint8Array): Promise<void>;
|
|
1151
|
+
}
|
|
1152
|
+
/** Thrown when the transport has no live connection: the op-stream path is
|
|
1153
|
+
* UNAVAILABLE (never silently degraded). The session catches this pre-start
|
|
1154
|
+
* and falls back to the legacy exec — the op provably never started. */
|
|
1155
|
+
declare class OpStreamUnavailableError extends Error {
|
|
1156
|
+
readonly name = "OpStreamUnavailableError";
|
|
1157
|
+
}
|
|
1158
|
+
/**
|
|
1159
|
+
* The minimal NATS surface the transport needs — mirrors the `nats`
|
|
1160
|
+
* `NatsConnection` subscribe/publish WITHOUT importing `nats` into the
|
|
1161
|
+
* agent-loop-free runtime leaf (the worker injects the live `@opengeni/events`
|
|
1162
|
+
* bus connection, exactly like `NatsRequestConnection` in control-rpc.ts).
|
|
1163
|
+
* `subscribe` returns the nats.js Subscription shape: an async iterable of
|
|
1164
|
+
* messages plus `unsubscribe()`.
|
|
1165
|
+
*/
|
|
1166
|
+
interface NatsOpStreamConnection {
|
|
1167
|
+
subscribe(subject: string): AsyncIterable<{
|
|
1168
|
+
data: Uint8Array;
|
|
1169
|
+
}> & {
|
|
1170
|
+
unsubscribe(): void;
|
|
1171
|
+
};
|
|
1172
|
+
publish(subject: string, payload: Uint8Array): void;
|
|
1173
|
+
}
|
|
1174
|
+
/**
|
|
1175
|
+
* The NATS-backed transport. Lazy memoized factory (identical discipline to
|
|
1176
|
+
* `NatsControlRpc`): the connection resolves on first use; a null factory
|
|
1177
|
+
* result or a dial failure surfaces `OpStreamUnavailableError` — the caller
|
|
1178
|
+
* (the session) falls back to the LEGACY exec path rather than failing the op,
|
|
1179
|
+
* so a NATS-less boot or a torn bus never strands a turn on a dead transport.
|
|
1180
|
+
*/
|
|
1181
|
+
declare class NatsOpStreamTransport implements OpStreamTransport {
|
|
1182
|
+
private readonly connect;
|
|
1183
|
+
private connection;
|
|
1184
|
+
private connecting;
|
|
1185
|
+
constructor(connect: () => Promise<NatsOpStreamConnection | null>);
|
|
1186
|
+
private resolveConnection;
|
|
1187
|
+
subscribe(subject: string, onMessage: (payload: Uint8Array) => void): Promise<OpStreamSubscription>;
|
|
1188
|
+
publish(subject: string, payload: Uint8Array): Promise<void>;
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
/** Initial send-credit window (bytes) when the caller does not size it. Matches
|
|
1192
|
+
* the wire default OpStart advertises (~4 MiB). */
|
|
1193
|
+
declare const OP_STREAM_DEFAULT_WINDOW_BYTES: number;
|
|
1194
|
+
/** Cumulative-ack repetition cadence while an op is live (ruling M1: acks are
|
|
1195
|
+
* best-effort and healed by repetition). */
|
|
1196
|
+
declare const OP_STREAM_ACK_INTERVAL_MS = 5000;
|
|
1197
|
+
/** Frame-silence threshold before the client starts probing with OpQuery. The
|
|
1198
|
+
* runner emits Progress every 5s while an op is alive and quiet, so 30s of
|
|
1199
|
+
* TOTAL silence means frames are not reaching us (or the op is gone). */
|
|
1200
|
+
declare const OP_STREAM_SILENCE_TIMEOUT_MS = 30000;
|
|
1201
|
+
/** How long a silent op is held open across a machine reconnect window before
|
|
1202
|
+
* the client fails it typed (PROTOCOL liveness rule: hold ≤120s, then only a
|
|
1203
|
+
* definitive answer fails the op). */
|
|
1204
|
+
declare const OP_STREAM_RECONNECT_HOLD_MS = 120000;
|
|
1205
|
+
/**
|
|
1206
|
+
* The durable-resume journal the worker adapts onto Temporal (design delta Δ1:
|
|
1207
|
+
* the attach generation is `Info.currentAttemptScheduledTimestampMs` — strictly
|
|
1208
|
+
* monotonic across worker-death redispatches, which `Info.attempt` is NOT under
|
|
1209
|
+
* `maximumAttempts: 1`). `persistSettled` folds the op's settled frontier into
|
|
1210
|
+
* the activity heartbeat details and MUST complete before the wire final ack
|
|
1211
|
+
* (the durable-before-wire-ack gate); the default no-journal client (tests,
|
|
1212
|
+
* non-activity callers) uses generation "1" and skips persistence.
|
|
1213
|
+
*/
|
|
1214
|
+
interface OpStreamJournal {
|
|
1215
|
+
/** The consumer's attach generation (u64 decimal string). */
|
|
1216
|
+
attachGeneration(): string;
|
|
1217
|
+
/** Durably record `opId` as settled at `exitSeq` BEFORE its final ack. */
|
|
1218
|
+
persistSettled(opId: string, exitSeq: string): void | Promise<void>;
|
|
1219
|
+
}
|
|
1220
|
+
/** A completed op-stream exec: the legacy-shaped response plus the healing
|
|
1221
|
+
* telemetry the session folds into its op observation. */
|
|
1222
|
+
interface OpStreamExecOutcome {
|
|
1223
|
+
response: ExecResponse;
|
|
1224
|
+
/** Attach/query heals that occurred after streaming began (gap replays,
|
|
1225
|
+
* silence probes that re-attached). >0 marks the op `healed`. */
|
|
1226
|
+
heals: number;
|
|
1227
|
+
/** OpStart-level retries (draining/blip) before the op was accepted. */
|
|
1228
|
+
startRetries: number;
|
|
1229
|
+
/** Total reassembled Data payload bytes (the observer's replyBytes). */
|
|
1230
|
+
replyBytes: number;
|
|
1231
|
+
}
|
|
1232
|
+
|
|
858
1233
|
/** The V4A-diff applier the SDK's apply_patch editor uses. The leaf cannot import
|
|
859
1234
|
* `@openai/agents`'s `applyDiff` (the agent-loop root the leaf forbids), so the
|
|
860
1235
|
* runtime barrel (`packages/runtime/src/index.ts`, which DOES import that root)
|
|
@@ -914,17 +1289,54 @@ interface SelfhostedRelayConfig {
|
|
|
914
1289
|
}
|
|
915
1290
|
/** The relay's default wss dial path (the `opengeni-relay` server route). */
|
|
916
1291
|
declare const SELFHOSTED_RELAY_STREAM_PATH = "/stream";
|
|
1292
|
+
/**
|
|
1293
|
+
* The op-stream injection (op-stream protocol v1.1 — see op-stream.ts). PRESENT
|
|
1294
|
+
* = the streaming exec transport is enabled for this session (the worker
|
|
1295
|
+
* injects it only when the runner advertised `Capabilities.op_stream` AND the
|
|
1296
|
+
* server flag is on — the leaf carries no flag logic). ABSENT = the legacy
|
|
1297
|
+
* monolithic exec, byte-identical to before. The timing knobs exist for tests.
|
|
1298
|
+
*/
|
|
1299
|
+
interface SelfhostedOpStreamDeps {
|
|
1300
|
+
transport: OpStreamTransport;
|
|
1301
|
+
/** The durable-resume journal (attach generation + settled-frontier persist).
|
|
1302
|
+
* Absent (tests, non-activity callers) ⇒ generation "1", no persistence. */
|
|
1303
|
+
journal?: OpStreamJournal;
|
|
1304
|
+
windowBytes?: number;
|
|
1305
|
+
ackIntervalMs?: number;
|
|
1306
|
+
silenceTimeoutMs?: number;
|
|
1307
|
+
reconnectHoldMs?: number;
|
|
1308
|
+
}
|
|
917
1309
|
interface SelfhostedSessionDeps {
|
|
918
1310
|
workspaceId: string;
|
|
919
1311
|
agentId: string;
|
|
920
1312
|
controlRpc: ControlRpc;
|
|
921
1313
|
relay: SelfhostedRelayConfig;
|
|
1314
|
+
/** Op-stream exec transport (present = enabled; see SelfhostedOpStreamDeps). */
|
|
1315
|
+
opStream?: SelfhostedOpStreamDeps;
|
|
922
1316
|
/** The lease/active epoch this session is fenced under (echoed on every
|
|
923
1317
|
* ControlRequest so the agent can reject a stale op with ERROR_CODE_FENCED).
|
|
924
1318
|
* Defaults to 0 (no fence) for the negotiation-only / test path. */
|
|
925
1319
|
epoch?: number;
|
|
926
|
-
/**
|
|
1320
|
+
/** The CONTROL-op timeout (ping/fs/desktop/pty and the exec wire fallback). The
|
|
1321
|
+
* agent's own control liveness must stay responsive, so this is short (the 30s
|
|
1322
|
+
* default). Override for tests or per-deployment tuning. */
|
|
927
1323
|
timeoutMs?: number;
|
|
1324
|
+
/**
|
|
1325
|
+
* The EXEC process deadline — distinct from `timeoutMs`. A real command
|
|
1326
|
+
* (compile, test run, install) routinely outlives the short control timeout, so
|
|
1327
|
+
* exec gets its own, much larger budget: the agent kills the child at this wall,
|
|
1328
|
+
* and the wire waits `execTimeoutMs + SELFHOSTED_EXEC_REPLY_GRACE_MS`. Omitted ⇒
|
|
1329
|
+
* falls back to `timeoutMs` (byte-identical to before this split), so any caller
|
|
1330
|
+
* that does not opt in keeps today's behavior.
|
|
1331
|
+
*/
|
|
1332
|
+
execTimeoutMs?: number;
|
|
1333
|
+
/** The clock the bounded control-op retry loop drives (sleep + jitter). Injected
|
|
1334
|
+
* so tests are deterministic; defaults to a real timer + `Math.random()`. */
|
|
1335
|
+
retryClock?: SelfhostedRetryClock;
|
|
1336
|
+
/** A fire-and-forget observer invoked once per completed control op (out-of-band
|
|
1337
|
+
* telemetry — metrics + `machine.*` events). The worker/api wire it to the sinks;
|
|
1338
|
+
* the leaf only invokes it (guarded so it can never break an op). Omitted ⇒ no-op. */
|
|
1339
|
+
onOp?: SelfhostedOpObserver;
|
|
928
1340
|
/**
|
|
929
1341
|
* The run's declared sandbox environment — the SAME `Record<string,string>` the
|
|
930
1342
|
* worker turn passes to `runtime.buildAgent`'s `sandboxEnvironment` (and that the
|
|
@@ -955,6 +1367,10 @@ interface SelfhostedExecResult {
|
|
|
955
1367
|
stdout: string;
|
|
956
1368
|
stderr: string;
|
|
957
1369
|
exitCode: number | null;
|
|
1370
|
+
/** True when the agent killed the child at the exec deadline. Additive to the
|
|
1371
|
+
* Channel-A superset (consumers that don't read it are unaffected); `execCommand`
|
|
1372
|
+
* reads it to surface the deadline hint on the stdout-only SDK path. */
|
|
1373
|
+
timedOut?: boolean;
|
|
958
1374
|
}
|
|
959
1375
|
/** The `exec` args the structural surface accepts (mirrors ChannelAExecArgs). */
|
|
960
1376
|
interface SelfhostedExecArgs {
|
|
@@ -986,7 +1402,14 @@ declare class SelfhostedSession {
|
|
|
986
1402
|
private readonly relay;
|
|
987
1403
|
private readonly epoch;
|
|
988
1404
|
private readonly timeoutMs;
|
|
1405
|
+
/** The exec process deadline (falls back to `timeoutMs` when unset). */
|
|
1406
|
+
private readonly execTimeoutMs;
|
|
1407
|
+
private readonly retryClock;
|
|
1408
|
+
private readonly onOp;
|
|
989
1409
|
private readonly subject;
|
|
1410
|
+
/** The op-stream exec client — constructed iff `deps.opStream` was injected
|
|
1411
|
+
* (= the streaming transport is enabled for this session). */
|
|
1412
|
+
private readonly opStreamClient;
|
|
990
1413
|
/** The session working directory — the path/cwd base every op is rooted under
|
|
991
1414
|
* (see `toMachinePath`). "" by default ⇒ today's workspace_root behavior. */
|
|
992
1415
|
private readonly workingDir;
|
|
@@ -1024,15 +1447,65 @@ declare class SelfhostedSession {
|
|
|
1024
1447
|
environment: Record<string, string>;
|
|
1025
1448
|
};
|
|
1026
1449
|
constructor(deps: SelfhostedSessionDeps);
|
|
1027
|
-
/**
|
|
1028
|
-
*
|
|
1029
|
-
|
|
1450
|
+
/** Invoke the injected per-op observer, guarded: a telemetry tap must NEVER break
|
|
1451
|
+
* an op (a throwing sink is swallowed). No-op when no observer is wired. */
|
|
1452
|
+
private emitOp;
|
|
1453
|
+
/**
|
|
1454
|
+
* Issue a control op, decoding the agent's reply or throwing the mapped
|
|
1455
|
+
* `SelfhostedControlError` on an AgentError (incl. a synthesized offline /
|
|
1456
|
+
* timeout error from the transport).
|
|
1457
|
+
*
|
|
1458
|
+
* Two failure modes are retried in-place (the decision is the pure
|
|
1459
|
+
* `decideSelfhostedRetry` policy):
|
|
1460
|
+
* - DRAINING — a pre-admission host-work backpressure rejection; the op never
|
|
1461
|
+
* started, so it is safe to re-issue for ANY op kind (bounded).
|
|
1462
|
+
* - a TIMEOUT / `agent_reconnecting` blip — re-issued ONCE, and ONLY for a
|
|
1463
|
+
* read-only idempotent-safe op. A timed-out MUTATION is NEVER re-issued: it
|
|
1464
|
+
* may already have executed on the machine (at-least-once), which would
|
|
1465
|
+
* duplicate the write/command. FENCED is intentionally NOT retried here — the
|
|
1466
|
+
* routing proxy retries it against a re-resolved backend under a fresh epoch.
|
|
1467
|
+
* Each attempt carries a FRESH `requestId` (a retry is a distinct request).
|
|
1468
|
+
*/
|
|
1030
1469
|
private call;
|
|
1470
|
+
/** The clamped exec process deadline (ms): the exec-specific budget when threaded,
|
|
1471
|
+
* else the control timeout. Shared by `exec()` (the wire deadline) and
|
|
1472
|
+
* `execCommand()` (the timed-out hint's "N-second limit"), so the two never drift. */
|
|
1473
|
+
private execDeadlineMs;
|
|
1031
1474
|
/** Channel-A `exec`: run a command on the machine and return its output. */
|
|
1032
1475
|
exec(args: SelfhostedExecArgs): Promise<SelfhostedExecResult>;
|
|
1476
|
+
/** The legacy monolithic exec request/reply — the permanent fallback wire
|
|
1477
|
+
* form (and the only form for runners without `op_stream`). */
|
|
1478
|
+
private execLegacy;
|
|
1479
|
+
/**
|
|
1480
|
+
* The op-stream exec path: the SAME `ExecRequest`, streamed (see
|
|
1481
|
+
* op-stream.ts). The durable op id comes from the tool-call correlation
|
|
1482
|
+
* context (`{callId}:{ordinal}` — B1) when this exec runs inside an SDK tool
|
|
1483
|
+
* invocation; a non-tool caller falls back to a random unique id (never
|
|
1484
|
+
* collides, merely not stable across a turn re-dispatch). Emits the SAME
|
|
1485
|
+
* per-op observation the legacy path does, with `replyBytes` filled from the
|
|
1486
|
+
* reassembled stream (the field the framed transport was designed to own).
|
|
1487
|
+
*/
|
|
1488
|
+
private execViaOpStream;
|
|
1489
|
+
/**
|
|
1490
|
+
* The turn-end op-stream durability hook: persists each settled op's frontier
|
|
1491
|
+
* to the journal, then final-acks it on the wire (licensing the runner to GC
|
|
1492
|
+
* its retained frames). The WORKER calls this after the turn's results are
|
|
1493
|
+
* durably recorded — never mid-turn (the durable-before-wire-ack ordering).
|
|
1494
|
+
* A no-op for sessions without the op-stream transport.
|
|
1495
|
+
*/
|
|
1496
|
+
finalizeOpStreamOps(): Promise<void>;
|
|
1033
1497
|
/** SDK shell capability `execCommand`: run a command and return its stdout (the
|
|
1034
1498
|
* `exec_command` tool). Selfhosted exec is non-interactive (no PTY) — `tty` is
|
|
1035
|
-
* ignored; `supportsPty()` is false so the SDK never offers a stdin session.
|
|
1499
|
+
* ignored; `supportsPty()` is false so the SDK never offers a stdin session.
|
|
1500
|
+
*
|
|
1501
|
+
* On a DEADLINE timeout the SDK path is stdout-only (the model sees `result.output`
|
|
1502
|
+
* and nothing else), so the deadline hint that `exec()` puts on stderr would be
|
|
1503
|
+
* invisible — a >deadline command would return `{"output":""}` and the model would
|
|
1504
|
+
* conclude "no output" with no idea it was killed. So when the result timed out we
|
|
1505
|
+
* surface the hint HERE, on stdout: the hint alone when stdout is empty, or appended
|
|
1506
|
+
* after a newline when there is partial output (a timed-out result is already not
|
|
1507
|
+
* cleanly parseable — silence is worse than an explanatory suffix). The structured
|
|
1508
|
+
* `exec()` result is left untouched for the Channel-A parsers. */
|
|
1036
1509
|
execCommand(args: {
|
|
1037
1510
|
cmd: string;
|
|
1038
1511
|
workdir?: string;
|
|
@@ -1171,8 +1644,11 @@ declare class SelfhostedSandboxClient {
|
|
|
1171
1644
|
private readonly defaultAgentId;
|
|
1172
1645
|
private readonly epoch;
|
|
1173
1646
|
private readonly timeoutMs;
|
|
1647
|
+
private readonly execTimeoutMs;
|
|
1174
1648
|
private readonly environment;
|
|
1175
1649
|
private readonly workingDir;
|
|
1650
|
+
private readonly onOp;
|
|
1651
|
+
private readonly opStream;
|
|
1176
1652
|
private controlRpcMemo;
|
|
1177
1653
|
constructor(opts: {
|
|
1178
1654
|
workspaceId: string;
|
|
@@ -1183,7 +1659,11 @@ declare class SelfhostedSandboxClient {
|
|
|
1183
1659
|
* resume path supplies it via deserializeSessionState. */
|
|
1184
1660
|
agentId?: string;
|
|
1185
1661
|
epoch?: number;
|
|
1662
|
+
/** The control-op timeout threaded into every bound session. */
|
|
1186
1663
|
timeoutMs?: number;
|
|
1664
|
+
/** The exec process deadline threaded into every bound session (distinct from
|
|
1665
|
+
* `timeoutMs`; falls back to it when unset). See SelfhostedSessionDeps.execTimeoutMs. */
|
|
1666
|
+
execTimeoutMs?: number;
|
|
1187
1667
|
/** The run's declared sandbox environment, threaded into every bound session's
|
|
1188
1668
|
* `state.manifest.environment` so the SDK's per-turn manifest-env delta is
|
|
1189
1669
|
* empty (validateNoEnvironmentDelta). See SelfhostedSessionDeps.environment.
|
|
@@ -1193,6 +1673,11 @@ declare class SelfhostedSandboxClient {
|
|
|
1193
1673
|
* cwd base; see SelfhostedSessionDeps.workingDir). Omitted/empty ⇒ the default
|
|
1194
1674
|
* workspace_root behavior. */
|
|
1195
1675
|
workingDir?: string;
|
|
1676
|
+
/** The per-op observer threaded into every bound session (out-of-band telemetry). */
|
|
1677
|
+
onOp?: SelfhostedOpObserver;
|
|
1678
|
+
/** The op-stream exec transport threaded into every bound session (present
|
|
1679
|
+
* = enabled; see SelfhostedOpStreamDeps). */
|
|
1680
|
+
opStream?: SelfhostedOpStreamDeps;
|
|
1196
1681
|
});
|
|
1197
1682
|
private controlRpc;
|
|
1198
1683
|
private bind;
|
|
@@ -1235,8 +1720,17 @@ interface SelfhostedSessionBuild {
|
|
|
1235
1720
|
environment?: Record<string, string>;
|
|
1236
1721
|
/** The session working directory (the path/cwd base). Null/absent ⇒ workspace_root. */
|
|
1237
1722
|
workingDir?: string | null;
|
|
1238
|
-
/**
|
|
1723
|
+
/** The control-op timeout (ping/fs/desktop/pty). Absent ⇒ the 30s default. */
|
|
1239
1724
|
timeoutMs?: number;
|
|
1725
|
+
/** The exec process deadline, distinct from `timeoutMs`. Absent ⇒ falls back to
|
|
1726
|
+
* `timeoutMs` (a real turn threads the long exec budget here). */
|
|
1727
|
+
execTimeoutMs?: number;
|
|
1728
|
+
/** The per-op observer (out-of-band telemetry). Absent ⇒ no-op. */
|
|
1729
|
+
onOp?: SelfhostedOpObserver;
|
|
1730
|
+
/** The op-stream exec transport (present = enabled for the session; the
|
|
1731
|
+
* worker injects it only when the runner advertised the capability AND the
|
|
1732
|
+
* server flag is on). Absent ⇒ the legacy exec, byte-identical to before. */
|
|
1733
|
+
opStream?: SelfhostedOpStreamDeps;
|
|
1240
1734
|
}
|
|
1241
1735
|
/**
|
|
1242
1736
|
* Build a live selfhosted session bound to a target machine: construct a request-
|
|
@@ -1273,6 +1767,31 @@ declare function buildSelfhostedBackendSession(deps: SelfhostedSessionBuild): Pr
|
|
|
1273
1767
|
*/
|
|
1274
1768
|
declare function isSelfhostedProviderNotFoundError(_error: unknown): false;
|
|
1275
1769
|
|
|
1770
|
+
/**
|
|
1771
|
+
* Sanitize a tool call id into a legal NATS subject TOKEN fragment. The op id
|
|
1772
|
+
* is interpolated into the per-op frame subject, whose tokens must not contain
|
|
1773
|
+
* whitespace/control characters or the subject-structure characters (`.`,
|
|
1774
|
+
* `*`, `>`); the runner refuses illegal ids loudly. The mapping is INJECTIVE
|
|
1775
|
+
* (each disallowed char becomes `_<hex>_`), so two distinct call ids can never
|
|
1776
|
+
* collide into one op id — a collision would merge two different execs through
|
|
1777
|
+
* the idempotent-OpStart dedup and return the wrong result.
|
|
1778
|
+
*/
|
|
1779
|
+
declare function sanitizeOpIdToken(raw: string): string;
|
|
1780
|
+
/**
|
|
1781
|
+
* Bind a tool-call correlation context around `fn` (the tool's execute). Every
|
|
1782
|
+
* durable op id minted inside — however deep in the transport — is
|
|
1783
|
+
* `{callId}:{ordinal}` with ordinals starting at 0 per invocation.
|
|
1784
|
+
*/
|
|
1785
|
+
declare function runWithToolCallCorrelation<T>(callId: string, fn: () => T): T;
|
|
1786
|
+
/**
|
|
1787
|
+
* Mint the next durable op id for the current tool invocation, or null when no
|
|
1788
|
+
* correlation context is bound (a non-tool caller, e.g. Channel-A structural
|
|
1789
|
+
* exec). Callers fall back to a random unique id — safe (never collides, never
|
|
1790
|
+
* wrongly dedups), merely not stable across a turn re-dispatch, which degrades
|
|
1791
|
+
* that one op to today's at-least-once semantics.
|
|
1792
|
+
*/
|
|
1793
|
+
declare function nextDurableOpId(): string | null;
|
|
1794
|
+
|
|
1276
1795
|
/**
|
|
1277
1796
|
* The structural slice of the M2 `@opengeni/db` `EnrollmentRecord` the selfhosted
|
|
1278
1797
|
* negotiation reads. Defined STRUCTURALLY (not imported from `@opengeni/db`) so
|
|
@@ -1288,6 +1807,13 @@ interface SelfhostedEnrollment {
|
|
|
1288
1807
|
allowScreenControl: boolean;
|
|
1289
1808
|
hasDisplay: boolean;
|
|
1290
1809
|
lastSeenAt: string | null;
|
|
1810
|
+
/** An un-cleared clean going-offline marker (the machine announced a typed
|
|
1811
|
+
* GoingOffline). When set, the derivation reads the machine OFFLINE immediately,
|
|
1812
|
+
* regardless of a still-fresh `lastSeenAt`. NULL ⇒ no pending goodbye. */
|
|
1813
|
+
wentOfflineAt: string | null;
|
|
1814
|
+
/** The typed reason string of the pending goodbye (rides alongside
|
|
1815
|
+
* `wentOfflineAt`; NULL when there is no un-cleared marker). */
|
|
1816
|
+
wentOfflineReason: string | null;
|
|
1291
1817
|
}
|
|
1292
1818
|
/** The derived liveness state of a selfhosted machine (the online/offline/
|
|
1293
1819
|
* reconnecting/consent/display matrix). */
|
|
@@ -1313,6 +1839,8 @@ declare const SELFHOSTED_RECONNECT_WINDOW_MS = 30000;
|
|
|
1313
1839
|
* (stale / never seen).
|
|
1314
1840
|
*
|
|
1315
1841
|
* - no enrollment / revoked → offline (the machine isn't enrolled).
|
|
1842
|
+
* - un-cleared clean goodbye → offline (an announced GoingOffline beats
|
|
1843
|
+
* last_seen aging AND a lingering probe).
|
|
1316
1844
|
* - probe responded → online.
|
|
1317
1845
|
* - probe missed, lastSeenAt recent → reconnecting (a transient blip).
|
|
1318
1846
|
* - probe missed, lastSeenAt stale → offline.
|
|
@@ -1519,6 +2047,7 @@ declare class RoutingSandboxSession implements RoutableBackendSession {
|
|
|
1519
2047
|
private readonly deps;
|
|
1520
2048
|
private readonly maxFenceRetries;
|
|
1521
2049
|
private cachedEpoch;
|
|
2050
|
+
private cachedSandboxId;
|
|
1522
2051
|
private cached;
|
|
1523
2052
|
private lastResolved;
|
|
1524
2053
|
desktopInput?: (event: unknown) => Promise<void>;
|
|
@@ -1565,6 +2094,25 @@ declare class RoutingSandboxSession implements RoutableBackendSession {
|
|
|
1565
2094
|
* race).
|
|
1566
2095
|
*/
|
|
1567
2096
|
private dispatch;
|
|
2097
|
+
/**
|
|
2098
|
+
* The failure-visibility boundary for the `exec_command` SDK capability tool — the
|
|
2099
|
+
* dominant fault surface, and the one whose thrown `SelfhostedControlError` reaches
|
|
2100
|
+
* the model wrapped by the SDK's generic tool-error function as
|
|
2101
|
+
* "…Please try again. Error: …" — actively wrong for a machine that is offline, a
|
|
2102
|
+
* consent that is not granted, or an oversized reply. Since the SDK closure-captures
|
|
2103
|
+
* its `errorFunction` (there is no seam to attach one to its internally-built tools),
|
|
2104
|
+
* we render the fault into the doctrine's four fields HERE and return it as the
|
|
2105
|
+
* tool's string result — legible, in-band, and free of the misleading wrapper.
|
|
2106
|
+
* (`apply_patch` already surfaces `error.message` via the SDK's own catch; the
|
|
2107
|
+
* skills / `view_image` tools consume their session methods internally, so they are
|
|
2108
|
+
* rendered elsewhere or left to their own SDK renderers — not this string boundary.)
|
|
2109
|
+
*
|
|
2110
|
+
* A FENCE error is re-thrown, NEVER rendered: `dispatch` already retries it against
|
|
2111
|
+
* a re-resolved backend, and a fence that escapes retries is a routing condition the
|
|
2112
|
+
* turn handles, not a model-facing fault. Any non-selfhosted error (a Modal fault, a
|
|
2113
|
+
* `RoutingUnsupportedError`) is re-thrown unchanged.
|
|
2114
|
+
*/
|
|
2115
|
+
private renderSelfhostedFaultOrThrow;
|
|
1568
2116
|
exec(args: unknown): Promise<unknown>;
|
|
1569
2117
|
execCommand(args: unknown): Promise<string>;
|
|
1570
2118
|
writeStdin(args: unknown): Promise<string>;
|
|
@@ -1580,10 +2128,17 @@ declare class RoutingSandboxSession implements RoutableBackendSession {
|
|
|
1580
2128
|
* the resolve the surrounding op already performed. Defaults false before the
|
|
1581
2129
|
* first resolve. */
|
|
1582
2130
|
supportsPty(): boolean;
|
|
1583
|
-
/** createEditor is a synchronous factory in the SDK surface
|
|
1584
|
-
*
|
|
1585
|
-
*
|
|
1586
|
-
*
|
|
2131
|
+
/** createEditor is a synchronous factory in the SDK surface. The SDK's filesystem
|
|
2132
|
+
* capability calls it ONCE at tool-BIND time — `FilesystemCapability.tools()`,
|
|
2133
|
+
* every turn, before any tool runs — and throws "Filesystem sandbox sessions must
|
|
2134
|
+
* provide createEditor()" if it returns falsy. When a backend is already resolved
|
|
2135
|
+
* (eager/selfhosted routing) we bind to its editor directly, byte-for-byte as
|
|
2136
|
+
* before. But under LAZY provisioning the backend is not established yet
|
|
2137
|
+
* (defaultResolved is the synthetic unprovisioned session with no editor), so a
|
|
2138
|
+
* direct delegate returns undefined and every lazy turn would die at bind. Return a
|
|
2139
|
+
* LAZY EDITOR PROXY instead: a non-null editor whose async ops resolve the active
|
|
2140
|
+
* backend (establishing the box on first use, via `dispatch`) and delegate to its
|
|
2141
|
+
* real editor — mirroring how this proxy defers exec/readFile. */
|
|
1587
2142
|
createEditor(runAs?: string): unknown;
|
|
1588
2143
|
resolveExposedPort(port: number): Promise<ExposedPortEndpoint>;
|
|
1589
2144
|
/** Serialize the active backend's session state. Used by the resume-by-id seam
|
|
@@ -1627,8 +2182,16 @@ interface ActiveBackendResolverDeps {
|
|
|
1627
2182
|
* absent, a modal swap target surfaces as unsupported (the caller validated
|
|
1628
2183
|
* liveness, so this is the "modal swap not wired in this context" guard). */
|
|
1629
2184
|
establishModalTarget?: (sandbox: RoutableSandbox) => Promise<RoutableBackendSession>;
|
|
1630
|
-
/**
|
|
2185
|
+
/** The selfhosted CONTROL-op timeout (ping/fs/desktop/pty) for a swap/pin target.
|
|
2186
|
+
* Absent ⇒ the session's 30s default. */
|
|
1631
2187
|
selfhostedTimeoutMs?: number;
|
|
2188
|
+
/** The selfhosted EXEC process deadline for a swap/pin target, distinct from the
|
|
2189
|
+
* control timeout (a swapped-to machine runs real commands too). Absent ⇒ falls
|
|
2190
|
+
* back to the control timeout, as in the session leaf. */
|
|
2191
|
+
selfhostedExecTimeoutMs?: number;
|
|
2192
|
+
/** The per-op observer threaded into a selfhosted swap/pin target (out-of-band
|
|
2193
|
+
* telemetry — metrics + machine.* events). Absent ⇒ no-op. */
|
|
2194
|
+
selfhostedOnOp?: SelfhostedOpObserver;
|
|
1632
2195
|
/**
|
|
1633
2196
|
* The run's declared sandbox environment — the SAME `Record<string,string>` the
|
|
1634
2197
|
* worker turn threads into the agent's TARGET manifest (and into the group box at
|
|
@@ -1657,13 +2220,77 @@ interface ActiveBackendResolverDeps {
|
|
|
1657
2220
|
epoch: number;
|
|
1658
2221
|
session: RoutableBackendSession;
|
|
1659
2222
|
};
|
|
2223
|
+
/**
|
|
2224
|
+
* Whether `defaultBackend` IS the session's home box — i.e. whether the null pointer
|
|
2225
|
+
* (== home) may resolve to it. Defaults to TRUE (omitted): a genuine machine-HOME turn
|
|
2226
|
+
* (home IS the machine) and a Modal-home turn established on its group box both pass/omit
|
|
2227
|
+
* true, so null resolves to `defaultBackend` as before. Set explicitly FALSE only for a
|
|
2228
|
+
* machine-primary turn of a Modal-HOME session (pinned to a machine, group box never
|
|
2229
|
+
* established this turn): the null pointer then has nothing to resolve to THIS turn, so
|
|
2230
|
+
* rather than silently keep serving the pinned machine as if the detach never happened,
|
|
2231
|
+
* the null branch throws a typed `home_unavailable_this_turn` error — the detach's pointer
|
|
2232
|
+
* commit stands and takes effect on the NEXT turn, which starts null and establishes the
|
|
2233
|
+
* home box normally. Lazily establishing the home box mid-turn on such a clear is a
|
|
2234
|
+
* deferred follow-up (issue #341); until then this makes the gap fail typed-and-specific,
|
|
2235
|
+
* never silent.
|
|
2236
|
+
*/
|
|
2237
|
+
defaultIsHome?: boolean;
|
|
1660
2238
|
}
|
|
2239
|
+
/** Why a persisted pointer / swap target cannot be turned into a live backend. A
|
|
2240
|
+
* typed discriminant so callers (turn-start reconcile, the swap tool, future
|
|
2241
|
+
* reports) can distinguish a stale pointer from an unaddressable enrollment from a
|
|
2242
|
+
* backend the turn context simply cannot establish — instead of string-matching
|
|
2243
|
+
* one opaque message (issue #341: "typed diagnostics that distinguish stale
|
|
2244
|
+
* pointer, offline enrollment, unsupported backend context, transient failure").
|
|
2245
|
+
* - `stale_pointer` — the pointed-at sandbox row is gone (deleted/orphaned).
|
|
2246
|
+
* - `offline_enrollment` — a selfhosted target carries no enrollment/agent id to address.
|
|
2247
|
+
* - `unsupported_backend_context` — a target no turn routing context can establish
|
|
2248
|
+
* (a non-group Modal sibling; an unknown kind).
|
|
2249
|
+
* - `transient_establishment` — a momentary establish failure worth a retry (reserved;
|
|
2250
|
+
* control-plane timeouts surface as their own typed error).
|
|
2251
|
+
* - `home_unavailable_this_turn` — the pointer was cleared to the session default (home)
|
|
2252
|
+
* mid-turn, but this turn started pinned to a machine and
|
|
2253
|
+
* never established the home box, so null has nothing to
|
|
2254
|
+
* resolve to THIS turn (the detach takes effect next turn). */
|
|
2255
|
+
type BackendUnresolvableCode = "stale_pointer" | "offline_enrollment" | "unsupported_backend_context" | "transient_establishment" | "home_unavailable_this_turn";
|
|
1661
2256
|
/** Thrown when a swap target cannot be resolved (unknown sandbox, or a modal
|
|
1662
|
-
* target with no establisher in this context). The caller maps it to a 409.
|
|
2257
|
+
* target with no establisher in this context). The caller maps it to a 409.
|
|
2258
|
+
* Carries a typed {@link BackendUnresolvableCode} in addition to the message. */
|
|
1663
2259
|
declare class ActiveBackendUnresolvableError extends Error {
|
|
1664
2260
|
readonly name = "ActiveBackendUnresolvableError";
|
|
1665
|
-
|
|
2261
|
+
readonly code: BackendUnresolvableCode;
|
|
2262
|
+
constructor(code: BackendUnresolvableCode, message: string);
|
|
1666
2263
|
}
|
|
2264
|
+
/** The outcome of {@link swapTargetEstablishability}. */
|
|
2265
|
+
type SwapTargetEstablishability = {
|
|
2266
|
+
ok: true;
|
|
2267
|
+
} | {
|
|
2268
|
+
ok: false;
|
|
2269
|
+
code: BackendUnresolvableCode;
|
|
2270
|
+
reason: string;
|
|
2271
|
+
};
|
|
2272
|
+
/**
|
|
2273
|
+
* THE SINGLE SOURCE OF TRUTH for "can a turn's routing context ESTABLISH this swap
|
|
2274
|
+
* target?" — shared by swap/seed ADMISSION (`fleet.resolveTarget`, before the CAS
|
|
2275
|
+
* pointer commit) and turn-time ESTABLISHMENT (this resolver + `wrapTurnBoxWithRouting`).
|
|
2276
|
+
* Admission MUST reject any target this predicate calls unestablishable, so the
|
|
2277
|
+
* pointer is never committed to a backend no turn can resume (issue #341 Shape 1:
|
|
2278
|
+
* a Modal-sibling swap was admitted + epoch-bumped, then every following op
|
|
2279
|
+
* stranded because no context wires an establisher for it).
|
|
2280
|
+
*
|
|
2281
|
+
* - the session's OWN group box (`isSessionGroup`) is the null/home target and is
|
|
2282
|
+
* always establishable (it IS the default backend);
|
|
2283
|
+
* - a `selfhosted` machine target is establishable (`buildSelfhostedBackendSession`);
|
|
2284
|
+
* admission's separate liveness probe — not this predicate — gates online-ness;
|
|
2285
|
+
* - a NON-group `modal` target is NOT establishable: no turn context wires
|
|
2286
|
+
* `establishModalTarget` (cross-group Modal resume-by-id is a future, billing-
|
|
2287
|
+
* touching feature — sandbox-routing.ts). Wiring that establisher is the ONE
|
|
2288
|
+
* toggle that flips this branch; keep admission and the resolver in lockstep.
|
|
2289
|
+
*/
|
|
2290
|
+
declare function swapTargetEstablishability(target: {
|
|
2291
|
+
kind: string;
|
|
2292
|
+
isSessionGroup: boolean;
|
|
2293
|
+
}): SwapTargetEstablishability;
|
|
1667
2294
|
/**
|
|
1668
2295
|
* Build the `resolveActiveBackend(pointer)` closure for a `RoutingSandboxSession`.
|
|
1669
2296
|
* The returned closure is re-invoked by the proxy whenever the active_epoch moves
|
|
@@ -1759,6 +2386,16 @@ type EstablishedSandboxSession = {
|
|
|
1759
2386
|
sessionState: unknown;
|
|
1760
2387
|
instanceId: string;
|
|
1761
2388
|
backendId: string;
|
|
2389
|
+
/** How this establish reached a live box: warm reattach by id ("resumed"),
|
|
2390
|
+
* fresh create ("created"), or fresh create + archive hydration
|
|
2391
|
+
* ("restored"). Callers use it to emit the durable sandbox.box.* lifecycle
|
|
2392
|
+
* events — box transitions were previously unrecorded, which made both
|
|
2393
|
+
* 2026-07-06 incidents near-unattributable. Optional so external/legacy
|
|
2394
|
+
* constructors of this shape stay valid. */
|
|
2395
|
+
origin?: "resumed" | "created" | "restored";
|
|
2396
|
+
/** Set when a create-authorized reattach found the envelope's box GONE
|
|
2397
|
+
* (provider NotFound) and fell through to cold-restore. */
|
|
2398
|
+
lostInstanceId?: string;
|
|
1762
2399
|
};
|
|
1763
2400
|
type SandboxCreatedCallback = (established: EstablishedSandboxSession) => Promise<void>;
|
|
1764
2401
|
/**
|
|
@@ -1777,21 +2414,21 @@ type SandboxCreatedCallback = (established: EstablishedSandboxSession) => Promis
|
|
|
1777
2414
|
*/
|
|
1778
2415
|
declare function isProviderSandboxNotFoundError(backendId: string, error: unknown): boolean;
|
|
1779
2416
|
/**
|
|
1780
|
-
*
|
|
1781
|
-
*
|
|
1782
|
-
*
|
|
1783
|
-
* turn activity). A null envelope means a cold session that was never warmed →
|
|
1784
|
-
* create() directly.
|
|
2417
|
+
* Establish from one recovery envelope under an explicit creation policy. The
|
|
2418
|
+
* envelope is the lease's box-identity descriptor (the same per-turn `_sandbox`
|
|
2419
|
+
* envelope upserted by the turn activity).
|
|
1785
2420
|
*
|
|
1786
2421
|
* - `opts.backendOverride ?? envelope.backendId ?? settings.sandboxBackend`
|
|
1787
2422
|
* selects the backend; the client is built for THAT backend (resume-by-id is
|
|
1788
2423
|
* fenced to the original provider).
|
|
1789
2424
|
* - warm reattach: deserialize the envelope sessionState → client.resume(state)
|
|
1790
|
-
* (no lock; R4-safe).
|
|
1791
|
-
* -
|
|
2425
|
+
* (no lock; R4-safe). `resume-only` propagates provider NotFound.
|
|
2426
|
+
* - `create-or-restore`: the elected owner may replace a missing warm instance,
|
|
2427
|
+
* or create directly from a cold/null envelope.
|
|
1792
2428
|
*/
|
|
1793
2429
|
declare function establishSandboxSessionFromEnvelope(settings: Settings, envelope: Record<string, unknown> | null, opts: {
|
|
1794
2430
|
sessionId: string;
|
|
2431
|
+
recovery: "create-or-restore" | "resume-only";
|
|
1795
2432
|
backendOverride?: SandboxBackend;
|
|
1796
2433
|
environment?: Record<string, string>;
|
|
1797
2434
|
onSandboxCreated?: SandboxCreatedCallback;
|
|
@@ -1811,4 +2448,4 @@ declare function establishSandboxSessionFromEnvelope(settings: Settings, envelop
|
|
|
1811
2448
|
*/
|
|
1812
2449
|
declare function serializeEstablishedSandboxEnvelope(established: EstablishedSandboxSession): Promise<Record<string, unknown> | null>;
|
|
1813
2450
|
|
|
1814
|
-
export {
|
|
2451
|
+
export { type OpStreamTransport as $, type ActiveBackendResolverDeps as A, type BackendUnresolvableCode as B, ChannelAConflictError as C, DEFAULT_DESKTOP_GEOMETRY as D, type EnsureDisplayStackOptions as E, FAULT_FIELD_WHAT_HAPPENED as F, type MockExecHandler as G, type ModalModuleLoader as H, type ModalOrphanSweepResult as I, type ModalSandboxAttribution as J, NatsControlRpc as K, type LiveModalSandboxLeaseAttribution as L, type MintStreamTokenInput as M, NEVER_SENT_DETAIL_KEY as N, type NatsOpStreamConnection as O, NatsOpStreamTransport as P, type NatsRequestConnection as Q, type RuntimeMetricsHooks as R, type NegotiationContext as S, type NumstatEntry as T, OP_STREAM_ACK_INTERVAL_MS as U, OP_STREAM_DEFAULT_WINDOW_BYTES as V, OP_STREAM_RECONNECT_HOLD_MS as W, OP_STREAM_SILENCE_TIMEOUT_MS as X, type OpStreamExecOutcome as Y, type OpStreamJournal as Z, type OpStreamSubscription as _, ActiveBackendUnresolvableError as a, type SwapTargetEstablishability as a$, OpStreamUnavailableError as a0, PROVIDER_REGISTRY as a1, type ProviderConstructionContext as a2, type ProviderRegistration as a3, REPOSITORY_DISCOVERY_LIMIT as a4, type RecordingArtifactMetadata as a5, type RecordingCodec as a6, type RecordingContentType as a7, RecordingError as a8, type RecordingProcess as a9, SandboxConfigError as aA, type SandboxCreatedCallback as aB, SandboxProviderUnavailableError as aC, SandboxResumeStateUnavailableError as aD, type SelfhostedApplyDiff as aE, SelfhostedControlError as aF, type SelfhostedEditor as aG, type SelfhostedEnrollment as aH, type SelfhostedExecArgs as aI, type SelfhostedExecResult as aJ, type SelfhostedImageOutput as aK, type SelfhostedLivenessState as aL, type SelfhostedNegotiationInput as aM, type SelfhostedOpObservation as aN, type SelfhostedOpObserver as aO, type SelfhostedOpStreamDeps as aP, type SelfhostedRelayConfig as aQ, type SelfhostedRetryClock as aR, type SelfhostedRetryDecision as aS, SelfhostedSandboxClient as aT, SelfhostedSession as aU, type SelfhostedSessionBuild as aV, type SelfhostedSessionDeps as aW, type SelfhostedSessionState as aX, type SelfhostedUnavailableReason as aY, type StartRecordingInput as aZ, StreamPortUnavailableError as a_, RecordingUnavailableError as aa, type RepositoryDiscoveryDegradedReason as ab, type RepositoryDiscoveryResult as ac, type ResolvedActiveBackend as ad, type RoutableBackendSession as ae, type RoutableSandbox as af, RoutingSandboxSession as ag, type RoutingSandboxSessionDeps as ah, type RoutingTransitionEvent as ai, RoutingUnsupportedError as aj, SELFHOSTED_DEFAULT_TIMEOUT_MS as ak, SELFHOSTED_DRAINING_MAX_RETRIES as al, SELFHOSTED_EXEC_DRAINING_MAX_RETRIES as am, SELFHOSTED_IDEMPOTENT_READONLY_OPS as an, SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES as ao, SELFHOSTED_NEVER_SENT_MAX_RETRIES as ap, SELFHOSTED_RECONNECT_WINDOW_MS as aq, SELFHOSTED_RELAY_STREAM_PATH as ar, SELFHOSTED_RETRY_BACKOFF_BASE_MS as as, SELFHOSTED_RETRY_BACKOFF_CAP_MS as at, SELFHOSTED_RETRY_BACKOFF_FACTOR as au, SELFHOSTED_TIMEOUT_MAX_RETRIES as av, STREAM_PORT as aw, STREAM_TOKEN_DEFAULT_TTL_SECONDS as ax, SandboxChannelAService as ay, type SandboxChannelAServiceOptions as az, type ActivePointer as b, serializeEstablishedSandboxEnvelope as b$, TERMINAL_SERVER_TIMEOUT_MS as b0, TerminalServerError as b1, TerminalServerUnsupportedError as b2, agentErrorToControlError as b3, assertDescriptorRegistryInvariants as b4, assertProviderRegistryInvariants as b5, assertSafeRelPath as b6, backendSupportsOs as b7, buildDisplayStackScript as b8, buildSelfhostedBackendSession as b9, makeActiveBackendResolver as bA, mintStreamToken as bB, modalSandboxAttributionEnvironment as bC, modalSandboxAttributionTags as bD, negotiateCapabilities as bE, negotiateSelfhostedCapabilities as bF, nextDurableOpId as bG, offlineAgentError as bH, offlineControlResponse as bI, opAckSubject as bJ, opFrameSubject as bK, parseExecBannerSessionId as bL, parseNumstatZ as bM, parsePorcelainV2 as bN, parseUnifiedPatch as bO, payloadTooLargeMessage as bP, readWorkspaceArchiveFromEnvelopeSessionState as bQ, recordingStorageKey as bR, renderSelfhostedFault as bS, restoredSandboxSessionStateFromEntry as bT, runWithToolCallCorrelation as bU, sandboxStateEntryFromRunState as bV, sanitizeOpIdToken as bW, selectBackend as bX, selfhostedFaultClass as bY, selfhostedLiveness as bZ, selfhostedRetryBackoffMs as b_, buildStreamUrl as ba, buildTerminalServerScript as bb, contentTypeForCodec as bc, createSandboxClient as bd, createSandboxClientForBackend as be, decideSelfhostedRetry as bf, decodeModalSnapshotId as bg, defaultSelfhostedRetryClock as bh, deletePriorPersistedSnapshot as bi, deleteRecordingArtifacts as bj, deserializeSandboxSessionStateEnvelope as bk, desktopCapableBackend as bl, drainingExhaustedError as bm, drainingMessage as bn, ensureDisplayStack as bo, ensureModalRegistryImage as bp, ensureTerminalServer as bq, establishSandboxSessionFromEnvelope as br, execDeadlineHint as bs, exposeStreamPort as bt, extForCodec as bu, inspectRecordingArtifact as bv, isExecSessionLostBanner as bw, isProviderSandboxNotFoundError as bx, isSelfhostedProviderNotFoundError as by, isWorkspaceEscapeError as bz, type ChannelAEmitter as c, setSelfhostedApplyDiff as c0, startRecording as c1, stopRecording as c2, stripExecBanner as c3, subjectFor as c4, swapTargetEstablishability as c5, sweepModalOrphanSandboxes as c6, tagModalSandbox as c7, tearDownDisplayStack as c8, tearDownTerminalServer as c9, terminateModalSandboxById as ca, timeoutAgentError as cb, timeoutControlResponse as cc, uploadRecordingArtifact as cd, verifyStreamToken as ce, type ChannelAExecArgs as d, type ChannelAExecResult as e, ChannelANotFoundError as f, type ChannelASession as g, ChannelAUnsupportedError as h, ChannelAValidationError as i, type ControlRpc as j, DISPLAY_STACK_TIMEOUT_MS as k, type DesktopGeometry as l, DisplayStackError as m, DisplayStackUnsupportedError as n, type EnsureDisplayStackResult as o, type EnsureTerminalServerOptions as p, type EnsureTerminalServerResult as q, type EstablishedSandboxSession as r, type ExposeStreamPortInput as s, type ExposeStreamPortResult as t, type ExposedPortEndpoint as u, FAULT_FIELD_WHAT_PRESERVED as v, FAULT_FIELD_WHAT_TO_TRY as w, FAULT_FIELD_WHICH_LAYER as x, MockAgentResponder as y, type MockAgentResponderOptions as z };
|