@opengeni/runtime 0.11.0 → 0.13.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-YUAFQEFY.js → chunk-JGUQSAG5.js} +2605 -279
- package/dist/chunk-JGUQSAG5.js.map +1 -0
- package/dist/chunk-SSB7VOLE.js +301 -0
- package/dist/chunk-SSB7VOLE.js.map +1 -0
- package/dist/{index-D6yi-JcU.d.ts → index-DU-KXVlx.d.ts} +364 -31
- package/dist/index.d.ts +103 -10
- package/dist/index.js +756 -761
- package/dist/index.js.map +1 -1
- package/dist/mcp-network.d.ts +80 -0
- package/dist/mcp-network.js +66 -0
- package/dist/mcp-network.js.map +1 -0
- package/dist/sandbox/index.d.ts +1 -1
- package/dist/sandbox/index.js +39 -1
- package/package.json +9 -4
- package/src/codex-tool-search.ts +20 -1
- package/src/index.ts +502 -85
- package/src/mcp-network.ts +343 -0
- package/src/sandbox/channel-a.ts +92 -71
- package/src/sandbox/display-stack.ts +860 -23
- package/src/sandbox/exec-banner.ts +82 -0
- package/src/sandbox/index.ts +367 -145
- package/src/sandbox/provider-errors.ts +175 -0
- package/src/sandbox/providers/modal.ts +29 -2
- package/src/sandbox/routing/backend-resolver.ts +15 -0
- package/src/sandbox/routing/routing-session.ts +730 -36
- package/src/sandbox/select.ts +8 -0
- package/src/sandbox/turn-tool-cancellation.ts +366 -92
- package/src/sandbox/workspace-archive.ts +289 -0
- package/src/sandbox-computer.ts +3 -1
- package/src/usage-telemetry.ts +495 -51
- package/dist/chunk-YUAFQEFY.js.map +0 -1
package/src/sandbox/index.ts
CHANGED
|
@@ -32,7 +32,14 @@ import type {
|
|
|
32
32
|
} from "@openai/agents/sandbox";
|
|
33
33
|
import { PROVIDER_REGISTRY } from "./providers";
|
|
34
34
|
import { SandboxConfigError, SandboxResumeStateUnavailableError } from "./errors";
|
|
35
|
-
import {
|
|
35
|
+
import { isProviderSandboxNotFoundError } from "./provider-errors";
|
|
36
|
+
import {
|
|
37
|
+
readVerifiedWorkspaceArchive,
|
|
38
|
+
verifyRestoredWorkspace,
|
|
39
|
+
WorkspaceArchiveIntegrityError,
|
|
40
|
+
type VerifiedWorkspaceArchive,
|
|
41
|
+
type WorkspaceArchiveDescriptor,
|
|
42
|
+
} from "./workspace-archive";
|
|
36
43
|
import type { RuntimeMetricsHooks } from "../metrics";
|
|
37
44
|
|
|
38
45
|
// Re-export the config-owned environment/port helpers from the leaf so the
|
|
@@ -62,6 +69,27 @@ export {
|
|
|
62
69
|
type ProviderRegistration,
|
|
63
70
|
type ProviderConstructionContext,
|
|
64
71
|
} from "./providers";
|
|
72
|
+
export {
|
|
73
|
+
classifyProviderSandboxFailure,
|
|
74
|
+
isProviderSandboxGoneDuringRoutedOperation,
|
|
75
|
+
isProviderSandboxNotFoundError,
|
|
76
|
+
isProviderSandboxTransientError,
|
|
77
|
+
type ProviderSandboxFailure,
|
|
78
|
+
type ProviderSandboxFailureKind,
|
|
79
|
+
} from "./provider-errors";
|
|
80
|
+
export {
|
|
81
|
+
WORKSPACE_ARCHIVE_DESCRIPTOR_VERSION,
|
|
82
|
+
WorkspaceArchiveIntegrityError,
|
|
83
|
+
captureVerifiedWorkspaceArchive,
|
|
84
|
+
fingerprintSandboxWorkspace,
|
|
85
|
+
parseWorkspaceArchiveDescriptor,
|
|
86
|
+
readVerifiedWorkspaceArchive,
|
|
87
|
+
verifyRestoredWorkspace,
|
|
88
|
+
type VerifiedWorkspaceArchive,
|
|
89
|
+
type WorkspaceArchiveDescriptor,
|
|
90
|
+
type WorkspaceArchiveIntegrityCode,
|
|
91
|
+
type WorkspaceTreeFingerprint,
|
|
92
|
+
} from "./workspace-archive";
|
|
65
93
|
export {
|
|
66
94
|
ensureModalRegistryImage,
|
|
67
95
|
modalSandboxAttributionEnvironment,
|
|
@@ -72,7 +100,9 @@ export {
|
|
|
72
100
|
type LiveModalSandboxLeaseAttribution,
|
|
73
101
|
type ModalModuleLoader,
|
|
74
102
|
type ModalOrphanSweepResult,
|
|
103
|
+
type ModalOrphanSweepTermination,
|
|
75
104
|
type ModalSandboxAttribution,
|
|
105
|
+
type RevalidateModalOrphanTermination,
|
|
76
106
|
} from "./providers/modal";
|
|
77
107
|
export {
|
|
78
108
|
selectBackend,
|
|
@@ -106,6 +136,11 @@ export {
|
|
|
106
136
|
ensureDisplayStack,
|
|
107
137
|
tearDownDisplayStack,
|
|
108
138
|
type DesktopGeometry,
|
|
139
|
+
type DisplayStackCallerKind,
|
|
140
|
+
type DisplayStackClassification,
|
|
141
|
+
type DisplayStackTelemetryContext,
|
|
142
|
+
type DisplayStackTelemetryEvent,
|
|
143
|
+
type DisplayStackTelemetryStatus,
|
|
109
144
|
type EnsureDisplayStackOptions,
|
|
110
145
|
type EnsureDisplayStackResult,
|
|
111
146
|
} from "./display-stack";
|
|
@@ -335,11 +370,18 @@ export {
|
|
|
335
370
|
// currently-active backend (Modal or selfhosted) — flippable mid-turn, single
|
|
336
371
|
// active at a time, fence-retrying on a swap race.
|
|
337
372
|
export {
|
|
373
|
+
RoutingBackendRecoveryRequiredError,
|
|
374
|
+
RoutingMutationOutcomeUnknownError,
|
|
375
|
+
RoutingRetainedProcessNotFoundError,
|
|
338
376
|
RoutingSandboxSession,
|
|
339
377
|
RoutingUnsupportedError,
|
|
340
378
|
type ActivePointer,
|
|
379
|
+
type DefaultBackendLossResult,
|
|
341
380
|
type RoutableBackendSession,
|
|
342
381
|
type ResolvedActiveBackend,
|
|
382
|
+
type RoutingRetainedProcess,
|
|
383
|
+
type RoutingRetainedProcessAdoption,
|
|
384
|
+
type RoutingRetainedProcessTerminalProof,
|
|
343
385
|
type RoutingSandboxSessionDeps,
|
|
344
386
|
type RoutingTransitionEvent,
|
|
345
387
|
} from "./routing/routing-session";
|
|
@@ -585,24 +627,6 @@ export function readWorkspaceArchiveFromEnvelopeSessionState(
|
|
|
585
627
|
}
|
|
586
628
|
}
|
|
587
629
|
|
|
588
|
-
function readWorkspaceArchivePairFromEnvelopeSessionState(sessionState: unknown): {
|
|
589
|
-
current?: Uint8Array;
|
|
590
|
-
previous?: Uint8Array;
|
|
591
|
-
} {
|
|
592
|
-
if (!sessionState || typeof sessionState !== "object") {
|
|
593
|
-
return {};
|
|
594
|
-
}
|
|
595
|
-
const state = sessionState as { workspaceArchivePrev?: unknown };
|
|
596
|
-
const current = readWorkspaceArchiveFromEnvelopeSessionState(sessionState);
|
|
597
|
-
const previous = readWorkspaceArchiveFromEnvelopeSessionState({
|
|
598
|
-
workspaceArchive: state.workspaceArchivePrev,
|
|
599
|
-
});
|
|
600
|
-
return {
|
|
601
|
-
...(current ? { current } : {}),
|
|
602
|
-
...(previous ? { previous } : {}),
|
|
603
|
-
};
|
|
604
|
-
}
|
|
605
|
-
|
|
606
630
|
// The native snapshot-ref prefixes the @openai/agents-extensions modal client
|
|
607
631
|
// encodes (snapshots.mjs `NATIVE_SNAPSHOT_PREFIXES`). The ref is
|
|
608
632
|
// `<PREFIX>\n{"snapshot_id":"...",...}`. We re-implement the decode here because
|
|
@@ -757,8 +781,113 @@ export type EstablishedSandboxSession = {
|
|
|
757
781
|
/** Set when a create-authorized reattach found the envelope's box GONE
|
|
758
782
|
* (provider NotFound) and fell through to cold-restore. */
|
|
759
783
|
lostInstanceId?: string;
|
|
784
|
+
/** Exact durable archive revision verified byte-for-byte after hydration. */
|
|
785
|
+
restoredArchive?: WorkspaceArchiveDescriptor;
|
|
760
786
|
};
|
|
761
787
|
|
|
788
|
+
export class SandboxExecReadinessError extends Error {
|
|
789
|
+
readonly name = "SandboxExecReadinessError";
|
|
790
|
+
|
|
791
|
+
constructor(
|
|
792
|
+
public readonly backend: string,
|
|
793
|
+
public readonly code: "exec_probe_unavailable" | "exec_probe_timeout" | "exec_probe_failed",
|
|
794
|
+
public readonly timeoutMs: number,
|
|
795
|
+
public readonly exitCode: number | null = null,
|
|
796
|
+
) {
|
|
797
|
+
super(
|
|
798
|
+
code === "exec_probe_timeout"
|
|
799
|
+
? `sandbox creation timed out waiting for ${backend} command readiness after ${timeoutMs}ms`
|
|
800
|
+
: code === "exec_probe_unavailable"
|
|
801
|
+
? `${backend} sandbox session does not expose an exec readiness probe`
|
|
802
|
+
: exitCode === null
|
|
803
|
+
? `${backend} sandbox exec readiness probe did not return a command exit code`
|
|
804
|
+
: `${backend} sandbox exec readiness probe failed with exit code ${exitCode}`,
|
|
805
|
+
);
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
function sandboxExecProbeExitCode(result: unknown): number | null {
|
|
810
|
+
if (typeof result === "string") {
|
|
811
|
+
const match = result.match(/Process exited with code (-?\d+)/);
|
|
812
|
+
return match ? Number(match[1]) : null;
|
|
813
|
+
}
|
|
814
|
+
if (!result || typeof result !== "object") return null;
|
|
815
|
+
const candidate = result as {
|
|
816
|
+
exitCode?: unknown;
|
|
817
|
+
exit_code?: unknown;
|
|
818
|
+
code?: unknown;
|
|
819
|
+
status?: unknown;
|
|
820
|
+
};
|
|
821
|
+
for (const value of [candidate.exitCode, candidate.exit_code, candidate.code, candidate.status]) {
|
|
822
|
+
if (typeof value === "number") return value;
|
|
823
|
+
}
|
|
824
|
+
return null;
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
function sandboxExecProbeStillRunning(result: unknown): boolean {
|
|
828
|
+
if (typeof result === "string") return /Process running with session ID \d+/u.test(result);
|
|
829
|
+
if (!result || typeof result !== "object") return false;
|
|
830
|
+
const candidate = result as { sessionId?: unknown; session_id?: unknown };
|
|
831
|
+
return typeof candidate.sessionId === "number" || typeof candidate.session_id === "number";
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
/** A provider handle is not workspace readiness. Modal can return a handle
|
|
835
|
+
* before its command router accepts exec, so every create path must pass this
|
|
836
|
+
* bounded probe before atomically publishing the lease warm/ready. */
|
|
837
|
+
export async function verifySandboxExecReadiness(
|
|
838
|
+
established: EstablishedSandboxSession,
|
|
839
|
+
timeoutMs = 15_000,
|
|
840
|
+
): Promise<void> {
|
|
841
|
+
if (established.backendId !== "modal") return;
|
|
842
|
+
const session = established.session as {
|
|
843
|
+
exec?: (args: {
|
|
844
|
+
cmd: string;
|
|
845
|
+
yieldTimeMs?: number;
|
|
846
|
+
maxOutputTokens?: number;
|
|
847
|
+
}) => Promise<unknown>;
|
|
848
|
+
execCommand?: (args: {
|
|
849
|
+
cmd: string;
|
|
850
|
+
yieldTimeMs?: number;
|
|
851
|
+
maxOutputTokens?: number;
|
|
852
|
+
}) => Promise<unknown>;
|
|
853
|
+
};
|
|
854
|
+
const run = session.exec ?? session.execCommand;
|
|
855
|
+
if (!run) {
|
|
856
|
+
throw new SandboxExecReadinessError(established.backendId, "exec_probe_unavailable", timeoutMs);
|
|
857
|
+
}
|
|
858
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
859
|
+
try {
|
|
860
|
+
const result = await Promise.race([
|
|
861
|
+
run.call(session, {
|
|
862
|
+
cmd: "true",
|
|
863
|
+
yieldTimeMs: 1_000,
|
|
864
|
+
maxOutputTokens: 1_000,
|
|
865
|
+
}),
|
|
866
|
+
new Promise<never>((_, reject) => {
|
|
867
|
+
timer = setTimeout(
|
|
868
|
+
() =>
|
|
869
|
+
reject(
|
|
870
|
+
new SandboxExecReadinessError(established.backendId, "exec_probe_timeout", timeoutMs),
|
|
871
|
+
),
|
|
872
|
+
timeoutMs,
|
|
873
|
+
);
|
|
874
|
+
if (timer && "unref" in timer && typeof timer.unref === "function") timer.unref();
|
|
875
|
+
}),
|
|
876
|
+
]);
|
|
877
|
+
const exitCode = sandboxExecProbeExitCode(result);
|
|
878
|
+
if (sandboxExecProbeStillRunning(result) || exitCode !== 0) {
|
|
879
|
+
throw new SandboxExecReadinessError(
|
|
880
|
+
established.backendId,
|
|
881
|
+
"exec_probe_failed",
|
|
882
|
+
timeoutMs,
|
|
883
|
+
exitCode,
|
|
884
|
+
);
|
|
885
|
+
}
|
|
886
|
+
} finally {
|
|
887
|
+
if (timer) clearTimeout(timer);
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
|
|
762
891
|
export type SandboxCreatedCallback = (established: EstablishedSandboxSession) => Promise<void>;
|
|
763
892
|
|
|
764
893
|
// The structural slice we need from a provider SandboxClient to resume by id and
|
|
@@ -785,64 +914,6 @@ type ResumeCapableClient = {
|
|
|
785
914
|
* an unrecognized error is treated as "not NotFound" (propagate), because a
|
|
786
915
|
* false-positive recreate is the dangerous direction (double-spawn).
|
|
787
916
|
*/
|
|
788
|
-
export function isProviderSandboxNotFoundError(backendId: string, error: unknown): boolean {
|
|
789
|
-
// selfhosted: agent-offline is NEVER a provider NotFound (the user's machine is
|
|
790
|
-
// not recreatable — a false NotFound would cold-create a RIVAL box). The
|
|
791
|
-
// selfhosted discriminator ALWAYS returns false; short-circuit so no goneMarker
|
|
792
|
-
// string match below can ever flip a selfhosted agent-offline error to true.
|
|
793
|
-
if (backendId === "selfhosted") {
|
|
794
|
-
return isSelfhostedProviderNotFoundError(error);
|
|
795
|
-
}
|
|
796
|
-
if (!error) {
|
|
797
|
-
return false;
|
|
798
|
-
}
|
|
799
|
-
const status =
|
|
800
|
-
(error as { status?: unknown; statusCode?: unknown }).status ??
|
|
801
|
-
(error as { statusCode?: unknown }).statusCode;
|
|
802
|
-
if (status === 404) {
|
|
803
|
-
return true;
|
|
804
|
-
}
|
|
805
|
-
const name =
|
|
806
|
-
typeof (error as { name?: unknown }).name === "string" ? (error as { name: string }).name : "";
|
|
807
|
-
const code =
|
|
808
|
-
typeof (error as { code?: unknown }).code === "string" ? (error as { code: string }).code : "";
|
|
809
|
-
const message =
|
|
810
|
-
error instanceof Error
|
|
811
|
-
? error.message
|
|
812
|
-
: typeof error === "string"
|
|
813
|
-
? error
|
|
814
|
-
: String((error as { message?: unknown })?.message ?? "");
|
|
815
|
-
const haystack = `${name} ${code} ${message}`.toLowerCase();
|
|
816
|
-
// Provider-agnostic "gone" markers (Modal: "sandbox … not found" / terminated;
|
|
817
|
-
// e2b/daytona/runloop: "not found" / "no longer running" / "terminated" /
|
|
818
|
-
// "does not exist"). Kept broad-but-conservative: it matches box-gone phrasing
|
|
819
|
-
// and never matches generic 5xx/transport errors.
|
|
820
|
-
const goneMarkers = [
|
|
821
|
-
"not found",
|
|
822
|
-
"no longer running",
|
|
823
|
-
"no longer exists",
|
|
824
|
-
"does not exist",
|
|
825
|
-
"doesn't exist",
|
|
826
|
-
"has been terminated",
|
|
827
|
-
"was terminated",
|
|
828
|
-
"is terminated",
|
|
829
|
-
"sandbox terminated",
|
|
830
|
-
"notfound",
|
|
831
|
-
"sandbox_not_found",
|
|
832
|
-
"box no longer running",
|
|
833
|
-
];
|
|
834
|
-
// A "running"/"already exists" resume-conflict is explicitly NOT NotFound — the
|
|
835
|
-
// box is alive; recreating would double-spawn.
|
|
836
|
-
if (
|
|
837
|
-
haystack.includes("already running") ||
|
|
838
|
-
haystack.includes("still running") ||
|
|
839
|
-
haystack.includes("already exists")
|
|
840
|
-
) {
|
|
841
|
-
return false;
|
|
842
|
-
}
|
|
843
|
-
return goneMarkers.some((marker) => haystack.includes(marker));
|
|
844
|
-
}
|
|
845
|
-
|
|
846
917
|
function readInstanceId(session: unknown): string {
|
|
847
918
|
const state = (session as { state?: Record<string, unknown> }).state ?? {};
|
|
848
919
|
const candidate =
|
|
@@ -903,6 +974,10 @@ export async function establishSandboxSessionFromEnvelope(
|
|
|
903
974
|
backendOverride?: SandboxBackend;
|
|
904
975
|
environment?: Record<string, string>;
|
|
905
976
|
onSandboxCreated?: SandboxCreatedCallback;
|
|
977
|
+
/** Called after archive hydration but immediately before the exact workspace
|
|
978
|
+
* fingerprint probe. Lease-aware callers persist `verifying` here so a box
|
|
979
|
+
* is never observable as ready while verification is in flight. */
|
|
980
|
+
onWorkspaceRestoreVerifying?: (descriptor: WorkspaceArchiveDescriptor) => Promise<void>;
|
|
906
981
|
metrics?: RuntimeMetricsHooks;
|
|
907
982
|
},
|
|
908
983
|
): Promise<EstablishedSandboxSession> {
|
|
@@ -951,19 +1026,35 @@ export async function establishSandboxSessionFromEnvelope(
|
|
|
951
1026
|
// - COLD lease re-warm (confirmDrainCold preserved a MINIMAL archive-only
|
|
952
1027
|
// envelope `{ sessionState: { workspaceArchive } }` — NO sandboxId, so the
|
|
953
1028
|
// warm-reattach branch must NOT try resume()-by-id; it cold-creates+hydrates).
|
|
954
|
-
const
|
|
955
|
-
|
|
1029
|
+
const archiveState =
|
|
1030
|
+
envelopeSessionState && typeof envelopeSessionState === "object"
|
|
1031
|
+
? (envelopeSessionState as {
|
|
1032
|
+
workspaceArchive?: unknown;
|
|
1033
|
+
workspaceArchiveMeta?: unknown;
|
|
1034
|
+
})
|
|
1035
|
+
: undefined;
|
|
1036
|
+
// Exactly one selected revision is restored. A present archive without its
|
|
1037
|
+
// revision/hash/tree descriptor is not recoverable truth and fails before any
|
|
1038
|
+
// provider create. `workspaceArchivePrev` is retained for explicit future
|
|
1039
|
+
// operator selection, never silently substituted in this attempt.
|
|
1040
|
+
const workspaceArchiveBase64 = archiveState?.workspaceArchive;
|
|
1041
|
+
const workspaceArchiveMetadata = archiveState?.workspaceArchiveMeta;
|
|
956
1042
|
|
|
957
1043
|
// create() a FRESH box, THEN replay the persisted /workspace snapshot via
|
|
958
1044
|
// session.hydrateWorkspace(archive) when one rode the envelope. hydrateWorkspace
|
|
959
1045
|
// decodes the snapshot-ref and swaps the box for one booted from the snapshot
|
|
960
|
-
// image (restoreSnapshotFilesystem)
|
|
961
|
-
//
|
|
962
|
-
//
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
1046
|
+
// image (restoreSnapshotFilesystem). No archive is valid only for a genuinely
|
|
1047
|
+
// new workspace; recovery callers select and verify an exact archive before
|
|
1048
|
+
// entering this seam. This is the SOLE archive-replay path, shared by the
|
|
1049
|
+
// NotFound warm-reattach path and the cold-restore branch (b) below.
|
|
1050
|
+
const coldRestore = async (resumeFallbackState?: unknown): Promise<EstablishedSandboxSession> => {
|
|
1051
|
+
// Parse/verify lazily: a warm resume-by-id does not consume its retained
|
|
1052
|
+
// archive, so a legacy live box remains resumable. Creation/restoration is
|
|
1053
|
+
// the boundary that requires complete durable metadata.
|
|
1054
|
+
const workspaceArchive: VerifiedWorkspaceArchive | null = readVerifiedWorkspaceArchive(
|
|
1055
|
+
workspaceArchiveBase64,
|
|
1056
|
+
workspaceArchiveMetadata,
|
|
1057
|
+
);
|
|
967
1058
|
const createStarted = Date.now();
|
|
968
1059
|
let restored: Awaited<ReturnType<NonNullable<typeof client.create>>>;
|
|
969
1060
|
try {
|
|
@@ -989,73 +1080,79 @@ export async function establishSandboxSessionFromEnvelope(
|
|
|
989
1080
|
throw createCallbackError;
|
|
990
1081
|
}
|
|
991
1082
|
}
|
|
992
|
-
// Whether an archive was ACTUALLY applied to /workspace — drives `origin`
|
|
993
|
-
// (and the worker's sandbox.box.created `hydrated` field). A clean-box
|
|
994
|
-
// fallback, a backend without hydrateWorkspace, or an all-archives-failed
|
|
995
|
-
// path leaves this false so we never report `hydrated: "archive"` for an
|
|
996
|
-
// empty workspace.
|
|
997
1083
|
let hydrationApplied = false;
|
|
998
|
-
if (
|
|
1084
|
+
if (workspaceArchive) {
|
|
999
1085
|
const hydrate = (restored as { hydrateWorkspace?: (data: Uint8Array) => Promise<void> })
|
|
1000
1086
|
.hydrateWorkspace;
|
|
1001
|
-
if (typeof hydrate
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1087
|
+
if (typeof hydrate !== "function") {
|
|
1088
|
+
await terminateCreatedSandbox(client, restored, restoredState);
|
|
1089
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
1090
|
+
"archive_hydration_failed",
|
|
1091
|
+
`sandbox backend ${client.backendId} cannot hydrate selected archive revision ${workspaceArchive.descriptor.revision}`,
|
|
1092
|
+
);
|
|
1093
|
+
}
|
|
1094
|
+
try {
|
|
1095
|
+
// hydrateWorkspace may internally replace the underlying box.
|
|
1096
|
+
await hydrate.call(restored, workspaceArchive.bytes);
|
|
1097
|
+
} catch (error) {
|
|
1098
|
+
await terminateCreatedSandbox(client, restored, (restored as { state?: unknown }).state);
|
|
1099
|
+
if (error instanceof WorkspaceArchiveIntegrityError) throw error;
|
|
1100
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
1101
|
+
"archive_hydration_failed",
|
|
1102
|
+
`failed to hydrate selected workspace archive revision ${workspaceArchive.descriptor.revision}`,
|
|
1103
|
+
{ retryable: true },
|
|
1104
|
+
);
|
|
1105
|
+
}
|
|
1106
|
+
// hydrateWorkspace may replace the provider box (Modal's native snapshot
|
|
1107
|
+
// restore does this). Attribute the newly-active identity immediately,
|
|
1108
|
+
// before restore-state marking, fingerprint verification, or any caller
|
|
1109
|
+
// can publish the box warm. The callback is the durable lease/tagging
|
|
1110
|
+
// boundary; if it cannot persist the replacement, the caller fails closed
|
|
1111
|
+
// and this exact replacement is terminated below.
|
|
1112
|
+
const hydratedState = (restored as { state?: unknown }).state;
|
|
1113
|
+
const hydratedInstanceId = readInstanceId(restored);
|
|
1114
|
+
if (hydratedInstanceId && hydratedInstanceId !== established.instanceId) {
|
|
1115
|
+
established = {
|
|
1116
|
+
client,
|
|
1117
|
+
session: restored,
|
|
1118
|
+
sessionState: hydratedState ?? resumeFallbackState,
|
|
1119
|
+
instanceId: hydratedInstanceId,
|
|
1120
|
+
backendId: client.backendId,
|
|
1121
|
+
};
|
|
1122
|
+
if (opts.onSandboxCreated) {
|
|
1010
1123
|
try {
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
console.info(
|
|
1016
|
-
`[sandbox] cold-restore hydrated workspace from ${candidate.label} archive`,
|
|
1017
|
-
);
|
|
1018
|
-
hydrated = true;
|
|
1019
|
-
break;
|
|
1020
|
-
} catch (hydrateError) {
|
|
1021
|
-
console.warn(
|
|
1022
|
-
`[sandbox] cold-restore failed to hydrate workspace from ${candidate.label} archive${
|
|
1023
|
-
candidate.label === "current" && workspaceArchives.previous
|
|
1024
|
-
? "; trying previous archive"
|
|
1025
|
-
: ""
|
|
1026
|
-
}`,
|
|
1027
|
-
hydrateError,
|
|
1028
|
-
);
|
|
1124
|
+
await opts.onSandboxCreated(established);
|
|
1125
|
+
} catch (createCallbackError) {
|
|
1126
|
+
await terminateCreatedSandbox(client, restored, hydratedState);
|
|
1127
|
+
throw createCallbackError;
|
|
1029
1128
|
}
|
|
1030
1129
|
}
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
hydrationApplied = true;
|
|
1039
|
-
const hydratedState = (restored as { state?: unknown }).state;
|
|
1040
|
-
const hydratedInstanceId = readInstanceId(restored);
|
|
1041
|
-
if (hydratedInstanceId && hydratedInstanceId !== established.instanceId) {
|
|
1042
|
-
established = {
|
|
1043
|
-
client,
|
|
1044
|
-
session: restored,
|
|
1045
|
-
sessionState: hydratedState ?? resumeFallbackState,
|
|
1046
|
-
instanceId: hydratedInstanceId,
|
|
1047
|
-
backendId: client.backendId,
|
|
1048
|
-
};
|
|
1049
|
-
if (opts.onSandboxCreated) {
|
|
1050
|
-
try {
|
|
1051
|
-
await opts.onSandboxCreated(established);
|
|
1052
|
-
} catch (createCallbackError) {
|
|
1053
|
-
await terminateCreatedSandbox(client, restored, hydratedState);
|
|
1054
|
-
throw createCallbackError;
|
|
1055
|
-
}
|
|
1056
|
-
}
|
|
1130
|
+
}
|
|
1131
|
+
if (opts.onWorkspaceRestoreVerifying) {
|
|
1132
|
+
try {
|
|
1133
|
+
await opts.onWorkspaceRestoreVerifying(workspaceArchive.descriptor);
|
|
1134
|
+
} catch (error) {
|
|
1135
|
+
await terminateCreatedSandbox(client, restored, (restored as { state?: unknown }).state);
|
|
1136
|
+
throw error;
|
|
1057
1137
|
}
|
|
1058
1138
|
}
|
|
1139
|
+
try {
|
|
1140
|
+
// Verification happens on the hydrated replacement before any caller
|
|
1141
|
+
// may publish it warm or route an operation to it.
|
|
1142
|
+
await verifyRestoredWorkspace(restored, workspaceArchive.descriptor);
|
|
1143
|
+
} catch (error) {
|
|
1144
|
+
await terminateCreatedSandbox(client, restored, (restored as { state?: unknown }).state);
|
|
1145
|
+
if (error instanceof WorkspaceArchiveIntegrityError) throw error;
|
|
1146
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
1147
|
+
"workspace_fingerprint_unavailable",
|
|
1148
|
+
`failed to verify selected workspace archive revision ${workspaceArchive.descriptor.revision}`,
|
|
1149
|
+
{ retryable: true },
|
|
1150
|
+
);
|
|
1151
|
+
}
|
|
1152
|
+
hydrationApplied = true;
|
|
1153
|
+
console.info(
|
|
1154
|
+
`[sandbox] cold-restore verified workspace archive revision ${workspaceArchive.descriptor.revision}`,
|
|
1155
|
+
);
|
|
1059
1156
|
}
|
|
1060
1157
|
restoredState = (restored as { state?: unknown }).state;
|
|
1061
1158
|
return {
|
|
@@ -1065,6 +1162,7 @@ export async function establishSandboxSessionFromEnvelope(
|
|
|
1065
1162
|
instanceId: readInstanceId(restored),
|
|
1066
1163
|
backendId: client.backendId,
|
|
1067
1164
|
origin: hydrationApplied ? ("restored" as const) : ("created" as const),
|
|
1165
|
+
...(workspaceArchive ? { restoredArchive: workspaceArchive.descriptor } : {}),
|
|
1068
1166
|
};
|
|
1069
1167
|
};
|
|
1070
1168
|
|
|
@@ -1239,3 +1337,127 @@ export async function serializeEstablishedSandboxEnvelope(
|
|
|
1239
1337
|
return null;
|
|
1240
1338
|
}
|
|
1241
1339
|
}
|
|
1340
|
+
|
|
1341
|
+
const PROVIDER_IDENTITY_FIELDS = [
|
|
1342
|
+
"sandboxId",
|
|
1343
|
+
"instanceId",
|
|
1344
|
+
"id",
|
|
1345
|
+
"hostId",
|
|
1346
|
+
"containerId",
|
|
1347
|
+
"workspaceRootPath",
|
|
1348
|
+
"agentId",
|
|
1349
|
+
] as const;
|
|
1350
|
+
|
|
1351
|
+
/**
|
|
1352
|
+
* Whether a replacement envelope contains a provider-resumable identity. An
|
|
1353
|
+
* archive-only envelope is intentionally valid recovery metadata, but it is
|
|
1354
|
+
* never proof that the newly-created provider box can be resumed or attached.
|
|
1355
|
+
*/
|
|
1356
|
+
export function hasPersistableSandboxProviderIdentity(
|
|
1357
|
+
envelope: Record<string, unknown> | null | undefined,
|
|
1358
|
+
): envelope is Record<string, unknown> {
|
|
1359
|
+
const sessionState =
|
|
1360
|
+
envelope?.sessionState && typeof envelope.sessionState === "object"
|
|
1361
|
+
? (envelope.sessionState as Record<string, unknown>)
|
|
1362
|
+
: null;
|
|
1363
|
+
const providerState =
|
|
1364
|
+
sessionState?.providerState && typeof sessionState.providerState === "object"
|
|
1365
|
+
? (sessionState.providerState as Record<string, unknown>)
|
|
1366
|
+
: null;
|
|
1367
|
+
return Boolean(
|
|
1368
|
+
providerState &&
|
|
1369
|
+
PROVIDER_IDENTITY_FIELDS.some(
|
|
1370
|
+
(field) => typeof providerState[field] === "string" && providerState[field].length > 0,
|
|
1371
|
+
),
|
|
1372
|
+
);
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1375
|
+
/** A replacement provider may not be published warm without a resumable
|
|
1376
|
+
* identity. Archive-only state is retained only when the caller rolls back to
|
|
1377
|
+
* cold and needs the verified workspace bytes for the next attempt. */
|
|
1378
|
+
export class SandboxReplacementProviderStateError extends Error {
|
|
1379
|
+
readonly code = "replacement_provider_state_unpersistable" as const;
|
|
1380
|
+
|
|
1381
|
+
constructor(public readonly backend: string) {
|
|
1382
|
+
super(
|
|
1383
|
+
`sandbox backend "${backend}" replacement provider state could not be serialized; refusing to publish an attachable lease`,
|
|
1384
|
+
);
|
|
1385
|
+
this.name = "SandboxReplacementProviderStateError";
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
export function requirePersistableReplacementSandboxEnvelope(
|
|
1390
|
+
envelope: Record<string, unknown> | null,
|
|
1391
|
+
backend: string,
|
|
1392
|
+
): Record<string, unknown> {
|
|
1393
|
+
if (!hasPersistableSandboxProviderIdentity(envelope)) {
|
|
1394
|
+
throw new SandboxReplacementProviderStateError(backend);
|
|
1395
|
+
}
|
|
1396
|
+
return envelope;
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
const DURABLE_WORKSPACE_ARCHIVE_FIELDS = [
|
|
1400
|
+
"workspaceArchive",
|
|
1401
|
+
"workspaceArchiveMeta",
|
|
1402
|
+
"workspaceArchivePrev",
|
|
1403
|
+
"workspaceArchivePrevMeta",
|
|
1404
|
+
"workspaceArchiveAt",
|
|
1405
|
+
] as const;
|
|
1406
|
+
|
|
1407
|
+
function durableWorkspaceArchiveFields(
|
|
1408
|
+
envelope: Record<string, unknown> | null,
|
|
1409
|
+
): Record<string, unknown> | null {
|
|
1410
|
+
const sessionState =
|
|
1411
|
+
envelope?.sessionState && typeof envelope.sessionState === "object"
|
|
1412
|
+
? (envelope.sessionState as Record<string, unknown>)
|
|
1413
|
+
: null;
|
|
1414
|
+
if (
|
|
1415
|
+
!sessionState ||
|
|
1416
|
+
typeof sessionState.workspaceArchive !== "string" ||
|
|
1417
|
+
sessionState.workspaceArchive.length === 0
|
|
1418
|
+
) {
|
|
1419
|
+
return null;
|
|
1420
|
+
}
|
|
1421
|
+
const fields: Record<string, unknown> = {};
|
|
1422
|
+
for (const key of DURABLE_WORKSPACE_ARCHIVE_FIELDS) {
|
|
1423
|
+
if (sessionState[key] !== undefined && sessionState[key] !== null) {
|
|
1424
|
+
fields[key] = sessionState[key];
|
|
1425
|
+
}
|
|
1426
|
+
}
|
|
1427
|
+
return fields;
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1430
|
+
/**
|
|
1431
|
+
* Build the only resume envelope that may be published for a newly-created
|
|
1432
|
+
* replacement sandbox. Historical state contributes durable archive pointers
|
|
1433
|
+
* only; it can never substitute for serialization of the replacement's provider
|
|
1434
|
+
* identity. If replacement serialization is unavailable or fails, publication
|
|
1435
|
+
* keeps an archive-only envelope (when one exists) or null. The new instance id
|
|
1436
|
+
* remains separately fenced on the lease, while later attach/resume fails closed
|
|
1437
|
+
* rather than targeting the dead provider that initiated rematerialization.
|
|
1438
|
+
*/
|
|
1439
|
+
export async function serializeReplacementSandboxEnvelope(
|
|
1440
|
+
established: EstablishedSandboxSession,
|
|
1441
|
+
archiveSource: Record<string, unknown> | null,
|
|
1442
|
+
): Promise<Record<string, unknown> | null> {
|
|
1443
|
+
const serialized = await serializeEstablishedSandboxEnvelope(established);
|
|
1444
|
+
const archiveFields = durableWorkspaceArchiveFields(archiveSource);
|
|
1445
|
+
if (!serialized && !archiveFields) {
|
|
1446
|
+
return null;
|
|
1447
|
+
}
|
|
1448
|
+
const serializedSessionState =
|
|
1449
|
+
serialized?.sessionState && typeof serialized.sessionState === "object"
|
|
1450
|
+
? (serialized.sessionState as Record<string, unknown>)
|
|
1451
|
+
: {};
|
|
1452
|
+
return {
|
|
1453
|
+
...(serialized ?? {}),
|
|
1454
|
+
// Never inherit a historical backend marker when the replacement could not
|
|
1455
|
+
// serialize. The separately-persisted resume_backend_id and this envelope
|
|
1456
|
+
// must both describe the replacement.
|
|
1457
|
+
backendId: established.backendId,
|
|
1458
|
+
sessionState: {
|
|
1459
|
+
...serializedSessionState,
|
|
1460
|
+
...(archiveFields ?? {}),
|
|
1461
|
+
},
|
|
1462
|
+
};
|
|
1463
|
+
}
|