@opengeni/worker-bundle 0.20.3 → 0.22.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +52 -0
- package/dist/activities/agent-turn/agent-build.d.ts +89 -0
- package/dist/activities/agent-turn/claim.d.ts +67 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
- package/dist/activities/agent-turn/codex.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
- package/dist/activities/agent-turn/errors.d.ts +253 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
- package/dist/activities/agent-turn/file-resources.d.ts +37 -0
- package/dist/activities/agent-turn/finalization.d.ts +42 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
- package/dist/activities/agent-turn/governance-model.d.ts +66 -0
- package/dist/activities/agent-turn/history-sink.d.ts +52 -0
- package/dist/activities/agent-turn/history.d.ts +63 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
- package/dist/activities/agent-turn/model-usage.d.ts +218 -0
- package/dist/activities/agent-turn/quiescence.d.ts +135 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
- package/dist/activities/agent-turn/run.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
- package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
- package/dist/activities/agent-turn/turn-context.d.ts +144 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
- package/dist/activities/agent-turn.d.ts +12 -954
- package/dist/activities/automations.d.ts +16 -0
- package/dist/activities/common.d.ts +1 -1
- package/dist/activities/connector-attachments.d.ts +13 -0
- package/dist/activities/documents.d.ts +1 -0
- package/dist/activities/environment.d.ts +26 -15
- package/dist/activities/generated-images.d.ts +1 -1
- package/dist/activities/git-credential-renewal.d.ts +3 -1
- package/dist/activities/goals.d.ts +4 -1
- package/dist/activities/google-drive-provider.d.ts +29 -0
- package/dist/activities/google-drive-publication.d.ts +17 -4
- package/dist/activities/incident-telemetry-authority.d.ts +52 -0
- package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
- package/dist/activities/knowledge-source-sync.d.ts +4 -0
- package/dist/activities/mcp-credentials.d.ts +9 -3
- package/dist/activities/open-suffix-resume.d.ts +77 -0
- package/dist/activities/packs.d.ts +3 -11
- package/dist/activities/parent-wake.d.ts +19 -0
- package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
- package/dist/activities/retained-screenshots.d.ts +1 -1
- package/dist/activities/rig-verification.d.ts +1 -1
- package/dist/activities/run-input.d.ts +9 -10
- package/dist/activities/sandbox-images.d.ts +1 -14
- package/dist/activities/sandbox-lease.d.ts +21 -3
- package/dist/activities/scheduled-tasks.d.ts +47 -0
- package/dist/activities/types.d.ts +23 -2
- package/dist/activities/video-media-validation.d.ts +1 -1
- package/dist/activities/video-output-retention.d.ts +1 -2
- package/dist/activities/xai-auth.d.ts +3 -0
- package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
- package/dist/activities-control-DKG22ONJ.js.map +1 -0
- package/dist/activities-control.d.ts +6 -2
- package/dist/activities-turn-4IMC3MAA.js +21326 -0
- package/dist/activities-turn-4IMC3MAA.js.map +1 -0
- package/dist/activities.d.ts +7 -3
- package/dist/artifact-materializer-entry.js +41 -22
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
- package/dist/chunk-CE7TPKBS.js.map +1 -0
- package/dist/chunk-CNYGVRKF.js +4698 -0
- package/dist/chunk-CNYGVRKF.js.map +1 -0
- package/dist/connection-provider-use.d.ts +18 -0
- package/dist/editable-artifact-materializer.d.ts +4 -3
- package/dist/github-rest-mcp.d.ts +26 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +58 -14
- package/dist/index.js.map +1 -1
- package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
- package/dist/model-context-contributions.d.ts +8 -4
- package/dist/observability-metrics.d.ts +63 -4
- package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
- package/dist/personal-github-git-credentials.d.ts +9 -0
- package/dist/pr-review-credentials.d.ts +6 -0
- package/dist/sandbox-archive-storage.d.ts +45 -0
- package/dist/sandbox-resume.d.ts +65 -6
- package/dist/sandbox-routing.d.ts +38 -2
- package/dist/scheduled-alert-occurrence.d.ts +33 -0
- package/dist/workflow-bundle.js +87 -28
- package/dist/workflows/activities.d.ts +1 -0
- package/dist/workflows/automations.d.ts +6 -0
- package/dist/workflows/document-index.d.ts +1 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +292 -0
- package/src/activities/agent-turn/agent-build.ts +802 -0
- package/src/activities/agent-turn/claim.ts +590 -0
- package/src/activities/agent-turn/codex-capacity.ts +720 -0
- package/src/activities/agent-turn/codex.ts +94 -0
- package/src/activities/agent-turn/compaction-prep.ts +722 -0
- package/src/activities/agent-turn/credential-leases.ts +240 -0
- package/src/activities/agent-turn/errors.ts +1068 -0
- package/src/activities/agent-turn/failure-settlement.ts +1436 -0
- package/src/activities/agent-turn/file-resources.ts +227 -0
- package/src/activities/agent-turn/finalization.ts +754 -0
- package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
- package/src/activities/agent-turn/governance-model.ts +465 -0
- package/src/activities/agent-turn/history-sink.ts +139 -0
- package/src/activities/agent-turn/history.ts +200 -0
- package/src/activities/agent-turn/media-artifacts.ts +359 -0
- package/src/activities/agent-turn/model-usage.ts +823 -0
- package/src/activities/agent-turn/quiescence.ts +353 -0
- package/src/activities/agent-turn/run-credentials.ts +761 -0
- package/src/activities/agent-turn/run.ts +1530 -0
- package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
- package/src/activities/agent-turn/sandbox-provision.ts +569 -0
- package/src/activities/agent-turn/sandbox-route.ts +358 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
- package/src/activities/agent-turn/stream-attempt.ts +1670 -0
- package/src/activities/agent-turn/tool-environment.ts +729 -0
- package/src/activities/agent-turn/tool-policy.ts +216 -0
- package/src/activities/agent-turn/tool-result-spill.ts +265 -0
- package/src/activities/agent-turn/turn-context.ts +281 -0
- package/src/activities/agent-turn/xai-capacity.ts +229 -0
- package/src/activities/agent-turn.ts +12 -12701
- package/src/activities/api-integrations.ts +3 -0
- package/src/activities/automations.ts +214 -0
- package/src/activities/codemode-dispatcher.ts +12 -1
- package/src/activities/common.ts +1 -5
- package/src/activities/connector-attachments.ts +168 -0
- package/src/activities/context-compaction.ts +18 -0
- package/src/activities/environment.ts +151 -55
- package/src/activities/generated-images.ts +26 -11
- package/src/activities/git-credential-renewal.ts +13 -3
- package/src/activities/goals.ts +85 -19
- package/src/activities/google-drive-provider.ts +178 -0
- package/src/activities/google-drive-publication.ts +147 -49
- package/src/activities/incident-telemetry-authority.ts +327 -0
- package/src/activities/incident-telemetry-preflight.ts +317 -0
- package/src/activities/knowledge-source-sync.ts +178 -37
- package/src/activities/mcp-credentials.ts +191 -17
- package/src/activities/open-suffix-resume.ts +422 -0
- package/src/activities/packs.ts +12 -60
- package/src/activities/parent-wake.ts +64 -14
- package/src/activities/retained-screenshot-reaper.ts +6 -3
- package/src/activities/retained-screenshots.ts +15 -47
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +102 -142
- package/src/activities/sandbox-images.ts +7 -166
- package/src/activities/sandbox-lease.ts +406 -15
- package/src/activities/scheduled-tasks.ts +2103 -411
- package/src/activities/session-state.ts +10 -2
- package/src/activities/types.ts +32 -2
- package/src/activities/video-media-validation.ts +3 -3
- package/src/activities/video-output-retention.ts +2 -57
- package/src/activities/xai-auth.ts +8 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/activity-services.ts +13 -4
- package/src/connection-provider-use.ts +36 -0
- package/src/editable-artifact-materializer-subprocess.ts +25 -14
- package/src/editable-artifact-materializer.ts +29 -18
- package/src/github-rest-mcp.ts +713 -0
- package/src/http.ts +16 -6
- package/src/index.ts +40 -5
- package/src/knowledge-source-sync-activity-policy.ts +2 -0
- package/src/model-context-contributions.ts +33 -10
- package/src/observability-metrics.ts +357 -0
- package/src/opensandbox-kubernetes-inventory.ts +292 -0
- package/src/personal-github-git-credentials.ts +331 -0
- package/src/pr-review-credentials.ts +163 -0
- package/src/sandbox-archive-storage.ts +159 -0
- package/src/sandbox-resume.ts +449 -63
- package/src/sandbox-routing.ts +314 -16
- package/src/scheduled-alert-occurrence.ts +176 -0
- package/src/workflows/activities.ts +18 -2
- package/src/workflows/automations.ts +11 -0
- package/src/workflows/session.ts +13 -3
- package/src/workflows.ts +1 -0
- package/dist/activities-control-BJSDRSIP.js.map +0 -1
- package/dist/activities-turn-J5YO7XXX.js +0 -17243
- package/dist/activities-turn-J5YO7XXX.js.map +0 -1
- package/dist/chunk-N5TJIFCE.js.map +0 -1
- package/dist/chunk-P6Z2SLAA.js +0 -3092
- package/dist/chunk-P6Z2SLAA.js.map +0 -1
package/src/sandbox-resume.ts
CHANGED
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
// Liveness between turns is the lease refcount; there is no keepalive loop.
|
|
22
22
|
|
|
23
23
|
import {
|
|
24
|
+
effectiveSandboxLifecycle,
|
|
24
25
|
sandboxArchiveCaptureTimeoutMs,
|
|
25
26
|
sandboxLifecycleTransitionWaitMs,
|
|
26
27
|
type Settings,
|
|
@@ -35,7 +36,7 @@ import {
|
|
|
35
36
|
failSandboxRematerialization,
|
|
36
37
|
failWarmingToCold,
|
|
37
38
|
getSandboxSessionEnvelope,
|
|
38
|
-
|
|
39
|
+
heartbeatLeaseHolderStatus,
|
|
39
40
|
markSandboxRestoreVerifying,
|
|
40
41
|
markWarmLeaseInstanceLost,
|
|
41
42
|
markSandboxCheckpointArtifactDeletePending,
|
|
@@ -43,6 +44,7 @@ import {
|
|
|
43
44
|
readLease,
|
|
44
45
|
registerSandboxCheckpointArtifact,
|
|
45
46
|
recordWarmingSandboxCreated,
|
|
47
|
+
reinstateTurnLeaseHolder,
|
|
46
48
|
releaseWorkspaceArchiveCapture,
|
|
47
49
|
releaseLeaseHolder,
|
|
48
50
|
touchLeaseHolder,
|
|
@@ -55,6 +57,7 @@ import {
|
|
|
55
57
|
import {
|
|
56
58
|
captureVerifiedWorkspaceArchive,
|
|
57
59
|
describeLegacyNativeSnapshotArchive,
|
|
60
|
+
inlineWorkspaceArchiveForRestore,
|
|
58
61
|
MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
59
62
|
SandboxExecReadinessError,
|
|
60
63
|
WorkspaceArchiveIntegrityError,
|
|
@@ -64,6 +67,7 @@ import {
|
|
|
64
67
|
providerWorkspaceCapturePolicy,
|
|
65
68
|
SandboxProviderContinuityUnavailableError,
|
|
66
69
|
requirePersistableReplacementSandboxEnvelope,
|
|
70
|
+
renewSandboxProviderExpiration,
|
|
67
71
|
modalSessionMatchesCheckpointProviderBinding,
|
|
68
72
|
resolveModalCheckpointProviderBindingForSession,
|
|
69
73
|
serializeReplacementSandboxEnvelope,
|
|
@@ -75,6 +79,15 @@ import {
|
|
|
75
79
|
type RuntimeMetricsHooks,
|
|
76
80
|
type WorkspaceArchiveDescriptor,
|
|
77
81
|
} from "@opengeni/runtime";
|
|
82
|
+
import type { ObjectStorage } from "@opengeni/storage";
|
|
83
|
+
import type { Observability } from "@opengeni/observability";
|
|
84
|
+
import { parseWorkspaceArchiveObjectRef } from "@opengeni/contracts";
|
|
85
|
+
import {
|
|
86
|
+
collectWorkspaceArchiveObjectKeys,
|
|
87
|
+
deleteUnpublishedWorkspaceArchiveObject,
|
|
88
|
+
deleteWorkspaceArchiveObjectKeys,
|
|
89
|
+
putVersion1TarArchiveOrInline,
|
|
90
|
+
} from "./sandbox-archive-storage";
|
|
78
91
|
|
|
79
92
|
// Re-exported for callers that just want the ack-kind union.
|
|
80
93
|
export type ResumeHolderKind = LeaseHolderKind;
|
|
@@ -107,11 +120,18 @@ export function isRetryableDegradedRestore(restore: {
|
|
|
107
120
|
export type SandboxResumeServices = {
|
|
108
121
|
db: Database;
|
|
109
122
|
settings: Settings;
|
|
123
|
+
objectStorage?: ObjectStorage | null;
|
|
110
124
|
/** Exact settings before a verified rig provider image overlaid the logical
|
|
111
125
|
* pack/deployment image. Fresh-create NotFound fallback must preserve an
|
|
112
126
|
* ID-only logical base instead of selecting the provider default. */
|
|
113
127
|
logicalFallbackSettings?: Settings;
|
|
114
128
|
sandboxMetrics?: RuntimeMetricsHooks;
|
|
129
|
+
/** Structured logger for lifecycle decisions (holder release reasons). Falls
|
|
130
|
+
* back to the console when a lean caller passes none. */
|
|
131
|
+
observability?: Pick<Observability, "info" | "warn">;
|
|
132
|
+
/** Test seam only: cadence of the private holder-liveness loop (default 10 s).
|
|
133
|
+
* Production keeps the default; the dead-worker reap horizon assumes it. */
|
|
134
|
+
holderLivenessIntervalMs?: number;
|
|
115
135
|
/**
|
|
116
136
|
* The logical turn-attempt lifetime, not merely the provider request lifetime.
|
|
117
137
|
* Some provider SDK calls cannot be interrupted. Aborting this signal still
|
|
@@ -140,7 +160,7 @@ export type ResumeBoxIds = {
|
|
|
140
160
|
* disclosure/attribution). For a singleton group this == sandboxGroupId. */
|
|
141
161
|
sessionId: string;
|
|
142
162
|
/** The backend the box runs on (sessions.sandbox_backend). */
|
|
143
|
-
backend:
|
|
163
|
+
backend: Settings["sandboxBackend"];
|
|
144
164
|
/** The OS axis (sessions.sandbox_os); default 'linux'. */
|
|
145
165
|
os?: string;
|
|
146
166
|
/**
|
|
@@ -189,6 +209,44 @@ export type ResumedTurnSandbox = {
|
|
|
189
209
|
release: (options?: { workspaceWritersQuiesced?: boolean }) => Promise<void>;
|
|
190
210
|
};
|
|
191
211
|
|
|
212
|
+
/**
|
|
213
|
+
* Structural ownership of the provider boundary that failed while establishing
|
|
214
|
+
* a logical sandbox provision. Object errors retain their identity and receive
|
|
215
|
+
* an out-of-band stage; primitive thrown values use a typed wrapper. In both
|
|
216
|
+
* cases the source diagnostic is unchanged and the stage is never guessed from
|
|
217
|
+
* prose.
|
|
218
|
+
*/
|
|
219
|
+
export type SandboxProvisionFailureStage = "create" | "resume" | "archive_recovery";
|
|
220
|
+
|
|
221
|
+
export class SandboxProvisionStageError extends Error {
|
|
222
|
+
readonly name = "SandboxProvisionStageError";
|
|
223
|
+
|
|
224
|
+
constructor(
|
|
225
|
+
public readonly stage: SandboxProvisionFailureStage,
|
|
226
|
+
public readonly source: unknown,
|
|
227
|
+
) {
|
|
228
|
+
super(source instanceof Error ? source.message : String(source), { cause: source });
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
const sandboxProvisionFailureStages = new WeakMap<object, SandboxProvisionFailureStage>();
|
|
233
|
+
|
|
234
|
+
export function sandboxProvisionFailureStage(error: unknown): SandboxProvisionFailureStage | null {
|
|
235
|
+
if (!error || typeof error !== "object") return null;
|
|
236
|
+
return (
|
|
237
|
+
sandboxProvisionFailureStages.get(error) ??
|
|
238
|
+
(error instanceof SandboxProvisionStageError ? error.stage : null)
|
|
239
|
+
);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function sandboxProvisionStageError(stage: SandboxProvisionFailureStage, error: unknown): unknown {
|
|
243
|
+
if (error && typeof error === "object") {
|
|
244
|
+
sandboxProvisionFailureStages.set(error, stage);
|
|
245
|
+
return error;
|
|
246
|
+
}
|
|
247
|
+
return new SandboxProvisionStageError(stage, error);
|
|
248
|
+
}
|
|
249
|
+
|
|
192
250
|
export abstract class SandboxWarmingTimeoutError extends Error {
|
|
193
251
|
abstract readonly code: "sandbox_exec_readiness_timeout" | "sandbox_sibling_warming_timeout";
|
|
194
252
|
|
|
@@ -277,10 +335,10 @@ async function sleep(ms: number): Promise<void> {
|
|
|
277
335
|
}
|
|
278
336
|
|
|
279
337
|
/**
|
|
280
|
-
*
|
|
281
|
-
* exec. The upstream session's yieldTimeMs starts only after
|
|
282
|
-
* returns, so it cannot bound that initial RPC. Probe it before
|
|
283
|
-
* lease as warm and enforce our own wall-clock deadline.
|
|
338
|
+
* A remote provider may return a sandbox handle before its command router
|
|
339
|
+
* accepts the first exec. The upstream session's yieldTimeMs starts only after
|
|
340
|
+
* sandbox.exec() returns, so it cannot bound that initial RPC. Probe it before
|
|
341
|
+
* publishing the lease as warm and enforce our own wall-clock deadline.
|
|
284
342
|
*/
|
|
285
343
|
export async function waitForSandboxExecReadiness(
|
|
286
344
|
established: EstablishedSandboxSession,
|
|
@@ -312,25 +370,42 @@ export function safeSnapshotError(error: unknown): {
|
|
|
312
370
|
errorCode: "snapshot_operation_failed";
|
|
313
371
|
status?: number;
|
|
314
372
|
origin: "sandbox-resume";
|
|
373
|
+
causeName?: string;
|
|
374
|
+
integrityCode?: string;
|
|
315
375
|
} {
|
|
316
376
|
const fields: {
|
|
317
377
|
errorClass: "SnapshotOperationError";
|
|
318
378
|
errorCode: "snapshot_operation_failed";
|
|
319
379
|
status?: number;
|
|
320
380
|
origin: "sandbox-resume";
|
|
381
|
+
causeName?: string;
|
|
382
|
+
integrityCode?: string;
|
|
321
383
|
} = {
|
|
322
384
|
errorClass: "SnapshotOperationError",
|
|
323
385
|
errorCode: "snapshot_operation_failed",
|
|
324
386
|
origin: "sandbox-resume",
|
|
325
387
|
};
|
|
326
388
|
try {
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
389
|
+
if (error && typeof error === "object") {
|
|
390
|
+
const candidate = error as {
|
|
391
|
+
name?: unknown;
|
|
392
|
+
code?: unknown;
|
|
393
|
+
status?: unknown;
|
|
394
|
+
statusCode?: unknown;
|
|
395
|
+
};
|
|
396
|
+
if (
|
|
397
|
+
typeof candidate.name === "string" &&
|
|
398
|
+
/^[A-Z][A-Za-z0-9]{2,62}Error$/u.test(candidate.name)
|
|
399
|
+
) {
|
|
400
|
+
fields.causeName = candidate.name;
|
|
401
|
+
}
|
|
402
|
+
if (typeof candidate.code === "string" && /^[a-z0-9_]{1,64}$/u.test(candidate.code)) {
|
|
403
|
+
fields.integrityCode = candidate.code;
|
|
404
|
+
}
|
|
405
|
+
const rawStatus = candidate.status ?? candidate.statusCode;
|
|
406
|
+
const status = Number(rawStatus);
|
|
407
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
|
|
408
|
+
}
|
|
334
409
|
} catch {
|
|
335
410
|
// Public diagnostics are best-effort and must never replace the exact
|
|
336
411
|
// internal snapshot failure.
|
|
@@ -461,19 +536,24 @@ function workspaceArchiveFieldsFromEnvelope(
|
|
|
461
536
|
? (envelope.sessionState as Record<string, unknown>)
|
|
462
537
|
: null;
|
|
463
538
|
const archive = sessionState?.workspaceArchive;
|
|
464
|
-
|
|
539
|
+
const archiveRef = parseWorkspaceArchiveObjectRef(sessionState?.workspaceArchiveRef);
|
|
540
|
+
const hasInline = typeof archive === "string" && archive.length > 0;
|
|
541
|
+
if (!hasInline && !archiveRef) {
|
|
465
542
|
return null;
|
|
466
543
|
}
|
|
467
544
|
const previous = sessionState?.workspaceArchivePrev;
|
|
545
|
+
const previousRef = parseWorkspaceArchiveObjectRef(sessionState?.workspaceArchivePrevRef);
|
|
468
546
|
const metadata = sessionState?.workspaceArchiveMeta;
|
|
469
547
|
const previousMetadata = sessionState?.workspaceArchivePrevMeta;
|
|
470
548
|
const capturedAt = sessionState?.workspaceArchiveAt;
|
|
471
549
|
return {
|
|
472
|
-
workspaceArchive: archive,
|
|
550
|
+
...(hasInline && !archiveRef ? { workspaceArchive: archive } : {}),
|
|
551
|
+
...(archiveRef ? { workspaceArchiveRef: archiveRef } : {}),
|
|
473
552
|
...(metadata !== undefined ? { workspaceArchiveMeta: metadata } : {}),
|
|
474
|
-
...(typeof previous === "string" && previous.length > 0
|
|
553
|
+
...(typeof previous === "string" && previous.length > 0 && !previousRef
|
|
475
554
|
? { workspaceArchivePrev: previous }
|
|
476
555
|
: {}),
|
|
556
|
+
...(previousRef ? { workspaceArchivePrevRef: previousRef } : {}),
|
|
477
557
|
...(previousMetadata !== undefined ? { workspaceArchivePrevMeta: previousMetadata } : {}),
|
|
478
558
|
...(typeof capturedAt === "string" && capturedAt.length > 0
|
|
479
559
|
? { workspaceArchiveAt: capturedAt }
|
|
@@ -481,6 +561,35 @@ function workspaceArchiveFieldsFromEnvelope(
|
|
|
481
561
|
};
|
|
482
562
|
}
|
|
483
563
|
|
|
564
|
+
async function materializeSpawnEnvelopeArchive(
|
|
565
|
+
envelope: unknown,
|
|
566
|
+
objectStorage: ObjectStorage | null | undefined,
|
|
567
|
+
): Promise<unknown> {
|
|
568
|
+
if (!envelope || typeof envelope !== "object") return envelope;
|
|
569
|
+
const record = envelope as Record<string, unknown>;
|
|
570
|
+
const sessionState =
|
|
571
|
+
record.sessionState &&
|
|
572
|
+
typeof record.sessionState === "object" &&
|
|
573
|
+
!Array.isArray(record.sessionState)
|
|
574
|
+
? (record.sessionState as Record<string, unknown>)
|
|
575
|
+
: null;
|
|
576
|
+
if (!sessionState || !parseWorkspaceArchiveObjectRef(sessionState.workspaceArchiveRef)) {
|
|
577
|
+
return envelope;
|
|
578
|
+
}
|
|
579
|
+
if (!objectStorage) {
|
|
580
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
581
|
+
"archive_base64_invalid",
|
|
582
|
+
"workspace archive object storage is not configured",
|
|
583
|
+
);
|
|
584
|
+
}
|
|
585
|
+
return {
|
|
586
|
+
...record,
|
|
587
|
+
sessionState: await inlineWorkspaceArchiveForRestore(sessionState, (key) =>
|
|
588
|
+
objectStorage.getObjectBytes(key),
|
|
589
|
+
),
|
|
590
|
+
};
|
|
591
|
+
}
|
|
592
|
+
|
|
484
593
|
function legacyNativeArchiveFromEnvelope(envelope: Record<string, unknown> | null | undefined) {
|
|
485
594
|
const sessionState =
|
|
486
595
|
envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null
|
|
@@ -495,6 +604,93 @@ function legacyNativeArchiveFromEnvelope(envelope: Record<string, unknown> | nul
|
|
|
495
604
|
);
|
|
496
605
|
}
|
|
497
606
|
|
|
607
|
+
export type SandboxDeadlineRotationCheckpointResult = {
|
|
608
|
+
/** The exact established epoch still carries a requested rotation and a
|
|
609
|
+
* complete current-generation archive: the turn may abort orderly. */
|
|
610
|
+
checkpointed: boolean;
|
|
611
|
+
/** What the defense-in-depth holder step observed before the capture. */
|
|
612
|
+
holder: Awaited<ReturnType<typeof reinstateTurnLeaseHolder>>["status"];
|
|
613
|
+
lease: Awaited<ReturnType<typeof readLease>>;
|
|
614
|
+
};
|
|
615
|
+
|
|
616
|
+
/**
|
|
617
|
+
* The orderly provider-deadline rotation checkpoint for one running turn. The
|
|
618
|
+
* turn-side lease heartbeat calls this once it observes a rotation-fenced lease
|
|
619
|
+
* at its established epoch/instance; `checkpointed: true` licenses the
|
|
620
|
+
* SandboxDeadlineRotationError abort that hands the same logical turn to the
|
|
621
|
+
* cold successor.
|
|
622
|
+
*
|
|
623
|
+
* Defense in depth: the warm capture claim requires exactly this turn's holder.
|
|
624
|
+
* If that holder was lost while the attempt is still the active writer (a stale
|
|
625
|
+
* release, a reap race), `reinstateTurnLeaseHolder` puts it back under the
|
|
626
|
+
* exact epoch/instance fence before the capture, so a lost holder cannot
|
|
627
|
+
* silently disable the orderly abort and leave the reaper to terminate the box
|
|
628
|
+
* under the running turn. Never captures when the epoch/instance changed.
|
|
629
|
+
*/
|
|
630
|
+
export async function persistSandboxDeadlineRotationCheckpoint(
|
|
631
|
+
services: SandboxResumeServices,
|
|
632
|
+
ids: {
|
|
633
|
+
accountId: string;
|
|
634
|
+
workspaceId: string;
|
|
635
|
+
sessionId: string;
|
|
636
|
+
turnId: string;
|
|
637
|
+
attemptId: string;
|
|
638
|
+
sandboxGroupId: string;
|
|
639
|
+
},
|
|
640
|
+
session: unknown,
|
|
641
|
+
expected: { leaseEpoch: number; instanceId: string },
|
|
642
|
+
signal?: AbortSignal,
|
|
643
|
+
): Promise<SandboxDeadlineRotationCheckpointResult> {
|
|
644
|
+
const { db, settings } = services;
|
|
645
|
+
// A cancelled attempt must never briefly regain its holder: its eager
|
|
646
|
+
// cancellation listener may already have released it, and reinstating here
|
|
647
|
+
// would pin the lease behind an attempt that is being torn down.
|
|
648
|
+
if (signal?.aborted) {
|
|
649
|
+
return { checkpointed: false, holder: "attempt_fenced", lease: null };
|
|
650
|
+
}
|
|
651
|
+
const reinstated = await reinstateTurnLeaseHolder(db, {
|
|
652
|
+
accountId: ids.accountId,
|
|
653
|
+
workspaceId: ids.workspaceId,
|
|
654
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
655
|
+
sessionId: ids.sessionId,
|
|
656
|
+
attemptId: ids.attemptId,
|
|
657
|
+
expectedEpoch: expected.leaseEpoch,
|
|
658
|
+
expectedInstanceId: expected.instanceId,
|
|
659
|
+
leaseTtlMs: settings.sandboxLeaseTtlMs,
|
|
660
|
+
});
|
|
661
|
+
if (reinstated.status === "reinstated") {
|
|
662
|
+
const message = "sandbox rotation checkpoint reinstated the live turn holder";
|
|
663
|
+
const fields = {
|
|
664
|
+
workspaceId: ids.workspaceId,
|
|
665
|
+
sessionId: ids.sessionId,
|
|
666
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
667
|
+
leaseEpoch: expected.leaseEpoch,
|
|
668
|
+
liveness: reinstated.lease.liveness,
|
|
669
|
+
};
|
|
670
|
+
if (services.observability) services.observability.warn(message, fields);
|
|
671
|
+
else console.warn(message, fields);
|
|
672
|
+
}
|
|
673
|
+
if (reinstated.status !== "present" && reinstated.status !== "reinstated") {
|
|
674
|
+
return { checkpointed: false, holder: reinstated.status, lease: null };
|
|
675
|
+
}
|
|
676
|
+
await maybePersistWarmWorkspaceSnapshot(
|
|
677
|
+
services,
|
|
678
|
+
ids,
|
|
679
|
+
session,
|
|
680
|
+
expected.leaseEpoch,
|
|
681
|
+
signal,
|
|
682
|
+
true,
|
|
683
|
+
);
|
|
684
|
+
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
685
|
+
const checkpointed =
|
|
686
|
+
lease?.leaseEpoch === expected.leaseEpoch &&
|
|
687
|
+
lease.instanceId === expected.instanceId &&
|
|
688
|
+
lease.rotationRequestedAt !== null &&
|
|
689
|
+
lease.archiveComplete &&
|
|
690
|
+
signal?.aborted !== true;
|
|
691
|
+
return { checkpointed, holder: reinstated.status, lease };
|
|
692
|
+
}
|
|
693
|
+
|
|
498
694
|
/**
|
|
499
695
|
* MID-SESSION /workspace snapshot (sandbox-file-persistence). The reaper's
|
|
500
696
|
* drain-persist only protects boxes the reaper itself kills; anything else —
|
|
@@ -535,12 +731,20 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
535
731
|
}
|
|
536
732
|
const persistable = session as {
|
|
537
733
|
persistWorkspace?: () => Promise<Uint8Array | undefined>;
|
|
734
|
+
persistWorkspaceTar?: () => Promise<Uint8Array | undefined>;
|
|
735
|
+
backendId?: unknown;
|
|
538
736
|
state?: {
|
|
539
737
|
workspacePersistence?: unknown;
|
|
540
738
|
providerState?: { workspacePersistence?: unknown };
|
|
541
739
|
};
|
|
542
740
|
};
|
|
543
|
-
if (
|
|
741
|
+
if (
|
|
742
|
+
typeof persistable.persistWorkspace !== "function" &&
|
|
743
|
+
typeof persistable.persistWorkspaceTar !== "function"
|
|
744
|
+
) {
|
|
745
|
+
console.error("mid-session workspace snapshot skipped (no persist primitive)", {
|
|
746
|
+
backendId: typeof persistable.backendId === "string" ? persistable.backendId : null,
|
|
747
|
+
});
|
|
544
748
|
return false;
|
|
545
749
|
}
|
|
546
750
|
// Filesystem and directory snapshots create retained Images without
|
|
@@ -559,12 +763,21 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
559
763
|
// This work cannot pause or mutate the box, so keeping it outside the claim
|
|
560
764
|
// minimizes the time that command admission must wait.
|
|
561
765
|
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
lease
|
|
567
|
-
|
|
766
|
+
const canWarmCapture =
|
|
767
|
+
lease?.liveness === "warm" && lease.instanceId !== null && lease.leaseEpoch === leaseEpoch;
|
|
768
|
+
const canForceDrainingCapture =
|
|
769
|
+
force === true &&
|
|
770
|
+
lease?.liveness === "draining" &&
|
|
771
|
+
lease.instanceId !== null &&
|
|
772
|
+
lease.leaseEpoch === leaseEpoch;
|
|
773
|
+
if (!lease || (!canWarmCapture && !canForceDrainingCapture)) {
|
|
774
|
+
console.error("mid-session workspace snapshot skipped (lease not warm)", {
|
|
775
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
776
|
+
expectedEpoch: leaseEpoch,
|
|
777
|
+
liveness: lease?.liveness ?? null,
|
|
778
|
+
leaseEpoch: lease?.leaseEpoch ?? null,
|
|
779
|
+
hasInstance: lease?.instanceId !== null && lease?.instanceId !== undefined,
|
|
780
|
+
});
|
|
568
781
|
return false;
|
|
569
782
|
}
|
|
570
783
|
// A checkpoint of this exact mutation generation already protects every
|
|
@@ -573,9 +786,18 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
573
786
|
if (lease.archiveComplete) {
|
|
574
787
|
return false;
|
|
575
788
|
}
|
|
789
|
+
if (lease.instanceId === null) {
|
|
790
|
+
return false;
|
|
791
|
+
}
|
|
576
792
|
const instanceId = lease.instanceId;
|
|
793
|
+
const captureLiveness: "warm" | "draining" = canWarmCapture ? "warm" : "draining";
|
|
577
794
|
const capturePolicy = providerWorkspaceCapturePolicy(lease.backend, session);
|
|
578
795
|
if (!capturePolicy || capturePolicy.liveInstance !== "preserved") {
|
|
796
|
+
console.error("mid-session workspace snapshot skipped (capture policy)", {
|
|
797
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
798
|
+
backend: lease.backend,
|
|
799
|
+
liveInstance: capturePolicy?.liveInstance ?? null,
|
|
800
|
+
});
|
|
579
801
|
return false;
|
|
580
802
|
}
|
|
581
803
|
const nativeModalPersistence =
|
|
@@ -595,19 +817,29 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
595
817
|
captureId,
|
|
596
818
|
expectedEpoch: leaseEpoch,
|
|
597
819
|
expectedInstanceId: instanceId,
|
|
598
|
-
liveness:
|
|
820
|
+
liveness: captureLiveness,
|
|
599
821
|
captureTimeoutMs,
|
|
600
822
|
minIntervalMs: force ? 0 : intervalMs,
|
|
601
823
|
providerReplaySafe: capturePolicy.takeover === "same_request",
|
|
602
824
|
takeoverSafe: capturePolicy.takeover !== "exclusive",
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
825
|
+
...(captureLiveness === "warm"
|
|
826
|
+
? {
|
|
827
|
+
warmAttempt: {
|
|
828
|
+
sessionId: ids.sessionId,
|
|
829
|
+
turnId: ids.turnId,
|
|
830
|
+
attemptId: ids.attemptId,
|
|
831
|
+
holderId: sandboxLeaseHolderIdForAttempt(ids.attemptId),
|
|
832
|
+
},
|
|
833
|
+
}
|
|
834
|
+
: {}),
|
|
609
835
|
});
|
|
610
|
-
if (claimed.status !== "claimed")
|
|
836
|
+
if (claimed.status !== "claimed") {
|
|
837
|
+
console.error("mid-session workspace snapshot skipped (capture claim)", {
|
|
838
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
839
|
+
status: claimed.status,
|
|
840
|
+
});
|
|
841
|
+
return false;
|
|
842
|
+
}
|
|
611
843
|
|
|
612
844
|
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
613
845
|
// Stamp WHEN this capture started: persistWarmSnapshot orders warm snapshots
|
|
@@ -657,12 +889,33 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
657
889
|
// of the exact admission gate; a late callback cannot release a successor.
|
|
658
890
|
const captureAndPublish = (async (): Promise<boolean> => {
|
|
659
891
|
let candidate: { id: string } | null = null;
|
|
892
|
+
let workspaceArchiveRef: Awaited<
|
|
893
|
+
ReturnType<typeof putVersion1TarArchiveOrInline>
|
|
894
|
+
>["workspaceArchiveRef"];
|
|
660
895
|
try {
|
|
661
896
|
const archive = await captureVerifiedWorkspaceArchive(session, capturedAtMs, {
|
|
662
897
|
requestId: claimed.claim.providerRequestId,
|
|
663
898
|
strategy: capturePolicy.strategy,
|
|
664
899
|
});
|
|
665
900
|
candidate = await registerCandidate(archive);
|
|
901
|
+
const priorLease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
902
|
+
const priorKeys = collectWorkspaceArchiveObjectKeys(
|
|
903
|
+
(priorLease?.resumeState as Record<string, unknown> | null | undefined) ?? null,
|
|
904
|
+
);
|
|
905
|
+
const published = await putVersion1TarArchiveOrInline({
|
|
906
|
+
backend: lease.backend,
|
|
907
|
+
objectStorage: services.objectStorage,
|
|
908
|
+
accountId: ids.accountId,
|
|
909
|
+
workspaceId: ids.workspaceId,
|
|
910
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
911
|
+
archive: {
|
|
912
|
+
bytes: archive.bytes,
|
|
913
|
+
descriptor: archive.descriptor,
|
|
914
|
+
base64: archive.base64,
|
|
915
|
+
},
|
|
916
|
+
...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
|
|
917
|
+
});
|
|
918
|
+
workspaceArchiveRef = published.workspaceArchiveRef;
|
|
666
919
|
const { wrote } = await persistWarmSnapshot(db, {
|
|
667
920
|
accountId: ids.accountId,
|
|
668
921
|
workspaceId: ids.workspaceId,
|
|
@@ -674,17 +927,40 @@ export async function maybePersistWarmWorkspaceSnapshot(
|
|
|
674
927
|
expectedInstanceId: instanceId,
|
|
675
928
|
expectedWorkspaceGeneration: claimed.claim.workspaceGeneration,
|
|
676
929
|
captureId,
|
|
677
|
-
workspaceArchive: archive.base64,
|
|
678
930
|
workspaceArchiveMeta: archive.descriptor,
|
|
931
|
+
...published,
|
|
679
932
|
checkpointArtifactId: candidate?.id ?? null,
|
|
680
933
|
minIntervalMs: force ? 0 : intervalMs,
|
|
681
934
|
capturedAtMs,
|
|
682
935
|
});
|
|
936
|
+
if (published.workspaceArchiveRef && services.objectStorage) {
|
|
937
|
+
if (!wrote) {
|
|
938
|
+
await deleteUnpublishedWorkspaceArchiveObject(
|
|
939
|
+
services.objectStorage,
|
|
940
|
+
published.workspaceArchiveRef,
|
|
941
|
+
services.sandboxMetrics,
|
|
942
|
+
);
|
|
943
|
+
} else {
|
|
944
|
+
const afterLease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
945
|
+
const afterKeys = collectWorkspaceArchiveObjectKeys(
|
|
946
|
+
(afterLease?.resumeState as Record<string, unknown> | null | undefined) ?? null,
|
|
947
|
+
);
|
|
948
|
+
await deleteWorkspaceArchiveObjectKeys(
|
|
949
|
+
services.objectStorage,
|
|
950
|
+
[...priorKeys].filter((key) => !afterKeys.has(key)),
|
|
951
|
+
).catch(() => undefined);
|
|
952
|
+
}
|
|
953
|
+
}
|
|
683
954
|
if (!wrote && candidate) {
|
|
684
955
|
await abandonCandidate(candidate.id, "snapshot_publication_fenced");
|
|
685
956
|
}
|
|
686
957
|
return wrote;
|
|
687
958
|
} catch (error) {
|
|
959
|
+
await deleteUnpublishedWorkspaceArchiveObject(
|
|
960
|
+
services.objectStorage,
|
|
961
|
+
workspaceArchiveRef,
|
|
962
|
+
services.sandboxMetrics,
|
|
963
|
+
);
|
|
688
964
|
if (candidate) await abandonCandidate(candidate.id, "snapshot_capture_failed");
|
|
689
965
|
throw error;
|
|
690
966
|
} finally {
|
|
@@ -913,36 +1189,122 @@ export async function resumeBoxForTurn(
|
|
|
913
1189
|
acquired.lease.liveness === "warm"
|
|
914
1190
|
? { expectedEpoch: acquired.lease.leaseEpoch, leaseTtlMs }
|
|
915
1191
|
: null;
|
|
1192
|
+
let providerRenewalTarget: { backend: Settings["sandboxBackend"]; instanceId: string } | null =
|
|
1193
|
+
acquired.lease.liveness === "warm" && acquired.lease.instanceId
|
|
1194
|
+
? {
|
|
1195
|
+
backend: ids.backend,
|
|
1196
|
+
instanceId: acquired.lease.instanceId,
|
|
1197
|
+
}
|
|
1198
|
+
: null;
|
|
1199
|
+
let providerRenewedAtMs = Date.now();
|
|
1200
|
+
let providerRenewalInFlight: Promise<void> | null = null;
|
|
1201
|
+
const maybeRenewProviderExpiration = async (force = false): Promise<void> => {
|
|
1202
|
+
const target = providerRenewalTarget;
|
|
1203
|
+
if (!target || providerRenewalInFlight) return;
|
|
1204
|
+
const ttlSeconds = effectiveSandboxLifecycle(settings, target.backend).renewableTtlSeconds;
|
|
1205
|
+
if (ttlSeconds === null) return;
|
|
1206
|
+
const intervalMs = Math.max(10_000, Math.floor((ttlSeconds * 1000) / 3));
|
|
1207
|
+
if (!force && Date.now() - providerRenewedAtMs < intervalMs) return;
|
|
1208
|
+
providerRenewalInFlight = renewSandboxProviderExpiration({
|
|
1209
|
+
backend: target.backend,
|
|
1210
|
+
settings,
|
|
1211
|
+
instanceId: target.instanceId,
|
|
1212
|
+
...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
|
|
1213
|
+
})
|
|
1214
|
+
.then(() => {
|
|
1215
|
+
providerRenewedAtMs = Date.now();
|
|
1216
|
+
})
|
|
1217
|
+
.finally(() => {
|
|
1218
|
+
providerRenewalInFlight = null;
|
|
1219
|
+
});
|
|
1220
|
+
await providerRenewalInFlight;
|
|
1221
|
+
};
|
|
1222
|
+
// Release ONLY on holder death. `heartbeatLeaseHolderStatus` separates "this
|
|
1223
|
+
// holder is gone / its attempt is no longer the active writer" from "the lease
|
|
1224
|
+
// could not be extended" (rotation requested, draining, another epoch). The
|
|
1225
|
+
// second is NOT a reason to release: once a provider-deadline rotation is
|
|
1226
|
+
// requested the lease heartbeat is fenced by design while the still-running
|
|
1227
|
+
// turn checkpoints, and the turn-side heartbeat owns that orderly
|
|
1228
|
+
// SandboxDeadlineRotationError abort. Dropping the live holder here instead
|
|
1229
|
+
// flips the lease to draining under the running turn, the reaper terminates
|
|
1230
|
+
// the box, and the checkpoint then fails `attempt_fenced` (the staging
|
|
1231
|
+
// ~61-minute box-age failure). Both loops therefore converge on
|
|
1232
|
+
// "holder alive -> never release"; this loop keeps touching the holder.
|
|
1233
|
+
const releaseDeadHolder = (reason: string): void => {
|
|
1234
|
+
const message = "sandbox resume holder released: holder no longer live";
|
|
1235
|
+
const fields = {
|
|
1236
|
+
workspaceId: ids.workspaceId,
|
|
1237
|
+
sessionId: ids.sessionId,
|
|
1238
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1239
|
+
holderKind: kind,
|
|
1240
|
+
reason,
|
|
1241
|
+
};
|
|
1242
|
+
if (services.observability) services.observability.warn(message, fields);
|
|
1243
|
+
else console.warn(message, fields);
|
|
1244
|
+
void release().catch(() => undefined);
|
|
1245
|
+
};
|
|
916
1246
|
holderLivenessTimer = setInterval(() => {
|
|
917
1247
|
const heartbeat = holderLeaseHeartbeat;
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
1248
|
+
if (heartbeat) {
|
|
1249
|
+
void heartbeatLeaseHolderStatus(db, {
|
|
1250
|
+
accountId: ids.accountId,
|
|
1251
|
+
workspaceId: ids.workspaceId,
|
|
1252
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1253
|
+
kind,
|
|
1254
|
+
holderId,
|
|
1255
|
+
expectedEpoch: heartbeat.expectedEpoch,
|
|
1256
|
+
leaseTtlMs: heartbeat.leaseTtlMs,
|
|
1257
|
+
})
|
|
1258
|
+
.then((status) => {
|
|
1259
|
+
if (heartbeat !== holderLeaseHeartbeat) return;
|
|
1260
|
+
if (!status.holderAlive) {
|
|
1261
|
+
// Reaped, released, or its exact attempt is no longer the active
|
|
1262
|
+
// writer. Stop this otherwise-unbounded provider operation and
|
|
1263
|
+
// idempotently drop any remaining holder state.
|
|
1264
|
+
releaseDeadHolder(status.fence);
|
|
1265
|
+
return;
|
|
1266
|
+
}
|
|
1267
|
+
if (status.fence === "epoch") {
|
|
1268
|
+
// The lease epoch never regresses: a newer epoch means the box this
|
|
1269
|
+
// holder was registered against is definitively superseded (lost
|
|
1270
|
+
// instance replaced, re-established by a later turn). The holder
|
|
1271
|
+
// row is stale authority for an instance that no longer exists;
|
|
1272
|
+
// drop it promptly instead of pinning a successor's lease.
|
|
1273
|
+
releaseDeadHolder(status.fence);
|
|
1274
|
+
return;
|
|
1275
|
+
}
|
|
1276
|
+
// The box is still serving this live holder: keep the provider TTL
|
|
1277
|
+
// renewed while the lease is ours (extended, or only rotation-fenced at
|
|
1278
|
+
// the same epoch). A liveness fence (draining at our epoch) means the
|
|
1279
|
+
// reaper now governs the instance; keep the holder but leave its
|
|
1280
|
+
// expiry alone.
|
|
1281
|
+
if (status.leaseExtended || status.fence === "rotation_requested") {
|
|
1282
|
+
void maybeRenewProviderExpiration().catch(() => undefined);
|
|
1283
|
+
}
|
|
927
1284
|
})
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
1285
|
+
.catch(() => undefined);
|
|
1286
|
+
return;
|
|
1287
|
+
}
|
|
1288
|
+
void touchLeaseHolder(db, {
|
|
1289
|
+
accountId: ids.accountId,
|
|
1290
|
+
workspaceId: ids.workspaceId,
|
|
1291
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1292
|
+
kind,
|
|
1293
|
+
holderId,
|
|
1294
|
+
})
|
|
936
1295
|
.then((touched) => {
|
|
1296
|
+
if (heartbeat !== holderLeaseHeartbeat) return;
|
|
937
1297
|
// A canonical turn holder is rejected once its exact attempt is no
|
|
938
|
-
// longer the active writer
|
|
939
|
-
//
|
|
940
|
-
if (!touched
|
|
941
|
-
|
|
1298
|
+
// longer the active writer (touch is attempt-fenced only; it never
|
|
1299
|
+
// consults lease authority, so false here always means holder death).
|
|
1300
|
+
if (!touched) {
|
|
1301
|
+
releaseDeadHolder("holder_gone");
|
|
1302
|
+
return;
|
|
942
1303
|
}
|
|
1304
|
+
void maybeRenewProviderExpiration().catch(() => undefined);
|
|
943
1305
|
})
|
|
944
1306
|
.catch(() => undefined);
|
|
945
|
-
}, 10_000);
|
|
1307
|
+
}, services.holderLivenessIntervalMs ?? 10_000);
|
|
946
1308
|
if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
|
|
947
1309
|
holderLivenessTimer.unref();
|
|
948
1310
|
}
|
|
@@ -1077,7 +1439,19 @@ export async function resumeBoxForTurn(
|
|
|
1077
1439
|
// `_sandbox` envelope is the per-session fallback. Without this a turn-first
|
|
1078
1440
|
// re-warm after a drain->cold would ignore the archive and start an EMPTY box.
|
|
1079
1441
|
const providerCreateStartedAt = new Date();
|
|
1080
|
-
const
|
|
1442
|
+
const persistArchiveSource =
|
|
1443
|
+
spawnEnvelope && typeof spawnEnvelope === "object"
|
|
1444
|
+
? (spawnEnvelope as Record<string, unknown>)
|
|
1445
|
+
: null;
|
|
1446
|
+
const hydrateEnvelopeRaw = await materializeSpawnEnvelopeArchive(
|
|
1447
|
+
persistArchiveSource,
|
|
1448
|
+
services.objectStorage,
|
|
1449
|
+
);
|
|
1450
|
+
const hydrateEnvelope =
|
|
1451
|
+
hydrateEnvelopeRaw && typeof hydrateEnvelopeRaw === "object"
|
|
1452
|
+
? (hydrateEnvelopeRaw as Record<string, unknown>)
|
|
1453
|
+
: null;
|
|
1454
|
+
const established = await establishSandboxSessionFromEnvelope(settings, hydrateEnvelope, {
|
|
1081
1455
|
sessionId: ids.sessionId,
|
|
1082
1456
|
recovery: "create-or-restore",
|
|
1083
1457
|
backendOverride: ids.backend as never,
|
|
@@ -1088,6 +1462,11 @@ export async function resumeBoxForTurn(
|
|
|
1088
1462
|
: {}),
|
|
1089
1463
|
onSandboxCreated: async (created) => {
|
|
1090
1464
|
createdEstablished = created;
|
|
1465
|
+
providerRenewalTarget = {
|
|
1466
|
+
backend: ids.backend,
|
|
1467
|
+
instanceId: created.instanceId,
|
|
1468
|
+
};
|
|
1469
|
+
providerRenewedAtMs = Date.now();
|
|
1091
1470
|
throwIfReleasedOrCancelled();
|
|
1092
1471
|
if (
|
|
1093
1472
|
rematerialization &&
|
|
@@ -1130,7 +1509,7 @@ export async function resumeBoxForTurn(
|
|
|
1130
1509
|
}
|
|
1131
1510
|
}
|
|
1132
1511
|
const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
|
|
1133
|
-
await serializeReplacementSandboxEnvelope(created,
|
|
1512
|
+
await serializeReplacementSandboxEnvelope(created, persistArchiveSource),
|
|
1134
1513
|
created.backendId,
|
|
1135
1514
|
);
|
|
1136
1515
|
const recorded = await recordWarmingSandboxCreated(db, {
|
|
@@ -1218,13 +1597,15 @@ export async function resumeBoxForTurn(
|
|
|
1218
1597
|
});
|
|
1219
1598
|
createdEstablished = established;
|
|
1220
1599
|
throwIfReleasedOrCancelled();
|
|
1221
|
-
// A sandbox handle is not sufficient evidence that
|
|
1222
|
-
// is live. Do not publish a warm lease until
|
|
1600
|
+
// A sandbox handle is not sufficient evidence that an asynchronous
|
|
1601
|
+
// provider's command router is live. Do not publish a warm lease until
|
|
1602
|
+
// one bounded no-op exec works.
|
|
1223
1603
|
// On timeout the catch below terminates the box and rolls warming -> cold,
|
|
1224
1604
|
// so the next turn cold-creates instead of hanging forever on first use.
|
|
1225
1605
|
await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
|
|
1226
1606
|
sandboxGroupId: ids.sandboxGroupId,
|
|
1227
1607
|
});
|
|
1608
|
+
await maybeRenewProviderExpiration(true);
|
|
1228
1609
|
throwIfReleasedOrCancelled();
|
|
1229
1610
|
// Fold the LIVE box into a re-resumable envelope and persist it as the
|
|
1230
1611
|
// lease's resume_state — exactly like the API-direct paths (channel-a.ts /
|
|
@@ -1244,7 +1625,7 @@ export async function resumeBoxForTurn(
|
|
|
1244
1625
|
// attempts terminate the replacement and fail closed; they never publish a
|
|
1245
1626
|
// clean or mixed workspace.
|
|
1246
1627
|
const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
|
|
1247
|
-
await serializeReplacementSandboxEnvelope(established,
|
|
1628
|
+
await serializeReplacementSandboxEnvelope(established, persistArchiveSource),
|
|
1248
1629
|
established.backendId,
|
|
1249
1630
|
);
|
|
1250
1631
|
throwIfReleasedOrCancelled();
|
|
@@ -1359,7 +1740,7 @@ export async function resumeBoxForTurn(
|
|
|
1359
1740
|
}
|
|
1360
1741
|
await release();
|
|
1361
1742
|
recordSandboxWarmingTimeout(services.sandboxMetrics, error);
|
|
1362
|
-
throw error;
|
|
1743
|
+
throw sandboxProvisionStageError(rematerialization ? "archive_recovery" : "create", error);
|
|
1363
1744
|
}
|
|
1364
1745
|
}
|
|
1365
1746
|
|
|
@@ -1401,8 +1782,8 @@ export async function resumeBoxForTurn(
|
|
|
1401
1782
|
});
|
|
1402
1783
|
throwIfReleasedOrCancelled();
|
|
1403
1784
|
// A durable `warm` row is an ownership assertion, not provider liveness.
|
|
1404
|
-
//
|
|
1405
|
-
//
|
|
1785
|
+
// A provider may have ended the exact box while OpenGeni was idle. Prove
|
|
1786
|
+
// the command router before handing the session to the
|
|
1406
1787
|
// agent so terminal evidence enters the atomic warm->cold recovery path
|
|
1407
1788
|
// below instead of surfacing inside a model-visible tool call.
|
|
1408
1789
|
if (services.verifyAttachedSandboxReadiness) {
|
|
@@ -1412,10 +1793,15 @@ export async function resumeBoxForTurn(
|
|
|
1412
1793
|
sandboxGroupId: ids.sandboxGroupId,
|
|
1413
1794
|
});
|
|
1414
1795
|
}
|
|
1796
|
+
providerRenewalTarget = {
|
|
1797
|
+
backend: ids.backend,
|
|
1798
|
+
instanceId: established.instanceId,
|
|
1799
|
+
};
|
|
1800
|
+
await maybeRenewProviderExpiration(true);
|
|
1415
1801
|
throwIfReleasedOrCancelled();
|
|
1416
1802
|
} catch (error) {
|
|
1417
1803
|
if (!isProviderSandboxNotFoundError(ids.backend, error)) {
|
|
1418
|
-
throw error;
|
|
1804
|
+
throw sandboxProvisionStageError("resume", error);
|
|
1419
1805
|
}
|
|
1420
1806
|
const marked = await markWarmLeaseInstanceLost(db, {
|
|
1421
1807
|
accountId: ids.accountId,
|