@opengeni/worker-bundle 0.28.3 → 0.28.4-canary.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/codex-selection-diagnostics.d.ts +1 -0
- package/dist/activities/agent-turn/mcp-operation-read-tool.d.ts +16 -0
- package/dist/activities/codex-fleet-shadow.d.ts +1 -1
- package/dist/activities/mcp-operation-authority.d.ts +16 -0
- package/dist/activities/mcp-operation-observer.d.ts +14 -0
- package/dist/activities/mcp-operation-reader.d.ts +45 -0
- package/dist/activities/mcp-operation-store.d.ts +6 -0
- package/dist/activities/sandbox-lease.d.ts +12 -4
- package/dist/{activities-control-UXGPMDL3.js → activities-control-MKGEOBXT.js} +90 -71
- package/dist/activities-control-MKGEOBXT.js.map +1 -0
- package/dist/{activities-turn-NOYDE7PX.js → activities-turn-5MZZ6HQA.js} +15691 -815
- package/dist/activities-turn-5MZZ6HQA.js.map +1 -0
- package/dist/artifact-materializer-entry.js +2 -0
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/artifact-outbox-entry.js +2 -0
- package/dist/artifact-outbox-entry.js.map +1 -1
- package/dist/{chunk-BARTLVBK.js → chunk-A4XQWVZK.js} +25 -25
- package/dist/chunk-A4XQWVZK.js.map +1 -0
- package/dist/{chunk-NNXLW6V7.js → chunk-OA5CSY5N.js} +147 -112
- package/dist/chunk-OA5CSY5N.js.map +1 -0
- package/dist/chunk-PZ5AY32C.js +10 -0
- package/dist/chunk-PZ5AY32C.js.map +1 -0
- package/dist/host-export.js +1 -0
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +12 -4
- package/dist/sandbox-archive-storage.d.ts +15 -3
- package/dist/sandbox-resume.d.ts +1 -1
- package/dist/sandbox-routing.d.ts +6 -1
- package/package.json +18 -17
- package/src/activities/agent-turn/claim.ts +3 -3
- package/src/activities/agent-turn/codex-capacity.ts +31 -53
- package/src/activities/agent-turn/codex-selection-diagnostics.ts +1 -0
- package/src/activities/agent-turn/finalization.ts +10 -5
- package/src/activities/agent-turn/mcp-operation-read-tool.ts +61 -0
- package/src/activities/agent-turn/sandbox-establish.ts +2 -0
- package/src/activities/agent-turn/skill-search.ts +2 -16
- package/src/activities/agent-turn/stream-attempt.ts +1 -1
- package/src/activities/agent-turn/tool-environment.ts +65 -0
- package/src/activities/codex-fleet-shadow.ts +3 -0
- package/src/activities/mcp-credentials.ts +94 -40
- package/src/activities/mcp-operation-authority.ts +76 -0
- package/src/activities/mcp-operation-observer.ts +86 -0
- package/src/activities/mcp-operation-reader.ts +108 -0
- package/src/activities/mcp-operation-store.ts +98 -0
- package/src/activities/sandbox-lease.ts +111 -73
- package/src/observability-metrics.ts +28 -24
- package/src/sandbox-archive-storage.ts +82 -18
- package/src/sandbox-resume.ts +79 -63
- package/src/sandbox-routing.ts +5 -1
- package/dist/activities-control-UXGPMDL3.js.map +0 -1
- package/dist/activities-turn-NOYDE7PX.js.map +0 -1
- package/dist/chunk-BARTLVBK.js.map +0 -1
- package/dist/chunk-NNXLW6V7.js.map +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { codexSelectionDiagnostics } from "@opengeni/db";
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { AttemptToolDefinition } from "@opengeni/codemode";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
declare const selectorSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
4
|
+
operationId: z.ZodString;
|
|
5
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
6
|
+
sourceTurnId: z.ZodString;
|
|
7
|
+
sourceCallId: z.ZodString;
|
|
8
|
+
}, z.core.$strict>]>;
|
|
9
|
+
export type McpOperationReadSelector = z.infer<typeof selectorSchema>;
|
|
10
|
+
/** Attempt-local adapter. The supplied read closure owns canonical session
|
|
11
|
+
* access, current provider authority, immutable binding checks and settlement.
|
|
12
|
+
* No caller-controlled provider parameters cross this tool boundary. */
|
|
13
|
+
export declare function createOperationReadAttemptToolDefinition(input: {
|
|
14
|
+
read: (selector: McpOperationReadSelector) => Promise<Record<string, unknown>>;
|
|
15
|
+
}): AttemptToolDefinition;
|
|
16
|
+
export {};
|
|
@@ -4,7 +4,7 @@ export declare const CODEX_FLEET_SHADOW_MAX_PAYLOAD_BYTES: number;
|
|
|
4
4
|
export type CodexFleetActualDecisionV1 = {
|
|
5
5
|
outcome: "selected" | "waiting" | "none";
|
|
6
6
|
candidateKey: string | null;
|
|
7
|
-
reason: "lease_reused" | "pin" | "rotation" | "active" | "all_capped" | "allocator_disabled" | "none";
|
|
7
|
+
reason: "lease_reused" | "affinity_reused" | "assigned" | "switched" | "pin" | "rotation" | "active" | "all_capped" | "allocator_disabled" | "none";
|
|
8
8
|
};
|
|
9
9
|
export type CodexFleetShadowComparisonV1 = "match" | "different_candidate" | "different_outcome" | "not_comparable_truncated";
|
|
10
10
|
export type CodexFleetShadowPayloadV1 = {
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { ConnectionUseAttribution } from "@opengeni/contracts/connection-authority";
|
|
2
|
+
import type { HostMcpAcceptedAuthority } from "@opengeni/contracts/host-mcp-bindings";
|
|
3
|
+
import type { ResolveConnectionCredentialInput } from "@opengeni/db";
|
|
4
|
+
/** Metadata only. Callers must obtain authority from a current live fence.
|
|
5
|
+
* This equality key is never authorization and never replaces physical-use checks. */
|
|
6
|
+
export declare function mcpOperationAuthorityDigest(request: ResolveConnectionCredentialInput, authority: {
|
|
7
|
+
native: ConnectionUseAttribution;
|
|
8
|
+
host?: never;
|
|
9
|
+
principal?: {
|
|
10
|
+
kind: string;
|
|
11
|
+
subjectId?: string;
|
|
12
|
+
};
|
|
13
|
+
} | {
|
|
14
|
+
host: HostMcpAcceptedAuthority;
|
|
15
|
+
native?: never;
|
|
16
|
+
}): string;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { Settings } from "@opengeni/config";
|
|
2
|
+
import type { AttemptToolEnvironment } from "@opengeni/codemode";
|
|
3
|
+
import { type ResolveConnectionCredentialInput, type ResolveConnectionCredentialResult } from "@opengeni/runtime";
|
|
4
|
+
import type { McpOperationReaderDependencies } from "./mcp-operation-reader.js";
|
|
5
|
+
/** Current-attempt adapter: no credential renewal authority survives this
|
|
6
|
+
* closure. The stored digest constrains fresh authority, never establishes it. */
|
|
7
|
+
export declare function createMcpOperationObserverResolver(input: {
|
|
8
|
+
settings: Pick<Settings, "mcpServers">;
|
|
9
|
+
workspaceId: string;
|
|
10
|
+
credentialSubjectId?: string;
|
|
11
|
+
assertAttempt: () => Promise<void>;
|
|
12
|
+
resolveCredential: (request: ResolveConnectionCredentialInput) => Promise<ResolveConnectionCredentialResult>;
|
|
13
|
+
getEnvironment: () => Promise<AttemptToolEnvironment | null>;
|
|
14
|
+
}): McpOperationReaderDependencies["resolveObserver"];
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { AttemptToolResult } from "@opengeni/contracts";
|
|
2
|
+
import { type McpObservationReceipt } from "@opengeni/runtime";
|
|
3
|
+
import type { McpOperationReadSelector } from "./agent-turn/mcp-operation-read-tool.js";
|
|
4
|
+
type TerminalReceipt = Extract<McpObservationReceipt, {
|
|
5
|
+
status: "completed";
|
|
6
|
+
}>;
|
|
7
|
+
/** Internal, credential-free projection loaded through canonical current
|
|
8
|
+
* attempt/session authorization. Never return this object directly to tools. */
|
|
9
|
+
export type McpOperationReadRecord = {
|
|
10
|
+
operationId: string;
|
|
11
|
+
sourceTurnId: string;
|
|
12
|
+
sourceCallId: string | null;
|
|
13
|
+
serverId: string;
|
|
14
|
+
originalTool: string;
|
|
15
|
+
observerTool: string;
|
|
16
|
+
argumentDigest: string;
|
|
17
|
+
destinationDigest: string;
|
|
18
|
+
authorityDigest: string;
|
|
19
|
+
originalOutcome: "captured" | "completed" | "outcome_unknown";
|
|
20
|
+
originalResult: AttemptToolResult | null;
|
|
21
|
+
receipt: TerminalReceipt | null;
|
|
22
|
+
};
|
|
23
|
+
type ObserverResolution = {
|
|
24
|
+
status: "unsupported" | "auth_needed" | "binding_changed";
|
|
25
|
+
} | {
|
|
26
|
+
status: "ready";
|
|
27
|
+
authorize: (phase: "before_request" | "before_delivery") => Promise<boolean>;
|
|
28
|
+
callObserver: (tool: string, args: Record<string, unknown>) => Promise<unknown>;
|
|
29
|
+
};
|
|
30
|
+
export type McpOperationReaderDependencies = {
|
|
31
|
+
/** Includes live attempt and original-principal/session disclosure checks. */
|
|
32
|
+
load: (selector: McpOperationReadSelector) => Promise<McpOperationReadRecord | null>;
|
|
33
|
+
claim: (operationId: string) => Promise<string | null>;
|
|
34
|
+
release: (operationId: string, claimId: string) => Promise<void>;
|
|
35
|
+
/** Claim-fenced transaction storing one immutable receipt. Must recheck
|
|
36
|
+
* canonical current authority and never rewrite the original outcome.
|
|
37
|
+
* Delivery uses the ordinary operation_read tool-result lifecycle. */
|
|
38
|
+
settle: (operationId: string, claimId: string, receipt: TerminalReceipt) => Promise<McpOperationReadRecord>;
|
|
39
|
+
/** Binds the current trusted config/connection to the persisted original
|
|
40
|
+
* digests. Model arguments may not supply or replace any of that authority. */
|
|
41
|
+
resolveObserver: (operation: McpOperationReadRecord) => Promise<ObserverResolution>;
|
|
42
|
+
};
|
|
43
|
+
/** One explicit read; no timer, inference wake, or mutation replay. */
|
|
44
|
+
export declare function readMcpOperation(selector: McpOperationReadSelector, dependencies: McpOperationReaderDependencies): Promise<Record<string, unknown>>;
|
|
45
|
+
export {};
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { Database } from "@opengeni/db";
|
|
2
|
+
import { type McpOperationAttempt } from "@opengeni/db/mcp-operations";
|
|
3
|
+
import type { McpOperationReaderDependencies } from "./mcp-operation-reader.js";
|
|
4
|
+
/** All DB calls remain bound to the actual accepted worker attempt. Locators
|
|
5
|
+
* supplied to operation_read cannot replace this scope or its principal. */
|
|
6
|
+
export declare function createMcpOperationReadStore(db: Database, attempt: McpOperationAttempt): Pick<McpOperationReaderDependencies, "load" | "claim" | "release" | "settle">;
|
|
@@ -2,7 +2,7 @@ import type { ProviderCommandPersistence } from "@opengeni/runtime";
|
|
|
2
2
|
import { type OpStatus } from "@opengeni/agent-proto";
|
|
3
3
|
import { listLiveModalSandboxLeaseAttributions, readLease, type ReapDrainable, type RetainedProcessProviderProof, type SandboxRetainedProcess, type LeaseSnapshot } from "@opengeni/db";
|
|
4
4
|
import { type ConnectedMachineBackgroundCommandClaim, type ConnectedMachineBackgroundCommandProof } from "@opengeni/db/session-background-commands";
|
|
5
|
-
import { createSandboxClientForBackend, inspectModalSandboxLifecycle, resolveModalCheckpointProviderBindingForSession, terminateModalSandboxById, type ModalOrphanSweepTermination, type ModalCheckpointProviderBinding, type ProviderWorkspaceCapturePolicy, type WorkspaceArchiveDescriptor, type ControlRpc } from "@opengeni/runtime/sandbox";
|
|
5
|
+
import { type VerifiedHostWorkspaceArchive, createSandboxClientForBackend, inspectModalSandboxLifecycle, resolveModalCheckpointProviderBindingForSession, terminateModalSandboxById, type ModalOrphanSweepTermination, type ModalCheckpointProviderBinding, type ProviderWorkspaceCapturePolicy, type WorkspaceArchiveDescriptor, type ControlRpc } from "@opengeni/runtime/sandbox";
|
|
6
6
|
import { type SandboxDrainActivityInput, type SandboxLeaseSweepMaintenanceInput } from "../sandbox-reaper-contract.js";
|
|
7
7
|
import type { ControlActivityServices as ActivityServices } from "./types.js";
|
|
8
8
|
import { type RetainedProcessReconciliationOutcome } from "../observability-metrics.js";
|
|
@@ -40,7 +40,7 @@ type CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;
|
|
|
40
40
|
* backends with no persistWorkspace — ensures a re-arm during the snapshot window
|
|
41
41
|
* aborts the terminate before client.delete()).
|
|
42
42
|
*/
|
|
43
|
-
export type PersistArchiveFn = (archiveBase64: string | null, archiveMetadata?: WorkspaceArchiveDescriptor, providerSession?: unknown, providerBinding?: Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForSession>> | null) => Promise<{
|
|
43
|
+
export type PersistArchiveFn = (archiveBase64: string | VerifiedHostWorkspaceArchive | null, archiveMetadata?: WorkspaceArchiveDescriptor, providerSession?: unknown, providerBinding?: Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForSession>> | null) => Promise<{
|
|
44
44
|
wrote: boolean;
|
|
45
45
|
archiveRevision?: string | null;
|
|
46
46
|
}>;
|
|
@@ -60,7 +60,7 @@ export type DrainCaptureDisposition = "capture_required" | "archive_published";
|
|
|
60
60
|
* Test seams may return the legacy boolean shorthand. Production returns the
|
|
61
61
|
* typed outcome so the cold commit can distinguish a clean stop from definitive
|
|
62
62
|
* provider disappearance before capture. */
|
|
63
|
-
export type TerminateBoxFn = (settings: ActivityServices["settings"], lease: NonNullable<Awaited<ReturnType<typeof readLease>>>, observability: ActivityServices["observability"], persistArchive: PersistArchiveFn, providerCaptureRequestId?: string, captureDisposition?: DrainCaptureDisposition, capturePolicy?: ProviderWorkspaceCapturePolicy | null) => Promise<boolean | ProviderTerminationOutcome>;
|
|
63
|
+
export type TerminateBoxFn = (settings: ActivityServices["settings"], lease: NonNullable<Awaited<ReturnType<typeof readLease>>>, observability: ActivityServices["observability"], persistArchive: PersistArchiveFn, providerCaptureRequestId?: string, captureDisposition?: DrainCaptureDisposition, capturePolicy?: ProviderWorkspaceCapturePolicy | null, diskBackedArchives?: boolean, releaseFailedCapture?: () => Promise<void>) => Promise<boolean | ProviderTerminationOutcome>;
|
|
64
64
|
export type SweepModalOrphansFn = (settings: ActivityServices["settings"], db: ActivityServices["db"], observability: ActivityServices["observability"]) => Promise<number>;
|
|
65
65
|
export type InspectOpenSandboxKubernetesInventoryFn = (input: {
|
|
66
66
|
namespace: string;
|
|
@@ -114,6 +114,14 @@ export declare class SandboxProviderCaptureTimeoutError extends Error {
|
|
|
114
114
|
readonly name = "SandboxProviderCaptureTimeoutError";
|
|
115
115
|
constructor(sandboxGroupId: string, backend: string, timeoutMs: number, leaseEpoch: number, instanceId: string);
|
|
116
116
|
}
|
|
117
|
+
export declare function awaitProviderCaptureWithLatePublication<T>(input: {
|
|
118
|
+
capture: Promise<T>;
|
|
119
|
+
timeoutMs: number;
|
|
120
|
+
timeoutError: SandboxProviderCaptureTimeoutError;
|
|
121
|
+
publishLate: (value: T) => Promise<void>;
|
|
122
|
+
observeLateFailure: (error: unknown) => void;
|
|
123
|
+
observeLatePublicationFailure: (error: unknown) => void;
|
|
124
|
+
}): Promise<T>;
|
|
117
125
|
export type SandboxLeaseSweepPlan = {
|
|
118
126
|
drainable: ReapDrainable[];
|
|
119
127
|
timeoutClass: SandboxDrainActivityInput["timeoutClass"];
|
|
@@ -199,4 +207,4 @@ export declare function sandboxDrainReason(lease: Pick<LeaseSnapshot, "rotationR
|
|
|
199
207
|
* draining until its adapter can prove termination. A truly instance-less or
|
|
200
208
|
* `none`-backed row is a no-op.
|
|
201
209
|
*/
|
|
202
|
-
export declare function terminateProviderBox(settings: ActivityServices["settings"], lease: NonNullable<Awaited<ReturnType<typeof readLease>>>, observability: ActivityServices["observability"], persistArchive: PersistArchiveFn, createClientForBackend?: CreateSandboxClientForBackendFn, terminateModalById?: typeof terminateModalSandboxById, providerCaptureRequestId?: string, captureDisposition?: DrainCaptureDisposition, claimedCapturePolicy?: ProviderWorkspaceCapturePolicy | null): Promise<ProviderTerminationOutcome>;
|
|
210
|
+
export declare function terminateProviderBox(settings: ActivityServices["settings"], lease: NonNullable<Awaited<ReturnType<typeof readLease>>>, observability: ActivityServices["observability"], persistArchive: PersistArchiveFn, createClientForBackend?: CreateSandboxClientForBackendFn, terminateModalById?: typeof terminateModalSandboxById, providerCaptureRequestId?: string, captureDisposition?: DrainCaptureDisposition, claimedCapturePolicy?: ProviderWorkspaceCapturePolicy | null, diskBackedArchives?: boolean, releaseFailedCapture?: () => Promise<void>): Promise<ProviderTerminationOutcome>;
|
|
@@ -10,18 +10,16 @@ import {
|
|
|
10
10
|
MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
11
11
|
assertReusableSessionRevivable,
|
|
12
12
|
captureConnectedCommandOutput,
|
|
13
|
-
collectWorkspaceArchiveObjectKeys,
|
|
14
13
|
createCodexCapacityActivities,
|
|
15
14
|
createGoalActivities,
|
|
16
15
|
createSharedActivityServices,
|
|
17
16
|
currentActivityContext,
|
|
18
|
-
deleteUnpublishedWorkspaceArchiveObject,
|
|
19
|
-
deleteWorkspaceArchiveObjectKeys,
|
|
20
17
|
deliverFailedChildTurnToParent,
|
|
21
18
|
evaluateIncidentTelemetryPreflight,
|
|
22
19
|
incidentTelemetryAuthorityFence,
|
|
23
20
|
incidentTelemetryPreflightDeclaration,
|
|
24
21
|
notifyParentOfChildIdle,
|
|
22
|
+
persistWorkspaceArchiveCandidate,
|
|
25
23
|
providerRecoveryCountFromMetadata,
|
|
26
24
|
putVersion1TarArchiveOrInline,
|
|
27
25
|
reconcileAutomaticSessionTitleFanout,
|
|
@@ -33,7 +31,7 @@ import {
|
|
|
33
31
|
scheduledUserMessagePayload,
|
|
34
32
|
withFirstPartyTools,
|
|
35
33
|
workflowIdForSession
|
|
36
|
-
} from "./chunk-
|
|
34
|
+
} from "./chunk-OA5CSY5N.js";
|
|
37
35
|
import {
|
|
38
36
|
inspectOpenSandboxKubernetesInventory,
|
|
39
37
|
recordCreditBalanceGauges,
|
|
@@ -53,7 +51,8 @@ import {
|
|
|
53
51
|
recordTurnsQueuedGauge,
|
|
54
52
|
recordWorkerDeathRecoveryMetrics,
|
|
55
53
|
runtimeMetricsHooksForObservability
|
|
56
|
-
} from "./chunk-
|
|
54
|
+
} from "./chunk-A4XQWVZK.js";
|
|
55
|
+
import "./chunk-PZ5AY32C.js";
|
|
57
56
|
|
|
58
57
|
// src/activities/browser-state-artifact-reaper.ts
|
|
59
58
|
import {
|
|
@@ -5200,6 +5199,7 @@ import {
|
|
|
5200
5199
|
appendSessionEventToSandboxGroup,
|
|
5201
5200
|
bindRetainedProcessProviderIdentity,
|
|
5202
5201
|
claimWorkspaceArchiveCapture,
|
|
5202
|
+
releaseWorkspaceArchiveCapture,
|
|
5203
5203
|
claimSandboxCheckpointArtifactsForGc,
|
|
5204
5204
|
claimTerminalRetainedProcesses,
|
|
5205
5205
|
countActiveRetainedProcessesByOwnerState,
|
|
@@ -5243,7 +5243,8 @@ import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
|
|
|
5243
5243
|
import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
|
|
5244
5244
|
import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
|
|
5245
5245
|
import {
|
|
5246
|
-
|
|
5246
|
+
captureWorkspaceArchiveForStorage,
|
|
5247
|
+
disposeWorkspaceArchive,
|
|
5247
5248
|
cancelSelfhostedOp,
|
|
5248
5249
|
assertConsistentSandboxProviderIdentity,
|
|
5249
5250
|
createSandboxClientForBackend,
|
|
@@ -5312,7 +5313,7 @@ async function awaitProviderCaptureWithLatePublication(input) {
|
|
|
5312
5313
|
]);
|
|
5313
5314
|
} catch (error) {
|
|
5314
5315
|
if (error !== timeoutError) throw error;
|
|
5315
|
-
void input.capture.then(input.publishLate).catch(input.
|
|
5316
|
+
void input.capture.then(input.publishLate, input.observeLateFailure).catch(input.observeLatePublicationFailure);
|
|
5316
5317
|
throw error;
|
|
5317
5318
|
} finally {
|
|
5318
5319
|
if (timeout) clearTimeout(timeout);
|
|
@@ -5362,7 +5363,7 @@ function sandboxDrainActivityAttempt() {
|
|
|
5362
5363
|
}
|
|
5363
5364
|
}
|
|
5364
5365
|
function createSandboxLeaseActivities(services, options = {}) {
|
|
5365
|
-
const terminateBox = options.terminateBox ?? (async (settings, lease, observability, persistArchive, providerCaptureRequestId, captureDisposition, capturePolicy) => await terminateProviderBox(
|
|
5366
|
+
const terminateBox = options.terminateBox ?? (async (settings, lease, observability, persistArchive, providerCaptureRequestId, captureDisposition, capturePolicy, diskBackedArchives, releaseFailedCapture) => await terminateProviderBox(
|
|
5366
5367
|
settings,
|
|
5367
5368
|
lease,
|
|
5368
5369
|
observability,
|
|
@@ -5371,7 +5372,9 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
5371
5372
|
terminateModalSandboxById,
|
|
5372
5373
|
providerCaptureRequestId,
|
|
5373
5374
|
captureDisposition,
|
|
5374
|
-
capturePolicy
|
|
5375
|
+
capturePolicy,
|
|
5376
|
+
diskBackedArchives,
|
|
5377
|
+
releaseFailedCapture
|
|
5375
5378
|
));
|
|
5376
5379
|
const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
5377
5380
|
const inspectOpenSandboxInventory = options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;
|
|
@@ -7044,7 +7047,7 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
7044
7047
|
}
|
|
7045
7048
|
const archiveMetrics = runtimeMetricsHooksForObservability(observability);
|
|
7046
7049
|
let checkpointArtifactId = null;
|
|
7047
|
-
if (archiveBase64 && archiveMetadata?.version === 2 && (archiveMetadata.provider === "modal_snapshot_filesystem" || archiveMetadata.provider === "modal_snapshot_directory")) {
|
|
7050
|
+
if (typeof archiveBase64 === "string" && archiveBase64 && archiveMetadata?.version === 2 && (archiveMetadata.provider === "modal_snapshot_filesystem" || archiveMetadata.provider === "modal_snapshot_directory")) {
|
|
7048
7051
|
if (!providerBinding) {
|
|
7049
7052
|
throw new Error("Modal native snapshot has no exact session provider identity");
|
|
7050
7053
|
}
|
|
@@ -7084,57 +7087,29 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
7084
7087
|
if (!archiveMetadata) {
|
|
7085
7088
|
throw new Error("Sandbox snapshot publication requires a verified archive descriptor");
|
|
7086
7089
|
}
|
|
7087
|
-
const
|
|
7088
|
-
lease.
|
|
7089
|
-
|
|
7090
|
-
|
|
7091
|
-
|
|
7092
|
-
|
|
7093
|
-
|
|
7094
|
-
|
|
7095
|
-
|
|
7096
|
-
|
|
7097
|
-
|
|
7098
|
-
|
|
7099
|
-
|
|
7100
|
-
|
|
7101
|
-
|
|
7102
|
-
|
|
7103
|
-
|
|
7104
|
-
});
|
|
7105
|
-
workspaceArchiveRef = published.workspaceArchiveRef;
|
|
7106
|
-
result = await persistDrainSnapshot(db, {
|
|
7090
|
+
const published = await putVersion1TarArchiveOrInline({
|
|
7091
|
+
backend: lease.backend,
|
|
7092
|
+
objectStorage,
|
|
7093
|
+
accountId,
|
|
7094
|
+
workspaceId: row.workspaceId,
|
|
7095
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
7096
|
+
archive: typeof archiveBase64 === "object" ? archiveBase64 : {
|
|
7097
|
+
bytes: Buffer.from(archiveBase64, "base64"),
|
|
7098
|
+
descriptor: archiveMetadata,
|
|
7099
|
+
base64: archiveBase64
|
|
7100
|
+
},
|
|
7101
|
+
metrics: archiveMetrics
|
|
7102
|
+
});
|
|
7103
|
+
result = await persistWorkspaceArchiveCandidate({
|
|
7104
|
+
objectStorage,
|
|
7105
|
+
...published.workspaceArchiveRef ? { ref: published.workspaceArchiveRef } : {},
|
|
7106
|
+
persist: () => persistDrainSnapshot(db, {
|
|
7107
7107
|
...baseInput,
|
|
7108
7108
|
workspaceArchiveMeta: archiveMetadata,
|
|
7109
7109
|
...published,
|
|
7110
7110
|
...checkpointArtifactId ? { checkpointArtifactId } : {}
|
|
7111
|
-
})
|
|
7112
|
-
|
|
7113
|
-
if (!result.wrote) {
|
|
7114
|
-
await deleteUnpublishedWorkspaceArchiveObject(
|
|
7115
|
-
objectStorage,
|
|
7116
|
-
published.workspaceArchiveRef,
|
|
7117
|
-
archiveMetrics
|
|
7118
|
-
);
|
|
7119
|
-
} else {
|
|
7120
|
-
const afterLease = await readLease(db, row.workspaceId, row.sandboxGroupId);
|
|
7121
|
-
const afterKeys = collectWorkspaceArchiveObjectKeys(
|
|
7122
|
-
afterLease?.resumeState ?? null
|
|
7123
|
-
);
|
|
7124
|
-
await deleteWorkspaceArchiveObjectKeys(
|
|
7125
|
-
objectStorage,
|
|
7126
|
-
[...priorKeys].filter((key) => !afterKeys.has(key))
|
|
7127
|
-
).catch(() => void 0);
|
|
7128
|
-
}
|
|
7129
|
-
}
|
|
7130
|
-
} catch (error) {
|
|
7131
|
-
await deleteUnpublishedWorkspaceArchiveObject(
|
|
7132
|
-
objectStorage,
|
|
7133
|
-
workspaceArchiveRef,
|
|
7134
|
-
archiveMetrics
|
|
7135
|
-
);
|
|
7136
|
-
throw error;
|
|
7137
|
-
}
|
|
7111
|
+
})
|
|
7112
|
+
});
|
|
7138
7113
|
}
|
|
7139
7114
|
if (!result.wrote && checkpointArtifactId) {
|
|
7140
7115
|
await markSandboxCheckpointArtifactDeletePending2(db, {
|
|
@@ -7157,7 +7132,19 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
7157
7132
|
persistArchive,
|
|
7158
7133
|
captureClaim?.providerRequestId ?? attempt.operationId,
|
|
7159
7134
|
captureDisposition,
|
|
7160
|
-
capturePolicy
|
|
7135
|
+
capturePolicy,
|
|
7136
|
+
Boolean(objectStorage),
|
|
7137
|
+
async () => {
|
|
7138
|
+
if (!captureClaim || !lease.instanceId) return;
|
|
7139
|
+
await releaseWorkspaceArchiveCapture(db, {
|
|
7140
|
+
accountId,
|
|
7141
|
+
workspaceId: row.workspaceId,
|
|
7142
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
7143
|
+
captureId: captureClaim.id,
|
|
7144
|
+
expectedEpoch: row.leaseEpoch,
|
|
7145
|
+
expectedInstanceId: lease.instanceId
|
|
7146
|
+
});
|
|
7147
|
+
}
|
|
7161
7148
|
);
|
|
7162
7149
|
const terminated = typeof termination === "boolean" ? termination : termination.terminated;
|
|
7163
7150
|
if (!terminated) {
|
|
@@ -7213,10 +7200,20 @@ function providerDrainLogIdentity(lease, backend) {
|
|
|
7213
7200
|
instanceId: lease.instanceId
|
|
7214
7201
|
};
|
|
7215
7202
|
}
|
|
7216
|
-
async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend, terminateModalById = terminateModalSandboxById, providerCaptureRequestId, captureDisposition = "capture_required", claimedCapturePolicy) {
|
|
7203
|
+
async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend, terminateModalById = terminateModalSandboxById, providerCaptureRequestId, captureDisposition = "capture_required", claimedCapturePolicy, diskBackedArchives = false, releaseFailedCapture) {
|
|
7217
7204
|
const durableBackendId = lease.resumeBackendId ?? lease.backend;
|
|
7218
7205
|
const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
|
|
7219
7206
|
const logIdentity = providerDrainLogIdentity(lease, backend);
|
|
7207
|
+
const releaseAfterCaptureFailure = async () => {
|
|
7208
|
+
try {
|
|
7209
|
+
await releaseFailedCapture?.();
|
|
7210
|
+
} catch (error) {
|
|
7211
|
+
observability.warn("sandbox reaper: failed capture claim release failed", {
|
|
7212
|
+
...logIdentity,
|
|
7213
|
+
error: error instanceof Error ? error.message : String(error)
|
|
7214
|
+
});
|
|
7215
|
+
}
|
|
7216
|
+
};
|
|
7220
7217
|
if (!backend || backend === "none") {
|
|
7221
7218
|
return { terminated: true, providerMissingBeforeCapture: false };
|
|
7222
7219
|
}
|
|
@@ -7335,10 +7332,15 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
7335
7332
|
) : null;
|
|
7336
7333
|
try {
|
|
7337
7334
|
if (captureDisposition !== "archive_published" && session?.persistWorkspace) {
|
|
7338
|
-
const capture =
|
|
7339
|
-
|
|
7340
|
-
|
|
7341
|
-
|
|
7335
|
+
const capture = captureWorkspaceArchiveForStorage(
|
|
7336
|
+
session,
|
|
7337
|
+
Date.now(),
|
|
7338
|
+
{
|
|
7339
|
+
requestId: providerCaptureRequestId ?? randomUUID2(),
|
|
7340
|
+
strategy: liveCapturePolicy.strategy
|
|
7341
|
+
},
|
|
7342
|
+
diskBackedArchives
|
|
7343
|
+
);
|
|
7342
7344
|
verifiedArchive = await awaitProviderCaptureWithLatePublication({
|
|
7343
7345
|
capture,
|
|
7344
7346
|
timeoutMs: settings.sandboxSnapshotTimeoutMs,
|
|
@@ -7352,7 +7354,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
7352
7354
|
publishLate: async (archive) => {
|
|
7353
7355
|
try {
|
|
7354
7356
|
const result = await persistArchive(
|
|
7355
|
-
archive.base64,
|
|
7357
|
+
archive.kind === "host_spool" ? archive : archive.base64,
|
|
7356
7358
|
archive.descriptor,
|
|
7357
7359
|
session,
|
|
7358
7360
|
checkpointBinding
|
|
@@ -7368,9 +7370,18 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
7368
7370
|
...logIdentity,
|
|
7369
7371
|
error: error instanceof Error ? error.message : String(error)
|
|
7370
7372
|
});
|
|
7373
|
+
} finally {
|
|
7374
|
+
await disposeWorkspaceArchive(archive);
|
|
7371
7375
|
}
|
|
7372
7376
|
},
|
|
7377
|
+
observeLatePublicationFailure: (error) => {
|
|
7378
|
+
observability.warn("sandbox reaper: late capture publication cleanup failed", {
|
|
7379
|
+
...logIdentity,
|
|
7380
|
+
error: error instanceof Error ? error.message : String(error)
|
|
7381
|
+
});
|
|
7382
|
+
},
|
|
7373
7383
|
observeLateFailure: (error) => {
|
|
7384
|
+
void releaseAfterCaptureFailure();
|
|
7374
7385
|
observability.warn("sandbox reaper: timed-out provider capture later failed", {
|
|
7375
7386
|
...logIdentity,
|
|
7376
7387
|
error: error instanceof Error ? error.message : String(error)
|
|
@@ -7392,6 +7403,9 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
7392
7403
|
error: error instanceof Error ? error.message : String(error)
|
|
7393
7404
|
}
|
|
7394
7405
|
);
|
|
7406
|
+
if (!(error instanceof SandboxProviderCaptureTimeoutError)) {
|
|
7407
|
+
await releaseAfterCaptureFailure();
|
|
7408
|
+
}
|
|
7395
7409
|
throw error;
|
|
7396
7410
|
}
|
|
7397
7411
|
if (captureDisposition === "archive_published") {
|
|
@@ -7399,12 +7413,17 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
7399
7413
|
...logIdentity
|
|
7400
7414
|
});
|
|
7401
7415
|
} else if (verifiedArchive) {
|
|
7402
|
-
|
|
7403
|
-
|
|
7404
|
-
|
|
7405
|
-
|
|
7406
|
-
|
|
7407
|
-
|
|
7416
|
+
let wrote;
|
|
7417
|
+
try {
|
|
7418
|
+
({ wrote } = await persistArchive(
|
|
7419
|
+
verifiedArchive.kind === "host_spool" ? verifiedArchive : verifiedArchive.base64,
|
|
7420
|
+
verifiedArchive.descriptor,
|
|
7421
|
+
session,
|
|
7422
|
+
checkpointBinding
|
|
7423
|
+
));
|
|
7424
|
+
} finally {
|
|
7425
|
+
await disposeWorkspaceArchive(verifiedArchive);
|
|
7426
|
+
}
|
|
7408
7427
|
if (!wrote) {
|
|
7409
7428
|
observability.info(
|
|
7410
7429
|
"sandbox reaper: lease re-armed during persist \u2014 leaving box RUNNING (no terminate)",
|
|
@@ -10350,4 +10369,4 @@ export {
|
|
|
10350
10369
|
createControlActivities,
|
|
10351
10370
|
createControlActivitiesFromServices
|
|
10352
10371
|
};
|
|
10353
|
-
//# sourceMappingURL=activities-control-
|
|
10372
|
+
//# sourceMappingURL=activities-control-MKGEOBXT.js.map
|