@opengeni/worker-bundle 0.20.3 → 0.22.1-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +52 -0
- package/dist/activities/agent-turn/agent-build.d.ts +89 -0
- package/dist/activities/agent-turn/claim.d.ts +67 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
- package/dist/activities/agent-turn/codex.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
- package/dist/activities/agent-turn/errors.d.ts +253 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
- package/dist/activities/agent-turn/file-resources.d.ts +37 -0
- package/dist/activities/agent-turn/finalization.d.ts +42 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
- package/dist/activities/agent-turn/governance-model.d.ts +66 -0
- package/dist/activities/agent-turn/history-sink.d.ts +52 -0
- package/dist/activities/agent-turn/history.d.ts +63 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
- package/dist/activities/agent-turn/model-usage.d.ts +218 -0
- package/dist/activities/agent-turn/quiescence.d.ts +135 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
- package/dist/activities/agent-turn/run.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
- package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
- package/dist/activities/agent-turn/turn-context.d.ts +144 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
- package/dist/activities/agent-turn.d.ts +12 -954
- package/dist/activities/automations.d.ts +16 -0
- package/dist/activities/common.d.ts +1 -1
- package/dist/activities/connector-attachments.d.ts +13 -0
- package/dist/activities/documents.d.ts +1 -0
- package/dist/activities/environment.d.ts +26 -15
- package/dist/activities/generated-images.d.ts +1 -1
- package/dist/activities/git-credential-renewal.d.ts +3 -1
- package/dist/activities/goals.d.ts +4 -1
- package/dist/activities/google-drive-provider.d.ts +29 -0
- package/dist/activities/google-drive-publication.d.ts +17 -4
- package/dist/activities/incident-telemetry-authority.d.ts +52 -0
- package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
- package/dist/activities/knowledge-source-sync.d.ts +4 -0
- package/dist/activities/mcp-credentials.d.ts +9 -3
- package/dist/activities/open-suffix-resume.d.ts +77 -0
- package/dist/activities/packs.d.ts +3 -11
- package/dist/activities/parent-wake.d.ts +19 -0
- package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
- package/dist/activities/retained-screenshots.d.ts +1 -1
- package/dist/activities/rig-verification.d.ts +1 -1
- package/dist/activities/run-input.d.ts +9 -10
- package/dist/activities/sandbox-images.d.ts +1 -14
- package/dist/activities/sandbox-lease.d.ts +21 -3
- package/dist/activities/scheduled-tasks.d.ts +47 -0
- package/dist/activities/types.d.ts +23 -2
- package/dist/activities/video-media-validation.d.ts +1 -1
- package/dist/activities/video-output-retention.d.ts +1 -2
- package/dist/activities/xai-auth.d.ts +3 -0
- package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
- package/dist/activities-control-DKG22ONJ.js.map +1 -0
- package/dist/activities-control.d.ts +6 -2
- package/dist/activities-turn-4IMC3MAA.js +21326 -0
- package/dist/activities-turn-4IMC3MAA.js.map +1 -0
- package/dist/activities.d.ts +7 -3
- package/dist/artifact-materializer-entry.js +41 -22
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
- package/dist/chunk-CE7TPKBS.js.map +1 -0
- package/dist/chunk-CNYGVRKF.js +4698 -0
- package/dist/chunk-CNYGVRKF.js.map +1 -0
- package/dist/connection-provider-use.d.ts +18 -0
- package/dist/editable-artifact-materializer.d.ts +4 -3
- package/dist/github-rest-mcp.d.ts +26 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +58 -14
- package/dist/index.js.map +1 -1
- package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
- package/dist/model-context-contributions.d.ts +8 -4
- package/dist/observability-metrics.d.ts +63 -4
- package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
- package/dist/personal-github-git-credentials.d.ts +9 -0
- package/dist/pr-review-credentials.d.ts +6 -0
- package/dist/sandbox-archive-storage.d.ts +45 -0
- package/dist/sandbox-resume.d.ts +65 -6
- package/dist/sandbox-routing.d.ts +38 -2
- package/dist/scheduled-alert-occurrence.d.ts +33 -0
- package/dist/workflow-bundle.js +87 -28
- package/dist/workflows/activities.d.ts +1 -0
- package/dist/workflows/automations.d.ts +6 -0
- package/dist/workflows/document-index.d.ts +1 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +292 -0
- package/src/activities/agent-turn/agent-build.ts +802 -0
- package/src/activities/agent-turn/claim.ts +590 -0
- package/src/activities/agent-turn/codex-capacity.ts +720 -0
- package/src/activities/agent-turn/codex.ts +94 -0
- package/src/activities/agent-turn/compaction-prep.ts +722 -0
- package/src/activities/agent-turn/credential-leases.ts +240 -0
- package/src/activities/agent-turn/errors.ts +1068 -0
- package/src/activities/agent-turn/failure-settlement.ts +1436 -0
- package/src/activities/agent-turn/file-resources.ts +227 -0
- package/src/activities/agent-turn/finalization.ts +754 -0
- package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
- package/src/activities/agent-turn/governance-model.ts +465 -0
- package/src/activities/agent-turn/history-sink.ts +139 -0
- package/src/activities/agent-turn/history.ts +200 -0
- package/src/activities/agent-turn/media-artifacts.ts +359 -0
- package/src/activities/agent-turn/model-usage.ts +823 -0
- package/src/activities/agent-turn/quiescence.ts +353 -0
- package/src/activities/agent-turn/run-credentials.ts +761 -0
- package/src/activities/agent-turn/run.ts +1530 -0
- package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
- package/src/activities/agent-turn/sandbox-provision.ts +569 -0
- package/src/activities/agent-turn/sandbox-route.ts +358 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
- package/src/activities/agent-turn/stream-attempt.ts +1670 -0
- package/src/activities/agent-turn/tool-environment.ts +729 -0
- package/src/activities/agent-turn/tool-policy.ts +216 -0
- package/src/activities/agent-turn/tool-result-spill.ts +265 -0
- package/src/activities/agent-turn/turn-context.ts +281 -0
- package/src/activities/agent-turn/xai-capacity.ts +229 -0
- package/src/activities/agent-turn.ts +12 -12701
- package/src/activities/api-integrations.ts +3 -0
- package/src/activities/automations.ts +214 -0
- package/src/activities/codemode-dispatcher.ts +12 -1
- package/src/activities/common.ts +1 -5
- package/src/activities/connector-attachments.ts +168 -0
- package/src/activities/context-compaction.ts +18 -0
- package/src/activities/environment.ts +151 -55
- package/src/activities/generated-images.ts +26 -11
- package/src/activities/git-credential-renewal.ts +13 -3
- package/src/activities/goals.ts +85 -19
- package/src/activities/google-drive-provider.ts +178 -0
- package/src/activities/google-drive-publication.ts +147 -49
- package/src/activities/incident-telemetry-authority.ts +327 -0
- package/src/activities/incident-telemetry-preflight.ts +317 -0
- package/src/activities/knowledge-source-sync.ts +178 -37
- package/src/activities/mcp-credentials.ts +191 -17
- package/src/activities/open-suffix-resume.ts +422 -0
- package/src/activities/packs.ts +12 -60
- package/src/activities/parent-wake.ts +64 -14
- package/src/activities/retained-screenshot-reaper.ts +6 -3
- package/src/activities/retained-screenshots.ts +15 -47
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +102 -142
- package/src/activities/sandbox-images.ts +7 -166
- package/src/activities/sandbox-lease.ts +406 -15
- package/src/activities/scheduled-tasks.ts +2103 -411
- package/src/activities/session-state.ts +10 -2
- package/src/activities/types.ts +32 -2
- package/src/activities/video-media-validation.ts +3 -3
- package/src/activities/video-output-retention.ts +2 -57
- package/src/activities/xai-auth.ts +8 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/activity-services.ts +13 -4
- package/src/connection-provider-use.ts +36 -0
- package/src/editable-artifact-materializer-subprocess.ts +25 -14
- package/src/editable-artifact-materializer.ts +29 -18
- package/src/github-rest-mcp.ts +713 -0
- package/src/http.ts +16 -6
- package/src/index.ts +40 -5
- package/src/knowledge-source-sync-activity-policy.ts +2 -0
- package/src/model-context-contributions.ts +33 -10
- package/src/observability-metrics.ts +357 -0
- package/src/opensandbox-kubernetes-inventory.ts +292 -0
- package/src/personal-github-git-credentials.ts +331 -0
- package/src/pr-review-credentials.ts +163 -0
- package/src/sandbox-archive-storage.ts +159 -0
- package/src/sandbox-resume.ts +449 -63
- package/src/sandbox-routing.ts +314 -16
- package/src/scheduled-alert-occurrence.ts +176 -0
- package/src/workflows/activities.ts +18 -2
- package/src/workflows/automations.ts +11 -0
- package/src/workflows/session.ts +13 -3
- package/src/workflows.ts +1 -0
- package/dist/activities-control-BJSDRSIP.js.map +0 -1
- package/dist/activities-turn-J5YO7XXX.js +0 -17243
- package/dist/activities-turn-J5YO7XXX.js.map +0 -1
- package/dist/chunk-N5TJIFCE.js.map +0 -1
- package/dist/chunk-P6Z2SLAA.js +0 -3092
- package/dist/chunk-P6Z2SLAA.js.map +0 -1
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
|
|
17
17
|
import { createHash, randomUUID } from "node:crypto";
|
|
18
18
|
import { Context } from "@temporalio/activity";
|
|
19
|
+
import { OpLostReason, OpState, type OpStatus } from "@opengeni/agent-proto";
|
|
19
20
|
import {
|
|
20
21
|
accrueWarmSeconds,
|
|
21
22
|
adoptLegacyModalCheckpointArtifact,
|
|
@@ -61,6 +62,15 @@ import {
|
|
|
61
62
|
type SandboxRetainedProcess,
|
|
62
63
|
type LeaseSnapshot,
|
|
63
64
|
} from "@opengeni/db";
|
|
65
|
+
import {
|
|
66
|
+
claimConnectedMachineSessionBackgroundCommands,
|
|
67
|
+
deferConnectedMachineBackgroundCommandReconciliation,
|
|
68
|
+
recordConnectedMachineBackgroundCommandProof,
|
|
69
|
+
settleClaimedConnectedMachineBackgroundCommand,
|
|
70
|
+
settleSessionBackgroundCommandForRetainedProcess,
|
|
71
|
+
type ConnectedMachineBackgroundCommandClaim,
|
|
72
|
+
type ConnectedMachineBackgroundCommandProof,
|
|
73
|
+
} from "@opengeni/db/session-background-commands";
|
|
64
74
|
import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
|
|
65
75
|
import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
|
|
66
76
|
import {
|
|
@@ -70,6 +80,7 @@ import {
|
|
|
70
80
|
// establishSandboxSessionFromEnvelope in strict resume-only mode so it can
|
|
71
81
|
// reuse the runtime's provider-ready proof without ever cold-restoring.
|
|
72
82
|
captureVerifiedWorkspaceArchive,
|
|
83
|
+
cancelSelfhostedOp,
|
|
73
84
|
assertConsistentSandboxProviderIdentity,
|
|
74
85
|
createSandboxClientForBackend,
|
|
75
86
|
deleteModalCheckpointSnapshot,
|
|
@@ -78,18 +89,22 @@ import {
|
|
|
78
89
|
inspectModalSandboxLifecycle,
|
|
79
90
|
isExecSessionLostBanner,
|
|
80
91
|
isProviderSandboxNotFoundError,
|
|
92
|
+
NatsControlRpc,
|
|
81
93
|
parseExecBannerExitCode,
|
|
82
94
|
prepareProviderForTeardownAfterCapture,
|
|
83
95
|
providerWorkspaceCapturePolicy,
|
|
84
96
|
resolveModalCheckpointProviderBindingForLiveSandbox,
|
|
85
97
|
resolveModalCheckpointProviderBinding,
|
|
86
98
|
resolveModalCheckpointProviderBindingForSession,
|
|
99
|
+
querySelfhostedOp,
|
|
87
100
|
resumeExactSandboxSession,
|
|
88
101
|
sandboxProviderContinuityForState,
|
|
89
102
|
sandboxBackendForSdkBackendId,
|
|
90
103
|
sandboxProviderInstanceIdFromEnvelope,
|
|
91
104
|
sandboxCommandExitCode,
|
|
92
105
|
sandboxCommandStillRunning,
|
|
106
|
+
SelfhostedControlError,
|
|
107
|
+
subjectFor,
|
|
93
108
|
sweepModalOrphanSandboxes,
|
|
94
109
|
terminateManagedSandboxSession,
|
|
95
110
|
terminateModalSandboxById,
|
|
@@ -98,6 +113,8 @@ import {
|
|
|
98
113
|
type ModalCheckpointProviderBinding,
|
|
99
114
|
type ProviderWorkspaceCapturePolicy,
|
|
100
115
|
type WorkspaceArchiveDescriptor,
|
|
116
|
+
type ControlRpc,
|
|
117
|
+
type NatsRequestConnection,
|
|
101
118
|
} from "@opengeni/runtime/sandbox";
|
|
102
119
|
import {
|
|
103
120
|
SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS,
|
|
@@ -113,6 +130,7 @@ import {
|
|
|
113
130
|
recordCreditBalanceGauges,
|
|
114
131
|
recordCreditMicros,
|
|
115
132
|
recordExpiredDrainingSandboxLeaseGauges,
|
|
133
|
+
recordOpenSandboxKubernetesInventoryGauges,
|
|
116
134
|
recordRetainedProcessInventoryGauges,
|
|
117
135
|
recordRetainedProcessReconciliation,
|
|
118
136
|
recordSandboxCheckpointArtifactGauges,
|
|
@@ -126,8 +144,19 @@ import {
|
|
|
126
144
|
recordSandboxOrphansTerminated,
|
|
127
145
|
recordSandboxRotationBacklogGauges,
|
|
128
146
|
recordTurnsQueuedGauge,
|
|
147
|
+
runtimeMetricsHooksForObservability,
|
|
129
148
|
} from "../observability-metrics";
|
|
130
|
-
import {
|
|
149
|
+
import {
|
|
150
|
+
inspectOpenSandboxKubernetesInventory,
|
|
151
|
+
type OpenSandboxKubernetesInventory,
|
|
152
|
+
} from "../opensandbox-kubernetes-inventory";
|
|
153
|
+
import type { ObjectStorage } from "@opengeni/storage";
|
|
154
|
+
import {
|
|
155
|
+
collectWorkspaceArchiveObjectKeys,
|
|
156
|
+
deleteUnpublishedWorkspaceArchiveObject,
|
|
157
|
+
deleteWorkspaceArchiveObjectKeys,
|
|
158
|
+
putVersion1TarArchiveOrInline,
|
|
159
|
+
} from "../sandbox-archive-storage";
|
|
131
160
|
|
|
132
161
|
export { sandboxLeaseTelemetryKey } from "@opengeni/observability";
|
|
133
162
|
|
|
@@ -218,6 +247,10 @@ export type SweepModalOrphansFn = (
|
|
|
218
247
|
observability: ActivityServices["observability"],
|
|
219
248
|
) => Promise<number>;
|
|
220
249
|
|
|
250
|
+
export type InspectOpenSandboxKubernetesInventoryFn = (input: {
|
|
251
|
+
namespace: string;
|
|
252
|
+
}) => Promise<OpenSandboxKubernetesInventory>;
|
|
253
|
+
|
|
221
254
|
export type SandboxLeaseActivityOptions = {
|
|
222
255
|
/** Override the provider terminate (tests spy this; defaults to the real
|
|
223
256
|
* resume-by-id + provider stop()). */
|
|
@@ -225,6 +258,8 @@ export type SandboxLeaseActivityOptions = {
|
|
|
225
258
|
/** Override the provider-side Modal orphan sweep (tests spy this; defaults to
|
|
226
259
|
* Modal list+tag comparison when Modal is configured). */
|
|
227
260
|
sweepModalOrphans?: SweepModalOrphansFn;
|
|
261
|
+
/** Override the read-only bounded OpenSandbox Kubernetes projection. */
|
|
262
|
+
inspectOpenSandboxKubernetesInventory?: InspectOpenSandboxKubernetesInventoryFn;
|
|
228
263
|
/** Override only the read-only provider process probe. The canonical DB
|
|
229
264
|
* settlement remains real in tests and production. */
|
|
230
265
|
probeRetainedProcess?: RetainedProcessProbeFn;
|
|
@@ -261,6 +296,7 @@ export type RetainedProcessProbeFn = (
|
|
|
261
296
|
settings: ActivityServices["settings"],
|
|
262
297
|
lease: LeaseSnapshot,
|
|
263
298
|
process: SandboxRetainedProcess,
|
|
299
|
+
mode?: "observe" | "cancel",
|
|
264
300
|
) => Promise<RetainedProcessProbeResult>;
|
|
265
301
|
|
|
266
302
|
export type HistoricalModalSandboxLifecycleProbeFn = typeof inspectModalSandboxLifecycle;
|
|
@@ -273,6 +309,8 @@ export type DrainableProviderProbeFn = (
|
|
|
273
309
|
export const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
|
|
274
310
|
export const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
|
|
275
311
|
export const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5_000;
|
|
312
|
+
export const CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;
|
|
313
|
+
export const CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
|
|
276
314
|
|
|
277
315
|
export class SandboxProviderCaptureTimeoutError extends Error {
|
|
278
316
|
readonly name = "SandboxProviderCaptureTimeoutError";
|
|
@@ -437,6 +475,8 @@ export function createSandboxLeaseActivities(
|
|
|
437
475
|
));
|
|
438
476
|
const sweepModalOrphans: SweepModalOrphansFn =
|
|
439
477
|
options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
478
|
+
const inspectOpenSandboxInventory =
|
|
479
|
+
options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;
|
|
440
480
|
const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
|
|
441
481
|
const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
|
|
442
482
|
async function prepareSandboxLeaseSweep(): Promise<SandboxLeaseSweepPlan> {
|
|
@@ -445,8 +485,19 @@ export function createSandboxLeaseActivities(
|
|
|
445
485
|
const { db, settings, observability } = await services();
|
|
446
486
|
const timing = sandboxDrainTiming(settings);
|
|
447
487
|
if (!settings.sandboxOwnershipEnabled) {
|
|
488
|
+
// Turns skip leases when the flag is off, but Computer/Browser attach
|
|
489
|
+
// still acquire them. Leaving drainable rows untouched strands Desktop
|
|
490
|
+
// behind rotation_in_progress forever. Inventory and drain those rows;
|
|
491
|
+
// do not request NEW deadline rotations or meter warm time.
|
|
492
|
+
const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
|
|
493
|
+
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
494
|
+
turnHolderTtlMs: settings.sandboxLeaseTtlMs,
|
|
495
|
+
interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
|
|
496
|
+
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
497
|
+
});
|
|
498
|
+
const drainable = drainableInventory.slice(0, SANDBOX_REAPER_CHILD_DISPATCH_LIMIT);
|
|
448
499
|
return {
|
|
449
|
-
drainable
|
|
500
|
+
drainable,
|
|
450
501
|
timeoutClass: timing.timeoutClass,
|
|
451
502
|
snapshotTimeoutMs: timing.snapshotTimeoutMs,
|
|
452
503
|
captureTimeoutMs: timing.captureTimeoutMs,
|
|
@@ -518,8 +569,7 @@ export function createSandboxLeaseActivities(
|
|
|
518
569
|
instanceId: input.target.instanceId,
|
|
519
570
|
});
|
|
520
571
|
try {
|
|
521
|
-
const { db, settings, observability } = await services();
|
|
522
|
-
if (!settings.sandboxOwnershipEnabled) return { status: "skipped" };
|
|
572
|
+
const { db, settings, observability, objectStorage } = await services();
|
|
523
573
|
assertSandboxDrainInputTiming(input);
|
|
524
574
|
const drainSettings =
|
|
525
575
|
settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs
|
|
@@ -542,6 +592,7 @@ export function createSandboxLeaseActivities(
|
|
|
542
592
|
terminateBox,
|
|
543
593
|
probeDrainableProvider,
|
|
544
594
|
captureAttempt,
|
|
595
|
+
objectStorage,
|
|
545
596
|
);
|
|
546
597
|
return { status: drainedCold ? "terminated" : "skipped" };
|
|
547
598
|
} catch (error) {
|
|
@@ -611,10 +662,21 @@ export function createSandboxLeaseActivities(
|
|
|
611
662
|
let forceDrained = 0;
|
|
612
663
|
const rotationsRequested = input.rotationsRequested;
|
|
613
664
|
let modalOrphansTerminated = 0;
|
|
665
|
+
|
|
666
|
+
// Connected Machine commands are not managed-sandbox lease state. Their
|
|
667
|
+
// immutable launch locators must reconcile even when managed ownership is
|
|
668
|
+
// disabled, otherwise an exact runner exit/loss proof can remain stranded.
|
|
669
|
+
await reconcileConnectedMachineBackgroundCommands(db, settings, observability, service.bus);
|
|
670
|
+
|
|
614
671
|
if (!settings.sandboxOwnershipEnabled) {
|
|
615
672
|
// Inventory remains useful while provider ownership is intentionally
|
|
616
673
|
// disabled, and this scheduled activity is its single projection owner.
|
|
617
|
-
await refreshQueueLeaseAndCreditGauges(
|
|
674
|
+
await refreshQueueLeaseAndCreditGauges(
|
|
675
|
+
db,
|
|
676
|
+
settings,
|
|
677
|
+
observability,
|
|
678
|
+
inspectOpenSandboxInventory,
|
|
679
|
+
);
|
|
618
680
|
return {
|
|
619
681
|
metered,
|
|
620
682
|
forceDrained,
|
|
@@ -683,7 +745,12 @@ export function createSandboxLeaseActivities(
|
|
|
683
745
|
},
|
|
684
746
|
);
|
|
685
747
|
|
|
686
|
-
await refreshQueueLeaseAndCreditGauges(
|
|
748
|
+
await refreshQueueLeaseAndCreditGauges(
|
|
749
|
+
db,
|
|
750
|
+
settings,
|
|
751
|
+
observability,
|
|
752
|
+
inspectOpenSandboxInventory,
|
|
753
|
+
);
|
|
687
754
|
|
|
688
755
|
if (
|
|
689
756
|
input.examined > 0 ||
|
|
@@ -789,6 +856,30 @@ const CHECKPOINT_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 60_000;
|
|
|
789
856
|
const CHECKPOINT_TOMBSTONE_PRUNE_LIMIT = 500;
|
|
790
857
|
const SANDBOX_MAINTENANCE_ITEM_CONCURRENCY = 8;
|
|
791
858
|
|
|
859
|
+
type ConnectedCommandReconciliationOutcome =
|
|
860
|
+
| "claim_failed"
|
|
861
|
+
| "provider_running"
|
|
862
|
+
| "provider_offline"
|
|
863
|
+
| "provider_error"
|
|
864
|
+
| "proof_exited"
|
|
865
|
+
| "proof_lost"
|
|
866
|
+
| "proof_checkpoint_failed"
|
|
867
|
+
| "settled_exited"
|
|
868
|
+
| "settled_lost"
|
|
869
|
+
| "settlement_failed"
|
|
870
|
+
| "defer_failed";
|
|
871
|
+
|
|
872
|
+
function recordConnectedCommandReconciliation(
|
|
873
|
+
observability: ActivityServices["observability"],
|
|
874
|
+
outcome: ConnectedCommandReconciliationOutcome,
|
|
875
|
+
): void {
|
|
876
|
+
observability.incrementCounter({
|
|
877
|
+
name: "opengeni_connected_background_command_reconciliation_total",
|
|
878
|
+
help: "Exact-locator Connected Machine background-command reconciliation outcomes.",
|
|
879
|
+
labels: { outcome },
|
|
880
|
+
});
|
|
881
|
+
}
|
|
882
|
+
|
|
792
883
|
async function forEachWithConcurrency<T>(
|
|
793
884
|
values: readonly T[],
|
|
794
885
|
concurrency: number,
|
|
@@ -934,6 +1025,199 @@ async function adoptLegacyModalCheckpointReceipts(
|
|
|
934
1025
|
return adopted;
|
|
935
1026
|
}
|
|
936
1027
|
|
|
1028
|
+
/** Reconcile commands only against the immutable physical locator copied at
|
|
1029
|
+
* adoption. A successor enrollment connection is intentionally irrelevant: no
|
|
1030
|
+
* query or cancellation is ever rebound to it. */
|
|
1031
|
+
async function reconcileConnectedMachineBackgroundCommands(
|
|
1032
|
+
db: ActivityServices["db"],
|
|
1033
|
+
settings: ActivityServices["settings"],
|
|
1034
|
+
observability: ActivityServices["observability"],
|
|
1035
|
+
bus: ActivityServices["bus"],
|
|
1036
|
+
): Promise<void> {
|
|
1037
|
+
const claimId = crypto.randomUUID();
|
|
1038
|
+
let claims: ConnectedMachineBackgroundCommandClaim[];
|
|
1039
|
+
try {
|
|
1040
|
+
claims = await claimConnectedMachineSessionBackgroundCommands(db, {
|
|
1041
|
+
claimId,
|
|
1042
|
+
limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
|
|
1043
|
+
claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS,
|
|
1044
|
+
});
|
|
1045
|
+
} catch (error) {
|
|
1046
|
+
recordConnectedCommandReconciliation(observability, "claim_failed");
|
|
1047
|
+
observability.warn("sandbox reaper: connected-command claim failed", {
|
|
1048
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1049
|
+
});
|
|
1050
|
+
return;
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
const controlRpc = new NatsControlRpc(
|
|
1054
|
+
async (): Promise<NatsRequestConnection | null> => bus.getRequestConnection(),
|
|
1055
|
+
);
|
|
1056
|
+
await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
|
|
1057
|
+
let proof = claim.proof;
|
|
1058
|
+
if (!proof) {
|
|
1059
|
+
try {
|
|
1060
|
+
proof = await probeConnectedMachineBackgroundCommand(
|
|
1061
|
+
claim,
|
|
1062
|
+
controlRpc,
|
|
1063
|
+
Math.min(
|
|
1064
|
+
settings.sandboxSelfhostedControlTimeoutMs,
|
|
1065
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
|
|
1066
|
+
),
|
|
1067
|
+
);
|
|
1068
|
+
if (!proof) {
|
|
1069
|
+
await deferConnectedCommandClaim(db, settings, observability, claim, "provider_running");
|
|
1070
|
+
return;
|
|
1071
|
+
}
|
|
1072
|
+
} catch (error) {
|
|
1073
|
+
const offline =
|
|
1074
|
+
error instanceof SelfhostedControlError &&
|
|
1075
|
+
(error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
|
|
1076
|
+
if (!offline) {
|
|
1077
|
+
observability.warn("sandbox reaper: connected-command provider probe failed", {
|
|
1078
|
+
commandId: claim.commandId,
|
|
1079
|
+
enrollmentId: claim.enrollmentId,
|
|
1080
|
+
connectionInstanceId: claim.connectionInstanceId,
|
|
1081
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1082
|
+
});
|
|
1083
|
+
}
|
|
1084
|
+
await deferConnectedCommandClaim(
|
|
1085
|
+
db,
|
|
1086
|
+
settings,
|
|
1087
|
+
observability,
|
|
1088
|
+
claim,
|
|
1089
|
+
offline ? "provider_offline" : "provider_error",
|
|
1090
|
+
);
|
|
1091
|
+
return;
|
|
1092
|
+
}
|
|
1093
|
+
|
|
1094
|
+
try {
|
|
1095
|
+
await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
|
|
1096
|
+
recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
|
|
1097
|
+
} catch (error) {
|
|
1098
|
+
recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
|
|
1099
|
+
observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
|
|
1100
|
+
commandId: claim.commandId,
|
|
1101
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1102
|
+
});
|
|
1103
|
+
return;
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
try {
|
|
1108
|
+
if (!(await settleClaimedConnectedMachineBackgroundCommand(db, { claim }))) {
|
|
1109
|
+
throw new Error("Connected command settlement lost its exact claim or proof");
|
|
1110
|
+
}
|
|
1111
|
+
recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
|
|
1112
|
+
} catch (error) {
|
|
1113
|
+
recordConnectedCommandReconciliation(observability, "settlement_failed");
|
|
1114
|
+
observability.warn("sandbox reaper: connected-command settlement failed", {
|
|
1115
|
+
commandId: claim.commandId,
|
|
1116
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1117
|
+
});
|
|
1118
|
+
await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
|
|
1119
|
+
}
|
|
1120
|
+
});
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
export async function probeConnectedMachineBackgroundCommand(
|
|
1124
|
+
claim: ConnectedMachineBackgroundCommandClaim,
|
|
1125
|
+
controlRpc: ControlRpc,
|
|
1126
|
+
controlTimeoutMs: number,
|
|
1127
|
+
observedAt = new Date(),
|
|
1128
|
+
): Promise<ConnectedMachineBackgroundCommandProof | null> {
|
|
1129
|
+
const input = {
|
|
1130
|
+
controlRpc,
|
|
1131
|
+
rpcSubject: subjectFor(
|
|
1132
|
+
claim.controlWorkspaceId,
|
|
1133
|
+
claim.enrollmentId,
|
|
1134
|
+
claim.connectionInstanceId,
|
|
1135
|
+
),
|
|
1136
|
+
opId: claim.opId,
|
|
1137
|
+
controlTimeoutMs,
|
|
1138
|
+
};
|
|
1139
|
+
const status =
|
|
1140
|
+
claim.state === "stopping" ? await cancelSelfhostedOp(input) : await querySelfhostedOp(input);
|
|
1141
|
+
return connectedCommandProofFromStatus(status, observedAt);
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
export function connectedCommandProofFromStatus(
|
|
1145
|
+
status: OpStatus,
|
|
1146
|
+
observedAt: Date,
|
|
1147
|
+
): ConnectedMachineBackgroundCommandProof | null {
|
|
1148
|
+
switch (status.state) {
|
|
1149
|
+
case OpState.OP_STATE_ACCEPTED:
|
|
1150
|
+
case OpState.OP_STATE_RUNNING:
|
|
1151
|
+
return null;
|
|
1152
|
+
case OpState.OP_STATE_COMPLETE: {
|
|
1153
|
+
const exit = status.exit;
|
|
1154
|
+
if (!exit) {
|
|
1155
|
+
throw new Error("Connected command completed without an exit record");
|
|
1156
|
+
}
|
|
1157
|
+
const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128);
|
|
1158
|
+
const reason = exit.cancelled
|
|
1159
|
+
? "op_cancelled"
|
|
1160
|
+
: exit.timedOut
|
|
1161
|
+
? "op_timed_out"
|
|
1162
|
+
: failureCode
|
|
1163
|
+
? `op_failure_${failureCode}`
|
|
1164
|
+
: "op_exit";
|
|
1165
|
+
return {
|
|
1166
|
+
outcome: "exited",
|
|
1167
|
+
exitCode: exit.exitCode,
|
|
1168
|
+
reason,
|
|
1169
|
+
observedAt,
|
|
1170
|
+
};
|
|
1171
|
+
}
|
|
1172
|
+
case OpState.OP_STATE_LOST:
|
|
1173
|
+
return {
|
|
1174
|
+
outcome: "lost",
|
|
1175
|
+
exitCode: null,
|
|
1176
|
+
reason:
|
|
1177
|
+
status.lostReason === OpLostReason.OP_LOST_REASON_EVICTED
|
|
1178
|
+
? "op_lost_evicted"
|
|
1179
|
+
: status.lostReason === OpLostReason.OP_LOST_REASON_AGENT_RESTARTED
|
|
1180
|
+
? "op_lost_agent_restarted"
|
|
1181
|
+
: "op_lost_unspecified",
|
|
1182
|
+
observedAt,
|
|
1183
|
+
};
|
|
1184
|
+
default:
|
|
1185
|
+
throw new Error(`Connected command returned invalid op state ${status.state}`);
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
|
|
1189
|
+
async function deferConnectedCommandClaim(
|
|
1190
|
+
db: ActivityServices["db"],
|
|
1191
|
+
settings: ActivityServices["settings"],
|
|
1192
|
+
observability: ActivityServices["observability"],
|
|
1193
|
+
claim: ConnectedMachineBackgroundCommandClaim,
|
|
1194
|
+
outcome: "provider_running" | "provider_offline" | "provider_error" | "settlement_failed",
|
|
1195
|
+
): Promise<void> {
|
|
1196
|
+
const exponent = Math.min(4, Math.max(0, claim.reconcileAttempts - 1));
|
|
1197
|
+
const retryAfterMs = Math.min(
|
|
1198
|
+
5 * 60_000,
|
|
1199
|
+
Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) * 2 ** exponent,
|
|
1200
|
+
);
|
|
1201
|
+
try {
|
|
1202
|
+
if (
|
|
1203
|
+
!(await deferConnectedMachineBackgroundCommandReconciliation(db, {
|
|
1204
|
+
claim,
|
|
1205
|
+
outcome,
|
|
1206
|
+
retryAfterMs,
|
|
1207
|
+
}))
|
|
1208
|
+
) {
|
|
1209
|
+
throw new Error("Connected command defer lost its exact claim");
|
|
1210
|
+
}
|
|
1211
|
+
recordConnectedCommandReconciliation(observability, outcome);
|
|
1212
|
+
} catch (error) {
|
|
1213
|
+
recordConnectedCommandReconciliation(observability, "defer_failed");
|
|
1214
|
+
observability.warn("sandbox reaper: connected-command defer failed", {
|
|
1215
|
+
commandId: claim.commandId,
|
|
1216
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1217
|
+
});
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
|
|
937
1221
|
async function reconcileTerminalRetainedProcesses(
|
|
938
1222
|
db: ActivityServices["db"],
|
|
939
1223
|
settings: ActivityServices["settings"],
|
|
@@ -1070,7 +1354,12 @@ async function reconcileTerminalRetainedProcesses(
|
|
|
1070
1354
|
}
|
|
1071
1355
|
} else {
|
|
1072
1356
|
try {
|
|
1073
|
-
observation = await probe(
|
|
1357
|
+
observation = await probe(
|
|
1358
|
+
settings,
|
|
1359
|
+
lease!,
|
|
1360
|
+
process,
|
|
1361
|
+
claim.ownerState === "background_stopping" ? "cancel" : "observe",
|
|
1362
|
+
);
|
|
1074
1363
|
} catch (error) {
|
|
1075
1364
|
observability.warn("sandbox reaper: retained-process provider probe failed", {
|
|
1076
1365
|
processId: process.id,
|
|
@@ -1150,6 +1439,15 @@ async function reconcileTerminalRetainedProcesses(
|
|
|
1150
1439
|
reason: proof.reason,
|
|
1151
1440
|
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
1152
1441
|
});
|
|
1442
|
+
await settleSessionBackgroundCommandForRetainedProcess(db, {
|
|
1443
|
+
accountId: process.accountId,
|
|
1444
|
+
workspaceId: process.workspaceId,
|
|
1445
|
+
sessionId: process.sessionId,
|
|
1446
|
+
retainedProcessId: process.id,
|
|
1447
|
+
outcome: proof.outcome,
|
|
1448
|
+
exitCode: proof.exitCode,
|
|
1449
|
+
reason: proof.reason,
|
|
1450
|
+
});
|
|
1153
1451
|
recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
|
|
1154
1452
|
} catch (error) {
|
|
1155
1453
|
recordRetainedProcessReconciliation(observability, "settlement_failed");
|
|
@@ -1244,6 +1542,7 @@ export async function probeRetainedProcessAtProvider(
|
|
|
1244
1542
|
settings: ActivityServices["settings"],
|
|
1245
1543
|
lease: LeaseSnapshot,
|
|
1246
1544
|
process: SandboxRetainedProcess,
|
|
1545
|
+
mode: "observe" | "cancel" = "observe",
|
|
1247
1546
|
): Promise<RetainedProcessProbeResult> {
|
|
1248
1547
|
if (
|
|
1249
1548
|
lease.id !== process.leaseId ||
|
|
@@ -1263,7 +1562,7 @@ export async function probeRetainedProcessAtProvider(
|
|
|
1263
1562
|
if (envelopeBackend !== undefined && envelopeBackend !== process.providerBackend) {
|
|
1264
1563
|
return { status: "deferred", reason: "identity_mismatch" };
|
|
1265
1564
|
}
|
|
1266
|
-
if (
|
|
1565
|
+
if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== process.providerInstanceId) {
|
|
1267
1566
|
return { status: "deferred", reason: "identity_mismatch" };
|
|
1268
1567
|
}
|
|
1269
1568
|
|
|
@@ -1378,7 +1677,7 @@ export async function probeRetainedProcessAtProvider(
|
|
|
1378
1677
|
result = await withRetainedProcessProbeTimeout(
|
|
1379
1678
|
session.writeStdin({
|
|
1380
1679
|
sessionId: process.providerSessionId,
|
|
1381
|
-
chars: "",
|
|
1680
|
+
chars: mode === "cancel" ? "\u0003" : "",
|
|
1382
1681
|
yieldTimeMs: 1_000,
|
|
1383
1682
|
maxOutputTokens: 2_000,
|
|
1384
1683
|
}),
|
|
@@ -1533,7 +1832,9 @@ async function accrueWarmTick(
|
|
|
1533
1832
|
|
|
1534
1833
|
async function refreshQueueLeaseAndCreditGauges(
|
|
1535
1834
|
db: ActivityServices["db"],
|
|
1835
|
+
settings: ActivityServices["settings"],
|
|
1536
1836
|
observability: ActivityServices["observability"],
|
|
1837
|
+
inspectOpenSandboxInventory: InspectOpenSandboxKubernetesInventoryFn,
|
|
1537
1838
|
): Promise<void> {
|
|
1538
1839
|
await Promise.all([
|
|
1539
1840
|
(async () => {
|
|
@@ -1584,6 +1885,24 @@ async function refreshQueueLeaseAndCreditGauges(
|
|
|
1584
1885
|
await countExpiredDrainingSandboxLeases(db),
|
|
1585
1886
|
);
|
|
1586
1887
|
}),
|
|
1888
|
+
...(settings.sandboxBackend === "opensandbox" &&
|
|
1889
|
+
settings.openSandboxKubernetesInventoryNamespace
|
|
1890
|
+
? [
|
|
1891
|
+
refreshSandboxInventoryGauge(
|
|
1892
|
+
observability,
|
|
1893
|
+
"opensandbox_kubernetes",
|
|
1894
|
+
"OpenSandbox Kubernetes",
|
|
1895
|
+
async () => {
|
|
1896
|
+
recordOpenSandboxKubernetesInventoryGauges(
|
|
1897
|
+
observability,
|
|
1898
|
+
await inspectOpenSandboxInventory({
|
|
1899
|
+
namespace: settings.openSandboxKubernetesInventoryNamespace!,
|
|
1900
|
+
}),
|
|
1901
|
+
);
|
|
1902
|
+
},
|
|
1903
|
+
),
|
|
1904
|
+
]
|
|
1905
|
+
: []),
|
|
1587
1906
|
(async () => {
|
|
1588
1907
|
try {
|
|
1589
1908
|
recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
|
|
@@ -1813,6 +2132,24 @@ async function probeDrainableProviderReadiness(
|
|
|
1813
2132
|
}
|
|
1814
2133
|
}
|
|
1815
2134
|
|
|
2135
|
+
export type SandboxDrainReason =
|
|
2136
|
+
| "idle"
|
|
2137
|
+
| "provider_deadline"
|
|
2138
|
+
| "operator"
|
|
2139
|
+
| "teardown_claim"
|
|
2140
|
+
| "rotation_requested";
|
|
2141
|
+
|
|
2142
|
+
/** The bounded drain reason carried on `sandbox.box.terminated`: the lease's
|
|
2143
|
+
* rotation reason when a rotation was requested, otherwise the ordinary idle
|
|
2144
|
+
* drain. A requested rotation without a recorded reason (legacy rows) is
|
|
2145
|
+
* reported as `rotation_requested` rather than guessed. */
|
|
2146
|
+
export function sandboxDrainReason(
|
|
2147
|
+
lease: Pick<LeaseSnapshot, "rotationRequestedAt" | "rotationReason">,
|
|
2148
|
+
): SandboxDrainReason {
|
|
2149
|
+
if (lease.rotationRequestedAt === null) return "idle";
|
|
2150
|
+
return lease.rotationReason ?? "rotation_requested";
|
|
2151
|
+
}
|
|
2152
|
+
|
|
1816
2153
|
/**
|
|
1817
2154
|
* Terminate one drainable box by id, then CAS its lease draining->cold under the
|
|
1818
2155
|
* epoch fence. Returns true when the lease went cold (the box is ours to stop
|
|
@@ -1837,6 +2174,7 @@ async function terminateDrainableBox(
|
|
|
1837
2174
|
terminateBox: TerminateBoxFn,
|
|
1838
2175
|
probeDrainableProvider: DrainableProviderProbeFn,
|
|
1839
2176
|
attempt: SandboxDrainCaptureAttempt,
|
|
2177
|
+
objectStorage: ObjectStorage | null,
|
|
1840
2178
|
): Promise<boolean> {
|
|
1841
2179
|
// Resolve the account for the RLS-scoped confirmDrainCold (the global sweep
|
|
1842
2180
|
// returns no account_id; the workspace->account map is the bootstrap read).
|
|
@@ -2092,6 +2430,7 @@ async function terminateDrainableBox(
|
|
|
2092
2430
|
if (!lease.instanceId || !captureClaim) {
|
|
2093
2431
|
return { wrote: false, archiveRevision: null };
|
|
2094
2432
|
}
|
|
2433
|
+
const archiveMetrics = runtimeMetricsHooksForObservability(observability);
|
|
2095
2434
|
let checkpointArtifactId: string | null = null;
|
|
2096
2435
|
if (
|
|
2097
2436
|
archiveBase64 &&
|
|
@@ -2138,12 +2477,59 @@ async function terminateDrainableBox(
|
|
|
2138
2477
|
if (!archiveMetadata) {
|
|
2139
2478
|
throw new Error("Sandbox snapshot publication requires a verified archive descriptor");
|
|
2140
2479
|
}
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2480
|
+
const priorKeys = collectWorkspaceArchiveObjectKeys(
|
|
2481
|
+
(lease.resumeState as Record<string, unknown> | null | undefined) ?? null,
|
|
2482
|
+
);
|
|
2483
|
+
let workspaceArchiveRef:
|
|
2484
|
+
| Awaited<ReturnType<typeof putVersion1TarArchiveOrInline>>["workspaceArchiveRef"]
|
|
2485
|
+
| undefined;
|
|
2486
|
+
try {
|
|
2487
|
+
const published = await putVersion1TarArchiveOrInline({
|
|
2488
|
+
backend: lease.backend,
|
|
2489
|
+
objectStorage,
|
|
2490
|
+
accountId,
|
|
2491
|
+
workspaceId: row.workspaceId,
|
|
2492
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
2493
|
+
archive: {
|
|
2494
|
+
bytes: Buffer.from(archiveBase64, "base64"),
|
|
2495
|
+
descriptor: archiveMetadata,
|
|
2496
|
+
base64: archiveBase64,
|
|
2497
|
+
},
|
|
2498
|
+
metrics: archiveMetrics,
|
|
2499
|
+
});
|
|
2500
|
+
workspaceArchiveRef = published.workspaceArchiveRef;
|
|
2501
|
+
result = await persistDrainSnapshot(db, {
|
|
2502
|
+
...baseInput,
|
|
2503
|
+
workspaceArchiveMeta: archiveMetadata,
|
|
2504
|
+
...published,
|
|
2505
|
+
...(checkpointArtifactId ? { checkpointArtifactId } : {}),
|
|
2506
|
+
});
|
|
2507
|
+
if (published.workspaceArchiveRef && objectStorage) {
|
|
2508
|
+
if (!result.wrote) {
|
|
2509
|
+
await deleteUnpublishedWorkspaceArchiveObject(
|
|
2510
|
+
objectStorage,
|
|
2511
|
+
published.workspaceArchiveRef,
|
|
2512
|
+
archiveMetrics,
|
|
2513
|
+
);
|
|
2514
|
+
} else {
|
|
2515
|
+
const afterLease = await readLease(db, row.workspaceId, row.sandboxGroupId);
|
|
2516
|
+
const afterKeys = collectWorkspaceArchiveObjectKeys(
|
|
2517
|
+
(afterLease?.resumeState as Record<string, unknown> | null | undefined) ?? null,
|
|
2518
|
+
);
|
|
2519
|
+
await deleteWorkspaceArchiveObjectKeys(
|
|
2520
|
+
objectStorage,
|
|
2521
|
+
[...priorKeys].filter((key) => !afterKeys.has(key)),
|
|
2522
|
+
).catch(() => undefined);
|
|
2523
|
+
}
|
|
2524
|
+
}
|
|
2525
|
+
} catch (error) {
|
|
2526
|
+
await deleteUnpublishedWorkspaceArchiveObject(
|
|
2527
|
+
objectStorage,
|
|
2528
|
+
workspaceArchiveRef,
|
|
2529
|
+
archiveMetrics,
|
|
2530
|
+
);
|
|
2531
|
+
throw error;
|
|
2532
|
+
}
|
|
2147
2533
|
}
|
|
2148
2534
|
if (!result.wrote && checkpointArtifactId) {
|
|
2149
2535
|
await markSandboxCheckpointArtifactDeletePending(db, {
|
|
@@ -2214,6 +2600,11 @@ async function terminateDrainableBox(
|
|
|
2214
2600
|
archiveRevision,
|
|
2215
2601
|
instanceId: lease.instanceId,
|
|
2216
2602
|
providerMissingBeforeCapture: providerMissing,
|
|
2603
|
+
// Why the box drained: the lease's bounded rotation reason when one
|
|
2604
|
+
// was requested (provider deadline, operator/image conflict, teardown
|
|
2605
|
+
// claim), otherwise the ordinary idle drain. Additive diagnostic
|
|
2606
|
+
// fact; no identifiers beyond the ones already carried above.
|
|
2607
|
+
drainReason: sandboxDrainReason(lease),
|
|
2217
2608
|
},
|
|
2218
2609
|
});
|
|
2219
2610
|
} catch (eventError) {
|