@opengeni/worker-bundle 0.28.1-canary.0 → 0.28.3-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -19,7 +19,7 @@ import {
19
19
  startContextCompactionPendingMonitor,
20
20
  startSessionRecoveryMonitor,
21
21
  startTurnCapacityMonitor
22
- } from "./chunk-BARTLVBK.js";
22
+ } from "./chunk-A4XQWVZK.js";
23
23
 
24
24
  // src/index.ts
25
25
  import {
@@ -781,7 +781,7 @@ function temporalWorkflowExecutionNotFound(error) {
781
781
  return isGrpcServiceError(error) && error.code === 5;
782
782
  }
783
783
  async function createDefaultWorkerActivities(role, dependencies = {}) {
784
- return role === "control" ? (await import("./activities-control-VVK7RA7F.js")).createControlActivities(dependencies) : (await import("./activities-turn-FOIZYY56.js")).createTurnActivities(dependencies);
784
+ return role === "control" ? (await import("./activities-control-JUBMDBDZ.js")).createControlActivities(dependencies) : (await import("./activities-turn-I2ZKGZE6.js")).createTurnActivities(dependencies);
785
785
  }
786
786
  function resolveOpenGeniWorkflowDefinition(moduleUrl = import.meta.url) {
787
787
  const modulePath = fileURLToPath(moduleUrl).replaceAll("\\", "/");
@@ -91,15 +91,23 @@ export declare function recordWorkerDeathRecoveryMetrics(observability: Observab
91
91
  export declare class TurnLifecycleMetrics {
92
92
  private readonly observability;
93
93
  private readonly options;
94
- private readonly turns;
94
+ private readonly attempts;
95
95
  private timer;
96
96
  constructor(observability: Observability, options?: {
97
97
  now?: () => number;
98
98
  refreshIntervalMs?: number;
99
99
  });
100
- start(turnId: string): void;
101
- progress(turnId: string): void;
102
- finish(turnId: string, outcome: TurnOutcome | null, durationSeconds?: number): void;
100
+ start(input: {
101
+ attemptId: string;
102
+ }): void;
103
+ progress(input: {
104
+ attemptId: string;
105
+ }): void;
106
+ finish(input: {
107
+ attemptId: string;
108
+ outcome: TurnOutcome | null;
109
+ durationSeconds?: number;
110
+ }): void;
103
111
  refreshGauges(): void;
104
112
  stop(): void;
105
113
  private oldestInflightAgeSeconds;
@@ -1,4 +1,5 @@
1
- import type { ObjectStorage } from "@opengeni/storage";
1
+ import { type ObjectStorage } from "@opengeni/storage";
2
+ import type { VerifiedHostWorkspaceArchive } from "@opengeni/runtime/sandbox";
2
3
  import { type WorkspaceArchiveDescriptor, type WorkspaceArchiveObjectRef } from "@opengeni/contracts";
3
4
  export declare class WorkspaceArchiveObjectStorageRequiredError extends Error {
4
5
  readonly code: "workspace_archive_object_storage_required";
@@ -13,15 +14,26 @@ export declare function putTarWorkspaceArchiveObject(input: {
13
14
  archive: {
14
15
  bytes: Uint8Array;
15
16
  descriptor: WorkspaceArchiveDescriptor;
16
- };
17
+ } | VerifiedHostWorkspaceArchive;
17
18
  }): Promise<WorkspaceArchiveObjectRef>;
19
+ /** The caller owns a fresh candidate returned by this module. A thrown or lost
20
+ * database response is UNKNOWN: it may have committed, so never delete on catch.
21
+ * Only the transaction's exact candidate disposition can authorize cleanup. */
22
+ export declare function persistWorkspaceArchiveCandidate<T extends {
23
+ wrote: boolean;
24
+ candidateDisposition?: "adopted" | "already_referenced" | "unused";
25
+ }>(input: {
26
+ objectStorage?: ObjectStorage | null;
27
+ ref?: WorkspaceArchiveObjectRef;
28
+ persist: () => Promise<T>;
29
+ }): Promise<T>;
18
30
  export declare function putVersion1TarArchiveOrInline(input: {
19
31
  backend: string;
20
32
  objectStorage?: ObjectStorage | null | undefined;
21
33
  accountId: string;
22
34
  workspaceId: string;
23
35
  sandboxGroupId: string;
24
- archive: {
36
+ archive: VerifiedHostWorkspaceArchive | {
25
37
  bytes: Uint8Array;
26
38
  descriptor: WorkspaceArchiveDescriptor;
27
39
  base64: string;
@@ -1,7 +1,7 @@
1
1
  import { type Settings } from "@opengeni/config";
2
2
  import { readLease, reinstateTurnLeaseHolder, SandboxLeaseSupersededError, type Database, type LeaseHolderKind } from "@opengeni/db";
3
3
  import { establishSandboxSessionFromEnvelope, type EstablishedSandboxSession, type RuntimeMetricsHooks } from "@opengeni/runtime";
4
- import type { ObjectStorage } from "@opengeni/storage";
4
+ import { type ObjectStorage } from "@opengeni/storage";
5
5
  import type { Observability } from "@opengeni/observability";
6
6
  export type ResumeHolderKind = LeaseHolderKind;
7
7
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@opengeni/worker-bundle",
3
- "version": "0.28.1-canary.0",
3
+ "version": "0.28.3-canary.0",
4
4
  "description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -54,22 +54,22 @@
54
54
  },
55
55
  "dependencies": {
56
56
  "@llamaindex/liteparse": "2.14.2",
57
- "@opengeni/agent-proto": "^0.5.1-canary.24",
58
- "@opengeni/capabilities": "^0.3.3-canary.6",
59
- "@opengeni/codemode": "^0.5.4-canary.0",
60
- "@opengeni/codex": "^0.2.22-canary.6",
61
- "@opengeni/config": "^1.1.0-canary.0",
62
- "@opengeni/contracts": "^3.0.0-canary.0",
63
- "@opengeni/core": "^2.9.1-canary.0",
64
- "@opengeni/db": "^4.3.0-canary.0",
65
- "@opengeni/documents": "^0.8.24-canary.0",
66
- "@opengeni/events": "^0.4.22-canary.0",
67
- "@opengeni/github": "^0.7.8-canary.0",
68
- "@opengeni/network": "^0.3.1-canary.6",
69
- "@opengeni/observability": "^0.8.24-canary.0",
70
- "@opengeni/runtime": "^2.5.1-canary.0",
71
- "@opengeni/storage": "^0.2.125-canary.0",
72
- "@opengeni/xai-subscription": "^0.1.4-canary.6",
57
+ "@opengeni/agent-proto": "^0.5.1-canary.25",
58
+ "@opengeni/capabilities": "^0.3.3-canary.7",
59
+ "@opengeni/codemode": "^0.5.5-canary.0",
60
+ "@opengeni/codex": "^0.2.22-canary.7",
61
+ "@opengeni/config": "^1.1.1-canary.0",
62
+ "@opengeni/contracts": "^3.0.1-canary.0",
63
+ "@opengeni/core": "^2.9.3-canary.0",
64
+ "@opengeni/db": "^4.3.2-canary.0",
65
+ "@opengeni/documents": "^0.8.26-canary.0",
66
+ "@opengeni/events": "^0.4.24-canary.0",
67
+ "@opengeni/github": "^0.7.9-canary.0",
68
+ "@opengeni/network": "^0.3.1-canary.7",
69
+ "@opengeni/observability": "^0.8.25-canary.0",
70
+ "@opengeni/runtime": "^2.5.2-canary.0",
71
+ "@opengeni/storage": "^0.2.126-canary.0",
72
+ "@opengeni/xai-subscription": "^0.1.4-canary.7",
73
73
  "@temporalio/activity": "^1.17.0",
74
74
  "@temporalio/client": "^1.17.0",
75
75
  "@temporalio/worker": "^1.17.0",
@@ -347,7 +347,7 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
347
347
  });
348
348
  return attached.action === "claimed" && attached.turn.id === turn.id;
349
349
  };
350
- turnLifecycleMetricsFor(observability).start(attempt.turnId);
350
+ turnLifecycleMetricsFor(observability).start({ attemptId: input.attemptId });
351
351
  // §7.5 P3 — pass the accepted billing attribution (externally funded turns
352
352
  // bypass OpenGeni credit/token gates)
353
353
  // AND the optional host `entitlements` port (when bound, its admitRun replaces
@@ -451,7 +451,7 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
451
451
  }
452
452
  recordCanonicalStartupMilestones(appended.canonicalStartupMilestones);
453
453
  if (inputs.length > 0) {
454
- turnLifecycleMetricsFor(observability).progress(attempt.turnId!);
454
+ turnLifecycleMetricsFor(observability).progress({ attemptId: input.attemptId });
455
455
  }
456
456
  activityContext?.heartbeat({
457
457
  ...heartbeatDetails,
@@ -515,7 +515,7 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
515
515
  return false;
516
516
  }
517
517
  recordCanonicalStartupMilestones(result.canonicalStartupMilestones);
518
- turnLifecycleMetricsFor(observability).progress(attempt.turnId!);
518
+ turnLifecycleMetricsFor(observability).progress({ attemptId: input.attemptId });
519
519
  await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, result.events);
520
520
  if (inputSettlement.turnStatus === "requires_action") {
521
521
  // The settlement transaction committed the parent's child_requires_action
@@ -745,12 +745,17 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
745
745
  status: finalizationError ? "cleanup_failed" : control.activityStatus,
746
746
  durationSeconds,
747
747
  });
748
- if (attempt.turnId && control.activityStatus !== "unknown") {
749
- turnLifecycleMetricsFor(observability).finish(
750
- attempt.turnId,
751
- control.turnMetricOutcome,
748
+ if (attempt.turnId) {
749
+ // This registry describes physical activity ownership, not durable turn
750
+ // settlement. Always clear the exact attempt when its activity unwinds,
751
+ // even when failure settlement could not classify an outcome. Otherwise
752
+ // a recovered successor can make progress while the predecessor leaves a
753
+ // process-local false stuck-turn alert behind.
754
+ turnLifecycleMetricsFor(observability).finish({
755
+ attemptId: input.attemptId,
756
+ outcome: control.turnMetricOutcome,
752
757
  durationSeconds,
753
- );
758
+ });
754
759
  }
755
760
  activitySpan.end({
756
761
  attributes: {
@@ -129,8 +129,17 @@ export const createSessionTitleModelUsageEventState = createCompactionModelUsage
129
129
 
130
130
  export function modelResponseContextSignal(
131
131
  state: ModelResponseEventState,
132
+ responseCountBeforeStream = 0,
132
133
  ): { revision: number; totalTokens: number } | null {
133
- return state.contextSignal;
134
+ const signal = state.contextSignal;
135
+ // A compaction retry creates a new SDK request counter, but usage identities
136
+ // remain activity-wide. Never bind a pre-stream report to a reused request
137
+ // ordinal; translate only this stream's reports without mutating usage state.
138
+ if (!signal || signal.revision <= responseCountBeforeStream) return null;
139
+ return {
140
+ revision: signal.revision - responseCountBeforeStream,
141
+ totalTokens: signal.totalTokens,
142
+ };
134
143
  }
135
144
 
136
145
  export function assertModelResponseLatencyMode(input: {
@@ -865,7 +865,8 @@ export async function runTurnStreamAttempt(
865
865
  },
866
866
  }
867
867
  : {}),
868
- contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
868
+ contextCompactionSignal: () =>
869
+ modelResponseContextSignal(modelResponseState, responseCountBeforeStream),
869
870
  contextCompactionRequested: () =>
870
871
  isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
871
872
  onModelPreparationPhase: (measurement) => {
@@ -1061,7 +1062,7 @@ export async function runTurnStreamAttempt(
1061
1062
  currentToolBatchCallIds = new Set<string>();
1062
1063
  currentToolBatchCompletedCallIds = new Set<string>();
1063
1064
  await historySink.reconcileConversationTruth();
1064
- turnLifecycleMetricsFor(observability).progress(attempt.turnId!);
1065
+ turnLifecycleMetricsFor(observability).progress({ attemptId: input.attemptId });
1065
1066
  modelCheckpointMemoryCollector.schedule(observability);
1066
1067
  try {
1067
1068
  await ensureRunAllowed(
@@ -286,6 +286,8 @@ export function goalContinuationPrompt(
286
286
  "Continuation behavior:",
287
287
  "- This goal persists across turns. A runtime boundary can end one turn without shrinking the objective; the next continuation resumes the same full objective.",
288
288
  "- Keep working until the requested end state is true and verified. Do not end the turn merely because one useful action completed, and do not redefine success around a smaller or easier task.",
289
+ "- An incomplete-status report is not a substitute for continuing the work. If the remaining problem can be investigated or addressed within your current authority, continue that work in this turn rather than returning another equivalent status-only final.",
290
+ "- Distinguish unfinished work from a blocker that actually requires human input or external change; use the existing waiting and blocked audits only when their conditions hold.",
289
291
  "- Temporary rough edges are acceptable while the work is moving in the right direction. Completion still requires the requested end state to be true and verified.",
290
292
  "",
291
293
  "Work from evidence:",
@@ -82,7 +82,10 @@ import {
82
82
  // box is typed as provider loss. Stale capture reconciliation separately uses
83
83
  // establishSandboxSessionFromEnvelope in strict resume-only mode so it can
84
84
  // reuse the runtime's provider-ready proof without ever cold-restoring.
85
- captureVerifiedWorkspaceArchive,
85
+ captureWorkspaceArchiveForStorage,
86
+ disposeWorkspaceArchive,
87
+ type VerifiedWorkspaceArchivePayload,
88
+ type VerifiedHostWorkspaceArchive,
86
89
  cancelSelfhostedOp,
87
90
  assertConsistentSandboxProviderIdentity,
88
91
  createSandboxClientForBackend,
@@ -160,9 +163,7 @@ import {
160
163
  } from "../opensandbox-kubernetes-inventory";
161
164
  import type { ObjectStorage } from "@opengeni/storage";
162
165
  import {
163
- collectWorkspaceArchiveObjectKeys,
164
- deleteUnpublishedWorkspaceArchiveObject,
165
- deleteWorkspaceArchiveObjectKeys,
166
+ persistWorkspaceArchiveCandidate,
166
167
  putVersion1TarArchiveOrInline,
167
168
  } from "../sandbox-archive-storage";
168
169
 
@@ -210,7 +211,7 @@ type DrainSandboxClient = {
210
211
  * aborts the terminate before client.delete()).
211
212
  */
212
213
  export type PersistArchiveFn = (
213
- archiveBase64: string | null,
214
+ archiveBase64: string | VerifiedHostWorkspaceArchive | null,
214
215
  archiveMetadata?: WorkspaceArchiveDescriptor,
215
216
  providerSession?: unknown,
216
217
  providerBinding?: Awaited<
@@ -247,6 +248,7 @@ export type TerminateBoxFn = (
247
248
  providerCaptureRequestId?: string,
248
249
  captureDisposition?: DrainCaptureDisposition,
249
250
  capturePolicy?: ProviderWorkspaceCapturePolicy | null,
251
+ diskBackedArchives?: boolean,
250
252
  ) => Promise<boolean | ProviderTerminationOutcome>;
251
253
 
252
254
  export type SweepModalOrphansFn = (
@@ -479,6 +481,7 @@ export function createSandboxLeaseActivities(
479
481
  providerCaptureRequestId,
480
482
  captureDisposition,
481
483
  capturePolicy,
484
+ diskBackedArchives,
482
485
  ) =>
483
486
  await terminateProviderBox(
484
487
  settings,
@@ -490,6 +493,7 @@ export function createSandboxLeaseActivities(
490
493
  providerCaptureRequestId,
491
494
  captureDisposition,
492
495
  capturePolicy,
496
+ diskBackedArchives,
493
497
  ));
494
498
  const sweepModalOrphans: SweepModalOrphansFn =
495
499
  options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
@@ -2675,7 +2679,7 @@ async function terminateDrainableBox(
2675
2679
  ? (lease.recovery.archive.current?.revision ?? null)
2676
2680
  : null;
2677
2681
  const persistArchive: PersistArchiveFn = async (
2678
- archiveBase64: string | null,
2682
+ archiveBase64: string | VerifiedHostWorkspaceArchive | null,
2679
2683
  archiveMetadata?: WorkspaceArchiveDescriptor,
2680
2684
  _providerSession?: unknown,
2681
2685
  providerBinding?: Awaited<
@@ -2691,6 +2695,7 @@ async function terminateDrainableBox(
2691
2695
  const archiveMetrics = runtimeMetricsHooksForObservability(observability);
2692
2696
  let checkpointArtifactId: string | null = null;
2693
2697
  if (
2698
+ typeof archiveBase64 === "string" &&
2694
2699
  archiveBase64 &&
2695
2700
  archiveMetadata?.version === 2 &&
2696
2701
  (archiveMetadata.provider === "modal_snapshot_filesystem" ||
@@ -2735,59 +2740,33 @@ async function terminateDrainableBox(
2735
2740
  if (!archiveMetadata) {
2736
2741
  throw new Error("Sandbox snapshot publication requires a verified archive descriptor");
2737
2742
  }
2738
- const priorKeys = collectWorkspaceArchiveObjectKeys(
2739
- (lease.resumeState as Record<string, unknown> | null | undefined) ?? null,
2740
- );
2741
- let workspaceArchiveRef:
2742
- | Awaited<ReturnType<typeof putVersion1TarArchiveOrInline>>["workspaceArchiveRef"]
2743
- | undefined;
2744
- try {
2745
- const published = await putVersion1TarArchiveOrInline({
2746
- backend: lease.backend,
2747
- objectStorage,
2748
- accountId,
2749
- workspaceId: row.workspaceId,
2750
- sandboxGroupId: row.sandboxGroupId,
2751
- archive: {
2752
- bytes: Buffer.from(archiveBase64, "base64"),
2753
- descriptor: archiveMetadata,
2754
- base64: archiveBase64,
2755
- },
2756
- metrics: archiveMetrics,
2757
- });
2758
- workspaceArchiveRef = published.workspaceArchiveRef;
2759
- result = await persistDrainSnapshot(db, {
2760
- ...baseInput,
2761
- workspaceArchiveMeta: archiveMetadata,
2762
- ...published,
2763
- ...(checkpointArtifactId ? { checkpointArtifactId } : {}),
2764
- });
2765
- if (published.workspaceArchiveRef && objectStorage) {
2766
- if (!result.wrote) {
2767
- await deleteUnpublishedWorkspaceArchiveObject(
2768
- objectStorage,
2769
- published.workspaceArchiveRef,
2770
- archiveMetrics,
2771
- );
2772
- } else {
2773
- const afterLease = await readLease(db, row.workspaceId, row.sandboxGroupId);
2774
- const afterKeys = collectWorkspaceArchiveObjectKeys(
2775
- (afterLease?.resumeState as Record<string, unknown> | null | undefined) ?? null,
2776
- );
2777
- await deleteWorkspaceArchiveObjectKeys(
2778
- objectStorage,
2779
- [...priorKeys].filter((key) => !afterKeys.has(key)),
2780
- ).catch(() => undefined);
2781
- }
2782
- }
2783
- } catch (error) {
2784
- await deleteUnpublishedWorkspaceArchiveObject(
2785
- objectStorage,
2786
- workspaceArchiveRef,
2787
- archiveMetrics,
2788
- );
2789
- throw error;
2790
- }
2743
+ const published = await putVersion1TarArchiveOrInline({
2744
+ backend: lease.backend,
2745
+ objectStorage,
2746
+ accountId,
2747
+ workspaceId: row.workspaceId,
2748
+ sandboxGroupId: row.sandboxGroupId,
2749
+ archive:
2750
+ typeof archiveBase64 === "object"
2751
+ ? archiveBase64
2752
+ : {
2753
+ bytes: Buffer.from(archiveBase64, "base64"),
2754
+ descriptor: archiveMetadata,
2755
+ base64: archiveBase64,
2756
+ },
2757
+ metrics: archiveMetrics,
2758
+ });
2759
+ result = await persistWorkspaceArchiveCandidate({
2760
+ objectStorage,
2761
+ ...(published.workspaceArchiveRef ? { ref: published.workspaceArchiveRef } : {}),
2762
+ persist: () =>
2763
+ persistDrainSnapshot(db, {
2764
+ ...baseInput,
2765
+ workspaceArchiveMeta: archiveMetadata,
2766
+ ...published,
2767
+ ...(checkpointArtifactId ? { checkpointArtifactId } : {}),
2768
+ }),
2769
+ });
2791
2770
  }
2792
2771
  if (!result.wrote && checkpointArtifactId) {
2793
2772
  await markSandboxCheckpointArtifactDeletePending(db, {
@@ -2823,6 +2802,7 @@ async function terminateDrainableBox(
2823
2802
  captureClaim?.providerRequestId ?? attempt.operationId,
2824
2803
  captureDisposition,
2825
2804
  capturePolicy,
2805
+ Boolean(objectStorage),
2826
2806
  );
2827
2807
  const terminated = typeof termination === "boolean" ? termination : termination.terminated;
2828
2808
  if (!terminated) {
@@ -2955,6 +2935,7 @@ export async function terminateProviderBox(
2955
2935
  providerCaptureRequestId?: string,
2956
2936
  captureDisposition: DrainCaptureDisposition = "capture_required",
2957
2937
  claimedCapturePolicy?: ProviderWorkspaceCapturePolicy | null,
2938
+ diskBackedArchives = false,
2958
2939
  ): Promise<ProviderTerminationOutcome> {
2959
2940
  const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;
2960
2941
  const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
@@ -3147,7 +3128,7 @@ export async function terminateProviderBox(
3147
3128
  // snapshot must re-throw BEFORE any terminate so files are never lost — the next
3148
3129
  // sweep retries, the provider idle-timeout is the backstop. A NotFound here (the
3149
3130
  // box raced gone between resume and persist) is success: nothing to persist.
3150
- let verifiedArchive: Awaited<ReturnType<typeof captureVerifiedWorkspaceArchive>> | undefined;
3131
+ let verifiedArchive: VerifiedWorkspaceArchivePayload | undefined;
3151
3132
  const sessionWorkspacePersistence = (
3152
3133
  session as { state?: { workspacePersistence?: unknown } } | undefined
3153
3134
  )?.state?.workspacePersistence;
@@ -3162,10 +3143,15 @@ export async function terminateProviderBox(
3162
3143
  : null;
3163
3144
  try {
3164
3145
  if (captureDisposition !== "archive_published" && session?.persistWorkspace) {
3165
- const capture = captureVerifiedWorkspaceArchive(session, Date.now(), {
3166
- requestId: providerCaptureRequestId ?? randomUUID(),
3167
- strategy: liveCapturePolicy.strategy,
3168
- });
3146
+ const capture = captureWorkspaceArchiveForStorage(
3147
+ session,
3148
+ Date.now(),
3149
+ {
3150
+ requestId: providerCaptureRequestId ?? randomUUID(),
3151
+ strategy: liveCapturePolicy.strategy,
3152
+ },
3153
+ diskBackedArchives,
3154
+ );
3169
3155
  verifiedArchive = await awaitProviderCaptureWithLatePublication({
3170
3156
  capture,
3171
3157
  timeoutMs: settings.sandboxSnapshotTimeoutMs,
@@ -3179,7 +3165,7 @@ export async function terminateProviderBox(
3179
3165
  publishLate: async (archive) => {
3180
3166
  try {
3181
3167
  const result = await persistArchive(
3182
- archive.base64,
3168
+ archive.kind === "host_spool" ? archive : archive.base64,
3183
3169
  archive.descriptor,
3184
3170
  session,
3185
3171
  checkpointBinding,
@@ -3197,6 +3183,8 @@ export async function terminateProviderBox(
3197
3183
  ...logIdentity,
3198
3184
  error: error instanceof Error ? error.message : String(error),
3199
3185
  });
3186
+ } finally {
3187
+ await disposeWorkspaceArchive(archive);
3200
3188
  }
3201
3189
  },
3202
3190
  observeLateFailure: (error) => {
@@ -3237,12 +3225,17 @@ export async function terminateProviderBox(
3237
3225
  ...logIdentity,
3238
3226
  });
3239
3227
  } else if (verifiedArchive) {
3240
- const { wrote } = await persistArchive(
3241
- verifiedArchive.base64,
3242
- verifiedArchive.descriptor,
3243
- session,
3244
- checkpointBinding,
3245
- );
3228
+ let wrote: boolean;
3229
+ try {
3230
+ ({ wrote } = await persistArchive(
3231
+ verifiedArchive.kind === "host_spool" ? verifiedArchive : verifiedArchive.base64,
3232
+ verifiedArchive.descriptor,
3233
+ session,
3234
+ checkpointBinding,
3235
+ ));
3236
+ } finally {
3237
+ await disposeWorkspaceArchive(verifiedArchive);
3238
+ }
3246
3239
  if (!wrote) {
3247
3240
  observability.info(
3248
3241
  "sandbox reaper: lease re-armed during persist — leaving box RUNNING (no terminate)",
@@ -413,7 +413,7 @@ export function recordWorkerDeathRecoveryMetrics(
413
413
  }
414
414
 
415
415
  export class TurnLifecycleMetrics {
416
- private readonly turns = new Map<string, { startedAt: number; lastProgressAt: number }>();
416
+ private readonly attempts = new Map<string, { startedAt: number; lastProgressAt: number }>();
417
417
  private timer: ReturnType<typeof setInterval> | null = null;
418
418
 
419
419
  constructor(
@@ -424,42 +424,46 @@ export class TurnLifecycleMetrics {
424
424
  } = {},
425
425
  ) {}
426
426
 
427
- start(turnId: string): void {
427
+ start(input: { attemptId: string }): void {
428
428
  const now = this.now();
429
- this.turns.set(turnId, { startedAt: now, lastProgressAt: now });
429
+ this.attempts.set(input.attemptId, { startedAt: now, lastProgressAt: now });
430
430
  this.ensureTimer();
431
431
  this.refreshGauges();
432
432
  }
433
433
 
434
- progress(turnId: string): void {
435
- const turn = this.turns.get(turnId);
436
- if (!turn) return;
437
- turn.lastProgressAt = this.now();
434
+ progress(input: { attemptId: string }): void {
435
+ const attempt = this.attempts.get(input.attemptId);
436
+ if (!attempt) return;
437
+ attempt.lastProgressAt = this.now();
438
438
  }
439
439
 
440
- finish(turnId: string, outcome: TurnOutcome | null, durationSeconds?: number): void {
441
- const startedAt = this.turns.get(turnId)?.startedAt;
440
+ finish(input: {
441
+ attemptId: string;
442
+ outcome: TurnOutcome | null;
443
+ durationSeconds?: number;
444
+ }): void {
445
+ const startedAt = this.attempts.get(input.attemptId)?.startedAt;
442
446
  if (startedAt !== undefined) {
443
- this.turns.delete(turnId);
447
+ this.attempts.delete(input.attemptId);
444
448
  }
445
- if (outcome) {
449
+ if (input.outcome) {
446
450
  const observedDuration =
447
- durationSeconds ??
451
+ input.durationSeconds ??
448
452
  (startedAt === undefined ? 0 : Math.max(0, (this.now() - startedAt) / 1000));
449
453
  this.observability.incrementCounter({
450
454
  name: "opengeni_turns_total",
451
455
  help: "Total agent turns by terminal outcome.",
452
- labels: { outcome },
456
+ labels: { outcome: input.outcome },
453
457
  });
454
458
  this.observability.observeHistogram({
455
459
  name: "opengeni_turn_duration_seconds",
456
460
  help: "Agent turn duration in seconds by terminal outcome.",
457
- labels: { outcome },
461
+ labels: { outcome: input.outcome },
458
462
  value: observedDuration,
459
463
  });
460
464
  }
461
465
  this.refreshGauges();
462
- if (this.turns.size === 0) {
466
+ if (this.attempts.size === 0) {
463
467
  this.stopTimer();
464
468
  }
465
469
  }
@@ -467,42 +471,42 @@ export class TurnLifecycleMetrics {
467
471
  refreshGauges(): void {
468
472
  this.observability.setGauge({
469
473
  name: "opengeni_turns_inflight",
470
- help: "Current number of in-flight agent turns in this worker process.",
471
- value: this.turns.size,
474
+ help: "Current number of in-flight physical agent-turn attempts in this worker process.",
475
+ value: this.attempts.size,
472
476
  });
473
477
  this.observability.setGauge({
474
478
  name: "opengeni_turn_oldest_inflight_age_seconds",
475
- help: "Age in seconds of the oldest in-flight agent turn in this worker process.",
479
+ help: "Age in seconds of the oldest in-flight physical agent-turn attempt in this worker process.",
476
480
  value: this.oldestInflightAgeSeconds(),
477
481
  });
478
482
  this.observability.setGauge({
479
483
  name: "opengeni_turn_oldest_no_progress_age_seconds",
480
- help: "Seconds since durable progress for the least recently progressing in-flight turn.",
484
+ help: "Seconds since durable progress for the least recently progressing in-flight physical agent-turn attempt.",
481
485
  value: this.oldestNoProgressAgeSeconds(),
482
486
  });
483
487
  }
484
488
 
485
489
  stop(): void {
486
- this.turns.clear();
490
+ this.attempts.clear();
487
491
  this.refreshGauges();
488
492
  this.stopTimer();
489
493
  }
490
494
 
491
495
  private oldestInflightAgeSeconds(): number {
492
- if (this.turns.size === 0) {
496
+ if (this.attempts.size === 0) {
493
497
  return 0;
494
498
  }
495
499
  let oldest = Number.POSITIVE_INFINITY;
496
- for (const { startedAt } of this.turns.values()) {
500
+ for (const { startedAt } of this.attempts.values()) {
497
501
  oldest = Math.min(oldest, startedAt);
498
502
  }
499
503
  return Math.max(0, (this.now() - oldest) / 1000);
500
504
  }
501
505
 
502
506
  private oldestNoProgressAgeSeconds(): number {
503
- if (this.turns.size === 0) return 0;
507
+ if (this.attempts.size === 0) return 0;
504
508
  let leastRecentProgress = Number.POSITIVE_INFINITY;
505
- for (const { lastProgressAt } of this.turns.values()) {
509
+ for (const { lastProgressAt } of this.attempts.values()) {
506
510
  leastRecentProgress = Math.min(leastRecentProgress, lastProgressAt);
507
511
  }
508
512
  return Math.max(0, (this.now() - leastRecentProgress) / 1000);