@opengeni/worker-bundle 0.28.1-canary.0 → 0.28.3-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/model-usage.d.ts +1 -1
- package/dist/activities/sandbox-lease.d.ts +4 -4
- package/dist/{activities-control-VVK7RA7F.js → activities-control-JUBMDBDZ.js} +55 -70
- package/dist/activities-control-JUBMDBDZ.js.map +1 -0
- package/dist/{activities-turn-FOIZYY56.js → activities-turn-I2ZKGZE6.js} +20 -15
- package/dist/activities-turn-I2ZKGZE6.js.map +1 -0
- package/dist/{chunk-BARTLVBK.js → chunk-A4XQWVZK.js} +25 -25
- package/dist/chunk-A4XQWVZK.js.map +1 -0
- package/dist/{chunk-AKP6BKM2.js → chunk-ZNIKZQMN.js} +146 -111
- package/dist/chunk-ZNIKZQMN.js.map +1 -0
- package/dist/index.js +2 -2
- package/dist/observability-metrics.d.ts +12 -4
- package/dist/sandbox-archive-storage.d.ts +15 -3
- package/dist/sandbox-resume.d.ts +1 -1
- package/package.json +17 -17
- package/src/activities/agent-turn/claim.ts +3 -3
- package/src/activities/agent-turn/finalization.ts +10 -5
- package/src/activities/agent-turn/model-usage.ts +10 -1
- package/src/activities/agent-turn/stream-attempt.ts +3 -2
- package/src/activities/goals.ts +2 -0
- package/src/activities/sandbox-lease.ts +64 -71
- package/src/observability-metrics.ts +28 -24
- package/src/sandbox-archive-storage.ts +82 -18
- package/src/sandbox-resume.ts +79 -63
- package/dist/activities-control-VVK7RA7F.js.map +0 -1
- package/dist/activities-turn-FOIZYY56.js.map +0 -1
- package/dist/chunk-AKP6BKM2.js.map +0 -1
- package/dist/chunk-BARTLVBK.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -19,7 +19,7 @@ import {
|
|
|
19
19
|
startContextCompactionPendingMonitor,
|
|
20
20
|
startSessionRecoveryMonitor,
|
|
21
21
|
startTurnCapacityMonitor
|
|
22
|
-
} from "./chunk-
|
|
22
|
+
} from "./chunk-A4XQWVZK.js";
|
|
23
23
|
|
|
24
24
|
// src/index.ts
|
|
25
25
|
import {
|
|
@@ -781,7 +781,7 @@ function temporalWorkflowExecutionNotFound(error) {
|
|
|
781
781
|
return isGrpcServiceError(error) && error.code === 5;
|
|
782
782
|
}
|
|
783
783
|
async function createDefaultWorkerActivities(role, dependencies = {}) {
|
|
784
|
-
return role === "control" ? (await import("./activities-control-
|
|
784
|
+
return role === "control" ? (await import("./activities-control-JUBMDBDZ.js")).createControlActivities(dependencies) : (await import("./activities-turn-I2ZKGZE6.js")).createTurnActivities(dependencies);
|
|
785
785
|
}
|
|
786
786
|
function resolveOpenGeniWorkflowDefinition(moduleUrl = import.meta.url) {
|
|
787
787
|
const modulePath = fileURLToPath(moduleUrl).replaceAll("\\", "/");
|
|
@@ -91,15 +91,23 @@ export declare function recordWorkerDeathRecoveryMetrics(observability: Observab
|
|
|
91
91
|
export declare class TurnLifecycleMetrics {
|
|
92
92
|
private readonly observability;
|
|
93
93
|
private readonly options;
|
|
94
|
-
private readonly
|
|
94
|
+
private readonly attempts;
|
|
95
95
|
private timer;
|
|
96
96
|
constructor(observability: Observability, options?: {
|
|
97
97
|
now?: () => number;
|
|
98
98
|
refreshIntervalMs?: number;
|
|
99
99
|
});
|
|
100
|
-
start(
|
|
101
|
-
|
|
102
|
-
|
|
100
|
+
start(input: {
|
|
101
|
+
attemptId: string;
|
|
102
|
+
}): void;
|
|
103
|
+
progress(input: {
|
|
104
|
+
attemptId: string;
|
|
105
|
+
}): void;
|
|
106
|
+
finish(input: {
|
|
107
|
+
attemptId: string;
|
|
108
|
+
outcome: TurnOutcome | null;
|
|
109
|
+
durationSeconds?: number;
|
|
110
|
+
}): void;
|
|
103
111
|
refreshGauges(): void;
|
|
104
112
|
stop(): void;
|
|
105
113
|
private oldestInflightAgeSeconds;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type ObjectStorage } from "@opengeni/storage";
|
|
2
|
+
import type { VerifiedHostWorkspaceArchive } from "@opengeni/runtime/sandbox";
|
|
2
3
|
import { type WorkspaceArchiveDescriptor, type WorkspaceArchiveObjectRef } from "@opengeni/contracts";
|
|
3
4
|
export declare class WorkspaceArchiveObjectStorageRequiredError extends Error {
|
|
4
5
|
readonly code: "workspace_archive_object_storage_required";
|
|
@@ -13,15 +14,26 @@ export declare function putTarWorkspaceArchiveObject(input: {
|
|
|
13
14
|
archive: {
|
|
14
15
|
bytes: Uint8Array;
|
|
15
16
|
descriptor: WorkspaceArchiveDescriptor;
|
|
16
|
-
};
|
|
17
|
+
} | VerifiedHostWorkspaceArchive;
|
|
17
18
|
}): Promise<WorkspaceArchiveObjectRef>;
|
|
19
|
+
/** The caller owns a fresh candidate returned by this module. A thrown or lost
|
|
20
|
+
* database response is UNKNOWN: it may have committed, so never delete on catch.
|
|
21
|
+
* Only the transaction's exact candidate disposition can authorize cleanup. */
|
|
22
|
+
export declare function persistWorkspaceArchiveCandidate<T extends {
|
|
23
|
+
wrote: boolean;
|
|
24
|
+
candidateDisposition?: "adopted" | "already_referenced" | "unused";
|
|
25
|
+
}>(input: {
|
|
26
|
+
objectStorage?: ObjectStorage | null;
|
|
27
|
+
ref?: WorkspaceArchiveObjectRef;
|
|
28
|
+
persist: () => Promise<T>;
|
|
29
|
+
}): Promise<T>;
|
|
18
30
|
export declare function putVersion1TarArchiveOrInline(input: {
|
|
19
31
|
backend: string;
|
|
20
32
|
objectStorage?: ObjectStorage | null | undefined;
|
|
21
33
|
accountId: string;
|
|
22
34
|
workspaceId: string;
|
|
23
35
|
sandboxGroupId: string;
|
|
24
|
-
archive: {
|
|
36
|
+
archive: VerifiedHostWorkspaceArchive | {
|
|
25
37
|
bytes: Uint8Array;
|
|
26
38
|
descriptor: WorkspaceArchiveDescriptor;
|
|
27
39
|
base64: string;
|
package/dist/sandbox-resume.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type Settings } from "@opengeni/config";
|
|
2
2
|
import { readLease, reinstateTurnLeaseHolder, SandboxLeaseSupersededError, type Database, type LeaseHolderKind } from "@opengeni/db";
|
|
3
3
|
import { establishSandboxSessionFromEnvelope, type EstablishedSandboxSession, type RuntimeMetricsHooks } from "@opengeni/runtime";
|
|
4
|
-
import type
|
|
4
|
+
import { type ObjectStorage } from "@opengeni/storage";
|
|
5
5
|
import type { Observability } from "@opengeni/observability";
|
|
6
6
|
export type ResumeHolderKind = LeaseHolderKind;
|
|
7
7
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opengeni/worker-bundle",
|
|
3
|
-
"version": "0.28.
|
|
3
|
+
"version": "0.28.3-canary.0",
|
|
4
4
|
"description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -54,22 +54,22 @@
|
|
|
54
54
|
},
|
|
55
55
|
"dependencies": {
|
|
56
56
|
"@llamaindex/liteparse": "2.14.2",
|
|
57
|
-
"@opengeni/agent-proto": "^0.5.1-canary.
|
|
58
|
-
"@opengeni/capabilities": "^0.3.3-canary.
|
|
59
|
-
"@opengeni/codemode": "^0.5.
|
|
60
|
-
"@opengeni/codex": "^0.2.22-canary.
|
|
61
|
-
"@opengeni/config": "^1.1.
|
|
62
|
-
"@opengeni/contracts": "^3.0.
|
|
63
|
-
"@opengeni/core": "^2.9.
|
|
64
|
-
"@opengeni/db": "^4.3.
|
|
65
|
-
"@opengeni/documents": "^0.8.
|
|
66
|
-
"@opengeni/events": "^0.4.
|
|
67
|
-
"@opengeni/github": "^0.7.
|
|
68
|
-
"@opengeni/network": "^0.3.1-canary.
|
|
69
|
-
"@opengeni/observability": "^0.8.
|
|
70
|
-
"@opengeni/runtime": "^2.5.
|
|
71
|
-
"@opengeni/storage": "^0.2.
|
|
72
|
-
"@opengeni/xai-subscription": "^0.1.4-canary.
|
|
57
|
+
"@opengeni/agent-proto": "^0.5.1-canary.25",
|
|
58
|
+
"@opengeni/capabilities": "^0.3.3-canary.7",
|
|
59
|
+
"@opengeni/codemode": "^0.5.5-canary.0",
|
|
60
|
+
"@opengeni/codex": "^0.2.22-canary.7",
|
|
61
|
+
"@opengeni/config": "^1.1.1-canary.0",
|
|
62
|
+
"@opengeni/contracts": "^3.0.1-canary.0",
|
|
63
|
+
"@opengeni/core": "^2.9.3-canary.0",
|
|
64
|
+
"@opengeni/db": "^4.3.2-canary.0",
|
|
65
|
+
"@opengeni/documents": "^0.8.26-canary.0",
|
|
66
|
+
"@opengeni/events": "^0.4.24-canary.0",
|
|
67
|
+
"@opengeni/github": "^0.7.9-canary.0",
|
|
68
|
+
"@opengeni/network": "^0.3.1-canary.7",
|
|
69
|
+
"@opengeni/observability": "^0.8.25-canary.0",
|
|
70
|
+
"@opengeni/runtime": "^2.5.2-canary.0",
|
|
71
|
+
"@opengeni/storage": "^0.2.126-canary.0",
|
|
72
|
+
"@opengeni/xai-subscription": "^0.1.4-canary.7",
|
|
73
73
|
"@temporalio/activity": "^1.17.0",
|
|
74
74
|
"@temporalio/client": "^1.17.0",
|
|
75
75
|
"@temporalio/worker": "^1.17.0",
|
|
@@ -347,7 +347,7 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
|
|
|
347
347
|
});
|
|
348
348
|
return attached.action === "claimed" && attached.turn.id === turn.id;
|
|
349
349
|
};
|
|
350
|
-
turnLifecycleMetricsFor(observability).start(
|
|
350
|
+
turnLifecycleMetricsFor(observability).start({ attemptId: input.attemptId });
|
|
351
351
|
// §7.5 P3 — pass the accepted billing attribution (externally funded turns
|
|
352
352
|
// bypass OpenGeni credit/token gates)
|
|
353
353
|
// AND the optional host `entitlements` port (when bound, its admitRun replaces
|
|
@@ -451,7 +451,7 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
|
|
|
451
451
|
}
|
|
452
452
|
recordCanonicalStartupMilestones(appended.canonicalStartupMilestones);
|
|
453
453
|
if (inputs.length > 0) {
|
|
454
|
-
turnLifecycleMetricsFor(observability).progress(
|
|
454
|
+
turnLifecycleMetricsFor(observability).progress({ attemptId: input.attemptId });
|
|
455
455
|
}
|
|
456
456
|
activityContext?.heartbeat({
|
|
457
457
|
...heartbeatDetails,
|
|
@@ -515,7 +515,7 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
|
|
|
515
515
|
return false;
|
|
516
516
|
}
|
|
517
517
|
recordCanonicalStartupMilestones(result.canonicalStartupMilestones);
|
|
518
|
-
turnLifecycleMetricsFor(observability).progress(
|
|
518
|
+
turnLifecycleMetricsFor(observability).progress({ attemptId: input.attemptId });
|
|
519
519
|
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, result.events);
|
|
520
520
|
if (inputSettlement.turnStatus === "requires_action") {
|
|
521
521
|
// The settlement transaction committed the parent's child_requires_action
|
|
@@ -745,12 +745,17 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
|
|
|
745
745
|
status: finalizationError ? "cleanup_failed" : control.activityStatus,
|
|
746
746
|
durationSeconds,
|
|
747
747
|
});
|
|
748
|
-
if (attempt.turnId
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
748
|
+
if (attempt.turnId) {
|
|
749
|
+
// This registry describes physical activity ownership, not durable turn
|
|
750
|
+
// settlement. Always clear the exact attempt when its activity unwinds,
|
|
751
|
+
// even when failure settlement could not classify an outcome. Otherwise
|
|
752
|
+
// a recovered successor can make progress while the predecessor leaves a
|
|
753
|
+
// process-local false stuck-turn alert behind.
|
|
754
|
+
turnLifecycleMetricsFor(observability).finish({
|
|
755
|
+
attemptId: input.attemptId,
|
|
756
|
+
outcome: control.turnMetricOutcome,
|
|
752
757
|
durationSeconds,
|
|
753
|
-
);
|
|
758
|
+
});
|
|
754
759
|
}
|
|
755
760
|
activitySpan.end({
|
|
756
761
|
attributes: {
|
|
@@ -129,8 +129,17 @@ export const createSessionTitleModelUsageEventState = createCompactionModelUsage
|
|
|
129
129
|
|
|
130
130
|
export function modelResponseContextSignal(
|
|
131
131
|
state: ModelResponseEventState,
|
|
132
|
+
responseCountBeforeStream = 0,
|
|
132
133
|
): { revision: number; totalTokens: number } | null {
|
|
133
|
-
|
|
134
|
+
const signal = state.contextSignal;
|
|
135
|
+
// A compaction retry creates a new SDK request counter, but usage identities
|
|
136
|
+
// remain activity-wide. Never bind a pre-stream report to a reused request
|
|
137
|
+
// ordinal; translate only this stream's reports without mutating usage state.
|
|
138
|
+
if (!signal || signal.revision <= responseCountBeforeStream) return null;
|
|
139
|
+
return {
|
|
140
|
+
revision: signal.revision - responseCountBeforeStream,
|
|
141
|
+
totalTokens: signal.totalTokens,
|
|
142
|
+
};
|
|
134
143
|
}
|
|
135
144
|
|
|
136
145
|
export function assertModelResponseLatencyMode(input: {
|
|
@@ -865,7 +865,8 @@ export async function runTurnStreamAttempt(
|
|
|
865
865
|
},
|
|
866
866
|
}
|
|
867
867
|
: {}),
|
|
868
|
-
contextCompactionSignal: () =>
|
|
868
|
+
contextCompactionSignal: () =>
|
|
869
|
+
modelResponseContextSignal(modelResponseState, responseCountBeforeStream),
|
|
869
870
|
contextCompactionRequested: () =>
|
|
870
871
|
isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
871
872
|
onModelPreparationPhase: (measurement) => {
|
|
@@ -1061,7 +1062,7 @@ export async function runTurnStreamAttempt(
|
|
|
1061
1062
|
currentToolBatchCallIds = new Set<string>();
|
|
1062
1063
|
currentToolBatchCompletedCallIds = new Set<string>();
|
|
1063
1064
|
await historySink.reconcileConversationTruth();
|
|
1064
|
-
turnLifecycleMetricsFor(observability).progress(
|
|
1065
|
+
turnLifecycleMetricsFor(observability).progress({ attemptId: input.attemptId });
|
|
1065
1066
|
modelCheckpointMemoryCollector.schedule(observability);
|
|
1066
1067
|
try {
|
|
1067
1068
|
await ensureRunAllowed(
|
package/src/activities/goals.ts
CHANGED
|
@@ -286,6 +286,8 @@ export function goalContinuationPrompt(
|
|
|
286
286
|
"Continuation behavior:",
|
|
287
287
|
"- This goal persists across turns. A runtime boundary can end one turn without shrinking the objective; the next continuation resumes the same full objective.",
|
|
288
288
|
"- Keep working until the requested end state is true and verified. Do not end the turn merely because one useful action completed, and do not redefine success around a smaller or easier task.",
|
|
289
|
+
"- An incomplete-status report is not a substitute for continuing the work. If the remaining problem can be investigated or addressed within your current authority, continue that work in this turn rather than returning another equivalent status-only final.",
|
|
290
|
+
"- Distinguish unfinished work from a blocker that actually requires human input or external change; use the existing waiting and blocked audits only when their conditions hold.",
|
|
289
291
|
"- Temporary rough edges are acceptable while the work is moving in the right direction. Completion still requires the requested end state to be true and verified.",
|
|
290
292
|
"",
|
|
291
293
|
"Work from evidence:",
|
|
@@ -82,7 +82,10 @@ import {
|
|
|
82
82
|
// box is typed as provider loss. Stale capture reconciliation separately uses
|
|
83
83
|
// establishSandboxSessionFromEnvelope in strict resume-only mode so it can
|
|
84
84
|
// reuse the runtime's provider-ready proof without ever cold-restoring.
|
|
85
|
-
|
|
85
|
+
captureWorkspaceArchiveForStorage,
|
|
86
|
+
disposeWorkspaceArchive,
|
|
87
|
+
type VerifiedWorkspaceArchivePayload,
|
|
88
|
+
type VerifiedHostWorkspaceArchive,
|
|
86
89
|
cancelSelfhostedOp,
|
|
87
90
|
assertConsistentSandboxProviderIdentity,
|
|
88
91
|
createSandboxClientForBackend,
|
|
@@ -160,9 +163,7 @@ import {
|
|
|
160
163
|
} from "../opensandbox-kubernetes-inventory";
|
|
161
164
|
import type { ObjectStorage } from "@opengeni/storage";
|
|
162
165
|
import {
|
|
163
|
-
|
|
164
|
-
deleteUnpublishedWorkspaceArchiveObject,
|
|
165
|
-
deleteWorkspaceArchiveObjectKeys,
|
|
166
|
+
persistWorkspaceArchiveCandidate,
|
|
166
167
|
putVersion1TarArchiveOrInline,
|
|
167
168
|
} from "../sandbox-archive-storage";
|
|
168
169
|
|
|
@@ -210,7 +211,7 @@ type DrainSandboxClient = {
|
|
|
210
211
|
* aborts the terminate before client.delete()).
|
|
211
212
|
*/
|
|
212
213
|
export type PersistArchiveFn = (
|
|
213
|
-
archiveBase64: string | null,
|
|
214
|
+
archiveBase64: string | VerifiedHostWorkspaceArchive | null,
|
|
214
215
|
archiveMetadata?: WorkspaceArchiveDescriptor,
|
|
215
216
|
providerSession?: unknown,
|
|
216
217
|
providerBinding?: Awaited<
|
|
@@ -247,6 +248,7 @@ export type TerminateBoxFn = (
|
|
|
247
248
|
providerCaptureRequestId?: string,
|
|
248
249
|
captureDisposition?: DrainCaptureDisposition,
|
|
249
250
|
capturePolicy?: ProviderWorkspaceCapturePolicy | null,
|
|
251
|
+
diskBackedArchives?: boolean,
|
|
250
252
|
) => Promise<boolean | ProviderTerminationOutcome>;
|
|
251
253
|
|
|
252
254
|
export type SweepModalOrphansFn = (
|
|
@@ -479,6 +481,7 @@ export function createSandboxLeaseActivities(
|
|
|
479
481
|
providerCaptureRequestId,
|
|
480
482
|
captureDisposition,
|
|
481
483
|
capturePolicy,
|
|
484
|
+
diskBackedArchives,
|
|
482
485
|
) =>
|
|
483
486
|
await terminateProviderBox(
|
|
484
487
|
settings,
|
|
@@ -490,6 +493,7 @@ export function createSandboxLeaseActivities(
|
|
|
490
493
|
providerCaptureRequestId,
|
|
491
494
|
captureDisposition,
|
|
492
495
|
capturePolicy,
|
|
496
|
+
diskBackedArchives,
|
|
493
497
|
));
|
|
494
498
|
const sweepModalOrphans: SweepModalOrphansFn =
|
|
495
499
|
options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
@@ -2675,7 +2679,7 @@ async function terminateDrainableBox(
|
|
|
2675
2679
|
? (lease.recovery.archive.current?.revision ?? null)
|
|
2676
2680
|
: null;
|
|
2677
2681
|
const persistArchive: PersistArchiveFn = async (
|
|
2678
|
-
archiveBase64: string | null,
|
|
2682
|
+
archiveBase64: string | VerifiedHostWorkspaceArchive | null,
|
|
2679
2683
|
archiveMetadata?: WorkspaceArchiveDescriptor,
|
|
2680
2684
|
_providerSession?: unknown,
|
|
2681
2685
|
providerBinding?: Awaited<
|
|
@@ -2691,6 +2695,7 @@ async function terminateDrainableBox(
|
|
|
2691
2695
|
const archiveMetrics = runtimeMetricsHooksForObservability(observability);
|
|
2692
2696
|
let checkpointArtifactId: string | null = null;
|
|
2693
2697
|
if (
|
|
2698
|
+
typeof archiveBase64 === "string" &&
|
|
2694
2699
|
archiveBase64 &&
|
|
2695
2700
|
archiveMetadata?.version === 2 &&
|
|
2696
2701
|
(archiveMetadata.provider === "modal_snapshot_filesystem" ||
|
|
@@ -2735,59 +2740,33 @@ async function terminateDrainableBox(
|
|
|
2735
2740
|
if (!archiveMetadata) {
|
|
2736
2741
|
throw new Error("Sandbox snapshot publication requires a verified archive descriptor");
|
|
2737
2742
|
}
|
|
2738
|
-
const
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
|
|
2761
|
-
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
|
|
2765
|
-
if (published.workspaceArchiveRef && objectStorage) {
|
|
2766
|
-
if (!result.wrote) {
|
|
2767
|
-
await deleteUnpublishedWorkspaceArchiveObject(
|
|
2768
|
-
objectStorage,
|
|
2769
|
-
published.workspaceArchiveRef,
|
|
2770
|
-
archiveMetrics,
|
|
2771
|
-
);
|
|
2772
|
-
} else {
|
|
2773
|
-
const afterLease = await readLease(db, row.workspaceId, row.sandboxGroupId);
|
|
2774
|
-
const afterKeys = collectWorkspaceArchiveObjectKeys(
|
|
2775
|
-
(afterLease?.resumeState as Record<string, unknown> | null | undefined) ?? null,
|
|
2776
|
-
);
|
|
2777
|
-
await deleteWorkspaceArchiveObjectKeys(
|
|
2778
|
-
objectStorage,
|
|
2779
|
-
[...priorKeys].filter((key) => !afterKeys.has(key)),
|
|
2780
|
-
).catch(() => undefined);
|
|
2781
|
-
}
|
|
2782
|
-
}
|
|
2783
|
-
} catch (error) {
|
|
2784
|
-
await deleteUnpublishedWorkspaceArchiveObject(
|
|
2785
|
-
objectStorage,
|
|
2786
|
-
workspaceArchiveRef,
|
|
2787
|
-
archiveMetrics,
|
|
2788
|
-
);
|
|
2789
|
-
throw error;
|
|
2790
|
-
}
|
|
2743
|
+
const published = await putVersion1TarArchiveOrInline({
|
|
2744
|
+
backend: lease.backend,
|
|
2745
|
+
objectStorage,
|
|
2746
|
+
accountId,
|
|
2747
|
+
workspaceId: row.workspaceId,
|
|
2748
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
2749
|
+
archive:
|
|
2750
|
+
typeof archiveBase64 === "object"
|
|
2751
|
+
? archiveBase64
|
|
2752
|
+
: {
|
|
2753
|
+
bytes: Buffer.from(archiveBase64, "base64"),
|
|
2754
|
+
descriptor: archiveMetadata,
|
|
2755
|
+
base64: archiveBase64,
|
|
2756
|
+
},
|
|
2757
|
+
metrics: archiveMetrics,
|
|
2758
|
+
});
|
|
2759
|
+
result = await persistWorkspaceArchiveCandidate({
|
|
2760
|
+
objectStorage,
|
|
2761
|
+
...(published.workspaceArchiveRef ? { ref: published.workspaceArchiveRef } : {}),
|
|
2762
|
+
persist: () =>
|
|
2763
|
+
persistDrainSnapshot(db, {
|
|
2764
|
+
...baseInput,
|
|
2765
|
+
workspaceArchiveMeta: archiveMetadata,
|
|
2766
|
+
...published,
|
|
2767
|
+
...(checkpointArtifactId ? { checkpointArtifactId } : {}),
|
|
2768
|
+
}),
|
|
2769
|
+
});
|
|
2791
2770
|
}
|
|
2792
2771
|
if (!result.wrote && checkpointArtifactId) {
|
|
2793
2772
|
await markSandboxCheckpointArtifactDeletePending(db, {
|
|
@@ -2823,6 +2802,7 @@ async function terminateDrainableBox(
|
|
|
2823
2802
|
captureClaim?.providerRequestId ?? attempt.operationId,
|
|
2824
2803
|
captureDisposition,
|
|
2825
2804
|
capturePolicy,
|
|
2805
|
+
Boolean(objectStorage),
|
|
2826
2806
|
);
|
|
2827
2807
|
const terminated = typeof termination === "boolean" ? termination : termination.terminated;
|
|
2828
2808
|
if (!terminated) {
|
|
@@ -2955,6 +2935,7 @@ export async function terminateProviderBox(
|
|
|
2955
2935
|
providerCaptureRequestId?: string,
|
|
2956
2936
|
captureDisposition: DrainCaptureDisposition = "capture_required",
|
|
2957
2937
|
claimedCapturePolicy?: ProviderWorkspaceCapturePolicy | null,
|
|
2938
|
+
diskBackedArchives = false,
|
|
2958
2939
|
): Promise<ProviderTerminationOutcome> {
|
|
2959
2940
|
const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;
|
|
2960
2941
|
const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
|
|
@@ -3147,7 +3128,7 @@ export async function terminateProviderBox(
|
|
|
3147
3128
|
// snapshot must re-throw BEFORE any terminate so files are never lost — the next
|
|
3148
3129
|
// sweep retries, the provider idle-timeout is the backstop. A NotFound here (the
|
|
3149
3130
|
// box raced gone between resume and persist) is success: nothing to persist.
|
|
3150
|
-
let verifiedArchive:
|
|
3131
|
+
let verifiedArchive: VerifiedWorkspaceArchivePayload | undefined;
|
|
3151
3132
|
const sessionWorkspacePersistence = (
|
|
3152
3133
|
session as { state?: { workspacePersistence?: unknown } } | undefined
|
|
3153
3134
|
)?.state?.workspacePersistence;
|
|
@@ -3162,10 +3143,15 @@ export async function terminateProviderBox(
|
|
|
3162
3143
|
: null;
|
|
3163
3144
|
try {
|
|
3164
3145
|
if (captureDisposition !== "archive_published" && session?.persistWorkspace) {
|
|
3165
|
-
const capture =
|
|
3166
|
-
|
|
3167
|
-
|
|
3168
|
-
|
|
3146
|
+
const capture = captureWorkspaceArchiveForStorage(
|
|
3147
|
+
session,
|
|
3148
|
+
Date.now(),
|
|
3149
|
+
{
|
|
3150
|
+
requestId: providerCaptureRequestId ?? randomUUID(),
|
|
3151
|
+
strategy: liveCapturePolicy.strategy,
|
|
3152
|
+
},
|
|
3153
|
+
diskBackedArchives,
|
|
3154
|
+
);
|
|
3169
3155
|
verifiedArchive = await awaitProviderCaptureWithLatePublication({
|
|
3170
3156
|
capture,
|
|
3171
3157
|
timeoutMs: settings.sandboxSnapshotTimeoutMs,
|
|
@@ -3179,7 +3165,7 @@ export async function terminateProviderBox(
|
|
|
3179
3165
|
publishLate: async (archive) => {
|
|
3180
3166
|
try {
|
|
3181
3167
|
const result = await persistArchive(
|
|
3182
|
-
archive.base64,
|
|
3168
|
+
archive.kind === "host_spool" ? archive : archive.base64,
|
|
3183
3169
|
archive.descriptor,
|
|
3184
3170
|
session,
|
|
3185
3171
|
checkpointBinding,
|
|
@@ -3197,6 +3183,8 @@ export async function terminateProviderBox(
|
|
|
3197
3183
|
...logIdentity,
|
|
3198
3184
|
error: error instanceof Error ? error.message : String(error),
|
|
3199
3185
|
});
|
|
3186
|
+
} finally {
|
|
3187
|
+
await disposeWorkspaceArchive(archive);
|
|
3200
3188
|
}
|
|
3201
3189
|
},
|
|
3202
3190
|
observeLateFailure: (error) => {
|
|
@@ -3237,12 +3225,17 @@ export async function terminateProviderBox(
|
|
|
3237
3225
|
...logIdentity,
|
|
3238
3226
|
});
|
|
3239
3227
|
} else if (verifiedArchive) {
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
|
|
3244
|
-
|
|
3245
|
-
|
|
3228
|
+
let wrote: boolean;
|
|
3229
|
+
try {
|
|
3230
|
+
({ wrote } = await persistArchive(
|
|
3231
|
+
verifiedArchive.kind === "host_spool" ? verifiedArchive : verifiedArchive.base64,
|
|
3232
|
+
verifiedArchive.descriptor,
|
|
3233
|
+
session,
|
|
3234
|
+
checkpointBinding,
|
|
3235
|
+
));
|
|
3236
|
+
} finally {
|
|
3237
|
+
await disposeWorkspaceArchive(verifiedArchive);
|
|
3238
|
+
}
|
|
3246
3239
|
if (!wrote) {
|
|
3247
3240
|
observability.info(
|
|
3248
3241
|
"sandbox reaper: lease re-armed during persist — leaving box RUNNING (no terminate)",
|
|
@@ -413,7 +413,7 @@ export function recordWorkerDeathRecoveryMetrics(
|
|
|
413
413
|
}
|
|
414
414
|
|
|
415
415
|
export class TurnLifecycleMetrics {
|
|
416
|
-
private readonly
|
|
416
|
+
private readonly attempts = new Map<string, { startedAt: number; lastProgressAt: number }>();
|
|
417
417
|
private timer: ReturnType<typeof setInterval> | null = null;
|
|
418
418
|
|
|
419
419
|
constructor(
|
|
@@ -424,42 +424,46 @@ export class TurnLifecycleMetrics {
|
|
|
424
424
|
} = {},
|
|
425
425
|
) {}
|
|
426
426
|
|
|
427
|
-
start(
|
|
427
|
+
start(input: { attemptId: string }): void {
|
|
428
428
|
const now = this.now();
|
|
429
|
-
this.
|
|
429
|
+
this.attempts.set(input.attemptId, { startedAt: now, lastProgressAt: now });
|
|
430
430
|
this.ensureTimer();
|
|
431
431
|
this.refreshGauges();
|
|
432
432
|
}
|
|
433
433
|
|
|
434
|
-
progress(
|
|
435
|
-
const
|
|
436
|
-
if (!
|
|
437
|
-
|
|
434
|
+
progress(input: { attemptId: string }): void {
|
|
435
|
+
const attempt = this.attempts.get(input.attemptId);
|
|
436
|
+
if (!attempt) return;
|
|
437
|
+
attempt.lastProgressAt = this.now();
|
|
438
438
|
}
|
|
439
439
|
|
|
440
|
-
finish(
|
|
441
|
-
|
|
440
|
+
finish(input: {
|
|
441
|
+
attemptId: string;
|
|
442
|
+
outcome: TurnOutcome | null;
|
|
443
|
+
durationSeconds?: number;
|
|
444
|
+
}): void {
|
|
445
|
+
const startedAt = this.attempts.get(input.attemptId)?.startedAt;
|
|
442
446
|
if (startedAt !== undefined) {
|
|
443
|
-
this.
|
|
447
|
+
this.attempts.delete(input.attemptId);
|
|
444
448
|
}
|
|
445
|
-
if (outcome) {
|
|
449
|
+
if (input.outcome) {
|
|
446
450
|
const observedDuration =
|
|
447
|
-
durationSeconds ??
|
|
451
|
+
input.durationSeconds ??
|
|
448
452
|
(startedAt === undefined ? 0 : Math.max(0, (this.now() - startedAt) / 1000));
|
|
449
453
|
this.observability.incrementCounter({
|
|
450
454
|
name: "opengeni_turns_total",
|
|
451
455
|
help: "Total agent turns by terminal outcome.",
|
|
452
|
-
labels: { outcome },
|
|
456
|
+
labels: { outcome: input.outcome },
|
|
453
457
|
});
|
|
454
458
|
this.observability.observeHistogram({
|
|
455
459
|
name: "opengeni_turn_duration_seconds",
|
|
456
460
|
help: "Agent turn duration in seconds by terminal outcome.",
|
|
457
|
-
labels: { outcome },
|
|
461
|
+
labels: { outcome: input.outcome },
|
|
458
462
|
value: observedDuration,
|
|
459
463
|
});
|
|
460
464
|
}
|
|
461
465
|
this.refreshGauges();
|
|
462
|
-
if (this.
|
|
466
|
+
if (this.attempts.size === 0) {
|
|
463
467
|
this.stopTimer();
|
|
464
468
|
}
|
|
465
469
|
}
|
|
@@ -467,42 +471,42 @@ export class TurnLifecycleMetrics {
|
|
|
467
471
|
refreshGauges(): void {
|
|
468
472
|
this.observability.setGauge({
|
|
469
473
|
name: "opengeni_turns_inflight",
|
|
470
|
-
help: "Current number of in-flight agent
|
|
471
|
-
value: this.
|
|
474
|
+
help: "Current number of in-flight physical agent-turn attempts in this worker process.",
|
|
475
|
+
value: this.attempts.size,
|
|
472
476
|
});
|
|
473
477
|
this.observability.setGauge({
|
|
474
478
|
name: "opengeni_turn_oldest_inflight_age_seconds",
|
|
475
|
-
help: "Age in seconds of the oldest in-flight agent
|
|
479
|
+
help: "Age in seconds of the oldest in-flight physical agent-turn attempt in this worker process.",
|
|
476
480
|
value: this.oldestInflightAgeSeconds(),
|
|
477
481
|
});
|
|
478
482
|
this.observability.setGauge({
|
|
479
483
|
name: "opengeni_turn_oldest_no_progress_age_seconds",
|
|
480
|
-
help: "Seconds since durable progress for the least recently progressing in-flight turn.",
|
|
484
|
+
help: "Seconds since durable progress for the least recently progressing in-flight physical agent-turn attempt.",
|
|
481
485
|
value: this.oldestNoProgressAgeSeconds(),
|
|
482
486
|
});
|
|
483
487
|
}
|
|
484
488
|
|
|
485
489
|
stop(): void {
|
|
486
|
-
this.
|
|
490
|
+
this.attempts.clear();
|
|
487
491
|
this.refreshGauges();
|
|
488
492
|
this.stopTimer();
|
|
489
493
|
}
|
|
490
494
|
|
|
491
495
|
private oldestInflightAgeSeconds(): number {
|
|
492
|
-
if (this.
|
|
496
|
+
if (this.attempts.size === 0) {
|
|
493
497
|
return 0;
|
|
494
498
|
}
|
|
495
499
|
let oldest = Number.POSITIVE_INFINITY;
|
|
496
|
-
for (const { startedAt } of this.
|
|
500
|
+
for (const { startedAt } of this.attempts.values()) {
|
|
497
501
|
oldest = Math.min(oldest, startedAt);
|
|
498
502
|
}
|
|
499
503
|
return Math.max(0, (this.now() - oldest) / 1000);
|
|
500
504
|
}
|
|
501
505
|
|
|
502
506
|
private oldestNoProgressAgeSeconds(): number {
|
|
503
|
-
if (this.
|
|
507
|
+
if (this.attempts.size === 0) return 0;
|
|
504
508
|
let leastRecentProgress = Number.POSITIVE_INFINITY;
|
|
505
|
-
for (const { lastProgressAt } of this.
|
|
509
|
+
for (const { lastProgressAt } of this.attempts.values()) {
|
|
506
510
|
leastRecentProgress = Math.min(leastRecentProgress, lastProgressAt);
|
|
507
511
|
}
|
|
508
512
|
return Math.max(0, (this.now() - leastRecentProgress) / 1000);
|