@opengeni/worker-bundle 0.20.3 → 0.22.1-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +52 -0
- package/dist/activities/agent-turn/agent-build.d.ts +89 -0
- package/dist/activities/agent-turn/claim.d.ts +67 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
- package/dist/activities/agent-turn/codex.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
- package/dist/activities/agent-turn/errors.d.ts +253 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
- package/dist/activities/agent-turn/file-resources.d.ts +37 -0
- package/dist/activities/agent-turn/finalization.d.ts +42 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
- package/dist/activities/agent-turn/governance-model.d.ts +66 -0
- package/dist/activities/agent-turn/history-sink.d.ts +52 -0
- package/dist/activities/agent-turn/history.d.ts +63 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
- package/dist/activities/agent-turn/model-usage.d.ts +218 -0
- package/dist/activities/agent-turn/quiescence.d.ts +135 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
- package/dist/activities/agent-turn/run.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
- package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
- package/dist/activities/agent-turn/turn-context.d.ts +144 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
- package/dist/activities/agent-turn.d.ts +12 -954
- package/dist/activities/automations.d.ts +16 -0
- package/dist/activities/common.d.ts +1 -1
- package/dist/activities/connector-attachments.d.ts +13 -0
- package/dist/activities/documents.d.ts +1 -0
- package/dist/activities/environment.d.ts +26 -15
- package/dist/activities/generated-images.d.ts +1 -1
- package/dist/activities/git-credential-renewal.d.ts +3 -1
- package/dist/activities/goals.d.ts +4 -1
- package/dist/activities/google-drive-provider.d.ts +29 -0
- package/dist/activities/google-drive-publication.d.ts +17 -4
- package/dist/activities/incident-telemetry-authority.d.ts +52 -0
- package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
- package/dist/activities/knowledge-source-sync.d.ts +4 -0
- package/dist/activities/mcp-credentials.d.ts +9 -3
- package/dist/activities/open-suffix-resume.d.ts +77 -0
- package/dist/activities/packs.d.ts +3 -11
- package/dist/activities/parent-wake.d.ts +19 -0
- package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
- package/dist/activities/retained-screenshots.d.ts +1 -1
- package/dist/activities/rig-verification.d.ts +1 -1
- package/dist/activities/run-input.d.ts +9 -10
- package/dist/activities/sandbox-images.d.ts +1 -14
- package/dist/activities/sandbox-lease.d.ts +21 -3
- package/dist/activities/scheduled-tasks.d.ts +47 -0
- package/dist/activities/types.d.ts +23 -2
- package/dist/activities/video-media-validation.d.ts +1 -1
- package/dist/activities/video-output-retention.d.ts +1 -2
- package/dist/activities/xai-auth.d.ts +3 -0
- package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
- package/dist/activities-control-DKG22ONJ.js.map +1 -0
- package/dist/activities-control.d.ts +6 -2
- package/dist/activities-turn-4IMC3MAA.js +21326 -0
- package/dist/activities-turn-4IMC3MAA.js.map +1 -0
- package/dist/activities.d.ts +7 -3
- package/dist/artifact-materializer-entry.js +41 -22
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
- package/dist/chunk-CE7TPKBS.js.map +1 -0
- package/dist/chunk-CNYGVRKF.js +4698 -0
- package/dist/chunk-CNYGVRKF.js.map +1 -0
- package/dist/connection-provider-use.d.ts +18 -0
- package/dist/editable-artifact-materializer.d.ts +4 -3
- package/dist/github-rest-mcp.d.ts +26 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +58 -14
- package/dist/index.js.map +1 -1
- package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
- package/dist/model-context-contributions.d.ts +8 -4
- package/dist/observability-metrics.d.ts +63 -4
- package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
- package/dist/personal-github-git-credentials.d.ts +9 -0
- package/dist/pr-review-credentials.d.ts +6 -0
- package/dist/sandbox-archive-storage.d.ts +45 -0
- package/dist/sandbox-resume.d.ts +65 -6
- package/dist/sandbox-routing.d.ts +38 -2
- package/dist/scheduled-alert-occurrence.d.ts +33 -0
- package/dist/workflow-bundle.js +87 -28
- package/dist/workflows/activities.d.ts +1 -0
- package/dist/workflows/automations.d.ts +6 -0
- package/dist/workflows/document-index.d.ts +1 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +292 -0
- package/src/activities/agent-turn/agent-build.ts +802 -0
- package/src/activities/agent-turn/claim.ts +590 -0
- package/src/activities/agent-turn/codex-capacity.ts +720 -0
- package/src/activities/agent-turn/codex.ts +94 -0
- package/src/activities/agent-turn/compaction-prep.ts +722 -0
- package/src/activities/agent-turn/credential-leases.ts +240 -0
- package/src/activities/agent-turn/errors.ts +1068 -0
- package/src/activities/agent-turn/failure-settlement.ts +1436 -0
- package/src/activities/agent-turn/file-resources.ts +227 -0
- package/src/activities/agent-turn/finalization.ts +754 -0
- package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
- package/src/activities/agent-turn/governance-model.ts +465 -0
- package/src/activities/agent-turn/history-sink.ts +139 -0
- package/src/activities/agent-turn/history.ts +200 -0
- package/src/activities/agent-turn/media-artifacts.ts +359 -0
- package/src/activities/agent-turn/model-usage.ts +823 -0
- package/src/activities/agent-turn/quiescence.ts +353 -0
- package/src/activities/agent-turn/run-credentials.ts +761 -0
- package/src/activities/agent-turn/run.ts +1530 -0
- package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
- package/src/activities/agent-turn/sandbox-provision.ts +569 -0
- package/src/activities/agent-turn/sandbox-route.ts +358 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
- package/src/activities/agent-turn/stream-attempt.ts +1670 -0
- package/src/activities/agent-turn/tool-environment.ts +729 -0
- package/src/activities/agent-turn/tool-policy.ts +216 -0
- package/src/activities/agent-turn/tool-result-spill.ts +265 -0
- package/src/activities/agent-turn/turn-context.ts +281 -0
- package/src/activities/agent-turn/xai-capacity.ts +229 -0
- package/src/activities/agent-turn.ts +12 -12701
- package/src/activities/api-integrations.ts +3 -0
- package/src/activities/automations.ts +214 -0
- package/src/activities/codemode-dispatcher.ts +12 -1
- package/src/activities/common.ts +1 -5
- package/src/activities/connector-attachments.ts +168 -0
- package/src/activities/context-compaction.ts +18 -0
- package/src/activities/environment.ts +151 -55
- package/src/activities/generated-images.ts +26 -11
- package/src/activities/git-credential-renewal.ts +13 -3
- package/src/activities/goals.ts +85 -19
- package/src/activities/google-drive-provider.ts +178 -0
- package/src/activities/google-drive-publication.ts +147 -49
- package/src/activities/incident-telemetry-authority.ts +327 -0
- package/src/activities/incident-telemetry-preflight.ts +317 -0
- package/src/activities/knowledge-source-sync.ts +178 -37
- package/src/activities/mcp-credentials.ts +191 -17
- package/src/activities/open-suffix-resume.ts +422 -0
- package/src/activities/packs.ts +12 -60
- package/src/activities/parent-wake.ts +64 -14
- package/src/activities/retained-screenshot-reaper.ts +6 -3
- package/src/activities/retained-screenshots.ts +15 -47
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +102 -142
- package/src/activities/sandbox-images.ts +7 -166
- package/src/activities/sandbox-lease.ts +406 -15
- package/src/activities/scheduled-tasks.ts +2103 -411
- package/src/activities/session-state.ts +10 -2
- package/src/activities/types.ts +32 -2
- package/src/activities/video-media-validation.ts +3 -3
- package/src/activities/video-output-retention.ts +2 -57
- package/src/activities/xai-auth.ts +8 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/activity-services.ts +13 -4
- package/src/connection-provider-use.ts +36 -0
- package/src/editable-artifact-materializer-subprocess.ts +25 -14
- package/src/editable-artifact-materializer.ts +29 -18
- package/src/github-rest-mcp.ts +713 -0
- package/src/http.ts +16 -6
- package/src/index.ts +40 -5
- package/src/knowledge-source-sync-activity-policy.ts +2 -0
- package/src/model-context-contributions.ts +33 -10
- package/src/observability-metrics.ts +357 -0
- package/src/opensandbox-kubernetes-inventory.ts +292 -0
- package/src/personal-github-git-credentials.ts +331 -0
- package/src/pr-review-credentials.ts +163 -0
- package/src/sandbox-archive-storage.ts +159 -0
- package/src/sandbox-resume.ts +449 -63
- package/src/sandbox-routing.ts +314 -16
- package/src/scheduled-alert-occurrence.ts +176 -0
- package/src/workflows/activities.ts +18 -2
- package/src/workflows/automations.ts +11 -0
- package/src/workflows/session.ts +13 -3
- package/src/workflows.ts +1 -0
- package/dist/activities-control-BJSDRSIP.js.map +0 -1
- package/dist/activities-turn-J5YO7XXX.js +0 -17243
- package/dist/activities-turn-J5YO7XXX.js.map +0 -1
- package/dist/chunk-N5TJIFCE.js.map +0 -1
- package/dist/chunk-P6Z2SLAA.js +0 -3092
- package/dist/chunk-P6Z2SLAA.js.map +0 -1
|
@@ -0,0 +1,569 @@
|
|
|
1
|
+
import {
|
|
2
|
+
SandboxLeaseRecoveryBlockedError,
|
|
3
|
+
SandboxLeaseSupersededError,
|
|
4
|
+
SandboxLeaseTransitionError,
|
|
5
|
+
SandboxImageConflictError,
|
|
6
|
+
SandboxRigConflictError,
|
|
7
|
+
} from "@opengeni/db";
|
|
8
|
+
import {
|
|
9
|
+
SandboxConfigError,
|
|
10
|
+
SandboxExactResumeInstanceUnavailableError,
|
|
11
|
+
SandboxExactResumeReplacedError,
|
|
12
|
+
SandboxExecReadinessError,
|
|
13
|
+
SandboxProviderContinuityUnavailableError,
|
|
14
|
+
SandboxProviderUnavailableError,
|
|
15
|
+
SandboxResumeIdentityMismatchError,
|
|
16
|
+
SandboxResumeIdentityUnavailableError,
|
|
17
|
+
SandboxResumeStateUnavailableError,
|
|
18
|
+
WorkspaceArchiveIntegrityError,
|
|
19
|
+
classifyProviderSandboxFailure,
|
|
20
|
+
} from "@opengeni/runtime";
|
|
21
|
+
import { type Settings } from "@opengeni/config";
|
|
22
|
+
import { CancelledFailure } from "@temporalio/activity";
|
|
23
|
+
import {
|
|
24
|
+
sandboxProvisionFailureStage,
|
|
25
|
+
SandboxProvisionStageError,
|
|
26
|
+
SandboxWarmingTimeoutError,
|
|
27
|
+
} from "../../sandbox-resume";
|
|
28
|
+
import {
|
|
29
|
+
type SandboxLogicalProvisionCategory,
|
|
30
|
+
type SandboxLogicalProvisionStage,
|
|
31
|
+
} from "../../observability-metrics";
|
|
32
|
+
import { randomUUID } from "node:crypto";
|
|
33
|
+
|
|
34
|
+
export type TurnSandboxProvisioner<T> = {
|
|
35
|
+
get(): Promise<T>;
|
|
36
|
+
hasStarted(): boolean;
|
|
37
|
+
waitForSettled(timeoutMs: number): Promise<T | null>;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
export type SandboxLogicalProvisionFailure = {
|
|
41
|
+
category: SandboxLogicalProvisionCategory;
|
|
42
|
+
stage: SandboxLogicalProvisionStage;
|
|
43
|
+
code: string;
|
|
44
|
+
expected: boolean;
|
|
45
|
+
retryable: boolean;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
export type TurnSandboxProvisionAttempt<T = unknown> = {
|
|
49
|
+
provisionId: string;
|
|
50
|
+
attempt: number;
|
|
51
|
+
outcome: "completed" | "retrying" | "failed";
|
|
52
|
+
durationMs: number;
|
|
53
|
+
result?: T;
|
|
54
|
+
error?: unknown;
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
export type TurnSandboxProvisionSettlement = {
|
|
58
|
+
provisionId: string;
|
|
59
|
+
internalAttempts: number;
|
|
60
|
+
durationMs: number;
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
export class TurnOperationCancelledError extends Error {
|
|
64
|
+
readonly name = "TurnOperationCancelledError";
|
|
65
|
+
|
|
66
|
+
constructor(readonly reason: unknown) {
|
|
67
|
+
super("Turn operation was cancelled with its owning turn", {
|
|
68
|
+
...(reason instanceof Error ? { cause: reason } : {}),
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export class SandboxDeadlineRotationError extends Error {
|
|
74
|
+
readonly name = "SandboxDeadlineRotationError";
|
|
75
|
+
|
|
76
|
+
constructor(
|
|
77
|
+
readonly sandboxGroupId: string,
|
|
78
|
+
readonly leaseEpoch: number,
|
|
79
|
+
) {
|
|
80
|
+
super(
|
|
81
|
+
`Sandbox ${sandboxGroupId} reached its provider rotation boundary at lease epoch ${leaseEpoch}`,
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Normalize a preparation/provisioning cancellation race back to the Temporal
|
|
88
|
+
* cancellation that owns the activity. Several provider APIs expose no portable
|
|
89
|
+
* abort primitive, so the worker stops awaiting them and disposes any late
|
|
90
|
+
* resource. The wrapper error must never fall through as an ordinary turn
|
|
91
|
+
* failure: doing so would omit the quiescence receipt and strand a committed
|
|
92
|
+
* Steer/Pause behind `control-pending`.
|
|
93
|
+
*/
|
|
94
|
+
export function turnOperationCancellationFailure(error: unknown): CancelledFailure | null {
|
|
95
|
+
if (error instanceof CancelledFailure) return error;
|
|
96
|
+
if (!(error instanceof TurnOperationCancelledError)) return null;
|
|
97
|
+
return error.reason instanceof CancelledFailure
|
|
98
|
+
? error.reason
|
|
99
|
+
: new CancelledFailure("TURN_SANDBOX_PROVISION_CANCELLED", [], error);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export function throwIfTurnOperationCancelled(signal: AbortSignal | undefined): void {
|
|
103
|
+
if (signal?.aborted) {
|
|
104
|
+
throw new TurnOperationCancelledError(signal.reason);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
export async function waitForTurnOperation<T>(
|
|
109
|
+
operation: Promise<T>,
|
|
110
|
+
signal: AbortSignal | undefined,
|
|
111
|
+
disposeLateResult: ((result: T) => Promise<void> | void) | undefined,
|
|
112
|
+
): Promise<T> {
|
|
113
|
+
if (!signal) return await operation;
|
|
114
|
+
|
|
115
|
+
let rejectCancellation: ((error: TurnOperationCancelledError) => void) | undefined;
|
|
116
|
+
const cancelled = new Promise<never>((_resolve, reject) => {
|
|
117
|
+
rejectCancellation = reject;
|
|
118
|
+
});
|
|
119
|
+
const cancel = (): void => {
|
|
120
|
+
rejectCancellation?.(new TurnOperationCancelledError(signal.reason));
|
|
121
|
+
};
|
|
122
|
+
signal.addEventListener("abort", cancel, { once: true });
|
|
123
|
+
if (signal.aborted) cancel();
|
|
124
|
+
|
|
125
|
+
try {
|
|
126
|
+
const result = await Promise.race([operation, cancelled]);
|
|
127
|
+
// Cancellation owns an exact turn boundary even when the provider result
|
|
128
|
+
// and AbortSignal settle in the same microtask checkpoint. Never let a
|
|
129
|
+
// just-resolved lease escape after the control was already committed.
|
|
130
|
+
if (signal.aborted) {
|
|
131
|
+
throw new TurnOperationCancelledError(signal.reason);
|
|
132
|
+
}
|
|
133
|
+
return result;
|
|
134
|
+
} catch (error) {
|
|
135
|
+
if (signal.aborted) {
|
|
136
|
+
// The provider establish call has no universal cancellation seam. It may
|
|
137
|
+
// finish after the Temporal activity has correctly stopped; dispose its
|
|
138
|
+
// late lease instead of letting a cancelled turn resurrect a holder/box.
|
|
139
|
+
void operation
|
|
140
|
+
.then(async (result) => await disposeLateResult?.(result))
|
|
141
|
+
.catch(() => undefined);
|
|
142
|
+
// A provider failure may settle in the same checkpoint as the committed
|
|
143
|
+
// control. The control is authoritative; retain its cancellation shape
|
|
144
|
+
// so the activity publishes quiescence instead of looking like an
|
|
145
|
+
// unrelated turn failure.
|
|
146
|
+
if (!(error instanceof TurnOperationCancelledError)) {
|
|
147
|
+
throw new TurnOperationCancelledError(signal.reason);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
throw error;
|
|
151
|
+
} finally {
|
|
152
|
+
signal.removeEventListener("abort", cancel);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export function sleep(ms: number): Promise<void> {
|
|
157
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export function sandboxProvisionErrorChain(error: unknown): unknown[] {
|
|
161
|
+
const chain: unknown[] = [];
|
|
162
|
+
const seen = new Set<object>();
|
|
163
|
+
let current = error;
|
|
164
|
+
for (let depth = 0; depth < 6; depth += 1) {
|
|
165
|
+
chain.push(current);
|
|
166
|
+
if (!current || typeof current !== "object" || seen.has(current)) break;
|
|
167
|
+
seen.add(current);
|
|
168
|
+
if (current instanceof SandboxProvisionStageError) {
|
|
169
|
+
current = current.source;
|
|
170
|
+
continue;
|
|
171
|
+
}
|
|
172
|
+
const cause = Reflect.get(current, "cause");
|
|
173
|
+
if (cause === undefined || cause === current) break;
|
|
174
|
+
current = cause;
|
|
175
|
+
}
|
|
176
|
+
return chain;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
export function firstProvisionError<T>(
|
|
180
|
+
chain: readonly unknown[],
|
|
181
|
+
constructor: abstract new (...args: never[]) => T,
|
|
182
|
+
): T | null {
|
|
183
|
+
return chain.find((value): value is T => value instanceof constructor) ?? null;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Closed, structural taxonomy for one logical provision failure. The classifier
|
|
188
|
+
* follows only typed wrapper/cause links and provider status/code fields; it
|
|
189
|
+
* never infers lifecycle truth from an arbitrary error message.
|
|
190
|
+
*/
|
|
191
|
+
export function classifySandboxLogicalProvisionFailure(
|
|
192
|
+
backend: string,
|
|
193
|
+
error: unknown,
|
|
194
|
+
): SandboxLogicalProvisionFailure {
|
|
195
|
+
const chain = sandboxProvisionErrorChain(error);
|
|
196
|
+
const wrappedStage =
|
|
197
|
+
chain.map((value) => sandboxProvisionFailureStage(value)).find((stage) => stage !== null) ??
|
|
198
|
+
null;
|
|
199
|
+
const warming =
|
|
200
|
+
chain.find(
|
|
201
|
+
(value): value is SandboxWarmingTimeoutError => value instanceof SandboxWarmingTimeoutError,
|
|
202
|
+
) ?? null;
|
|
203
|
+
if (warming) {
|
|
204
|
+
const execReadiness = warming.stage === "exec_readiness";
|
|
205
|
+
return {
|
|
206
|
+
category: execReadiness ? "exec_readiness" : "sibling_warming",
|
|
207
|
+
stage: execReadiness ? "exec_readiness" : "sibling_warming",
|
|
208
|
+
code: warming.code,
|
|
209
|
+
expected: false,
|
|
210
|
+
retryable: false,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
const readiness = firstProvisionError(chain, SandboxExecReadinessError);
|
|
214
|
+
if (readiness) {
|
|
215
|
+
return {
|
|
216
|
+
category: "exec_readiness",
|
|
217
|
+
stage: "exec_readiness",
|
|
218
|
+
code: readiness.code,
|
|
219
|
+
expected: false,
|
|
220
|
+
retryable: false,
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
const superseded = firstProvisionError(chain, SandboxLeaseSupersededError);
|
|
224
|
+
if (superseded) {
|
|
225
|
+
return {
|
|
226
|
+
category: "lease_superseded",
|
|
227
|
+
stage: "lease_admission",
|
|
228
|
+
code: "lease_superseded",
|
|
229
|
+
expected: true,
|
|
230
|
+
retryable: true,
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
const transition = firstProvisionError(chain, SandboxLeaseTransitionError);
|
|
234
|
+
if (transition) {
|
|
235
|
+
return {
|
|
236
|
+
category: "drain_capture_wait",
|
|
237
|
+
stage: "lifecycle_wait",
|
|
238
|
+
code: transition.reason,
|
|
239
|
+
expected: true,
|
|
240
|
+
retryable: true,
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
const deadlineRotation = firstProvisionError(chain, SandboxDeadlineRotationError);
|
|
244
|
+
if (deadlineRotation) {
|
|
245
|
+
return {
|
|
246
|
+
category: "drain_capture_wait",
|
|
247
|
+
stage: "lifecycle_wait",
|
|
248
|
+
code: "provider_deadline_rotation",
|
|
249
|
+
expected: true,
|
|
250
|
+
retryable: false,
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
const recoveryBlocked = firstProvisionError(chain, SandboxLeaseRecoveryBlockedError);
|
|
254
|
+
if (recoveryBlocked) {
|
|
255
|
+
return {
|
|
256
|
+
category: "archive_recovery",
|
|
257
|
+
stage: "archive_recovery",
|
|
258
|
+
code: recoveryBlocked.code,
|
|
259
|
+
expected: false,
|
|
260
|
+
retryable: false,
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
const archive = firstProvisionError(chain, WorkspaceArchiveIntegrityError);
|
|
264
|
+
if (archive) {
|
|
265
|
+
return {
|
|
266
|
+
category: "archive_recovery",
|
|
267
|
+
stage: "archive_recovery",
|
|
268
|
+
code: archive.code,
|
|
269
|
+
expected: false,
|
|
270
|
+
retryable: archive.retryable,
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
const continuity = firstProvisionError(chain, SandboxProviderContinuityUnavailableError);
|
|
274
|
+
if (continuity) {
|
|
275
|
+
return {
|
|
276
|
+
category: "archive_recovery",
|
|
277
|
+
stage: "archive_recovery",
|
|
278
|
+
code: "provider_continuity_unavailable",
|
|
279
|
+
expected: false,
|
|
280
|
+
retryable: continuity.retryable,
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
const configuration = chain.find(
|
|
284
|
+
(value) =>
|
|
285
|
+
value instanceof SandboxConfigError ||
|
|
286
|
+
value instanceof SandboxProviderUnavailableError ||
|
|
287
|
+
value instanceof SandboxImageConflictError ||
|
|
288
|
+
value instanceof SandboxRigConflictError,
|
|
289
|
+
);
|
|
290
|
+
if (configuration) {
|
|
291
|
+
return {
|
|
292
|
+
category: "configuration",
|
|
293
|
+
stage: wrappedStage ?? "configuration",
|
|
294
|
+
code:
|
|
295
|
+
configuration instanceof SandboxImageConflictError
|
|
296
|
+
? "image_conflict"
|
|
297
|
+
: configuration instanceof SandboxRigConflictError
|
|
298
|
+
? "rig_conflict"
|
|
299
|
+
: configuration instanceof SandboxProviderUnavailableError
|
|
300
|
+
? "provider_unavailable"
|
|
301
|
+
: "sandbox_config",
|
|
302
|
+
expected: false,
|
|
303
|
+
retryable: false,
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
const resumeFailure = chain.find(
|
|
307
|
+
(value) =>
|
|
308
|
+
value instanceof SandboxResumeStateUnavailableError ||
|
|
309
|
+
value instanceof SandboxResumeIdentityMismatchError ||
|
|
310
|
+
value instanceof SandboxResumeIdentityUnavailableError ||
|
|
311
|
+
value instanceof SandboxExactResumeReplacedError ||
|
|
312
|
+
value instanceof SandboxExactResumeInstanceUnavailableError,
|
|
313
|
+
);
|
|
314
|
+
if (resumeFailure) {
|
|
315
|
+
return {
|
|
316
|
+
category: "resume",
|
|
317
|
+
stage: "resume",
|
|
318
|
+
code:
|
|
319
|
+
resumeFailure instanceof SandboxResumeStateUnavailableError
|
|
320
|
+
? "resume_state_unavailable"
|
|
321
|
+
: resumeFailure instanceof SandboxResumeIdentityMismatchError
|
|
322
|
+
? "resume_identity_mismatch"
|
|
323
|
+
: resumeFailure instanceof SandboxResumeIdentityUnavailableError
|
|
324
|
+
? "resume_identity_unavailable"
|
|
325
|
+
: resumeFailure instanceof SandboxExactResumeReplacedError
|
|
326
|
+
? "exact_resume_replaced"
|
|
327
|
+
: "exact_resume_instance_unavailable",
|
|
328
|
+
expected: false,
|
|
329
|
+
retryable: false,
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
const providerFailure = classifyProviderSandboxFailure(backend, error);
|
|
333
|
+
if (providerFailure.kind === "transient_transport") {
|
|
334
|
+
return {
|
|
335
|
+
category: "provider_transport",
|
|
336
|
+
stage: wrappedStage ?? "provider_transport",
|
|
337
|
+
code: providerFailure.diagnostic,
|
|
338
|
+
expected: false,
|
|
339
|
+
// A typed transport fault is attributable, but a create outcome may be
|
|
340
|
+
// ambiguous. Classification never licenses replay on its own.
|
|
341
|
+
retryable: false,
|
|
342
|
+
};
|
|
343
|
+
}
|
|
344
|
+
if (wrappedStage) {
|
|
345
|
+
return {
|
|
346
|
+
category: wrappedStage,
|
|
347
|
+
stage: wrappedStage,
|
|
348
|
+
code: `${wrappedStage}_failed`,
|
|
349
|
+
expected: false,
|
|
350
|
+
retryable: false,
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
return {
|
|
354
|
+
category: "unknown",
|
|
355
|
+
stage: "unknown",
|
|
356
|
+
code: "unknown",
|
|
357
|
+
expected: false,
|
|
358
|
+
retryable: false,
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
export function isLazySandboxProvisionRetryable(error: unknown): boolean {
|
|
363
|
+
const chain = sandboxProvisionErrorChain(error);
|
|
364
|
+
if (
|
|
365
|
+
chain.some(
|
|
366
|
+
(value) =>
|
|
367
|
+
value instanceof SandboxImageConflictError ||
|
|
368
|
+
value instanceof SandboxRigConflictError ||
|
|
369
|
+
value instanceof SandboxLeaseRecoveryBlockedError,
|
|
370
|
+
)
|
|
371
|
+
) {
|
|
372
|
+
return false;
|
|
373
|
+
}
|
|
374
|
+
const archive = firstProvisionError(chain, WorkspaceArchiveIntegrityError);
|
|
375
|
+
if (archive) {
|
|
376
|
+
return archive.retryable;
|
|
377
|
+
}
|
|
378
|
+
if (
|
|
379
|
+
chain.some(
|
|
380
|
+
(value) =>
|
|
381
|
+
value instanceof SandboxLeaseSupersededError ||
|
|
382
|
+
value instanceof SandboxLeaseTransitionError,
|
|
383
|
+
)
|
|
384
|
+
) {
|
|
385
|
+
return true;
|
|
386
|
+
}
|
|
387
|
+
if (chain.some((value) => value instanceof SandboxWarmingTimeoutError)) {
|
|
388
|
+
return false;
|
|
389
|
+
}
|
|
390
|
+
// Provider/transport text is not durable evidence that creation never
|
|
391
|
+
// happened. Retrying an ambiguous unknown here can create a second box; only
|
|
392
|
+
// typed, ownership-fenced lifecycle outcomes above are safe to replay.
|
|
393
|
+
return false;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/** Short workflow-visible anti-churn pacing after a lifecycle transition. The
|
|
397
|
+
* next acquire waits on the exact durable claim itself, so this is deliberately
|
|
398
|
+
* not a guessed snapshot-plus-schedules completion time. */
|
|
399
|
+
export function sandboxDeadlineRotationRecoveryDelayMs(
|
|
400
|
+
settings: Pick<Settings, "sandboxLeaseReaperPeriodMs">,
|
|
401
|
+
): number {
|
|
402
|
+
// The next attempt waits on the durable lifecycle claim itself. This delay is
|
|
403
|
+
// only anti-churn pacing, not an estimate of snapshot + future schedules.
|
|
404
|
+
return Math.min(5_000, settings.sandboxLeaseReaperPeriodMs);
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
export function createTurnSandboxProvisioner<T>(
|
|
408
|
+
establish: () => Promise<T>,
|
|
409
|
+
options: {
|
|
410
|
+
maxRetries?: number;
|
|
411
|
+
backoffMs?: number;
|
|
412
|
+
signal?: AbortSignal;
|
|
413
|
+
provisionIdFactory?: () => string;
|
|
414
|
+
onStarted?: (
|
|
415
|
+
settlement: Pick<TurnSandboxProvisionSettlement, "provisionId">,
|
|
416
|
+
) => Promise<void> | void;
|
|
417
|
+
onAttemptSettled?: (attempt: TurnSandboxProvisionAttempt<T>) => Promise<void> | void;
|
|
418
|
+
beforeCompleted?: (
|
|
419
|
+
result: T,
|
|
420
|
+
settlement: TurnSandboxProvisionSettlement,
|
|
421
|
+
) => Promise<void> | void;
|
|
422
|
+
onCompleted?: (result: T, settlement: TurnSandboxProvisionSettlement) => Promise<void> | void;
|
|
423
|
+
onFailed?: (error: unknown, settlement: TurnSandboxProvisionSettlement) => Promise<void> | void;
|
|
424
|
+
disposeResult?: (result: T) => Promise<void> | void;
|
|
425
|
+
} = {},
|
|
426
|
+
): TurnSandboxProvisioner<T> {
|
|
427
|
+
const maxRetries = options.maxRetries ?? 2;
|
|
428
|
+
const backoffMs = options.backoffMs ?? 250;
|
|
429
|
+
let memo: Promise<T> | null = null;
|
|
430
|
+
|
|
431
|
+
const run = async (
|
|
432
|
+
provisionId: string,
|
|
433
|
+
onAttempt: (attempt: number) => void,
|
|
434
|
+
): Promise<{ result: T; internalAttempts: number }> => {
|
|
435
|
+
let internalAttempts = 0;
|
|
436
|
+
while (true) {
|
|
437
|
+
internalAttempts += 1;
|
|
438
|
+
onAttempt(internalAttempts);
|
|
439
|
+
const startedAt = performance.now();
|
|
440
|
+
try {
|
|
441
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
442
|
+
const operation = establish();
|
|
443
|
+
const result = await waitForTurnOperation(operation, options.signal, options.disposeResult);
|
|
444
|
+
await options.onAttemptSettled?.({
|
|
445
|
+
provisionId,
|
|
446
|
+
attempt: internalAttempts,
|
|
447
|
+
outcome: "completed",
|
|
448
|
+
durationMs: performance.now() - startedAt,
|
|
449
|
+
result,
|
|
450
|
+
});
|
|
451
|
+
return { result, internalAttempts };
|
|
452
|
+
} catch (error) {
|
|
453
|
+
const retrying = !(
|
|
454
|
+
error instanceof TurnOperationCancelledError ||
|
|
455
|
+
internalAttempts > maxRetries ||
|
|
456
|
+
!isLazySandboxProvisionRetryable(error)
|
|
457
|
+
);
|
|
458
|
+
await options.onAttemptSettled?.({
|
|
459
|
+
provisionId,
|
|
460
|
+
attempt: internalAttempts,
|
|
461
|
+
outcome: retrying ? "retrying" : "failed",
|
|
462
|
+
durationMs: performance.now() - startedAt,
|
|
463
|
+
error,
|
|
464
|
+
});
|
|
465
|
+
if (!retrying) {
|
|
466
|
+
throw error;
|
|
467
|
+
}
|
|
468
|
+
await sleep(backoffMs * internalAttempts);
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
};
|
|
472
|
+
|
|
473
|
+
return {
|
|
474
|
+
get(): Promise<T> {
|
|
475
|
+
if (!memo) {
|
|
476
|
+
const provisionId = options.provisionIdFactory?.() ?? randomUUID();
|
|
477
|
+
memo = (async () => {
|
|
478
|
+
const startedAt = performance.now();
|
|
479
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
480
|
+
await options.onStarted?.({ provisionId });
|
|
481
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
482
|
+
let result: T | undefined;
|
|
483
|
+
let internalAttempts = 0;
|
|
484
|
+
let hasResult = false;
|
|
485
|
+
try {
|
|
486
|
+
const runResult = await run(provisionId, (attempt) => {
|
|
487
|
+
internalAttempts = attempt;
|
|
488
|
+
});
|
|
489
|
+
result = runResult.result;
|
|
490
|
+
internalAttempts = runResult.internalAttempts;
|
|
491
|
+
hasResult = true;
|
|
492
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
493
|
+
await options.beforeCompleted?.(result, {
|
|
494
|
+
provisionId,
|
|
495
|
+
internalAttempts,
|
|
496
|
+
durationMs: performance.now() - startedAt,
|
|
497
|
+
});
|
|
498
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
499
|
+
await options.onCompleted?.(result, {
|
|
500
|
+
provisionId,
|
|
501
|
+
internalAttempts,
|
|
502
|
+
durationMs: performance.now() - startedAt,
|
|
503
|
+
});
|
|
504
|
+
return result;
|
|
505
|
+
} catch (error) {
|
|
506
|
+
if (hasResult) {
|
|
507
|
+
await options.disposeResult?.(result as T);
|
|
508
|
+
}
|
|
509
|
+
if (!(error instanceof TurnOperationCancelledError)) {
|
|
510
|
+
await options.onFailed?.(error, {
|
|
511
|
+
provisionId,
|
|
512
|
+
internalAttempts: Math.max(1, internalAttempts),
|
|
513
|
+
durationMs: performance.now() - startedAt,
|
|
514
|
+
});
|
|
515
|
+
}
|
|
516
|
+
throw error;
|
|
517
|
+
}
|
|
518
|
+
})().catch((error) => {
|
|
519
|
+
// A terminal lease/archive/configuration result cannot change within
|
|
520
|
+
// this frozen turn. Keep its rejected promise memoized so later model
|
|
521
|
+
// tool calls observe the same typed result without re-entering
|
|
522
|
+
// provisioning, emitting another failure event, or creating another
|
|
523
|
+
// provider attempt. Only a retryable failure releases the single-flight
|
|
524
|
+
// for a later operation to re-read the lease/provider state.
|
|
525
|
+
if (isLazySandboxProvisionRetryable(error)) {
|
|
526
|
+
memo = null;
|
|
527
|
+
}
|
|
528
|
+
throw error;
|
|
529
|
+
});
|
|
530
|
+
}
|
|
531
|
+
return memo;
|
|
532
|
+
},
|
|
533
|
+
hasStarted(): boolean {
|
|
534
|
+
return memo !== null;
|
|
535
|
+
},
|
|
536
|
+
async waitForSettled(timeoutMs: number): Promise<T | null> {
|
|
537
|
+
if (!memo) {
|
|
538
|
+
return null;
|
|
539
|
+
}
|
|
540
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
541
|
+
try {
|
|
542
|
+
return await Promise.race([
|
|
543
|
+
memo.catch(() => null),
|
|
544
|
+
new Promise<null>((resolve) => {
|
|
545
|
+
timer = setTimeout(() => resolve(null), timeoutMs);
|
|
546
|
+
}),
|
|
547
|
+
]);
|
|
548
|
+
} finally {
|
|
549
|
+
if (timer) {
|
|
550
|
+
clearTimeout(timer);
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
},
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
/**
|
|
558
|
+
* Decide whether THIS turn may send OpenAI's `prompt_cache_key` request field.
|
|
559
|
+
*
|
|
560
|
+
* Accepted transports:
|
|
561
|
+
* - legacy/built-in OpenAI or Azure Responses fallback (resolvedModel null);
|
|
562
|
+
* - resolved built-in OpenAI/Azure providers;
|
|
563
|
+
* - ChatGPT/Codex subscription backend (its strict allowlist permits the field).
|
|
564
|
+
*
|
|
565
|
+
* Registry API-key providers are intentionally excluded. Fireworks' prompt-cache
|
|
566
|
+
* docs prescribe `user` or `x-session-affinity`, not `prompt_cache_key`; Z.AI/GLM
|
|
567
|
+
* documents automatic context caching plus `user_id`. Sending OpenAI-only fields
|
|
568
|
+
* to unknown OpenAI-compatible providers risks unsupported-parameter 400s.
|
|
569
|
+
*/
|