@opengeni/worker-bundle 0.22.8 → 0.23.3-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/agent-build.d.ts +1 -4
- package/dist/activities/agent-turn/claim.d.ts +2 -4
- package/dist/activities/agent-turn/codex.d.ts +6 -5
- package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
- package/dist/activities/agent-turn/errors.d.ts +13 -0
- package/dist/activities/agent-turn/finalization.d.ts +1 -3
- package/dist/activities/agent-turn/quiescence.d.ts +39 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
- package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +3 -32
- package/dist/activities/agent-turn/turn-context.d.ts +1 -8
- package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
- package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
- package/dist/activities/codex-rotation.d.ts +2 -0
- package/dist/activities/session-state.d.ts +2 -1
- package/dist/{activities-control-VZIIV7RX.js → activities-control-52W5IDWM.js} +125 -13
- package/dist/activities-control-52W5IDWM.js.map +1 -0
- package/dist/{activities-turn-5RGQDBFV.js → activities-turn-A2G2KLSY.js} +534 -482
- package/dist/activities-turn-A2G2KLSY.js.map +1 -0
- package/dist/{chunk-NAVSAZ56.js → chunk-GFNXVZVQ.js} +160 -1
- package/dist/chunk-GFNXVZVQ.js.map +1 -0
- package/dist/{chunk-O4HDX5SC.js → chunk-IPVMCAW5.js} +144 -15
- package/dist/chunk-IPVMCAW5.js.map +1 -0
- package/dist/index.js +42 -2
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +28 -0
- package/dist/personal-github-git-credentials.d.ts +1 -0
- package/dist/unhandled-rejection-boundary.d.ts +29 -0
- package/package.json +18 -18
- package/src/activities/agent-turn/agent-build.ts +15 -31
- package/src/activities/agent-turn/claim.ts +29 -68
- package/src/activities/agent-turn/codex.ts +19 -10
- package/src/activities/agent-turn/compaction-prep.ts +50 -1
- package/src/activities/agent-turn/errors.ts +78 -0
- package/src/activities/agent-turn/failure-settlement.ts +30 -19
- package/src/activities/agent-turn/finalization.ts +24 -21
- package/src/activities/agent-turn/history.ts +5 -1
- package/src/activities/agent-turn/quiescence.ts +86 -0
- package/src/activities/agent-turn/run-credentials.ts +12 -1
- package/src/activities/agent-turn/run.ts +14 -11
- package/src/activities/agent-turn/sandbox-establish.ts +46 -11
- package/src/activities/agent-turn/sandbox-route.ts +21 -25
- package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
- package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
- package/src/activities/agent-turn/stream-attempt.ts +98 -14
- package/src/activities/agent-turn/tool-environment.ts +10 -13
- package/src/activities/agent-turn/tool-policy.ts +3 -55
- package/src/activities/agent-turn/turn-context.ts +8 -14
- package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
- package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
- package/src/activities/codex-capacity.ts +13 -5
- package/src/activities/codex-rotation.ts +17 -0
- package/src/activities/goals.ts +10 -4
- package/src/activities/retained-screenshots.ts +104 -21
- package/src/activities/sandbox-lease.ts +7 -4
- package/src/activities/scheduled-tasks.ts +128 -4
- package/src/activities/session-state.ts +8 -1
- package/src/github-rest-mcp.ts +6 -3
- package/src/http.ts +3 -0
- package/src/index.ts +19 -0
- package/src/observability-metrics.ts +200 -0
- package/src/personal-github-git-credentials.ts +45 -4
- package/src/sandbox-resume.ts +1 -1
- package/src/sandbox-routing.ts +7 -4
- package/src/unhandled-rejection-boundary.ts +73 -0
- package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
- package/dist/activities/recording.d.ts +0 -90
- package/dist/activities-control-VZIIV7RX.js.map +0 -1
- package/dist/activities-turn-5RGQDBFV.js.map +0 -1
- package/dist/chunk-NAVSAZ56.js.map +0 -1
- package/dist/chunk-O4HDX5SC.js.map +0 -1
- package/src/activities/recording.ts +0 -279
|
@@ -3,6 +3,7 @@ import {
|
|
|
3
3
|
ApprovalRunStateLimitExceededError,
|
|
4
4
|
isRetryableDatabaseTransportFailure,
|
|
5
5
|
isSessionEventPersistenceError,
|
|
6
|
+
SandboxLeaseTransitionError,
|
|
6
7
|
} from "@opengeni/db";
|
|
7
8
|
import {
|
|
8
9
|
ActiveBackendUnresolvableError,
|
|
@@ -10,6 +11,7 @@ import {
|
|
|
10
11
|
EmptyCompactionSummaryError,
|
|
11
12
|
isMcpRequestTimeoutError,
|
|
12
13
|
isMcpTransportConnectivityError,
|
|
14
|
+
isModalTaskExecStartDnsResolutionError,
|
|
13
15
|
RoutingWorkspaceRootChangedError,
|
|
14
16
|
SelfhostedWorkspaceRootChangedError,
|
|
15
17
|
UNKNOWN_MODEL_FINISH_REASON_CODE,
|
|
@@ -104,6 +106,7 @@ export function providerRecoveryResult(input: {
|
|
|
104
106
|
? (providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS)
|
|
105
107
|
: input.failureCode === "provider_unavailable" ||
|
|
106
108
|
input.failureCode === "upstream_connectivity_unavailable" ||
|
|
109
|
+
input.failureCode === "sandbox_command_start_unavailable" ||
|
|
107
110
|
input.failureCode === "mcp_transport_timeout" ||
|
|
108
111
|
input.failureCode === "mcp_transport_unavailable" ||
|
|
109
112
|
input.failureCode === POST_COMPACTION_CONTINUATION_EMPTY_CODE
|
|
@@ -151,6 +154,13 @@ export function providerRecoveryCountFromMetadata(metadata: Record<string, unkno
|
|
|
151
154
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
|
|
152
155
|
}
|
|
153
156
|
|
|
157
|
+
export function providerRecoveryCountAfterModelRequestPhase(
|
|
158
|
+
currentCount: number,
|
|
159
|
+
phase: string,
|
|
160
|
+
): number {
|
|
161
|
+
return phase === "completed" ? 0 : currentCount;
|
|
162
|
+
}
|
|
163
|
+
|
|
154
164
|
export function headerValue(headers: unknown, name: string): string | null {
|
|
155
165
|
if (!headers || typeof headers !== "object") return null;
|
|
156
166
|
const getter = (headers as { get?: unknown }).get;
|
|
@@ -367,6 +377,66 @@ export function isWorkerShutdownCancellation(error: unknown): boolean {
|
|
|
367
377
|
return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
|
|
368
378
|
}
|
|
369
379
|
|
|
380
|
+
export type SandboxLifecycleTransitionDiagnostic = {
|
|
381
|
+
sandboxGroupId: string;
|
|
382
|
+
leaseEpoch: number;
|
|
383
|
+
reason: "capture_in_progress" | "rotation_in_progress" | "provider_recovery_in_progress";
|
|
384
|
+
};
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* Recover a typed sandbox lifecycle transition through the structural wrappers
|
|
388
|
+
* used by parallel Agents SDK function-tool execution. Never infer transition
|
|
389
|
+
* truth from message text: only the original class or an exact, fully-shaped
|
|
390
|
+
* cross-package error object is accepted.
|
|
391
|
+
*/
|
|
392
|
+
export function sandboxLifecycleTransitionDiagnostic(
|
|
393
|
+
error: unknown,
|
|
394
|
+
): SandboxLifecycleTransitionDiagnostic | null {
|
|
395
|
+
const pending: unknown[] = [error];
|
|
396
|
+
const seen = new WeakSet<object>();
|
|
397
|
+
let inspected = 0;
|
|
398
|
+
|
|
399
|
+
while (pending.length > 0 && inspected < 64) {
|
|
400
|
+
const current = pending.shift();
|
|
401
|
+
inspected += 1;
|
|
402
|
+
if (!current || typeof current !== "object" || seen.has(current)) continue;
|
|
403
|
+
seen.add(current);
|
|
404
|
+
|
|
405
|
+
try {
|
|
406
|
+
const record = current as Record<string, unknown>;
|
|
407
|
+
const reason = record.reason;
|
|
408
|
+
if (
|
|
409
|
+
(current instanceof SandboxLeaseTransitionError ||
|
|
410
|
+
record.name === "SandboxLeaseTransitionError") &&
|
|
411
|
+
typeof record.sandboxGroupId === "string" &&
|
|
412
|
+
record.sandboxGroupId.length > 0 &&
|
|
413
|
+
typeof record.leaseEpoch === "number" &&
|
|
414
|
+
Number.isSafeInteger(record.leaseEpoch) &&
|
|
415
|
+
record.leaseEpoch >= 0 &&
|
|
416
|
+
(reason === "capture_in_progress" ||
|
|
417
|
+
reason === "rotation_in_progress" ||
|
|
418
|
+
reason === "provider_recovery_in_progress")
|
|
419
|
+
) {
|
|
420
|
+
return {
|
|
421
|
+
sandboxGroupId: record.sandboxGroupId,
|
|
422
|
+
leaseEpoch: record.leaseEpoch,
|
|
423
|
+
reason,
|
|
424
|
+
};
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
for (const key of ["cause", "error"] as const) {
|
|
428
|
+
const nested = record[key];
|
|
429
|
+
if (nested && typeof nested === "object") pending.push(nested);
|
|
430
|
+
}
|
|
431
|
+
if (Array.isArray(record.errors)) pending.push(...record.errors.slice(0, 32));
|
|
432
|
+
} catch {
|
|
433
|
+
// A hostile proxy or getter is not durable lifecycle evidence.
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
return null;
|
|
438
|
+
}
|
|
439
|
+
|
|
370
440
|
/**
|
|
371
441
|
* Recognize the one active-route transition that cannot finish inside its
|
|
372
442
|
* originating attempt. A Modal-home session may start on a Connected Machine
|
|
@@ -812,6 +882,14 @@ export function agentRunFailurePayload(
|
|
|
812
882
|
retryable: true,
|
|
813
883
|
};
|
|
814
884
|
}
|
|
885
|
+
if (isModalTaskExecStartDnsResolutionError(error)) {
|
|
886
|
+
return {
|
|
887
|
+
error:
|
|
888
|
+
"The managed sandbox command transport was temporarily unreachable before the command started. The same turn will retry after a short delay.",
|
|
889
|
+
code: "sandbox_command_start_unavailable",
|
|
890
|
+
retryable: true,
|
|
891
|
+
};
|
|
892
|
+
}
|
|
815
893
|
// An accepted Codex stream with no terminal response is malformed/partial,
|
|
816
894
|
// not provider backpressure. Replaying the same accepted turn could repeat
|
|
817
895
|
// model or tool effects, so this marked transport failure must outrank the
|
|
@@ -18,7 +18,6 @@ import {
|
|
|
18
18
|
settleCodexCredentialFailover,
|
|
19
19
|
readLease,
|
|
20
20
|
SandboxLeaseSupersededError,
|
|
21
|
-
SandboxLeaseTransitionError,
|
|
22
21
|
isSessionEventPersistenceError,
|
|
23
22
|
} from "@opengeni/db";
|
|
24
23
|
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
@@ -61,6 +60,7 @@ import {
|
|
|
61
60
|
escapedMcpTimeoutRecoveryFailure,
|
|
62
61
|
preClaimAdmissionFailure,
|
|
63
62
|
isWorkerShutdownCancellation,
|
|
63
|
+
sandboxLifecycleTransitionDiagnostic,
|
|
64
64
|
sandboxRouteTransitionCode,
|
|
65
65
|
safeErrorDiagnostic,
|
|
66
66
|
classifyXaiCredentialFailure,
|
|
@@ -159,20 +159,20 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
159
159
|
// recoverable control-plane states, never session failures. The latter is
|
|
160
160
|
// paced briefly; the next acquire waits on the durable claim and resumes
|
|
161
161
|
// immediately when it clears.
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
) {
|
|
162
|
+
const lifecycleTransition = sandboxLifecycleTransitionDiagnostic(error);
|
|
163
|
+
const leaseControlError =
|
|
164
|
+
error instanceof SandboxLeaseSupersededError ? error : lifecycleTransition;
|
|
165
|
+
if (leaseControlError && recoveryTurnId) {
|
|
167
166
|
try {
|
|
168
|
-
const fencedLease = await readLease(
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
167
|
+
const fencedLease = await readLease(
|
|
168
|
+
db,
|
|
169
|
+
input.workspaceId,
|
|
170
|
+
leaseControlError.sandboxGroupId,
|
|
171
|
+
).catch(() => null);
|
|
172
172
|
const rotationPending =
|
|
173
173
|
fencedLease?.rotationRequestedAt != null ||
|
|
174
|
-
|
|
175
|
-
const transitionPending = lifecycleTransition || rotationPending;
|
|
174
|
+
lifecycleTransition?.reason === "rotation_in_progress";
|
|
175
|
+
const transitionPending = lifecycleTransition !== null || rotationPending;
|
|
176
176
|
const deadlineRotationPending =
|
|
177
177
|
rotationPending && fencedLease?.rotationReason === "provider_deadline";
|
|
178
178
|
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
@@ -182,20 +182,20 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
182
182
|
attemptId: input.attemptId,
|
|
183
183
|
reason: deadlineRotationPending
|
|
184
184
|
? "sandbox_deadline_rotation"
|
|
185
|
-
: lifecycleTransition
|
|
185
|
+
: lifecycleTransition !== null
|
|
186
186
|
? "sandbox_lifecycle_transition"
|
|
187
187
|
: "sandbox_lease_superseded",
|
|
188
188
|
...(transitionPending
|
|
189
189
|
? {
|
|
190
190
|
detail: {
|
|
191
|
-
sandboxGroupId:
|
|
192
|
-
leaseEpoch:
|
|
191
|
+
sandboxGroupId: leaseControlError.sandboxGroupId,
|
|
192
|
+
leaseEpoch: leaseControlError.leaseEpoch,
|
|
193
193
|
...(rotationPending
|
|
194
194
|
? {
|
|
195
195
|
rotationReason: fencedLease?.rotationReason ?? "operator",
|
|
196
196
|
}
|
|
197
197
|
: {}),
|
|
198
|
-
...(lifecycleTransition ? { transitionReason:
|
|
198
|
+
...(lifecycleTransition ? { transitionReason: lifecycleTransition.reason } : {}),
|
|
199
199
|
},
|
|
200
200
|
}
|
|
201
201
|
: {}),
|
|
@@ -647,7 +647,11 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
647
647
|
status: "error",
|
|
648
648
|
lastError: "model request was forbidden for this credential",
|
|
649
649
|
}
|
|
650
|
-
: {
|
|
650
|
+
: {
|
|
651
|
+
kind: "cooldown",
|
|
652
|
+
until: cooldownUntil!,
|
|
653
|
+
cooldownKind: codexCredentialFailure.kind,
|
|
654
|
+
},
|
|
651
655
|
})
|
|
652
656
|
: false;
|
|
653
657
|
if (!statePersisted && leases.codex.holderId && leases.codex.generation !== null) {
|
|
@@ -903,6 +907,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
903
907
|
let rotationResumeIdleUntilReset = false; // circuit-breaker fall (Finding 1b) ⇒ MANDATORY hold
|
|
904
908
|
let allCappedResetAt: Date | null = null; // set ⇒ every account capped; idle until this
|
|
905
909
|
let capacityAuthoritativeResetAt: Date | null = null;
|
|
910
|
+
let capacityNeedsBoundedRefresh = false;
|
|
906
911
|
if (providerTurn.effectiveCodexCredentialId) {
|
|
907
912
|
const [rotation, sessionCodex] = await Promise.all([
|
|
908
913
|
getCodexRotationSettings(db, input.workspaceId).catch(() => null),
|
|
@@ -941,6 +946,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
941
946
|
input.workspaceId,
|
|
942
947
|
providerTurn.effectiveCodexCredentialId,
|
|
943
948
|
until,
|
|
949
|
+
"quota",
|
|
944
950
|
).catch(() => null);
|
|
945
951
|
const cooldownPersisted = cooldownMutation?.result ?? false;
|
|
946
952
|
if (cooldownMutation) {
|
|
@@ -953,7 +959,9 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
953
959
|
// write, so stamp the just-cooled account so the engine excludes it now. The
|
|
954
960
|
// serving account is thus walked AT MOST ONCE per turn (invariant 4: bounded).
|
|
955
961
|
const fresh = accounts.map((a) =>
|
|
956
|
-
a.id === providerTurn.effectiveCodexCredentialId
|
|
962
|
+
a.id === providerTurn.effectiveCodexCredentialId
|
|
963
|
+
? { ...a, exhaustedUntil: until, exhaustedKind: "quota" as const }
|
|
964
|
+
: a,
|
|
957
965
|
);
|
|
958
966
|
if (reactiveSharded) {
|
|
959
967
|
// AM-5: RE-SHARD over the healthy survivors (the just-capped serving account is
|
|
@@ -1017,6 +1025,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
1017
1025
|
rotated = true;
|
|
1018
1026
|
allCappedResetAt = earliestCodexReset(fresh, new Date());
|
|
1019
1027
|
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
|
|
1028
|
+
capacityNeedsBoundedRefresh = true;
|
|
1020
1029
|
}
|
|
1021
1030
|
} else {
|
|
1022
1031
|
const decision = chooseRotationActive({
|
|
@@ -1048,6 +1057,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
1048
1057
|
rotated = true;
|
|
1049
1058
|
allCappedResetAt = decision.earliestResetAt;
|
|
1050
1059
|
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
|
|
1060
|
+
capacityNeedsBoundedRefresh = true;
|
|
1051
1061
|
}
|
|
1052
1062
|
// kind:"none" → fall through to today's single-account idle.
|
|
1053
1063
|
}
|
|
@@ -1088,7 +1098,8 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
1088
1098
|
goalId: goalActive && goal ? goal.id : null,
|
|
1089
1099
|
goalVersion: goalActive && goal ? goal.version : null,
|
|
1090
1100
|
earliestResetAt: providerResetAt,
|
|
1091
|
-
resetKind:
|
|
1101
|
+
resetKind:
|
|
1102
|
+
providerResetAt && !capacityNeedsBoundedRefresh ? "authoritative" : "bounded_refresh",
|
|
1092
1103
|
failurePayload,
|
|
1093
1104
|
...(leases.codex.holderId && leases.codex.generation !== null
|
|
1094
1105
|
? {
|
|
@@ -23,7 +23,11 @@ import type {
|
|
|
23
23
|
import { captureWorkspaceRevision, openFreshWorkspaceCaptureSession } from "../workspace-capture";
|
|
24
24
|
import { type ChannelASession } from "@opengeni/runtime/sandbox";
|
|
25
25
|
import { createModelCheckpointMemoryCollector } from "../../model-checkpoint-memory-collector";
|
|
26
|
-
import {
|
|
26
|
+
import {
|
|
27
|
+
makeMachineOpObserver,
|
|
28
|
+
recordSessionEventAppendPhase,
|
|
29
|
+
turnLifecycleMetricsFor,
|
|
30
|
+
} from "../../observability-metrics";
|
|
27
31
|
import {
|
|
28
32
|
maybePersistWarmWorkspaceSnapshot,
|
|
29
33
|
waitForWarmSnapshot,
|
|
@@ -34,6 +38,7 @@ import { safeErrorDiagnostic, safeErrorForTelemetry } from "./errors";
|
|
|
34
38
|
import {
|
|
35
39
|
assertPhysicalToolQuiescenceForCancellation,
|
|
36
40
|
assertSessionAttemptQuiescenceRecoveryDurable,
|
|
41
|
+
armTurnQuiescenceWatchdog,
|
|
37
42
|
clearAttemptCredentialsWithSettledFence,
|
|
38
43
|
drainAttemptOwnedSandboxWriters,
|
|
39
44
|
persistOrSignalSessionAttemptQuiescence,
|
|
@@ -46,7 +51,6 @@ import type {
|
|
|
46
51
|
AttemptIdentityState,
|
|
47
52
|
EventingState,
|
|
48
53
|
ProviderTurnState,
|
|
49
|
-
RecordingState,
|
|
50
54
|
RenewalState,
|
|
51
55
|
SandboxRuntimeState,
|
|
52
56
|
TurnControlState,
|
|
@@ -76,7 +80,6 @@ export type TurnFinalizationDeps = {
|
|
|
76
80
|
attempt: AttemptIdentityState;
|
|
77
81
|
sandboxState: SandboxRuntimeState;
|
|
78
82
|
renewals: RenewalState;
|
|
79
|
-
recordingState: RecordingState;
|
|
80
83
|
eventing: EventingState;
|
|
81
84
|
providerTurn: ProviderTurnState;
|
|
82
85
|
workspaceRefs: WorkspaceRefState;
|
|
@@ -87,7 +90,6 @@ export type TurnFinalizationDeps = {
|
|
|
87
90
|
) => Promise<T>;
|
|
88
91
|
requireResolvedSandboxForMutation: (message: string) => ResumedTurnSandbox;
|
|
89
92
|
stopLeaseHeartbeat: () => void;
|
|
90
|
-
abandonActiveRecording: (reason: string, disposition?: "failed" | "discard") => Promise<void>;
|
|
91
93
|
turnCompletionMemoryCollector: ReturnType<typeof createModelCheckpointMemoryCollector>;
|
|
92
94
|
};
|
|
93
95
|
|
|
@@ -115,14 +117,12 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
|
|
|
115
117
|
attempt,
|
|
116
118
|
sandboxState,
|
|
117
119
|
renewals,
|
|
118
|
-
recordingState,
|
|
119
120
|
eventing,
|
|
120
121
|
providerTurn,
|
|
121
122
|
workspaceRefs,
|
|
122
123
|
runWorkspaceMutationForSandbox,
|
|
123
124
|
requireResolvedSandboxForMutation,
|
|
124
125
|
stopLeaseHeartbeat,
|
|
125
|
-
abandonActiveRecording,
|
|
126
126
|
turnCompletionMemoryCollector,
|
|
127
127
|
} = deps;
|
|
128
128
|
// This is the logical ownership boundary. Abort before any fallible
|
|
@@ -137,6 +137,17 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
|
|
|
137
137
|
let finalizationError: unknown;
|
|
138
138
|
let physicalToolQuiescenceConfirmed = !control.acknowledgeQuiescence;
|
|
139
139
|
let quiescenceReceiptOrProofDurable = !control.acknowledgeQuiescence;
|
|
140
|
+
const disarmQuiescenceWatchdog = armTurnQuiescenceWatchdog({
|
|
141
|
+
enabled: control.acknowledgeQuiescence,
|
|
142
|
+
onTimeout: () => {
|
|
143
|
+
observability.error("turn quiescence drain exceeded hard containment deadline", {
|
|
144
|
+
"opengeni.session_id": input.sessionId,
|
|
145
|
+
"opengeni.turn_id": attempt.turnId ?? "",
|
|
146
|
+
"opengeni.attempt_id": input.attemptId,
|
|
147
|
+
});
|
|
148
|
+
},
|
|
149
|
+
terminateWorker: () => process.exit(1),
|
|
150
|
+
});
|
|
140
151
|
const finalizerSignal = turnFinalizerCancellationSignal(
|
|
141
152
|
cancellationSignal,
|
|
142
153
|
control.activityStatus,
|
|
@@ -303,6 +314,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
|
|
|
303
314
|
},
|
|
304
315
|
});
|
|
305
316
|
quiescenceReceiptOrProofDurable = true;
|
|
317
|
+
disarmQuiescenceWatchdog();
|
|
306
318
|
if (recoveryMode === "signal") {
|
|
307
319
|
observability.info("agent turn quiescence proof handed to workflow recovery", {
|
|
308
320
|
"opengeni.session_id": input.sessionId,
|
|
@@ -332,6 +344,10 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
|
|
|
332
344
|
...event,
|
|
333
345
|
turnId: attempt.turnId ?? null,
|
|
334
346
|
})),
|
|
347
|
+
{
|
|
348
|
+
onAppendPhase: (observation) =>
|
|
349
|
+
recordSessionEventAppendPhase(observability, observation),
|
|
350
|
+
},
|
|
335
351
|
).catch(() => undefined),
|
|
336
352
|
finalizerSignal,
|
|
337
353
|
);
|
|
@@ -435,10 +451,8 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
|
|
|
435
451
|
// Workbench v2 turn-end workspace capture — runs FIRST in
|
|
436
452
|
// the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
|
|
437
453
|
// tool ran before this finally, so /workspace is already final; capture is
|
|
438
|
-
// FS-equivalent to the
|
|
439
|
-
//
|
|
440
|
-
// preparedTools.close() (which tears down tools / computer-use / the display
|
|
441
|
-
// stack and is what starts the Modal box exiting a few seconds later) —
|
|
454
|
+
// FS-equivalent to the warm snapshot (neither mutates workspace files).
|
|
455
|
+
// Running it here — BEFORE preparedTools.close() tears down tools —
|
|
442
456
|
// gives capture the full live-box margin instead of racing the teardown
|
|
443
457
|
// tail, which was dropping 100% of captures on real Modal desktop boxes
|
|
444
458
|
// ("request cancelled due to container exiting", 0 rows). External module:
|
|
@@ -556,17 +570,6 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
|
|
|
556
570
|
() => undefined,
|
|
557
571
|
);
|
|
558
572
|
}
|
|
559
|
-
// A recording normally closes inside the attempt-fenced turn settlement.
|
|
560
|
-
// Reaching finally with one still active means settlement threw, never ran,
|
|
561
|
-
// or lost ownership. Stop ffmpeg and mark only this exact attempt-owned row
|
|
562
|
-
// failed; publish no event and leave the artifact recoverable on the box.
|
|
563
|
-
await waitForTurnFinalizerStep(
|
|
564
|
-
abandonActiveRecording(
|
|
565
|
-
"activity ended without recording settlement",
|
|
566
|
-
recordingState.didComputerUse ? "failed" : "discard",
|
|
567
|
-
),
|
|
568
|
-
finalizerSignal,
|
|
569
|
-
);
|
|
570
573
|
if (sandboxState.resolvedSandbox) {
|
|
571
574
|
// TURN-END mid-session snapshot (sandbox-file-persistence): fold the
|
|
572
575
|
// turn's finished /workspace onto the lease before releasing the holder,
|
|
@@ -169,7 +169,11 @@ export function pendingToolCallFromSdkEvent(event: unknown): {
|
|
|
169
169
|
|
|
170
170
|
/** Tools whose intentional visual result must survive inline-media sanitization. */
|
|
171
171
|
export function toolCallProducesRetainableSessionImage(name: string | null): boolean {
|
|
172
|
-
return
|
|
172
|
+
return (
|
|
173
|
+
name === "computer_screenshot" ||
|
|
174
|
+
name === "view_image" ||
|
|
175
|
+
name === "interaction__computer_observe"
|
|
176
|
+
);
|
|
173
177
|
}
|
|
174
178
|
|
|
175
179
|
export function completedToolCallFromSdkEvent(event: unknown): {
|
|
@@ -79,6 +79,32 @@ export function assertSessionAttemptQuiescenceRecoveryDurable(input: {
|
|
|
79
79
|
|
|
80
80
|
export const QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS = 250;
|
|
81
81
|
export const QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS = 5_000;
|
|
82
|
+
export const TURN_QUIESCENCE_WATCHDOG_MS = 5 * 60_000;
|
|
83
|
+
|
|
84
|
+
/** A fenced activity must never heartbeat forever while its physical writer
|
|
85
|
+
* drain is stuck. The last-resort worker exit stops every in-process writer;
|
|
86
|
+
* Kubernetes replaces the pod and the workflow's heartbeat-lease reconciler
|
|
87
|
+
* then commits the exact quiescence receipt. */
|
|
88
|
+
export function armTurnQuiescenceWatchdog(input: {
|
|
89
|
+
enabled: boolean;
|
|
90
|
+
timeoutMs?: number;
|
|
91
|
+
onTimeout?: () => void;
|
|
92
|
+
terminateWorker: () => void;
|
|
93
|
+
}): () => void {
|
|
94
|
+
if (!input.enabled) return () => undefined;
|
|
95
|
+
let armed = true;
|
|
96
|
+
const timer = setTimeout(() => {
|
|
97
|
+
if (!armed) return;
|
|
98
|
+
input.onTimeout?.();
|
|
99
|
+
input.terminateWorker();
|
|
100
|
+
}, input.timeoutMs ?? TURN_QUIESCENCE_WATCHDOG_MS);
|
|
101
|
+
timer.unref?.();
|
|
102
|
+
return () => {
|
|
103
|
+
if (!armed) return;
|
|
104
|
+
armed = false;
|
|
105
|
+
clearTimeout(timer);
|
|
106
|
+
};
|
|
107
|
+
}
|
|
82
108
|
|
|
83
109
|
/** Persist the authoritative receipt or durably hand the exact physical proof
|
|
84
110
|
* to Temporal. This retries signal delivery, not DB eligibility or workflow
|
|
@@ -253,6 +279,66 @@ export async function waitForTurnStreamCleanup(
|
|
|
253
279
|
await waitForTurnFinalizerStep(providerCompleted, signal);
|
|
254
280
|
}
|
|
255
281
|
|
|
282
|
+
export class IncompleteAgentStreamError extends Error {
|
|
283
|
+
readonly name = "IncompleteAgentStreamError";
|
|
284
|
+
|
|
285
|
+
constructor(message = "Agent stream ended without a successful terminal output") {
|
|
286
|
+
super(message);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* Prove that SDK completion authority did not report a failure or runtime
|
|
292
|
+
* cancellation. The Agents SDK closes its ReadableStream immediately when its
|
|
293
|
+
* run signal aborts, so iterator EOF is not completion authority: the separate
|
|
294
|
+
* `completed` promise carries the eventual run failure.
|
|
295
|
+
*
|
|
296
|
+
* Only Temporal cancellation detaches cleanup. Internal runtime cancellation
|
|
297
|
+
* such as a provider-deadline sandbox rotation must escape to the turn failure
|
|
298
|
+
* settlement path, which checkpoints and redispatches the same logical turn.
|
|
299
|
+
*/
|
|
300
|
+
export async function assertSuccessfulAgentStreamCompletion(input: {
|
|
301
|
+
batcherFlush: Promise<unknown>;
|
|
302
|
+
stream: {
|
|
303
|
+
completed: Promise<void>;
|
|
304
|
+
error: unknown;
|
|
305
|
+
};
|
|
306
|
+
temporalCancellationSignal: AbortSignal | undefined;
|
|
307
|
+
runtimeCancellationSignal: AbortSignal;
|
|
308
|
+
}): Promise<void> {
|
|
309
|
+
await waitForTurnStreamCleanup(
|
|
310
|
+
input.batcherFlush,
|
|
311
|
+
input.stream.completed,
|
|
312
|
+
input.temporalCancellationSignal,
|
|
313
|
+
);
|
|
314
|
+
|
|
315
|
+
if (input.runtimeCancellationSignal.aborted) {
|
|
316
|
+
throw (
|
|
317
|
+
input.runtimeCancellationSignal.reason ??
|
|
318
|
+
new IncompleteAgentStreamError("Agent stream was cancelled without a terminal reason")
|
|
319
|
+
);
|
|
320
|
+
}
|
|
321
|
+
if (input.stream.error !== null && input.stream.error !== undefined) {
|
|
322
|
+
throw input.stream.error;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/** Call after any more-specific empty-stream recovery has had priority. */
|
|
327
|
+
export function assertAgentStreamNotCancelled(cancelled: boolean): void {
|
|
328
|
+
if (cancelled) {
|
|
329
|
+
throw new IncompleteAgentStreamError("Agent stream was cancelled without a terminal result");
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/** Interruption streams do not produce a normal final output. Call this only
|
|
334
|
+
* after the interruption branch has settled any required action. */
|
|
335
|
+
export function requireAgentStreamFinalOutput(finalOutput: unknown | undefined): unknown {
|
|
336
|
+
if (finalOutput === undefined) {
|
|
337
|
+
throw new IncompleteAgentStreamError();
|
|
338
|
+
}
|
|
339
|
+
return finalOutput;
|
|
340
|
+
}
|
|
341
|
+
|
|
256
342
|
/**
|
|
257
343
|
* Terminal settlement closes the active attempt before turn finalization. The
|
|
258
344
|
* ordinary workspace admission therefore correctly returns `attempt_fenced`
|
|
@@ -56,6 +56,7 @@ import {
|
|
|
56
56
|
sandboxEstablishPolicyDecision,
|
|
57
57
|
ensureTurnModalRegistryImage,
|
|
58
58
|
sandboxArtifactRuntimeAdmission,
|
|
59
|
+
sandboxSettingsForRoute,
|
|
59
60
|
} from "./sandbox-route";
|
|
60
61
|
import type { ClaimTurnOk } from "./claim";
|
|
61
62
|
import type { GovernanceModelOk } from "./governance-model";
|
|
@@ -134,6 +135,16 @@ export async function prepareRunCredentials(deps: PrepareRunCredentialsDeps) {
|
|
|
134
135
|
connectionScope,
|
|
135
136
|
runWorkspaceMutationForSandbox,
|
|
136
137
|
} = deps;
|
|
138
|
+
// A machine-home row carries selfhosted path defaults, but clearing its
|
|
139
|
+
// pointer explicitly selects the deployment-managed group. Build the stable
|
|
140
|
+
// manifest environment with that effective provider just as Channel-A and
|
|
141
|
+
// viewer readiness do; otherwise the verified managed box and the agent can
|
|
142
|
+
// disagree on backend-aware HOME/token-file paths.
|
|
143
|
+
const sandboxEnvironmentSettings = sandboxSettingsForRoute({
|
|
144
|
+
runSettings,
|
|
145
|
+
machinePrimary,
|
|
146
|
+
groupBoxBackend,
|
|
147
|
+
});
|
|
137
148
|
|
|
138
149
|
const runCredentialResolver =
|
|
139
150
|
effectiveRunCredentialBackend === "none"
|
|
@@ -321,7 +332,7 @@ export async function prepareRunCredentials(deps: PrepareRunCredentialsDeps) {
|
|
|
321
332
|
codemodeTokenExpiresAt: sandboxCodemodeTokenExpiresAt,
|
|
322
333
|
} = await waitForTurnOperation(
|
|
323
334
|
sandboxEnvironmentForRun(
|
|
324
|
-
|
|
335
|
+
sandboxEnvironmentSettings,
|
|
325
336
|
turnResources,
|
|
326
337
|
// Rig default sets merged BELOW the session set (session wins); rig-less
|
|
327
338
|
// turns pass exactly workspaceVariableSet?.values (byte-for-byte today).
|
|
@@ -45,6 +45,7 @@ import { createTurnCredentialLeases } from "./credential-leases";
|
|
|
45
45
|
import { createTurnMediaArtifacts } from "./media-artifacts";
|
|
46
46
|
import { createTurnHistorySink } from "./history-sink";
|
|
47
47
|
import { checkpointHistoryBeforeProviderDispatch } from "./provider-dispatch-barrier";
|
|
48
|
+
import { providerRecoveryCountAfterModelRequestPhase } from "./errors";
|
|
48
49
|
import { sandboxRunAs } from "@opengeni/runtime";
|
|
49
50
|
import { randomUUID } from "node:crypto";
|
|
50
51
|
import { createModelCheckpointMemoryCollector } from "../../model-checkpoint-memory-collector";
|
|
@@ -124,7 +125,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
124
125
|
billingState,
|
|
125
126
|
sandboxState,
|
|
126
127
|
renewals,
|
|
127
|
-
recordingState,
|
|
128
128
|
eventing,
|
|
129
129
|
workspaceRefs,
|
|
130
130
|
providerTurn,
|
|
@@ -222,7 +222,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
222
222
|
activityContext,
|
|
223
223
|
sandboxRotationController,
|
|
224
224
|
sandboxState,
|
|
225
|
-
recordingState,
|
|
226
225
|
eventing,
|
|
227
226
|
attempt,
|
|
228
227
|
});
|
|
@@ -230,9 +229,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
230
229
|
requireResolvedSandboxForMutation,
|
|
231
230
|
runWorkspaceMutationForSandbox,
|
|
232
231
|
stopLeaseHeartbeat,
|
|
233
|
-
abandonActiveRecording,
|
|
234
232
|
flushRuntimeBatcher,
|
|
235
|
-
maybeStartOnTurnRecording,
|
|
236
233
|
finalizeTurnOpStreamOps,
|
|
237
234
|
} = sandboxRuntime;
|
|
238
235
|
|
|
@@ -308,7 +305,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
308
305
|
db,
|
|
309
306
|
bus,
|
|
310
307
|
runtime,
|
|
311
|
-
objectStorage,
|
|
312
308
|
observability,
|
|
313
309
|
entitlements,
|
|
314
310
|
wakeSessionWorkflow,
|
|
@@ -320,13 +316,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
320
316
|
attempt,
|
|
321
317
|
billingState,
|
|
322
318
|
sandboxState,
|
|
323
|
-
recordingState,
|
|
324
319
|
eventing,
|
|
325
320
|
leases,
|
|
326
321
|
media,
|
|
327
322
|
claimedResult,
|
|
328
323
|
acknowledgeLostAttemptOwnership,
|
|
329
|
-
abandonActiveRecording,
|
|
330
324
|
});
|
|
331
325
|
if ("exit" in claimed) return claimed.exit;
|
|
332
326
|
if (!eventing.publish || !eventing.settle) {
|
|
@@ -350,6 +344,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
350
344
|
trigger,
|
|
351
345
|
humanInputResume,
|
|
352
346
|
interactionInterventionResume,
|
|
347
|
+
attachPendingUpdatesAfterOpenSuffix,
|
|
353
348
|
throwIfWorkerShuttingDown,
|
|
354
349
|
throwIfTurnCancelled,
|
|
355
350
|
opJournal,
|
|
@@ -605,6 +600,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
605
600
|
},
|
|
606
601
|
},
|
|
607
602
|
]);
|
|
603
|
+
attempt.providerRecoveryCount = providerRecoveryCountAfterModelRequestPhase(
|
|
604
|
+
attempt.providerRecoveryCount,
|
|
605
|
+
event.phase,
|
|
606
|
+
);
|
|
608
607
|
} catch (error) {
|
|
609
608
|
auditOutcome = "failed";
|
|
610
609
|
throw error;
|
|
@@ -750,6 +749,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
750
749
|
},
|
|
751
750
|
},
|
|
752
751
|
]);
|
|
752
|
+
attempt.providerRecoveryCount = providerRecoveryCountAfterModelRequestPhase(
|
|
753
|
+
attempt.providerRecoveryCount,
|
|
754
|
+
event.phase,
|
|
755
|
+
);
|
|
753
756
|
} catch (error) {
|
|
754
757
|
auditOutcome = "failed";
|
|
755
758
|
throw error;
|
|
@@ -834,6 +837,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
834
837
|
compactionModeOptions,
|
|
835
838
|
compactionOnlyTurn,
|
|
836
839
|
compactionSummarizerFor,
|
|
840
|
+
settleDeferredSteerAfterCompaction,
|
|
837
841
|
} = compactionPrep.ok;
|
|
838
842
|
|
|
839
843
|
const toolPolicy = await prepareTurnToolPolicy({
|
|
@@ -1112,8 +1116,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1112
1116
|
eventing,
|
|
1113
1117
|
attempt,
|
|
1114
1118
|
sandboxState,
|
|
1115
|
-
recordingState,
|
|
1116
|
-
maybeStartOnTurnRecording,
|
|
1117
1119
|
providerTurn,
|
|
1118
1120
|
media,
|
|
1119
1121
|
leases,
|
|
@@ -1278,6 +1280,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1278
1280
|
compactionModeOptions,
|
|
1279
1281
|
compactionOnlyTurn,
|
|
1280
1282
|
compactionSummarizerFor,
|
|
1283
|
+
settleDeferredSteerAfterCompaction,
|
|
1281
1284
|
agent,
|
|
1282
1285
|
});
|
|
1283
1286
|
if ("exit" in postAgentCompaction) return postAgentCompaction.exit;
|
|
@@ -1456,12 +1459,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1456
1459
|
turn,
|
|
1457
1460
|
trigger,
|
|
1458
1461
|
humanInputResume,
|
|
1462
|
+
attachPendingUpdatesAfterOpenSuffix,
|
|
1459
1463
|
agent,
|
|
1460
1464
|
resolvedModel,
|
|
1461
1465
|
providerApi,
|
|
1462
1466
|
runSettings,
|
|
1463
1467
|
turnTools,
|
|
1464
1468
|
compactSummarizer,
|
|
1469
|
+
settleDeferredSteerAfterCompaction,
|
|
1465
1470
|
compactionModelHistoryProjector,
|
|
1466
1471
|
generatedImageHistoryProjector,
|
|
1467
1472
|
modelHistoryProjector,
|
|
@@ -1525,14 +1530,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1525
1530
|
attempt,
|
|
1526
1531
|
sandboxState,
|
|
1527
1532
|
renewals,
|
|
1528
|
-
recordingState,
|
|
1529
1533
|
eventing,
|
|
1530
1534
|
providerTurn,
|
|
1531
1535
|
workspaceRefs,
|
|
1532
1536
|
runWorkspaceMutationForSandbox,
|
|
1533
1537
|
requireResolvedSandboxForMutation,
|
|
1534
1538
|
stopLeaseHeartbeat,
|
|
1535
|
-
abandonActiveRecording,
|
|
1536
1539
|
turnCompletionMemoryCollector,
|
|
1537
1540
|
});
|
|
1538
1541
|
}
|