@opengeni/worker-bundle 0.22.8 → 0.23.4-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +10 -2
- package/dist/activities/agent-turn/agent-build.d.ts +1 -4
- package/dist/activities/agent-turn/claim.d.ts +3 -4
- package/dist/activities/agent-turn/codex.d.ts +8 -7
- package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
- package/dist/activities/agent-turn/errors.d.ts +13 -0
- package/dist/activities/agent-turn/finalization.d.ts +1 -3
- package/dist/activities/agent-turn/list-models.d.ts +16 -0
- package/dist/activities/agent-turn/model-usage.d.ts +10 -1
- package/dist/activities/agent-turn/quiescence.d.ts +39 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
- package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +1 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +7 -5
- package/dist/activities/agent-turn/turn-context.d.ts +3 -8
- package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
- package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
- package/dist/activities/automations.d.ts +1 -0
- package/dist/activities/capabilities.d.ts +2 -1
- package/dist/activities/codex-rotation.d.ts +2 -0
- package/dist/activities/goals.d.ts +11 -0
- package/dist/activities/image-generation-references.d.ts +2 -1
- package/dist/activities/scheduled-tasks.d.ts +1 -0
- package/dist/activities/session-state.d.ts +2 -1
- package/dist/activities/types.d.ts +8 -0
- package/dist/{activities-control-VZIIV7RX.js → activities-control-SGU2WEVB.js} +328 -107
- package/dist/activities-control-SGU2WEVB.js.map +1 -0
- package/dist/activities-control.d.ts +2 -0
- package/dist/{activities-turn-5RGQDBFV.js → activities-turn-7EAQXW6Q.js} +828 -540
- package/dist/activities-turn-7EAQXW6Q.js.map +1 -0
- package/dist/activities.d.ts +2 -0
- package/dist/{chunk-O4HDX5SC.js → chunk-S2QVRVJW.js} +227 -36
- package/dist/chunk-S2QVRVJW.js.map +1 -0
- package/dist/{chunk-NAVSAZ56.js → chunk-TJIPCHMP.js} +162 -2
- package/dist/chunk-TJIPCHMP.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +69 -6
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +29 -0
- package/dist/personal-github-git-credentials.d.ts +1 -0
- package/dist/unhandled-rejection-boundary.d.ts +29 -0
- package/dist/workflow-bundle.js +26 -3
- package/dist/workflows/activities.d.ts +5 -0
- package/dist/workflows/automations.d.ts +5 -0
- package/package.json +18 -18
- package/src/activities/agent-run-admission.ts +7 -1
- package/src/activities/agent-turn/admission.ts +28 -6
- package/src/activities/agent-turn/agent-build.ts +15 -31
- package/src/activities/agent-turn/claim.ts +53 -72
- package/src/activities/agent-turn/codex.ts +21 -12
- package/src/activities/agent-turn/compaction-prep.ts +52 -1
- package/src/activities/agent-turn/errors.ts +78 -0
- package/src/activities/agent-turn/failure-settlement.ts +30 -19
- package/src/activities/agent-turn/finalization.ts +24 -21
- package/src/activities/agent-turn/history.ts +5 -1
- package/src/activities/agent-turn/list-models.ts +101 -0
- package/src/activities/agent-turn/model-usage.ts +88 -42
- package/src/activities/agent-turn/quiescence.ts +86 -0
- package/src/activities/agent-turn/run-credentials.ts +12 -1
- package/src/activities/agent-turn/run.ts +20 -14
- package/src/activities/agent-turn/sandbox-establish.ts +46 -11
- package/src/activities/agent-turn/sandbox-route.ts +21 -25
- package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
- package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
- package/src/activities/agent-turn/stream-attempt.ts +106 -14
- package/src/activities/agent-turn/tool-environment.ts +72 -14
- package/src/activities/agent-turn/tool-policy.ts +7 -6
- package/src/activities/agent-turn/turn-context.ts +12 -14
- package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
- package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
- package/src/activities/automations.ts +122 -67
- package/src/activities/capabilities.ts +72 -2
- package/src/activities/codex-capacity.ts +13 -5
- package/src/activities/codex-rotation.ts +17 -0
- package/src/activities/goals.ts +117 -35
- package/src/activities/image-generation-references.ts +9 -9
- package/src/activities/retained-screenshots.ts +104 -21
- package/src/activities/sandbox-lease.ts +7 -4
- package/src/activities/scheduled-tasks.ts +248 -30
- package/src/activities/session-state.ts +8 -1
- package/src/activities/site-auth-maintenance.ts +12 -12
- package/src/activities/types.ts +6 -0
- package/src/activities.ts +1 -0
- package/src/activity-services.ts +4 -1
- package/src/github-rest-mcp.ts +8 -4
- package/src/http.ts +3 -0
- package/src/index.ts +38 -4
- package/src/observability-metrics.ts +202 -0
- package/src/personal-github-git-credentials.ts +45 -4
- package/src/sandbox-resume.ts +1 -1
- package/src/sandbox-routing.ts +7 -4
- package/src/unhandled-rejection-boundary.ts +73 -0
- package/src/workflows/activities.ts +15 -0
- package/src/workflows/automations.ts +21 -2
- package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
- package/dist/activities/recording.d.ts +0 -90
- package/dist/activities-control-VZIIV7RX.js.map +0 -1
- package/dist/activities-turn-5RGQDBFV.js.map +0 -1
- package/dist/chunk-NAVSAZ56.js.map +0 -1
- package/dist/chunk-O4HDX5SC.js.map +0 -1
- package/src/activities/recording.ts +0 -279
|
@@ -3,8 +3,8 @@ import {
|
|
|
3
3
|
fetchCodexUsageForAccount,
|
|
4
4
|
type CodexAccountStatus,
|
|
5
5
|
} from "@opengeni/db";
|
|
6
|
-
import { type
|
|
7
|
-
import {
|
|
6
|
+
import { type ResolvedModelProvider, type Settings } from "@opengeni/config";
|
|
7
|
+
import { codexAccountNeedsLiveCapacityRefresh } from "../codex-rotation";
|
|
8
8
|
import {
|
|
9
9
|
refreshCodexUsageAndRepairCapacityWaiters,
|
|
10
10
|
signalPendingCodexCapacityWakeTargets,
|
|
@@ -19,7 +19,7 @@ export function codexWorkspaceMetricKey(workspaceId: string): string {
|
|
|
19
19
|
/** Stable public request identity across partial resumes and activity retries. */
|
|
20
20
|
export function acceptsPromptCacheKeyForTurn(
|
|
21
21
|
resolvedModel: {
|
|
22
|
-
provider: { kind:
|
|
22
|
+
provider: { kind: ResolvedModelProvider["kind"]; builtin?: boolean };
|
|
23
23
|
} | null,
|
|
24
24
|
): boolean {
|
|
25
25
|
if (!resolvedModel) {
|
|
@@ -37,9 +37,10 @@ export function acceptsPromptCacheKeyForTurn(
|
|
|
37
37
|
* would idle-loop forever. Before idling, refresh LIVE usage for every connected account
|
|
38
38
|
* the cache marks exhausted (bounded to the account count), which re-writes the
|
|
39
39
|
* cache columns, then return the re-read rows so the ranker can pick up a genuinely-reset
|
|
40
|
-
* window THIS turn.
|
|
41
|
-
*
|
|
42
|
-
*
|
|
40
|
+
* window THIS turn. Typed quota cooldowns are also refreshed because the provider may
|
|
41
|
+
* replace an allowance cycle before its original reset; revision fencing makes that
|
|
42
|
+
* repair safe. Generic backpressure and legacy-unknown cooldowns are never reconciled.
|
|
43
|
+
* A refresh/read failure is swallowed (fall back to the pre-refresh rows + bounded idle).
|
|
43
44
|
*/
|
|
44
45
|
export async function refreshCappedCodexUsageRows(
|
|
45
46
|
db: ActivityServices["db"],
|
|
@@ -48,7 +49,12 @@ export async function refreshCappedCodexUsageRows(
|
|
|
48
49
|
accounts: Array<
|
|
49
50
|
Pick<
|
|
50
51
|
CodexAccountStatus,
|
|
51
|
-
|
|
52
|
+
| "id"
|
|
53
|
+
| "status"
|
|
54
|
+
| "primaryUsedPercent"
|
|
55
|
+
| "secondaryUsedPercent"
|
|
56
|
+
| "exhaustedUntil"
|
|
57
|
+
| "exhaustedKind"
|
|
52
58
|
>
|
|
53
59
|
>,
|
|
54
60
|
capacitySignals: {
|
|
@@ -59,15 +65,18 @@ export async function refreshCappedCodexUsageRows(
|
|
|
59
65
|
Array<
|
|
60
66
|
Pick<
|
|
61
67
|
CodexAccountStatus,
|
|
62
|
-
|
|
68
|
+
| "id"
|
|
69
|
+
| "status"
|
|
70
|
+
| "primaryUsedPercent"
|
|
71
|
+
| "secondaryUsedPercent"
|
|
72
|
+
| "exhaustedUntil"
|
|
73
|
+
| "exhaustedKind"
|
|
63
74
|
>
|
|
64
75
|
>
|
|
65
76
|
> {
|
|
77
|
+
const now = new Date();
|
|
66
78
|
const stale = accounts.filter(
|
|
67
|
-
(
|
|
68
|
-
a.status === "active" &&
|
|
69
|
-
((a.primaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT ||
|
|
70
|
-
(a.secondaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT),
|
|
79
|
+
(account) => account.status === "active" && codexAccountNeedsLiveCapacityRefresh(account, now),
|
|
71
80
|
);
|
|
72
81
|
if (stale.length === 0) {
|
|
73
82
|
return accounts;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { isSessionCompactionRequested } from "@opengeni/db";
|
|
1
|
+
import { hasPendingSteerAfterContextCompaction, isSessionCompactionRequested } from "@opengeni/db";
|
|
2
2
|
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
3
3
|
import {
|
|
4
4
|
appendSessionInstructions,
|
|
@@ -102,6 +102,7 @@ export type CompactionPrepOk = {
|
|
|
102
102
|
compactionModeOptions: NonNullable<Parameters<typeof maybeCompactContext>[5]>;
|
|
103
103
|
compactionOnlyTurn: boolean;
|
|
104
104
|
compactionSummarizerFor: (systemInstructions?: string) => CompactionSummarizer;
|
|
105
|
+
settleDeferredSteerAfterCompaction: () => Promise<RunAgentTurnResult | null>;
|
|
105
106
|
};
|
|
106
107
|
|
|
107
108
|
export type CompactionPrepOutcome = { exit: RunAgentTurnResult } | { ok: CompactionPrepOk };
|
|
@@ -114,6 +115,7 @@ export type PostAgentCompactionDeps = CompactionPrepDeps & {
|
|
|
114
115
|
compactionModeOptions: CompactionPrepOk["compactionModeOptions"];
|
|
115
116
|
compactionOnlyTurn: boolean;
|
|
116
117
|
compactionSummarizerFor: CompactionPrepOk["compactionSummarizerFor"];
|
|
118
|
+
settleDeferredSteerAfterCompaction: CompactionPrepOk["settleDeferredSteerAfterCompaction"];
|
|
117
119
|
agent: ReturnType<ActivityServices["runtime"]["buildAgent"]>;
|
|
118
120
|
};
|
|
119
121
|
|
|
@@ -186,6 +188,8 @@ export async function prepareCompaction(deps: CompactionPrepDeps): Promise<Compa
|
|
|
186
188
|
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
187
189
|
model: resolvedModel?.configured.id ?? turn.model,
|
|
188
190
|
externallyBilled: billingState.isExternallyBilledTurn,
|
|
191
|
+
chargesOpenGeniCredits: billingState.chargesOpenGeniCredits,
|
|
192
|
+
countsTowardTokenCap: billingState.countsTowardTokenCap,
|
|
189
193
|
turnAttemptId: input.attemptId,
|
|
190
194
|
servingCredentialId: providerTurn.effectiveCodexCredentialId,
|
|
191
195
|
priorSessionCredentialId: providerTurn.priorSessionCodexCredentialId,
|
|
@@ -298,6 +302,39 @@ export async function prepareCompaction(deps: CompactionPrepDeps): Promise<Compa
|
|
|
298
302
|
const compactionOnlyTurn = turn.source === "compaction";
|
|
299
303
|
const remoteV2CompactionNeedsAgentPrefix =
|
|
300
304
|
Boolean(remoteCompactionRequester) && session.codexCompactionMode === "remote_v2";
|
|
305
|
+
const settleDeferredSteerAfterCompaction = async (): Promise<RunAgentTurnResult | null> => {
|
|
306
|
+
if (compactionOnlyTurn) return null;
|
|
307
|
+
const pending = await hasPendingSteerAfterContextCompaction(db, {
|
|
308
|
+
accountId: input.accountId,
|
|
309
|
+
workspaceId: input.workspaceId,
|
|
310
|
+
sessionId: input.sessionId,
|
|
311
|
+
turnId: turn.id,
|
|
312
|
+
executionGeneration: attempt.executionGeneration,
|
|
313
|
+
attemptId: input.attemptId,
|
|
314
|
+
});
|
|
315
|
+
if (!pending) return null;
|
|
316
|
+
if (
|
|
317
|
+
!(await eventing.settle({
|
|
318
|
+
events: [
|
|
319
|
+
{
|
|
320
|
+
type: "turn.superseded",
|
|
321
|
+
payload: { reason: "steer", deferredUntilCompaction: true },
|
|
322
|
+
},
|
|
323
|
+
{ type: "session.status.changed", payload: { status: "queued" } },
|
|
324
|
+
],
|
|
325
|
+
turnStatus: "superseded",
|
|
326
|
+
sessionStatus: "queued",
|
|
327
|
+
activeTurnId: null,
|
|
328
|
+
}))
|
|
329
|
+
) {
|
|
330
|
+
return claimedResult({ status: "cancelled" });
|
|
331
|
+
}
|
|
332
|
+
// Metrics currently group user-directed supersession with cancellation;
|
|
333
|
+
// the durable turn outcome remains the precise `superseded` truth.
|
|
334
|
+
control.turnMetricOutcome = "cancelled";
|
|
335
|
+
control.activityStatus = "idle";
|
|
336
|
+
return claimedResult({ status: "idle" });
|
|
337
|
+
};
|
|
301
338
|
if (compactionOnlyTurn && !remoteV2CompactionNeedsAgentPrefix) {
|
|
302
339
|
const compactionInstructions = appendWorkspaceMemory(
|
|
303
340
|
appendSessionInstructions(
|
|
@@ -443,6 +480,7 @@ export async function prepareCompaction(deps: CompactionPrepDeps): Promise<Compa
|
|
|
443
480
|
compactionModeOptions,
|
|
444
481
|
compactionOnlyTurn,
|
|
445
482
|
compactionSummarizerFor,
|
|
483
|
+
settleDeferredSteerAfterCompaction,
|
|
446
484
|
},
|
|
447
485
|
};
|
|
448
486
|
}
|
|
@@ -468,6 +506,7 @@ export async function runPostAgentCompaction(
|
|
|
468
506
|
compactionModeOptions,
|
|
469
507
|
compactionOnlyTurn,
|
|
470
508
|
compactionSummarizerFor,
|
|
509
|
+
settleDeferredSteerAfterCompaction,
|
|
471
510
|
compactionModelHistoryProjector,
|
|
472
511
|
media,
|
|
473
512
|
agent,
|
|
@@ -661,6 +700,16 @@ export async function runPostAgentCompaction(
|
|
|
661
700
|
}
|
|
662
701
|
await publishCompactionOutcomeEvents(outcome.events);
|
|
663
702
|
}
|
|
703
|
+
if (
|
|
704
|
+
outcome.events.some(
|
|
705
|
+
(event) =>
|
|
706
|
+
event.type === "session.context.compacted" ||
|
|
707
|
+
event.type === "session.context.compaction.skipped",
|
|
708
|
+
)
|
|
709
|
+
) {
|
|
710
|
+
const deferredSteer = await settleDeferredSteerAfterCompaction();
|
|
711
|
+
if (deferredSteer) return { exit: deferredSteer };
|
|
712
|
+
}
|
|
664
713
|
} catch (compactError) {
|
|
665
714
|
if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
|
|
666
715
|
if (compactError instanceof TurnAttemptFencedError) throw compactError;
|
|
@@ -680,6 +729,8 @@ export async function runPostAgentCompaction(
|
|
|
680
729
|
},
|
|
681
730
|
);
|
|
682
731
|
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
732
|
+
const deferredSteer = await settleDeferredSteerAfterCompaction();
|
|
733
|
+
if (deferredSteer) return { exit: deferredSteer };
|
|
683
734
|
const errorMessage = String(compactionFailureReasonFromError(compactError));
|
|
684
735
|
observability.error("context compaction failed", {
|
|
685
736
|
sessionId: input.sessionId,
|
|
@@ -3,6 +3,7 @@ import {
|
|
|
3
3
|
ApprovalRunStateLimitExceededError,
|
|
4
4
|
isRetryableDatabaseTransportFailure,
|
|
5
5
|
isSessionEventPersistenceError,
|
|
6
|
+
SandboxLeaseTransitionError,
|
|
6
7
|
} from "@opengeni/db";
|
|
7
8
|
import {
|
|
8
9
|
ActiveBackendUnresolvableError,
|
|
@@ -10,6 +11,7 @@ import {
|
|
|
10
11
|
EmptyCompactionSummaryError,
|
|
11
12
|
isMcpRequestTimeoutError,
|
|
12
13
|
isMcpTransportConnectivityError,
|
|
14
|
+
isModalTaskExecStartDnsResolutionError,
|
|
13
15
|
RoutingWorkspaceRootChangedError,
|
|
14
16
|
SelfhostedWorkspaceRootChangedError,
|
|
15
17
|
UNKNOWN_MODEL_FINISH_REASON_CODE,
|
|
@@ -104,6 +106,7 @@ export function providerRecoveryResult(input: {
|
|
|
104
106
|
? (providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS)
|
|
105
107
|
: input.failureCode === "provider_unavailable" ||
|
|
106
108
|
input.failureCode === "upstream_connectivity_unavailable" ||
|
|
109
|
+
input.failureCode === "sandbox_command_start_unavailable" ||
|
|
107
110
|
input.failureCode === "mcp_transport_timeout" ||
|
|
108
111
|
input.failureCode === "mcp_transport_unavailable" ||
|
|
109
112
|
input.failureCode === POST_COMPACTION_CONTINUATION_EMPTY_CODE
|
|
@@ -151,6 +154,13 @@ export function providerRecoveryCountFromMetadata(metadata: Record<string, unkno
|
|
|
151
154
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
|
|
152
155
|
}
|
|
153
156
|
|
|
157
|
+
export function providerRecoveryCountAfterModelRequestPhase(
|
|
158
|
+
currentCount: number,
|
|
159
|
+
phase: string,
|
|
160
|
+
): number {
|
|
161
|
+
return phase === "completed" ? 0 : currentCount;
|
|
162
|
+
}
|
|
163
|
+
|
|
154
164
|
export function headerValue(headers: unknown, name: string): string | null {
|
|
155
165
|
if (!headers || typeof headers !== "object") return null;
|
|
156
166
|
const getter = (headers as { get?: unknown }).get;
|
|
@@ -367,6 +377,66 @@ export function isWorkerShutdownCancellation(error: unknown): boolean {
|
|
|
367
377
|
return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
|
|
368
378
|
}
|
|
369
379
|
|
|
380
|
+
export type SandboxLifecycleTransitionDiagnostic = {
|
|
381
|
+
sandboxGroupId: string;
|
|
382
|
+
leaseEpoch: number;
|
|
383
|
+
reason: "capture_in_progress" | "rotation_in_progress" | "provider_recovery_in_progress";
|
|
384
|
+
};
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* Recover a typed sandbox lifecycle transition through the structural wrappers
|
|
388
|
+
* used by parallel Agents SDK function-tool execution. Never infer transition
|
|
389
|
+
* truth from message text: only the original class or an exact, fully-shaped
|
|
390
|
+
* cross-package error object is accepted.
|
|
391
|
+
*/
|
|
392
|
+
export function sandboxLifecycleTransitionDiagnostic(
|
|
393
|
+
error: unknown,
|
|
394
|
+
): SandboxLifecycleTransitionDiagnostic | null {
|
|
395
|
+
const pending: unknown[] = [error];
|
|
396
|
+
const seen = new WeakSet<object>();
|
|
397
|
+
let inspected = 0;
|
|
398
|
+
|
|
399
|
+
while (pending.length > 0 && inspected < 64) {
|
|
400
|
+
const current = pending.shift();
|
|
401
|
+
inspected += 1;
|
|
402
|
+
if (!current || typeof current !== "object" || seen.has(current)) continue;
|
|
403
|
+
seen.add(current);
|
|
404
|
+
|
|
405
|
+
try {
|
|
406
|
+
const record = current as Record<string, unknown>;
|
|
407
|
+
const reason = record.reason;
|
|
408
|
+
if (
|
|
409
|
+
(current instanceof SandboxLeaseTransitionError ||
|
|
410
|
+
record.name === "SandboxLeaseTransitionError") &&
|
|
411
|
+
typeof record.sandboxGroupId === "string" &&
|
|
412
|
+
record.sandboxGroupId.length > 0 &&
|
|
413
|
+
typeof record.leaseEpoch === "number" &&
|
|
414
|
+
Number.isSafeInteger(record.leaseEpoch) &&
|
|
415
|
+
record.leaseEpoch >= 0 &&
|
|
416
|
+
(reason === "capture_in_progress" ||
|
|
417
|
+
reason === "rotation_in_progress" ||
|
|
418
|
+
reason === "provider_recovery_in_progress")
|
|
419
|
+
) {
|
|
420
|
+
return {
|
|
421
|
+
sandboxGroupId: record.sandboxGroupId,
|
|
422
|
+
leaseEpoch: record.leaseEpoch,
|
|
423
|
+
reason,
|
|
424
|
+
};
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
for (const key of ["cause", "error"] as const) {
|
|
428
|
+
const nested = record[key];
|
|
429
|
+
if (nested && typeof nested === "object") pending.push(nested);
|
|
430
|
+
}
|
|
431
|
+
if (Array.isArray(record.errors)) pending.push(...record.errors.slice(0, 32));
|
|
432
|
+
} catch {
|
|
433
|
+
// A hostile proxy or getter is not durable lifecycle evidence.
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
return null;
|
|
438
|
+
}
|
|
439
|
+
|
|
370
440
|
/**
|
|
371
441
|
* Recognize the one active-route transition that cannot finish inside its
|
|
372
442
|
* originating attempt. A Modal-home session may start on a Connected Machine
|
|
@@ -812,6 +882,14 @@ export function agentRunFailurePayload(
|
|
|
812
882
|
retryable: true,
|
|
813
883
|
};
|
|
814
884
|
}
|
|
885
|
+
if (isModalTaskExecStartDnsResolutionError(error)) {
|
|
886
|
+
return {
|
|
887
|
+
error:
|
|
888
|
+
"The managed sandbox command transport was temporarily unreachable before the command started. The same turn will retry after a short delay.",
|
|
889
|
+
code: "sandbox_command_start_unavailable",
|
|
890
|
+
retryable: true,
|
|
891
|
+
};
|
|
892
|
+
}
|
|
815
893
|
// An accepted Codex stream with no terminal response is malformed/partial,
|
|
816
894
|
// not provider backpressure. Replaying the same accepted turn could repeat
|
|
817
895
|
// model or tool effects, so this marked transport failure must outrank the
|
|
@@ -18,7 +18,6 @@ import {
|
|
|
18
18
|
settleCodexCredentialFailover,
|
|
19
19
|
readLease,
|
|
20
20
|
SandboxLeaseSupersededError,
|
|
21
|
-
SandboxLeaseTransitionError,
|
|
22
21
|
isSessionEventPersistenceError,
|
|
23
22
|
} from "@opengeni/db";
|
|
24
23
|
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
@@ -61,6 +60,7 @@ import {
|
|
|
61
60
|
escapedMcpTimeoutRecoveryFailure,
|
|
62
61
|
preClaimAdmissionFailure,
|
|
63
62
|
isWorkerShutdownCancellation,
|
|
63
|
+
sandboxLifecycleTransitionDiagnostic,
|
|
64
64
|
sandboxRouteTransitionCode,
|
|
65
65
|
safeErrorDiagnostic,
|
|
66
66
|
classifyXaiCredentialFailure,
|
|
@@ -159,20 +159,20 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
159
159
|
// recoverable control-plane states, never session failures. The latter is
|
|
160
160
|
// paced briefly; the next acquire waits on the durable claim and resumes
|
|
161
161
|
// immediately when it clears.
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
) {
|
|
162
|
+
const lifecycleTransition = sandboxLifecycleTransitionDiagnostic(error);
|
|
163
|
+
const leaseControlError =
|
|
164
|
+
error instanceof SandboxLeaseSupersededError ? error : lifecycleTransition;
|
|
165
|
+
if (leaseControlError && recoveryTurnId) {
|
|
167
166
|
try {
|
|
168
|
-
const fencedLease = await readLease(
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
167
|
+
const fencedLease = await readLease(
|
|
168
|
+
db,
|
|
169
|
+
input.workspaceId,
|
|
170
|
+
leaseControlError.sandboxGroupId,
|
|
171
|
+
).catch(() => null);
|
|
172
172
|
const rotationPending =
|
|
173
173
|
fencedLease?.rotationRequestedAt != null ||
|
|
174
|
-
|
|
175
|
-
const transitionPending = lifecycleTransition || rotationPending;
|
|
174
|
+
lifecycleTransition?.reason === "rotation_in_progress";
|
|
175
|
+
const transitionPending = lifecycleTransition !== null || rotationPending;
|
|
176
176
|
const deadlineRotationPending =
|
|
177
177
|
rotationPending && fencedLease?.rotationReason === "provider_deadline";
|
|
178
178
|
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
@@ -182,20 +182,20 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
182
182
|
attemptId: input.attemptId,
|
|
183
183
|
reason: deadlineRotationPending
|
|
184
184
|
? "sandbox_deadline_rotation"
|
|
185
|
-
: lifecycleTransition
|
|
185
|
+
: lifecycleTransition !== null
|
|
186
186
|
? "sandbox_lifecycle_transition"
|
|
187
187
|
: "sandbox_lease_superseded",
|
|
188
188
|
...(transitionPending
|
|
189
189
|
? {
|
|
190
190
|
detail: {
|
|
191
|
-
sandboxGroupId:
|
|
192
|
-
leaseEpoch:
|
|
191
|
+
sandboxGroupId: leaseControlError.sandboxGroupId,
|
|
192
|
+
leaseEpoch: leaseControlError.leaseEpoch,
|
|
193
193
|
...(rotationPending
|
|
194
194
|
? {
|
|
195
195
|
rotationReason: fencedLease?.rotationReason ?? "operator",
|
|
196
196
|
}
|
|
197
197
|
: {}),
|
|
198
|
-
...(lifecycleTransition ? { transitionReason:
|
|
198
|
+
...(lifecycleTransition ? { transitionReason: lifecycleTransition.reason } : {}),
|
|
199
199
|
},
|
|
200
200
|
}
|
|
201
201
|
: {}),
|
|
@@ -647,7 +647,11 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
647
647
|
status: "error",
|
|
648
648
|
lastError: "model request was forbidden for this credential",
|
|
649
649
|
}
|
|
650
|
-
: {
|
|
650
|
+
: {
|
|
651
|
+
kind: "cooldown",
|
|
652
|
+
until: cooldownUntil!,
|
|
653
|
+
cooldownKind: codexCredentialFailure.kind,
|
|
654
|
+
},
|
|
651
655
|
})
|
|
652
656
|
: false;
|
|
653
657
|
if (!statePersisted && leases.codex.holderId && leases.codex.generation !== null) {
|
|
@@ -903,6 +907,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
903
907
|
let rotationResumeIdleUntilReset = false; // circuit-breaker fall (Finding 1b) ⇒ MANDATORY hold
|
|
904
908
|
let allCappedResetAt: Date | null = null; // set ⇒ every account capped; idle until this
|
|
905
909
|
let capacityAuthoritativeResetAt: Date | null = null;
|
|
910
|
+
let capacityNeedsBoundedRefresh = false;
|
|
906
911
|
if (providerTurn.effectiveCodexCredentialId) {
|
|
907
912
|
const [rotation, sessionCodex] = await Promise.all([
|
|
908
913
|
getCodexRotationSettings(db, input.workspaceId).catch(() => null),
|
|
@@ -941,6 +946,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
941
946
|
input.workspaceId,
|
|
942
947
|
providerTurn.effectiveCodexCredentialId,
|
|
943
948
|
until,
|
|
949
|
+
"quota",
|
|
944
950
|
).catch(() => null);
|
|
945
951
|
const cooldownPersisted = cooldownMutation?.result ?? false;
|
|
946
952
|
if (cooldownMutation) {
|
|
@@ -953,7 +959,9 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
953
959
|
// write, so stamp the just-cooled account so the engine excludes it now. The
|
|
954
960
|
// serving account is thus walked AT MOST ONCE per turn (invariant 4: bounded).
|
|
955
961
|
const fresh = accounts.map((a) =>
|
|
956
|
-
a.id === providerTurn.effectiveCodexCredentialId
|
|
962
|
+
a.id === providerTurn.effectiveCodexCredentialId
|
|
963
|
+
? { ...a, exhaustedUntil: until, exhaustedKind: "quota" as const }
|
|
964
|
+
: a,
|
|
957
965
|
);
|
|
958
966
|
if (reactiveSharded) {
|
|
959
967
|
// AM-5: RE-SHARD over the healthy survivors (the just-capped serving account is
|
|
@@ -1017,6 +1025,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
1017
1025
|
rotated = true;
|
|
1018
1026
|
allCappedResetAt = earliestCodexReset(fresh, new Date());
|
|
1019
1027
|
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
|
|
1028
|
+
capacityNeedsBoundedRefresh = true;
|
|
1020
1029
|
}
|
|
1021
1030
|
} else {
|
|
1022
1031
|
const decision = chooseRotationActive({
|
|
@@ -1048,6 +1057,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
1048
1057
|
rotated = true;
|
|
1049
1058
|
allCappedResetAt = decision.earliestResetAt;
|
|
1050
1059
|
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
|
|
1060
|
+
capacityNeedsBoundedRefresh = true;
|
|
1051
1061
|
}
|
|
1052
1062
|
// kind:"none" → fall through to today's single-account idle.
|
|
1053
1063
|
}
|
|
@@ -1088,7 +1098,8 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
1088
1098
|
goalId: goalActive && goal ? goal.id : null,
|
|
1089
1099
|
goalVersion: goalActive && goal ? goal.version : null,
|
|
1090
1100
|
earliestResetAt: providerResetAt,
|
|
1091
|
-
resetKind:
|
|
1101
|
+
resetKind:
|
|
1102
|
+
providerResetAt && !capacityNeedsBoundedRefresh ? "authoritative" : "bounded_refresh",
|
|
1092
1103
|
failurePayload,
|
|
1093
1104
|
...(leases.codex.holderId && leases.codex.generation !== null
|
|
1094
1105
|
? {
|
|
@@ -23,7 +23,11 @@ import type {
|
|
|
23
23
|
import { captureWorkspaceRevision, openFreshWorkspaceCaptureSession } from "../workspace-capture";
|
|
24
24
|
import { type ChannelASession } from "@opengeni/runtime/sandbox";
|
|
25
25
|
import { createModelCheckpointMemoryCollector } from "../../model-checkpoint-memory-collector";
|
|
26
|
-
import {
|
|
26
|
+
import {
|
|
27
|
+
makeMachineOpObserver,
|
|
28
|
+
recordSessionEventAppendPhase,
|
|
29
|
+
turnLifecycleMetricsFor,
|
|
30
|
+
} from "../../observability-metrics";
|
|
27
31
|
import {
|
|
28
32
|
maybePersistWarmWorkspaceSnapshot,
|
|
29
33
|
waitForWarmSnapshot,
|
|
@@ -34,6 +38,7 @@ import { safeErrorDiagnostic, safeErrorForTelemetry } from "./errors";
|
|
|
34
38
|
import {
|
|
35
39
|
assertPhysicalToolQuiescenceForCancellation,
|
|
36
40
|
assertSessionAttemptQuiescenceRecoveryDurable,
|
|
41
|
+
armTurnQuiescenceWatchdog,
|
|
37
42
|
clearAttemptCredentialsWithSettledFence,
|
|
38
43
|
drainAttemptOwnedSandboxWriters,
|
|
39
44
|
persistOrSignalSessionAttemptQuiescence,
|
|
@@ -46,7 +51,6 @@ import type {
|
|
|
46
51
|
AttemptIdentityState,
|
|
47
52
|
EventingState,
|
|
48
53
|
ProviderTurnState,
|
|
49
|
-
RecordingState,
|
|
50
54
|
RenewalState,
|
|
51
55
|
SandboxRuntimeState,
|
|
52
56
|
TurnControlState,
|
|
@@ -76,7 +80,6 @@ export type TurnFinalizationDeps = {
|
|
|
76
80
|
attempt: AttemptIdentityState;
|
|
77
81
|
sandboxState: SandboxRuntimeState;
|
|
78
82
|
renewals: RenewalState;
|
|
79
|
-
recordingState: RecordingState;
|
|
80
83
|
eventing: EventingState;
|
|
81
84
|
providerTurn: ProviderTurnState;
|
|
82
85
|
workspaceRefs: WorkspaceRefState;
|
|
@@ -87,7 +90,6 @@ export type TurnFinalizationDeps = {
|
|
|
87
90
|
) => Promise<T>;
|
|
88
91
|
requireResolvedSandboxForMutation: (message: string) => ResumedTurnSandbox;
|
|
89
92
|
stopLeaseHeartbeat: () => void;
|
|
90
|
-
abandonActiveRecording: (reason: string, disposition?: "failed" | "discard") => Promise<void>;
|
|
91
93
|
turnCompletionMemoryCollector: ReturnType<typeof createModelCheckpointMemoryCollector>;
|
|
92
94
|
};
|
|
93
95
|
|
|
@@ -115,14 +117,12 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
|
|
|
115
117
|
attempt,
|
|
116
118
|
sandboxState,
|
|
117
119
|
renewals,
|
|
118
|
-
recordingState,
|
|
119
120
|
eventing,
|
|
120
121
|
providerTurn,
|
|
121
122
|
workspaceRefs,
|
|
122
123
|
runWorkspaceMutationForSandbox,
|
|
123
124
|
requireResolvedSandboxForMutation,
|
|
124
125
|
stopLeaseHeartbeat,
|
|
125
|
-
abandonActiveRecording,
|
|
126
126
|
turnCompletionMemoryCollector,
|
|
127
127
|
} = deps;
|
|
128
128
|
// This is the logical ownership boundary. Abort before any fallible
|
|
@@ -137,6 +137,17 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
|
|
|
137
137
|
let finalizationError: unknown;
|
|
138
138
|
let physicalToolQuiescenceConfirmed = !control.acknowledgeQuiescence;
|
|
139
139
|
let quiescenceReceiptOrProofDurable = !control.acknowledgeQuiescence;
|
|
140
|
+
const disarmQuiescenceWatchdog = armTurnQuiescenceWatchdog({
|
|
141
|
+
enabled: control.acknowledgeQuiescence,
|
|
142
|
+
onTimeout: () => {
|
|
143
|
+
observability.error("turn quiescence drain exceeded hard containment deadline", {
|
|
144
|
+
"opengeni.session_id": input.sessionId,
|
|
145
|
+
"opengeni.turn_id": attempt.turnId ?? "",
|
|
146
|
+
"opengeni.attempt_id": input.attemptId,
|
|
147
|
+
});
|
|
148
|
+
},
|
|
149
|
+
terminateWorker: () => process.exit(1),
|
|
150
|
+
});
|
|
140
151
|
const finalizerSignal = turnFinalizerCancellationSignal(
|
|
141
152
|
cancellationSignal,
|
|
142
153
|
control.activityStatus,
|
|
@@ -303,6 +314,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
|
|
|
303
314
|
},
|
|
304
315
|
});
|
|
305
316
|
quiescenceReceiptOrProofDurable = true;
|
|
317
|
+
disarmQuiescenceWatchdog();
|
|
306
318
|
if (recoveryMode === "signal") {
|
|
307
319
|
observability.info("agent turn quiescence proof handed to workflow recovery", {
|
|
308
320
|
"opengeni.session_id": input.sessionId,
|
|
@@ -332,6 +344,10 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
|
|
|
332
344
|
...event,
|
|
333
345
|
turnId: attempt.turnId ?? null,
|
|
334
346
|
})),
|
|
347
|
+
{
|
|
348
|
+
onAppendPhase: (observation) =>
|
|
349
|
+
recordSessionEventAppendPhase(observability, observation),
|
|
350
|
+
},
|
|
335
351
|
).catch(() => undefined),
|
|
336
352
|
finalizerSignal,
|
|
337
353
|
);
|
|
@@ -435,10 +451,8 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
|
|
|
435
451
|
// Workbench v2 turn-end workspace capture — runs FIRST in
|
|
436
452
|
// the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
|
|
437
453
|
// tool ran before this finally, so /workspace is already final; capture is
|
|
438
|
-
// FS-equivalent to the
|
|
439
|
-
//
|
|
440
|
-
// preparedTools.close() (which tears down tools / computer-use / the display
|
|
441
|
-
// stack and is what starts the Modal box exiting a few seconds later) —
|
|
454
|
+
// FS-equivalent to the warm snapshot (neither mutates workspace files).
|
|
455
|
+
// Running it here — BEFORE preparedTools.close() tears down tools —
|
|
442
456
|
// gives capture the full live-box margin instead of racing the teardown
|
|
443
457
|
// tail, which was dropping 100% of captures on real Modal desktop boxes
|
|
444
458
|
// ("request cancelled due to container exiting", 0 rows). External module:
|
|
@@ -556,17 +570,6 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
|
|
|
556
570
|
() => undefined,
|
|
557
571
|
);
|
|
558
572
|
}
|
|
559
|
-
// A recording normally closes inside the attempt-fenced turn settlement.
|
|
560
|
-
// Reaching finally with one still active means settlement threw, never ran,
|
|
561
|
-
// or lost ownership. Stop ffmpeg and mark only this exact attempt-owned row
|
|
562
|
-
// failed; publish no event and leave the artifact recoverable on the box.
|
|
563
|
-
await waitForTurnFinalizerStep(
|
|
564
|
-
abandonActiveRecording(
|
|
565
|
-
"activity ended without recording settlement",
|
|
566
|
-
recordingState.didComputerUse ? "failed" : "discard",
|
|
567
|
-
),
|
|
568
|
-
finalizerSignal,
|
|
569
|
-
);
|
|
570
573
|
if (sandboxState.resolvedSandbox) {
|
|
571
574
|
// TURN-END mid-session snapshot (sandbox-file-persistence): fold the
|
|
572
575
|
// turn's finished /workspace onto the lease before releasing the holder,
|
|
@@ -169,7 +169,11 @@ export function pendingToolCallFromSdkEvent(event: unknown): {
|
|
|
169
169
|
|
|
170
170
|
/** Tools whose intentional visual result must survive inline-media sanitization. */
|
|
171
171
|
export function toolCallProducesRetainableSessionImage(name: string | null): boolean {
|
|
172
|
-
return
|
|
172
|
+
return (
|
|
173
|
+
name === "computer_screenshot" ||
|
|
174
|
+
name === "view_image" ||
|
|
175
|
+
name === "interaction__computer_observe"
|
|
176
|
+
);
|
|
173
177
|
}
|
|
174
178
|
|
|
175
179
|
export function completedToolCallFromSdkEvent(event: unknown): {
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import type { AttemptToolDefinition } from "@opengeni/codemode";
|
|
2
|
+
import {
|
|
3
|
+
compareModelPickerOrder,
|
|
4
|
+
modelPickerBillingClassFor,
|
|
5
|
+
} from "@opengeni/contracts/model-picker-order";
|
|
6
|
+
import type { WorkspaceModelSelection } from "@opengeni/core";
|
|
7
|
+
|
|
8
|
+
export const LIST_MODELS_TOOL_NAME = "list_models";
|
|
9
|
+
|
|
10
|
+
export const LIST_MODELS_TOOL_DESCRIPTION =
|
|
11
|
+
"List the models this workspace can select right now, in picker order, with deployment-defined cost and optional guidance. This does not switch the current session model; use an ID with session_create or the human model picker.";
|
|
12
|
+
|
|
13
|
+
function safeLineField(value: string): string {
|
|
14
|
+
return value
|
|
15
|
+
.replace(/[\r\n|]/gu, " ")
|
|
16
|
+
.replace(/\s+/gu, " ")
|
|
17
|
+
.trim();
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function renderListModels(input: {
|
|
21
|
+
currentModelId: string | null;
|
|
22
|
+
selections: readonly WorkspaceModelSelection[];
|
|
23
|
+
modelNotes: Readonly<Record<string, string>>;
|
|
24
|
+
}): string {
|
|
25
|
+
const lines: string[] = [];
|
|
26
|
+
if (input.currentModelId) {
|
|
27
|
+
lines.push(`Current: ${safeLineField(input.currentModelId)}`);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const selectable = input.selections
|
|
31
|
+
.filter((entry) => entry.availability.selectable)
|
|
32
|
+
.map((selection) => ({
|
|
33
|
+
selection,
|
|
34
|
+
billingClass: modelPickerBillingClassFor(selection.model),
|
|
35
|
+
selectable: selection.availability.selectable,
|
|
36
|
+
label: selection.model.label,
|
|
37
|
+
}))
|
|
38
|
+
.sort(compareModelPickerOrder);
|
|
39
|
+
if (selectable.length === 0) {
|
|
40
|
+
lines.push("No models are available in this workspace.");
|
|
41
|
+
return lines.join("\n");
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
for (const { selection } of selectable) {
|
|
45
|
+
const { model } = selection;
|
|
46
|
+
const fields = [safeLineField(model.id), safeLineField(model.label), model.cost];
|
|
47
|
+
const note = input.modelNotes[model.id];
|
|
48
|
+
if (note) fields.push(safeLineField(note));
|
|
49
|
+
lines.push(`- ${fields.join(" | ")}`);
|
|
50
|
+
}
|
|
51
|
+
return lines.join("\n");
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function createListModelsAttemptToolDefinition(input: {
|
|
55
|
+
currentModelId: string | null;
|
|
56
|
+
load: () => Promise<{
|
|
57
|
+
selections: readonly WorkspaceModelSelection[];
|
|
58
|
+
modelNotes: Readonly<Record<string, string>>;
|
|
59
|
+
}>;
|
|
60
|
+
}): AttemptToolDefinition {
|
|
61
|
+
return {
|
|
62
|
+
identity: { serverId: "opengeni", toolName: LIST_MODELS_TOOL_NAME },
|
|
63
|
+
modelName: LIST_MODELS_TOOL_NAME,
|
|
64
|
+
codemodePath: ["opengeni", LIST_MODELS_TOOL_NAME],
|
|
65
|
+
title: "List selectable models",
|
|
66
|
+
description: LIST_MODELS_TOOL_DESCRIPTION,
|
|
67
|
+
inputSchema: {
|
|
68
|
+
type: "object",
|
|
69
|
+
properties: {},
|
|
70
|
+
additionalProperties: false,
|
|
71
|
+
},
|
|
72
|
+
annotations: {
|
|
73
|
+
title: "List selectable models",
|
|
74
|
+
readOnlyHint: true,
|
|
75
|
+
destructiveHint: false,
|
|
76
|
+
idempotentHint: true,
|
|
77
|
+
openWorldHint: false,
|
|
78
|
+
},
|
|
79
|
+
source: "opengeni",
|
|
80
|
+
approval: "none",
|
|
81
|
+
execute: async (args) => {
|
|
82
|
+
if (Object.keys(args).length !== 0) {
|
|
83
|
+
throw new Error("list_models accepts no arguments");
|
|
84
|
+
}
|
|
85
|
+
const catalog = await input.load();
|
|
86
|
+
return {
|
|
87
|
+
isError: false,
|
|
88
|
+
content: [
|
|
89
|
+
{
|
|
90
|
+
type: "text",
|
|
91
|
+
text: renderListModels({
|
|
92
|
+
currentModelId: input.currentModelId,
|
|
93
|
+
selections: catalog.selections,
|
|
94
|
+
modelNotes: catalog.modelNotes,
|
|
95
|
+
}),
|
|
96
|
+
},
|
|
97
|
+
],
|
|
98
|
+
};
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
}
|