@opengeni/worker-bundle 0.7.5 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-BJUW5LCM.js +325 -0
- package/dist/chunk-BJUW5LCM.js.map +1 -0
- package/dist/host-export.d.ts +70 -0
- package/dist/host-export.js +7 -0
- package/dist/host-export.js.map +1 -0
- package/dist/index.d.ts +98 -6
- package/dist/index.js +2812 -813
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +12358 -0
- package/package.json +18 -12
- package/src/activities/agent-turn.ts +2045 -545
- package/src/activities/capabilities.ts +9 -2
- package/src/activities/codex-capacity.ts +6 -11
- package/src/activities/codex-rotation.ts +75 -86
- package/src/activities/context-compaction.ts +104 -24
- package/src/activities/environment.ts +367 -92
- package/src/activities/expiring-material-renewal.ts +143 -0
- package/src/activities/goals.ts +4 -0
- package/src/activities/mcp-credentials.ts +38 -0
- package/src/activities/packs.ts +115 -2
- package/src/activities/run-credential-renewal.ts +70 -0
- package/src/activities/run-credentials.ts +132 -0
- package/src/activities/run-input.ts +23 -0
- package/src/activities/sandbox-lease.ts +1 -1
- package/src/activities/scheduled-tasks.ts +21 -0
- package/src/activities/session-state.ts +72 -0
- package/src/activities/streaming.ts +1 -0
- package/src/activities/toolspace-token-renewal.ts +57 -0
- package/src/activities/types.ts +44 -6
- package/src/activities/workspace-capture.ts +113 -93
- package/src/activities.ts +10 -2
- package/src/host-export-pump.ts +416 -0
- package/src/host-export.ts +22 -0
- package/src/http.ts +77 -28
- package/src/index.ts +360 -156
- package/src/sandbox-resume.ts +47 -1
- package/src/sandbox-routing.ts +1 -1
- package/src/worker-service-lifecycle.ts +104 -0
- package/src/workflows/activities.ts +13 -7
- package/src/workflows/session.ts +219 -23
- package/src/workflows.ts +1 -0
|
@@ -11,8 +11,11 @@ import {
|
|
|
11
11
|
setActiveSandbox,
|
|
12
12
|
requireFile,
|
|
13
13
|
getSessionEvent,
|
|
14
|
+
getSessionRootId,
|
|
14
15
|
getSessionGoal,
|
|
15
16
|
getLatestRunState,
|
|
17
|
+
getHumanInputResumeForEvent,
|
|
18
|
+
getSessionHumanInputRequest,
|
|
16
19
|
isCodexBilledTurn,
|
|
17
20
|
workspaceCodexSubscriptionActive,
|
|
18
21
|
acquireCodexCredentialLease,
|
|
@@ -47,7 +50,6 @@ import {
|
|
|
47
50
|
nextSessionHistoryPosition,
|
|
48
51
|
settleCodexCredentialLeaseLoss,
|
|
49
52
|
settleCodexCredentialFailover,
|
|
50
|
-
saveRunState,
|
|
51
53
|
upsertSandboxSessionEnvelope,
|
|
52
54
|
setSessionLastInputTokensForTurnAttempt,
|
|
53
55
|
sumUsageQuantity,
|
|
@@ -55,16 +57,19 @@ import {
|
|
|
55
57
|
accrueWarmSeconds,
|
|
56
58
|
getMaterializedSandboxFileResources,
|
|
57
59
|
markSandboxFileResourcesMaterialized,
|
|
60
|
+
areGitHubRepositoriesAllowedForWorkspace,
|
|
58
61
|
SandboxLeaseSupersededError,
|
|
59
62
|
SandboxImageConflictError,
|
|
60
|
-
|
|
63
|
+
isSessionEventPersistenceError,
|
|
61
64
|
getEnrollment,
|
|
62
65
|
abandonRecordingForTurnAttempt,
|
|
66
|
+
markSessionAttemptQuiesced,
|
|
63
67
|
type AppendEventInput,
|
|
64
68
|
type ActiveSandboxPointer,
|
|
65
69
|
type SandboxRecord,
|
|
66
70
|
type CodexCredentialLeaseResult,
|
|
67
71
|
type CodexCredentialLeaseSelectionContext,
|
|
72
|
+
type ApplySessionTurnSettlementInput,
|
|
68
73
|
type SessionTurnRecordingSettlement,
|
|
69
74
|
} from "@opengeni/db";
|
|
70
75
|
import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
|
|
@@ -81,10 +86,18 @@ import {
|
|
|
81
86
|
appendWorkspaceMemory,
|
|
82
87
|
composeAgentInstructions,
|
|
83
88
|
summarizeForCompaction,
|
|
89
|
+
CompactionProviderResponseError,
|
|
90
|
+
EmptyCompactionSummaryError,
|
|
84
91
|
ensureModalRegistryImage,
|
|
85
92
|
findCompactionNeededError,
|
|
86
93
|
materializeSandboxFileDownloads,
|
|
87
|
-
|
|
94
|
+
materializeRunCredentials,
|
|
95
|
+
clearRunCredentials,
|
|
96
|
+
clearRunCredentialsForAttempt,
|
|
97
|
+
withRunCredentialsSession,
|
|
98
|
+
refreshGitCredentialBindingTokenFiles,
|
|
99
|
+
refreshToolspaceTokenFile,
|
|
100
|
+
toolspaceTokenFileFromEnvironment,
|
|
88
101
|
sandboxFileDownloadFailureNote,
|
|
89
102
|
SUMMARY_BUFFER_TOKENS,
|
|
90
103
|
runOwnedSandboxSetup,
|
|
@@ -95,12 +108,17 @@ import {
|
|
|
95
108
|
type ComputerToolMode,
|
|
96
109
|
type ModelResponseUsage,
|
|
97
110
|
type BuildAgentOptions,
|
|
111
|
+
type TurnToolCancellationFence,
|
|
98
112
|
type BackendUnresolvableCode,
|
|
99
113
|
type EstablishedSandboxSession,
|
|
100
114
|
type GitCredentialTokenWriterSession,
|
|
115
|
+
type NormalizedRunCredentialMaterial,
|
|
116
|
+
type RunCredentialCommandSession,
|
|
117
|
+
type ToolspaceTokenWriterSession,
|
|
101
118
|
deleteRecordingArtifacts,
|
|
102
119
|
stopRecording as stopRecordingOnBox,
|
|
103
120
|
} from "@opengeni/runtime";
|
|
121
|
+
import { connectionTokenResolverForTurn } from "./mcp-credentials";
|
|
104
122
|
import {
|
|
105
123
|
builtinProviderId,
|
|
106
124
|
calculateModelUsageCostMicros,
|
|
@@ -120,6 +138,7 @@ import {
|
|
|
120
138
|
settingsWithSessionMcpServersForRun,
|
|
121
139
|
} from "./capabilities";
|
|
122
140
|
import {
|
|
141
|
+
CODEX_USAGE_EXHAUSTED_PCT,
|
|
123
142
|
authoritativeCodexCapacityResetAt,
|
|
124
143
|
chooseRotationActive,
|
|
125
144
|
classifyCodexPin,
|
|
@@ -154,25 +173,46 @@ import { mergeResourceRefs, mergeToolRefs } from "./common";
|
|
|
154
173
|
import { maybeCompactContext } from "./context-compaction";
|
|
155
174
|
import { TurnAttemptFencedError } from "./turn-attempt-fenced";
|
|
156
175
|
import {
|
|
176
|
+
gitCredentialAuthorityForTurn,
|
|
177
|
+
gitHubTokenMintSelections,
|
|
157
178
|
loadWorkspaceEnvironmentForRunWithCredentials,
|
|
179
|
+
mintSandboxToolspaceToken,
|
|
158
180
|
mintRunGitCredentials,
|
|
181
|
+
mintRunGitCredentialBinding,
|
|
159
182
|
sandboxEnvironmentForRun,
|
|
183
|
+
type GitHubTokenMintAuthorization,
|
|
160
184
|
type MintedRunGitCredentials,
|
|
161
185
|
} from "./environment";
|
|
162
186
|
import {
|
|
163
187
|
startGitCredentialRenewalLoop,
|
|
164
188
|
type GitCredentialRenewalController,
|
|
165
189
|
} from "./git-credential-renewal";
|
|
190
|
+
import {
|
|
191
|
+
RUN_CREDENTIAL_EXPIRY_LEAD_MS,
|
|
192
|
+
startRunCredentialRenewalLoop,
|
|
193
|
+
type RunCredentialRenewalController,
|
|
194
|
+
} from "./run-credential-renewal";
|
|
195
|
+
import {
|
|
196
|
+
TOOLSPACE_TOKEN_EXPIRY_LEAD_MS,
|
|
197
|
+
startToolspaceTokenRenewalLoop,
|
|
198
|
+
type ToolspaceTokenRenewalController,
|
|
199
|
+
} from "./toolspace-token-renewal";
|
|
200
|
+
import {
|
|
201
|
+
bindRunCredentialResolver,
|
|
202
|
+
runCredentialAuthNeededPayloads,
|
|
203
|
+
runCredentialModelNote,
|
|
204
|
+
} from "./run-credentials";
|
|
166
205
|
import { withCodexAppsTool, withFirstPartyTools } from "./goals";
|
|
167
206
|
import {
|
|
168
207
|
mergeRigDefaultVariableSetEnvironment,
|
|
169
208
|
resolveWorkspaceAgentInstructions,
|
|
170
209
|
resolveWorkspacePackRuntime,
|
|
210
|
+
resolveWorkspaceSkillLibraryRuntime,
|
|
171
211
|
settingsWithPackSandboxImage,
|
|
172
212
|
settingsWithRigImage,
|
|
173
213
|
} from "./packs";
|
|
174
214
|
import { deliverFailedChildTurnToParent } from "./parent-wake";
|
|
175
|
-
import { createSecretRedactor, identityRedactor } from "./redaction";
|
|
215
|
+
import { createSecretRedactor, identityRedactor, type SecretForRedaction } from "./redaction";
|
|
176
216
|
import { applyCodexHistoryStrip, turnInput, type TurnCodexAccount } from "./run-input";
|
|
177
217
|
import {
|
|
178
218
|
createRuntimeBatcher,
|
|
@@ -180,7 +220,12 @@ import {
|
|
|
180
220
|
nextStreamEvent,
|
|
181
221
|
startActivityHeartbeat,
|
|
182
222
|
} from "./streaming";
|
|
183
|
-
import type {
|
|
223
|
+
import type {
|
|
224
|
+
ActivityServices,
|
|
225
|
+
RunAgentTurnInput,
|
|
226
|
+
RunAgentTurnResult,
|
|
227
|
+
SessionAttemptQuiescenceProof,
|
|
228
|
+
} from "./types";
|
|
184
229
|
import {
|
|
185
230
|
resumeBoxForTurn,
|
|
186
231
|
acquireSelfhostedLeaseForTurn,
|
|
@@ -229,7 +274,6 @@ import {
|
|
|
229
274
|
import {
|
|
230
275
|
CAPABILITY_DESCRIPTORS,
|
|
231
276
|
evaluateWorkspaceModelPolicy,
|
|
232
|
-
type GitCredentialProvider,
|
|
233
277
|
type ResourceRef,
|
|
234
278
|
type SessionEvent,
|
|
235
279
|
type SessionEventType,
|
|
@@ -293,6 +337,28 @@ export function codexWorkspaceMetricKey(workspaceId: string): string {
|
|
|
293
337
|
return createHash("sha256").update(workspaceId).digest("hex").slice(0, 12);
|
|
294
338
|
}
|
|
295
339
|
|
|
340
|
+
/** Stable public request identity across partial resumes and activity retries. */
|
|
341
|
+
export function stableHumanInputRequestId(
|
|
342
|
+
sessionId: string,
|
|
343
|
+
turnId: string,
|
|
344
|
+
toolCallId: string,
|
|
345
|
+
): string {
|
|
346
|
+
const hex = createHash("sha256")
|
|
347
|
+
.update("opengeni-human-input-v1\0")
|
|
348
|
+
.update(sessionId)
|
|
349
|
+
.update("\0")
|
|
350
|
+
.update(turnId)
|
|
351
|
+
.update("\0")
|
|
352
|
+
.update(toolCallId)
|
|
353
|
+
.digest("hex")
|
|
354
|
+
.slice(0, 32)
|
|
355
|
+
.split("");
|
|
356
|
+
hex[12] = "5";
|
|
357
|
+
hex[16] = ["8", "9", "a", "b"][Number.parseInt(hex[16] ?? "0", 16) % 4] ?? "8";
|
|
358
|
+
const value = hex.join("");
|
|
359
|
+
return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(12, 16)}-${value.slice(16, 20)}-${value.slice(20)}`;
|
|
360
|
+
}
|
|
361
|
+
|
|
296
362
|
/**
|
|
297
363
|
* True when this activity attempt was cancelled because its hosting worker is
|
|
298
364
|
* shutting down gracefully (SIGTERM during a deploy), as opposed to a
|
|
@@ -302,12 +368,229 @@ export function isWorkerShutdownCancellation(error: unknown): boolean {
|
|
|
302
368
|
return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
|
|
303
369
|
}
|
|
304
370
|
|
|
371
|
+
/**
|
|
372
|
+
* Review captures and protective snapshots are cache/persistence housekeeping,
|
|
373
|
+
* never part of cancellation correctness. A control-fenced or Temporal-cancelled
|
|
374
|
+
* attempt must release its physical activity promptly so Steer/Pause can advance.
|
|
375
|
+
*/
|
|
376
|
+
export function shouldRunTurnEndWorkspacePersistence(input: {
|
|
377
|
+
activityStatus: RunAgentTurnResult["status"] | "unknown";
|
|
378
|
+
cancellationRequested: boolean;
|
|
379
|
+
}): boolean {
|
|
380
|
+
return input.activityStatus !== "cancelled" && !input.cancellationRequested;
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* Temporal cancellation is delivery/transport state, never proof that the
|
|
385
|
+
* dying activity crossed its mandatory sandbox-tool fence. If that fence
|
|
386
|
+
* fails, surface the fence failure instead of retaining a misleading typed
|
|
387
|
+
* cancellation; replacement admission remains closed because no durable
|
|
388
|
+
* quiescence receipt was written.
|
|
389
|
+
*/
|
|
390
|
+
export function assertPhysicalToolQuiescenceForCancellation(input: {
|
|
391
|
+
acknowledgeQuiescence: boolean;
|
|
392
|
+
physicalToolQuiescenceConfirmed: boolean;
|
|
393
|
+
failure: unknown;
|
|
394
|
+
}): void {
|
|
395
|
+
if (!input.acknowledgeQuiescence || input.physicalToolQuiescenceConfirmed) return;
|
|
396
|
+
if (input.failure instanceof Error) throw input.failure;
|
|
397
|
+
throw new Error("Physical sandbox-tool quiescence could not be confirmed", {
|
|
398
|
+
cause: input.failure,
|
|
399
|
+
});
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/** A physically drained attempt must leave one durable recovery producer. The
|
|
403
|
+
* direct Postgres receipt is preferred; after its bounded retries exhaust, an
|
|
404
|
+
* exact Temporal proof signal is sufficient because the workflow persists it
|
|
405
|
+
* through an independently retrying DB-only control activity. */
|
|
406
|
+
export function assertSessionAttemptQuiescenceRecoveryDurable(input: {
|
|
407
|
+
acknowledgeQuiescence: boolean;
|
|
408
|
+
physicalToolQuiescenceConfirmed: boolean;
|
|
409
|
+
receiptOrProofDurable: boolean;
|
|
410
|
+
failure: unknown;
|
|
411
|
+
}): void {
|
|
412
|
+
if (
|
|
413
|
+
!input.acknowledgeQuiescence ||
|
|
414
|
+
!input.physicalToolQuiescenceConfirmed ||
|
|
415
|
+
input.receiptOrProofDurable
|
|
416
|
+
) {
|
|
417
|
+
return;
|
|
418
|
+
}
|
|
419
|
+
if (input.failure instanceof Error) throw input.failure;
|
|
420
|
+
throw new Error("Physical quiescence had no durable receipt or recovery proof", {
|
|
421
|
+
cause: input.failure,
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
const QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS = 250;
|
|
426
|
+
const QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS = 5_000;
|
|
427
|
+
|
|
428
|
+
/** Persist the authoritative receipt or durably hand the exact physical proof
|
|
429
|
+
* to Temporal. This retries signal delivery, not DB eligibility or workflow
|
|
430
|
+
* state. The proof object never changes between attempts. */
|
|
431
|
+
export async function persistOrSignalSessionAttemptQuiescence(input: {
|
|
432
|
+
proof: SessionAttemptQuiescenceProof;
|
|
433
|
+
persistReceipt: () => Promise<SessionEvent[]>;
|
|
434
|
+
publishEvents: (events: SessionEvent[]) => Promise<unknown>;
|
|
435
|
+
signalProof: ActivityServices["signalSessionAttemptQuiesced"];
|
|
436
|
+
sleep?: (ms: number) => Promise<void>;
|
|
437
|
+
heartbeat?: (attempt: number, delayMs: number) => void;
|
|
438
|
+
onReceiptFailure?: (error: unknown) => void;
|
|
439
|
+
onPublishFailure?: (error: unknown) => void;
|
|
440
|
+
onSignalFailure?: (error: unknown, attempt: number, delayMs: number) => void;
|
|
441
|
+
}): Promise<"receipt" | "signal"> {
|
|
442
|
+
let events: SessionEvent[];
|
|
443
|
+
try {
|
|
444
|
+
events = await input.persistReceipt();
|
|
445
|
+
} catch (receiptError) {
|
|
446
|
+
input.onReceiptFailure?.(receiptError);
|
|
447
|
+
if (!input.signalProof) {
|
|
448
|
+
throw new Error("Session-attempt quiescence proof signaler is unavailable", {
|
|
449
|
+
cause: receiptError,
|
|
450
|
+
});
|
|
451
|
+
}
|
|
452
|
+
const delay = input.sleep ?? sleep;
|
|
453
|
+
let retryMs = QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS;
|
|
454
|
+
let attempt = 1;
|
|
455
|
+
for (;;) {
|
|
456
|
+
try {
|
|
457
|
+
await input.signalProof(input.proof);
|
|
458
|
+
return "signal";
|
|
459
|
+
} catch (signalError) {
|
|
460
|
+
input.onSignalFailure?.(signalError, attempt, retryMs);
|
|
461
|
+
try {
|
|
462
|
+
input.heartbeat?.(attempt, retryMs);
|
|
463
|
+
} catch {
|
|
464
|
+
// Heartbeat telemetry is not proof delivery and cannot replace or
|
|
465
|
+
// interrupt the exact signal retry loop.
|
|
466
|
+
}
|
|
467
|
+
await delay(retryMs);
|
|
468
|
+
retryMs = Math.min(retryMs * 2, QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS);
|
|
469
|
+
attempt += 1;
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
try {
|
|
475
|
+
await input.publishEvents(events);
|
|
476
|
+
} catch (publishError) {
|
|
477
|
+
// Postgres already committed quiesced_at, the queue event, and the wake.
|
|
478
|
+
// NATS is live fanout only; never misclassify its failure as receipt loss.
|
|
479
|
+
input.onPublishFailure?.(publishError);
|
|
480
|
+
}
|
|
481
|
+
return "receipt";
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
/**
|
|
485
|
+
* Cross the non-detachable physical boundary for one dying attempt. Temporal's
|
|
486
|
+
* cancellation signal is intentionally absent: cancellation is transport, not
|
|
487
|
+
* proof that sandbox tools and attempt-owned credential writers have drained.
|
|
488
|
+
* A replacement may be admitted only after this function resolves and the
|
|
489
|
+
* caller durably persists or signals the exact quiescence receipt.
|
|
490
|
+
*/
|
|
491
|
+
export async function drainAttemptOwnedSandboxWriters(input: {
|
|
492
|
+
toolCancellationFence: Pick<TurnToolCancellationFence, "cancel" | "waitForQuiescence"> | null;
|
|
493
|
+
cancellationReason?: unknown;
|
|
494
|
+
toolspaceTokenRenewal: Pick<ToolspaceTokenRenewalController, "stop"> | null;
|
|
495
|
+
runCredentialRenewal: Pick<RunCredentialRenewalController, "stop"> | null;
|
|
496
|
+
}): Promise<void> {
|
|
497
|
+
if (input.toolCancellationFence) {
|
|
498
|
+
input.toolCancellationFence.cancel(
|
|
499
|
+
input.cancellationReason ?? new Error("TURN_ATTEMPT_FENCED"),
|
|
500
|
+
);
|
|
501
|
+
await input.toolCancellationFence.waitForQuiescence();
|
|
502
|
+
}
|
|
503
|
+
await input.toolspaceTokenRenewal?.stop();
|
|
504
|
+
await input.runCredentialRenewal?.stop();
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/**
|
|
508
|
+
* Await a finalizer operation only while this Temporal activity still owns its
|
|
509
|
+
* execution window. Once Pause/Steer cancellation arrives, the operation keeps
|
|
510
|
+
* its own rejection handler and may finish its idempotent, attempt-scoped
|
|
511
|
+
* cleanup in the background, but it cannot pin activity terminalization or
|
|
512
|
+
* delay the separately receipt-gated replacement dispatch.
|
|
513
|
+
*/
|
|
514
|
+
export async function waitForTurnFinalizerStep<T>(
|
|
515
|
+
operation: Promise<T>,
|
|
516
|
+
signal: AbortSignal | undefined,
|
|
517
|
+
): Promise<T | undefined> {
|
|
518
|
+
if (!signal) return await operation;
|
|
519
|
+
if (signal.aborted) {
|
|
520
|
+
void operation.catch(() => undefined);
|
|
521
|
+
return undefined;
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
let resolveCancellation: (() => void) | undefined;
|
|
525
|
+
const cancelled = new Promise<undefined>((resolve) => {
|
|
526
|
+
resolveCancellation = () => resolve(undefined);
|
|
527
|
+
});
|
|
528
|
+
const cancel = (): void => {
|
|
529
|
+
void operation.catch(() => undefined);
|
|
530
|
+
resolveCancellation?.();
|
|
531
|
+
};
|
|
532
|
+
signal.addEventListener("abort", cancel, { once: true });
|
|
533
|
+
try {
|
|
534
|
+
return await Promise.race([operation, cancelled]);
|
|
535
|
+
} finally {
|
|
536
|
+
signal.removeEventListener("abort", cancel);
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
/**
|
|
541
|
+
* Flush provider-facing stream state while the attempt still owns its activity
|
|
542
|
+
* window. On Pause/Steer, both promises are detached with rejection handlers:
|
|
543
|
+
* neither a runtime batcher nor an uncooperative provider completion promise
|
|
544
|
+
* may pin the activity behind cancellation after durable writes are fenced.
|
|
545
|
+
*/
|
|
546
|
+
export async function waitForTurnStreamCleanup(
|
|
547
|
+
batcherFlush: Promise<unknown>,
|
|
548
|
+
providerCompleted: Promise<unknown>,
|
|
549
|
+
signal: AbortSignal | undefined,
|
|
550
|
+
): Promise<void> {
|
|
551
|
+
await waitForTurnFinalizerStep(batcherFlush, signal);
|
|
552
|
+
await waitForTurnFinalizerStep(providerCompleted, signal);
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
function turnFinalizerCancellationSignal(
|
|
556
|
+
temporalSignal: AbortSignal | undefined,
|
|
557
|
+
activityStatus: RunAgentTurnResult["status"] | "unknown",
|
|
558
|
+
): AbortSignal | undefined {
|
|
559
|
+
if (activityStatus !== "cancelled" || temporalSignal?.aborted) return temporalSignal;
|
|
560
|
+
const fenced = new AbortController();
|
|
561
|
+
fenced.abort(new Error("TURN_ATTEMPT_FENCED"));
|
|
562
|
+
return fenced.signal;
|
|
563
|
+
}
|
|
564
|
+
|
|
305
565
|
function compactionFailureReason(reason: string): string {
|
|
306
566
|
return reason.startsWith("compaction summarization failed:")
|
|
307
567
|
? reason
|
|
308
568
|
: `compaction summarization failed: ${reason}`;
|
|
309
569
|
}
|
|
310
570
|
|
|
571
|
+
function compactionFailureReasonFromError(error: unknown): string {
|
|
572
|
+
if (
|
|
573
|
+
error instanceof CompactionProviderResponseError ||
|
|
574
|
+
error instanceof EmptyCompactionSummaryError
|
|
575
|
+
) {
|
|
576
|
+
return compactionFailureReason(error.message);
|
|
577
|
+
}
|
|
578
|
+
const errorName = error instanceof Error && error.name ? error.name : "unknown error";
|
|
579
|
+
return compactionFailureReason(`unexpected ${errorName}`);
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
function isCompactionSummaryFailure(error: unknown): boolean {
|
|
583
|
+
return (
|
|
584
|
+
error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError
|
|
585
|
+
);
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
export function shouldRecoverCompactionProviderFailure(error: unknown): boolean {
|
|
589
|
+
if (!(error instanceof CompactionProviderResponseError)) return false;
|
|
590
|
+
if (isCodexTransportError(error) && classifyCodexUsageLimitError(error)) return true;
|
|
591
|
+
return agentRunFailurePayload(error).retryable === true;
|
|
592
|
+
}
|
|
593
|
+
|
|
311
594
|
export function classifyContextWindowOverflowError(
|
|
312
595
|
error: unknown,
|
|
313
596
|
): { message: string; code?: string; detail?: string } | null {
|
|
@@ -440,6 +723,28 @@ export function modelUsageSourceKey(input: {
|
|
|
440
723
|
return input.dispatchId ? `${input.dispatchId}:${input.positionalKey}` : input.positionalKey;
|
|
441
724
|
}
|
|
442
725
|
|
|
726
|
+
export function providerContextTokens(
|
|
727
|
+
usage:
|
|
728
|
+
| {
|
|
729
|
+
inputTokens?: number;
|
|
730
|
+
outputTokens?: number;
|
|
731
|
+
totalTokens?: number;
|
|
732
|
+
}
|
|
733
|
+
| null
|
|
734
|
+
| undefined,
|
|
735
|
+
): number | null {
|
|
736
|
+
const total = usage?.totalTokens;
|
|
737
|
+
if (typeof total === "number" && Number.isFinite(total) && total > 0) {
|
|
738
|
+
return total;
|
|
739
|
+
}
|
|
740
|
+
const input = usage?.inputTokens;
|
|
741
|
+
if (typeof input !== "number" || !Number.isFinite(input) || input <= 0) {
|
|
742
|
+
return null;
|
|
743
|
+
}
|
|
744
|
+
const output = usage?.outputTokens;
|
|
745
|
+
return input + (typeof output === "number" && Number.isFinite(output) && output > 0 ? output : 0);
|
|
746
|
+
}
|
|
747
|
+
|
|
443
748
|
/**
|
|
444
749
|
* A provider call has already consumed tokens by the time its usage frame is
|
|
445
750
|
* available. Losing the Codex credential lease at the renewal checkpoint must
|
|
@@ -565,12 +870,13 @@ export function historyRowsToAppend(
|
|
|
565
870
|
// persistedHistoryCount to preserve the pre-compaction behaviour (contiguous
|
|
566
871
|
// positions from 0) when callers do not pass an explicit next position.
|
|
567
872
|
nextPosition: number = persistedHistoryCount,
|
|
873
|
+
toolOutputTruncationTokens?: number,
|
|
568
874
|
): {
|
|
569
875
|
rows: Array<{ position: number; item: Record<string, unknown> }>;
|
|
570
876
|
nextWatermark: number;
|
|
571
877
|
nextPosition: number;
|
|
572
878
|
} {
|
|
573
|
-
const sanitized = sanitizeHistoryItemsForModel(rawHistory).filter(
|
|
879
|
+
const sanitized = sanitizeHistoryItemsForModel(rawHistory, toolOutputTruncationTokens).filter(
|
|
574
880
|
(item) => !isEphemeralInternalContext(item),
|
|
575
881
|
);
|
|
576
882
|
if (sanitized.length <= persistedHistoryCount) {
|
|
@@ -916,6 +1222,86 @@ export type TurnSandboxProvisioner<T> = {
|
|
|
916
1222
|
waitForSettled(timeoutMs: number): Promise<T | null>;
|
|
917
1223
|
};
|
|
918
1224
|
|
|
1225
|
+
export class TurnOperationCancelledError extends Error {
|
|
1226
|
+
readonly name = "TurnOperationCancelledError";
|
|
1227
|
+
|
|
1228
|
+
constructor(readonly reason: unknown) {
|
|
1229
|
+
super("Turn operation was cancelled with its owning turn", {
|
|
1230
|
+
...(reason instanceof Error ? { cause: reason } : {}),
|
|
1231
|
+
});
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
/**
|
|
1236
|
+
* Normalize a preparation/provisioning cancellation race back to the Temporal
|
|
1237
|
+
* cancellation that owns the activity. Several provider APIs expose no portable
|
|
1238
|
+
* abort primitive, so the worker stops awaiting them and disposes any late
|
|
1239
|
+
* resource. The wrapper error must never fall through as an ordinary turn
|
|
1240
|
+
* failure: doing so would omit the quiescence receipt and strand a committed
|
|
1241
|
+
* Steer/Pause behind `control-pending`.
|
|
1242
|
+
*/
|
|
1243
|
+
export function turnOperationCancellationFailure(error: unknown): CancelledFailure | null {
|
|
1244
|
+
if (error instanceof CancelledFailure) return error;
|
|
1245
|
+
if (!(error instanceof TurnOperationCancelledError)) return null;
|
|
1246
|
+
return error.reason instanceof CancelledFailure
|
|
1247
|
+
? error.reason
|
|
1248
|
+
: new CancelledFailure("TURN_SANDBOX_PROVISION_CANCELLED", [], error);
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
function throwIfTurnOperationCancelled(signal: AbortSignal | undefined): void {
|
|
1252
|
+
if (signal?.aborted) {
|
|
1253
|
+
throw new TurnOperationCancelledError(signal.reason);
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
export async function waitForTurnOperation<T>(
|
|
1258
|
+
operation: Promise<T>,
|
|
1259
|
+
signal: AbortSignal | undefined,
|
|
1260
|
+
disposeLateResult: ((result: T) => Promise<void> | void) | undefined,
|
|
1261
|
+
): Promise<T> {
|
|
1262
|
+
if (!signal) return await operation;
|
|
1263
|
+
|
|
1264
|
+
let rejectCancellation: ((error: TurnOperationCancelledError) => void) | undefined;
|
|
1265
|
+
const cancelled = new Promise<never>((_resolve, reject) => {
|
|
1266
|
+
rejectCancellation = reject;
|
|
1267
|
+
});
|
|
1268
|
+
const cancel = (): void => {
|
|
1269
|
+
rejectCancellation?.(new TurnOperationCancelledError(signal.reason));
|
|
1270
|
+
};
|
|
1271
|
+
signal.addEventListener("abort", cancel, { once: true });
|
|
1272
|
+
if (signal.aborted) cancel();
|
|
1273
|
+
|
|
1274
|
+
try {
|
|
1275
|
+
const result = await Promise.race([operation, cancelled]);
|
|
1276
|
+
// Cancellation owns an exact turn boundary even when the provider result
|
|
1277
|
+
// and AbortSignal settle in the same microtask checkpoint. Never let a
|
|
1278
|
+
// just-resolved lease escape after the control was already committed.
|
|
1279
|
+
if (signal.aborted) {
|
|
1280
|
+
throw new TurnOperationCancelledError(signal.reason);
|
|
1281
|
+
}
|
|
1282
|
+
return result;
|
|
1283
|
+
} catch (error) {
|
|
1284
|
+
if (signal.aborted) {
|
|
1285
|
+
// The provider establish call has no universal cancellation seam. It may
|
|
1286
|
+
// finish after the Temporal activity has correctly stopped; dispose its
|
|
1287
|
+
// late lease instead of letting a cancelled turn resurrect a holder/box.
|
|
1288
|
+
void operation
|
|
1289
|
+
.then(async (result) => await disposeLateResult?.(result))
|
|
1290
|
+
.catch(() => undefined);
|
|
1291
|
+
// A provider failure may settle in the same checkpoint as the committed
|
|
1292
|
+
// control. The control is authoritative; retain its cancellation shape
|
|
1293
|
+
// so the activity publishes quiescence instead of looking like an
|
|
1294
|
+
// unrelated turn failure.
|
|
1295
|
+
if (!(error instanceof TurnOperationCancelledError)) {
|
|
1296
|
+
throw new TurnOperationCancelledError(signal.reason);
|
|
1297
|
+
}
|
|
1298
|
+
}
|
|
1299
|
+
throw error;
|
|
1300
|
+
} finally {
|
|
1301
|
+
signal.removeEventListener("abort", cancel);
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
|
|
919
1305
|
function sleep(ms: number): Promise<void> {
|
|
920
1306
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
921
1307
|
}
|
|
@@ -938,9 +1324,11 @@ export function createTurnSandboxProvisioner<T>(
|
|
|
938
1324
|
options: {
|
|
939
1325
|
maxRetries?: number;
|
|
940
1326
|
backoffMs?: number;
|
|
1327
|
+
signal?: AbortSignal;
|
|
941
1328
|
onStarted?: () => Promise<void> | void;
|
|
942
|
-
onCompleted?: () => Promise<void> | void;
|
|
1329
|
+
onCompleted?: (result: T) => Promise<void> | void;
|
|
943
1330
|
onFailed?: (error: unknown) => Promise<void> | void;
|
|
1331
|
+
disposeResult?: (result: T) => Promise<void> | void;
|
|
944
1332
|
} = {},
|
|
945
1333
|
): TurnSandboxProvisioner<T> {
|
|
946
1334
|
const maxRetries = options.maxRetries ?? 2;
|
|
@@ -951,9 +1339,15 @@ export function createTurnSandboxProvisioner<T>(
|
|
|
951
1339
|
let attempt = 0;
|
|
952
1340
|
while (true) {
|
|
953
1341
|
try {
|
|
954
|
-
|
|
1342
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
1343
|
+
const operation = establish();
|
|
1344
|
+
return await waitForTurnOperation(operation, options.signal, options.disposeResult);
|
|
955
1345
|
} catch (error) {
|
|
956
|
-
if (
|
|
1346
|
+
if (
|
|
1347
|
+
error instanceof TurnOperationCancelledError ||
|
|
1348
|
+
attempt >= maxRetries ||
|
|
1349
|
+
!isLazySandboxProvisionRetryable(error)
|
|
1350
|
+
) {
|
|
957
1351
|
throw error;
|
|
958
1352
|
}
|
|
959
1353
|
attempt += 1;
|
|
@@ -966,13 +1360,24 @@ export function createTurnSandboxProvisioner<T>(
|
|
|
966
1360
|
get(): Promise<T> {
|
|
967
1361
|
if (!memo) {
|
|
968
1362
|
memo = (async () => {
|
|
1363
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
969
1364
|
await options.onStarted?.();
|
|
1365
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
1366
|
+
let result: T | undefined;
|
|
1367
|
+
let hasResult = false;
|
|
970
1368
|
try {
|
|
971
|
-
|
|
972
|
-
|
|
1369
|
+
result = await run();
|
|
1370
|
+
hasResult = true;
|
|
1371
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
1372
|
+
await options.onCompleted?.(result);
|
|
973
1373
|
return result;
|
|
974
1374
|
} catch (error) {
|
|
975
|
-
|
|
1375
|
+
if (hasResult) {
|
|
1376
|
+
await options.disposeResult?.(result as T);
|
|
1377
|
+
}
|
|
1378
|
+
if (!(error instanceof TurnOperationCancelledError)) {
|
|
1379
|
+
await options.onFailed?.(error);
|
|
1380
|
+
}
|
|
976
1381
|
throw error;
|
|
977
1382
|
}
|
|
978
1383
|
})().catch((error) => {
|
|
@@ -1041,7 +1446,7 @@ export function acceptsPromptCacheKeyForTurn(
|
|
|
1041
1446
|
* The turn hot path never refreshes Codex usage — only the usage API route does — so a
|
|
1042
1447
|
* window that has actually reset still reads OVER-threshold from the stale cache, which
|
|
1043
1448
|
* would idle-loop forever. Before idling, refresh LIVE usage for every connected account
|
|
1044
|
-
* the cache marks
|
|
1449
|
+
* the cache marks exhausted (bounded to the account count), which re-writes the
|
|
1045
1450
|
* cache columns, then return the re-read rows so the ranker can pick up a genuinely-reset
|
|
1046
1451
|
* window THIS turn. A refresh/read failure is swallowed (fall back to the pre-refresh rows
|
|
1047
1452
|
* + the bounded idle). Cooling (429'd) accounts are NOT refreshed: their exhaustedUntil
|
|
@@ -1057,11 +1462,11 @@ async function refreshCappedCodexUsageRows(
|
|
|
1057
1462
|
wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
|
|
1058
1463
|
},
|
|
1059
1464
|
): Promise<CodexAccountStatus[]> {
|
|
1060
|
-
const nearPct = settings.codexRotationNearExhaustionPct;
|
|
1061
1465
|
const stale = accounts.filter(
|
|
1062
1466
|
(a) =>
|
|
1063
1467
|
a.status === "active" &&
|
|
1064
|
-
((a.primaryUsedPercent ?? 0) >=
|
|
1468
|
+
((a.primaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT ||
|
|
1469
|
+
(a.secondaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT),
|
|
1065
1470
|
);
|
|
1066
1471
|
if (stale.length === 0) {
|
|
1067
1472
|
return accounts;
|
|
@@ -1097,11 +1502,22 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1097
1502
|
objectStorage,
|
|
1098
1503
|
observability,
|
|
1099
1504
|
wakeSessionWorkflow,
|
|
1505
|
+
signalSessionAttemptQuiesced,
|
|
1100
1506
|
signalCodexCapacityWorkflow,
|
|
1101
1507
|
entitlements,
|
|
1102
1508
|
connectionCredentials,
|
|
1103
1509
|
} = await services();
|
|
1104
1510
|
const activityContext = currentActivityContext();
|
|
1511
|
+
const cancellationSignal = activityContext?.cancellationSignal;
|
|
1512
|
+
let cancellationRequestedAt: number | null = cancellationSignal?.aborted
|
|
1513
|
+
? performance.now()
|
|
1514
|
+
: null;
|
|
1515
|
+
const noteCancellationRequested = (): void => {
|
|
1516
|
+
cancellationRequestedAt ??= performance.now();
|
|
1517
|
+
};
|
|
1518
|
+
cancellationSignal?.addEventListener("abort", noteCancellationRequested, {
|
|
1519
|
+
once: true,
|
|
1520
|
+
});
|
|
1105
1521
|
const dispatchId = activityContext?.info.activityId ?? randomUUID();
|
|
1106
1522
|
const activityStarted = performance.now();
|
|
1107
1523
|
const activitySpan = observability.startSpan("worker.run_agent_segment", {
|
|
@@ -1112,6 +1528,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1112
1528
|
let activityStatus: RunAgentTurnResult["status"] | "unknown" = "unknown";
|
|
1113
1529
|
let turnMetricOutcome: TurnOutcome | null = null;
|
|
1114
1530
|
let activityError: unknown;
|
|
1531
|
+
let acknowledgeQuiescence = false;
|
|
1532
|
+
const acknowledgeLostAttemptOwnership = (): void => {
|
|
1533
|
+
// A stale terminal/recovery settlement can lose either to a benign
|
|
1534
|
+
// successor or to Pause/Steer closing this exact attempt. Only the
|
|
1535
|
+
// receipt transaction can distinguish those cases after the hard tool
|
|
1536
|
+
// fence: allowUninterrupted makes the benign case an event-free no-op.
|
|
1537
|
+
acknowledgeQuiescence = true;
|
|
1538
|
+
noteCancellationRequested();
|
|
1539
|
+
};
|
|
1115
1540
|
let turnId: string | undefined;
|
|
1116
1541
|
let triggerEventId: string | undefined;
|
|
1117
1542
|
const claimedResult = (
|
|
@@ -1139,13 +1564,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1139
1564
|
// Still required by credential-loss/capacity settlements, whose own
|
|
1140
1565
|
// recovery transactions fence against worker-death redispatches.
|
|
1141
1566
|
let redispatchesAtDispatch = 0;
|
|
1142
|
-
const saveRunStateFenced = async (state: Parameters<typeof saveRunState>[1]): Promise<void> => {
|
|
1143
|
-
if (!(await saveRunState(db, state))) {
|
|
1144
|
-
throw new TurnAttemptFencedError(
|
|
1145
|
-
"turn execution generation was fenced while saving run state",
|
|
1146
|
-
);
|
|
1147
|
-
}
|
|
1148
|
-
};
|
|
1149
1567
|
const setLastInputTokensFenced = async (lastInputTokens: number): Promise<void> => {
|
|
1150
1568
|
if (!turnId || executionGeneration <= 0) {
|
|
1151
1569
|
throw new Error("Turn attempt was not initialized before token accounting");
|
|
@@ -1164,7 +1582,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1164
1582
|
}
|
|
1165
1583
|
};
|
|
1166
1584
|
let heartbeatTimer: ReturnType<typeof startActivityHeartbeat> | undefined;
|
|
1167
|
-
//
|
|
1585
|
+
// credential allocator: one workspace-local idempotent credential holder per running
|
|
1168
1586
|
// Codex turn. The DB row is the cross-replica fairness primitive; this timer
|
|
1169
1587
|
// only extends its short TTL. A killed worker stops heartbeating and the
|
|
1170
1588
|
// holder self-expires. Other workspaces never see or share this holder.
|
|
@@ -1295,11 +1713,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1295
1713
|
// runs it refreshes expires_at epoch-fenced so a legit multi-day turn is
|
|
1296
1714
|
// never TTL-reaped. Cleared in finally. Only set when the flag resolved a box.
|
|
1297
1715
|
let leaseHeartbeatTimer: ReturnType<typeof setInterval> | undefined;
|
|
1298
|
-
//
|
|
1716
|
+
// credential-renewal policy: the worker, not the model, owns renewal of run-scoped Git
|
|
1299
1717
|
// credentials for a multi-day turn. The controller is attached only after
|
|
1300
1718
|
// the initial seed reached a real cloud box and is drained before capture.
|
|
1301
|
-
let
|
|
1719
|
+
let gitCredentialRenewals: GitCredentialRenewalController[] = [];
|
|
1302
1720
|
let gitCredentialRenewalClosed = false;
|
|
1721
|
+
// Generic host-owned run material has its own attempt-scoped renewal and
|
|
1722
|
+
// write handle. It is always drained and wiped before workspace capture.
|
|
1723
|
+
let runCredentialRenewal: RunCredentialRenewalController | null = null;
|
|
1724
|
+
let runCredentialRenewalClosed = false;
|
|
1725
|
+
let runCredentialSession: RunCredentialCommandSession | null = null;
|
|
1726
|
+
// The delegated Toolspace bearer has a one-hour TTL. Renewal is attempt-
|
|
1727
|
+
// owned and attaches only after the initial token file reached a real
|
|
1728
|
+
// sandbox session; finalization drains an in-flight replacement.
|
|
1729
|
+
let toolspaceTokenRenewal: ToolspaceTokenRenewalController | null = null;
|
|
1730
|
+
let toolspaceTokenRenewalClosed = false;
|
|
1303
1731
|
// MID-SESSION snapshot single-flight guard: the heartbeat tick fires every
|
|
1304
1732
|
// 10s but a Modal filesystem snapshot can take longer — never overlap two
|
|
1305
1733
|
// captures on one box. The in-flight capture's promise is held so the
|
|
@@ -1349,6 +1777,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1349
1777
|
await current?.flush().catch(() => undefined);
|
|
1350
1778
|
};
|
|
1351
1779
|
let preparedTools: Awaited<ReturnType<OpenGeniRuntime["prepareTools"]>> | null = null;
|
|
1780
|
+
const toolCancellationFenceRef: {
|
|
1781
|
+
current: TurnToolCancellationFence | null;
|
|
1782
|
+
} = {
|
|
1783
|
+
current: null,
|
|
1784
|
+
};
|
|
1352
1785
|
let publish: TurnEventPublisher | null = null;
|
|
1353
1786
|
let settle:
|
|
1354
1787
|
| ((input: {
|
|
@@ -1362,10 +1795,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1362
1795
|
| "requires_action";
|
|
1363
1796
|
sessionStatus: SessionStatus;
|
|
1364
1797
|
activeTurnId: string | null;
|
|
1798
|
+
consumeRequestedCompactionFailure?: boolean;
|
|
1799
|
+
runState?: ApplySessionTurnSettlementInput["runState"];
|
|
1365
1800
|
}) => Promise<boolean>)
|
|
1366
1801
|
| null = null;
|
|
1367
1802
|
let turnStartedPublished = false;
|
|
1368
1803
|
let stream: Awaited<ReturnType<OpenGeniRuntime["runStream"]>> | undefined;
|
|
1804
|
+
// Reconciliation is declared before provider routing so every turn-end path
|
|
1805
|
+
// can share one closure. It cannot run until `stream` exists, by which time
|
|
1806
|
+
// this value has been rebound to the turn's resolved model policy.
|
|
1807
|
+
let modelRunSettings: Settings = settings;
|
|
1369
1808
|
const publishSandboxLifecycleEvents = async (sandbox: ResumedTurnSandbox): Promise<void> => {
|
|
1370
1809
|
const established = sandbox.established;
|
|
1371
1810
|
if (publish && established.origin && established.origin !== "resumed") {
|
|
@@ -1465,16 +1904,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1465
1904
|
// selfhosted target has no persistWorkspace anyway. Best-effort +
|
|
1466
1905
|
// single-flight; throttling lives in the helper.
|
|
1467
1906
|
const snapshotSession = setupBoxSession;
|
|
1468
|
-
|
|
1907
|
+
const snapshotTurnId = turnId;
|
|
1908
|
+
if (snapshotSession && snapshotTurnId && !snapshotInFlight) {
|
|
1469
1909
|
snapshotInFlight = maybePersistWarmWorkspaceSnapshot(
|
|
1470
1910
|
{ db, settings },
|
|
1471
1911
|
{
|
|
1472
1912
|
accountId: input.accountId,
|
|
1473
1913
|
workspaceId: input.workspaceId,
|
|
1914
|
+
sessionId: input.sessionId,
|
|
1915
|
+
turnId: snapshotTurnId,
|
|
1916
|
+
attemptId: input.attemptId,
|
|
1474
1917
|
sandboxGroupId: heartbeatGroupId,
|
|
1475
1918
|
},
|
|
1476
1919
|
snapshotSession,
|
|
1477
1920
|
heartbeatEpoch,
|
|
1921
|
+
activityContext?.cancellationSignal,
|
|
1478
1922
|
)
|
|
1479
1923
|
.then(async (persisted) => {
|
|
1480
1924
|
if (persisted && publish) {
|
|
@@ -1602,6 +2046,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1602
2046
|
rawHistory as Array<Record<string, unknown>>,
|
|
1603
2047
|
persistedHistoryCount,
|
|
1604
2048
|
nextHistoryPosition,
|
|
2049
|
+
modelRunSettings.modelToolOutputTruncationTokens,
|
|
1605
2050
|
);
|
|
1606
2051
|
const hasModelOrToolProgress = rows.some((row) =>
|
|
1607
2052
|
isModelOrToolProgressHistoryItem(row.item),
|
|
@@ -1621,6 +2066,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1621
2066
|
// runs, so this is the turn's effective account. The read path uses
|
|
1622
2067
|
// it to strip cross-account reasoning.encrypted_content next turn.
|
|
1623
2068
|
producerCodexCredentialId: effectiveCodexCredentialId,
|
|
2069
|
+
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
1624
2070
|
items: rows,
|
|
1625
2071
|
});
|
|
1626
2072
|
if (!appended) {
|
|
@@ -1651,6 +2097,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1651
2097
|
// Reassigned after the variable set loads; the publish closure is
|
|
1652
2098
|
// created (and used for turn.started) before the variableSet is available.
|
|
1653
2099
|
let redact: (payload: unknown) => unknown = identityRedactor;
|
|
2100
|
+
const secretRedactions = new Map<string, string>();
|
|
2101
|
+
const publishedRunCredentialNotices = new Set<string>();
|
|
2102
|
+
const registerSecretRedactions = (secrets: SecretForRedaction[]): void => {
|
|
2103
|
+
for (const secret of secrets) {
|
|
2104
|
+
if (!secretRedactions.has(secret.value)) secretRedactions.set(secret.value, secret.name);
|
|
2105
|
+
}
|
|
2106
|
+
redact = createSecretRedactor(
|
|
2107
|
+
[...secretRedactions].map(([value, name]) => ({ name, value })),
|
|
2108
|
+
);
|
|
2109
|
+
};
|
|
1654
2110
|
let variableSetId = "";
|
|
1655
2111
|
// Rig telemetry (M3): set once the session loads; empty string for a rig-less
|
|
1656
2112
|
// turn (mirrors variableSetId). Read by the activity span's finally block.
|
|
@@ -1718,6 +2174,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1718
2174
|
if (!trigger) {
|
|
1719
2175
|
throw new Error(`Trigger event not found: ${triggerEventId}`);
|
|
1720
2176
|
}
|
|
2177
|
+
const humanInputResume = await getHumanInputResumeForEvent(
|
|
2178
|
+
db,
|
|
2179
|
+
input.workspaceId,
|
|
2180
|
+
input.sessionId,
|
|
2181
|
+
trigger,
|
|
2182
|
+
);
|
|
1721
2183
|
triggerType = trigger.type;
|
|
1722
2184
|
executionGeneration = turn.executionGeneration;
|
|
1723
2185
|
const latestTurnState = await getLatestRunState(db, input.workspaceId, input.sessionId);
|
|
@@ -1745,13 +2207,17 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1745
2207
|
// §7.5 P3 — pass BOTH the codex predicate (codex-plan turns bypass the gate)
|
|
1746
2208
|
// AND the optional host `entitlements` port (when bound, its admitRun replaces
|
|
1747
2209
|
// the local credit read). Unset port → today's local-ledger path.
|
|
1748
|
-
await
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
2210
|
+
await waitForTurnOperation(
|
|
2211
|
+
ensureRunAllowed(
|
|
2212
|
+
settings,
|
|
2213
|
+
db,
|
|
2214
|
+
input.accountId,
|
|
2215
|
+
input.workspaceId,
|
|
2216
|
+
isCodexTurn,
|
|
2217
|
+
entitlements,
|
|
2218
|
+
),
|
|
2219
|
+
cancellationSignal,
|
|
2220
|
+
undefined,
|
|
1755
2221
|
);
|
|
1756
2222
|
// Setup (variableSet load, MCP connects, sandbox restore) does not
|
|
1757
2223
|
// stream and so never observes cancellation on its own; these explicit
|
|
@@ -1763,6 +2229,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1763
2229
|
throw reason;
|
|
1764
2230
|
}
|
|
1765
2231
|
};
|
|
2232
|
+
const throwIfTurnCancelled = () => throwIfTurnOperationCancelled(cancellationSignal);
|
|
1766
2233
|
// ONE shared details object for every heartbeat this activity sends (each
|
|
1767
2234
|
// site spreads it + its own phase), so cross-site fields — the op-stream
|
|
1768
2235
|
// settled roster in particular — survive last-write-wins instead of being
|
|
@@ -1858,6 +2325,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1858
2325
|
producerSeq: ++producerSeq,
|
|
1859
2326
|
};
|
|
1860
2327
|
}
|
|
2328
|
+
const compactionRequestFailure = inputSettlement.consumeRequestedCompactionFailure
|
|
2329
|
+
? {
|
|
2330
|
+
reason: "summarization_failed" as const,
|
|
2331
|
+
producerId,
|
|
2332
|
+
producerSeq: ++producerSeq,
|
|
2333
|
+
}
|
|
2334
|
+
: undefined;
|
|
1861
2335
|
const inputs = inputSettlement.events.map((event) => ({
|
|
1862
2336
|
...event,
|
|
1863
2337
|
payload: redact(event.payload),
|
|
@@ -1874,9 +2348,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1874
2348
|
sessionStatus: inputSettlement.sessionStatus,
|
|
1875
2349
|
activeTurnId: inputSettlement.activeTurnId,
|
|
1876
2350
|
events: inputs,
|
|
2351
|
+
...(inputSettlement.runState ? { runState: inputSettlement.runState } : {}),
|
|
1877
2352
|
...(recordingMutation ? { recording: recordingMutation } : {}),
|
|
2353
|
+
...(compactionRequestFailure ? { compactionRequestFailure } : {}),
|
|
1878
2354
|
});
|
|
1879
2355
|
if (result.action === "stale") {
|
|
2356
|
+
// The terminal write can lose to a control transaction before the
|
|
2357
|
+
// workflow delivers Temporal cancellation. That control may settle
|
|
2358
|
+
// the already-closed attempt as rejected_stale, so returning without
|
|
2359
|
+
// this flag would strand its replacement behind quiesced_at forever.
|
|
2360
|
+
// Enter the same hard tool-fence/receipt path as an explicit
|
|
2361
|
+
// TurnAttemptFencedError. If ownership was lost for an unrelated
|
|
2362
|
+
// reason, allowUninterrupted makes the receipt transaction a no-op.
|
|
2363
|
+
acknowledgeLostAttemptOwnership();
|
|
1880
2364
|
if (recordingForSettlement) {
|
|
1881
2365
|
await abandonActiveRecording(
|
|
1882
2366
|
"recording settlement lost attempt ownership",
|
|
@@ -1917,6 +2401,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1917
2401
|
// visibly starts: nothing ran yet, so the same inference starts cleanly
|
|
1918
2402
|
// on a healthy worker.
|
|
1919
2403
|
throwIfWorkerShuttingDown();
|
|
2404
|
+
throwIfTurnCancelled();
|
|
1920
2405
|
if (
|
|
1921
2406
|
!(await settle({
|
|
1922
2407
|
events: [
|
|
@@ -1953,7 +2438,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1953
2438
|
sessionPinSource,
|
|
1954
2439
|
sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
|
|
1955
2440
|
continuationCredentialId: continuationCodexCredentialId,
|
|
1956
|
-
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
1957
2441
|
now: new Date(),
|
|
1958
2442
|
});
|
|
1959
2443
|
|
|
@@ -2075,7 +2559,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2075
2559
|
strategy: leased.rotationStrategy as CodexRotationStrategy,
|
|
2076
2560
|
rotationEnabled: leased.rotationEnabled,
|
|
2077
2561
|
});
|
|
2078
|
-
//
|
|
2562
|
+
// pin policy pin persistence follows the atomic credential allocator selection. The selector
|
|
2079
2563
|
// already ran exact-turn reuse before policy filtering and vetoed pointer
|
|
2080
2564
|
// movement for manual/policy homes; this write only records the NEXT turn's
|
|
2081
2565
|
// policy home (or clears a policy pin whose strategy is no longer active).
|
|
@@ -2163,7 +2647,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2163
2647
|
if (codexLeaseHeld) startCodexLeaseHeartbeat();
|
|
2164
2648
|
|
|
2165
2649
|
const eligibleCount = leased.accounts.filter((account) =>
|
|
2166
|
-
isCodexCredentialEligible(account,
|
|
2650
|
+
isCodexCredentialEligible(account, new Date()),
|
|
2167
2651
|
).length;
|
|
2168
2652
|
const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
|
|
2169
2653
|
observability.incrementCounter({
|
|
@@ -2340,7 +2824,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2340
2824
|
if (goalActive && goal) {
|
|
2341
2825
|
const authoritativeResetAt = authoritativeCodexCapacityResetAt(
|
|
2342
2826
|
leased.accounts,
|
|
2343
|
-
settings.codexRotationNearExhaustionPct,
|
|
2344
2827
|
new Date(),
|
|
2345
2828
|
);
|
|
2346
2829
|
const armed = await armCodexCapacityWait(db, {
|
|
@@ -2482,7 +2965,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2482
2965
|
// Resolved after turn.started so a composition conflict (two enabled
|
|
2483
2966
|
// packs declaring images) fails the turn with its plain error instead
|
|
2484
2967
|
// of failing the activity opaquely.
|
|
2485
|
-
const packRuntime = await
|
|
2968
|
+
const [packRuntime, skillLibraryRuntime] = await Promise.all([
|
|
2969
|
+
resolveWorkspacePackRuntime(db, input.workspaceId),
|
|
2970
|
+
resolveWorkspaceSkillLibraryRuntime(db, input.workspaceId),
|
|
2971
|
+
]);
|
|
2486
2972
|
// RIG BINDING (M3): load the session's FROZEN rig version (resolved+frozen
|
|
2487
2973
|
// at create). Everything rig-derived below (image precedence, env default
|
|
2488
2974
|
// sets, setup hook, credential hooks, doctrine, lease/telemetry stamps) is
|
|
@@ -2552,7 +3038,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2552
3038
|
// raw window, effective ceiling, and auto-compact limit are distinct live
|
|
2553
3039
|
// catalog values and must reach pre-turn compaction, history guards, and
|
|
2554
3040
|
// every model call together.
|
|
2555
|
-
|
|
3041
|
+
modelRunSettings = resolvedModel
|
|
2556
3042
|
? settingsWithResolvedModelContext(runSettings, resolvedModel.configured)
|
|
2557
3043
|
: runSettings;
|
|
2558
3044
|
// WORKSPACE MODEL POLICY — the authoritative hard gate. Runs immediately
|
|
@@ -2653,6 +3139,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2653
3139
|
workspaceId: input.workspaceId,
|
|
2654
3140
|
sessionId: input.sessionId,
|
|
2655
3141
|
turnId: turn.id,
|
|
3142
|
+
turnAttemptId: input.attemptId,
|
|
2656
3143
|
model: resolvedModel?.configured.id ?? turn.model,
|
|
2657
3144
|
isCodexTurn,
|
|
2658
3145
|
usage: usage.usage,
|
|
@@ -2725,25 +3212,69 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2725
3212
|
);
|
|
2726
3213
|
let outcome: Awaited<ReturnType<typeof maybeCompactContext>> | null = null;
|
|
2727
3214
|
if (requested) {
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
3215
|
+
try {
|
|
3216
|
+
outcome = await waitForTurnOperation(
|
|
3217
|
+
maybeCompactContext(
|
|
3218
|
+
db,
|
|
3219
|
+
modelRunSettings,
|
|
3220
|
+
{
|
|
3221
|
+
accountId: input.accountId,
|
|
3222
|
+
workspaceId: input.workspaceId,
|
|
3223
|
+
sessionId: input.sessionId,
|
|
3224
|
+
turnId: turn.id,
|
|
3225
|
+
executionGeneration,
|
|
3226
|
+
attemptId: input.attemptId,
|
|
3227
|
+
},
|
|
3228
|
+
session.lastInputTokens,
|
|
3229
|
+
compactionSummarizerFor(compactionInstructions),
|
|
3230
|
+
{
|
|
3231
|
+
force: true,
|
|
3232
|
+
clearRequestedCompaction: true,
|
|
3233
|
+
trigger: "operator",
|
|
3234
|
+
},
|
|
3235
|
+
),
|
|
3236
|
+
cancellationSignal,
|
|
3237
|
+
undefined,
|
|
3238
|
+
);
|
|
3239
|
+
} catch (error) {
|
|
3240
|
+
// Codex retries retryable checkpoint-provider failures rather than
|
|
3241
|
+
// treating them as a semantic compaction result. Keep the operator
|
|
3242
|
+
// request pending and let the ordinary same-turn provider/capacity
|
|
3243
|
+
// recovery path re-dispatch this exact maintenance execution.
|
|
3244
|
+
if (shouldRecoverCompactionProviderFailure(error)) throw error;
|
|
3245
|
+
if (!isCompactionSummaryFailure(error)) throw error;
|
|
3246
|
+
const errorMessage = compactionFailureReasonFromError(error);
|
|
3247
|
+
if (
|
|
3248
|
+
!(await settle!({
|
|
3249
|
+
events: [
|
|
3250
|
+
{
|
|
3251
|
+
type: "turn.failed",
|
|
3252
|
+
payload: {
|
|
3253
|
+
error: errorMessage,
|
|
3254
|
+
code: "context_compaction_failed",
|
|
3255
|
+
retryable: false,
|
|
3256
|
+
recovery: "user_message",
|
|
3257
|
+
compacted: false,
|
|
3258
|
+
},
|
|
3259
|
+
},
|
|
3260
|
+
{
|
|
3261
|
+
type: "session.status.changed",
|
|
3262
|
+
payload: { status: "idle" },
|
|
3263
|
+
},
|
|
3264
|
+
],
|
|
3265
|
+
turnStatus: "failed",
|
|
3266
|
+
sessionStatus: "idle",
|
|
3267
|
+
activeTurnId: null,
|
|
3268
|
+
consumeRequestedCompactionFailure: true,
|
|
3269
|
+
}))
|
|
3270
|
+
) {
|
|
3271
|
+
return claimedResult({ status: "cancelled" });
|
|
3272
|
+
}
|
|
3273
|
+
turnMetricOutcome = "failed";
|
|
3274
|
+
activityStatus = "idle";
|
|
3275
|
+
activityError = error;
|
|
3276
|
+
return claimedResult({ status: "idle" });
|
|
3277
|
+
}
|
|
2747
3278
|
if (outcome.events.length > 0) {
|
|
2748
3279
|
if (outcome.compacted) {
|
|
2749
3280
|
recordContextCompaction(observability, "operator");
|
|
@@ -2794,18 +3325,22 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2794
3325
|
runSettings,
|
|
2795
3326
|
withFirstPartyTools(runSettings, mergeToolRefs(session.tools, turn.tools)),
|
|
2796
3327
|
);
|
|
2797
|
-
// §7.6
|
|
3328
|
+
// §7.6 connection-credential provider — load (and decrypt) the variable set via the host
|
|
2798
3329
|
// `sandboxSecrets` provider when bound; unset → today's local decrypt.
|
|
2799
3330
|
const connectionScope = {
|
|
2800
3331
|
accountId: input.accountId,
|
|
2801
3332
|
workspaceId: input.workspaceId,
|
|
2802
3333
|
};
|
|
2803
|
-
const workspaceVariableSet = await
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
3334
|
+
const workspaceVariableSet = await waitForTurnOperation(
|
|
3335
|
+
loadWorkspaceEnvironmentForRunWithCredentials(
|
|
3336
|
+
db,
|
|
3337
|
+
runSettings,
|
|
3338
|
+
connectionScope,
|
|
3339
|
+
session.variableSetId,
|
|
3340
|
+
connectionCredentials?.sandboxSecrets,
|
|
3341
|
+
),
|
|
3342
|
+
cancellationSignal,
|
|
3343
|
+
undefined,
|
|
2809
3344
|
);
|
|
2810
3345
|
variableSetId = workspaceVariableSet?.id ?? "";
|
|
2811
3346
|
// RIG DEFAULT VARIABLE SETS (M3): decrypt the frozen rig version's default
|
|
@@ -2820,12 +3355,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2820
3355
|
// set already relies on), keeping validateNoEnvironmentDelta empty.
|
|
2821
3356
|
const rigDefaultEnvironmentValues: Record<string, string> = {};
|
|
2822
3357
|
for (const rigDefaultVariableSetId of rigVersion?.defaultVariableSetIds ?? []) {
|
|
2823
|
-
const rigDefaultSet = await
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
3358
|
+
const rigDefaultSet = await waitForTurnOperation(
|
|
3359
|
+
loadWorkspaceEnvironmentForRunWithCredentials(
|
|
3360
|
+
db,
|
|
3361
|
+
runSettings,
|
|
3362
|
+
connectionScope,
|
|
3363
|
+
rigDefaultVariableSetId,
|
|
3364
|
+
connectionCredentials?.sandboxSecrets,
|
|
3365
|
+
),
|
|
3366
|
+
cancellationSignal,
|
|
3367
|
+
undefined,
|
|
2829
3368
|
);
|
|
2830
3369
|
Object.assign(rigDefaultEnvironmentValues, rigDefaultSet?.values ?? {});
|
|
2831
3370
|
}
|
|
@@ -2836,7 +3375,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2836
3375
|
);
|
|
2837
3376
|
// Redact EVERY exported secret value (rig defaults + session set) from turn
|
|
2838
3377
|
// output, not just the session set's.
|
|
2839
|
-
|
|
3378
|
+
registerSecretRedactions(
|
|
2840
3379
|
Object.entries(sandboxWorkspaceEnvironmentValues).map(([name, value]) => ({ name, value })),
|
|
2841
3380
|
);
|
|
2842
3381
|
// EFFECTIVE compute backend, resolved ONCE at turn start (Case B + Stage D
|
|
@@ -2909,7 +3448,56 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2909
3448
|
settings.sandboxOwnershipEnabled && runSettings.sandboxBackend !== "none"
|
|
2910
3449
|
? groupBoxBackend
|
|
2911
3450
|
: runSettings.sandboxBackend;
|
|
2912
|
-
|
|
3451
|
+
const effectiveRunCredentialBackend = activeSandboxBackend ?? groupBoxBackend;
|
|
3452
|
+
const runCredentialResolver =
|
|
3453
|
+
effectiveRunCredentialBackend === "none"
|
|
3454
|
+
? null
|
|
3455
|
+
: await waitForTurnOperation(
|
|
3456
|
+
bindRunCredentialResolver({
|
|
3457
|
+
db,
|
|
3458
|
+
connectionCredentials: connectionCredentials ?? null,
|
|
3459
|
+
accountId: input.accountId,
|
|
3460
|
+
workspaceId: input.workspaceId,
|
|
3461
|
+
session,
|
|
3462
|
+
turn,
|
|
3463
|
+
attemptId: input.attemptId,
|
|
3464
|
+
effectiveSandboxBackend: effectiveRunCredentialBackend,
|
|
3465
|
+
variableSet: workspaceVariableSet
|
|
3466
|
+
? {
|
|
3467
|
+
id: workspaceVariableSet.id,
|
|
3468
|
+
name: workspaceVariableSet.name,
|
|
3469
|
+
}
|
|
3470
|
+
: null,
|
|
3471
|
+
}),
|
|
3472
|
+
cancellationSignal,
|
|
3473
|
+
undefined,
|
|
3474
|
+
);
|
|
3475
|
+
const initialRunCredentialMaterial = runCredentialResolver
|
|
3476
|
+
? await waitForTurnOperation(
|
|
3477
|
+
runCredentialResolver.resolve({
|
|
3478
|
+
purpose: "provision",
|
|
3479
|
+
forceRefresh: false,
|
|
3480
|
+
}),
|
|
3481
|
+
cancellationSignal,
|
|
3482
|
+
undefined,
|
|
3483
|
+
)
|
|
3484
|
+
: null;
|
|
3485
|
+
if (initialRunCredentialMaterial) {
|
|
3486
|
+
registerSecretRedactions(initialRunCredentialMaterial.redactions);
|
|
3487
|
+
for (const payload of runCredentialAuthNeededPayloads(initialRunCredentialMaterial)) {
|
|
3488
|
+
publishedRunCredentialNotices.add(JSON.stringify(payload));
|
|
3489
|
+
await publish!([{ type: "credential.auth_needed", payload }], true);
|
|
3490
|
+
}
|
|
3491
|
+
}
|
|
3492
|
+
const runCredentialsNote = initialRunCredentialMaterial
|
|
3493
|
+
? runCredentialModelNote(initialRunCredentialMaterial)
|
|
3494
|
+
: undefined;
|
|
3495
|
+
throwIfTurnOperationCancelled(cancellationSignal);
|
|
3496
|
+
await waitForTurnOperation(
|
|
3497
|
+
ensureTurnModalRegistryImage(runSettings, sandboxCreationBackend),
|
|
3498
|
+
cancellationSignal,
|
|
3499
|
+
undefined,
|
|
3500
|
+
);
|
|
2913
3501
|
const establishPolicy: "eager" | "on-demand" =
|
|
2914
3502
|
lazyProvisionEnabled(settings) && !machinePrimary && runSettings.sandboxBackend !== "none"
|
|
2915
3503
|
? "on-demand"
|
|
@@ -2933,91 +3521,348 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2933
3521
|
// provider may supply it; unset still self-mints GitHub from settings.
|
|
2934
3522
|
// gitToken/gitTokens are undefined on the selfhosted skip path (the machine
|
|
2935
3523
|
// uses its own git creds).
|
|
3524
|
+
if (activeSandboxBackend !== "selfhosted") {
|
|
3525
|
+
await assertGitHubResourcesRemainAuthorized(db, input.workspaceId, turnResources);
|
|
3526
|
+
}
|
|
3527
|
+
const authorizeGitHubTokenMint: GitHubTokenMintAuthorization = async (selection) => {
|
|
3528
|
+
await assertGitHubTokenMintSelectionAuthorized(
|
|
3529
|
+
db,
|
|
3530
|
+
input.workspaceId,
|
|
3531
|
+
selection.installationId,
|
|
3532
|
+
selection.repositoryIds,
|
|
3533
|
+
);
|
|
3534
|
+
};
|
|
3535
|
+
// Git and MCP credentials share one lineage snapshot for this turn. A
|
|
3536
|
+
// host that supplies both ports must never see two independently resolved
|
|
3537
|
+
// roots for the same execution merely because the call sites are far apart.
|
|
3538
|
+
const needsHostCredentialRoot = Boolean(
|
|
3539
|
+
connectionCredentials?.gitCredentials || connectionCredentials?.mcpCredentials,
|
|
3540
|
+
);
|
|
3541
|
+
const hostCredentialRootSessionId = needsHostCredentialRoot
|
|
3542
|
+
? await getSessionRootId(db, input.workspaceId, input.sessionId)
|
|
3543
|
+
: null;
|
|
3544
|
+
if (needsHostCredentialRoot && !hostCredentialRootSessionId) {
|
|
3545
|
+
throw new Error(`cannot resolve host credentials for missing session ${input.sessionId}`);
|
|
3546
|
+
}
|
|
3547
|
+
const gitCredentialAuthority =
|
|
3548
|
+
connectionCredentials?.gitCredentials && hostCredentialRootSessionId
|
|
3549
|
+
? gitCredentialAuthorityForTurn({
|
|
3550
|
+
sessionId: input.sessionId,
|
|
3551
|
+
rootSessionId: hostCredentialRootSessionId,
|
|
3552
|
+
attemptId: input.attemptId,
|
|
3553
|
+
turn,
|
|
3554
|
+
})
|
|
3555
|
+
: undefined;
|
|
2936
3556
|
const {
|
|
2937
3557
|
environment: sandboxEnvironment,
|
|
2938
3558
|
gitToken: sandboxGitToken,
|
|
2939
3559
|
gitTokens: sandboxGitTokens,
|
|
2940
3560
|
gitTokenExpiresAt: sandboxGitTokenExpiresAt,
|
|
3561
|
+
gitCredentialBindings: sandboxGitCredentialBindings,
|
|
2941
3562
|
toolspaceToken: sandboxToolspaceToken,
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
|
|
2948
|
-
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
activeSandboxBackend
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
|
|
2955
|
-
|
|
2956
|
-
|
|
3563
|
+
toolspaceTokenExpiresAt: sandboxToolspaceTokenExpiresAt,
|
|
3564
|
+
} = await waitForTurnOperation(
|
|
3565
|
+
sandboxEnvironmentForRun(
|
|
3566
|
+
runSettings,
|
|
3567
|
+
turnResources,
|
|
3568
|
+
// Rig default sets merged BELOW the session set (session wins); rig-less
|
|
3569
|
+
// turns pass exactly workspaceVariableSet?.values (byte-for-byte today).
|
|
3570
|
+
sandboxWorkspaceEnvironmentValues,
|
|
3571
|
+
{
|
|
3572
|
+
skipGitHubToken: activeSandboxBackend === "selfhosted",
|
|
3573
|
+
deferGitHubToken:
|
|
3574
|
+
activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
|
|
3575
|
+
scope: connectionScope,
|
|
3576
|
+
...(gitCredentialAuthority ? { authority: gitCredentialAuthority } : {}),
|
|
3577
|
+
gitCredentials: connectionCredentials?.gitCredentials,
|
|
3578
|
+
authorizeGitHubTokenMint,
|
|
3579
|
+
sessionId: input.sessionId,
|
|
3580
|
+
runId: turnId,
|
|
3581
|
+
},
|
|
3582
|
+
),
|
|
3583
|
+
cancellationSignal,
|
|
3584
|
+
undefined,
|
|
2957
3585
|
);
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
? {
|
|
2961
|
-
gitTokens: sandboxGitTokens,
|
|
2962
|
-
expiresAt: sandboxGitTokenExpiresAt ?? {},
|
|
2963
|
-
}
|
|
3586
|
+
const sandboxToolspaceTokenFile = sandboxToolspaceToken
|
|
3587
|
+
? toolspaceTokenFileFromEnvironment(sandboxEnvironment, input.sessionId)
|
|
2964
3588
|
: undefined;
|
|
3589
|
+
|
|
3590
|
+
const initialGitCredentials: MintedRunGitCredentials | undefined =
|
|
3591
|
+
sandboxGitCredentialBindings
|
|
3592
|
+
? {
|
|
3593
|
+
bindings: sandboxGitCredentialBindings,
|
|
3594
|
+
gitTokens: sandboxGitTokens ?? {},
|
|
3595
|
+
expiresAt: sandboxGitTokenExpiresAt ?? {},
|
|
3596
|
+
}
|
|
3597
|
+
: undefined;
|
|
2965
3598
|
const attachGitCredentialRenewal = async (
|
|
2966
3599
|
tokenSession: GitCredentialTokenWriterSession,
|
|
2967
3600
|
initial: MintedRunGitCredentials | undefined,
|
|
2968
3601
|
): Promise<void> => {
|
|
2969
|
-
if (!initial ||
|
|
2970
|
-
const previous =
|
|
2971
|
-
|
|
2972
|
-
await previous
|
|
3602
|
+
if (!initial || initial.bindings.length === 0) return;
|
|
3603
|
+
const previous = gitCredentialRenewals;
|
|
3604
|
+
gitCredentialRenewals = [];
|
|
3605
|
+
await Promise.all(previous.map(async (controller) => await controller.stop()));
|
|
2973
3606
|
if (gitCredentialRenewalClosed) return;
|
|
2974
3607
|
|
|
2975
|
-
const
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
3608
|
+
const controllers = initial.bindings.map((initialBinding) => {
|
|
3609
|
+
let pendingBinding: typeof initialBinding | undefined;
|
|
3610
|
+
return startGitCredentialRenewalLoop({
|
|
3611
|
+
expectedProviders: [initialBinding.provider],
|
|
3612
|
+
initialExpiresAt: initialBinding.expiresAt
|
|
3613
|
+
? { [initialBinding.provider]: initialBinding.expiresAt }
|
|
3614
|
+
: {},
|
|
3615
|
+
mint: async () => {
|
|
3616
|
+
const binding = await mintRunGitCredentialBinding(
|
|
3617
|
+
runSettings,
|
|
3618
|
+
turnResources,
|
|
3619
|
+
initialBinding.provider,
|
|
3620
|
+
initialBinding.credentialBindingId,
|
|
3621
|
+
{
|
|
3622
|
+
scope: connectionScope,
|
|
3623
|
+
...(gitCredentialAuthority ? { authority: gitCredentialAuthority } : {}),
|
|
3624
|
+
gitCredentials: connectionCredentials?.gitCredentials,
|
|
3625
|
+
authorizeGitHubTokenMint,
|
|
3626
|
+
},
|
|
3627
|
+
);
|
|
3628
|
+
pendingBinding = binding;
|
|
3629
|
+
return binding
|
|
3630
|
+
? {
|
|
3631
|
+
bindings: [binding],
|
|
3632
|
+
gitTokens: { [binding.provider]: binding.token },
|
|
3633
|
+
expiresAt: binding.expiresAt ? { [binding.provider]: binding.expiresAt } : {},
|
|
3634
|
+
}
|
|
3635
|
+
: undefined;
|
|
3636
|
+
},
|
|
3637
|
+
write: async () => {
|
|
3638
|
+
if (!pendingBinding) {
|
|
3639
|
+
throw new Error("credential renewal produced no binding token");
|
|
3640
|
+
}
|
|
3641
|
+
const runAs = sandboxRunAs(runSettings);
|
|
3642
|
+
await refreshGitCredentialBindingTokenFiles(tokenSession, [pendingBinding], {
|
|
3643
|
+
...(runAs ? { runAs } : {}),
|
|
3644
|
+
...(toolCancellationFenceRef.current
|
|
3645
|
+
? {
|
|
3646
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
3647
|
+
toolCancellationFenceRef.current,
|
|
3648
|
+
),
|
|
3649
|
+
}
|
|
3650
|
+
: {}),
|
|
2996
3651
|
});
|
|
2997
|
-
}
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3652
|
+
},
|
|
3653
|
+
onSuccess: ({ providers: renewedProviders }) => {
|
|
3654
|
+
for (const provider of renewedProviders) {
|
|
3655
|
+
observability.incrementCounter({
|
|
3656
|
+
name: "opengeni_git_credential_renewals_total",
|
|
3657
|
+
help: "Host-managed Git credential renewal attempts by provider and outcome.",
|
|
3658
|
+
labels: { provider, outcome: "completed" },
|
|
3659
|
+
});
|
|
3660
|
+
}
|
|
3661
|
+
},
|
|
3662
|
+
onFailure: ({ providers: failedProviders, retryDelayMs, errorClass }) => {
|
|
3663
|
+
for (const provider of failedProviders) {
|
|
3664
|
+
observability.incrementCounter({
|
|
3665
|
+
name: "opengeni_git_credential_renewals_total",
|
|
3666
|
+
help: "Host-managed Git credential renewal attempts by provider and outcome.",
|
|
3667
|
+
labels: { provider, outcome: "error" },
|
|
3668
|
+
});
|
|
3669
|
+
}
|
|
3670
|
+
observability.warn("Sandbox Git credential renewal failed; retry scheduled", {
|
|
3671
|
+
sessionId: input.sessionId,
|
|
3672
|
+
turnId,
|
|
3673
|
+
providers: failedProviders.join(","),
|
|
3674
|
+
errorClass,
|
|
3675
|
+
retryDelayMs,
|
|
3005
3676
|
});
|
|
3006
|
-
}
|
|
3007
|
-
|
|
3677
|
+
},
|
|
3678
|
+
});
|
|
3679
|
+
});
|
|
3680
|
+
if (gitCredentialRenewalClosed) {
|
|
3681
|
+
await Promise.all(controllers.map(async (controller) => await controller.stop()));
|
|
3682
|
+
return;
|
|
3683
|
+
}
|
|
3684
|
+
gitCredentialRenewals = controllers;
|
|
3685
|
+
};
|
|
3686
|
+
|
|
3687
|
+
const attachToolspaceTokenRenewal = async (
|
|
3688
|
+
tokenSession: ToolspaceTokenWriterSession,
|
|
3689
|
+
initialExpiresAt = sandboxToolspaceTokenExpiresAt,
|
|
3690
|
+
): Promise<void> => {
|
|
3691
|
+
if (!sandboxToolspaceToken || !initialExpiresAt) return;
|
|
3692
|
+
const previous = toolspaceTokenRenewal;
|
|
3693
|
+
toolspaceTokenRenewal = null;
|
|
3694
|
+
await previous?.stop();
|
|
3695
|
+
if (toolspaceTokenRenewalClosed) return;
|
|
3696
|
+
|
|
3697
|
+
const mint = async () =>
|
|
3698
|
+
await mintSandboxToolspaceToken(runSettings, connectionScope, input.sessionId, turn.id);
|
|
3699
|
+
const write = async (material: NonNullable<Awaited<ReturnType<typeof mint>>>) => {
|
|
3700
|
+
const runAs = sandboxRunAs(runSettings);
|
|
3701
|
+
await refreshToolspaceTokenFile(tokenSession, material.token, {
|
|
3702
|
+
...(runAs ? { runAs } : {}),
|
|
3703
|
+
...(sandboxToolspaceTokenFile
|
|
3704
|
+
? {
|
|
3705
|
+
tokenFile: sandboxToolspaceTokenFile,
|
|
3706
|
+
legacyTokenFile: sandboxEnvironment.OPENGENI_TOOLSPACE_TOKEN_FILE!,
|
|
3707
|
+
}
|
|
3708
|
+
: {}),
|
|
3709
|
+
...(toolCancellationFenceRef.current
|
|
3710
|
+
? {
|
|
3711
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
3712
|
+
toolCancellationFenceRef.current,
|
|
3713
|
+
),
|
|
3714
|
+
}
|
|
3715
|
+
: {}),
|
|
3716
|
+
});
|
|
3717
|
+
};
|
|
3718
|
+
let renewalExpiresAt = initialExpiresAt;
|
|
3719
|
+
if (renewalExpiresAt.getTime() <= Date.now() + TOOLSPACE_TOKEN_EXPIRY_LEAD_MS) {
|
|
3720
|
+
const fresh = await mint();
|
|
3721
|
+
if (!fresh) {
|
|
3722
|
+
throw new Error("Toolspace token mint became unavailable during sandbox setup");
|
|
3723
|
+
}
|
|
3724
|
+
await write(fresh);
|
|
3725
|
+
renewalExpiresAt = fresh.expiresAt;
|
|
3726
|
+
}
|
|
3727
|
+
const controller = startToolspaceTokenRenewalLoop({
|
|
3728
|
+
initialExpiresAt: renewalExpiresAt,
|
|
3729
|
+
mint,
|
|
3730
|
+
write,
|
|
3731
|
+
onSuccess: () => {
|
|
3732
|
+
observability.incrementCounter({
|
|
3733
|
+
name: "opengeni_toolspace_token_renewals_total",
|
|
3734
|
+
help: "Sandbox Toolspace token renewal attempts by outcome.",
|
|
3735
|
+
labels: { outcome: "completed" },
|
|
3736
|
+
});
|
|
3737
|
+
},
|
|
3738
|
+
onFailure: ({ retryDelayMs, errorClass }) => {
|
|
3739
|
+
observability.incrementCounter({
|
|
3740
|
+
name: "opengeni_toolspace_token_renewals_total",
|
|
3741
|
+
help: "Sandbox Toolspace token renewal attempts by outcome.",
|
|
3742
|
+
labels: { outcome: "error" },
|
|
3743
|
+
});
|
|
3744
|
+
observability.warn("Sandbox Toolspace token renewal failed; retry scheduled", {
|
|
3008
3745
|
sessionId: input.sessionId,
|
|
3009
3746
|
turnId,
|
|
3010
|
-
providers: failedProviders.join(","),
|
|
3011
3747
|
errorClass,
|
|
3012
3748
|
retryDelayMs,
|
|
3013
3749
|
});
|
|
3014
3750
|
},
|
|
3015
3751
|
});
|
|
3016
|
-
if (
|
|
3752
|
+
if (toolspaceTokenRenewalClosed) {
|
|
3753
|
+
await controller.stop();
|
|
3754
|
+
return;
|
|
3755
|
+
}
|
|
3756
|
+
toolspaceTokenRenewal = controller;
|
|
3757
|
+
};
|
|
3758
|
+
|
|
3759
|
+
const attachRunCredentialRenewal = async (
|
|
3760
|
+
credentialSession: RunCredentialCommandSession,
|
|
3761
|
+
): Promise<void> => {
|
|
3762
|
+
if (!runCredentialResolver) return;
|
|
3763
|
+
const previous = runCredentialRenewal;
|
|
3764
|
+
runCredentialRenewal = null;
|
|
3765
|
+
await previous?.stop();
|
|
3766
|
+
if (runCredentialRenewalClosed) return;
|
|
3767
|
+
runCredentialSession = credentialSession;
|
|
3768
|
+
|
|
3769
|
+
if (!initialRunCredentialMaterial) {
|
|
3770
|
+
await clearRunCredentials(
|
|
3771
|
+
credentialSession,
|
|
3772
|
+
input.sessionId,
|
|
3773
|
+
toolCancellationFenceRef.current
|
|
3774
|
+
? toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
3775
|
+
toolCancellationFenceRef.current,
|
|
3776
|
+
)
|
|
3777
|
+
: undefined,
|
|
3778
|
+
);
|
|
3779
|
+
return;
|
|
3780
|
+
}
|
|
3781
|
+
|
|
3782
|
+
const write = async (
|
|
3783
|
+
material: NormalizedRunCredentialMaterial | null,
|
|
3784
|
+
pruneOtherAttempts = false,
|
|
3785
|
+
): Promise<void> => {
|
|
3786
|
+
if (!material) {
|
|
3787
|
+
await clearRunCredentialsForAttempt(credentialSession, {
|
|
3788
|
+
sessionId: input.sessionId,
|
|
3789
|
+
attemptId: input.attemptId,
|
|
3790
|
+
executionGeneration,
|
|
3791
|
+
});
|
|
3792
|
+
return;
|
|
3793
|
+
}
|
|
3794
|
+
registerSecretRedactions(material.redactions);
|
|
3795
|
+
await materializeRunCredentials(credentialSession, material, {
|
|
3796
|
+
sessionId: input.sessionId,
|
|
3797
|
+
attemptId: input.attemptId,
|
|
3798
|
+
executionGeneration,
|
|
3799
|
+
...(pruneOtherAttempts ? { pruneOtherAttempts: true } : {}),
|
|
3800
|
+
...(!pruneOtherAttempts ? { pruneSupersededGenerations: true } : {}),
|
|
3801
|
+
...(material.authNeeded.length > 0 &&
|
|
3802
|
+
Object.keys(material.environment).length === 0 &&
|
|
3803
|
+
material.files.length === 0
|
|
3804
|
+
? { prunePreviousGenerations: true }
|
|
3805
|
+
: {}),
|
|
3806
|
+
...(toolCancellationFenceRef.current
|
|
3807
|
+
? {
|
|
3808
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
3809
|
+
toolCancellationFenceRef.current,
|
|
3810
|
+
),
|
|
3811
|
+
}
|
|
3812
|
+
: {}),
|
|
3813
|
+
});
|
|
3814
|
+
for (const payload of runCredentialAuthNeededPayloads(material)) {
|
|
3815
|
+
const key = JSON.stringify(payload);
|
|
3816
|
+
if (publishedRunCredentialNotices.has(key)) continue;
|
|
3817
|
+
publishedRunCredentialNotices.add(key);
|
|
3818
|
+
await publish!([{ type: "credential.auth_needed", payload }], true);
|
|
3819
|
+
}
|
|
3820
|
+
};
|
|
3821
|
+
|
|
3822
|
+
const initialExpiryMs = initialRunCredentialMaterial.expiresAt?.getTime() ?? null;
|
|
3823
|
+
const seed =
|
|
3824
|
+
initialExpiryMs !== null && initialExpiryMs <= Date.now() + RUN_CREDENTIAL_EXPIRY_LEAD_MS
|
|
3825
|
+
? await runCredentialResolver.resolve({
|
|
3826
|
+
purpose: "provision",
|
|
3827
|
+
forceRefresh: true,
|
|
3828
|
+
})
|
|
3829
|
+
: initialRunCredentialMaterial;
|
|
3830
|
+
await write(seed, true);
|
|
3831
|
+
if (runCredentialRenewalClosed) return;
|
|
3832
|
+
const controller = startRunCredentialRenewalLoop({
|
|
3833
|
+
initialExpiresAt: seed?.expiresAt ?? null,
|
|
3834
|
+
resolve: async () =>
|
|
3835
|
+
await runCredentialResolver.resolve({
|
|
3836
|
+
purpose: "renewal",
|
|
3837
|
+
forceRefresh: true,
|
|
3838
|
+
}),
|
|
3839
|
+
write: async (material) => await write(material),
|
|
3840
|
+
onSuccess: ({ authNeeded }) => {
|
|
3841
|
+
observability.incrementCounter({
|
|
3842
|
+
name: "opengeni_run_credential_renewals_total",
|
|
3843
|
+
help: "Host-managed run credential renewal attempts by outcome.",
|
|
3844
|
+
labels: { outcome: authNeeded ? "auth_needed" : "completed" },
|
|
3845
|
+
});
|
|
3846
|
+
},
|
|
3847
|
+
onFailure: ({ retryDelayMs, errorClass }) => {
|
|
3848
|
+
observability.incrementCounter({
|
|
3849
|
+
name: "opengeni_run_credential_renewals_total",
|
|
3850
|
+
help: "Host-managed run credential renewal attempts by outcome.",
|
|
3851
|
+
labels: { outcome: "error" },
|
|
3852
|
+
});
|
|
3853
|
+
observability.warn("Host run credential renewal failed; retry scheduled", {
|
|
3854
|
+
sessionId: input.sessionId,
|
|
3855
|
+
turnId,
|
|
3856
|
+
errorClass,
|
|
3857
|
+
retryDelayMs,
|
|
3858
|
+
});
|
|
3859
|
+
},
|
|
3860
|
+
});
|
|
3861
|
+
if (runCredentialRenewalClosed) {
|
|
3017
3862
|
await controller.stop();
|
|
3018
3863
|
return;
|
|
3019
3864
|
}
|
|
3020
|
-
|
|
3865
|
+
runCredentialRenewal = controller;
|
|
3021
3866
|
};
|
|
3022
3867
|
|
|
3023
3868
|
// P1.2 ownership inversion (gated, default OFF). With the flag off this
|
|
@@ -3083,16 +3928,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3083
3928
|
// (buildSelfhostedBackendSession); EstablishedSandboxSession widens it.
|
|
3084
3929
|
machinePrimarySession =
|
|
3085
3930
|
established.session as import("@opengeni/runtime").SelfhostedSession;
|
|
3086
|
-
const lease = await
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
|
|
3931
|
+
const lease = await waitForTurnOperation(
|
|
3932
|
+
acquireSelfhostedLeaseForTurn(
|
|
3933
|
+
{ db, settings },
|
|
3934
|
+
{
|
|
3935
|
+
accountId: input.accountId,
|
|
3936
|
+
workspaceId: input.workspaceId,
|
|
3937
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
3938
|
+
sessionId: input.sessionId,
|
|
3939
|
+
},
|
|
3940
|
+
"turn",
|
|
3941
|
+
sandboxHolderId,
|
|
3942
|
+
),
|
|
3943
|
+
cancellationSignal,
|
|
3944
|
+
async (lateLease) => await lateLease.release(),
|
|
3096
3945
|
);
|
|
3097
3946
|
setupBoxSession = established.session;
|
|
3098
3947
|
resolvedSandbox = {
|
|
@@ -3137,45 +3986,49 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3137
3986
|
// proxy's first default-pointer op. A chat-only turn never calls it, so
|
|
3138
3987
|
// no lease row, no provider box, no warm-meter interval.
|
|
3139
3988
|
} else {
|
|
3140
|
-
resolvedSandbox = await
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
|
|
3156
|
-
|
|
3157
|
-
|
|
3158
|
-
|
|
3159
|
-
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3165
|
-
|
|
3166
|
-
|
|
3167
|
-
|
|
3168
|
-
|
|
3169
|
-
|
|
3170
|
-
|
|
3171
|
-
|
|
3172
|
-
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3176
|
-
|
|
3177
|
-
|
|
3178
|
-
|
|
3989
|
+
resolvedSandbox = await waitForTurnOperation(
|
|
3990
|
+
resumeBoxForTurn(
|
|
3991
|
+
{
|
|
3992
|
+
db,
|
|
3993
|
+
settings,
|
|
3994
|
+
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
3995
|
+
onSandboxLost: publishSandboxLost,
|
|
3996
|
+
},
|
|
3997
|
+
{
|
|
3998
|
+
accountId: input.accountId,
|
|
3999
|
+
workspaceId: input.workspaceId,
|
|
4000
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
4001
|
+
sessionId: input.sessionId,
|
|
4002
|
+
// groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
|
|
4003
|
+
// is not machine-primary resumes a real cloud group box (the
|
|
4004
|
+
// deployment default), never a "selfhosted" box (which would throw
|
|
4005
|
+
// for lack of a bound agentId).
|
|
4006
|
+
backend: groupBoxBackend,
|
|
4007
|
+
os: session.sandboxOs,
|
|
4008
|
+
environment: sandboxEnvironment,
|
|
4009
|
+
// IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
|
|
4010
|
+
// this run resolves. The lease stamps it + conflicts on a live shared box
|
|
4011
|
+
// running a DIFFERENT image (solo → recreate on the new image; N-holders →
|
|
4012
|
+
// SandboxImageConflictError surfaced as an actionable turn error). Prefer the
|
|
4013
|
+
// explicit Modal image ref, else the docker image. The selfhosted branch
|
|
4014
|
+
// (establishSelfhostedTurnSession/acquireSelfhostedLeaseForTurn) NEVER passes
|
|
4015
|
+
// an image — B3 lives only on this Modal else-branch.
|
|
4016
|
+
...((runSettings.modalImageRef ?? runSettings.dockerImage)
|
|
4017
|
+
? {
|
|
4018
|
+
image: runSettings.modalImageRef ?? runSettings.dockerImage,
|
|
4019
|
+
}
|
|
4020
|
+
: {}),
|
|
4021
|
+
// RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
|
|
4022
|
+
// conflicts on a live shared box set up under a different rig (solo
|
|
4023
|
+
// recreate / N-holders SandboxRigConflictError). Omitted for a rig-less
|
|
4024
|
+
// turn -> never stamped or enforced (shares exactly as today).
|
|
4025
|
+
...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
|
|
4026
|
+
},
|
|
4027
|
+
"turn",
|
|
4028
|
+
sandboxHolderId,
|
|
4029
|
+
),
|
|
4030
|
+
cancellationSignal,
|
|
4031
|
+
async (lateSandbox) => await lateSandbox.release(),
|
|
3179
4032
|
);
|
|
3180
4033
|
setupBoxSession = resolvedSandbox.established.session;
|
|
3181
4034
|
// Durable box-lifecycle events (sandbox-file-persistence observability):
|
|
@@ -3216,42 +4069,65 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3216
4069
|
}
|
|
3217
4070
|
}
|
|
3218
4071
|
|
|
3219
|
-
const fileResourceDownloads = await
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
4072
|
+
const fileResourceDownloads = await waitForTurnOperation(
|
|
4073
|
+
sandboxFileDownloadsForRun(
|
|
4074
|
+
runSettings,
|
|
4075
|
+
db,
|
|
4076
|
+
objectStorage,
|
|
4077
|
+
input.workspaceId,
|
|
4078
|
+
turnResources,
|
|
4079
|
+
),
|
|
4080
|
+
cancellationSignal,
|
|
4081
|
+
undefined,
|
|
3225
4082
|
);
|
|
3226
4083
|
throwIfWorkerShuttingDown();
|
|
4084
|
+
throwIfTurnCancelled();
|
|
4085
|
+
const mcpCredentialRootSessionId =
|
|
4086
|
+
connectionCredentials?.mcpCredentials && hostCredentialRootSessionId
|
|
4087
|
+
? hostCredentialRootSessionId
|
|
4088
|
+
: input.sessionId;
|
|
3227
4089
|
// Wrap MCP prep in the codex ALS so the codex_apps connect handshake
|
|
3228
4090
|
// (initialize + tools/list) can resolve the per-workspace bearer from
|
|
3229
4091
|
// codexRequestStorage (runtime/codexAppsMcpRequestInit). withCodex is the
|
|
3230
4092
|
// identity on every non-codex turn, so this is a no-op for existing paths.
|
|
3231
|
-
const resolveCredential =
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
|
|
3244
|
-
|
|
3245
|
-
|
|
3246
|
-
|
|
3247
|
-
|
|
3248
|
-
|
|
3249
|
-
|
|
3250
|
-
|
|
3251
|
-
|
|
3252
|
-
|
|
3253
|
-
|
|
3254
|
-
|
|
4093
|
+
const resolveCredential = connectionTokenResolverForTurn({
|
|
4094
|
+
db,
|
|
4095
|
+
settings: runSettings,
|
|
4096
|
+
connectionCredentials: connectionCredentials ?? null,
|
|
4097
|
+
accountId: input.accountId,
|
|
4098
|
+
workspaceId: input.workspaceId,
|
|
4099
|
+
sessionId: input.sessionId,
|
|
4100
|
+
rootSessionId: mcpCredentialRootSessionId,
|
|
4101
|
+
attemptId: input.attemptId,
|
|
4102
|
+
turn,
|
|
4103
|
+
});
|
|
4104
|
+
preparedTools = await waitForTurnOperation(
|
|
4105
|
+
withCodex(() =>
|
|
4106
|
+
runtime.prepareTools(runSettings, turnTools, {
|
|
4107
|
+
accountId: input.accountId,
|
|
4108
|
+
workspaceId: input.workspaceId,
|
|
4109
|
+
sessionId: input.sessionId,
|
|
4110
|
+
// Sign the calling turn into the first-party token so tools classify
|
|
4111
|
+
// the caller by its own identity (sacred-pause guard), not the racy
|
|
4112
|
+
// live active pointer.
|
|
4113
|
+
...(turnId ? { turnId } : {}),
|
|
4114
|
+
attemptId: input.attemptId,
|
|
4115
|
+
executionGeneration,
|
|
4116
|
+
subjectId: "worker:first-party-mcp",
|
|
4117
|
+
subjectLabel: "OpenGeni worker",
|
|
4118
|
+
resolveCredential,
|
|
4119
|
+
onAuthNeeded: async (payload) => {
|
|
4120
|
+
await publish!([{ type: "tool.auth_needed", payload }], true);
|
|
4121
|
+
},
|
|
4122
|
+
// Manager-style sessions carry a creation-validated permission set
|
|
4123
|
+
// for their first-party MCP token; null keeps the fixed default.
|
|
4124
|
+
...(session.firstPartyMcpPermissions?.length
|
|
4125
|
+
? { firstPartyPermissions: session.firstPartyMcpPermissions }
|
|
4126
|
+
: {}),
|
|
4127
|
+
}),
|
|
4128
|
+
),
|
|
4129
|
+
cancellationSignal,
|
|
4130
|
+
async (latePreparedTools) => await latePreparedTools.close().catch(() => undefined),
|
|
3255
4131
|
);
|
|
3256
4132
|
// Genesis turn = the first user turn (no assistant history reconciled
|
|
3257
4133
|
// yet). Durable Postgres state (countSessionHistoryItems includes
|
|
@@ -3292,11 +4168,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3292
4168
|
: {};
|
|
3293
4169
|
const agent = runtime.buildAgent(modelRunSettings, turnResources, {
|
|
3294
4170
|
reasoningEffort: turn.reasoningEffort,
|
|
4171
|
+
...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
|
|
3295
4172
|
genesisTitleHint: isGenesisTurn,
|
|
3296
4173
|
persistentSessionSettings: {
|
|
3297
4174
|
titleIsSet: Boolean(session.title?.trim()),
|
|
3298
4175
|
},
|
|
3299
4176
|
sandboxEnvironment,
|
|
4177
|
+
...(cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {}),
|
|
4178
|
+
onToolCancellationFence: (fence) => {
|
|
4179
|
+
toolCancellationFenceRef.current = fence;
|
|
4180
|
+
},
|
|
3300
4181
|
// TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
|
|
3301
4182
|
// seed. ONLY when the effective backend is NOT selfhosted (the connected
|
|
3302
4183
|
// machine uses its own git creds — mirrors the skipGitHubToken gate above)
|
|
@@ -3306,6 +4187,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3306
4187
|
...(activeSandboxBackend !== "selfhosted" && sandboxGitTokens
|
|
3307
4188
|
? { gitTokenSeeds: sandboxGitTokens }
|
|
3308
4189
|
: {}),
|
|
4190
|
+
...(activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings
|
|
4191
|
+
? { gitCredentialBindings: sandboxGitCredentialBindings }
|
|
4192
|
+
: {}),
|
|
3309
4193
|
...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
|
|
3310
4194
|
? { gitTokenSeed: sandboxGitToken }
|
|
3311
4195
|
: {}),
|
|
@@ -3316,7 +4200,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3316
4200
|
// than toolspace:call for its own session (own-session-bound, turn TTL,
|
|
3317
4201
|
// budgeted, approval-tools excluded). The runtime seeds it to the box's
|
|
3318
4202
|
// token file over the same exec channel, off-manifest, on every backend.
|
|
3319
|
-
...(sandboxToolspaceToken
|
|
4203
|
+
...(sandboxToolspaceToken
|
|
4204
|
+
? {
|
|
4205
|
+
toolspaceTokenSeed: sandboxToolspaceToken,
|
|
4206
|
+
toolspaceTokenSessionId: input.sessionId,
|
|
4207
|
+
}
|
|
4208
|
+
: {}),
|
|
3320
4209
|
...(activeSandboxBackend ? { activeSandboxBackend } : {}),
|
|
3321
4210
|
fileResourceDownloads,
|
|
3322
4211
|
mcpServers: preparedTools.mcpServers,
|
|
@@ -3384,12 +4273,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3384
4273
|
await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
|
|
3385
4274
|
},
|
|
3386
4275
|
...(packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {}),
|
|
4276
|
+
...(skillLibraryRuntime.skillLibrarySkills.length > 0
|
|
4277
|
+
? {
|
|
4278
|
+
skillLibrarySkills: skillLibraryRuntime.skillLibrarySkills,
|
|
4279
|
+
skillLibrarySelections: skillLibraryRuntime.skillLibrarySelections,
|
|
4280
|
+
}
|
|
4281
|
+
: {}),
|
|
3387
4282
|
...(workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {}),
|
|
3388
4283
|
...(workspaceMemory ? { workspaceMemory } : {}),
|
|
3389
4284
|
// Per-session persona tier (session > workspace > deployment default).
|
|
3390
4285
|
// Composed system-level AFTER the workspace persona so it refines it for
|
|
3391
4286
|
// this one session; absent ⇒ byte-identical to today's composition.
|
|
3392
4287
|
...(session.instructions ? { sessionInstructions: session.instructions } : {}),
|
|
4288
|
+
// Exact host context captured when this turn was accepted. It is
|
|
4289
|
+
// system-level and disappears with the turn rather than entering chat.
|
|
4290
|
+
...(turn.turnInstructions ? { turnInstructions: turn.turnInstructions } : {}),
|
|
3393
4291
|
...workspaceEnvironmentOption,
|
|
3394
4292
|
// RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
|
|
3395
4293
|
// the frozen version carries a non-empty script), and the rig credential
|
|
@@ -3414,6 +4312,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3414
4312
|
}
|
|
3415
4313
|
: {}),
|
|
3416
4314
|
});
|
|
4315
|
+
if (modelRunSettings.sandboxBackend !== "none" && toolCancellationFenceRef.current === null) {
|
|
4316
|
+
throw new Error(
|
|
4317
|
+
"Sandbox agent construction did not install the mandatory turn tool cancellation fence",
|
|
4318
|
+
);
|
|
4319
|
+
}
|
|
3417
4320
|
if (establishPolicy === "on-demand" && sandboxHolderId && sandboxGroupId) {
|
|
3418
4321
|
const lazyHolderId = sandboxHolderId;
|
|
3419
4322
|
const lazyGroupId = sandboxGroupId;
|
|
@@ -3424,18 +4327,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3424
4327
|
const lazyClient = {
|
|
3425
4328
|
backendId: sdkBackendIdForSandboxBackend(groupBoxBackend),
|
|
3426
4329
|
} as EstablishedSandboxSession["client"];
|
|
3427
|
-
let lazyEstablishmentOrigin: EstablishedSandboxSession["origin"] | null = null;
|
|
3428
4330
|
turnSandboxProvisioner = createTurnSandboxProvisioner<ResumedTurnSandbox>(
|
|
3429
4331
|
async () => {
|
|
3430
4332
|
throwIfWorkerShuttingDown();
|
|
4333
|
+
throwIfTurnCancelled();
|
|
3431
4334
|
const lazyGitCredentials =
|
|
3432
4335
|
activeSandboxBackend === "selfhosted"
|
|
3433
4336
|
? undefined
|
|
3434
4337
|
: await mintRunGitCredentials(runSettings, turnResources, {
|
|
3435
4338
|
scope: connectionScope,
|
|
4339
|
+
...(gitCredentialAuthority ? { authority: gitCredentialAuthority } : {}),
|
|
3436
4340
|
gitCredentials: connectionCredentials?.gitCredentials,
|
|
4341
|
+
authorizeGitHubTokenMint,
|
|
3437
4342
|
});
|
|
3438
4343
|
const lazyGitTokens = lazyGitCredentials?.gitTokens;
|
|
4344
|
+
const lazyToolspaceToken = sandboxToolspaceToken
|
|
4345
|
+
? await mintSandboxToolspaceToken(
|
|
4346
|
+
runSettings,
|
|
4347
|
+
connectionScope,
|
|
4348
|
+
input.sessionId,
|
|
4349
|
+
turn.id,
|
|
4350
|
+
)
|
|
4351
|
+
: undefined;
|
|
3439
4352
|
const provisioned = await resumeBoxForTurn(
|
|
3440
4353
|
{
|
|
3441
4354
|
db,
|
|
@@ -3460,13 +4373,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3460
4373
|
"turn",
|
|
3461
4374
|
lazyHolderId,
|
|
3462
4375
|
);
|
|
3463
|
-
lazyEstablishmentOrigin = provisioned.established.origin ?? null;
|
|
3464
|
-
setupBoxSession = provisioned.established.session;
|
|
3465
4376
|
await publishSandboxLifecycleEvents(provisioned);
|
|
4377
|
+
await attachRunCredentialRenewal(
|
|
4378
|
+
provisioned.established.session as RunCredentialCommandSession,
|
|
4379
|
+
);
|
|
4380
|
+
const provisionedSetupSession = initialRunCredentialMaterial
|
|
4381
|
+
? withRunCredentialsSession(
|
|
4382
|
+
provisioned.established.session as object,
|
|
4383
|
+
input.sessionId,
|
|
4384
|
+
)
|
|
4385
|
+
: provisioned.established.session;
|
|
3466
4386
|
await runOwnedSandboxSetup(
|
|
3467
4387
|
agent,
|
|
3468
4388
|
provisioned.established.session as never,
|
|
3469
|
-
|
|
4389
|
+
provisionedSetupSession as never,
|
|
3470
4390
|
{
|
|
3471
4391
|
settings: runSettings,
|
|
3472
4392
|
environment: sandboxEnvironment,
|
|
@@ -3474,8 +4394,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3474
4394
|
await publish?.([{ type: event.type, payload: event.payload }], true);
|
|
3475
4395
|
},
|
|
3476
4396
|
...(lazyGitTokens ? { gitTokenSeedsOverride: lazyGitTokens } : {}),
|
|
4397
|
+
...(lazyGitCredentials?.bindings
|
|
4398
|
+
? { gitCredentialBindingsOverride: lazyGitCredentials.bindings }
|
|
4399
|
+
: {}),
|
|
4400
|
+
...(lazyToolspaceToken
|
|
4401
|
+
? { toolspaceTokenSeedOverride: lazyToolspaceToken.token }
|
|
4402
|
+
: {}),
|
|
4403
|
+
...(toolCancellationFenceRef.current
|
|
4404
|
+
? {
|
|
4405
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
4406
|
+
toolCancellationFenceRef.current,
|
|
4407
|
+
),
|
|
4408
|
+
}
|
|
4409
|
+
: {}),
|
|
3477
4410
|
},
|
|
3478
4411
|
);
|
|
4412
|
+
await attachToolspaceTokenRenewal(
|
|
4413
|
+
provisioned.established.session as ToolspaceTokenWriterSession,
|
|
4414
|
+
lazyToolspaceToken?.expiresAt,
|
|
4415
|
+
);
|
|
3479
4416
|
await attachGitCredentialRenewal(
|
|
3480
4417
|
provisioned.established.session as GitCredentialTokenWriterSession,
|
|
3481
4418
|
lazyGitCredentials,
|
|
@@ -3488,11 +4425,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3488
4425
|
// already holds the proxy directly (injected as lazyOwnedSandbox.session), so it
|
|
3489
4426
|
// gets per-op routing; the worker-side handle (resolvedSandbox: release,
|
|
3490
4427
|
// heartbeat, computer-use recording) wants the real box, unproxied.
|
|
3491
|
-
resolvedSandbox = provisioned;
|
|
3492
|
-
startLeaseHeartbeat(provisioned, activeSandboxBackend ?? groupBoxBackend);
|
|
3493
4428
|
return provisioned;
|
|
3494
4429
|
},
|
|
3495
4430
|
{
|
|
4431
|
+
...(activityContext ? { signal: activityContext.cancellationSignal } : {}),
|
|
3496
4432
|
onStarted: async () => {
|
|
3497
4433
|
await publish?.(
|
|
3498
4434
|
[
|
|
@@ -3504,19 +4440,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3504
4440
|
true,
|
|
3505
4441
|
);
|
|
3506
4442
|
},
|
|
3507
|
-
onCompleted: async () => {
|
|
4443
|
+
onCompleted: async (provisioned) => {
|
|
3508
4444
|
await publish?.(
|
|
3509
4445
|
[
|
|
3510
4446
|
{
|
|
3511
4447
|
type: "sandbox.operation.completed",
|
|
3512
4448
|
payload: {
|
|
3513
4449
|
name: "sandbox.provision",
|
|
3514
|
-
...(
|
|
4450
|
+
...(provisioned.established.origin
|
|
4451
|
+
? { origin: provisioned.established.origin }
|
|
4452
|
+
: {}),
|
|
3515
4453
|
},
|
|
3516
4454
|
},
|
|
3517
4455
|
],
|
|
3518
4456
|
true,
|
|
3519
4457
|
);
|
|
4458
|
+
throwIfTurnOperationCancelled(activityContext?.cancellationSignal);
|
|
4459
|
+
startLeaseHeartbeat(provisioned, activeSandboxBackend ?? groupBoxBackend);
|
|
4460
|
+
setupBoxSession = provisioned.established.session;
|
|
4461
|
+
resolvedSandbox = provisioned;
|
|
3520
4462
|
},
|
|
3521
4463
|
onFailed: async (error) => {
|
|
3522
4464
|
await publish?.(
|
|
@@ -3532,6 +4474,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3532
4474
|
true,
|
|
3533
4475
|
);
|
|
3534
4476
|
},
|
|
4477
|
+
disposeResult: async (provisioned) => {
|
|
4478
|
+
await provisioned.release().catch(() => undefined);
|
|
4479
|
+
},
|
|
3535
4480
|
},
|
|
3536
4481
|
);
|
|
3537
4482
|
lazyOwnedSandbox = wrapLazyTurnBoxWithRouting(
|
|
@@ -3566,36 +4511,41 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3566
4511
|
// RunState verbatim and recovering attempts already compacted, if needed,
|
|
3567
4512
|
// before the first attempt's model boundary.
|
|
3568
4513
|
if (triggerType === "user.message" || triggerType === "system.update.delivered") {
|
|
4514
|
+
let forced = false;
|
|
3569
4515
|
try {
|
|
3570
4516
|
// Operator /compact (the slash command) sets a durable request flag;
|
|
3571
4517
|
// observe it without consuming it so a failed/stale attempt cannot
|
|
3572
4518
|
// lose the request. The replacement transaction clears it on success.
|
|
3573
|
-
|
|
3574
|
-
const outcome = await
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
|
|
3588
|
-
|
|
3589
|
-
|
|
3590
|
-
|
|
3591
|
-
|
|
3592
|
-
|
|
3593
|
-
|
|
3594
|
-
|
|
3595
|
-
|
|
3596
|
-
|
|
3597
|
-
|
|
3598
|
-
|
|
4519
|
+
forced = await isSessionCompactionRequested(db, input.workspaceId, input.sessionId);
|
|
4520
|
+
const outcome = await waitForTurnOperation(
|
|
4521
|
+
maybeCompactContext(
|
|
4522
|
+
db,
|
|
4523
|
+
modelRunSettings,
|
|
4524
|
+
{
|
|
4525
|
+
accountId: input.accountId,
|
|
4526
|
+
workspaceId: input.workspaceId,
|
|
4527
|
+
sessionId: input.sessionId,
|
|
4528
|
+
turnId: turnId!,
|
|
4529
|
+
executionGeneration,
|
|
4530
|
+
attemptId: input.attemptId,
|
|
4531
|
+
},
|
|
4532
|
+
session.lastInputTokens,
|
|
4533
|
+
// Provider-aware summarizer: when the turn's model resolved to a
|
|
4534
|
+
// registry provider, summarize on THAT provider's client + wire API
|
|
4535
|
+
// (a chat provider can't summarize through OpenAI/Azure). Null
|
|
4536
|
+
// resolution uses the built-in Responses summarizer with the same
|
|
4537
|
+
// session prompt-cache key as the main model calls.
|
|
4538
|
+
compactSummarizer,
|
|
4539
|
+
forced
|
|
4540
|
+
? {
|
|
4541
|
+
force: true,
|
|
4542
|
+
clearRequestedCompaction: true,
|
|
4543
|
+
trigger: "operator",
|
|
4544
|
+
}
|
|
4545
|
+
: { trigger: "auto" },
|
|
4546
|
+
),
|
|
4547
|
+
cancellationSignal,
|
|
4548
|
+
undefined,
|
|
3599
4549
|
);
|
|
3600
4550
|
if (outcome.compacted) {
|
|
3601
4551
|
const compactionTrigger = forced ? "operator" : undefined;
|
|
@@ -3608,12 +4558,41 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3608
4558
|
);
|
|
3609
4559
|
}
|
|
3610
4560
|
} catch (compactError) {
|
|
4561
|
+
if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
|
|
4562
|
+
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
4563
|
+
const errorMessage = compactionFailureReasonFromError(compactError);
|
|
3611
4564
|
observability.error("context compaction failed", {
|
|
3612
4565
|
sessionId: input.sessionId,
|
|
3613
4566
|
turnId,
|
|
3614
|
-
error:
|
|
4567
|
+
error: errorMessage,
|
|
3615
4568
|
});
|
|
3616
|
-
|
|
4569
|
+
if (
|
|
4570
|
+
!(await settle!({
|
|
4571
|
+
events: [
|
|
4572
|
+
{
|
|
4573
|
+
type: "turn.failed",
|
|
4574
|
+
payload: {
|
|
4575
|
+
error: errorMessage,
|
|
4576
|
+
code: "context_compaction_failed",
|
|
4577
|
+
retryable: false,
|
|
4578
|
+
recovery: "user_message",
|
|
4579
|
+
compacted: false,
|
|
4580
|
+
},
|
|
4581
|
+
},
|
|
4582
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
4583
|
+
],
|
|
4584
|
+
turnStatus: "failed",
|
|
4585
|
+
sessionStatus: "idle",
|
|
4586
|
+
activeTurnId: null,
|
|
4587
|
+
...(forced ? { consumeRequestedCompactionFailure: true } : {}),
|
|
4588
|
+
}))
|
|
4589
|
+
) {
|
|
4590
|
+
return claimedResult({ status: "cancelled" });
|
|
4591
|
+
}
|
|
4592
|
+
turnMetricOutcome = "failed";
|
|
4593
|
+
activityStatus = "idle";
|
|
4594
|
+
activityError = compactError;
|
|
4595
|
+
return claimedResult({ status: "idle" });
|
|
3617
4596
|
}
|
|
3618
4597
|
}
|
|
3619
4598
|
let fileMaterializationFailures: SandboxFileDownloadFailure[] = [];
|
|
@@ -3646,6 +4625,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3646
4625
|
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
3647
4626
|
},
|
|
3648
4627
|
...(runAs ? { runAs } : {}),
|
|
4628
|
+
...(toolCancellationFenceRef.current
|
|
4629
|
+
? {
|
|
4630
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
4631
|
+
toolCancellationFenceRef.current,
|
|
4632
|
+
),
|
|
4633
|
+
}
|
|
4634
|
+
: {}),
|
|
3649
4635
|
},
|
|
3650
4636
|
);
|
|
3651
4637
|
fileMaterializationFailures = materialized.failures;
|
|
@@ -3693,6 +4679,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3693
4679
|
turnId: activeTurnId,
|
|
3694
4680
|
recovering: turn.executionGeneration > 1,
|
|
3695
4681
|
...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
|
|
4682
|
+
...(runCredentialsNote ? { runCredentialsNote } : {}),
|
|
3696
4683
|
},
|
|
3697
4684
|
);
|
|
3698
4685
|
runInput = prepared.input;
|
|
@@ -3740,29 +4727,33 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3740
4727
|
triggerLabel: "overflow" | "proactive" | "operator",
|
|
3741
4728
|
recoverySignalTokens: number | null,
|
|
3742
4729
|
) => {
|
|
3743
|
-
const outcome = await
|
|
3744
|
-
|
|
3745
|
-
|
|
3746
|
-
|
|
3747
|
-
|
|
3748
|
-
|
|
3749
|
-
|
|
3750
|
-
|
|
3751
|
-
|
|
3752
|
-
|
|
3753
|
-
|
|
3754
|
-
|
|
3755
|
-
|
|
3756
|
-
|
|
3757
|
-
|
|
3758
|
-
|
|
3759
|
-
|
|
3760
|
-
|
|
3761
|
-
|
|
3762
|
-
|
|
3763
|
-
|
|
3764
|
-
|
|
3765
|
-
|
|
4730
|
+
const outcome = await waitForTurnOperation(
|
|
4731
|
+
maybeCompactContext(
|
|
4732
|
+
db,
|
|
4733
|
+
modelRunSettings,
|
|
4734
|
+
{
|
|
4735
|
+
accountId: input.accountId,
|
|
4736
|
+
workspaceId: input.workspaceId,
|
|
4737
|
+
sessionId: input.sessionId,
|
|
4738
|
+
turnId: activeTurnId,
|
|
4739
|
+
executionGeneration,
|
|
4740
|
+
attemptId: input.attemptId,
|
|
4741
|
+
},
|
|
4742
|
+
// Never reuse the persisted prior-turn signal for recovery. Proactive
|
|
4743
|
+
// guards provide their exact current signal; provider overflows do not,
|
|
4744
|
+
// so null derives decision metadata from active history. Forced
|
|
4745
|
+
// recovery proves progress separately by comparing the replacement
|
|
4746
|
+
// with the current active-history estimate in maybeCompactContext.
|
|
4747
|
+
recoverySignalTokens,
|
|
4748
|
+
compactSummarizer,
|
|
4749
|
+
{
|
|
4750
|
+
force: true,
|
|
4751
|
+
...(triggerLabel === "operator" ? { clearRequestedCompaction: true } : {}),
|
|
4752
|
+
trigger: triggerLabel,
|
|
4753
|
+
},
|
|
4754
|
+
),
|
|
4755
|
+
cancellationSignal,
|
|
4756
|
+
undefined,
|
|
3766
4757
|
);
|
|
3767
4758
|
if (outcome.events.length > 0) {
|
|
3768
4759
|
if (outcome.compacted) {
|
|
@@ -3785,10 +4776,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3785
4776
|
stream = undefined;
|
|
3786
4777
|
batcher = null;
|
|
3787
4778
|
let responseUsageCount = 0;
|
|
4779
|
+
// The SDK emits every processed call item for one model response before
|
|
4780
|
+
// it emits any result for that response. Keep that response-local batch
|
|
4781
|
+
// in memory so an orphan from an older response cannot pin later stable
|
|
4782
|
+
// calls. Durable recovery remains call/result based and needs no batch
|
|
4783
|
+
// schema or compatibility state.
|
|
4784
|
+
let currentToolBatchCallIds = new Set<string>();
|
|
4785
|
+
let currentToolBatchCompletedCallIds = new Set<string>();
|
|
3788
4786
|
// Actual input tokens of the most recent model response this turn; the
|
|
3789
4787
|
// pre-read trigger for the NEXT turn. Persisted at every turn-end path.
|
|
3790
|
-
let
|
|
4788
|
+
let lastProviderContextTokensObserved: number | null = null;
|
|
4789
|
+
let providerContextRevision = 0;
|
|
3791
4790
|
throwIfWorkerShuttingDown();
|
|
4791
|
+
throwIfTurnCancelled();
|
|
3792
4792
|
const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
|
|
3793
4793
|
const runStreamOnce = (): ReturnType<OpenGeniRuntime["runStream"]> =>
|
|
3794
4794
|
runtime.runStream(agent, runInput!, modelRunSettings, {
|
|
@@ -3831,9 +4831,48 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3831
4831
|
},
|
|
3832
4832
|
}
|
|
3833
4833
|
: {}),
|
|
3834
|
-
|
|
4834
|
+
...(sandboxToolspaceToken && sandboxToolspaceTokenExpiresAt && !lazyOwnedSandbox
|
|
4835
|
+
? {
|
|
4836
|
+
onToolspaceTokenSessionReady: async (
|
|
4837
|
+
tokenSession: ToolspaceTokenWriterSession,
|
|
4838
|
+
) => {
|
|
4839
|
+
const renewalSession =
|
|
4840
|
+
activeSandboxBackend === "selfhosted"
|
|
4841
|
+
? tokenSession
|
|
4842
|
+
: ((setupBoxSession as ToolspaceTokenWriterSession | null) ?? tokenSession);
|
|
4843
|
+
await attachToolspaceTokenRenewal(renewalSession);
|
|
4844
|
+
},
|
|
4845
|
+
}
|
|
4846
|
+
: {}),
|
|
4847
|
+
...(runCredentialResolver
|
|
4848
|
+
? {
|
|
4849
|
+
runCredentialSessionId: input.sessionId,
|
|
4850
|
+
...(!lazyOwnedSandbox
|
|
4851
|
+
? {
|
|
4852
|
+
onRunCredentialSessionReady: async (
|
|
4853
|
+
credentialSession: RunCredentialCommandSession,
|
|
4854
|
+
) => {
|
|
4855
|
+
const pinnedCredentialSession = setupBoxSession
|
|
4856
|
+
? (setupBoxSession as RunCredentialCommandSession)
|
|
4857
|
+
: credentialSession;
|
|
4858
|
+
await attachRunCredentialRenewal(pinnedCredentialSession);
|
|
4859
|
+
},
|
|
4860
|
+
}
|
|
4861
|
+
: {}),
|
|
4862
|
+
}
|
|
4863
|
+
: {}),
|
|
4864
|
+
contextCompactionSignal: () =>
|
|
4865
|
+
lastProviderContextTokensObserved === null
|
|
4866
|
+
? null
|
|
4867
|
+
: {
|
|
4868
|
+
revision: providerContextRevision,
|
|
4869
|
+
totalTokens: lastProviderContextTokensObserved,
|
|
4870
|
+
},
|
|
3835
4871
|
contextCompactionRequested: () =>
|
|
3836
4872
|
isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
4873
|
+
...(toolCancellationFenceRef.current
|
|
4874
|
+
? { turnToolCancellationFence: toolCancellationFenceRef.current }
|
|
4875
|
+
: {}),
|
|
3837
4876
|
});
|
|
3838
4877
|
if (codexLeaseLost) {
|
|
3839
4878
|
throw new Error("Codex credential lease expired before the model run");
|
|
@@ -3865,6 +4904,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3865
4904
|
streamDone = true;
|
|
3866
4905
|
break;
|
|
3867
4906
|
}
|
|
4907
|
+
let stableToolCallIdsToClear: string[] | null = null;
|
|
4908
|
+
let completedCurrentToolBatch = false;
|
|
3868
4909
|
const responseUsage = modelResponseUsageFromSdkEvent(next.value);
|
|
3869
4910
|
if (responseUsage) {
|
|
3870
4911
|
await recordCompletedModelCallBeforeOwnershipFences({
|
|
@@ -3890,6 +4931,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3890
4931
|
workspaceId: input.workspaceId,
|
|
3891
4932
|
sessionId: input.sessionId,
|
|
3892
4933
|
turnId: activeTurnId,
|
|
4934
|
+
turnAttemptId: input.attemptId,
|
|
3893
4935
|
model: turn.model,
|
|
3894
4936
|
isCodexTurn,
|
|
3895
4937
|
usage: responseUsage.usage,
|
|
@@ -3915,7 +4957,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3915
4957
|
const observed = responseUsage.usage?.inputTokens;
|
|
3916
4958
|
if (typeof observed === "number" && observed > 0) {
|
|
3917
4959
|
recordModelInputTokens(observability, streamProvider, observed);
|
|
3918
|
-
|
|
4960
|
+
}
|
|
4961
|
+
const observedTotal = providerContextTokens(responseUsage.usage);
|
|
4962
|
+
if (observedTotal !== null) {
|
|
4963
|
+
lastProviderContextTokensObserved = observedTotal;
|
|
4964
|
+
providerContextRevision += 1;
|
|
3919
4965
|
}
|
|
3920
4966
|
// Prompt-cache efficiency for this response — same usage frame as the
|
|
3921
4967
|
// input-token accounting above, so the two are always consistent.
|
|
@@ -3931,6 +4977,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3931
4977
|
}
|
|
3932
4978
|
},
|
|
3933
4979
|
});
|
|
4980
|
+
currentToolBatchCallIds = new Set<string>();
|
|
4981
|
+
currentToolBatchCompletedCallIds = new Set<string>();
|
|
3934
4982
|
await reconcileConversationTruth();
|
|
3935
4983
|
try {
|
|
3936
4984
|
await ensureRunAllowed(
|
|
@@ -3973,6 +5021,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3973
5021
|
"turn attempt ended while recording an in-flight tool call",
|
|
3974
5022
|
);
|
|
3975
5023
|
}
|
|
5024
|
+
currentToolBatchCallIds.add(pendingToolCall.callId);
|
|
3976
5025
|
}
|
|
3977
5026
|
const completedToolCall = completedToolCallFromSdkEvent(next.value);
|
|
3978
5027
|
if (completedToolCall) {
|
|
@@ -3994,27 +5043,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3994
5043
|
if (!recorded.accepted) {
|
|
3995
5044
|
throw new TurnAttemptFencedError(
|
|
3996
5045
|
"turn attempt ended while recording a tool-call result",
|
|
3997
|
-
);
|
|
3998
|
-
}
|
|
3999
|
-
|
|
4000
|
-
|
|
4001
|
-
|
|
4002
|
-
|
|
4003
|
-
|
|
4004
|
-
|
|
4005
|
-
|
|
4006
|
-
|
|
4007
|
-
|
|
4008
|
-
|
|
4009
|
-
|
|
4010
|
-
|
|
4011
|
-
|
|
4012
|
-
|
|
4013
|
-
|
|
4014
|
-
|
|
4015
|
-
|
|
4016
|
-
|
|
4017
|
-
|
|
5046
|
+
);
|
|
5047
|
+
}
|
|
5048
|
+
const belongsToCurrentBatch = currentToolBatchCallIds.has(completedToolCall.callId);
|
|
5049
|
+
if (belongsToCurrentBatch) {
|
|
5050
|
+
currentToolBatchCompletedCallIds.add(completedToolCall.callId);
|
|
5051
|
+
}
|
|
5052
|
+
const currentBatchIsStable =
|
|
5053
|
+
belongsToCurrentBatch &&
|
|
5054
|
+
currentToolBatchCallIds.size > 0 &&
|
|
5055
|
+
currentToolBatchCompletedCallIds.size === currentToolBatchCallIds.size;
|
|
5056
|
+
const standaloneStableResult =
|
|
5057
|
+
!belongsToCurrentBatch && currentToolBatchCallIds.size === 0;
|
|
5058
|
+
if (currentBatchIsStable || standaloneStableResult) {
|
|
5059
|
+
// Persist the SDK's now-stable complete call/result batch. Keep
|
|
5060
|
+
// the receipts until the normalized tool-output event below is
|
|
5061
|
+
// durably flushed: recovery then covers every crash boundary
|
|
5062
|
+
// without either losing or duplicating the UI projection.
|
|
5063
|
+
await reconcileConversationTruth({ requireDurable: true });
|
|
5064
|
+
stableToolCallIdsToClear = currentBatchIsStable
|
|
5065
|
+
? [...currentToolBatchCallIds]
|
|
5066
|
+
: [completedToolCall.callId];
|
|
5067
|
+
completedCurrentToolBatch = currentBatchIsStable;
|
|
4018
5068
|
}
|
|
4019
5069
|
}
|
|
4020
5070
|
const normalized = normalizeSdkEvent(next.value);
|
|
@@ -4022,6 +5072,26 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4022
5072
|
streamTiming.onEvent(event.type);
|
|
4023
5073
|
await batcher.push(event);
|
|
4024
5074
|
}
|
|
5075
|
+
if (stableToolCallIdsToClear) {
|
|
5076
|
+
const cleared = await clearDurablePendingSessionToolCalls(db, {
|
|
5077
|
+
accountId: input.accountId,
|
|
5078
|
+
workspaceId: input.workspaceId,
|
|
5079
|
+
sessionId: input.sessionId,
|
|
5080
|
+
turnId: activeTurnId,
|
|
5081
|
+
executionGeneration,
|
|
5082
|
+
attemptId: input.attemptId,
|
|
5083
|
+
callIds: stableToolCallIdsToClear,
|
|
5084
|
+
});
|
|
5085
|
+
if (!cleared.accepted) {
|
|
5086
|
+
throw new TurnAttemptFencedError(
|
|
5087
|
+
"turn attempt ended while finalizing tool-call results",
|
|
5088
|
+
);
|
|
5089
|
+
}
|
|
5090
|
+
if (completedCurrentToolBatch) {
|
|
5091
|
+
currentToolBatchCallIds = new Set<string>();
|
|
5092
|
+
currentToolBatchCompletedCallIds = new Set<string>();
|
|
5093
|
+
}
|
|
5094
|
+
}
|
|
4025
5095
|
}
|
|
4026
5096
|
} finally {
|
|
4027
5097
|
if (!streamDone) {
|
|
@@ -4036,14 +5106,27 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4036
5106
|
void iterator.return?.().catch(() => undefined);
|
|
4037
5107
|
}
|
|
4038
5108
|
}
|
|
4039
|
-
await
|
|
4040
|
-
|
|
5109
|
+
await waitForTurnStreamCleanup(
|
|
5110
|
+
batcher.flush(),
|
|
5111
|
+
stream.completed.catch(() => undefined),
|
|
5112
|
+
cancellationSignal,
|
|
5113
|
+
);
|
|
4041
5114
|
if (responseUsageCount === 0) {
|
|
4042
5115
|
const aggregateUsage = stream.state.usage;
|
|
4043
5116
|
const aggregateInput = (aggregateUsage as { inputTokens?: unknown } | undefined)
|
|
4044
5117
|
?.inputTokens;
|
|
4045
5118
|
if (typeof aggregateInput === "number" && aggregateInput > 0) {
|
|
4046
|
-
|
|
5119
|
+
const aggregateContext = providerContextTokens(
|
|
5120
|
+
aggregateUsage as {
|
|
5121
|
+
inputTokens?: number;
|
|
5122
|
+
outputTokens?: number;
|
|
5123
|
+
totalTokens?: number;
|
|
5124
|
+
},
|
|
5125
|
+
);
|
|
5126
|
+
if (aggregateContext !== null) {
|
|
5127
|
+
lastProviderContextTokensObserved = aggregateContext;
|
|
5128
|
+
providerContextRevision += 1;
|
|
5129
|
+
}
|
|
4047
5130
|
}
|
|
4048
5131
|
const aggregateSourceKey = modelUsageSourceKey({
|
|
4049
5132
|
responseId: null,
|
|
@@ -4075,6 +5158,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4075
5158
|
workspaceId: input.workspaceId,
|
|
4076
5159
|
sessionId: input.sessionId,
|
|
4077
5160
|
turnId: activeTurnId,
|
|
5161
|
+
turnAttemptId: input.attemptId,
|
|
4078
5162
|
model: turn.model,
|
|
4079
5163
|
isCodexTurn,
|
|
4080
5164
|
usage: aggregateUsage,
|
|
@@ -4108,23 +5192,59 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4108
5192
|
if (stream.interruptions.length > 0) {
|
|
4109
5193
|
await reconcileConversationTruth();
|
|
4110
5194
|
const approvals = runtime.serializeApprovals(stream.interruptions);
|
|
4111
|
-
|
|
4112
|
-
|
|
4113
|
-
|
|
4114
|
-
|
|
4115
|
-
|
|
4116
|
-
|
|
4117
|
-
|
|
4118
|
-
|
|
4119
|
-
|
|
4120
|
-
|
|
4121
|
-
|
|
4122
|
-
|
|
4123
|
-
|
|
5195
|
+
const humanInputInterruptions =
|
|
5196
|
+
runtime.serializeHumanInputRequests?.(stream.interruptions) ?? [];
|
|
5197
|
+
const humanInputRequests = await Promise.all(
|
|
5198
|
+
humanInputInterruptions.map(async (interruption) => {
|
|
5199
|
+
const id = stableHumanInputRequestId(
|
|
5200
|
+
input.sessionId,
|
|
5201
|
+
activeTurnId,
|
|
5202
|
+
interruption.toolCallId,
|
|
5203
|
+
);
|
|
5204
|
+
const existing = await getSessionHumanInputRequest(
|
|
5205
|
+
db,
|
|
5206
|
+
input.workspaceId,
|
|
5207
|
+
input.sessionId,
|
|
5208
|
+
id,
|
|
5209
|
+
);
|
|
5210
|
+
if (existing && existing.status !== "pending") {
|
|
5211
|
+
throw new Error(`Settled human-input request ${id} reappeared as an interruption`);
|
|
5212
|
+
}
|
|
5213
|
+
const expiresAt = existing?.expiresAt
|
|
5214
|
+
? new Date(existing.expiresAt)
|
|
5215
|
+
: interruption.input.expiresInSeconds
|
|
5216
|
+
? new Date(Date.now() + interruption.input.expiresInSeconds * 1000)
|
|
5217
|
+
: null;
|
|
5218
|
+
return {
|
|
5219
|
+
id,
|
|
5220
|
+
toolCallId: interruption.toolCallId,
|
|
5221
|
+
questions: interruption.input.questions,
|
|
5222
|
+
allowSkip: interruption.input.allowSkip,
|
|
5223
|
+
expiresAt,
|
|
5224
|
+
isNew: existing === null,
|
|
5225
|
+
};
|
|
5226
|
+
}),
|
|
5227
|
+
);
|
|
5228
|
+
const requestEvents = humanInputRequests
|
|
5229
|
+
.filter((request) => request.isNew)
|
|
5230
|
+
.map((request) => ({
|
|
5231
|
+
type: "session.humanInput.requested" as const,
|
|
5232
|
+
payload: {
|
|
5233
|
+
request: {
|
|
5234
|
+
id: request.id,
|
|
5235
|
+
questions: request.questions,
|
|
5236
|
+
allowSkip: request.allowSkip,
|
|
5237
|
+
expiresAt: request.expiresAt?.toISOString() ?? null,
|
|
5238
|
+
},
|
|
5239
|
+
},
|
|
5240
|
+
}));
|
|
4124
5241
|
if (
|
|
4125
5242
|
!(await settle!({
|
|
4126
5243
|
events: [
|
|
4127
|
-
|
|
5244
|
+
...requestEvents,
|
|
5245
|
+
...(approvals.length > 0
|
|
5246
|
+
? [{ type: "session.requiresAction" as const, payload: { approvals } }]
|
|
5247
|
+
: []),
|
|
4128
5248
|
{
|
|
4129
5249
|
type: "session.status.changed",
|
|
4130
5250
|
payload: { status: "requires_action" },
|
|
@@ -4133,6 +5253,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4133
5253
|
turnStatus: "requires_action",
|
|
4134
5254
|
sessionStatus: "requires_action",
|
|
4135
5255
|
activeTurnId,
|
|
5256
|
+
runState: {
|
|
5257
|
+
serializedRunState: stream.state.toString(),
|
|
5258
|
+
pendingApprovals: approvals,
|
|
5259
|
+
frozenCodexCredentialId: effectiveCodexCredentialId,
|
|
5260
|
+
humanInputRequests: humanInputRequests.map(
|
|
5261
|
+
({ isNew: _isNew, ...request }) => request,
|
|
5262
|
+
),
|
|
5263
|
+
},
|
|
4136
5264
|
}))
|
|
4137
5265
|
) {
|
|
4138
5266
|
return claimedResult({ status: "cancelled" });
|
|
@@ -4182,6 +5310,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4182
5310
|
unit: "run",
|
|
4183
5311
|
sourceResourceType: "session_turn",
|
|
4184
5312
|
sourceResourceId: activeTurnId,
|
|
5313
|
+
sessionId: input.sessionId,
|
|
5314
|
+
turnId: activeTurnId,
|
|
5315
|
+
turnAttemptId: input.attemptId,
|
|
4185
5316
|
idempotencyKey: `usage:agent_run.completed:${activeTurnId}`,
|
|
4186
5317
|
});
|
|
4187
5318
|
activityStatus = "idle";
|
|
@@ -4242,9 +5373,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4242
5373
|
}
|
|
4243
5374
|
}
|
|
4244
5375
|
} catch (compactError) {
|
|
4245
|
-
|
|
4246
|
-
|
|
4247
|
-
|
|
5376
|
+
// Transient checkpoint-provider failures recover this same accepted
|
|
5377
|
+
// turn through the normal provider/capacity path. They are not an
|
|
5378
|
+
// empty summary and must not create a new goal continuation.
|
|
5379
|
+
if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
|
|
5380
|
+
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
5381
|
+
compactionFailureMessage = compactionFailureReasonFromError(compactError);
|
|
4248
5382
|
observability.warn("context compaction recovery compaction failed", {
|
|
4249
5383
|
sessionId: input.sessionId,
|
|
4250
5384
|
turnId: activeTurnId,
|
|
@@ -4276,6 +5410,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4276
5410
|
turnStatus: "failed",
|
|
4277
5411
|
sessionStatus: "idle",
|
|
4278
5412
|
activeTurnId: null,
|
|
5413
|
+
...(recoveryKind === "operator" ? { consumeRequestedCompactionFailure: true } : {}),
|
|
4279
5414
|
}))
|
|
4280
5415
|
) {
|
|
4281
5416
|
return claimedResult({ status: "cancelled" });
|
|
@@ -4283,7 +5418,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4283
5418
|
turnMetricOutcome = "failed";
|
|
4284
5419
|
activityStatus = "idle";
|
|
4285
5420
|
activityError = attemptError;
|
|
4286
|
-
|
|
5421
|
+
// The failed turn settlement already defers ordinary internal
|
|
5422
|
+
// updates and makes the delivered goal-continuation receipt
|
|
5423
|
+
// terminal. End this workflow run as well: returning plain idle
|
|
5424
|
+
// would immediately synthesize another goal continuation against
|
|
5425
|
+
// the unchanged active history and repeat the same failed
|
|
5426
|
+
// compaction. A new human/API prompt, Steer, or explicitly requested
|
|
5427
|
+
// Compact remains a durable explicit wake and may retry; ordinary
|
|
5428
|
+
// machine updates stay pending for that actionable wake.
|
|
5429
|
+
return claimedResult({ status: "idle", deferredUntilWake: true });
|
|
4287
5430
|
}
|
|
4288
5431
|
// Codex parity: compaction remains inside the same logical turn and
|
|
4289
5432
|
// the same activity. Rebuild the model-visible history from the
|
|
@@ -4326,6 +5469,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4326
5469
|
reason: "sandbox_lease_superseded",
|
|
4327
5470
|
});
|
|
4328
5471
|
if (recovery.action === "stale") {
|
|
5472
|
+
acknowledgeLostAttemptOwnership();
|
|
4329
5473
|
activityStatus = "cancelled";
|
|
4330
5474
|
turnMetricOutcome = "cancelled";
|
|
4331
5475
|
return claimedResult({ status: "cancelled" });
|
|
@@ -4344,7 +5488,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4344
5488
|
throw recoveryError;
|
|
4345
5489
|
}
|
|
4346
5490
|
}
|
|
4347
|
-
|
|
5491
|
+
const cancellationFailure = turnOperationCancellationFailure(error);
|
|
5492
|
+
if (
|
|
5493
|
+
cancellationFailure &&
|
|
5494
|
+
isWorkerShutdownCancellation(cancellationFailure) &&
|
|
5495
|
+
recoveryTurnId
|
|
5496
|
+
) {
|
|
4348
5497
|
try {
|
|
4349
5498
|
await flushRuntimeBatcher();
|
|
4350
5499
|
await reconcileConversationTruth();
|
|
@@ -4363,6 +5512,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4363
5512
|
reason: "worker_shutdown",
|
|
4364
5513
|
});
|
|
4365
5514
|
if (recovery.action === "stale") {
|
|
5515
|
+
acknowledgeLostAttemptOwnership();
|
|
4366
5516
|
activityStatus = "cancelled";
|
|
4367
5517
|
turnMetricOutcome = "cancelled";
|
|
4368
5518
|
return claimedResult({ status: "cancelled" });
|
|
@@ -4387,26 +5537,35 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4387
5537
|
if (error instanceof TurnAttemptFencedError) {
|
|
4388
5538
|
activityStatus = "cancelled";
|
|
4389
5539
|
activityError = error;
|
|
4390
|
-
|
|
5540
|
+
acknowledgeQuiescence = true;
|
|
5541
|
+
noteCancellationRequested();
|
|
5542
|
+
await waitForTurnFinalizerStep(
|
|
5543
|
+
flushRuntimeBatcher(),
|
|
5544
|
+
turnFinalizerCancellationSignal(cancellationSignal, activityStatus),
|
|
5545
|
+
);
|
|
4391
5546
|
// Ownership already moved to a newer attempt or an authoritative
|
|
4392
|
-
// control transaction. Surface
|
|
4393
|
-
// normal
|
|
4394
|
-
//
|
|
4395
|
-
//
|
|
4396
|
-
// server and leaves the worker task detached indefinitely.
|
|
5547
|
+
// control transaction. Surface the exact transport cancellation rather
|
|
5548
|
+
// than a normal result. Temporal terminalization remains diagnostic
|
|
5549
|
+
// only; replacement admission waits for the activity-owned durable
|
|
5550
|
+
// quiescence receipt written from the hard tool fence below.
|
|
4397
5551
|
turnMetricOutcome = "cancelled";
|
|
4398
5552
|
throw new CancelledFailure("TURN_ATTEMPT_FENCED", [], error);
|
|
4399
5553
|
}
|
|
4400
|
-
if (
|
|
5554
|
+
if (cancellationFailure) {
|
|
4401
5555
|
activityStatus = "cancelled";
|
|
4402
5556
|
activityError = error;
|
|
4403
|
-
|
|
5557
|
+
acknowledgeQuiescence = true;
|
|
5558
|
+
noteCancellationRequested();
|
|
5559
|
+
await waitForTurnFinalizerStep(
|
|
5560
|
+
flushRuntimeBatcher(),
|
|
5561
|
+
turnFinalizerCancellationSignal(cancellationSignal, activityStatus),
|
|
5562
|
+
);
|
|
4404
5563
|
// The workflow owns cancellation settlement: Pause/Steer controls use
|
|
4405
5564
|
// settleSessionControl, and heartbeat timeouts use worker-death
|
|
4406
5565
|
// recovery. A dying activity must never append a
|
|
4407
5566
|
// competing cancellation or mutate the turn/session on its own.
|
|
4408
5567
|
turnMetricOutcome = "cancelled";
|
|
4409
|
-
throw
|
|
5568
|
+
throw cancellationFailure;
|
|
4410
5569
|
}
|
|
4411
5570
|
// The SDK's per-segment turn cap is a pacing valve, not a failure: end
|
|
4412
5571
|
// the turn gracefully and idle the session so an active goal continues
|
|
@@ -4448,6 +5607,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4448
5607
|
unit: "run",
|
|
4449
5608
|
sourceResourceType: "session_turn",
|
|
4450
5609
|
sourceResourceId: turnId,
|
|
5610
|
+
sessionId: input.sessionId,
|
|
5611
|
+
turnId,
|
|
5612
|
+
turnAttemptId: input.attemptId,
|
|
4451
5613
|
idempotencyKey: `usage:agent_run.completed:${turnId}`,
|
|
4452
5614
|
});
|
|
4453
5615
|
activityStatus = "idle";
|
|
@@ -4613,12 +5775,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4613
5775
|
: {}),
|
|
4614
5776
|
}
|
|
4615
5777
|
: null;
|
|
4616
|
-
const cooldownUntil = codexCredentialCooldownUntil(
|
|
4617
|
-
codexCredentialFailure,
|
|
4618
|
-
serving,
|
|
4619
|
-
settings.codexRotationNearExhaustionPct,
|
|
4620
|
-
now,
|
|
4621
|
-
);
|
|
5778
|
+
const cooldownUntil = codexCredentialCooldownUntil(codexCredentialFailure, serving, now);
|
|
4622
5779
|
const statePersisted =
|
|
4623
5780
|
codexLeaseHolderId && codexLeaseGeneration !== null
|
|
4624
5781
|
? await quarantineCodexCredentialForLease(db, {
|
|
@@ -4663,7 +5820,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4663
5820
|
activeCredentialId: rotation.activeCredentialId,
|
|
4664
5821
|
priorCredentialId: effectiveCodexCredentialId,
|
|
4665
5822
|
accounts,
|
|
4666
|
-
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
4667
5823
|
now: new Date(),
|
|
4668
5824
|
usedConnectors: serving?.connectorNamespaces ?? [],
|
|
4669
5825
|
})
|
|
@@ -4725,9 +5881,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4725
5881
|
if (settlement.action === "stale") {
|
|
4726
5882
|
// One transaction proves both exact-holder recovery (including a
|
|
4727
5883
|
// just-expired or reaped lease row) and successor/control-gate
|
|
4728
|
-
// rejection.
|
|
4729
|
-
|
|
4730
|
-
|
|
5884
|
+
// rejection. Cross the hard tool fence so a control-gate loss can
|
|
5885
|
+
// write its quiescence receipt; a successor-only loss is a no-op.
|
|
5886
|
+
acknowledgeLostAttemptOwnership();
|
|
5887
|
+
activityStatus = "cancelled";
|
|
5888
|
+
turnMetricOutcome = "cancelled";
|
|
4731
5889
|
return claimedResult({ status: "recovering" });
|
|
4732
5890
|
}
|
|
4733
5891
|
}
|
|
@@ -4784,7 +5942,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4784
5942
|
const until = codexCredentialCooldownUntil(
|
|
4785
5943
|
{ kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds },
|
|
4786
5944
|
serving,
|
|
4787
|
-
settings.codexRotationNearExhaustionPct,
|
|
4788
5945
|
new Date(),
|
|
4789
5946
|
)!;
|
|
4790
5947
|
// Finding 1a: INSPECT the cooldown-write result. A swallowed best-effort
|
|
@@ -4822,7 +5979,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4822
5979
|
const newHome = shardCredentialForSession({
|
|
4823
5980
|
sessionId: input.sessionId,
|
|
4824
5981
|
accounts: fresh,
|
|
4825
|
-
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
4826
5982
|
now: new Date(),
|
|
4827
5983
|
});
|
|
4828
5984
|
if (newHome) {
|
|
@@ -4871,16 +6027,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4871
6027
|
} else {
|
|
4872
6028
|
// Every account capped/cooling → idle until the earliest reset across all.
|
|
4873
6029
|
rotated = true;
|
|
4874
|
-
allCappedResetAt = earliestCodexReset(
|
|
4875
|
-
|
|
4876
|
-
settings.codexRotationNearExhaustionPct,
|
|
4877
|
-
new Date(),
|
|
4878
|
-
);
|
|
4879
|
-
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(
|
|
4880
|
-
fresh,
|
|
4881
|
-
settings.codexRotationNearExhaustionPct,
|
|
4882
|
-
new Date(),
|
|
4883
|
-
);
|
|
6030
|
+
allCappedResetAt = earliestCodexReset(fresh, new Date());
|
|
6031
|
+
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
|
|
4884
6032
|
}
|
|
4885
6033
|
} else {
|
|
4886
6034
|
const decision = chooseRotationActive({
|
|
@@ -4888,7 +6036,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4888
6036
|
activeCredentialId: rotation.activeCredentialId,
|
|
4889
6037
|
priorCredentialId: effectiveCodexCredentialId,
|
|
4890
6038
|
accounts: fresh,
|
|
4891
|
-
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
4892
6039
|
now: new Date(),
|
|
4893
6040
|
// P4: the just-capped serving account's connector set is the proxy for
|
|
4894
6041
|
// "what this session has access to" — prefer a covering failover target.
|
|
@@ -4915,11 +6062,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4915
6062
|
} else if (decision.kind === "allCapped") {
|
|
4916
6063
|
rotated = true;
|
|
4917
6064
|
allCappedResetAt = decision.earliestResetAt;
|
|
4918
|
-
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(
|
|
4919
|
-
fresh,
|
|
4920
|
-
settings.codexRotationNearExhaustionPct,
|
|
4921
|
-
new Date(),
|
|
4922
|
-
);
|
|
6065
|
+
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
|
|
4923
6066
|
}
|
|
4924
6067
|
// kind:"none" → fall through to today's single-account idle.
|
|
4925
6068
|
}
|
|
@@ -5094,6 +6237,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5094
6237
|
unit: "run",
|
|
5095
6238
|
sourceResourceType: "session_turn",
|
|
5096
6239
|
sourceResourceId: turnId,
|
|
6240
|
+
sessionId: input.sessionId,
|
|
6241
|
+
turnId,
|
|
6242
|
+
turnAttemptId: input.attemptId,
|
|
5097
6243
|
idempotencyKey: `usage:agent_run.completed:${turnId}`,
|
|
5098
6244
|
});
|
|
5099
6245
|
activityStatus = "idle";
|
|
@@ -5106,6 +6252,34 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5106
6252
|
// it after a pacing delay. This is independent of goal state and never
|
|
5107
6253
|
// relies on a synthetic continuation prompt.
|
|
5108
6254
|
const failure = agentRunFailurePayload(error);
|
|
6255
|
+
if (isSessionEventPersistenceError(error)) {
|
|
6256
|
+
// Never pass the original Drizzle/postgres-js error to telemetry: its
|
|
6257
|
+
// nested cause may contain raw SQL and bound parameters. The typed DB
|
|
6258
|
+
// boundary retains only SQLSTATE, stage, correlation, and safe catalog
|
|
6259
|
+
// identifiers. Provider inference has already happened and is NOT
|
|
6260
|
+
// retried by this terminal classification.
|
|
6261
|
+
observability.error("session event persistence failed", {
|
|
6262
|
+
accountId: input.accountId,
|
|
6263
|
+
workspaceId: input.workspaceId,
|
|
6264
|
+
sessionId: input.sessionId,
|
|
6265
|
+
turnId,
|
|
6266
|
+
attemptId: input.attemptId,
|
|
6267
|
+
code: error.details.code,
|
|
6268
|
+
sqlState: error.details.sqlState ?? "unknown",
|
|
6269
|
+
stage: error.details.stage,
|
|
6270
|
+
eventTypes: error.details.eventTypes.join(","),
|
|
6271
|
+
correlationId: error.details.correlationId,
|
|
6272
|
+
attempts: error.details.attempts,
|
|
6273
|
+
retryOutcome: error.details.retryOutcome,
|
|
6274
|
+
dbSeverity: error.details.database.severity,
|
|
6275
|
+
dbSchema: error.details.database.schema,
|
|
6276
|
+
dbTable: error.details.database.table,
|
|
6277
|
+
dbColumn: error.details.database.column,
|
|
6278
|
+
dbDataType: error.details.database.dataType,
|
|
6279
|
+
dbConstraint: error.details.database.constraint,
|
|
6280
|
+
dbRoutine: error.details.database.routine,
|
|
6281
|
+
});
|
|
6282
|
+
}
|
|
5109
6283
|
if (failure.retryable && publish && turnId && turnStartedPublished) {
|
|
5110
6284
|
const recoveryResult = providerRecoveryResult();
|
|
5111
6285
|
await flushRuntimeBatcher();
|
|
@@ -5122,6 +6296,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5122
6296
|
},
|
|
5123
6297
|
});
|
|
5124
6298
|
if (recovery.action === "stale") {
|
|
6299
|
+
acknowledgeLostAttemptOwnership();
|
|
5125
6300
|
activityStatus = "cancelled";
|
|
5126
6301
|
turnMetricOutcome = "cancelled";
|
|
5127
6302
|
return claimedResult({ status: "cancelled" });
|
|
@@ -5173,215 +6348,490 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5173
6348
|
);
|
|
5174
6349
|
return claimedResult({ status: "failed" });
|
|
5175
6350
|
} finally {
|
|
5176
|
-
|
|
5177
|
-
|
|
5178
|
-
|
|
5179
|
-
|
|
5180
|
-
const
|
|
5181
|
-
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5185
|
-
|
|
5186
|
-
|
|
5187
|
-
|
|
5188
|
-
|
|
5189
|
-
|
|
5190
|
-
|
|
5191
|
-
|
|
5192
|
-
|
|
5193
|
-
|
|
5194
|
-
|
|
5195
|
-
|
|
5196
|
-
|
|
5197
|
-
|
|
5198
|
-
|
|
5199
|
-
|
|
5200
|
-
|
|
5201
|
-
|
|
5202
|
-
|
|
5203
|
-
|
|
5204
|
-
|
|
5205
|
-
|
|
5206
|
-
|
|
5207
|
-
|
|
5208
|
-
|
|
5209
|
-
|
|
5210
|
-
|
|
5211
|
-
|
|
5212
|
-
|
|
5213
|
-
|
|
5214
|
-
|
|
5215
|
-
|
|
5216
|
-
|
|
5217
|
-
|
|
5218
|
-
|
|
5219
|
-
|
|
5220
|
-
|
|
5221
|
-
|
|
5222
|
-
|
|
5223
|
-
|
|
5224
|
-
|
|
5225
|
-
|
|
5226
|
-
|
|
5227
|
-
|
|
5228
|
-
|
|
5229
|
-
|
|
5230
|
-
|
|
5231
|
-
|
|
5232
|
-
|
|
5233
|
-
|
|
5234
|
-
|
|
5235
|
-
|
|
5236
|
-
|
|
5237
|
-
|
|
5238
|
-
|
|
5239
|
-
|
|
6351
|
+
const finalizationStarted = performance.now();
|
|
6352
|
+
let finalizationError: unknown;
|
|
6353
|
+
let physicalToolQuiescenceConfirmed = !acknowledgeQuiescence;
|
|
6354
|
+
let quiescenceReceiptOrProofDurable = !acknowledgeQuiescence;
|
|
6355
|
+
const finalizerSignal = turnFinalizerCancellationSignal(cancellationSignal, activityStatus);
|
|
6356
|
+
try {
|
|
6357
|
+
const toolCancellationFence = toolCancellationFenceRef.current;
|
|
6358
|
+
// Toolspace and run-credential renewals are attempt-owned sandbox
|
|
6359
|
+
// writers. Capture and close them before crossing the same authoritative
|
|
6360
|
+
// boundary as tool calls; none of these drains may race the already-
|
|
6361
|
+
// aborted Temporal signal.
|
|
6362
|
+
toolspaceTokenRenewalClosed = true;
|
|
6363
|
+
const toolspaceRenewalToStop =
|
|
6364
|
+
toolspaceTokenRenewal as ToolspaceTokenRenewalController | null;
|
|
6365
|
+
toolspaceTokenRenewal = null;
|
|
6366
|
+
runCredentialRenewalClosed = true;
|
|
6367
|
+
const runRenewalToStop = runCredentialRenewal as RunCredentialRenewalController | null;
|
|
6368
|
+
runCredentialRenewal = null;
|
|
6369
|
+
await drainAttemptOwnedSandboxWriters({
|
|
6370
|
+
toolCancellationFence: acknowledgeQuiescence ? toolCancellationFence : null,
|
|
6371
|
+
cancellationReason: cancellationSignal?.reason,
|
|
6372
|
+
toolspaceTokenRenewal: toolspaceRenewalToStop,
|
|
6373
|
+
runCredentialRenewal: runRenewalToStop,
|
|
6374
|
+
});
|
|
6375
|
+
if (acknowledgeQuiescence) {
|
|
6376
|
+
// A cancellation before sandbox-backed capabilities exist still has
|
|
6377
|
+
// no tool controller to drain. Sandbox agent construction fails closed
|
|
6378
|
+
// when a backend exists but no controller was installed. Renewal
|
|
6379
|
+
// writers, when present, were drained above in either case.
|
|
6380
|
+
physicalToolQuiescenceConfirmed = true;
|
|
6381
|
+
}
|
|
6382
|
+
if (acknowledgeQuiescence && physicalToolQuiescenceConfirmed) {
|
|
6383
|
+
// This receipt is part of the hard cancellation boundary, not
|
|
6384
|
+
// housekeeping. Persist it immediately after the sandbox/tool fence
|
|
6385
|
+
// and before lease, cache, recording, or provider cleanup. Its
|
|
6386
|
+
// transaction also enqueues the exact workflow wake that will admit
|
|
6387
|
+
// the replacement; Temporal activity terminalization does neither.
|
|
6388
|
+
const proof: SessionAttemptQuiescenceProof = {
|
|
6389
|
+
accountId: input.accountId,
|
|
6390
|
+
workspaceId: input.workspaceId,
|
|
6391
|
+
sessionId: input.sessionId,
|
|
6392
|
+
attemptId: input.attemptId,
|
|
6393
|
+
workflowId: input.workflowId,
|
|
6394
|
+
workflowRunId: input.workflowRunId,
|
|
6395
|
+
activityId: dispatchId,
|
|
6396
|
+
};
|
|
6397
|
+
const recoveryMode = await persistOrSignalSessionAttemptQuiescence({
|
|
6398
|
+
proof,
|
|
6399
|
+
persistReceipt: async () =>
|
|
6400
|
+
await markSessionAttemptQuiesced(db, {
|
|
6401
|
+
accountId: input.accountId,
|
|
6402
|
+
workspaceId: input.workspaceId,
|
|
6403
|
+
sessionId: input.sessionId,
|
|
6404
|
+
attemptId: input.attemptId,
|
|
6405
|
+
temporalWorkflowId: input.workflowId,
|
|
6406
|
+
temporalWorkflowRunId: input.workflowRunId,
|
|
6407
|
+
temporalActivityId: dispatchId,
|
|
6408
|
+
allowUninterrupted: true,
|
|
6409
|
+
}),
|
|
6410
|
+
publishEvents: async (events) => {
|
|
6411
|
+
await waitForTurnFinalizerStep(
|
|
6412
|
+
publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events),
|
|
6413
|
+
finalizerSignal,
|
|
6414
|
+
);
|
|
6415
|
+
},
|
|
6416
|
+
signalProof: signalSessionAttemptQuiesced,
|
|
6417
|
+
heartbeat: (attempt, retryMs) => {
|
|
6418
|
+
activityContext?.heartbeat({
|
|
6419
|
+
phase: "quiescence-proof-delivery",
|
|
6420
|
+
sessionId: input.sessionId,
|
|
6421
|
+
attemptId: input.attemptId,
|
|
6422
|
+
deliveryAttempt: attempt,
|
|
6423
|
+
retryMs,
|
|
6424
|
+
at: new Date().toISOString(),
|
|
6425
|
+
});
|
|
6426
|
+
},
|
|
6427
|
+
onReceiptFailure: (error) => {
|
|
6428
|
+
console.error("agent turn quiescence receipt exhausted; signalling proof", error);
|
|
6429
|
+
},
|
|
6430
|
+
onPublishFailure: (error) => {
|
|
6431
|
+
console.error("agent turn quiescence event fanout failed", error);
|
|
6432
|
+
},
|
|
6433
|
+
onSignalFailure: (error, attempt, retryMs) => {
|
|
6434
|
+
console.error("agent turn quiescence proof signal failed; retrying", {
|
|
6435
|
+
error,
|
|
6436
|
+
attempt,
|
|
6437
|
+
retryMs,
|
|
6438
|
+
});
|
|
6439
|
+
},
|
|
6440
|
+
});
|
|
6441
|
+
quiescenceReceiptOrProofDurable = true;
|
|
6442
|
+
if (recoveryMode === "signal") {
|
|
6443
|
+
observability.info("agent turn quiescence proof handed to workflow recovery", {
|
|
6444
|
+
"opengeni.session_id": input.sessionId,
|
|
6445
|
+
"opengeni.attempt_id": input.attemptId,
|
|
6446
|
+
"opengeni.workflow_run_id": input.workflowRunId,
|
|
6447
|
+
"opengeni.activity_id": dispatchId,
|
|
6448
|
+
});
|
|
6449
|
+
}
|
|
6450
|
+
}
|
|
6451
|
+
// Deleting this attempt's credential generation is post-receipt
|
|
6452
|
+
// housekeeping: pointer mutation is serialized with successor
|
|
6453
|
+
// activation and the attempt-qualified check cannot erase a successor.
|
|
6454
|
+
// It may therefore detach on cancellation without weakening the writer
|
|
6455
|
+
// drain that gates the receipt above.
|
|
6456
|
+
const credentialSessionToClear = runCredentialSession;
|
|
6457
|
+
runCredentialSession = null;
|
|
6458
|
+
if (credentialSessionToClear) {
|
|
6459
|
+
const cleanup = clearRunCredentialsForAttempt(credentialSessionToClear, {
|
|
6460
|
+
sessionId: input.sessionId,
|
|
6461
|
+
attemptId: input.attemptId,
|
|
6462
|
+
executionGeneration,
|
|
6463
|
+
}).catch((error: unknown) => {
|
|
6464
|
+
try {
|
|
6465
|
+
observability.incrementCounter({
|
|
6466
|
+
name: "opengeni_run_credential_cleanup_total",
|
|
6467
|
+
help: "Attempt-owned run credential cleanup outcomes.",
|
|
6468
|
+
labels: { outcome: "error" },
|
|
6469
|
+
});
|
|
6470
|
+
observability.warn("Attempt-owned run credential cleanup failed", {
|
|
6471
|
+
sessionId: input.sessionId,
|
|
6472
|
+
turnId,
|
|
6473
|
+
attemptId: input.attemptId,
|
|
6474
|
+
errorClass: error instanceof Error ? error.name : "UnknownError",
|
|
6475
|
+
});
|
|
6476
|
+
} catch {
|
|
6477
|
+
// Cleanup observability must not own finalizer liveness.
|
|
6478
|
+
}
|
|
6479
|
+
});
|
|
6480
|
+
await waitForTurnFinalizerStep(cleanup, finalizerSignal);
|
|
6481
|
+
}
|
|
6482
|
+
gitCredentialRenewalClosed = true;
|
|
6483
|
+
const renewalsToStop = gitCredentialRenewals;
|
|
6484
|
+
gitCredentialRenewals = [];
|
|
6485
|
+
for (const renewal of renewalsToStop) {
|
|
6486
|
+
await waitForTurnFinalizerStep(renewal.stop(), finalizerSignal);
|
|
6487
|
+
}
|
|
6488
|
+
// Drain the buffered Connected Machine op events (infra failures + healed
|
|
6489
|
+
// recoveries) to durable session events — awaited, best-effort, never blocking
|
|
6490
|
+
// the turn. Sync observer → buffer → single awaited append here (no unawaited
|
|
6491
|
+
// DB write inside the activity). Scoped to this turn; skipped if no turnId
|
|
6492
|
+
// (the op ran under a turn, so on the normal path turnId is set).
|
|
6493
|
+
const machineOpEvents = machineOpObserver.drainEvents();
|
|
6494
|
+
if (machineOpEvents.length > 0 && turnId && executionGeneration > 0) {
|
|
6495
|
+
await waitForTurnFinalizerStep(
|
|
6496
|
+
appendAndPublishTurnEventsFenced(
|
|
6497
|
+
db,
|
|
6498
|
+
bus,
|
|
6499
|
+
input.workspaceId,
|
|
6500
|
+
input.sessionId,
|
|
6501
|
+
turnId,
|
|
6502
|
+
executionGeneration,
|
|
6503
|
+
input.attemptId,
|
|
6504
|
+
machineOpEvents.map((event) => ({
|
|
6505
|
+
...event,
|
|
6506
|
+
turnId: turnId ?? null,
|
|
6507
|
+
})),
|
|
6508
|
+
).catch(() => undefined),
|
|
6509
|
+
finalizerSignal,
|
|
6510
|
+
);
|
|
6511
|
+
}
|
|
6512
|
+
// Multi-account P4: flush the serving account's free per-turn caches ONCE,
|
|
6513
|
+
// best-effort (same discipline as today's usage write). Both writers skip
|
|
6514
|
+
// version/updatedAt, so neither can race the token-refresh CAS.
|
|
6515
|
+
if (effectiveCodexCredentialId) {
|
|
6516
|
+
// Part A: the latest scraped usage-header snapshot → the P2 usage cache. A
|
|
6517
|
+
// full both-windows snapshot (parseCodexUsageHeaders gates on both), so this
|
|
6518
|
+
// is byte-identical to the /wham/usage write — no partial-window clobber.
|
|
6519
|
+
if (latestCodexUsage) {
|
|
6520
|
+
const usageMutation = await waitForTurnFinalizerStep(
|
|
6521
|
+
recordCodexAccountUsageWithWakeTargets(
|
|
6522
|
+
db,
|
|
6523
|
+
input.workspaceId,
|
|
6524
|
+
effectiveCodexCredentialId,
|
|
6525
|
+
latestCodexUsage,
|
|
6526
|
+
).catch(() => null),
|
|
6527
|
+
finalizerSignal,
|
|
6528
|
+
);
|
|
6529
|
+
if (usageMutation) {
|
|
6530
|
+
await waitForTurnFinalizerStep(
|
|
6531
|
+
signalCodexCapacityWakeTargets(
|
|
6532
|
+
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
6533
|
+
usageMutation.wakeTargets,
|
|
6534
|
+
),
|
|
6535
|
+
finalizerSignal,
|
|
6536
|
+
);
|
|
6537
|
+
}
|
|
6538
|
+
}
|
|
6539
|
+
// Part B.1: the connector namespaces codex_apps listed this turn → the
|
|
6540
|
+
// connector-set cache. NON-EMPTY-only: a flaky/empty tools/list must never
|
|
6541
|
+
// overwrite a known set with [] (false coverage drop). Read by reference
|
|
6542
|
+
// AFTER the run, so every tools/list this turn has accumulated.
|
|
6543
|
+
const connectorNamespaces = preparedTools?.codexConnectorNamespaces;
|
|
6544
|
+
if (connectorNamespaces && connectorNamespaces.size > 0) {
|
|
6545
|
+
await waitForTurnFinalizerStep(
|
|
6546
|
+
recordCodexAccountConnectors(db, input.workspaceId, effectiveCodexCredentialId, [
|
|
6547
|
+
...connectorNamespaces,
|
|
6548
|
+
]).catch(() => undefined),
|
|
6549
|
+
finalizerSignal,
|
|
5240
6550
|
);
|
|
5241
6551
|
}
|
|
5242
6552
|
}
|
|
5243
|
-
|
|
5244
|
-
|
|
5245
|
-
|
|
5246
|
-
// AFTER the run, so every tools/list this turn has accumulated.
|
|
5247
|
-
const connectorNamespaces = preparedTools?.codexConnectorNamespaces;
|
|
5248
|
-
if (connectorNamespaces && connectorNamespaces.size > 0) {
|
|
5249
|
-
await recordCodexAccountConnectors(db, input.workspaceId, effectiveCodexCredentialId, [
|
|
5250
|
-
...connectorNamespaces,
|
|
5251
|
-
]).catch(() => undefined);
|
|
6553
|
+
if (codexLeaseHeartbeatTimer) {
|
|
6554
|
+
clearInterval(codexLeaseHeartbeatTimer);
|
|
6555
|
+
codexLeaseHeartbeatTimer = undefined;
|
|
5252
6556
|
}
|
|
5253
|
-
|
|
5254
|
-
|
|
5255
|
-
|
|
5256
|
-
|
|
5257
|
-
|
|
5258
|
-
|
|
5259
|
-
|
|
5260
|
-
|
|
5261
|
-
|
|
5262
|
-
|
|
5263
|
-
|
|
5264
|
-
|
|
5265
|
-
|
|
5266
|
-
).catch(() => undefined);
|
|
5267
|
-
codexLeaseHeld = false;
|
|
5268
|
-
}
|
|
5269
|
-
// Workbench v2 turn-end workspace capture (dossier §10.1) — runs FIRST in
|
|
5270
|
-
// the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
|
|
5271
|
-
// tool ran before this finally, so /workspace is already final; capture is
|
|
5272
|
-
// FS-equivalent to the already-settled recording preparation and the warm
|
|
5273
|
-
// snapshot (neither mutates workspace files). Running it here — BEFORE
|
|
5274
|
-
// preparedTools.close() (which tears down tools / computer-use / the display
|
|
5275
|
-
// stack and is what starts the Modal box exiting a few seconds later) —
|
|
5276
|
-
// gives capture the full live-box margin instead of racing the teardown
|
|
5277
|
-
// tail, which was dropping 100% of captures on real Modal desktop boxes
|
|
5278
|
-
// ("request cancelled due to container exiting", 0 rows). External module:
|
|
5279
|
-
// self-capped at 60s, best-effort (never throws past its boundary),
|
|
5280
|
-
// epoch-fenced, and it NEVER closes the box. The emitted
|
|
5281
|
-
// workspace.revision.captured event is ANNOUNCE-ONLY (metadata, never
|
|
5282
|
-
// content).
|
|
5283
|
-
if (resolvedSandbox && setupBoxSession && sandboxGroupId) {
|
|
5284
|
-
// Stop new heartbeat snapshot/meter ticks so a mid-turn snapshot cannot
|
|
5285
|
-
// start concurrently with capture, then drain any in-flight snapshot
|
|
5286
|
-
// (bounded) — capture and the warm snapshot both exec on the box, so
|
|
5287
|
-
// sequence them, exactly as the turn-end snapshot placement did.
|
|
5288
|
-
if (leaseHeartbeatTimer) {
|
|
5289
|
-
clearInterval(leaseHeartbeatTimer);
|
|
5290
|
-
leaseHeartbeatTimer = undefined;
|
|
6557
|
+
if (codexLeaseHeld && turnId && codexLeaseHolderId && codexLeaseGeneration !== null) {
|
|
6558
|
+
await waitForTurnFinalizerStep(
|
|
6559
|
+
releaseCodexCredentialLease(
|
|
6560
|
+
db,
|
|
6561
|
+
input.accountId,
|
|
6562
|
+
input.workspaceId,
|
|
6563
|
+
turnId,
|
|
6564
|
+
codexLeaseHolderId,
|
|
6565
|
+
codexLeaseGeneration,
|
|
6566
|
+
).catch(() => undefined),
|
|
6567
|
+
finalizerSignal,
|
|
6568
|
+
);
|
|
6569
|
+
codexLeaseHeld = false;
|
|
5291
6570
|
}
|
|
5292
|
-
|
|
5293
|
-
|
|
6571
|
+
// Workbench v2 turn-end workspace capture — runs FIRST in
|
|
6572
|
+
// the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
|
|
6573
|
+
// tool ran before this finally, so /workspace is already final; capture is
|
|
6574
|
+
// FS-equivalent to the already-settled recording preparation and the warm
|
|
6575
|
+
// snapshot (neither mutates workspace files). Running it here — BEFORE
|
|
6576
|
+
// preparedTools.close() (which tears down tools / computer-use / the display
|
|
6577
|
+
// stack and is what starts the Modal box exiting a few seconds later) —
|
|
6578
|
+
// gives capture the full live-box margin instead of racing the teardown
|
|
6579
|
+
// tail, which was dropping 100% of captures on real Modal desktop boxes
|
|
6580
|
+
// ("request cancelled due to container exiting", 0 rows). External module:
|
|
6581
|
+
// self-capped at 60s, best-effort (never throws past its boundary),
|
|
6582
|
+
// epoch-fenced, and it NEVER closes the box. The emitted
|
|
6583
|
+
// workspace.revision.captured event is ANNOUNCE-ONLY (metadata, never
|
|
6584
|
+
// content).
|
|
6585
|
+
if (process.env.OPENGENI_TEST_SCENARIO === "sandbox") {
|
|
6586
|
+
console.log(
|
|
6587
|
+
`[sandbox-e2e] capture preflight ownership=${settings.sandboxOwnershipEnabled} enabled=${settings.workspaceCaptureEnabled} resolved=${Boolean(resolvedSandbox)} session=${Boolean(setupBoxSession)} group=${Boolean(sandboxGroupId)} storage=${Boolean(objectStorage)}`,
|
|
6588
|
+
);
|
|
5294
6589
|
}
|
|
5295
|
-
|
|
5296
|
-
|
|
5297
|
-
|
|
5298
|
-
settings,
|
|
5299
|
-
publish: async (events) => {
|
|
5300
|
-
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events);
|
|
5301
|
-
},
|
|
5302
|
-
session: setupBoxSession as ChannelASession,
|
|
5303
|
-
leaseEpoch: resolvedSandbox.leaseEpoch,
|
|
5304
|
-
sandboxGroupId,
|
|
5305
|
-
accountId: input.accountId,
|
|
5306
|
-
workspaceId: input.workspaceId,
|
|
5307
|
-
sessionId: input.sessionId,
|
|
5308
|
-
turnId: turnId ?? null,
|
|
5309
|
-
observability,
|
|
6590
|
+
const runTurnEndPersistence = shouldRunTurnEndWorkspacePersistence({
|
|
6591
|
+
activityStatus,
|
|
6592
|
+
cancellationRequested: finalizerSignal?.aborted === true,
|
|
5310
6593
|
});
|
|
5311
|
-
|
|
5312
|
-
|
|
5313
|
-
|
|
5314
|
-
|
|
5315
|
-
|
|
5316
|
-
|
|
5317
|
-
|
|
5318
|
-
|
|
5319
|
-
|
|
5320
|
-
|
|
5321
|
-
|
|
5322
|
-
|
|
5323
|
-
|
|
5324
|
-
|
|
5325
|
-
|
|
5326
|
-
clearInterval(leaseHeartbeatTimer);
|
|
5327
|
-
}
|
|
5328
|
-
// A recording normally closes inside the attempt-fenced turn settlement.
|
|
5329
|
-
// Reaching finally with one still active means settlement threw, never ran,
|
|
5330
|
-
// or lost ownership. Stop ffmpeg and mark only this exact attempt-owned row
|
|
5331
|
-
// failed; publish no event and leave the artifact recoverable on the box.
|
|
5332
|
-
await abandonActiveRecording(
|
|
5333
|
-
"activity ended without recording settlement",
|
|
5334
|
-
didComputerUse ? "failed" : "discard",
|
|
5335
|
-
);
|
|
5336
|
-
if (resolvedSandbox) {
|
|
5337
|
-
// TURN-END mid-session snapshot (sandbox-file-persistence): fold the
|
|
5338
|
-
// turn's finished /workspace onto the lease before releasing the holder,
|
|
5339
|
-
// so the work this turn just produced survives any unclean box death in
|
|
5340
|
-
// the idle window ahead. Throttled by the same interval as the heartbeat
|
|
5341
|
-
// tick (a short turn right after a snapshot skips — bounded-loss contract
|
|
5342
|
-
// is the interval, not per-turn). Best-effort and time-capped by the
|
|
5343
|
-
// helper's own failure discipline; never delays release on failure.
|
|
5344
|
-
if (setupBoxSession && sandboxGroupId) {
|
|
5345
|
-
// Single-flight vs the heartbeat capture: the timer is already cleared
|
|
5346
|
-
// above, but a capture it launched may still be in flight — and that
|
|
5347
|
-
// capture predates the turn's final writes. Wait for it, but only up
|
|
5348
|
-
// to the snapshot timeout: release must never depend on an unbounded
|
|
5349
|
-
// provider capture.
|
|
6594
|
+
if (
|
|
6595
|
+
runTurnEndPersistence &&
|
|
6596
|
+
turnId &&
|
|
6597
|
+
resolvedSandbox &&
|
|
6598
|
+
setupBoxSession &&
|
|
6599
|
+
sandboxGroupId
|
|
6600
|
+
) {
|
|
6601
|
+
// Stop new heartbeat snapshot/meter ticks so a mid-turn snapshot cannot
|
|
6602
|
+
// start concurrently with capture, then drain any in-flight snapshot
|
|
6603
|
+
// (bounded) — capture and the warm snapshot both exec on the box, so
|
|
6604
|
+
// sequence them, exactly as the turn-end snapshot placement did.
|
|
6605
|
+
if (leaseHeartbeatTimer) {
|
|
6606
|
+
clearInterval(leaseHeartbeatTimer);
|
|
6607
|
+
leaseHeartbeatTimer = undefined;
|
|
6608
|
+
}
|
|
5350
6609
|
if (snapshotInFlight) {
|
|
5351
|
-
await waitForWarmSnapshot(
|
|
6610
|
+
await waitForWarmSnapshot(
|
|
6611
|
+
snapshotInFlight,
|
|
6612
|
+
settings.sandboxSnapshotTimeoutMs,
|
|
6613
|
+
finalizerSignal,
|
|
6614
|
+
);
|
|
5352
6615
|
}
|
|
5353
|
-
|
|
5354
|
-
|
|
5355
|
-
|
|
5356
|
-
|
|
5357
|
-
|
|
5358
|
-
|
|
6616
|
+
await captureWorkspaceRevision({
|
|
6617
|
+
db,
|
|
6618
|
+
objectStorage,
|
|
6619
|
+
settings,
|
|
6620
|
+
publish: async (events) => {
|
|
6621
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events);
|
|
5359
6622
|
},
|
|
5360
|
-
setupBoxSession,
|
|
5361
|
-
resolvedSandbox.leaseEpoch,
|
|
6623
|
+
session: setupBoxSession as ChannelASession,
|
|
6624
|
+
leaseEpoch: resolvedSandbox.leaseEpoch,
|
|
6625
|
+
sandboxGroupId,
|
|
6626
|
+
accountId: input.accountId,
|
|
6627
|
+
workspaceId: input.workspaceId,
|
|
6628
|
+
sessionId: input.sessionId,
|
|
6629
|
+
turnId,
|
|
6630
|
+
attemptId: input.attemptId,
|
|
6631
|
+
observability,
|
|
6632
|
+
...(finalizerSignal ? { signal: finalizerSignal } : {}),
|
|
6633
|
+
});
|
|
6634
|
+
}
|
|
6635
|
+
if (preparedTools) {
|
|
6636
|
+
await waitForTurnFinalizerStep(
|
|
6637
|
+
preparedTools.close().catch(() => undefined),
|
|
6638
|
+
finalizerSignal,
|
|
5362
6639
|
);
|
|
5363
|
-
|
|
5364
|
-
|
|
6640
|
+
}
|
|
6641
|
+
if (heartbeatTimer) {
|
|
6642
|
+
clearInterval(heartbeatTimer);
|
|
6643
|
+
}
|
|
6644
|
+
if (turnSandboxProvisioner?.hasStarted()) {
|
|
6645
|
+
await waitForTurnFinalizerStep(
|
|
6646
|
+
turnSandboxProvisioner.waitForSettled(30_000),
|
|
6647
|
+
finalizerSignal,
|
|
6648
|
+
);
|
|
6649
|
+
}
|
|
6650
|
+
// P1.2: stop the lease-TTL refresh, release the turn holder (idempotent
|
|
6651
|
+
// delete-my-row; refcount-- and warm->draining if it hit 0 with no turns),
|
|
6652
|
+
// and DROP the in-memory handle. Release NEVER stops the box — the reaper
|
|
6653
|
+
// (P1.3) issues the provider stop() past the drain grace at refcount 0; the
|
|
6654
|
+
// box rides the provider idle-timeout in the meantime. Best-effort: a
|
|
6655
|
+
// release failure must never mask the turn's real outcome.
|
|
6656
|
+
if (leaseHeartbeatTimer) {
|
|
6657
|
+
clearInterval(leaseHeartbeatTimer);
|
|
6658
|
+
}
|
|
6659
|
+
// A recording normally closes inside the attempt-fenced turn settlement.
|
|
6660
|
+
// Reaching finally with one still active means settlement threw, never ran,
|
|
6661
|
+
// or lost ownership. Stop ffmpeg and mark only this exact attempt-owned row
|
|
6662
|
+
// failed; publish no event and leave the artifact recoverable on the box.
|
|
6663
|
+
await waitForTurnFinalizerStep(
|
|
6664
|
+
abandonActiveRecording(
|
|
6665
|
+
"activity ended without recording settlement",
|
|
6666
|
+
didComputerUse ? "failed" : "discard",
|
|
6667
|
+
),
|
|
6668
|
+
finalizerSignal,
|
|
6669
|
+
);
|
|
6670
|
+
if (resolvedSandbox) {
|
|
6671
|
+
// TURN-END mid-session snapshot (sandbox-file-persistence): fold the
|
|
6672
|
+
// turn's finished /workspace onto the lease before releasing the holder,
|
|
6673
|
+
// so the work this turn just produced survives any unclean box death in
|
|
6674
|
+
// the idle window ahead. Throttled by the same interval as the heartbeat
|
|
6675
|
+
// tick (a short turn right after a snapshot skips — bounded-loss contract
|
|
6676
|
+
// is the interval, not per-turn). Best-effort and time-capped by the
|
|
6677
|
+
// helper's own failure discipline; never delays release on failure.
|
|
6678
|
+
const settledTurnId = turnId;
|
|
6679
|
+
if (runTurnEndPersistence && setupBoxSession && sandboxGroupId && settledTurnId) {
|
|
6680
|
+
// Single-flight vs the heartbeat capture: the timer is already cleared
|
|
6681
|
+
// above, but a capture it launched may still be in flight — and that
|
|
6682
|
+
// capture predates the turn's final writes. Wait for it, but only up
|
|
6683
|
+
// to the snapshot timeout: release must never depend on an unbounded
|
|
6684
|
+
// provider capture.
|
|
6685
|
+
if (snapshotInFlight) {
|
|
6686
|
+
await waitForWarmSnapshot(
|
|
6687
|
+
snapshotInFlight,
|
|
6688
|
+
settings.sandboxSnapshotTimeoutMs,
|
|
6689
|
+
finalizerSignal,
|
|
6690
|
+
);
|
|
6691
|
+
}
|
|
6692
|
+
const persisted = await maybePersistWarmWorkspaceSnapshot(
|
|
6693
|
+
{ db, settings },
|
|
5365
6694
|
{
|
|
5366
|
-
|
|
5367
|
-
|
|
6695
|
+
accountId: input.accountId,
|
|
6696
|
+
workspaceId: input.workspaceId,
|
|
6697
|
+
sessionId: input.sessionId,
|
|
6698
|
+
turnId: settledTurnId,
|
|
6699
|
+
attemptId: input.attemptId,
|
|
6700
|
+
sandboxGroupId,
|
|
5368
6701
|
},
|
|
5369
|
-
|
|
6702
|
+
setupBoxSession,
|
|
6703
|
+
resolvedSandbox.leaseEpoch,
|
|
6704
|
+
finalizerSignal,
|
|
6705
|
+
);
|
|
6706
|
+
if (persisted && publish) {
|
|
6707
|
+
await publish([
|
|
6708
|
+
{
|
|
6709
|
+
type: "sandbox.box.snapshot",
|
|
6710
|
+
payload: { trigger: "turn-end" },
|
|
6711
|
+
},
|
|
6712
|
+
]).catch(() => undefined);
|
|
6713
|
+
}
|
|
6714
|
+
// NB workspace capture no longer runs here — it moved to
|
|
6715
|
+
// the TOP of this finally (before preparedTools.close) so it completes
|
|
6716
|
+
// while the box is still solidly alive, instead of racing the turn-end
|
|
6717
|
+
// teardown that was killing 100% of captures on real Modal desktop boxes.
|
|
5370
6718
|
}
|
|
5371
|
-
|
|
5372
|
-
//
|
|
5373
|
-
|
|
5374
|
-
|
|
6719
|
+
const sandboxToRelease = resolvedSandbox;
|
|
6720
|
+
resolvedSandbox = null; // drop ownership now; the exact-holder release may finish later
|
|
6721
|
+
await waitForTurnFinalizerStep(
|
|
6722
|
+
sandboxToRelease.release().catch((releaseError) => {
|
|
6723
|
+
console.error("sandbox lease release failed (turn outcome unaffected)", releaseError);
|
|
6724
|
+
}),
|
|
6725
|
+
finalizerSignal,
|
|
6726
|
+
);
|
|
6727
|
+
}
|
|
6728
|
+
} catch (error) {
|
|
6729
|
+
finalizationError ??= error;
|
|
6730
|
+
console.error("agent turn finalization failed (turn outcome unaffected)", error);
|
|
6731
|
+
} finally {
|
|
6732
|
+
cancellationSignal?.removeEventListener("abort", noteCancellationRequested);
|
|
6733
|
+
const completedAt = performance.now();
|
|
6734
|
+
const durationSeconds = (completedAt - activityStarted) / 1000;
|
|
6735
|
+
const finalizationDurationSeconds = (completedAt - finalizationStarted) / 1000;
|
|
6736
|
+
observability.observeHistogram({
|
|
6737
|
+
name: "opengeni_turn_finalization_duration_seconds",
|
|
6738
|
+
help: "Agent turn finalization duration, including workspace housekeeping and lease release.",
|
|
6739
|
+
labels: {
|
|
6740
|
+
cancellation_requested: String(cancellationRequestedAt !== null),
|
|
6741
|
+
},
|
|
6742
|
+
value: finalizationDurationSeconds,
|
|
6743
|
+
});
|
|
6744
|
+
if (cancellationRequestedAt !== null) {
|
|
6745
|
+
const physicalCancellationDurationSeconds =
|
|
6746
|
+
(completedAt - cancellationRequestedAt) / 1000;
|
|
6747
|
+
observability.observeHistogram({
|
|
6748
|
+
name: "opengeni_turn_physical_cancellation_duration_seconds",
|
|
6749
|
+
help: "Time from Temporal cancellation delivery until the activity physically stops.",
|
|
6750
|
+
value: physicalCancellationDurationSeconds,
|
|
6751
|
+
});
|
|
6752
|
+
observability.info("agent turn physical cancellation completed", {
|
|
6753
|
+
"opengeni.session_id": input.sessionId,
|
|
6754
|
+
"opengeni.turn_id": turnId ?? "",
|
|
6755
|
+
"opengeni.attempt_id": input.attemptId,
|
|
6756
|
+
"opengeni.physical_cancellation_duration_ms": Math.round(
|
|
6757
|
+
physicalCancellationDurationSeconds * 1000,
|
|
6758
|
+
),
|
|
6759
|
+
});
|
|
6760
|
+
}
|
|
6761
|
+
observability.recordWorkerActivity({
|
|
6762
|
+
activity: "runAgentTurn",
|
|
6763
|
+
status: finalizationError ? "cleanup_failed" : activityStatus,
|
|
6764
|
+
durationSeconds,
|
|
6765
|
+
});
|
|
6766
|
+
if (turnId && activityStatus !== "unknown") {
|
|
6767
|
+
turnLifecycleMetricsFor(observability).finish(turnId, turnMetricOutcome, durationSeconds);
|
|
5375
6768
|
}
|
|
5376
|
-
|
|
5377
|
-
|
|
6769
|
+
activitySpan.end({
|
|
6770
|
+
attributes: {
|
|
6771
|
+
"opengeni.turn_id": turnId ?? "",
|
|
6772
|
+
"opengeni.status": activityStatus,
|
|
6773
|
+
"opengeni.variable_set_id": variableSetId,
|
|
6774
|
+
"opengeni.rig_id": rigId,
|
|
6775
|
+
"opengeni.rig_version_id": rigVersionId,
|
|
6776
|
+
"opengeni.codex_credential_id": effectiveCodexCredentialId ?? "",
|
|
6777
|
+
"opengeni.duration_ms": Math.round(durationSeconds * 1000),
|
|
6778
|
+
"opengeni.finalization_duration_ms": Math.round(finalizationDurationSeconds * 1000),
|
|
6779
|
+
},
|
|
6780
|
+
error: finalizationError ?? activityError,
|
|
6781
|
+
});
|
|
6782
|
+
assertPhysicalToolQuiescenceForCancellation({
|
|
6783
|
+
acknowledgeQuiescence,
|
|
6784
|
+
physicalToolQuiescenceConfirmed,
|
|
6785
|
+
failure: finalizationError,
|
|
6786
|
+
});
|
|
6787
|
+
assertSessionAttemptQuiescenceRecoveryDurable({
|
|
6788
|
+
acknowledgeQuiescence,
|
|
6789
|
+
physicalToolQuiescenceConfirmed,
|
|
6790
|
+
receiptOrProofDurable: quiescenceReceiptOrProofDurable,
|
|
6791
|
+
failure: finalizationError,
|
|
5378
6792
|
});
|
|
5379
|
-
resolvedSandbox = null; // drop the handle; the box survives the turn
|
|
5380
6793
|
}
|
|
5381
6794
|
}
|
|
5382
6795
|
};
|
|
5383
6796
|
}
|
|
5384
6797
|
|
|
6798
|
+
async function assertGitHubResourcesRemainAuthorized(
|
|
6799
|
+
db: Parameters<typeof areGitHubRepositoriesAllowedForWorkspace>[0],
|
|
6800
|
+
workspaceId: string,
|
|
6801
|
+
resources: import("@opengeni/contracts").ResourceRef[],
|
|
6802
|
+
): Promise<void> {
|
|
6803
|
+
// Must check exactly what sandboxEnvironmentForRun would mint a token for,
|
|
6804
|
+
// so the selection is derived from the same extraction as the mint path.
|
|
6805
|
+
for (const selection of gitHubTokenMintSelections(resources)) {
|
|
6806
|
+
await assertGitHubTokenMintSelectionAuthorized(
|
|
6807
|
+
db,
|
|
6808
|
+
workspaceId,
|
|
6809
|
+
selection.installationId,
|
|
6810
|
+
selection.repositoryIds,
|
|
6811
|
+
);
|
|
6812
|
+
}
|
|
6813
|
+
}
|
|
6814
|
+
|
|
6815
|
+
async function assertGitHubTokenMintSelectionAuthorized(
|
|
6816
|
+
db: Parameters<typeof areGitHubRepositoriesAllowedForWorkspace>[0],
|
|
6817
|
+
workspaceId: string,
|
|
6818
|
+
installationId: number,
|
|
6819
|
+
repositoryIds: number[],
|
|
6820
|
+
): Promise<void> {
|
|
6821
|
+
if (
|
|
6822
|
+
!(await areGitHubRepositoriesAllowedForWorkspace(
|
|
6823
|
+
db,
|
|
6824
|
+
workspaceId,
|
|
6825
|
+
installationId,
|
|
6826
|
+
repositoryIds,
|
|
6827
|
+
))
|
|
6828
|
+
) {
|
|
6829
|
+
throw new Error(
|
|
6830
|
+
"This workspace no longer authorizes one or more GitHub repositories attached to the session",
|
|
6831
|
+
);
|
|
6832
|
+
}
|
|
6833
|
+
}
|
|
6834
|
+
|
|
5385
6835
|
/**
|
|
5386
6836
|
* True when the error is transient upstream backpressure — a model-provider 5xx,
|
|
5387
6837
|
* a "server had a bad minute" body, or a dropped/again-able network connection —
|
|
@@ -5433,6 +6883,12 @@ export function agentRunFailurePayload(error: unknown): {
|
|
|
5433
6883
|
code?: string;
|
|
5434
6884
|
retryable?: boolean;
|
|
5435
6885
|
detail?: string;
|
|
6886
|
+
correlationId?: string;
|
|
6887
|
+
stage?: string;
|
|
6888
|
+
sqlState?: string | null;
|
|
6889
|
+
attempts?: number;
|
|
6890
|
+
retryOutcome?: string;
|
|
6891
|
+
database?: Record<string, string>;
|
|
5436
6892
|
} {
|
|
5437
6893
|
const message = error instanceof Error ? error.message : String(error);
|
|
5438
6894
|
const status =
|
|
@@ -5443,6 +6899,41 @@ export function agentRunFailurePayload(error: unknown): {
|
|
|
5443
6899
|
typeof error === "object" && error !== null && "code" in error
|
|
5444
6900
|
? String((error as { code?: unknown }).code)
|
|
5445
6901
|
: undefined;
|
|
6902
|
+
// An accepted Codex stream with no terminal response is malformed/partial,
|
|
6903
|
+
// not provider backpressure. Replaying the same accepted turn could repeat
|
|
6904
|
+
// model or tool effects, so this marked transport failure must outrank the
|
|
6905
|
+
// generic 5xx retry classifier (CodexStreamingTerminalError uses status 502).
|
|
6906
|
+
if (isCodexTransportError(error) && code === "invalid_sse_terminal") {
|
|
6907
|
+
return {
|
|
6908
|
+
error: "The Codex response stream ended without a terminal response",
|
|
6909
|
+
code: "invalid_sse_terminal",
|
|
6910
|
+
retryable: false,
|
|
6911
|
+
};
|
|
6912
|
+
}
|
|
6913
|
+
if (isSessionEventPersistenceError(error)) {
|
|
6914
|
+
const { details } = error;
|
|
6915
|
+
const eventLabel = details.eventTypes.join(", ") || "session events";
|
|
6916
|
+
const failureLabel =
|
|
6917
|
+
details.code === "db_deadlock"
|
|
6918
|
+
? "Database deadlock"
|
|
6919
|
+
: details.code === "db_serialization_failure"
|
|
6920
|
+
? "Database serialization failure"
|
|
6921
|
+
: "Database failure";
|
|
6922
|
+
return {
|
|
6923
|
+
error: `${failureLabel} while persisting ${eventLabel}. The completed provider call and external effects were not retried.`,
|
|
6924
|
+
code: details.code,
|
|
6925
|
+
detail:
|
|
6926
|
+
details.retryOutcome === "exhausted"
|
|
6927
|
+
? `The idempotent persistence transaction failed after ${details.attempts} attempts.`
|
|
6928
|
+
: "The database rejected the idempotent persistence transaction.",
|
|
6929
|
+
correlationId: details.correlationId,
|
|
6930
|
+
stage: details.stage,
|
|
6931
|
+
sqlState: details.sqlState,
|
|
6932
|
+
attempts: details.attempts,
|
|
6933
|
+
retryOutcome: details.retryOutcome,
|
|
6934
|
+
...(Object.keys(details.database).length > 0 ? { database: details.database } : {}),
|
|
6935
|
+
};
|
|
6936
|
+
}
|
|
5446
6937
|
// A ChatGPT/Codex usage cap is a HARD limit, not transient backpressure: it
|
|
5447
6938
|
// must NOT be reported as a generic, retryable rate-limit (which would loop a
|
|
5448
6939
|
// goal against a capped backend). Surface a precise, actionable message with
|
|
@@ -5511,7 +7002,6 @@ export function codexCredentialCooldownUntil(
|
|
|
5511
7002
|
CodexAccountStatus,
|
|
5512
7003
|
"primaryUsedPercent" | "primaryResetAt" | "secondaryUsedPercent" | "secondaryResetAt"
|
|
5513
7004
|
> | null,
|
|
5514
|
-
nearExhaustionPct: number,
|
|
5515
7005
|
now: Date,
|
|
5516
7006
|
): Date | null {
|
|
5517
7007
|
if (failure.kind === "auth" || failure.kind === "forbidden") {
|
|
@@ -5533,7 +7023,7 @@ export function codexCredentialCooldownUntil(
|
|
|
5533
7023
|
]
|
|
5534
7024
|
.filter(
|
|
5535
7025
|
(window): window is { used: number; reset: Date } =>
|
|
5536
|
-
(window.used ?? 0) >=
|
|
7026
|
+
(window.used ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT &&
|
|
5537
7027
|
window.reset instanceof Date &&
|
|
5538
7028
|
window.reset.getTime() > now.getTime(),
|
|
5539
7029
|
)
|
|
@@ -5826,6 +7316,7 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
5826
7316
|
workspaceId: string;
|
|
5827
7317
|
sessionId: string;
|
|
5828
7318
|
turnId: string;
|
|
7319
|
+
turnAttemptId: string;
|
|
5829
7320
|
model: string;
|
|
5830
7321
|
isCodexTurn: boolean;
|
|
5831
7322
|
usage?: ModelUsageInput | null;
|
|
@@ -5857,6 +7348,9 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
5857
7348
|
unit: "usd_micros",
|
|
5858
7349
|
sourceResourceType: "model_response",
|
|
5859
7350
|
sourceResourceId: `${input.turnId}:${input.sourceKey}`,
|
|
7351
|
+
sessionId: input.sessionId,
|
|
7352
|
+
turnId: input.turnId,
|
|
7353
|
+
turnAttemptId: input.turnAttemptId,
|
|
5860
7354
|
idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
|
|
5861
7355
|
});
|
|
5862
7356
|
return;
|
|
@@ -5870,6 +7364,9 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
5870
7364
|
unit: "tokens",
|
|
5871
7365
|
sourceResourceType: "model_response",
|
|
5872
7366
|
sourceResourceId: `${input.turnId}:${input.sourceKey}`,
|
|
7367
|
+
sessionId: input.sessionId,
|
|
7368
|
+
turnId: input.turnId,
|
|
7369
|
+
turnAttemptId: input.turnAttemptId,
|
|
5873
7370
|
idempotencyKey: `usage:model.tokens:${input.turnId}:${input.sourceKey}`,
|
|
5874
7371
|
});
|
|
5875
7372
|
}
|
|
@@ -5889,6 +7386,9 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
5889
7386
|
unit: "usd_micros",
|
|
5890
7387
|
sourceResourceType: "model_response",
|
|
5891
7388
|
sourceResourceId: `${input.turnId}:${input.sourceKey}`,
|
|
7389
|
+
sessionId: input.sessionId,
|
|
7390
|
+
turnId: input.turnId,
|
|
7391
|
+
turnAttemptId: input.turnAttemptId,
|
|
5892
7392
|
idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
|
|
5893
7393
|
});
|
|
5894
7394
|
if (costMicros > 0) {
|