@opengeni/worker-bundle 0.16.21 → 0.16.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +61 -14
- package/dist/activities/context-compaction.d.ts +2 -0
- package/dist/activities/environment.d.ts +1 -0
- package/dist/activities/expiring-material-renewal.d.ts +5 -3
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
- package/dist/activities/retained-screenshots.d.ts +93 -0
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +2 -0
- package/dist/activities/toolspace-token-renewal.d.ts +1 -1
- package/dist/activities/workspace-capture.d.ts +1 -0
- package/dist/activities.d.ts +3 -0
- package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
- package/dist/chunk-RAMNYSAW.js.map +1 -0
- package/dist/host-export.js +1 -1
- package/dist/index.js +1317 -390
- package/dist/index.js.map +1 -1
- package/dist/sandbox-resume.d.ts +6 -0
- package/dist/sandbox-routing.d.ts +11 -6
- package/dist/workflow-bundle.js +2 -1
- package/package.json +12 -12
- package/src/activities/agent-turn.ts +482 -360
- package/src/activities/codex-capacity.ts +10 -1
- package/src/activities/codex-rotation.ts +19 -5
- package/src/activities/context-compaction.ts +9 -3
- package/src/activities/environment.ts +13 -24
- package/src/activities/expiring-material-renewal.ts +16 -7
- package/src/activities/git-credential-renewal.ts +3 -3
- package/src/activities/recording.ts +4 -5
- package/src/activities/retained-screenshot-reaper.ts +202 -0
- package/src/activities/retained-screenshots.ts +892 -0
- package/src/activities/rig-verification.ts +13 -34
- package/src/activities/run-credential-renewal.ts +9 -2
- package/src/activities/run-input.ts +18 -4
- package/src/activities/scheduled-tasks.ts +19 -3
- package/src/activities/toolspace-token-renewal.ts +9 -2
- package/src/activities/workspace-capture.ts +33 -3
- package/src/activities.ts +10 -2
- package/src/host-export-pump.ts +28 -11
- package/src/http.ts +3 -6
- package/src/observability-metrics.ts +3 -3
- package/src/sandbox-resume.ts +45 -12
- package/src/sandbox-routing.ts +30 -15
- package/src/worker-service-lifecycle.ts +8 -4
- package/src/workflows/file-upload-reaper.ts +4 -1
- package/dist/activities/redaction.d.ts +0 -1
- package/dist/chunk-BJUW5LCM.js.map +0 -1
- package/src/activities/redaction.ts +0 -9
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
getCodexRotationSettings,
|
|
28
28
|
getWorkspaceModelPolicy,
|
|
29
29
|
getWorkspaceGrant,
|
|
30
|
+
getWorkspace,
|
|
30
31
|
listCodexAccountStatuses,
|
|
31
32
|
fetchCodexUsageForAccount,
|
|
32
33
|
getSessionCodexState,
|
|
@@ -92,6 +93,7 @@ import {
|
|
|
92
93
|
modelTerminalResponseFromSdkEvent,
|
|
93
94
|
normalizeModelCallUsage,
|
|
94
95
|
normalizeSdkEvent,
|
|
96
|
+
normalizeProtocolJsonValue,
|
|
95
97
|
projectHistoryForProvider,
|
|
96
98
|
sanitizeHistoryItemsForModel,
|
|
97
99
|
projectModelInputForCapabilities,
|
|
@@ -137,7 +139,6 @@ import {
|
|
|
137
139
|
type ModelCallUsageNormalization,
|
|
138
140
|
type BuildAgentOptions,
|
|
139
141
|
type ConnectorActionPolicyHooks,
|
|
140
|
-
type RunAgentStreamOptions,
|
|
141
142
|
type TurnToolCancellationFence,
|
|
142
143
|
type BackendUnresolvableCode,
|
|
143
144
|
type EstablishedSandboxSession,
|
|
@@ -263,19 +264,12 @@ import {
|
|
|
263
264
|
import { withFirstPartyTools } from "./goals";
|
|
264
265
|
import {
|
|
265
266
|
mergeRigDefaultVariableSetEnvironment,
|
|
266
|
-
resolveWorkspaceAgentInstructions,
|
|
267
267
|
resolveWorkspacePackRuntime,
|
|
268
268
|
resolveWorkspaceSkillLibraryRuntime,
|
|
269
269
|
settingsWithPackSandboxImage,
|
|
270
270
|
settingsWithRigImage,
|
|
271
271
|
} from "./packs";
|
|
272
272
|
import { deliverFailedChildTurnToParent } from "./parent-wake";
|
|
273
|
-
import {
|
|
274
|
-
createSecretRedactor,
|
|
275
|
-
isCredentialHeaderName,
|
|
276
|
-
redactSensitiveText,
|
|
277
|
-
type SecretForRedaction,
|
|
278
|
-
} from "./redaction";
|
|
279
273
|
import {
|
|
280
274
|
createModelHistoryAttachmentProjector,
|
|
281
275
|
turnInput,
|
|
@@ -323,6 +317,8 @@ import {
|
|
|
323
317
|
recordCreditMicros,
|
|
324
318
|
recordModelCacheTokens,
|
|
325
319
|
recordModelInputTokens,
|
|
320
|
+
recordSessionEventAppendLatency,
|
|
321
|
+
recordSessionEventPublishLatency,
|
|
326
322
|
runtimeMetricsHooksForObservability,
|
|
327
323
|
StreamTimingMetrics,
|
|
328
324
|
turnLifecycleMetricsFor,
|
|
@@ -334,6 +330,16 @@ import {
|
|
|
334
330
|
prepareRecordingForSettlement,
|
|
335
331
|
type ActiveRecording,
|
|
336
332
|
} from "./recording";
|
|
333
|
+
import {
|
|
334
|
+
collectRetainedScreenshotReceipts,
|
|
335
|
+
collectRetainedScreenshotRunStateReceipts,
|
|
336
|
+
compactRetainedScreenshotHistory,
|
|
337
|
+
compactRetainedScreenshotRunState,
|
|
338
|
+
materializeRetainedScreenshotHistory,
|
|
339
|
+
materializeRetainedScreenshotRunState,
|
|
340
|
+
retainComputerScreenshot,
|
|
341
|
+
typedScreenshotFromSdkEvent,
|
|
342
|
+
} from "./retained-screenshots";
|
|
337
343
|
import { captureWorkspaceRevision, openFreshWorkspaceCaptureSession } from "./workspace-capture";
|
|
338
344
|
import type { ChannelASession } from "@opengeni/runtime/sandbox";
|
|
339
345
|
import { createObjectStorage, type ObjectStorage } from "@opengeni/storage";
|
|
@@ -347,9 +353,11 @@ import {
|
|
|
347
353
|
DEFAULT_FIRST_PARTY_MCP_TOOLS,
|
|
348
354
|
evaluateWorkspaceModelPolicy,
|
|
349
355
|
readTurnExecutionPolicyV1,
|
|
356
|
+
resolveWorkspaceAgentHumanInputEnabled,
|
|
350
357
|
resourceMountPath,
|
|
351
358
|
type LatencyMode,
|
|
352
359
|
type ResourceRef,
|
|
360
|
+
type RetainedArtifactMetadata,
|
|
353
361
|
type SessionEvent,
|
|
354
362
|
type SessionEventType,
|
|
355
363
|
type SessionStatus,
|
|
@@ -364,6 +372,21 @@ import { createHash, randomUUID } from "node:crypto";
|
|
|
364
372
|
export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
|
|
365
373
|
const PROVIDER_CONNECTIVITY_BACKOFF_MS = [2_000, 5_000, 15_000, 30_000, 60_000] as const;
|
|
366
374
|
|
|
375
|
+
export class WorkspaceHumanInputDisabledError extends Error {
|
|
376
|
+
constructor(kind: "resume" | "interruption") {
|
|
377
|
+
super(`Workspace policy rejects structured human-input ${kind}`);
|
|
378
|
+
this.name = "WorkspaceHumanInputDisabledError";
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
export function assertWorkspaceHumanInputAllowed(
|
|
383
|
+
enabled: boolean,
|
|
384
|
+
kind: "resume" | "interruption",
|
|
385
|
+
attempted: boolean,
|
|
386
|
+
): void {
|
|
387
|
+
if (!enabled && attempted) throw new WorkspaceHumanInputDisabledError(kind);
|
|
388
|
+
}
|
|
389
|
+
|
|
367
390
|
/** Broad personal lookup is allowed only for a direct human/API command. */
|
|
368
391
|
export function credentialSubjectIdForTurnInitiator(
|
|
369
392
|
turn: Pick<SessionTurn, "source" | "initiator" | "initiatorContext">,
|
|
@@ -831,6 +854,33 @@ export async function waitForTurnStreamCleanup(
|
|
|
831
854
|
await waitForTurnFinalizerStep(providerCompleted, signal);
|
|
832
855
|
}
|
|
833
856
|
|
|
857
|
+
/**
|
|
858
|
+
* Terminal settlement closes the active attempt before turn finalization. The
|
|
859
|
+
* ordinary workspace admission therefore correctly returns `attempt_fenced`
|
|
860
|
+
* for the final attempt-qualified credential deletion. Retry only that exact
|
|
861
|
+
* settled-attempt case directly; every other fence/status and the direct
|
|
862
|
+
* deletion itself remain fail-closed.
|
|
863
|
+
*/
|
|
864
|
+
export async function clearAttemptCredentialsWithSettledFence(input: {
|
|
865
|
+
activityStatus: RunAgentTurnResult["status"] | "unknown";
|
|
866
|
+
runWorkspaceFencedClear: () => Promise<void>;
|
|
867
|
+
clearExactAttempt: () => Promise<void>;
|
|
868
|
+
onSettledAttemptFence: () => void;
|
|
869
|
+
}): Promise<void> {
|
|
870
|
+
try {
|
|
871
|
+
await input.runWorkspaceFencedClear();
|
|
872
|
+
} catch (error) {
|
|
873
|
+
const settledAttemptFence =
|
|
874
|
+
error instanceof Error &&
|
|
875
|
+
error.name === "SandboxWorkspaceMutationFencedError" &&
|
|
876
|
+
(error as Error & { code?: unknown }).code === "attempt_fenced" &&
|
|
877
|
+
(input.activityStatus === "idle" || input.activityStatus === "failed");
|
|
878
|
+
if (!settledAttemptFence) throw error;
|
|
879
|
+
input.onSettledAttemptFence();
|
|
880
|
+
await input.clearExactAttempt();
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
|
|
834
884
|
function turnFinalizerCancellationSignal(
|
|
835
885
|
temporalSignal: AbortSignal | undefined,
|
|
836
886
|
activityStatus: RunAgentTurnResult["status"] | "unknown",
|
|
@@ -847,84 +897,94 @@ function compactionFailureReason(reason: string): string {
|
|
|
847
897
|
: `compaction summarization failed: ${reason}`;
|
|
848
898
|
}
|
|
849
899
|
|
|
900
|
+
export type MandatoryHistoryPersistenceStage = "history_append" | "sandbox_envelope";
|
|
901
|
+
|
|
902
|
+
/**
|
|
903
|
+
* Preserve the exact storage failure for the permission-controlled turn event
|
|
904
|
+
* while attaching only a stable, non-secret operation token to public worker
|
|
905
|
+
* diagnostics. Mandatory history failures remain terminal and never inherit a
|
|
906
|
+
* provider-retry classification from coincidental source text or status.
|
|
907
|
+
*/
|
|
908
|
+
export class MandatoryHistoryPersistenceError extends Error {
|
|
909
|
+
readonly name = "MandatoryHistoryPersistenceError";
|
|
910
|
+
readonly cause: unknown;
|
|
911
|
+
|
|
912
|
+
constructor(
|
|
913
|
+
readonly stage: MandatoryHistoryPersistenceStage,
|
|
914
|
+
cause: unknown,
|
|
915
|
+
) {
|
|
916
|
+
super(cause instanceof Error ? cause.message : String(cause));
|
|
917
|
+
this.cause = cause;
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
export async function runMandatoryHistoryPersistenceStep<T>(
|
|
922
|
+
stage: MandatoryHistoryPersistenceStage,
|
|
923
|
+
operation: () => Promise<T>,
|
|
924
|
+
): Promise<T> {
|
|
925
|
+
try {
|
|
926
|
+
return await operation();
|
|
927
|
+
} catch (cause) {
|
|
928
|
+
throw new MandatoryHistoryPersistenceError(stage, cause);
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
|
|
850
932
|
export type SafeErrorDiagnostic = {
|
|
851
|
-
|
|
852
|
-
|
|
933
|
+
errorClass: "WorkerOperationError";
|
|
934
|
+
errorCode: "worker_operation_failed";
|
|
853
935
|
status?: number;
|
|
854
|
-
|
|
936
|
+
origin: "worker";
|
|
937
|
+
historyPersistenceStage?: MandatoryHistoryPersistenceStage;
|
|
855
938
|
};
|
|
856
939
|
|
|
857
940
|
/**
|
|
858
941
|
* Produce the only exception shape allowed in worker logs. It deliberately
|
|
859
|
-
* excludes stack, cause, response/request
|
|
860
|
-
* properties
|
|
942
|
+
* excludes the arbitrary source message, stack, cause, response/request
|
|
943
|
+
* bodies, and enumerable properties. Exact failure content belongs in the
|
|
944
|
+
* permission-controlled session event, not stdout or telemetry.
|
|
861
945
|
*/
|
|
862
|
-
export function safeErrorDiagnostic(
|
|
863
|
-
error: unknown,
|
|
864
|
-
redactText: (value: string) => string = redactSensitiveText,
|
|
865
|
-
): SafeErrorDiagnostic {
|
|
866
|
-
const rawName = error instanceof Error ? error.name : "Error";
|
|
867
|
-
const name = /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error";
|
|
868
|
-
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
946
|
+
export function safeErrorDiagnostic(error: unknown): SafeErrorDiagnostic {
|
|
869
947
|
const diagnostic: SafeErrorDiagnostic = {
|
|
870
|
-
|
|
871
|
-
|
|
948
|
+
errorClass: "WorkerOperationError",
|
|
949
|
+
errorCode: "worker_operation_failed",
|
|
950
|
+
origin: "worker",
|
|
872
951
|
};
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
if (Number.isInteger(status) && status >= 100 && status <= 599) {
|
|
879
|
-
diagnostic.status = status;
|
|
952
|
+
try {
|
|
953
|
+
let statusSource = error;
|
|
954
|
+
if (error instanceof MandatoryHistoryPersistenceError) {
|
|
955
|
+
diagnostic.historyPersistenceStage = error.stage;
|
|
956
|
+
statusSource = error.cause;
|
|
880
957
|
}
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
958
|
+
if (statusSource && typeof statusSource === "object") {
|
|
959
|
+
const status = Number(
|
|
960
|
+
(statusSource as { status?: unknown; statusCode?: unknown }).status ??
|
|
961
|
+
(statusSource as { statusCode?: unknown }).statusCode,
|
|
962
|
+
);
|
|
963
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) {
|
|
964
|
+
diagnostic.status = status;
|
|
965
|
+
}
|
|
885
966
|
}
|
|
967
|
+
} catch {
|
|
968
|
+
// Public diagnostics are best-effort and must never replace the exact
|
|
969
|
+
// internal worker failure.
|
|
886
970
|
}
|
|
887
971
|
return diagnostic;
|
|
888
972
|
}
|
|
889
973
|
|
|
890
|
-
function safeErrorForTelemetry(error: unknown
|
|
891
|
-
const diagnostic = safeErrorDiagnostic(error
|
|
892
|
-
const safe = new Error(
|
|
893
|
-
|
|
974
|
+
function safeErrorForTelemetry(error: unknown): Error {
|
|
975
|
+
const diagnostic = safeErrorDiagnostic(error);
|
|
976
|
+
const safe = new Error("worker operation failed") as Error & {
|
|
977
|
+
code?: string;
|
|
978
|
+
status?: number;
|
|
979
|
+
origin?: string;
|
|
980
|
+
};
|
|
981
|
+
safe.name = "WorkerOperationError";
|
|
982
|
+
safe.code = diagnostic.errorCode;
|
|
983
|
+
if (diagnostic.status !== undefined) safe.status = diagnostic.status;
|
|
984
|
+
safe.origin = diagnostic.origin;
|
|
894
985
|
return safe;
|
|
895
986
|
}
|
|
896
987
|
|
|
897
|
-
export function headerSecretRedactions(
|
|
898
|
-
prefix: string,
|
|
899
|
-
headers: Readonly<Record<string, string>> | undefined,
|
|
900
|
-
credentialHeaderNames?: Iterable<string>,
|
|
901
|
-
): SecretForRedaction[] {
|
|
902
|
-
const discovered: SecretForRedaction[] = [];
|
|
903
|
-
const explicitCredentialHeaders = credentialHeaderNames
|
|
904
|
-
? new Set([...credentialHeaderNames].map((name) => name.toLowerCase()))
|
|
905
|
-
: null;
|
|
906
|
-
for (const [headerName, value] of Object.entries(headers ?? {})) {
|
|
907
|
-
if (
|
|
908
|
-
!explicitCredentialHeaders?.has(headerName.toLowerCase()) &&
|
|
909
|
-
!isCredentialHeaderName(headerName)
|
|
910
|
-
) {
|
|
911
|
-
continue;
|
|
912
|
-
}
|
|
913
|
-
const safeHeaderName = headerName.toUpperCase().replace(/[^A-Z0-9]+/g, "_");
|
|
914
|
-
discovered.push({ name: `${prefix}_${safeHeaderName || "HEADER"}`, value });
|
|
915
|
-
if (/^(?:proxy-)?authorization$/i.test(headerName)) {
|
|
916
|
-
const credential = value.match(/^\s*[A-Za-z][A-Za-z0-9_-]*\s+(.+)\s*$/)?.[1];
|
|
917
|
-
if (credential) {
|
|
918
|
-
discovered.push({
|
|
919
|
-
name: `${prefix}_${safeHeaderName || "AUTHORIZATION"}_CREDENTIAL`,
|
|
920
|
-
value: credential,
|
|
921
|
-
});
|
|
922
|
-
}
|
|
923
|
-
}
|
|
924
|
-
}
|
|
925
|
-
return discovered;
|
|
926
|
-
}
|
|
927
|
-
|
|
928
988
|
function compactionFailureReasonFromError(error: unknown): string {
|
|
929
989
|
if (
|
|
930
990
|
error instanceof CompactionProviderResponseError ||
|
|
@@ -1044,14 +1104,14 @@ function collectErrorStrings(value: unknown, seen = new WeakSet<object>()): stri
|
|
|
1044
1104
|
* an earlier slice — the orphaned tool output that 400s the Responses API and
|
|
1045
1105
|
* bricks the session on every replay.
|
|
1046
1106
|
*
|
|
1047
|
-
* Defending:
|
|
1107
|
+
* Defending: structurally repair the full current history into an API-valid sequence (the
|
|
1048
1108
|
* same pure rules the read path uses), then append only the new tail beyond the
|
|
1049
1109
|
* watermark. A trailing dangling call is dropped here and re-evaluated next
|
|
1050
1110
|
* pass once its result lands, so a call and its result are written together at
|
|
1051
1111
|
* consecutive positions and a result is never persisted without its call. The
|
|
1052
|
-
* watermark advances to the
|
|
1112
|
+
* watermark advances to the repaired length — never past anything unwritten —
|
|
1053
1113
|
* so a non-monotonic history can never desync it. When previously-persisted
|
|
1054
|
-
* rows already exceed the
|
|
1114
|
+
* rows already exceed the repaired length (e.g. legacy orphans written before
|
|
1055
1115
|
* this fix), nothing new is appended and the watermark holds steady.
|
|
1056
1116
|
*/
|
|
1057
1117
|
/**
|
|
@@ -1518,14 +1578,9 @@ export async function emitModelCallUsage(input: {
|
|
|
1518
1578
|
if (!authoritative) return false;
|
|
1519
1579
|
try {
|
|
1520
1580
|
input.observability.info("model call usage", {
|
|
1521
|
-
accountId: input.accountId,
|
|
1522
|
-
workspaceId: input.workspaceId,
|
|
1523
|
-
sessionId: input.sessionId,
|
|
1524
|
-
turnId: input.turnId,
|
|
1525
1581
|
provider: input.provider,
|
|
1526
1582
|
providerApi: input.providerApi,
|
|
1527
1583
|
model: input.model,
|
|
1528
|
-
sourceKey: input.sourceKey,
|
|
1529
1584
|
inputTokens: telemetry.inputTokens,
|
|
1530
1585
|
outputTokens: telemetry.outputTokens,
|
|
1531
1586
|
cachedTokens: telemetry.cachedTokens,
|
|
@@ -1543,7 +1598,6 @@ export async function emitModelCallUsage(input: {
|
|
|
1543
1598
|
provider: input.provider,
|
|
1544
1599
|
providerApi: input.providerApi,
|
|
1545
1600
|
model: input.model,
|
|
1546
|
-
sourceKey: input.sourceKey,
|
|
1547
1601
|
rejectedFields: normalizedUsage.rejectedFields.join(","),
|
|
1548
1602
|
});
|
|
1549
1603
|
}
|
|
@@ -1590,16 +1644,13 @@ export function historyRowsToAppend(
|
|
|
1590
1644
|
// positions from 0) when callers do not pass an explicit next position.
|
|
1591
1645
|
nextPosition: number = persistedHistoryCount,
|
|
1592
1646
|
toolOutputTruncationTokens?: number,
|
|
1593
|
-
redactValue: (value: unknown) => unknown = (value) => value,
|
|
1594
1647
|
): {
|
|
1595
1648
|
rows: Array<{ position: number; item: Record<string, unknown> }>;
|
|
1596
1649
|
nextWatermark: number;
|
|
1597
1650
|
nextPosition: number;
|
|
1598
1651
|
} {
|
|
1599
|
-
const
|
|
1600
|
-
|
|
1601
|
-
);
|
|
1602
|
-
if (sanitized.length <= persistedHistoryCount) {
|
|
1652
|
+
const modelReady = sanitizeHistoryItemsForModel(rawHistory, toolOutputTruncationTokens);
|
|
1653
|
+
if (modelReady.length <= persistedHistoryCount) {
|
|
1603
1654
|
return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
|
|
1604
1655
|
}
|
|
1605
1656
|
// Canonical model-facing inputs (including machine-input batches) are
|
|
@@ -1609,16 +1660,22 @@ export function historyRowsToAppend(
|
|
|
1609
1660
|
// notes—must not accidentally become conversation memory during reconciliation.
|
|
1610
1661
|
// Advance the in-memory watermark past them, but only allocate durable
|
|
1611
1662
|
// positions to actual model/tool output.
|
|
1612
|
-
const rows =
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1663
|
+
const rows: Array<{ position: number; item: Record<string, unknown> }> = [];
|
|
1664
|
+
for (const [offset, item] of modelReady.slice(persistedHistoryCount).entries()) {
|
|
1665
|
+
if (item.type === "message" && item.role === "system") {
|
|
1666
|
+
continue;
|
|
1667
|
+
}
|
|
1668
|
+
rows.push({
|
|
1669
|
+
position: nextPosition + rows.length,
|
|
1670
|
+
item: normalizeProtocolJsonValue(
|
|
1671
|
+
item as Record<string, unknown>,
|
|
1672
|
+
`$[${persistedHistoryCount + offset}]`,
|
|
1673
|
+
),
|
|
1674
|
+
});
|
|
1675
|
+
}
|
|
1619
1676
|
return {
|
|
1620
1677
|
rows,
|
|
1621
|
-
nextWatermark:
|
|
1678
|
+
nextWatermark: modelReady.length,
|
|
1622
1679
|
nextPosition: nextPosition + rows.length,
|
|
1623
1680
|
};
|
|
1624
1681
|
}
|
|
@@ -1660,16 +1717,7 @@ export function selectRejectedProviderArtifactHistoryIds(
|
|
|
1660
1717
|
}
|
|
1661
1718
|
|
|
1662
1719
|
/** Literal final per-call boundary for replayed and current-turn model input. */
|
|
1663
|
-
export function
|
|
1664
|
-
redactValue: (value: unknown) => unknown,
|
|
1665
|
-
): NonNullable<RunAgentStreamOptions["callModelInputFilter"]> {
|
|
1666
|
-
return ({ modelData }) => ({
|
|
1667
|
-
...modelData,
|
|
1668
|
-
input: redactValue(modelData.input) as typeof modelData.input,
|
|
1669
|
-
});
|
|
1670
|
-
}
|
|
1671
|
-
|
|
1672
|
-
function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
|
|
1720
|
+
export function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
|
|
1673
1721
|
if (item.type === "message") {
|
|
1674
1722
|
return item.role === "assistant";
|
|
1675
1723
|
}
|
|
@@ -2371,6 +2419,34 @@ export function codexCredentialLeaseDeadlineExpired(
|
|
|
2371
2419
|
);
|
|
2372
2420
|
}
|
|
2373
2421
|
|
|
2422
|
+
type OpStreamFinalizer = {
|
|
2423
|
+
finalizeOpStreamOps(): Promise<void>;
|
|
2424
|
+
};
|
|
2425
|
+
|
|
2426
|
+
/** Release retained Connected Machine output only after the turn is durable. */
|
|
2427
|
+
export async function finalizeDurableTurnOpStreams(
|
|
2428
|
+
sessions: readonly unknown[],
|
|
2429
|
+
fallback: OpStreamFinalizer | null,
|
|
2430
|
+
): Promise<void> {
|
|
2431
|
+
const candidates = new Set<OpStreamFinalizer>();
|
|
2432
|
+
for (const session of sessions) {
|
|
2433
|
+
const candidate = session as Partial<OpStreamFinalizer> | null;
|
|
2434
|
+
if (typeof candidate?.finalizeOpStreamOps === "function") {
|
|
2435
|
+
candidates.add(candidate as OpStreamFinalizer);
|
|
2436
|
+
}
|
|
2437
|
+
}
|
|
2438
|
+
if (candidates.size === 0 && fallback) {
|
|
2439
|
+
candidates.add(fallback);
|
|
2440
|
+
}
|
|
2441
|
+
for (const candidate of candidates) {
|
|
2442
|
+
try {
|
|
2443
|
+
await candidate.finalizeOpStreamOps();
|
|
2444
|
+
} catch {
|
|
2445
|
+
// The runner's retention TTL owns the fallback.
|
|
2446
|
+
}
|
|
2447
|
+
}
|
|
2448
|
+
}
|
|
2449
|
+
|
|
2374
2450
|
export function createRunAgentTurnActivity(services: () => Promise<ActivityServices>) {
|
|
2375
2451
|
return async function runAgentTurn(input: RunAgentTurnInput): Promise<RunAgentTurnResult> {
|
|
2376
2452
|
const {
|
|
@@ -2429,6 +2505,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2429
2505
|
acknowledgeQuiescence = true;
|
|
2430
2506
|
noteCancellationRequested();
|
|
2431
2507
|
};
|
|
2508
|
+
const acknowledgeRecoveryQuiescence = (): void => {
|
|
2509
|
+
// requestSessionTurnRecovery closed this exact attempt and durably
|
|
2510
|
+
// recorded the replacement cause. The activity still owns the physical
|
|
2511
|
+
// tool/credential boundary until finally drains it and publishes the
|
|
2512
|
+
// exact quiescence receipt; a workflow result alone is never that proof.
|
|
2513
|
+
acknowledgeQuiescence = true;
|
|
2514
|
+
noteCancellationRequested();
|
|
2515
|
+
};
|
|
2432
2516
|
let turnId: string | undefined;
|
|
2433
2517
|
let triggerEventId: string | undefined;
|
|
2434
2518
|
const claimedResult = (
|
|
@@ -2562,7 +2646,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2562
2646
|
workspaceId: input.workspaceId,
|
|
2563
2647
|
turnId,
|
|
2564
2648
|
reason,
|
|
2565
|
-
|
|
2649
|
+
...safeErrorDiagnostic(error),
|
|
2566
2650
|
});
|
|
2567
2651
|
observability.incrementCounter({
|
|
2568
2652
|
name: "opengeni_codex_lease_renewals_total",
|
|
@@ -2594,8 +2678,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2594
2678
|
return resolvedSandbox;
|
|
2595
2679
|
};
|
|
2596
2680
|
// The machine-primary SelfhostedSession (the UNWRAPPED backend, not the
|
|
2597
|
-
// routing proxy)
|
|
2598
|
-
//
|
|
2681
|
+
// routing proxy). Kept as a fallback finalizer; the routing proxy normally
|
|
2682
|
+
// aggregates it together with every machine reached after a mid-turn swap.
|
|
2599
2683
|
let machinePrimarySession: import("@opengeni/runtime").SelfhostedSession | null = null;
|
|
2600
2684
|
let lazyOwnedSandbox: EstablishedSandboxSession | null = null;
|
|
2601
2685
|
let turnSandboxProvisioner: TurnSandboxProvisioner<ResumedTurnSandbox> | null = null;
|
|
@@ -2604,6 +2688,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2604
2688
|
// THIS handle so a mid-turn sandbox_swap can never re-route those execs onto a
|
|
2605
2689
|
// connected machine (the user's real computer).
|
|
2606
2690
|
let setupBoxSession: unknown = null;
|
|
2691
|
+
const finalizeTurnOpStreamOps = async (): Promise<void> => {
|
|
2692
|
+
await finalizeDurableTurnOpStreams(
|
|
2693
|
+
[lazyOwnedSandbox?.session, resolvedSandbox?.established.session],
|
|
2694
|
+
machinePrimarySession,
|
|
2695
|
+
);
|
|
2696
|
+
};
|
|
2607
2697
|
// A same-target API repair can replace the home provider while this turn is
|
|
2608
2698
|
// alive. Keep setup/snapshot persistence on the rebound raw session while
|
|
2609
2699
|
// preserving the SDK-owned routing proxy for eager turns. Lazy turns hold the
|
|
@@ -2951,7 +3041,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2951
3041
|
sessionId: input.sessionId,
|
|
2952
3042
|
sandboxGroupId: heartbeatGroupId,
|
|
2953
3043
|
leaseEpoch: heartbeatEpoch,
|
|
2954
|
-
|
|
3044
|
+
...safeErrorDiagnostic(error),
|
|
2955
3045
|
});
|
|
2956
3046
|
})
|
|
2957
3047
|
.finally(() => {
|
|
@@ -3104,6 +3194,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3104
3194
|
// stable and this scalar append watermark is valid again. The sanitizer
|
|
3105
3195
|
// remains the final call/result pairing guard for every other reconcile.
|
|
3106
3196
|
let persistedHistoryCount = 0;
|
|
3197
|
+
const retainedScreenshotReceiptsByCallId = new Map<string, RetainedArtifactMetadata>();
|
|
3198
|
+
const computerScreenshotCallIds = new Set<string>();
|
|
3199
|
+
const materializeScreenshotHistory = async (history: Array<Record<string, unknown>>) => {
|
|
3200
|
+
collectRetainedScreenshotReceipts(history, retainedScreenshotReceiptsByCallId);
|
|
3201
|
+
return await materializeRetainedScreenshotHistory({
|
|
3202
|
+
db,
|
|
3203
|
+
objectStorage,
|
|
3204
|
+
workspaceId: input.workspaceId,
|
|
3205
|
+
sessionId: input.sessionId,
|
|
3206
|
+
history,
|
|
3207
|
+
});
|
|
3208
|
+
};
|
|
3209
|
+
const materializeScreenshotRunState = async (serialized: string) => {
|
|
3210
|
+
collectRetainedScreenshotRunStateReceipts(serialized, retainedScreenshotReceiptsByCallId);
|
|
3211
|
+
return await materializeRetainedScreenshotRunState({
|
|
3212
|
+
db,
|
|
3213
|
+
objectStorage,
|
|
3214
|
+
workspaceId: input.workspaceId,
|
|
3215
|
+
sessionId: input.sessionId,
|
|
3216
|
+
serialized,
|
|
3217
|
+
});
|
|
3218
|
+
};
|
|
3107
3219
|
let providerArtifactCandidates: Awaited<
|
|
3108
3220
|
ReturnType<typeof turnInput>
|
|
3109
3221
|
>["providerArtifactCandidates"] = {
|
|
@@ -3121,15 +3233,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3121
3233
|
if (!stream || !turnId) {
|
|
3122
3234
|
return;
|
|
3123
3235
|
}
|
|
3236
|
+
const durableTurnId = turnId;
|
|
3124
3237
|
try {
|
|
3125
3238
|
const rawHistory = (stream.state as { history?: unknown[] }).history;
|
|
3126
3239
|
if (Array.isArray(rawHistory)) {
|
|
3127
|
-
const
|
|
3240
|
+
const durableHistory = compactRetainedScreenshotHistory(
|
|
3128
3241
|
rawHistory as Array<Record<string, unknown>>,
|
|
3242
|
+
retainedScreenshotReceiptsByCallId,
|
|
3243
|
+
);
|
|
3244
|
+
const { rows, nextWatermark, nextPosition } = historyRowsToAppend(
|
|
3245
|
+
durableHistory,
|
|
3129
3246
|
persistedHistoryCount,
|
|
3130
3247
|
nextHistoryPosition,
|
|
3131
3248
|
modelRunSettings.modelToolOutputTruncationTokens,
|
|
3132
|
-
(value) => redact(value),
|
|
3133
3249
|
);
|
|
3134
3250
|
const hasModelOrToolProgress = rows.some((row) =>
|
|
3135
3251
|
isModelOrToolProgressHistoryItem(row.item),
|
|
@@ -3137,21 +3253,23 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3137
3253
|
const shouldAppendRows =
|
|
3138
3254
|
rows.length > 0 && (!options.skipInputOnlyRows || hasModelOrToolProgress);
|
|
3139
3255
|
if (shouldAppendRows) {
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3256
|
+
await runMandatoryHistoryPersistenceStep("history_append", async () => {
|
|
3257
|
+
const appended = await appendSessionHistoryItems(db, {
|
|
3258
|
+
accountId: input.accountId,
|
|
3259
|
+
workspaceId: input.workspaceId,
|
|
3260
|
+
sessionId: input.sessionId,
|
|
3261
|
+
turnId: durableTurnId,
|
|
3262
|
+
expectedExecutionGeneration: executionGeneration,
|
|
3263
|
+
expectedAttemptId: input.attemptId,
|
|
3264
|
+
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
3265
|
+
items: rows,
|
|
3266
|
+
});
|
|
3267
|
+
if (!appended) {
|
|
3268
|
+
throw new TurnAttemptFencedError(
|
|
3269
|
+
"turn execution generation was fenced while saving conversation history",
|
|
3270
|
+
);
|
|
3271
|
+
}
|
|
3149
3272
|
});
|
|
3150
|
-
if (!appended) {
|
|
3151
|
-
throw new TurnAttemptFencedError(
|
|
3152
|
-
"turn execution generation was fenced while saving conversation history",
|
|
3153
|
-
);
|
|
3154
|
-
}
|
|
3155
3273
|
}
|
|
3156
3274
|
if (shouldAppendRows || !options.skipInputOnlyRows) {
|
|
3157
3275
|
persistedHistoryCount = nextWatermark;
|
|
@@ -3160,37 +3278,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3160
3278
|
}
|
|
3161
3279
|
const envelope = sandboxStateEntryFromRunState(stream.state);
|
|
3162
3280
|
if (envelope) {
|
|
3163
|
-
await
|
|
3164
|
-
|
|
3165
|
-
|
|
3166
|
-
|
|
3167
|
-
|
|
3168
|
-
|
|
3281
|
+
await runMandatoryHistoryPersistenceStep("sandbox_envelope", () =>
|
|
3282
|
+
upsertSandboxSessionEnvelope(db, {
|
|
3283
|
+
accountId: input.accountId,
|
|
3284
|
+
workspaceId: input.workspaceId,
|
|
3285
|
+
sessionId: input.sessionId,
|
|
3286
|
+
envelope,
|
|
3287
|
+
}),
|
|
3288
|
+
);
|
|
3169
3289
|
}
|
|
3170
3290
|
} catch (persistError) {
|
|
3171
3291
|
console.error(
|
|
3172
3292
|
"session history dual-write failed (run unaffected)",
|
|
3173
|
-
safeErrorDiagnostic(persistError
|
|
3293
|
+
safeErrorDiagnostic(persistError),
|
|
3174
3294
|
);
|
|
3175
3295
|
if (options.requireDurable) throw persistError;
|
|
3176
3296
|
}
|
|
3177
3297
|
};
|
|
3178
|
-
// Reassigned after the variable set loads; the publish closure is
|
|
3179
|
-
// created (and used for turn.started) before the variableSet is available.
|
|
3180
|
-
// Generic auth/cookie/signed-URL/token classification is active before any
|
|
3181
|
-
// exact run-secret provenance is available. Registered values strengthen
|
|
3182
|
-
// this boundary as credentials are resolved and renewed during the turn.
|
|
3183
|
-
let redact: (payload: unknown) => unknown = createSecretRedactor([]);
|
|
3184
|
-
const secretRedactions = new Map<string, string>();
|
|
3185
3298
|
const publishedRunCredentialNotices = new Set<string>();
|
|
3186
|
-
|
|
3187
|
-
for (const secret of secrets) {
|
|
3188
|
-
if (!secretRedactions.has(secret.value)) secretRedactions.set(secret.value, secret.name);
|
|
3189
|
-
}
|
|
3190
|
-
redact = createSecretRedactor(
|
|
3191
|
-
[...secretRedactions].map(([value, name]) => ({ name, value })),
|
|
3192
|
-
);
|
|
3193
|
-
};
|
|
3299
|
+
|
|
3194
3300
|
let variableSetId = "";
|
|
3195
3301
|
// Rig telemetry (M3): set once the session loads; empty string for a rig-less
|
|
3196
3302
|
// turn (mirrors variableSetId). Read by the activity span's finally block.
|
|
@@ -3377,7 +3483,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3377
3483
|
) => {
|
|
3378
3484
|
const inputs = events.map((event) => ({
|
|
3379
3485
|
...event,
|
|
3380
|
-
payload:
|
|
3486
|
+
payload: event.payload,
|
|
3381
3487
|
turnId: turnId!,
|
|
3382
3488
|
producerId,
|
|
3383
3489
|
producerSeq: ++producerSeq,
|
|
@@ -3391,6 +3497,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3391
3497
|
executionGeneration,
|
|
3392
3498
|
input.attemptId,
|
|
3393
3499
|
inputs,
|
|
3500
|
+
{
|
|
3501
|
+
onAppend: ({ durationSeconds }) =>
|
|
3502
|
+
recordSessionEventAppendLatency(observability, { durationSeconds }),
|
|
3503
|
+
onPublish: ({ durationSeconds }) =>
|
|
3504
|
+
recordSessionEventPublishLatency(observability, { durationSeconds }),
|
|
3505
|
+
},
|
|
3394
3506
|
);
|
|
3395
3507
|
if (inputs.length > 0 && !appended.accepted) {
|
|
3396
3508
|
throw new TurnAttemptFencedError("turn execution generation was fenced");
|
|
@@ -3445,13 +3557,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3445
3557
|
: undefined;
|
|
3446
3558
|
const inputs = inputSettlement.events.map((event) => ({
|
|
3447
3559
|
...event,
|
|
3448
|
-
payload:
|
|
3560
|
+
payload: event.payload,
|
|
3449
3561
|
turnId: turnId!,
|
|
3450
3562
|
producerId,
|
|
3451
3563
|
producerSeq: ++producerSeq,
|
|
3452
3564
|
}));
|
|
3453
|
-
const
|
|
3454
|
-
?
|
|
3565
|
+
const runState = inputSettlement.runState
|
|
3566
|
+
? {
|
|
3567
|
+
...inputSettlement.runState,
|
|
3568
|
+
serializedRunState: compactRetainedScreenshotRunState(
|
|
3569
|
+
inputSettlement.runState.serializedRunState,
|
|
3570
|
+
retainedScreenshotReceiptsByCallId,
|
|
3571
|
+
),
|
|
3572
|
+
}
|
|
3455
3573
|
: undefined;
|
|
3456
3574
|
const result = await applySessionTurnSettlement(db, input.workspaceId, {
|
|
3457
3575
|
sessionId: input.sessionId,
|
|
@@ -3462,7 +3580,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3462
3580
|
sessionStatus: inputSettlement.sessionStatus,
|
|
3463
3581
|
activeTurnId: inputSettlement.activeTurnId,
|
|
3464
3582
|
events: inputs,
|
|
3465
|
-
...(
|
|
3583
|
+
...(runState ? { runState } : {}),
|
|
3466
3584
|
...(recordingMutation ? { recording: recordingMutation } : {}),
|
|
3467
3585
|
...(compactionRequestFailure ? { compactionRequestFailure } : {}),
|
|
3468
3586
|
});
|
|
@@ -3820,10 +3938,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3820
3938
|
labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
|
|
3821
3939
|
});
|
|
3822
3940
|
observability.warn("Codex adaptive fleet shadow decision failed open", {
|
|
3823
|
-
workspaceId: input.workspaceId,
|
|
3824
3941
|
stage: shadowResult.stage,
|
|
3825
3942
|
reason: shadowResult.reason,
|
|
3826
|
-
|
|
3943
|
+
errorClass: "CodexFleetShadowOperationError",
|
|
3944
|
+
errorCode: "codex_fleet_shadow_failed",
|
|
3945
|
+
origin: "worker",
|
|
3827
3946
|
payloadBytes: shadowResult.payloadBytes,
|
|
3828
3947
|
});
|
|
3829
3948
|
}
|
|
@@ -4173,15 +4292,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4173
4292
|
attemptId: input.attemptId,
|
|
4174
4293
|
executionGeneration: turn.executionGeneration,
|
|
4175
4294
|
};
|
|
4176
|
-
const [
|
|
4177
|
-
|
|
4178
|
-
|
|
4179
|
-
|
|
4180
|
-
|
|
4181
|
-
|
|
4182
|
-
|
|
4183
|
-
|
|
4184
|
-
|
|
4295
|
+
const [workspace, instructionPolicySnapshot, preferenceSnapshot] = await Promise.all([
|
|
4296
|
+
getWorkspace(db, input.workspaceId),
|
|
4297
|
+
getOrCreateWorkspaceInstructionPolicySnapshot(db, governanceClaims),
|
|
4298
|
+
getOrCreatePreferenceRegistrySnapshot(db, governanceClaims).catch((error) => {
|
|
4299
|
+
if (error instanceof PreferenceRegistryInitiatorError) return null;
|
|
4300
|
+
throw error;
|
|
4301
|
+
}),
|
|
4302
|
+
]);
|
|
4303
|
+
if (!workspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
|
|
4304
|
+
const workspaceAgentInstructions = workspace.agentInstructions;
|
|
4305
|
+
const agentHumanInputEnabled = resolveWorkspaceAgentHumanInputEnabled(workspace.settings);
|
|
4306
|
+
assertWorkspaceHumanInputAllowed(agentHumanInputEnabled, "resume", humanInputResume !== null);
|
|
4185
4307
|
const workspaceGovernance = renderWorkspaceGovernanceContext({
|
|
4186
4308
|
instructionPolicy: instructionPolicySnapshot,
|
|
4187
4309
|
preferences: preferenceSnapshot,
|
|
@@ -4206,33 +4328,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4206
4328
|
openaiReasoningEffort: turn.reasoningEffort,
|
|
4207
4329
|
sandboxBackend: turn.sandboxBackend,
|
|
4208
4330
|
};
|
|
4209
|
-
const sessionMcpHeaderNames = new Map<string, readonly string[]>();
|
|
4210
4331
|
const runSettings = await settingsWithSessionMcpServersForRun(
|
|
4211
4332
|
db,
|
|
4212
4333
|
input.workspaceId,
|
|
4213
4334
|
input.sessionId,
|
|
4214
4335
|
input.attemptId,
|
|
4215
4336
|
baseRunSettings,
|
|
4216
|
-
{
|
|
4217
|
-
onResolvedServers: (servers) => {
|
|
4218
|
-
for (const server of servers) {
|
|
4219
|
-
sessionMcpHeaderNames.set(server.id, server.headerNames);
|
|
4220
|
-
}
|
|
4221
|
-
},
|
|
4222
|
-
},
|
|
4223
4337
|
);
|
|
4224
|
-
|
|
4225
|
-
...(runSettings.accessKey
|
|
4226
|
-
? [{ name: "OPENGENI_ACCESS_KEY", value: runSettings.accessKey }]
|
|
4227
|
-
: []),
|
|
4228
|
-
...runSettings.mcpServers.flatMap((server) => {
|
|
4229
|
-
return headerSecretRedactions(
|
|
4230
|
-
`MCP_${server.id.toUpperCase()}_STATIC`,
|
|
4231
|
-
server.headers,
|
|
4232
|
-
sessionMcpHeaderNames.get(server.id),
|
|
4233
|
-
);
|
|
4234
|
-
}),
|
|
4235
|
-
]);
|
|
4338
|
+
|
|
4236
4339
|
// Multi-provider per-turn routing → the provider gating (compaction mode,
|
|
4237
4340
|
// hosted web search, encrypted reasoning, context window) the agent and
|
|
4238
4341
|
// compaction summarizer must use; null falls back to the legacy global
|
|
@@ -4263,49 +4366,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4263
4366
|
projectModelInputForCapabilities(await attachmentProjector(items), modelInputPolicy);
|
|
4264
4367
|
const compactionModelHistoryProjector = async (items: Array<Record<string, unknown>>) =>
|
|
4265
4368
|
await modelHistoryProjector(projectHistoryForProvider(items, providerApi));
|
|
4266
|
-
const selectedProvider = resolvedModel?.provider;
|
|
4267
|
-
const publicProviderHeaders = new Set(
|
|
4268
|
-
selectedProvider?.publicDefaultHeaderNames?.map((name) => name.toLowerCase()) ?? [],
|
|
4269
|
-
);
|
|
4270
|
-
const publicProviderQueries = new Set(
|
|
4271
|
-
selectedProvider?.publicDefaultQueryNames?.map((name) => name.toLowerCase()) ?? [],
|
|
4272
|
-
);
|
|
4273
|
-
const fallbackProviderApiKey =
|
|
4274
|
-
capabilitySettings.openaiProvider === "azure"
|
|
4275
|
-
? (capabilitySettings.azureOpenaiApiKey ?? capabilitySettings.azureOpenaiAdToken)
|
|
4276
|
-
: capabilitySettings.openaiApiKey;
|
|
4277
|
-
registerSecretRedactions([
|
|
4278
|
-
...(selectedProvider?.apiKey
|
|
4279
|
-
? [
|
|
4280
|
-
{
|
|
4281
|
-
name: `MODEL_${selectedProvider.id.toUpperCase()}_API_KEY`,
|
|
4282
|
-
value: selectedProvider.apiKey,
|
|
4283
|
-
},
|
|
4284
|
-
]
|
|
4285
|
-
: !resolvedModel && fallbackProviderApiKey
|
|
4286
|
-
? [
|
|
4287
|
-
{
|
|
4288
|
-
name: "MODEL_PROVIDER_API_KEY",
|
|
4289
|
-
value: fallbackProviderApiKey,
|
|
4290
|
-
},
|
|
4291
|
-
]
|
|
4292
|
-
: []),
|
|
4293
|
-
...Object.entries(selectedProvider?.defaultHeaders ?? {}).flatMap(([name, value]) =>
|
|
4294
|
-
publicProviderHeaders.has(name.toLowerCase())
|
|
4295
|
-
? []
|
|
4296
|
-
: [
|
|
4297
|
-
{
|
|
4298
|
-
name: `MODEL_PROVIDER_HEADER_${name.toUpperCase()}`,
|
|
4299
|
-
value,
|
|
4300
|
-
},
|
|
4301
|
-
],
|
|
4302
|
-
),
|
|
4303
|
-
...Object.entries(selectedProvider?.defaultQuery ?? {}).flatMap(([name, value]) =>
|
|
4304
|
-
publicProviderQueries.has(name.toLowerCase())
|
|
4305
|
-
? []
|
|
4306
|
-
: [{ name: `MODEL_PROVIDER_QUERY_${name.toUpperCase()}`, value }],
|
|
4307
|
-
),
|
|
4308
|
-
]);
|
|
4309
4369
|
// Bind the provider/model catalog's context policy to every model-facing
|
|
4310
4370
|
// path for this turn. In particular, Codex subscription turns must not
|
|
4311
4371
|
// inherit the deployment's OpenAI/Azure mode or 1.05M context defaults:
|
|
@@ -4360,15 +4420,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4360
4420
|
input.workspaceId,
|
|
4361
4421
|
effectiveCodexCredentialId ?? "",
|
|
4362
4422
|
);
|
|
4363
|
-
const registerSnapshot = async (
|
|
4364
|
-
snapshotPromise: ReturnType<typeof resolver.getToken>,
|
|
4365
|
-
) => {
|
|
4366
|
-
const snapshot = await snapshotPromise;
|
|
4367
|
-
registerSecretRedactions([
|
|
4368
|
-
{ name: "CODEX_ACCESS_TOKEN", value: snapshot.accessToken },
|
|
4369
|
-
]);
|
|
4370
|
-
return snapshot;
|
|
4371
|
-
};
|
|
4372
4423
|
return {
|
|
4373
4424
|
clientVersion: CODEX_CLIENT_VERSION,
|
|
4374
4425
|
// Backend sticky cache-routing key — the SAME id as the body's
|
|
@@ -4378,8 +4429,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4378
4429
|
// (per-request shard lottery = prod's measured 48.6% on sol);
|
|
4379
4430
|
// with it, resends pin to the warm shard (Codex CLI parity).
|
|
4380
4431
|
sessionId: input.sessionId,
|
|
4381
|
-
getToken: () =>
|
|
4382
|
-
refresh: () =>
|
|
4432
|
+
getToken: () => resolver.getToken(),
|
|
4433
|
+
refresh: () => resolver.refresh(),
|
|
4383
4434
|
resolveModel: buildModelResolver(
|
|
4384
4435
|
CODEX_FALLBACK_MODEL_SLUGS,
|
|
4385
4436
|
CODEX_FALLBACK_MODEL_SLUGS[0],
|
|
@@ -4622,6 +4673,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4622
4673
|
force: true,
|
|
4623
4674
|
clearRequestedCompaction: true,
|
|
4624
4675
|
trigger: "operator",
|
|
4676
|
+
materializeHistory: materializeScreenshotHistory,
|
|
4625
4677
|
projectModelInput: compactionModelHistoryProjector,
|
|
4626
4678
|
...compactionModeOptions,
|
|
4627
4679
|
},
|
|
@@ -4654,7 +4706,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4654
4706
|
},
|
|
4655
4707
|
);
|
|
4656
4708
|
if (!isCompactionSummaryFailure(error)) throw error;
|
|
4657
|
-
const errorMessage = String(
|
|
4709
|
+
const errorMessage = String(compactionFailureReasonFromError(error));
|
|
4658
4710
|
if (
|
|
4659
4711
|
!(await settle!({
|
|
4660
4712
|
events: [
|
|
@@ -4787,11 +4839,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4787
4839
|
rigDefaultEnvironmentValues,
|
|
4788
4840
|
workspaceVariableSet?.values ?? {},
|
|
4789
4841
|
);
|
|
4790
|
-
// Redact EVERY exported secret value (rig defaults + session set) from turn
|
|
4791
|
-
// output, not just the session set's.
|
|
4792
|
-
registerSecretRedactions(
|
|
4793
|
-
Object.entries(sandboxWorkspaceEnvironmentValues).map(([name, value]) => ({ name, value })),
|
|
4794
|
-
);
|
|
4795
4842
|
// EFFECTIVE compute backend, resolved ONCE at turn start (Case B + Stage D
|
|
4796
4843
|
// D1-lite) and reused for EVERY downstream decision: the env mint (skip
|
|
4797
4844
|
// inert platform git tokens for a machine turn), the establish path (no phantom Modal
|
|
@@ -4897,7 +4944,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4897
4944
|
)
|
|
4898
4945
|
: null;
|
|
4899
4946
|
if (initialRunCredentialMaterial) {
|
|
4900
|
-
registerSecretRedactions(initialRunCredentialMaterial.redactions);
|
|
4901
4947
|
for (const payload of runCredentialAuthNeededPayloads(initialRunCredentialMaterial)) {
|
|
4902
4948
|
publishedRunCredentialNotices.add(JSON.stringify(payload));
|
|
4903
4949
|
await publish!([{ type: "credential.auth_needed", payload }], true);
|
|
@@ -4913,7 +4959,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4913
4959
|
undefined,
|
|
4914
4960
|
);
|
|
4915
4961
|
const establishPolicy: "eager" | "on-demand" =
|
|
4916
|
-
lazyProvisionEnabled(settings) &&
|
|
4962
|
+
lazyProvisionEnabled(settings) &&
|
|
4963
|
+
!machinePrimary &&
|
|
4964
|
+
runSettings.sandboxBackend !== "none" &&
|
|
4965
|
+
// Resolved run credentials must be written to one exact leased sandbox
|
|
4966
|
+
// before agent execution. A warm active pointer can bypass the lazy
|
|
4967
|
+
// provisioner entirely, which previously skipped materialization.
|
|
4968
|
+
!initialRunCredentialMaterial
|
|
4917
4969
|
? "on-demand"
|
|
4918
4970
|
: "eager";
|
|
4919
4971
|
// Computed exactly ONCE per turn and reused for BOTH the box manifest
|
|
@@ -4987,6 +5039,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4987
5039
|
sandboxWorkspaceEnvironmentValues,
|
|
4988
5040
|
{
|
|
4989
5041
|
skipGitHubToken: activeSandboxBackend === "selfhosted",
|
|
5042
|
+
skipToolspace: activeSandboxBackend === "selfhosted",
|
|
4990
5043
|
deferGitHubToken:
|
|
4991
5044
|
activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
|
|
4992
5045
|
scope: connectionScope,
|
|
@@ -4999,19 +5052,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4999
5052
|
cancellationSignal,
|
|
5000
5053
|
undefined,
|
|
5001
5054
|
);
|
|
5002
|
-
|
|
5003
|
-
...Object.entries(sandboxGitTokens ?? {}).flatMap(([provider, value]) =>
|
|
5004
|
-
value ? [{ name: `GIT_${provider.toUpperCase()}_TOKEN`, value }] : [],
|
|
5005
|
-
),
|
|
5006
|
-
...(sandboxGitToken ? [{ name: "GIT_TOKEN", value: sandboxGitToken }] : []),
|
|
5007
|
-
...((sandboxGitCredentialBindings ?? []).map((binding) => ({
|
|
5008
|
-
name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
|
|
5009
|
-
value: binding.token,
|
|
5010
|
-
})) satisfies SecretForRedaction[]),
|
|
5011
|
-
...(sandboxToolspaceToken
|
|
5012
|
-
? [{ name: "TOOLSPACE_TOKEN", value: sandboxToolspaceToken }]
|
|
5013
|
-
: []),
|
|
5014
|
-
]);
|
|
5055
|
+
|
|
5015
5056
|
const sandboxToolspaceTokenFile = sandboxToolspaceToken
|
|
5016
5057
|
? toolspaceTokenFileFromEnvironment(sandboxEnvironment, input.sessionId)
|
|
5017
5058
|
: undefined;
|
|
@@ -5057,12 +5098,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5057
5098
|
);
|
|
5058
5099
|
if (binding) {
|
|
5059
5100
|
assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
|
|
5060
|
-
registerSecretRedactions([
|
|
5061
|
-
{
|
|
5062
|
-
name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
|
|
5063
|
-
value: binding.token,
|
|
5064
|
-
},
|
|
5065
|
-
]);
|
|
5066
5101
|
}
|
|
5067
5102
|
pendingBinding = binding;
|
|
5068
5103
|
return binding
|
|
@@ -5150,7 +5185,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5150
5185
|
toolspaceAuthority,
|
|
5151
5186
|
);
|
|
5152
5187
|
if (material) {
|
|
5153
|
-
registerSecretRedactions([{ name: "TOOLSPACE_TOKEN", value: material.token }]);
|
|
5154
5188
|
}
|
|
5155
5189
|
return material;
|
|
5156
5190
|
};
|
|
@@ -5277,7 +5311,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5277
5311
|
);
|
|
5278
5312
|
return;
|
|
5279
5313
|
}
|
|
5280
|
-
|
|
5314
|
+
|
|
5281
5315
|
await runWorkspaceMutationForSandbox(
|
|
5282
5316
|
requireTargetSandbox(),
|
|
5283
5317
|
"runCredentialMaterialization",
|
|
@@ -5435,6 +5469,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5435
5469
|
db,
|
|
5436
5470
|
settings,
|
|
5437
5471
|
bus,
|
|
5472
|
+
opJournal,
|
|
5438
5473
|
onOp: machineOpObserver.observer,
|
|
5439
5474
|
onSandboxOperation: sandboxOperationObserver,
|
|
5440
5475
|
onHomeSandboxRebound,
|
|
@@ -5544,6 +5579,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5544
5579
|
db,
|
|
5545
5580
|
settings,
|
|
5546
5581
|
bus,
|
|
5582
|
+
opJournal,
|
|
5547
5583
|
onSandboxOperation: sandboxOperationObserver,
|
|
5548
5584
|
onHomeSandboxLost: publishSandboxLost,
|
|
5549
5585
|
onHomeSandboxRebound,
|
|
@@ -5628,9 +5664,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5628
5664
|
const resolveCredential: typeof rawResolveCredential = async (request) => {
|
|
5629
5665
|
const result = await resolveFrozenCredential(request);
|
|
5630
5666
|
if (result.status === "ok") {
|
|
5631
|
-
registerSecretRedactions(
|
|
5632
|
-
headerSecretRedactions("MCP", result.headers, Object.keys(result.headers)),
|
|
5633
|
-
);
|
|
5634
5667
|
}
|
|
5635
5668
|
return result;
|
|
5636
5669
|
};
|
|
@@ -5652,9 +5685,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5652
5685
|
}) => Promise<T>,
|
|
5653
5686
|
): Promise<T> => {
|
|
5654
5687
|
const snapshot = await resolver.getToken();
|
|
5655
|
-
|
|
5656
|
-
{ name: "CODEX_APPS_ACCESS_TOKEN", value: snapshot.accessToken },
|
|
5657
|
-
]);
|
|
5688
|
+
|
|
5658
5689
|
return await withCodexAppsRequestAuthorization(
|
|
5659
5690
|
db,
|
|
5660
5691
|
{ workspaceId: input.workspaceId, credentialId: codexAppsCredentialId },
|
|
@@ -5769,6 +5800,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5769
5800
|
latencyMode: turnExecutionPolicy.latencyMode,
|
|
5770
5801
|
...(serviceTier ? { serviceTier } : {}),
|
|
5771
5802
|
...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
|
|
5803
|
+
humanInputEnabled: agentHumanInputEnabled,
|
|
5772
5804
|
genesisTitleHint: isGenesisTurn,
|
|
5773
5805
|
persistentSessionSettings: {
|
|
5774
5806
|
titleIsSet: Boolean(session.title?.trim()),
|
|
@@ -5793,14 +5825,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5793
5825
|
...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
|
|
5794
5826
|
? { gitTokenSeed: sandboxGitToken }
|
|
5795
5827
|
: {}),
|
|
5796
|
-
// Toolspace is
|
|
5797
|
-
//
|
|
5798
|
-
//
|
|
5799
|
-
// machine's ONLY path to programmatic tool calling and grants no more
|
|
5800
|
-
// than toolspace:call for its own session (own-session-bound,
|
|
5801
|
-
// short-lived and renewable, per-active-turn budgeted, approval-tools
|
|
5802
|
-
// excluded). The runtime seeds it to the box's token file over the same
|
|
5803
|
-
// exec channel, off-manifest, on every backend.
|
|
5828
|
+
// Toolspace delivery is managed-sandbox-only. Connected Machines own any
|
|
5829
|
+
// manually configured API credentials and must never be contacted during
|
|
5830
|
+
// turn admission merely to seed OpenGeni tooling.
|
|
5804
5831
|
...(sandboxToolspaceToken
|
|
5805
5832
|
? {
|
|
5806
5833
|
toolspaceTokenSeed: sandboxToolspaceToken,
|
|
@@ -6108,6 +6135,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6108
6135
|
db,
|
|
6109
6136
|
settings,
|
|
6110
6137
|
bus,
|
|
6138
|
+
opJournal,
|
|
6111
6139
|
onOp: machineOpObserver.observer,
|
|
6112
6140
|
onSandboxOperation: sandboxOperationObserver,
|
|
6113
6141
|
onHomeSandboxLost: publishSandboxLost,
|
|
@@ -6206,7 +6234,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6206
6234
|
},
|
|
6207
6235
|
);
|
|
6208
6236
|
if (!isCompactionSummaryFailure(error)) throw error;
|
|
6209
|
-
const errorMessage = String(
|
|
6237
|
+
const errorMessage = String(compactionFailureReasonFromError(error));
|
|
6210
6238
|
if (
|
|
6211
6239
|
!(await settle!({
|
|
6212
6240
|
events: [
|
|
@@ -6308,11 +6336,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6308
6336
|
force: true,
|
|
6309
6337
|
clearRequestedCompaction: true,
|
|
6310
6338
|
trigger: "operator",
|
|
6339
|
+
materializeHistory: materializeScreenshotHistory,
|
|
6311
6340
|
projectModelInput: compactionModelHistoryProjector,
|
|
6312
6341
|
...compactionModeOptions,
|
|
6313
6342
|
}
|
|
6314
6343
|
: {
|
|
6315
6344
|
trigger: "auto",
|
|
6345
|
+
materializeHistory: materializeScreenshotHistory,
|
|
6316
6346
|
projectModelInput: compactionModelHistoryProjector,
|
|
6317
6347
|
...compactionModeOptions,
|
|
6318
6348
|
},
|
|
@@ -6346,11 +6376,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6346
6376
|
},
|
|
6347
6377
|
);
|
|
6348
6378
|
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
6349
|
-
const errorMessage = String(
|
|
6379
|
+
const errorMessage = String(compactionFailureReasonFromError(compactError));
|
|
6350
6380
|
observability.error("context compaction failed", {
|
|
6351
6381
|
sessionId: input.sessionId,
|
|
6352
6382
|
turnId,
|
|
6353
|
-
|
|
6383
|
+
...safeErrorDiagnostic(compactError),
|
|
6354
6384
|
});
|
|
6355
6385
|
if (
|
|
6356
6386
|
!(await settle!({
|
|
@@ -6465,6 +6495,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6465
6495
|
...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
|
|
6466
6496
|
...(runCredentialsNote ? { runCredentialsNote } : {}),
|
|
6467
6497
|
providerApi,
|
|
6498
|
+
materializeModelHistory: materializeScreenshotHistory,
|
|
6499
|
+
materializeSerializedRunState: materializeScreenshotRunState,
|
|
6468
6500
|
projectModelHistory: modelHistoryProjector,
|
|
6469
6501
|
});
|
|
6470
6502
|
runInput = prepared.input;
|
|
@@ -6478,16 +6510,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6478
6510
|
// total rows no longer equal max(position)+1. Pre-compaction both reduce to
|
|
6479
6511
|
// the old total-count value, so the common path is unchanged.
|
|
6480
6512
|
//
|
|
6481
|
-
// CRITICAL: seed from the
|
|
6513
|
+
// CRITICAL: seed from the structurally repaired active-row length, not the raw active
|
|
6482
6514
|
// count. `prepareRunInput` builds `state.history` from
|
|
6483
6515
|
// `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
|
|
6484
6516
|
// rows (a legacy orphan/dangling pair), the in-memory history this turn
|
|
6485
6517
|
// starts from is K shorter than the raw row count. The reconcile slices the
|
|
6486
|
-
//
|
|
6518
|
+
// repaired `state.history` off `persistedHistoryCount`; seeding it from
|
|
6487
6519
|
// the raw count (K too high) skips K genuinely-new items, and a
|
|
6488
6520
|
// `function_call` left in that skipped region can later have its
|
|
6489
6521
|
// `function_call_result` persisted alone — the orphan that 400s on replay
|
|
6490
|
-
// and bricks the session (issue-61). The
|
|
6522
|
+
// and bricks the session (issue-61). The repaired seed is already
|
|
6491
6523
|
// orphan-free, so it is a stable prefix of the re-sanitized history and the
|
|
6492
6524
|
// slice begins exactly at the first genuinely-new item.
|
|
6493
6525
|
persistedHistoryCount = prepared.persistedHistoryCount;
|
|
@@ -6525,6 +6557,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6525
6557
|
force: true,
|
|
6526
6558
|
...(triggerLabel === "operator" ? { clearRequestedCompaction: true } : {}),
|
|
6527
6559
|
trigger: triggerLabel,
|
|
6560
|
+
materializeHistory: materializeScreenshotHistory,
|
|
6528
6561
|
projectModelInput: compactionModelHistoryProjector,
|
|
6529
6562
|
...compactionModeOptions,
|
|
6530
6563
|
},
|
|
@@ -6573,6 +6606,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6573
6606
|
// its first-operation provisioner above.
|
|
6574
6607
|
if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
|
|
6575
6608
|
const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
|
|
6609
|
+
// `deferredSetup: true` below tells the runtime that the worker owns
|
|
6610
|
+
// platform setup, so the runtime intentionally skips its credential
|
|
6611
|
+
// session callback. Materialize host-managed run credentials here
|
|
6612
|
+
// before any setup command (including provider login hooks), then
|
|
6613
|
+
// decorate setup commands so they source the active generation.
|
|
6614
|
+
await attachRunCredentialRenewal(
|
|
6615
|
+
eagerSetupSession as RunCredentialCommandSession,
|
|
6616
|
+
resolvedSandbox,
|
|
6617
|
+
);
|
|
6618
|
+
const eagerCredentialSetupSession = initialRunCredentialMaterial
|
|
6619
|
+
? withRunCredentialsSession(eagerSetupSession as object, input.sessionId)
|
|
6620
|
+
: eagerSetupSession;
|
|
6576
6621
|
await runWorkspaceMutationForSandbox(
|
|
6577
6622
|
resolvedSandbox,
|
|
6578
6623
|
"eagerOwnedSandboxSetup",
|
|
@@ -6580,7 +6625,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6580
6625
|
await runOwnedSandboxSetup(
|
|
6581
6626
|
agent,
|
|
6582
6627
|
ownedEstablished.session as never,
|
|
6583
|
-
|
|
6628
|
+
eagerCredentialSetupSession as never,
|
|
6584
6629
|
{
|
|
6585
6630
|
settings: runSettings,
|
|
6586
6631
|
environment: sandboxEnvironment,
|
|
@@ -6652,9 +6697,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6652
6697
|
tokenSession: ToolspaceTokenWriterSession,
|
|
6653
6698
|
) => {
|
|
6654
6699
|
const renewalSession =
|
|
6655
|
-
|
|
6656
|
-
? tokenSession
|
|
6657
|
-
: ((setupBoxSession as ToolspaceTokenWriterSession | null) ?? tokenSession);
|
|
6700
|
+
(setupBoxSession as ToolspaceTokenWriterSession | null) ?? tokenSession;
|
|
6658
6701
|
await attachToolspaceTokenRenewal(renewalSession);
|
|
6659
6702
|
},
|
|
6660
6703
|
}
|
|
@@ -6679,7 +6722,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6679
6722
|
contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
|
|
6680
6723
|
contextCompactionRequested: () =>
|
|
6681
6724
|
isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
6682
|
-
callModelInputFilter: secretRedactionModelInputFilter((value) => redact(value)),
|
|
6683
6725
|
...(toolCancellationFenceRef.current
|
|
6684
6726
|
? { turnToolCancellationFence: toolCancellationFenceRef.current }
|
|
6685
6727
|
: {}),
|
|
@@ -6717,6 +6759,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6717
6759
|
}
|
|
6718
6760
|
let stableToolCallIdsToClear: string[] | null = null;
|
|
6719
6761
|
let completedCurrentToolBatch = false;
|
|
6762
|
+
let retainedScreenshotMetadata: RetainedArtifactMetadata | null = null;
|
|
6720
6763
|
const responseResult = await processModelResponseTerminalEvent({
|
|
6721
6764
|
event: next.value,
|
|
6722
6765
|
state: modelResponseState,
|
|
@@ -6770,7 +6813,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6770
6813
|
// conversation context preserved for the post-top-up resume.
|
|
6771
6814
|
let serializedRunState: string | null = null;
|
|
6772
6815
|
try {
|
|
6773
|
-
serializedRunState = String(
|
|
6816
|
+
serializedRunState = String(stream.state.toString());
|
|
6774
6817
|
} catch {
|
|
6775
6818
|
serializedRunState = null;
|
|
6776
6819
|
}
|
|
@@ -6792,7 +6835,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6792
6835
|
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
6793
6836
|
callId: pendingToolCall.callId,
|
|
6794
6837
|
callType: pendingToolCall.callType,
|
|
6795
|
-
callItem:
|
|
6838
|
+
callItem: pendingToolCall.callItem as Record<string, unknown>,
|
|
6796
6839
|
});
|
|
6797
6840
|
if (!registered.accepted) {
|
|
6798
6841
|
throw new TurnAttemptFencedError(
|
|
@@ -6800,9 +6843,39 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6800
6843
|
);
|
|
6801
6844
|
}
|
|
6802
6845
|
currentToolBatchCallIds.add(pendingToolCall.callId);
|
|
6846
|
+
if (pendingToolCall.callName === "computer_screenshot") {
|
|
6847
|
+
computerScreenshotCallIds.add(pendingToolCall.callId);
|
|
6848
|
+
}
|
|
6803
6849
|
}
|
|
6804
6850
|
const completedToolCall = completedToolCallFromSdkEvent(next.value);
|
|
6805
6851
|
if (completedToolCall) {
|
|
6852
|
+
const typedScreenshot = typedScreenshotFromSdkEvent(next.value);
|
|
6853
|
+
if (
|
|
6854
|
+
typedScreenshot &&
|
|
6855
|
+
typedScreenshot.callId === completedToolCall.callId &&
|
|
6856
|
+
computerScreenshotCallIds.has(completedToolCall.callId)
|
|
6857
|
+
) {
|
|
6858
|
+
retainedScreenshotMetadata = await retainComputerScreenshot({
|
|
6859
|
+
db,
|
|
6860
|
+
objectStorage,
|
|
6861
|
+
accountId: input.accountId,
|
|
6862
|
+
workspaceId: input.workspaceId,
|
|
6863
|
+
sessionId: input.sessionId,
|
|
6864
|
+
turnId: activeTurnId,
|
|
6865
|
+
attemptId: input.attemptId,
|
|
6866
|
+
output: typedScreenshot,
|
|
6867
|
+
});
|
|
6868
|
+
retainedScreenshotReceiptsByCallId.set(
|
|
6869
|
+
completedToolCall.callId,
|
|
6870
|
+
retainedScreenshotMetadata,
|
|
6871
|
+
);
|
|
6872
|
+
}
|
|
6873
|
+
const durableResultItem = retainedScreenshotMetadata
|
|
6874
|
+
? (compactRetainedScreenshotHistory(
|
|
6875
|
+
[completedToolCall.resultItem],
|
|
6876
|
+
retainedScreenshotReceiptsByCallId,
|
|
6877
|
+
)[0] ?? completedToolCall.resultItem)
|
|
6878
|
+
: completedToolCall.resultItem;
|
|
6806
6879
|
// Keep every parallel result in the attempt ledger until the full
|
|
6807
6880
|
// call batch has settled. The SDK's computed history is
|
|
6808
6881
|
// non-monotonic while parallel calls complete (a later call may
|
|
@@ -6817,7 +6890,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6817
6890
|
attemptId: input.attemptId,
|
|
6818
6891
|
callId: completedToolCall.callId,
|
|
6819
6892
|
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
6820
|
-
resultItem:
|
|
6893
|
+
resultItem: durableResultItem as Record<string, unknown>,
|
|
6821
6894
|
});
|
|
6822
6895
|
if (!recorded.accepted) {
|
|
6823
6896
|
throw new TurnAttemptFencedError(
|
|
@@ -6846,7 +6919,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6846
6919
|
completedCurrentToolBatch = currentBatchIsStable;
|
|
6847
6920
|
}
|
|
6848
6921
|
}
|
|
6849
|
-
const normalized = normalizeSdkEvent(
|
|
6922
|
+
const normalized = normalizeSdkEvent(
|
|
6923
|
+
next.value,
|
|
6924
|
+
retainedScreenshotMetadata
|
|
6925
|
+
? {
|
|
6926
|
+
toolOutputOverride: retainedScreenshotMetadata,
|
|
6927
|
+
retainedOutputEvidence: retainedScreenshotMetadata.available
|
|
6928
|
+
? retainedScreenshotMetadata
|
|
6929
|
+
: {
|
|
6930
|
+
available: false,
|
|
6931
|
+
reason: retainedScreenshotMetadata.reason,
|
|
6932
|
+
},
|
|
6933
|
+
}
|
|
6934
|
+
: {},
|
|
6935
|
+
);
|
|
6850
6936
|
for (const event of normalized) {
|
|
6851
6937
|
streamTiming.onEvent(event.type);
|
|
6852
6938
|
await batcher.push(event);
|
|
@@ -6870,6 +6956,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6870
6956
|
currentToolBatchCallIds = new Set<string>();
|
|
6871
6957
|
currentToolBatchCompletedCallIds = new Set<string>();
|
|
6872
6958
|
}
|
|
6959
|
+
for (const callId of stableToolCallIdsToClear) {
|
|
6960
|
+
computerScreenshotCallIds.delete(callId);
|
|
6961
|
+
}
|
|
6873
6962
|
}
|
|
6874
6963
|
}
|
|
6875
6964
|
} finally {
|
|
@@ -6983,6 +7072,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6983
7072
|
const approvals = runtime.serializeApprovals(stream.interruptions);
|
|
6984
7073
|
const humanInputInterruptions =
|
|
6985
7074
|
runtime.serializeHumanInputRequests?.(stream.interruptions) ?? [];
|
|
7075
|
+
const latestWorkspace = await getWorkspace(db, input.workspaceId);
|
|
7076
|
+
if (!latestWorkspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
|
|
7077
|
+
assertWorkspaceHumanInputAllowed(
|
|
7078
|
+
resolveWorkspaceAgentHumanInputEnabled(latestWorkspace.settings),
|
|
7079
|
+
"interruption",
|
|
7080
|
+
humanInputInterruptions.length > 0,
|
|
7081
|
+
);
|
|
6986
7082
|
const humanInputRequests = await Promise.all(
|
|
6987
7083
|
humanInputInterruptions.map(async (interruption) => {
|
|
6988
7084
|
const id = stableHumanInputRequestId(
|
|
@@ -7058,25 +7154,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7058
7154
|
) {
|
|
7059
7155
|
return claimedResult({ status: "cancelled" });
|
|
7060
7156
|
}
|
|
7157
|
+
// The interruption and its preceding tool results are now durable.
|
|
7158
|
+
await finalizeTurnOpStreamOps();
|
|
7061
7159
|
activityStatus = "requires_action";
|
|
7062
7160
|
return claimedResult({ status: "requires_action" });
|
|
7063
7161
|
}
|
|
7064
7162
|
|
|
7065
|
-
const finalOutput = String(
|
|
7066
|
-
await reconcileConversationTruth();
|
|
7163
|
+
const finalOutput = String(stream.finalOutput ?? "");
|
|
7164
|
+
await reconcileConversationTruth({ requireDurable: true });
|
|
7067
7165
|
// Op-stream durability fence: the tool outputs are now durably in the
|
|
7068
7166
|
// history store (a redispatch would NOT re-execute them), so this
|
|
7069
7167
|
// turn's settled ops may advance their acked frontier — journal persist
|
|
7070
7168
|
// then wire final ack (licensing the runner to GC its retained
|
|
7071
7169
|
// frames). Best-effort: a miss leaves the runner's retention TTL to
|
|
7072
7170
|
// reap, never fails a completed turn.
|
|
7073
|
-
|
|
7074
|
-
try {
|
|
7075
|
-
await machinePrimarySession.finalizeOpStreamOps();
|
|
7076
|
-
} catch {
|
|
7077
|
-
// The runner's retention TTL owns the fallback.
|
|
7078
|
-
}
|
|
7079
|
-
}
|
|
7171
|
+
await finalizeTurnOpStreamOps();
|
|
7080
7172
|
if (
|
|
7081
7173
|
!(await settle!({
|
|
7082
7174
|
events: [
|
|
@@ -7143,8 +7235,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7143
7235
|
sessionId: input.sessionId,
|
|
7144
7236
|
turnId: activeTurnId,
|
|
7145
7237
|
reason: recoveryKind,
|
|
7146
|
-
|
|
7147
|
-
error: String(redact(overflow?.message ?? compactionNeeded?.message ?? "")),
|
|
7238
|
+
...safeErrorDiagnostic(attemptError),
|
|
7148
7239
|
signalTokens: compactionNeeded?.signalTokens,
|
|
7149
7240
|
thresholdTokens: compactionNeeded?.thresholdTokens,
|
|
7150
7241
|
});
|
|
@@ -7189,13 +7280,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7189
7280
|
);
|
|
7190
7281
|
compactionRequestCleared = landmark.requestConsumed;
|
|
7191
7282
|
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
7192
|
-
compactionFailureMessage = String(
|
|
7193
|
-
redact(compactionFailureReasonFromError(compactError)),
|
|
7194
|
-
);
|
|
7283
|
+
compactionFailureMessage = String(compactionFailureReasonFromError(compactError));
|
|
7195
7284
|
observability.warn("context compaction recovery compaction failed", {
|
|
7196
7285
|
sessionId: input.sessionId,
|
|
7197
7286
|
turnId: activeTurnId,
|
|
7198
|
-
|
|
7287
|
+
...safeErrorDiagnostic(compactError),
|
|
7199
7288
|
});
|
|
7200
7289
|
}
|
|
7201
7290
|
if (!compactionHandled) {
|
|
@@ -7306,6 +7395,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7306
7395
|
turnMetricOutcome = "cancelled";
|
|
7307
7396
|
return claimedResult({ status: "cancelled" });
|
|
7308
7397
|
}
|
|
7398
|
+
acknowledgeRecoveryQuiescence();
|
|
7309
7399
|
await publishDurableSessionEvents(
|
|
7310
7400
|
bus,
|
|
7311
7401
|
input.workspaceId,
|
|
@@ -7325,7 +7415,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7325
7415
|
} catch (recoveryError) {
|
|
7326
7416
|
console.error(
|
|
7327
7417
|
"sandbox lease supersession recovery failed",
|
|
7328
|
-
safeErrorDiagnostic(recoveryError
|
|
7418
|
+
safeErrorDiagnostic(recoveryError),
|
|
7329
7419
|
);
|
|
7330
7420
|
throw recoveryError;
|
|
7331
7421
|
}
|
|
@@ -7357,6 +7447,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7357
7447
|
turnMetricOutcome = "cancelled";
|
|
7358
7448
|
return claimedResult({ status: "cancelled" });
|
|
7359
7449
|
}
|
|
7450
|
+
acknowledgeRecoveryQuiescence();
|
|
7360
7451
|
await publishDurableSessionEvents(
|
|
7361
7452
|
bus,
|
|
7362
7453
|
input.workspaceId,
|
|
@@ -7372,7 +7463,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7372
7463
|
} catch (recoveryError) {
|
|
7373
7464
|
console.error(
|
|
7374
7465
|
"sandbox deadline rotation recovery failed",
|
|
7375
|
-
safeErrorDiagnostic(recoveryError
|
|
7466
|
+
safeErrorDiagnostic(recoveryError),
|
|
7376
7467
|
);
|
|
7377
7468
|
throw recoveryError;
|
|
7378
7469
|
}
|
|
@@ -7406,6 +7497,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7406
7497
|
turnMetricOutcome = "cancelled";
|
|
7407
7498
|
return claimedResult({ status: "cancelled" });
|
|
7408
7499
|
}
|
|
7500
|
+
acknowledgeRecoveryQuiescence();
|
|
7409
7501
|
await publishDurableSessionEvents(
|
|
7410
7502
|
bus,
|
|
7411
7503
|
input.workspaceId,
|
|
@@ -7421,7 +7513,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7421
7513
|
// the turn through a second cancellation path.
|
|
7422
7514
|
console.error(
|
|
7423
7515
|
"worker-shutdown recovery checkpoint failed",
|
|
7424
|
-
safeErrorDiagnostic(recoveryError
|
|
7516
|
+
safeErrorDiagnostic(recoveryError),
|
|
7425
7517
|
);
|
|
7426
7518
|
throw recoveryError;
|
|
7427
7519
|
}
|
|
@@ -7520,11 +7612,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7520
7612
|
await reconcileConversationTruth({ requireDurable: true });
|
|
7521
7613
|
}
|
|
7522
7614
|
checkpointDurable = true;
|
|
7523
|
-
} catch
|
|
7615
|
+
} catch {
|
|
7524
7616
|
observability.warn("Codex lease-loss checkpoint failed; refusing automatic turn replay", {
|
|
7525
|
-
|
|
7526
|
-
|
|
7527
|
-
|
|
7617
|
+
errorClass: "CodexCheckpointOperationError",
|
|
7618
|
+
errorCode: "codex_lease_loss_checkpoint_failed",
|
|
7619
|
+
origin: "worker",
|
|
7528
7620
|
});
|
|
7529
7621
|
}
|
|
7530
7622
|
|
|
@@ -7631,16 +7723,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7631
7723
|
await flushRuntimeBatcher();
|
|
7632
7724
|
await reconcileConversationTruth({ requireDurable: true });
|
|
7633
7725
|
checkpointDurable = true;
|
|
7634
|
-
} catch
|
|
7726
|
+
} catch {
|
|
7635
7727
|
observability.incrementCounter({
|
|
7636
7728
|
name: "opengeni_codex_failover_checkpoints_total",
|
|
7637
7729
|
help: "Durable Codex failover checkpoint attempts by outcome.",
|
|
7638
7730
|
labels: { workspace_key: codexWorkspaceKey, outcome: "failed" },
|
|
7639
7731
|
});
|
|
7640
7732
|
observability.warn("Codex failover checkpoint failed; refusing automatic replay", {
|
|
7641
|
-
|
|
7642
|
-
|
|
7643
|
-
|
|
7733
|
+
errorClass: "CodexCheckpointOperationError",
|
|
7734
|
+
errorCode: "codex_failover_checkpoint_failed",
|
|
7735
|
+
origin: "worker",
|
|
7644
7736
|
});
|
|
7645
7737
|
}
|
|
7646
7738
|
|
|
@@ -8193,6 +8285,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8193
8285
|
return claimedResult({ status: "cancelled" });
|
|
8194
8286
|
}
|
|
8195
8287
|
if (recovery.action === "recovering") {
|
|
8288
|
+
acknowledgeRecoveryQuiescence();
|
|
8196
8289
|
await publishDurableSessionEvents(
|
|
8197
8290
|
bus,
|
|
8198
8291
|
input.workspaceId,
|
|
@@ -8211,15 +8304,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8211
8304
|
// truth, recover this SAME accepted turn, then let the workflow re-claim
|
|
8212
8305
|
// it after a pacing delay. This is independent of goal state and never
|
|
8213
8306
|
// relies on a synthetic continuation prompt.
|
|
8214
|
-
const failure =
|
|
8307
|
+
const failure = agentRunFailurePayload(error, { isCodexTurn }) as ReturnType<
|
|
8215
8308
|
typeof agentRunFailurePayload
|
|
8216
8309
|
>;
|
|
8217
8310
|
if (isSessionEventPersistenceError(error)) {
|
|
8218
|
-
//
|
|
8219
|
-
//
|
|
8220
|
-
// boundary retains only SQLSTATE, stage, correlation, and safe catalog
|
|
8221
|
-
// identifiers. Provider inference has already happened and is NOT
|
|
8222
|
-
// retried by this terminal classification.
|
|
8311
|
+
// Preserve the exact source message in the internal runtime diagnostic;
|
|
8312
|
+
// SQLSTATE/catalog facts remain separate classification attributes.
|
|
8223
8313
|
observability.error("session event persistence failed", {
|
|
8224
8314
|
accountId: input.accountId,
|
|
8225
8315
|
workspaceId: input.workspaceId,
|
|
@@ -8240,6 +8330,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8240
8330
|
dbDataType: error.details.database.dataType,
|
|
8241
8331
|
dbConstraint: error.details.database.constraint,
|
|
8242
8332
|
dbRoutine: error.details.database.routine,
|
|
8333
|
+
error: error.message,
|
|
8243
8334
|
});
|
|
8244
8335
|
}
|
|
8245
8336
|
if (failure.retryable && publish && turnId && turnStartedPublished) {
|
|
@@ -8271,6 +8362,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8271
8362
|
turnMetricOutcome = "cancelled";
|
|
8272
8363
|
return claimedResult({ status: "cancelled" });
|
|
8273
8364
|
}
|
|
8365
|
+
acknowledgeRecoveryQuiescence();
|
|
8274
8366
|
await publishDurableSessionEvents(
|
|
8275
8367
|
bus,
|
|
8276
8368
|
input.workspaceId,
|
|
@@ -8376,18 +8468,38 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8376
8468
|
const credentialSessionToClear = runCredentialSession;
|
|
8377
8469
|
runCredentialSession = null;
|
|
8378
8470
|
if (credentialSessionToClear) {
|
|
8379
|
-
|
|
8380
|
-
|
|
8381
|
-
|
|
8382
|
-
|
|
8383
|
-
|
|
8384
|
-
|
|
8385
|
-
|
|
8386
|
-
|
|
8387
|
-
|
|
8388
|
-
|
|
8389
|
-
|
|
8390
|
-
|
|
8471
|
+
const clearAttemptCredentials = async (): Promise<void> =>
|
|
8472
|
+
await clearRunCredentialsForAttempt(credentialSessionToClear, {
|
|
8473
|
+
sessionId: input.sessionId,
|
|
8474
|
+
attemptId: input.attemptId,
|
|
8475
|
+
executionGeneration,
|
|
8476
|
+
});
|
|
8477
|
+
await clearAttemptCredentialsWithSettledFence({
|
|
8478
|
+
activityStatus,
|
|
8479
|
+
runWorkspaceFencedClear: async () =>
|
|
8480
|
+
await runWorkspaceMutationForSandbox(
|
|
8481
|
+
requireResolvedSandboxForMutation(
|
|
8482
|
+
"Run credential cleanup has no exact sandbox lease target",
|
|
8483
|
+
),
|
|
8484
|
+
"runCredentialAttemptClear",
|
|
8485
|
+
clearAttemptCredentials,
|
|
8486
|
+
),
|
|
8487
|
+
clearExactAttempt: clearAttemptCredentials,
|
|
8488
|
+
onSettledAttemptFence: () => {
|
|
8489
|
+
// Terminal settlement closes the active attempt before this finally
|
|
8490
|
+
// runs, so the ordinary workspace admission correctly rejects it.
|
|
8491
|
+
// The attempt-qualified delete is nevertheless successor-safe: it
|
|
8492
|
+
// removes only this attempt/generation and clears the pointer only
|
|
8493
|
+
// when it still names that exact generation. Finish that cleanup
|
|
8494
|
+
// directly so a successful turn can continue into workspace capture,
|
|
8495
|
+
// tool teardown, and lease release.
|
|
8496
|
+
observability.info("retrying exact run credential cleanup after turn settlement", {
|
|
8497
|
+
"opengeni.session_id": input.sessionId,
|
|
8498
|
+
"opengeni.turn_id": turnId ?? "",
|
|
8499
|
+
"opengeni.attempt_id": input.attemptId,
|
|
8500
|
+
});
|
|
8501
|
+
},
|
|
8502
|
+
});
|
|
8391
8503
|
}
|
|
8392
8504
|
await drainAttemptOwnedSandboxWriters({
|
|
8393
8505
|
// Normal turn completion owns the same process boundary as
|
|
@@ -8455,18 +8567,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8455
8567
|
onReceiptFailure: (error) => {
|
|
8456
8568
|
console.error(
|
|
8457
8569
|
"agent turn quiescence receipt exhausted; signalling proof",
|
|
8458
|
-
safeErrorDiagnostic(error
|
|
8570
|
+
safeErrorDiagnostic(error),
|
|
8459
8571
|
);
|
|
8460
8572
|
},
|
|
8461
8573
|
onPublishFailure: (error) => {
|
|
8462
8574
|
console.error(
|
|
8463
8575
|
"agent turn quiescence event fanout failed",
|
|
8464
|
-
safeErrorDiagnostic(error
|
|
8576
|
+
safeErrorDiagnostic(error),
|
|
8465
8577
|
);
|
|
8466
8578
|
},
|
|
8467
8579
|
onSignalFailure: (error, attempt, retryMs) => {
|
|
8468
8580
|
console.error("agent turn quiescence proof signal failed; retrying", {
|
|
8469
|
-
error: safeErrorDiagnostic(error
|
|
8581
|
+
error: safeErrorDiagnostic(error),
|
|
8470
8582
|
attempt,
|
|
8471
8583
|
retryMs,
|
|
8472
8584
|
});
|
|
@@ -8714,7 +8826,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8714
8826
|
sandboxToRelease.release().catch((releaseError) => {
|
|
8715
8827
|
console.error(
|
|
8716
8828
|
"sandbox lease release failed (turn outcome unaffected)",
|
|
8717
|
-
safeErrorDiagnostic(releaseError
|
|
8829
|
+
safeErrorDiagnostic(releaseError),
|
|
8718
8830
|
);
|
|
8719
8831
|
}),
|
|
8720
8832
|
finalizerSignal,
|
|
@@ -8724,7 +8836,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8724
8836
|
finalizationError ??= error;
|
|
8725
8837
|
console.error(
|
|
8726
8838
|
"agent turn finalization failed (turn outcome unaffected)",
|
|
8727
|
-
safeErrorDiagnostic(error
|
|
8839
|
+
safeErrorDiagnostic(error),
|
|
8728
8840
|
);
|
|
8729
8841
|
} finally {
|
|
8730
8842
|
// If fallible finalization exited before the ordinary exact-holder
|
|
@@ -8785,9 +8897,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8785
8897
|
},
|
|
8786
8898
|
error:
|
|
8787
8899
|
finalizationError || activityError
|
|
8788
|
-
? safeErrorForTelemetry(finalizationError ?? activityError
|
|
8789
|
-
String(redact(value)),
|
|
8790
|
-
)
|
|
8900
|
+
? safeErrorForTelemetry(finalizationError ?? activityError)
|
|
8791
8901
|
: undefined,
|
|
8792
8902
|
});
|
|
8793
8903
|
assertPhysicalToolQuiescenceForCancellation({
|
|
@@ -8919,7 +9029,19 @@ export function agentRunFailurePayload(
|
|
|
8919
9029
|
attempts?: number;
|
|
8920
9030
|
retryOutcome?: string;
|
|
8921
9031
|
database?: Record<string, string>;
|
|
9032
|
+
historyPersistenceStage?: MandatoryHistoryPersistenceStage;
|
|
8922
9033
|
} {
|
|
9034
|
+
if (error instanceof MandatoryHistoryPersistenceError) {
|
|
9035
|
+
const underlying = isSessionEventPersistenceError(error.cause)
|
|
9036
|
+
? agentRunFailurePayload(error.cause, options)
|
|
9037
|
+
: {
|
|
9038
|
+
error: error.cause instanceof Error ? error.cause.message : String(error.cause),
|
|
9039
|
+
};
|
|
9040
|
+
return {
|
|
9041
|
+
...underlying,
|
|
9042
|
+
historyPersistenceStage: error.stage,
|
|
9043
|
+
};
|
|
9044
|
+
}
|
|
8923
9045
|
const message = error instanceof Error ? error.message : String(error);
|
|
8924
9046
|
const status =
|
|
8925
9047
|
typeof error === "object" && error !== null
|
|
@@ -8945,15 +9067,8 @@ export function agentRunFailurePayload(
|
|
|
8945
9067
|
}
|
|
8946
9068
|
if (isSessionEventPersistenceError(error)) {
|
|
8947
9069
|
const { details } = error;
|
|
8948
|
-
const eventLabel = details.eventTypes.join(", ") || "session events";
|
|
8949
|
-
const failureLabel =
|
|
8950
|
-
details.code === "db_deadlock"
|
|
8951
|
-
? "Database deadlock"
|
|
8952
|
-
: details.code === "db_serialization_failure"
|
|
8953
|
-
? "Database serialization failure"
|
|
8954
|
-
: "Database failure";
|
|
8955
9070
|
return {
|
|
8956
|
-
error:
|
|
9071
|
+
error: error.message,
|
|
8957
9072
|
code: details.code,
|
|
8958
9073
|
detail:
|
|
8959
9074
|
details.retryOutcome === "exhausted"
|
|
@@ -9015,6 +9130,7 @@ export function agentRunFailurePayload(
|
|
|
9015
9130
|
"A required MCP server was temporarily unreachable. The same turn will retry after a short delay.",
|
|
9016
9131
|
code: "mcp_transport_unavailable",
|
|
9017
9132
|
retryable: true,
|
|
9133
|
+
detail: message,
|
|
9018
9134
|
};
|
|
9019
9135
|
}
|
|
9020
9136
|
if (
|
|
@@ -9232,9 +9348,10 @@ export function codexUsageLimitFailurePayload(
|
|
|
9232
9348
|
// goal re-evaluates at most this far out (it will re-pause if still capped).
|
|
9233
9349
|
const CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 60_000; // 1h
|
|
9234
9350
|
|
|
9235
|
-
function pendingToolCallFromSdkEvent(event: unknown): {
|
|
9351
|
+
export function pendingToolCallFromSdkEvent(event: unknown): {
|
|
9236
9352
|
callId: string;
|
|
9237
9353
|
callType: string;
|
|
9354
|
+
callName: string | null;
|
|
9238
9355
|
callItem: Record<string, unknown>;
|
|
9239
9356
|
} | null {
|
|
9240
9357
|
if (!event || typeof event !== "object") return null;
|
|
@@ -9255,10 +9372,16 @@ function pendingToolCallFromSdkEvent(event: unknown): {
|
|
|
9255
9372
|
if (typeof callId !== "string" || callId.length === 0 || typeof callType !== "string") {
|
|
9256
9373
|
return null;
|
|
9257
9374
|
}
|
|
9258
|
-
|
|
9375
|
+
const callItem = normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]');
|
|
9376
|
+
return {
|
|
9377
|
+
callId,
|
|
9378
|
+
callType,
|
|
9379
|
+
callName: typeof raw.name === "string" ? raw.name : null,
|
|
9380
|
+
callItem,
|
|
9381
|
+
};
|
|
9259
9382
|
}
|
|
9260
9383
|
|
|
9261
|
-
function completedToolCallFromSdkEvent(event: unknown): {
|
|
9384
|
+
export function completedToolCallFromSdkEvent(event: unknown): {
|
|
9262
9385
|
callId: string;
|
|
9263
9386
|
resultItem: Record<string, unknown>;
|
|
9264
9387
|
} | null {
|
|
@@ -9273,7 +9396,11 @@ function completedToolCallFromSdkEvent(event: unknown): {
|
|
|
9273
9396
|
? (item.rawItem as Record<string, unknown>)
|
|
9274
9397
|
: {};
|
|
9275
9398
|
const callId = raw.callId ?? raw.call_id ?? item.id;
|
|
9276
|
-
|
|
9399
|
+
if (typeof callId !== "string" || callId.length === 0) return null;
|
|
9400
|
+
return {
|
|
9401
|
+
callId,
|
|
9402
|
+
resultItem: normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]'),
|
|
9403
|
+
};
|
|
9277
9404
|
}
|
|
9278
9405
|
|
|
9279
9406
|
/**
|
|
@@ -9574,12 +9701,7 @@ export async function recordAuthoritativeModelCallFact(input: {
|
|
|
9574
9701
|
});
|
|
9575
9702
|
} catch (error) {
|
|
9576
9703
|
input.observability.warn("model call fact persist failed", {
|
|
9577
|
-
|
|
9578
|
-
workspaceId: input.workspaceId,
|
|
9579
|
-
sessionId: input.sessionId,
|
|
9580
|
-
turnId: input.turnId,
|
|
9581
|
-
sourceKey: input.sourceKey,
|
|
9582
|
-
error: error instanceof Error ? error.message : String(error),
|
|
9704
|
+
...safeErrorDiagnostic(error),
|
|
9583
9705
|
});
|
|
9584
9706
|
}
|
|
9585
9707
|
}
|