@opengeni/worker-bundle 0.16.17 → 0.16.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/dist/activities/agent-turn.d.ts +83 -32
  2. package/dist/activities/context-compaction.d.ts +2 -0
  3. package/dist/activities/environment.d.ts +1 -0
  4. package/dist/activities/expiring-material-renewal.d.ts +5 -3
  5. package/dist/activities/file-upload-reaper.d.ts +6 -0
  6. package/dist/activities/git-credential-renewal.d.ts +1 -1
  7. package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
  8. package/dist/activities/retained-screenshots.d.ts +93 -0
  9. package/dist/activities/run-credential-renewal.d.ts +1 -1
  10. package/dist/activities/run-input.d.ts +2 -0
  11. package/dist/activities/toolspace-token-renewal.d.ts +1 -1
  12. package/dist/activities/workspace-capture.d.ts +1 -0
  13. package/dist/activities.d.ts +3 -0
  14. package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
  15. package/dist/chunk-RAMNYSAW.js.map +1 -0
  16. package/dist/host-export.js +1 -1
  17. package/dist/index.js +1465 -453
  18. package/dist/index.js.map +1 -1
  19. package/dist/sandbox-resume.d.ts +6 -0
  20. package/dist/sandbox-routing.d.ts +11 -6
  21. package/dist/workflow-bundle.js +2 -1
  22. package/package.json +12 -12
  23. package/src/activities/agent-turn.ts +559 -434
  24. package/src/activities/codex-capacity.ts +10 -1
  25. package/src/activities/codex-rotation.ts +19 -5
  26. package/src/activities/context-compaction.ts +9 -5
  27. package/src/activities/environment.ts +13 -24
  28. package/src/activities/expiring-material-renewal.ts +16 -7
  29. package/src/activities/file-upload-reaper.ts +108 -7
  30. package/src/activities/git-credential-renewal.ts +3 -3
  31. package/src/activities/recording.ts +4 -5
  32. package/src/activities/retained-screenshot-reaper.ts +202 -0
  33. package/src/activities/retained-screenshots.ts +892 -0
  34. package/src/activities/rig-verification.ts +13 -34
  35. package/src/activities/run-credential-renewal.ts +9 -2
  36. package/src/activities/run-input.ts +18 -4
  37. package/src/activities/sandbox-lease.ts +5 -2
  38. package/src/activities/scheduled-tasks.ts +19 -3
  39. package/src/activities/toolspace-token-renewal.ts +9 -2
  40. package/src/activities/workspace-capture.ts +33 -3
  41. package/src/activities.ts +10 -2
  42. package/src/host-export-pump.ts +28 -11
  43. package/src/http.ts +3 -6
  44. package/src/observability-metrics.ts +3 -3
  45. package/src/sandbox-resume.ts +45 -12
  46. package/src/sandbox-routing.ts +35 -17
  47. package/src/worker-service-lifecycle.ts +8 -4
  48. package/src/workflows/file-upload-reaper.ts +4 -1
  49. package/dist/activities/redaction.d.ts +0 -1
  50. package/dist/chunk-BJUW5LCM.js.map +0 -1
  51. package/src/activities/redaction.ts +0 -9
@@ -27,6 +27,7 @@ import {
27
27
  getCodexRotationSettings,
28
28
  getWorkspaceModelPolicy,
29
29
  getWorkspaceGrant,
30
+ getWorkspace,
30
31
  listCodexAccountStatuses,
31
32
  fetchCodexUsageForAccount,
32
33
  getSessionCodexState,
@@ -89,9 +90,10 @@ import {
89
90
  sandboxStateEntryFromRunState,
90
91
  maxTurnsExceededRunState,
91
92
  modelResponseServiceTierFromSdkEvent,
92
- modelResponseUsageFromSdkEvent,
93
+ modelTerminalResponseFromSdkEvent,
93
94
  normalizeModelCallUsage,
94
95
  normalizeSdkEvent,
96
+ normalizeProtocolJsonValue,
95
97
  projectHistoryForProvider,
96
98
  sanitizeHistoryItemsForModel,
97
99
  projectModelInputForCapabilities,
@@ -126,6 +128,7 @@ import {
126
128
  runOwnedSandboxSetup,
127
129
  RoutingMutationOutcomeUnknownError,
128
130
  WorkspaceArchiveIntegrityError,
131
+ sdkBackendIdForSandboxBackend,
129
132
  swapTargetEstablishability,
130
133
  type SandboxFileDownload,
131
134
  type SandboxFileDownloadFailure,
@@ -136,7 +139,6 @@ import {
136
139
  type ModelCallUsageNormalization,
137
140
  type BuildAgentOptions,
138
141
  type ConnectorActionPolicyHooks,
139
- type RunAgentStreamOptions,
140
142
  type TurnToolCancellationFence,
141
143
  type BackendUnresolvableCode,
142
144
  type EstablishedSandboxSession,
@@ -262,19 +264,12 @@ import {
262
264
  import { withFirstPartyTools } from "./goals";
263
265
  import {
264
266
  mergeRigDefaultVariableSetEnvironment,
265
- resolveWorkspaceAgentInstructions,
266
267
  resolveWorkspacePackRuntime,
267
268
  resolveWorkspaceSkillLibraryRuntime,
268
269
  settingsWithPackSandboxImage,
269
270
  settingsWithRigImage,
270
271
  } from "./packs";
271
272
  import { deliverFailedChildTurnToParent } from "./parent-wake";
272
- import {
273
- createSecretRedactor,
274
- isCredentialHeaderName,
275
- redactSensitiveText,
276
- type SecretForRedaction,
277
- } from "./redaction";
278
273
  import {
279
274
  createModelHistoryAttachmentProjector,
280
275
  turnInput,
@@ -322,6 +317,8 @@ import {
322
317
  recordCreditMicros,
323
318
  recordModelCacheTokens,
324
319
  recordModelInputTokens,
320
+ recordSessionEventAppendLatency,
321
+ recordSessionEventPublishLatency,
325
322
  runtimeMetricsHooksForObservability,
326
323
  StreamTimingMetrics,
327
324
  turnLifecycleMetricsFor,
@@ -333,6 +330,16 @@ import {
333
330
  prepareRecordingForSettlement,
334
331
  type ActiveRecording,
335
332
  } from "./recording";
333
+ import {
334
+ collectRetainedScreenshotReceipts,
335
+ collectRetainedScreenshotRunStateReceipts,
336
+ compactRetainedScreenshotHistory,
337
+ compactRetainedScreenshotRunState,
338
+ materializeRetainedScreenshotHistory,
339
+ materializeRetainedScreenshotRunState,
340
+ retainComputerScreenshot,
341
+ typedScreenshotFromSdkEvent,
342
+ } from "./retained-screenshots";
336
343
  import { captureWorkspaceRevision, openFreshWorkspaceCaptureSession } from "./workspace-capture";
337
344
  import type { ChannelASession } from "@opengeni/runtime/sandbox";
338
345
  import { createObjectStorage, type ObjectStorage } from "@opengeni/storage";
@@ -346,9 +353,11 @@ import {
346
353
  DEFAULT_FIRST_PARTY_MCP_TOOLS,
347
354
  evaluateWorkspaceModelPolicy,
348
355
  readTurnExecutionPolicyV1,
356
+ resolveWorkspaceAgentHumanInputEnabled,
349
357
  resourceMountPath,
350
358
  type LatencyMode,
351
359
  type ResourceRef,
360
+ type RetainedArtifactMetadata,
352
361
  type SessionEvent,
353
362
  type SessionEventType,
354
363
  type SessionStatus,
@@ -363,6 +372,21 @@ import { createHash, randomUUID } from "node:crypto";
363
372
  export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
364
373
  const PROVIDER_CONNECTIVITY_BACKOFF_MS = [2_000, 5_000, 15_000, 30_000, 60_000] as const;
365
374
 
375
+ export class WorkspaceHumanInputDisabledError extends Error {
376
+ constructor(kind: "resume" | "interruption") {
377
+ super(`Workspace policy rejects structured human-input ${kind}`);
378
+ this.name = "WorkspaceHumanInputDisabledError";
379
+ }
380
+ }
381
+
382
+ export function assertWorkspaceHumanInputAllowed(
383
+ enabled: boolean,
384
+ kind: "resume" | "interruption",
385
+ attempted: boolean,
386
+ ): void {
387
+ if (!enabled && attempted) throw new WorkspaceHumanInputDisabledError(kind);
388
+ }
389
+
366
390
  /** Broad personal lookup is allowed only for a direct human/API command. */
367
391
  export function credentialSubjectIdForTurnInitiator(
368
392
  turn: Pick<SessionTurn, "source" | "initiator" | "initiatorContext">,
@@ -830,6 +854,33 @@ export async function waitForTurnStreamCleanup(
830
854
  await waitForTurnFinalizerStep(providerCompleted, signal);
831
855
  }
832
856
 
857
+ /**
858
+ * Terminal settlement closes the active attempt before turn finalization. The
859
+ * ordinary workspace admission therefore correctly returns `attempt_fenced`
860
+ * for the final attempt-qualified credential deletion. Retry only that exact
861
+ * settled-attempt case directly; every other fence/status and the direct
862
+ * deletion itself remain fail-closed.
863
+ */
864
+ export async function clearAttemptCredentialsWithSettledFence(input: {
865
+ activityStatus: RunAgentTurnResult["status"] | "unknown";
866
+ runWorkspaceFencedClear: () => Promise<void>;
867
+ clearExactAttempt: () => Promise<void>;
868
+ onSettledAttemptFence: () => void;
869
+ }): Promise<void> {
870
+ try {
871
+ await input.runWorkspaceFencedClear();
872
+ } catch (error) {
873
+ const settledAttemptFence =
874
+ error instanceof Error &&
875
+ error.name === "SandboxWorkspaceMutationFencedError" &&
876
+ (error as Error & { code?: unknown }).code === "attempt_fenced" &&
877
+ (input.activityStatus === "idle" || input.activityStatus === "failed");
878
+ if (!settledAttemptFence) throw error;
879
+ input.onSettledAttemptFence();
880
+ await input.clearExactAttempt();
881
+ }
882
+ }
883
+
833
884
  function turnFinalizerCancellationSignal(
834
885
  temporalSignal: AbortSignal | undefined,
835
886
  activityStatus: RunAgentTurnResult["status"] | "unknown",
@@ -846,84 +897,94 @@ function compactionFailureReason(reason: string): string {
846
897
  : `compaction summarization failed: ${reason}`;
847
898
  }
848
899
 
900
+ export type MandatoryHistoryPersistenceStage = "history_append" | "sandbox_envelope";
901
+
902
+ /**
903
+ * Preserve the exact storage failure for the permission-controlled turn event
904
+ * while attaching only a stable, non-secret operation token to public worker
905
+ * diagnostics. Mandatory history failures remain terminal and never inherit a
906
+ * provider-retry classification from coincidental source text or status.
907
+ */
908
+ export class MandatoryHistoryPersistenceError extends Error {
909
+ readonly name = "MandatoryHistoryPersistenceError";
910
+ readonly cause: unknown;
911
+
912
+ constructor(
913
+ readonly stage: MandatoryHistoryPersistenceStage,
914
+ cause: unknown,
915
+ ) {
916
+ super(cause instanceof Error ? cause.message : String(cause));
917
+ this.cause = cause;
918
+ }
919
+ }
920
+
921
+ export async function runMandatoryHistoryPersistenceStep<T>(
922
+ stage: MandatoryHistoryPersistenceStage,
923
+ operation: () => Promise<T>,
924
+ ): Promise<T> {
925
+ try {
926
+ return await operation();
927
+ } catch (cause) {
928
+ throw new MandatoryHistoryPersistenceError(stage, cause);
929
+ }
930
+ }
931
+
849
932
  export type SafeErrorDiagnostic = {
850
- name: string;
851
- message: string;
933
+ errorClass: "WorkerOperationError";
934
+ errorCode: "worker_operation_failed";
852
935
  status?: number;
853
- code?: string;
936
+ origin: "worker";
937
+ historyPersistenceStage?: MandatoryHistoryPersistenceStage;
854
938
  };
855
939
 
856
940
  /**
857
941
  * Produce the only exception shape allowed in worker logs. It deliberately
858
- * excludes stack, cause, response/request bodies, and arbitrary enumerable
859
- * properties while retaining bounded, redacted diagnostics.
942
+ * excludes the arbitrary source message, stack, cause, response/request
943
+ * bodies, and enumerable properties. Exact failure content belongs in the
944
+ * permission-controlled session event, not stdout or telemetry.
860
945
  */
861
- export function safeErrorDiagnostic(
862
- error: unknown,
863
- redactText: (value: string) => string = redactSensitiveText,
864
- ): SafeErrorDiagnostic {
865
- const rawName = error instanceof Error ? error.name : "Error";
866
- const name = /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error";
867
- const rawMessage = error instanceof Error ? error.message : String(error);
946
+ export function safeErrorDiagnostic(error: unknown): SafeErrorDiagnostic {
868
947
  const diagnostic: SafeErrorDiagnostic = {
869
- name,
870
- message: redactText(rawMessage).slice(0, 2_048),
948
+ errorClass: "WorkerOperationError",
949
+ errorCode: "worker_operation_failed",
950
+ origin: "worker",
871
951
  };
872
- if (error && typeof error === "object") {
873
- const status = Number(
874
- (error as { status?: unknown; statusCode?: unknown }).status ??
875
- (error as { statusCode?: unknown }).statusCode,
876
- );
877
- if (Number.isInteger(status) && status >= 100 && status <= 599) {
878
- diagnostic.status = status;
952
+ try {
953
+ let statusSource = error;
954
+ if (error instanceof MandatoryHistoryPersistenceError) {
955
+ diagnostic.historyPersistenceStage = error.stage;
956
+ statusSource = error.cause;
879
957
  }
880
- const rawCode = (error as { code?: unknown }).code;
881
- if (typeof rawCode === "string" || typeof rawCode === "number") {
882
- const code = redactText(String(rawCode)).slice(0, 80);
883
- if (/^[A-Za-z0-9_.:-]+$/.test(code)) diagnostic.code = code;
958
+ if (statusSource && typeof statusSource === "object") {
959
+ const status = Number(
960
+ (statusSource as { status?: unknown; statusCode?: unknown }).status ??
961
+ (statusSource as { statusCode?: unknown }).statusCode,
962
+ );
963
+ if (Number.isInteger(status) && status >= 100 && status <= 599) {
964
+ diagnostic.status = status;
965
+ }
884
966
  }
967
+ } catch {
968
+ // Public diagnostics are best-effort and must never replace the exact
969
+ // internal worker failure.
885
970
  }
886
971
  return diagnostic;
887
972
  }
888
973
 
889
- function safeErrorForTelemetry(error: unknown, redactText: (value: string) => string): Error {
890
- const diagnostic = safeErrorDiagnostic(error, redactText);
891
- const safe = new Error(diagnostic.message);
892
- safe.name = diagnostic.name;
974
+ function safeErrorForTelemetry(error: unknown): Error {
975
+ const diagnostic = safeErrorDiagnostic(error);
976
+ const safe = new Error("worker operation failed") as Error & {
977
+ code?: string;
978
+ status?: number;
979
+ origin?: string;
980
+ };
981
+ safe.name = "WorkerOperationError";
982
+ safe.code = diagnostic.errorCode;
983
+ if (diagnostic.status !== undefined) safe.status = diagnostic.status;
984
+ safe.origin = diagnostic.origin;
893
985
  return safe;
894
986
  }
895
987
 
896
- export function headerSecretRedactions(
897
- prefix: string,
898
- headers: Readonly<Record<string, string>> | undefined,
899
- credentialHeaderNames?: Iterable<string>,
900
- ): SecretForRedaction[] {
901
- const discovered: SecretForRedaction[] = [];
902
- const explicitCredentialHeaders = credentialHeaderNames
903
- ? new Set([...credentialHeaderNames].map((name) => name.toLowerCase()))
904
- : null;
905
- for (const [headerName, value] of Object.entries(headers ?? {})) {
906
- if (
907
- !explicitCredentialHeaders?.has(headerName.toLowerCase()) &&
908
- !isCredentialHeaderName(headerName)
909
- ) {
910
- continue;
911
- }
912
- const safeHeaderName = headerName.toUpperCase().replace(/[^A-Z0-9]+/g, "_");
913
- discovered.push({ name: `${prefix}_${safeHeaderName || "HEADER"}`, value });
914
- if (/^(?:proxy-)?authorization$/i.test(headerName)) {
915
- const credential = value.match(/^\s*[A-Za-z][A-Za-z0-9_-]*\s+(.+)\s*$/)?.[1];
916
- if (credential) {
917
- discovered.push({
918
- name: `${prefix}_${safeHeaderName || "AUTHORIZATION"}_CREDENTIAL`,
919
- value: credential,
920
- });
921
- }
922
- }
923
- }
924
- return discovered;
925
- }
926
-
927
988
  function compactionFailureReasonFromError(error: unknown): string {
928
989
  if (
929
990
  error instanceof CompactionProviderResponseError ||
@@ -1043,14 +1104,14 @@ function collectErrorStrings(value: unknown, seen = new WeakSet<object>()): stri
1043
1104
  * an earlier slice — the orphaned tool output that 400s the Responses API and
1044
1105
  * bricks the session on every replay.
1045
1106
  *
1046
- * Defending: sanitize the full current history into an API-valid sequence (the
1107
+ * Defending: structurally repair the full current history into an API-valid sequence (the
1047
1108
  * same pure rules the read path uses), then append only the new tail beyond the
1048
1109
  * watermark. A trailing dangling call is dropped here and re-evaluated next
1049
1110
  * pass once its result lands, so a call and its result are written together at
1050
1111
  * consecutive positions and a result is never persisted without its call. The
1051
- * watermark advances to the sanitized length — never past anything unwritten —
1112
+ * watermark advances to the repaired length — never past anything unwritten —
1052
1113
  * so a non-monotonic history can never desync it. When previously-persisted
1053
- * rows already exceed the sanitized length (e.g. legacy orphans written before
1114
+ * rows already exceed the repaired length (e.g. legacy orphans written before
1054
1115
  * this fix), nothing new is appended and the watermark holds steady.
1055
1116
  */
1056
1117
  /**
@@ -1123,10 +1184,9 @@ type TurnEventPublisher = (
1123
1184
  immediate?: boolean,
1124
1185
  ) => Promise<{ events: SessionEvent[]; accepted: boolean }>;
1125
1186
 
1126
- export type ModelResponseUsageEventState = {
1127
- responseUsageCount: number;
1128
- providerContextRevision: number;
1129
- lastProviderContextTokensObserved: number | null;
1187
+ export type ModelResponseEventState = {
1188
+ responseCount: number;
1189
+ contextSignal: { revision: number; totalTokens: number } | null;
1130
1190
  claimedSourceKeys: Set<string>;
1131
1191
  };
1132
1192
 
@@ -1135,13 +1195,12 @@ export type CompactionModelUsageEventState = {
1135
1195
  claimedSourceKeys: Set<string>;
1136
1196
  };
1137
1197
 
1138
- export function createModelResponseUsageEventState(
1198
+ export function createModelResponseEventState(
1139
1199
  claimedSourceKeys: Set<string> = new Set<string>(),
1140
- ): ModelResponseUsageEventState {
1200
+ ): ModelResponseEventState {
1141
1201
  return {
1142
- responseUsageCount: 0,
1143
- providerContextRevision: 0,
1144
- lastProviderContextTokensObserved: null,
1202
+ responseCount: 0,
1203
+ contextSignal: null,
1145
1204
  claimedSourceKeys,
1146
1205
  };
1147
1206
  }
@@ -1152,15 +1211,10 @@ export function createCompactionModelUsageEventState(
1152
1211
  return { usageCount: 0, claimedSourceKeys };
1153
1212
  }
1154
1213
 
1155
- export function modelResponseUsageContextSignal(
1156
- state: ModelResponseUsageEventState,
1214
+ export function modelResponseContextSignal(
1215
+ state: ModelResponseEventState,
1157
1216
  ): { revision: number; totalTokens: number } | null {
1158
- return state.lastProviderContextTokensObserved === null
1159
- ? null
1160
- : {
1161
- revision: state.providerContextRevision,
1162
- totalTokens: state.lastProviderContextTokensObserved,
1163
- };
1217
+ return state.contextSignal;
1164
1218
  }
1165
1219
 
1166
1220
  export function assertModelResponseLatencyMode(input: {
@@ -1195,19 +1249,20 @@ export function assertModelResponseLatencyMode(input: {
1195
1249
  }
1196
1250
 
1197
1251
  /**
1198
- * Process one SDK stream event through the exact production usage path.
1252
+ * Process one SDK terminal-response event through the production authority path.
1199
1253
  *
1200
1254
  * The pinned Responses SDK mirrors one provider terminal response as both a
1201
1255
  * normalized `response_done` and a raw `model/response.completed` event. Claim
1202
- * the stable response/source key before lease renewal or any usage side effect,
1203
- * and advance the positional ordinal only for a newly claimed response. The
1204
- * durable `agent.model.usage` source-key fence remains the cross-restart
1205
- * authority: a replay may retry the idempotent billing write, but it cannot
1206
- * advance process-local metrics, provider context, or attempt-owned signals.
1256
+ * the stable response/source key before lease renewal or any side effect, and
1257
+ * use that one positional ordinal for both response identity and same-run
1258
+ * context binding. A response without usage still clears attempt-owned token
1259
+ * state. When usage exists, the durable `agent.model.usage` source-key fence
1260
+ * remains the cross-restart authority: a replay may retry the idempotent billing
1261
+ * write, but it cannot advance metrics, context, or attempt-owned signals.
1207
1262
  */
1208
- export async function processModelResponseUsageEvent(input: {
1209
- event: Parameters<typeof modelResponseUsageFromSdkEvent>[0];
1210
- state: ModelResponseUsageEventState;
1263
+ export async function processModelResponseTerminalEvent(input: {
1264
+ event: Parameters<typeof modelTerminalResponseFromSdkEvent>[0];
1265
+ state: ModelResponseEventState;
1211
1266
  dispatchId: string | null;
1212
1267
  settings: Settings;
1213
1268
  db: ActivityServices["db"];
@@ -1230,20 +1285,25 @@ export async function processModelResponseUsageEvent(input: {
1230
1285
  renewLease: () => Promise<void>;
1231
1286
  leaseLost: () => boolean;
1232
1287
  leaseLostMessage: string;
1233
- setLastInputTokens: (tokens: number) => Promise<void>;
1288
+ setLastInputTokens: (tokens: number | null) => Promise<void>;
1234
1289
  }): Promise<
1235
- | { status: "not_usage" }
1290
+ | { status: "not_response" }
1236
1291
  | { status: "duplicate"; sourceKey: string }
1237
- | { status: "processed"; sourceKey: string; authoritative: boolean }
1292
+ | {
1293
+ status: "processed";
1294
+ sourceKey: string;
1295
+ authoritative: boolean;
1296
+ usageReported: boolean;
1297
+ }
1238
1298
  > {
1239
- const responseUsage = modelResponseUsageFromSdkEvent(input.event);
1240
- if (!responseUsage) {
1241
- return { status: "not_usage" };
1299
+ const terminal = modelTerminalResponseFromSdkEvent(input.event);
1300
+ if (!terminal) {
1301
+ return { status: "not_response" };
1242
1302
  }
1243
1303
 
1244
- const responseOrdinal = input.state.responseUsageCount + 1;
1304
+ const responseOrdinal = input.state.responseCount + 1;
1245
1305
  const sourceKey = modelUsageSourceKey({
1246
- responseId: responseUsage.responseId,
1306
+ responseId: terminal.responseId,
1247
1307
  dispatchId: input.dispatchId,
1248
1308
  positionalKey: `response-${responseOrdinal}`,
1249
1309
  });
@@ -1251,20 +1311,25 @@ export async function processModelResponseUsageEvent(input: {
1251
1311
  return { status: "duplicate", sourceKey };
1252
1312
  }
1253
1313
  input.state.claimedSourceKeys.add(sourceKey);
1254
- input.state.responseUsageCount = responseOrdinal;
1314
+ input.state.responseCount = responseOrdinal;
1315
+
1316
+ const responseUsage = terminal.usage;
1255
1317
 
1256
- const normalizedUsage = normalizeModelCallUsage(responseUsage.usage);
1318
+ const normalizedUsage = normalizeModelCallUsage(responseUsage?.usage);
1257
1319
  const accountContext = modelCallAccountContext({
1258
1320
  servingCredentialId: input.servingCredentialId,
1259
1321
  priorSessionCredentialId: input.priorSessionCredentialId,
1260
1322
  isFirstCallOfTurn: responseOrdinal === 1,
1261
1323
  });
1262
- let authoritative = false;
1324
+ // A terminal response without usage still authoritatively replaces the
1325
+ // attempt-owned context signal. There is simply no billing event to claim.
1326
+ let authoritative = responseUsage === null;
1263
1327
  await recordCompletedModelCallBeforeOwnershipFences({
1264
1328
  renewLease: input.renewLease,
1265
1329
  leaseLost: input.leaseLost,
1266
1330
  leaseLostMessage: input.leaseLostMessage,
1267
1331
  recordUsage: async () => {
1332
+ if (!responseUsage) return;
1268
1333
  const billing = await recordModelUsageAndDebitCredits(input.settings, input.db, {
1269
1334
  accountId: input.accountId,
1270
1335
  workspaceId: input.workspaceId,
@@ -1322,17 +1387,22 @@ export async function processModelResponseUsageEvent(input: {
1322
1387
  recordAttemptSignals: async () => {
1323
1388
  if (!authoritative) return;
1324
1389
  const observedTotal = normalizedUsage.totalTokens;
1325
- if (observedTotal !== null) {
1326
- input.state.lastProviderContextTokensObserved = observedTotal;
1327
- input.state.providerContextRevision += 1;
1328
- }
1390
+ input.state.contextSignal =
1391
+ observedTotal !== null && observedTotal > 0
1392
+ ? { revision: responseOrdinal, totalTokens: observedTotal }
1393
+ : null;
1329
1394
  const observedInput = normalizedUsage.telemetry.inputTokens;
1330
- if (observedInput !== null && observedInput > 0) {
1331
- await input.setLastInputTokens(observedInput);
1332
- }
1395
+ await input.setLastInputTokens(
1396
+ observedInput !== null && observedInput > 0 ? observedInput : null,
1397
+ );
1333
1398
  },
1334
1399
  });
1335
- return { status: "processed", sourceKey, authoritative };
1400
+ return {
1401
+ status: "processed",
1402
+ sourceKey,
1403
+ authoritative,
1404
+ usageReported: responseUsage !== null,
1405
+ };
1336
1406
  }
1337
1407
 
1338
1408
  /**
@@ -1508,14 +1578,9 @@ export async function emitModelCallUsage(input: {
1508
1578
  if (!authoritative) return false;
1509
1579
  try {
1510
1580
  input.observability.info("model call usage", {
1511
- accountId: input.accountId,
1512
- workspaceId: input.workspaceId,
1513
- sessionId: input.sessionId,
1514
- turnId: input.turnId,
1515
1581
  provider: input.provider,
1516
1582
  providerApi: input.providerApi,
1517
1583
  model: input.model,
1518
- sourceKey: input.sourceKey,
1519
1584
  inputTokens: telemetry.inputTokens,
1520
1585
  outputTokens: telemetry.outputTokens,
1521
1586
  cachedTokens: telemetry.cachedTokens,
@@ -1533,7 +1598,6 @@ export async function emitModelCallUsage(input: {
1533
1598
  provider: input.provider,
1534
1599
  providerApi: input.providerApi,
1535
1600
  model: input.model,
1536
- sourceKey: input.sourceKey,
1537
1601
  rejectedFields: normalizedUsage.rejectedFields.join(","),
1538
1602
  });
1539
1603
  }
@@ -1580,16 +1644,13 @@ export function historyRowsToAppend(
1580
1644
  // positions from 0) when callers do not pass an explicit next position.
1581
1645
  nextPosition: number = persistedHistoryCount,
1582
1646
  toolOutputTruncationTokens?: number,
1583
- redactValue: (value: unknown) => unknown = (value) => value,
1584
1647
  ): {
1585
1648
  rows: Array<{ position: number; item: Record<string, unknown> }>;
1586
1649
  nextWatermark: number;
1587
1650
  nextPosition: number;
1588
1651
  } {
1589
- const sanitized = sanitizeHistoryItemsForModel(rawHistory, toolOutputTruncationTokens).map(
1590
- (item) => redactValue(item) as Record<string, unknown>,
1591
- );
1592
- if (sanitized.length <= persistedHistoryCount) {
1652
+ const modelReady = sanitizeHistoryItemsForModel(rawHistory, toolOutputTruncationTokens);
1653
+ if (modelReady.length <= persistedHistoryCount) {
1593
1654
  return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
1594
1655
  }
1595
1656
  // Canonical model-facing inputs (including machine-input batches) are
@@ -1599,16 +1660,22 @@ export function historyRowsToAppend(
1599
1660
  // notes—must not accidentally become conversation memory during reconciliation.
1600
1661
  // Advance the in-memory watermark past them, but only allocate durable
1601
1662
  // positions to actual model/tool output.
1602
- const rows = sanitized
1603
- .slice(persistedHistoryCount)
1604
- .filter((item) => !(item.type === "message" && item.role === "system"))
1605
- .map((item, offset) => ({
1606
- position: nextPosition + offset,
1607
- item: item as Record<string, unknown>,
1608
- }));
1663
+ const rows: Array<{ position: number; item: Record<string, unknown> }> = [];
1664
+ for (const [offset, item] of modelReady.slice(persistedHistoryCount).entries()) {
1665
+ if (item.type === "message" && item.role === "system") {
1666
+ continue;
1667
+ }
1668
+ rows.push({
1669
+ position: nextPosition + rows.length,
1670
+ item: normalizeProtocolJsonValue(
1671
+ item as Record<string, unknown>,
1672
+ `$[${persistedHistoryCount + offset}]`,
1673
+ ),
1674
+ });
1675
+ }
1609
1676
  return {
1610
1677
  rows,
1611
- nextWatermark: sanitized.length,
1678
+ nextWatermark: modelReady.length,
1612
1679
  nextPosition: nextPosition + rows.length,
1613
1680
  };
1614
1681
  }
@@ -1650,16 +1717,7 @@ export function selectRejectedProviderArtifactHistoryIds(
1650
1717
  }
1651
1718
 
1652
1719
  /** Literal final per-call boundary for replayed and current-turn model input. */
1653
- export function secretRedactionModelInputFilter(
1654
- redactValue: (value: unknown) => unknown,
1655
- ): NonNullable<RunAgentStreamOptions["callModelInputFilter"]> {
1656
- return ({ modelData }) => ({
1657
- ...modelData,
1658
- input: redactValue(modelData.input) as typeof modelData.input,
1659
- });
1660
- }
1661
-
1662
- function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
1720
+ export function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
1663
1721
  if (item.type === "message") {
1664
1722
  return item.role === "assistant";
1665
1723
  }
@@ -2272,10 +2330,6 @@ export function createTurnSandboxProvisioner<T>(
2272
2330
  };
2273
2331
  }
2274
2332
 
2275
- function sdkBackendIdForSandboxBackend(backend: Settings["sandboxBackend"]): string {
2276
- return backend === "local" ? "unix_local" : backend;
2277
- }
2278
-
2279
2333
  /**
2280
2334
  * Decide whether THIS turn may send OpenAI's `prompt_cache_key` request field.
2281
2335
  *
@@ -2365,6 +2419,34 @@ export function codexCredentialLeaseDeadlineExpired(
2365
2419
  );
2366
2420
  }
2367
2421
 
2422
+ type OpStreamFinalizer = {
2423
+ finalizeOpStreamOps(): Promise<void>;
2424
+ };
2425
+
2426
+ /** Release retained Connected Machine output only after the turn is durable. */
2427
+ export async function finalizeDurableTurnOpStreams(
2428
+ sessions: readonly unknown[],
2429
+ fallback: OpStreamFinalizer | null,
2430
+ ): Promise<void> {
2431
+ const candidates = new Set<OpStreamFinalizer>();
2432
+ for (const session of sessions) {
2433
+ const candidate = session as Partial<OpStreamFinalizer> | null;
2434
+ if (typeof candidate?.finalizeOpStreamOps === "function") {
2435
+ candidates.add(candidate as OpStreamFinalizer);
2436
+ }
2437
+ }
2438
+ if (candidates.size === 0 && fallback) {
2439
+ candidates.add(fallback);
2440
+ }
2441
+ for (const candidate of candidates) {
2442
+ try {
2443
+ await candidate.finalizeOpStreamOps();
2444
+ } catch {
2445
+ // The runner's retention TTL owns the fallback.
2446
+ }
2447
+ }
2448
+ }
2449
+
2368
2450
  export function createRunAgentTurnActivity(services: () => Promise<ActivityServices>) {
2369
2451
  return async function runAgentTurn(input: RunAgentTurnInput): Promise<RunAgentTurnResult> {
2370
2452
  const {
@@ -2423,6 +2505,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2423
2505
  acknowledgeQuiescence = true;
2424
2506
  noteCancellationRequested();
2425
2507
  };
2508
+ const acknowledgeRecoveryQuiescence = (): void => {
2509
+ // requestSessionTurnRecovery closed this exact attempt and durably
2510
+ // recorded the replacement cause. The activity still owns the physical
2511
+ // tool/credential boundary until finally drains it and publishes the
2512
+ // exact quiescence receipt; a workflow result alone is never that proof.
2513
+ acknowledgeQuiescence = true;
2514
+ noteCancellationRequested();
2515
+ };
2426
2516
  let turnId: string | undefined;
2427
2517
  let triggerEventId: string | undefined;
2428
2518
  const claimedResult = (
@@ -2454,7 +2544,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2454
2544
  // Still required by credential-loss/capacity settlements, whose own
2455
2545
  // recovery transactions fence against worker-death redispatches.
2456
2546
  let redispatchesAtDispatch = 0;
2457
- const setLastInputTokensFenced = async (lastInputTokens: number): Promise<void> => {
2547
+ const setLastInputTokensFenced = async (lastInputTokens: number | null): Promise<void> => {
2458
2548
  if (!turnId || executionGeneration <= 0) {
2459
2549
  throw new Error("Turn attempt was not initialized before token accounting");
2460
2550
  }
@@ -2556,7 +2646,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2556
2646
  workspaceId: input.workspaceId,
2557
2647
  turnId,
2558
2648
  reason,
2559
- errorName: error instanceof Error ? error.name : "unknown",
2649
+ ...safeErrorDiagnostic(error),
2560
2650
  });
2561
2651
  observability.incrementCounter({
2562
2652
  name: "opengeni_codex_lease_renewals_total",
@@ -2588,8 +2678,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2588
2678
  return resolvedSandbox;
2589
2679
  };
2590
2680
  // The machine-primary SelfhostedSession (the UNWRAPPED backend, not the
2591
- // routing proxy): held so the turn's completion can final-ack this turn's
2592
- // settled op-stream ops AFTER the results are durably persisted.
2681
+ // routing proxy). Kept as a fallback finalizer; the routing proxy normally
2682
+ // aggregates it together with every machine reached after a mid-turn swap.
2593
2683
  let machinePrimarySession: import("@opengeni/runtime").SelfhostedSession | null = null;
2594
2684
  let lazyOwnedSandbox: EstablishedSandboxSession | null = null;
2595
2685
  let turnSandboxProvisioner: TurnSandboxProvisioner<ResumedTurnSandbox> | null = null;
@@ -2598,6 +2688,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2598
2688
  // THIS handle so a mid-turn sandbox_swap can never re-route those execs onto a
2599
2689
  // connected machine (the user's real computer).
2600
2690
  let setupBoxSession: unknown = null;
2691
+ const finalizeTurnOpStreamOps = async (): Promise<void> => {
2692
+ await finalizeDurableTurnOpStreams(
2693
+ [lazyOwnedSandbox?.session, resolvedSandbox?.established.session],
2694
+ machinePrimarySession,
2695
+ );
2696
+ };
2601
2697
  // A same-target API repair can replace the home provider while this turn is
2602
2698
  // alive. Keep setup/snapshot persistence on the rebound raw session while
2603
2699
  // preserving the SDK-owned routing proxy for eager turns. Lazy turns hold the
@@ -2945,7 +3041,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2945
3041
  sessionId: input.sessionId,
2946
3042
  sandboxGroupId: heartbeatGroupId,
2947
3043
  leaseEpoch: heartbeatEpoch,
2948
- error: error instanceof Error ? error.message : String(error),
3044
+ ...safeErrorDiagnostic(error),
2949
3045
  });
2950
3046
  })
2951
3047
  .finally(() => {
@@ -3098,6 +3194,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3098
3194
  // stable and this scalar append watermark is valid again. The sanitizer
3099
3195
  // remains the final call/result pairing guard for every other reconcile.
3100
3196
  let persistedHistoryCount = 0;
3197
+ const retainedScreenshotReceiptsByCallId = new Map<string, RetainedArtifactMetadata>();
3198
+ const computerScreenshotCallIds = new Set<string>();
3199
+ const materializeScreenshotHistory = async (history: Array<Record<string, unknown>>) => {
3200
+ collectRetainedScreenshotReceipts(history, retainedScreenshotReceiptsByCallId);
3201
+ return await materializeRetainedScreenshotHistory({
3202
+ db,
3203
+ objectStorage,
3204
+ workspaceId: input.workspaceId,
3205
+ sessionId: input.sessionId,
3206
+ history,
3207
+ });
3208
+ };
3209
+ const materializeScreenshotRunState = async (serialized: string) => {
3210
+ collectRetainedScreenshotRunStateReceipts(serialized, retainedScreenshotReceiptsByCallId);
3211
+ return await materializeRetainedScreenshotRunState({
3212
+ db,
3213
+ objectStorage,
3214
+ workspaceId: input.workspaceId,
3215
+ sessionId: input.sessionId,
3216
+ serialized,
3217
+ });
3218
+ };
3101
3219
  let providerArtifactCandidates: Awaited<
3102
3220
  ReturnType<typeof turnInput>
3103
3221
  >["providerArtifactCandidates"] = {
@@ -3115,15 +3233,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3115
3233
  if (!stream || !turnId) {
3116
3234
  return;
3117
3235
  }
3236
+ const durableTurnId = turnId;
3118
3237
  try {
3119
3238
  const rawHistory = (stream.state as { history?: unknown[] }).history;
3120
3239
  if (Array.isArray(rawHistory)) {
3121
- const { rows, nextWatermark, nextPosition } = historyRowsToAppend(
3240
+ const durableHistory = compactRetainedScreenshotHistory(
3122
3241
  rawHistory as Array<Record<string, unknown>>,
3242
+ retainedScreenshotReceiptsByCallId,
3243
+ );
3244
+ const { rows, nextWatermark, nextPosition } = historyRowsToAppend(
3245
+ durableHistory,
3123
3246
  persistedHistoryCount,
3124
3247
  nextHistoryPosition,
3125
3248
  modelRunSettings.modelToolOutputTruncationTokens,
3126
- (value) => redact(value),
3127
3249
  );
3128
3250
  const hasModelOrToolProgress = rows.some((row) =>
3129
3251
  isModelOrToolProgressHistoryItem(row.item),
@@ -3131,21 +3253,23 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3131
3253
  const shouldAppendRows =
3132
3254
  rows.length > 0 && (!options.skipInputOnlyRows || hasModelOrToolProgress);
3133
3255
  if (shouldAppendRows) {
3134
- const appended = await appendSessionHistoryItems(db, {
3135
- accountId: input.accountId,
3136
- workspaceId: input.workspaceId,
3137
- sessionId: input.sessionId,
3138
- turnId,
3139
- expectedExecutionGeneration: executionGeneration,
3140
- expectedAttemptId: input.attemptId,
3141
- modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
3142
- items: rows,
3256
+ await runMandatoryHistoryPersistenceStep("history_append", async () => {
3257
+ const appended = await appendSessionHistoryItems(db, {
3258
+ accountId: input.accountId,
3259
+ workspaceId: input.workspaceId,
3260
+ sessionId: input.sessionId,
3261
+ turnId: durableTurnId,
3262
+ expectedExecutionGeneration: executionGeneration,
3263
+ expectedAttemptId: input.attemptId,
3264
+ modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
3265
+ items: rows,
3266
+ });
3267
+ if (!appended) {
3268
+ throw new TurnAttemptFencedError(
3269
+ "turn execution generation was fenced while saving conversation history",
3270
+ );
3271
+ }
3143
3272
  });
3144
- if (!appended) {
3145
- throw new TurnAttemptFencedError(
3146
- "turn execution generation was fenced while saving conversation history",
3147
- );
3148
- }
3149
3273
  }
3150
3274
  if (shouldAppendRows || !options.skipInputOnlyRows) {
3151
3275
  persistedHistoryCount = nextWatermark;
@@ -3154,37 +3278,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3154
3278
  }
3155
3279
  const envelope = sandboxStateEntryFromRunState(stream.state);
3156
3280
  if (envelope) {
3157
- await upsertSandboxSessionEnvelope(db, {
3158
- accountId: input.accountId,
3159
- workspaceId: input.workspaceId,
3160
- sessionId: input.sessionId,
3161
- envelope,
3162
- });
3281
+ await runMandatoryHistoryPersistenceStep("sandbox_envelope", () =>
3282
+ upsertSandboxSessionEnvelope(db, {
3283
+ accountId: input.accountId,
3284
+ workspaceId: input.workspaceId,
3285
+ sessionId: input.sessionId,
3286
+ envelope,
3287
+ }),
3288
+ );
3163
3289
  }
3164
3290
  } catch (persistError) {
3165
3291
  console.error(
3166
3292
  "session history dual-write failed (run unaffected)",
3167
- safeErrorDiagnostic(persistError, (value) => String(redact(value))),
3293
+ safeErrorDiagnostic(persistError),
3168
3294
  );
3169
3295
  if (options.requireDurable) throw persistError;
3170
3296
  }
3171
3297
  };
3172
- // Reassigned after the variable set loads; the publish closure is
3173
- // created (and used for turn.started) before the variableSet is available.
3174
- // Generic auth/cookie/signed-URL/token classification is active before any
3175
- // exact run-secret provenance is available. Registered values strengthen
3176
- // this boundary as credentials are resolved and renewed during the turn.
3177
- let redact: (payload: unknown) => unknown = createSecretRedactor([]);
3178
- const secretRedactions = new Map<string, string>();
3179
3298
  const publishedRunCredentialNotices = new Set<string>();
3180
- const registerSecretRedactions = (secrets: SecretForRedaction[]): void => {
3181
- for (const secret of secrets) {
3182
- if (!secretRedactions.has(secret.value)) secretRedactions.set(secret.value, secret.name);
3183
- }
3184
- redact = createSecretRedactor(
3185
- [...secretRedactions].map(([value, name]) => ({ name, value })),
3186
- );
3187
- };
3299
+
3188
3300
  let variableSetId = "";
3189
3301
  // Rig telemetry (M3): set once the session loads; empty string for a rig-less
3190
3302
  // turn (mirrors variableSetId). Read by the activity span's finally block.
@@ -3371,7 +3483,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3371
3483
  ) => {
3372
3484
  const inputs = events.map((event) => ({
3373
3485
  ...event,
3374
- payload: redact(event.payload),
3486
+ payload: event.payload,
3375
3487
  turnId: turnId!,
3376
3488
  producerId,
3377
3489
  producerSeq: ++producerSeq,
@@ -3385,6 +3497,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3385
3497
  executionGeneration,
3386
3498
  input.attemptId,
3387
3499
  inputs,
3500
+ {
3501
+ onAppend: ({ durationSeconds }) =>
3502
+ recordSessionEventAppendLatency(observability, { durationSeconds }),
3503
+ onPublish: ({ durationSeconds }) =>
3504
+ recordSessionEventPublishLatency(observability, { durationSeconds }),
3505
+ },
3388
3506
  );
3389
3507
  if (inputs.length > 0 && !appended.accepted) {
3390
3508
  throw new TurnAttemptFencedError("turn execution generation was fenced");
@@ -3439,13 +3557,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3439
3557
  : undefined;
3440
3558
  const inputs = inputSettlement.events.map((event) => ({
3441
3559
  ...event,
3442
- payload: redact(event.payload),
3560
+ payload: event.payload,
3443
3561
  turnId: turnId!,
3444
3562
  producerId,
3445
3563
  producerSeq: ++producerSeq,
3446
3564
  }));
3447
- const redactedRunState = inputSettlement.runState
3448
- ? (redact(inputSettlement.runState) as typeof inputSettlement.runState)
3565
+ const runState = inputSettlement.runState
3566
+ ? {
3567
+ ...inputSettlement.runState,
3568
+ serializedRunState: compactRetainedScreenshotRunState(
3569
+ inputSettlement.runState.serializedRunState,
3570
+ retainedScreenshotReceiptsByCallId,
3571
+ ),
3572
+ }
3449
3573
  : undefined;
3450
3574
  const result = await applySessionTurnSettlement(db, input.workspaceId, {
3451
3575
  sessionId: input.sessionId,
@@ -3456,7 +3580,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3456
3580
  sessionStatus: inputSettlement.sessionStatus,
3457
3581
  activeTurnId: inputSettlement.activeTurnId,
3458
3582
  events: inputs,
3459
- ...(redactedRunState ? { runState: redactedRunState } : {}),
3583
+ ...(runState ? { runState } : {}),
3460
3584
  ...(recordingMutation ? { recording: recordingMutation } : {}),
3461
3585
  ...(compactionRequestFailure ? { compactionRequestFailure } : {}),
3462
3586
  });
@@ -3814,10 +3938,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3814
3938
  labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
3815
3939
  });
3816
3940
  observability.warn("Codex adaptive fleet shadow decision failed open", {
3817
- workspaceId: input.workspaceId,
3818
3941
  stage: shadowResult.stage,
3819
3942
  reason: shadowResult.reason,
3820
- errorName: shadowResult.errorName,
3943
+ errorClass: "CodexFleetShadowOperationError",
3944
+ errorCode: "codex_fleet_shadow_failed",
3945
+ origin: "worker",
3821
3946
  payloadBytes: shadowResult.payloadBytes,
3822
3947
  });
3823
3948
  }
@@ -4167,15 +4292,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4167
4292
  attemptId: input.attemptId,
4168
4293
  executionGeneration: turn.executionGeneration,
4169
4294
  };
4170
- const [workspaceAgentInstructions, instructionPolicySnapshot, preferenceSnapshot] =
4171
- await Promise.all([
4172
- resolveWorkspaceAgentInstructions(db, input.workspaceId),
4173
- getOrCreateWorkspaceInstructionPolicySnapshot(db, governanceClaims),
4174
- getOrCreatePreferenceRegistrySnapshot(db, governanceClaims).catch((error) => {
4175
- if (error instanceof PreferenceRegistryInitiatorError) return null;
4176
- throw error;
4177
- }),
4178
- ]);
4295
+ const [workspace, instructionPolicySnapshot, preferenceSnapshot] = await Promise.all([
4296
+ getWorkspace(db, input.workspaceId),
4297
+ getOrCreateWorkspaceInstructionPolicySnapshot(db, governanceClaims),
4298
+ getOrCreatePreferenceRegistrySnapshot(db, governanceClaims).catch((error) => {
4299
+ if (error instanceof PreferenceRegistryInitiatorError) return null;
4300
+ throw error;
4301
+ }),
4302
+ ]);
4303
+ if (!workspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
4304
+ const workspaceAgentInstructions = workspace.agentInstructions;
4305
+ const agentHumanInputEnabled = resolveWorkspaceAgentHumanInputEnabled(workspace.settings);
4306
+ assertWorkspaceHumanInputAllowed(agentHumanInputEnabled, "resume", humanInputResume !== null);
4179
4307
  const workspaceGovernance = renderWorkspaceGovernanceContext({
4180
4308
  instructionPolicy: instructionPolicySnapshot,
4181
4309
  preferences: preferenceSnapshot,
@@ -4200,33 +4328,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4200
4328
  openaiReasoningEffort: turn.reasoningEffort,
4201
4329
  sandboxBackend: turn.sandboxBackend,
4202
4330
  };
4203
- const sessionMcpHeaderNames = new Map<string, readonly string[]>();
4204
4331
  const runSettings = await settingsWithSessionMcpServersForRun(
4205
4332
  db,
4206
4333
  input.workspaceId,
4207
4334
  input.sessionId,
4208
4335
  input.attemptId,
4209
4336
  baseRunSettings,
4210
- {
4211
- onResolvedServers: (servers) => {
4212
- for (const server of servers) {
4213
- sessionMcpHeaderNames.set(server.id, server.headerNames);
4214
- }
4215
- },
4216
- },
4217
4337
  );
4218
- registerSecretRedactions([
4219
- ...(runSettings.accessKey
4220
- ? [{ name: "OPENGENI_ACCESS_KEY", value: runSettings.accessKey }]
4221
- : []),
4222
- ...runSettings.mcpServers.flatMap((server) => {
4223
- return headerSecretRedactions(
4224
- `MCP_${server.id.toUpperCase()}_STATIC`,
4225
- server.headers,
4226
- sessionMcpHeaderNames.get(server.id),
4227
- );
4228
- }),
4229
- ]);
4338
+
4230
4339
  // Multi-provider per-turn routing → the provider gating (compaction mode,
4231
4340
  // hosted web search, encrypted reasoning, context window) the agent and
4232
4341
  // compaction summarizer must use; null falls back to the legacy global
@@ -4257,49 +4366,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4257
4366
  projectModelInputForCapabilities(await attachmentProjector(items), modelInputPolicy);
4258
4367
  const compactionModelHistoryProjector = async (items: Array<Record<string, unknown>>) =>
4259
4368
  await modelHistoryProjector(projectHistoryForProvider(items, providerApi));
4260
- const selectedProvider = resolvedModel?.provider;
4261
- const publicProviderHeaders = new Set(
4262
- selectedProvider?.publicDefaultHeaderNames?.map((name) => name.toLowerCase()) ?? [],
4263
- );
4264
- const publicProviderQueries = new Set(
4265
- selectedProvider?.publicDefaultQueryNames?.map((name) => name.toLowerCase()) ?? [],
4266
- );
4267
- const fallbackProviderApiKey =
4268
- capabilitySettings.openaiProvider === "azure"
4269
- ? (capabilitySettings.azureOpenaiApiKey ?? capabilitySettings.azureOpenaiAdToken)
4270
- : capabilitySettings.openaiApiKey;
4271
- registerSecretRedactions([
4272
- ...(selectedProvider?.apiKey
4273
- ? [
4274
- {
4275
- name: `MODEL_${selectedProvider.id.toUpperCase()}_API_KEY`,
4276
- value: selectedProvider.apiKey,
4277
- },
4278
- ]
4279
- : !resolvedModel && fallbackProviderApiKey
4280
- ? [
4281
- {
4282
- name: "MODEL_PROVIDER_API_KEY",
4283
- value: fallbackProviderApiKey,
4284
- },
4285
- ]
4286
- : []),
4287
- ...Object.entries(selectedProvider?.defaultHeaders ?? {}).flatMap(([name, value]) =>
4288
- publicProviderHeaders.has(name.toLowerCase())
4289
- ? []
4290
- : [
4291
- {
4292
- name: `MODEL_PROVIDER_HEADER_${name.toUpperCase()}`,
4293
- value,
4294
- },
4295
- ],
4296
- ),
4297
- ...Object.entries(selectedProvider?.defaultQuery ?? {}).flatMap(([name, value]) =>
4298
- publicProviderQueries.has(name.toLowerCase())
4299
- ? []
4300
- : [{ name: `MODEL_PROVIDER_QUERY_${name.toUpperCase()}`, value }],
4301
- ),
4302
- ]);
4303
4369
  // Bind the provider/model catalog's context policy to every model-facing
4304
4370
  // path for this turn. In particular, Codex subscription turns must not
4305
4371
  // inherit the deployment's OpenAI/Azure mode or 1.05M context defaults:
@@ -4354,15 +4420,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4354
4420
  input.workspaceId,
4355
4421
  effectiveCodexCredentialId ?? "",
4356
4422
  );
4357
- const registerSnapshot = async (
4358
- snapshotPromise: ReturnType<typeof resolver.getToken>,
4359
- ) => {
4360
- const snapshot = await snapshotPromise;
4361
- registerSecretRedactions([
4362
- { name: "CODEX_ACCESS_TOKEN", value: snapshot.accessToken },
4363
- ]);
4364
- return snapshot;
4365
- };
4366
4423
  return {
4367
4424
  clientVersion: CODEX_CLIENT_VERSION,
4368
4425
  // Backend sticky cache-routing key — the SAME id as the body's
@@ -4372,8 +4429,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4372
4429
  // (per-request shard lottery = prod's measured 48.6% on sol);
4373
4430
  // with it, resends pin to the warm shard (Codex CLI parity).
4374
4431
  sessionId: input.sessionId,
4375
- getToken: () => registerSnapshot(resolver.getToken()),
4376
- refresh: () => registerSnapshot(resolver.refresh()),
4432
+ getToken: () => resolver.getToken(),
4433
+ refresh: () => resolver.refresh(),
4377
4434
  resolveModel: buildModelResolver(
4378
4435
  CODEX_FALLBACK_MODEL_SLUGS,
4379
4436
  CODEX_FALLBACK_MODEL_SLUGS[0],
@@ -4616,6 +4673,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4616
4673
  force: true,
4617
4674
  clearRequestedCompaction: true,
4618
4675
  trigger: "operator",
4676
+ materializeHistory: materializeScreenshotHistory,
4619
4677
  projectModelInput: compactionModelHistoryProjector,
4620
4678
  ...compactionModeOptions,
4621
4679
  },
@@ -4648,7 +4706,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4648
4706
  },
4649
4707
  );
4650
4708
  if (!isCompactionSummaryFailure(error)) throw error;
4651
- const errorMessage = String(redact(compactionFailureReasonFromError(error)));
4709
+ const errorMessage = String(compactionFailureReasonFromError(error));
4652
4710
  if (
4653
4711
  !(await settle!({
4654
4712
  events: [
@@ -4781,11 +4839,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4781
4839
  rigDefaultEnvironmentValues,
4782
4840
  workspaceVariableSet?.values ?? {},
4783
4841
  );
4784
- // Redact EVERY exported secret value (rig defaults + session set) from turn
4785
- // output, not just the session set's.
4786
- registerSecretRedactions(
4787
- Object.entries(sandboxWorkspaceEnvironmentValues).map(([name, value]) => ({ name, value })),
4788
- );
4789
4842
  // EFFECTIVE compute backend, resolved ONCE at turn start (Case B + Stage D
4790
4843
  // D1-lite) and reused for EVERY downstream decision: the env mint (skip
4791
4844
  // inert platform git tokens for a machine turn), the establish path (no phantom Modal
@@ -4891,7 +4944,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4891
4944
  )
4892
4945
  : null;
4893
4946
  if (initialRunCredentialMaterial) {
4894
- registerSecretRedactions(initialRunCredentialMaterial.redactions);
4895
4947
  for (const payload of runCredentialAuthNeededPayloads(initialRunCredentialMaterial)) {
4896
4948
  publishedRunCredentialNotices.add(JSON.stringify(payload));
4897
4949
  await publish!([{ type: "credential.auth_needed", payload }], true);
@@ -4907,7 +4959,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4907
4959
  undefined,
4908
4960
  );
4909
4961
  const establishPolicy: "eager" | "on-demand" =
4910
- lazyProvisionEnabled(settings) && !machinePrimary && runSettings.sandboxBackend !== "none"
4962
+ lazyProvisionEnabled(settings) &&
4963
+ !machinePrimary &&
4964
+ runSettings.sandboxBackend !== "none" &&
4965
+ // Resolved run credentials must be written to one exact leased sandbox
4966
+ // before agent execution. A warm active pointer can bypass the lazy
4967
+ // provisioner entirely, which previously skipped materialization.
4968
+ !initialRunCredentialMaterial
4911
4969
  ? "on-demand"
4912
4970
  : "eager";
4913
4971
  // Computed exactly ONCE per turn and reused for BOTH the box manifest
@@ -4981,6 +5039,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4981
5039
  sandboxWorkspaceEnvironmentValues,
4982
5040
  {
4983
5041
  skipGitHubToken: activeSandboxBackend === "selfhosted",
5042
+ skipToolspace: activeSandboxBackend === "selfhosted",
4984
5043
  deferGitHubToken:
4985
5044
  activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
4986
5045
  scope: connectionScope,
@@ -4993,19 +5052,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4993
5052
  cancellationSignal,
4994
5053
  undefined,
4995
5054
  );
4996
- registerSecretRedactions([
4997
- ...Object.entries(sandboxGitTokens ?? {}).flatMap(([provider, value]) =>
4998
- value ? [{ name: `GIT_${provider.toUpperCase()}_TOKEN`, value }] : [],
4999
- ),
5000
- ...(sandboxGitToken ? [{ name: "GIT_TOKEN", value: sandboxGitToken }] : []),
5001
- ...((sandboxGitCredentialBindings ?? []).map((binding) => ({
5002
- name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
5003
- value: binding.token,
5004
- })) satisfies SecretForRedaction[]),
5005
- ...(sandboxToolspaceToken
5006
- ? [{ name: "TOOLSPACE_TOKEN", value: sandboxToolspaceToken }]
5007
- : []),
5008
- ]);
5055
+
5009
5056
  const sandboxToolspaceTokenFile = sandboxToolspaceToken
5010
5057
  ? toolspaceTokenFileFromEnvironment(sandboxEnvironment, input.sessionId)
5011
5058
  : undefined;
@@ -5051,12 +5098,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5051
5098
  );
5052
5099
  if (binding) {
5053
5100
  assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
5054
- registerSecretRedactions([
5055
- {
5056
- name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
5057
- value: binding.token,
5058
- },
5059
- ]);
5060
5101
  }
5061
5102
  pendingBinding = binding;
5062
5103
  return binding
@@ -5144,7 +5185,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5144
5185
  toolspaceAuthority,
5145
5186
  );
5146
5187
  if (material) {
5147
- registerSecretRedactions([{ name: "TOOLSPACE_TOKEN", value: material.token }]);
5148
5188
  }
5149
5189
  return material;
5150
5190
  };
@@ -5271,7 +5311,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5271
5311
  );
5272
5312
  return;
5273
5313
  }
5274
- registerSecretRedactions(material.redactions);
5314
+
5275
5315
  await runWorkspaceMutationForSandbox(
5276
5316
  requireTargetSandbox(),
5277
5317
  "runCredentialMaterialization",
@@ -5429,6 +5469,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5429
5469
  db,
5430
5470
  settings,
5431
5471
  bus,
5472
+ opJournal,
5432
5473
  onOp: machineOpObserver.observer,
5433
5474
  onSandboxOperation: sandboxOperationObserver,
5434
5475
  onHomeSandboxRebound,
@@ -5538,6 +5579,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5538
5579
  db,
5539
5580
  settings,
5540
5581
  bus,
5582
+ opJournal,
5541
5583
  onSandboxOperation: sandboxOperationObserver,
5542
5584
  onHomeSandboxLost: publishSandboxLost,
5543
5585
  onHomeSandboxRebound,
@@ -5622,9 +5664,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5622
5664
  const resolveCredential: typeof rawResolveCredential = async (request) => {
5623
5665
  const result = await resolveFrozenCredential(request);
5624
5666
  if (result.status === "ok") {
5625
- registerSecretRedactions(
5626
- headerSecretRedactions("MCP", result.headers, Object.keys(result.headers)),
5627
- );
5628
5667
  }
5629
5668
  return result;
5630
5669
  };
@@ -5646,9 +5685,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5646
5685
  }) => Promise<T>,
5647
5686
  ): Promise<T> => {
5648
5687
  const snapshot = await resolver.getToken();
5649
- registerSecretRedactions([
5650
- { name: "CODEX_APPS_ACCESS_TOKEN", value: snapshot.accessToken },
5651
- ]);
5688
+
5652
5689
  return await withCodexAppsRequestAuthorization(
5653
5690
  db,
5654
5691
  { workspaceId: input.workspaceId, credentialId: codexAppsCredentialId },
@@ -5763,6 +5800,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5763
5800
  latencyMode: turnExecutionPolicy.latencyMode,
5764
5801
  ...(serviceTier ? { serviceTier } : {}),
5765
5802
  ...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
5803
+ humanInputEnabled: agentHumanInputEnabled,
5766
5804
  genesisTitleHint: isGenesisTurn,
5767
5805
  persistentSessionSettings: {
5768
5806
  titleIsSet: Boolean(session.title?.trim()),
@@ -5787,14 +5825,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5787
5825
  ...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
5788
5826
  ? { gitTokenSeed: sandboxGitToken }
5789
5827
  : {}),
5790
- // Toolspace is delivered on EVERY backend including selfhosted. The git-
5791
- // token skip does NOT transfer: that token is inert on a connected
5792
- // machine (it uses its own git creds), but the toolspace token is the
5793
- // machine's ONLY path to programmatic tool calling and grants no more
5794
- // than toolspace:call for its own session (own-session-bound,
5795
- // short-lived and renewable, per-active-turn budgeted, approval-tools
5796
- // excluded). The runtime seeds it to the box's token file over the same
5797
- // exec channel, off-manifest, on every backend.
5828
+ // Toolspace delivery is managed-sandbox-only. Connected Machines own any
5829
+ // manually configured API credentials and must never be contacted during
5830
+ // turn admission merely to seed OpenGeni tooling.
5798
5831
  ...(sandboxToolspaceToken
5799
5832
  ? {
5800
5833
  toolspaceTokenSeed: sandboxToolspaceToken,
@@ -6102,6 +6135,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6102
6135
  db,
6103
6136
  settings,
6104
6137
  bus,
6138
+ opJournal,
6105
6139
  onOp: machineOpObserver.observer,
6106
6140
  onSandboxOperation: sandboxOperationObserver,
6107
6141
  onHomeSandboxLost: publishSandboxLost,
@@ -6200,7 +6234,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6200
6234
  },
6201
6235
  );
6202
6236
  if (!isCompactionSummaryFailure(error)) throw error;
6203
- const errorMessage = String(redact(compactionFailureReasonFromError(error)));
6237
+ const errorMessage = String(compactionFailureReasonFromError(error));
6204
6238
  if (
6205
6239
  !(await settle!({
6206
6240
  events: [
@@ -6302,11 +6336,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6302
6336
  force: true,
6303
6337
  clearRequestedCompaction: true,
6304
6338
  trigger: "operator",
6339
+ materializeHistory: materializeScreenshotHistory,
6305
6340
  projectModelInput: compactionModelHistoryProjector,
6306
6341
  ...compactionModeOptions,
6307
6342
  }
6308
6343
  : {
6309
6344
  trigger: "auto",
6345
+ materializeHistory: materializeScreenshotHistory,
6310
6346
  projectModelInput: compactionModelHistoryProjector,
6311
6347
  ...compactionModeOptions,
6312
6348
  },
@@ -6340,11 +6376,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6340
6376
  },
6341
6377
  );
6342
6378
  if (!isCompactionSummaryFailure(compactError)) throw compactError;
6343
- const errorMessage = String(redact(compactionFailureReasonFromError(compactError)));
6379
+ const errorMessage = String(compactionFailureReasonFromError(compactError));
6344
6380
  observability.error("context compaction failed", {
6345
6381
  sessionId: input.sessionId,
6346
6382
  turnId,
6347
- error: errorMessage,
6383
+ ...safeErrorDiagnostic(compactError),
6348
6384
  });
6349
6385
  if (
6350
6386
  !(await settle!({
@@ -6459,6 +6495,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6459
6495
  ...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
6460
6496
  ...(runCredentialsNote ? { runCredentialsNote } : {}),
6461
6497
  providerApi,
6498
+ materializeModelHistory: materializeScreenshotHistory,
6499
+ materializeSerializedRunState: materializeScreenshotRunState,
6462
6500
  projectModelHistory: modelHistoryProjector,
6463
6501
  });
6464
6502
  runInput = prepared.input;
@@ -6472,16 +6510,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6472
6510
  // total rows no longer equal max(position)+1. Pre-compaction both reduce to
6473
6511
  // the old total-count value, so the common path is unchanged.
6474
6512
  //
6475
- // CRITICAL: seed from the SANITIZED active-row length, not the raw active
6513
+ // CRITICAL: seed from the structurally repaired active-row length, not the raw active
6476
6514
  // count. `prepareRunInput` builds `state.history` from
6477
6515
  // `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
6478
6516
  // rows (a legacy orphan/dangling pair), the in-memory history this turn
6479
6517
  // starts from is K shorter than the raw row count. The reconcile slices the
6480
- // re-sanitized `state.history` off `persistedHistoryCount`; seeding it from
6518
+ // repaired `state.history` off `persistedHistoryCount`; seeding it from
6481
6519
  // the raw count (K too high) skips K genuinely-new items, and a
6482
6520
  // `function_call` left in that skipped region can later have its
6483
6521
  // `function_call_result` persisted alone — the orphan that 400s on replay
6484
- // and bricks the session (issue-61). The sanitized seed is already
6522
+ // and bricks the session (issue-61). The repaired seed is already
6485
6523
  // orphan-free, so it is a stable prefix of the re-sanitized history and the
6486
6524
  // slice begins exactly at the first genuinely-new item.
6487
6525
  persistedHistoryCount = prepared.persistedHistoryCount;
@@ -6509,16 +6547,17 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6509
6547
  attemptId: input.attemptId,
6510
6548
  },
6511
6549
  // Never reuse the persisted prior-turn signal for recovery. Proactive
6512
- // guards provide their exact current signal; provider overflows do not,
6513
- // so null derives decision metadata from active history. Forced
6514
- // recovery proves progress separately by comparing the replacement
6515
- // with the current active-history estimate in maybeCompactContext.
6550
+ // guards provide their exact current provider-based signal; provider
6551
+ // overflows do not, so their diagnostic signal stays unknown/zero.
6552
+ // Forced recovery proves progress separately by comparing the
6553
+ // replacement with the current active-history estimate.
6516
6554
  recoverySignalTokens,
6517
6555
  compactSummarizer,
6518
6556
  {
6519
6557
  force: true,
6520
6558
  ...(triggerLabel === "operator" ? { clearRequestedCompaction: true } : {}),
6521
6559
  trigger: triggerLabel,
6560
+ materializeHistory: materializeScreenshotHistory,
6522
6561
  projectModelInput: compactionModelHistoryProjector,
6523
6562
  ...compactionModeOptions,
6524
6563
  },
@@ -6535,13 +6574,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6535
6574
  return outcome;
6536
6575
  };
6537
6576
 
6538
- // Keep response usage identity across every stream attempt in this
6577
+ // Keep response identity across every stream attempt in this
6539
6578
  // activity. Context compaction retries the same logical dispatch by
6540
6579
  // calling runStreamAttempt again; resetting this state there would reuse
6541
6580
  // the first no-response-ID fallback key and suppress a real model call.
6542
- const modelResponseUsageState = createModelResponseUsageEventState(
6543
- claimedModelUsageSourceKeys,
6544
- );
6581
+ const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
6545
6582
  const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
6546
6583
  if (!runInput) {
6547
6584
  throw new Error("Run input was not prepared");
@@ -6555,6 +6592,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6555
6592
  // schema or compatibility state.
6556
6593
  let currentToolBatchCallIds = new Set<string>();
6557
6594
  let currentToolBatchCompletedCallIds = new Set<string>();
6595
+ let streamSawPerResponseUsage = false;
6558
6596
  // Actual input tokens of the most recent model response this turn; the
6559
6597
  // pre-read trigger for the NEXT turn. Persisted at every turn-end path.
6560
6598
  throwIfWorkerShuttingDown();
@@ -6568,6 +6606,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6568
6606
  // its first-operation provisioner above.
6569
6607
  if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
6570
6608
  const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
6609
+ // `deferredSetup: true` below tells the runtime that the worker owns
6610
+ // platform setup, so the runtime intentionally skips its credential
6611
+ // session callback. Materialize host-managed run credentials here
6612
+ // before any setup command (including provider login hooks), then
6613
+ // decorate setup commands so they source the active generation.
6614
+ await attachRunCredentialRenewal(
6615
+ eagerSetupSession as RunCredentialCommandSession,
6616
+ resolvedSandbox,
6617
+ );
6618
+ const eagerCredentialSetupSession = initialRunCredentialMaterial
6619
+ ? withRunCredentialsSession(eagerSetupSession as object, input.sessionId)
6620
+ : eagerSetupSession;
6571
6621
  await runWorkspaceMutationForSandbox(
6572
6622
  resolvedSandbox,
6573
6623
  "eagerOwnedSandboxSetup",
@@ -6575,7 +6625,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6575
6625
  await runOwnedSandboxSetup(
6576
6626
  agent,
6577
6627
  ownedEstablished.session as never,
6578
- eagerSetupSession as never,
6628
+ eagerCredentialSetupSession as never,
6579
6629
  {
6580
6630
  settings: runSettings,
6581
6631
  environment: sandboxEnvironment,
@@ -6647,9 +6697,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6647
6697
  tokenSession: ToolspaceTokenWriterSession,
6648
6698
  ) => {
6649
6699
  const renewalSession =
6650
- activeSandboxBackend === "selfhosted"
6651
- ? tokenSession
6652
- : ((setupBoxSession as ToolspaceTokenWriterSession | null) ?? tokenSession);
6700
+ (setupBoxSession as ToolspaceTokenWriterSession | null) ?? tokenSession;
6653
6701
  await attachToolspaceTokenRenewal(renewalSession);
6654
6702
  },
6655
6703
  }
@@ -6671,10 +6719,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6671
6719
  : {}),
6672
6720
  }
6673
6721
  : {}),
6674
- contextCompactionSignal: () => modelResponseUsageContextSignal(modelResponseUsageState),
6722
+ contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
6675
6723
  contextCompactionRequested: () =>
6676
6724
  isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
6677
- callModelInputFilter: secretRedactionModelInputFilter((value) => redact(value)),
6678
6725
  ...(toolCancellationFenceRef.current
6679
6726
  ? { turnToolCancellationFence: toolCancellationFenceRef.current }
6680
6727
  : {}),
@@ -6712,9 +6759,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6712
6759
  }
6713
6760
  let stableToolCallIdsToClear: string[] | null = null;
6714
6761
  let completedCurrentToolBatch = false;
6715
- const responseUsageResult = await processModelResponseUsageEvent({
6762
+ let retainedScreenshotMetadata: RetainedArtifactMetadata | null = null;
6763
+ const responseResult = await processModelResponseTerminalEvent({
6716
6764
  event: next.value,
6717
- state: modelResponseUsageState,
6765
+ state: modelResponseState,
6718
6766
  dispatchId: modelUsageDispatchId,
6719
6767
  settings,
6720
6768
  db,
@@ -6745,7 +6793,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6745
6793
  model: turn.model,
6746
6794
  ...(resolvedModel?.provider.id ? { providerId: resolvedModel.provider.id } : {}),
6747
6795
  });
6748
- if (responseUsageResult.status === "processed") {
6796
+ if (responseResult.status === "processed") {
6797
+ streamSawPerResponseUsage ||= responseResult.usageReported;
6749
6798
  currentToolBatchCallIds = new Set<string>();
6750
6799
  currentToolBatchCompletedCallIds = new Set<string>();
6751
6800
  await reconcileConversationTruth();
@@ -6764,7 +6813,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6764
6813
  // conversation context preserved for the post-top-up resume.
6765
6814
  let serializedRunState: string | null = null;
6766
6815
  try {
6767
- serializedRunState = String(redact(stream.state.toString()));
6816
+ serializedRunState = String(stream.state.toString());
6768
6817
  } catch {
6769
6818
  serializedRunState = null;
6770
6819
  }
@@ -6786,7 +6835,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6786
6835
  modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
6787
6836
  callId: pendingToolCall.callId,
6788
6837
  callType: pendingToolCall.callType,
6789
- callItem: redact(pendingToolCall.callItem) as Record<string, unknown>,
6838
+ callItem: pendingToolCall.callItem as Record<string, unknown>,
6790
6839
  });
6791
6840
  if (!registered.accepted) {
6792
6841
  throw new TurnAttemptFencedError(
@@ -6794,9 +6843,39 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6794
6843
  );
6795
6844
  }
6796
6845
  currentToolBatchCallIds.add(pendingToolCall.callId);
6846
+ if (pendingToolCall.callName === "computer_screenshot") {
6847
+ computerScreenshotCallIds.add(pendingToolCall.callId);
6848
+ }
6797
6849
  }
6798
6850
  const completedToolCall = completedToolCallFromSdkEvent(next.value);
6799
6851
  if (completedToolCall) {
6852
+ const typedScreenshot = typedScreenshotFromSdkEvent(next.value);
6853
+ if (
6854
+ typedScreenshot &&
6855
+ typedScreenshot.callId === completedToolCall.callId &&
6856
+ computerScreenshotCallIds.has(completedToolCall.callId)
6857
+ ) {
6858
+ retainedScreenshotMetadata = await retainComputerScreenshot({
6859
+ db,
6860
+ objectStorage,
6861
+ accountId: input.accountId,
6862
+ workspaceId: input.workspaceId,
6863
+ sessionId: input.sessionId,
6864
+ turnId: activeTurnId,
6865
+ attemptId: input.attemptId,
6866
+ output: typedScreenshot,
6867
+ });
6868
+ retainedScreenshotReceiptsByCallId.set(
6869
+ completedToolCall.callId,
6870
+ retainedScreenshotMetadata,
6871
+ );
6872
+ }
6873
+ const durableResultItem = retainedScreenshotMetadata
6874
+ ? (compactRetainedScreenshotHistory(
6875
+ [completedToolCall.resultItem],
6876
+ retainedScreenshotReceiptsByCallId,
6877
+ )[0] ?? completedToolCall.resultItem)
6878
+ : completedToolCall.resultItem;
6800
6879
  // Keep every parallel result in the attempt ledger until the full
6801
6880
  // call batch has settled. The SDK's computed history is
6802
6881
  // non-monotonic while parallel calls complete (a later call may
@@ -6811,7 +6890,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6811
6890
  attemptId: input.attemptId,
6812
6891
  callId: completedToolCall.callId,
6813
6892
  modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
6814
- resultItem: redact(completedToolCall.resultItem) as Record<string, unknown>,
6893
+ resultItem: durableResultItem as Record<string, unknown>,
6815
6894
  });
6816
6895
  if (!recorded.accepted) {
6817
6896
  throw new TurnAttemptFencedError(
@@ -6840,7 +6919,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6840
6919
  completedCurrentToolBatch = currentBatchIsStable;
6841
6920
  }
6842
6921
  }
6843
- const normalized = normalizeSdkEvent(next.value);
6922
+ const normalized = normalizeSdkEvent(
6923
+ next.value,
6924
+ retainedScreenshotMetadata
6925
+ ? {
6926
+ toolOutputOverride: retainedScreenshotMetadata,
6927
+ retainedOutputEvidence: retainedScreenshotMetadata.available
6928
+ ? retainedScreenshotMetadata
6929
+ : {
6930
+ available: false,
6931
+ reason: retainedScreenshotMetadata.reason,
6932
+ },
6933
+ }
6934
+ : {},
6935
+ );
6844
6936
  for (const event of normalized) {
6845
6937
  streamTiming.onEvent(event.type);
6846
6938
  await batcher.push(event);
@@ -6864,6 +6956,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6864
6956
  currentToolBatchCallIds = new Set<string>();
6865
6957
  currentToolBatchCompletedCallIds = new Set<string>();
6866
6958
  }
6959
+ for (const callId of stableToolCallIdsToClear) {
6960
+ computerScreenshotCallIds.delete(callId);
6961
+ }
6867
6962
  }
6868
6963
  }
6869
6964
  } finally {
@@ -6884,7 +6979,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6884
6979
  stream.completed.catch(() => undefined),
6885
6980
  cancellationSignal,
6886
6981
  );
6887
- if (modelResponseUsageState.responseUsageCount === 0) {
6982
+ if (!streamSawPerResponseUsage) {
6888
6983
  const aggregateUsage = stream.state.usage;
6889
6984
  const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
6890
6985
  const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
@@ -6895,8 +6990,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6895
6990
  });
6896
6991
  if (!claimedModelUsageSourceKeys.has(aggregateSourceKey)) {
6897
6992
  claimedModelUsageSourceKeys.add(aggregateSourceKey);
6898
- // The single aggregate frame is this turn's only model-usage record, so
6899
- // it is the first (account-switch surfaces here just like a first response).
6993
+ // The aggregate frame is only a billing fallback when no terminal
6994
+ // response exposed usage. It is not final-request context authority.
6900
6995
  const aggregateAccountCtx = modelCallAccountContext({
6901
6996
  servingCredentialId: effectiveCodexCredentialId,
6902
6997
  priorSessionCredentialId: priorSessionCodexCredentialId,
@@ -6963,14 +7058,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6963
7058
  },
6964
7059
  recordAttemptSignals: async () => {
6965
7060
  if (!aggregateAuthoritative) return;
6966
- if (normalizedAggregateUsage.totalTokens !== null) {
6967
- modelResponseUsageState.lastProviderContextTokensObserved =
6968
- normalizedAggregateUsage.totalTokens;
6969
- modelResponseUsageState.providerContextRevision += 1;
6970
- }
6971
- if (aggregateInput !== null && aggregateInput > 0) {
6972
- await setLastInputTokensFenced(aggregateInput);
6973
- }
7061
+ // Stream aggregates can cover multiple requests and therefore
7062
+ // cannot identify the final request that compaction must bind.
7063
+ // They remain billing/telemetry fallback data only.
7064
+ modelResponseState.contextSignal = null;
7065
+ await setLastInputTokensFenced(null);
6974
7066
  },
6975
7067
  });
6976
7068
  }
@@ -6980,6 +7072,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6980
7072
  const approvals = runtime.serializeApprovals(stream.interruptions);
6981
7073
  const humanInputInterruptions =
6982
7074
  runtime.serializeHumanInputRequests?.(stream.interruptions) ?? [];
7075
+ const latestWorkspace = await getWorkspace(db, input.workspaceId);
7076
+ if (!latestWorkspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
7077
+ assertWorkspaceHumanInputAllowed(
7078
+ resolveWorkspaceAgentHumanInputEnabled(latestWorkspace.settings),
7079
+ "interruption",
7080
+ humanInputInterruptions.length > 0,
7081
+ );
6983
7082
  const humanInputRequests = await Promise.all(
6984
7083
  humanInputInterruptions.map(async (interruption) => {
6985
7084
  const id = stableHumanInputRequestId(
@@ -7055,25 +7154,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7055
7154
  ) {
7056
7155
  return claimedResult({ status: "cancelled" });
7057
7156
  }
7157
+ // The interruption and its preceding tool results are now durable.
7158
+ await finalizeTurnOpStreamOps();
7058
7159
  activityStatus = "requires_action";
7059
7160
  return claimedResult({ status: "requires_action" });
7060
7161
  }
7061
7162
 
7062
- const finalOutput = String(redact(String(stream.finalOutput ?? "")));
7063
- await reconcileConversationTruth();
7163
+ const finalOutput = String(stream.finalOutput ?? "");
7164
+ await reconcileConversationTruth({ requireDurable: true });
7064
7165
  // Op-stream durability fence: the tool outputs are now durably in the
7065
7166
  // history store (a redispatch would NOT re-execute them), so this
7066
7167
  // turn's settled ops may advance their acked frontier — journal persist
7067
7168
  // then wire final ack (licensing the runner to GC its retained
7068
7169
  // frames). Best-effort: a miss leaves the runner's retention TTL to
7069
7170
  // reap, never fails a completed turn.
7070
- if (machinePrimarySession) {
7071
- try {
7072
- await machinePrimarySession.finalizeOpStreamOps();
7073
- } catch {
7074
- // The runner's retention TTL owns the fallback.
7075
- }
7076
- }
7171
+ await finalizeTurnOpStreamOps();
7077
7172
  if (
7078
7173
  !(await settle!({
7079
7174
  events: [
@@ -7140,8 +7235,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7140
7235
  sessionId: input.sessionId,
7141
7236
  turnId: activeTurnId,
7142
7237
  reason: recoveryKind,
7143
- code: overflow?.code ? String(redact(overflow.code)) : undefined,
7144
- error: String(redact(overflow?.message ?? compactionNeeded?.message ?? "")),
7238
+ ...safeErrorDiagnostic(attemptError),
7145
7239
  signalTokens: compactionNeeded?.signalTokens,
7146
7240
  thresholdTokens: compactionNeeded?.thresholdTokens,
7147
7241
  });
@@ -7186,13 +7280,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7186
7280
  );
7187
7281
  compactionRequestCleared = landmark.requestConsumed;
7188
7282
  if (!isCompactionSummaryFailure(compactError)) throw compactError;
7189
- compactionFailureMessage = String(
7190
- redact(compactionFailureReasonFromError(compactError)),
7191
- );
7283
+ compactionFailureMessage = String(compactionFailureReasonFromError(compactError));
7192
7284
  observability.warn("context compaction recovery compaction failed", {
7193
7285
  sessionId: input.sessionId,
7194
7286
  turnId: activeTurnId,
7195
- error: compactionFailureMessage,
7287
+ ...safeErrorDiagnostic(compactError),
7196
7288
  });
7197
7289
  }
7198
7290
  if (!compactionHandled) {
@@ -7303,6 +7395,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7303
7395
  turnMetricOutcome = "cancelled";
7304
7396
  return claimedResult({ status: "cancelled" });
7305
7397
  }
7398
+ acknowledgeRecoveryQuiescence();
7306
7399
  await publishDurableSessionEvents(
7307
7400
  bus,
7308
7401
  input.workspaceId,
@@ -7322,7 +7415,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7322
7415
  } catch (recoveryError) {
7323
7416
  console.error(
7324
7417
  "sandbox lease supersession recovery failed",
7325
- safeErrorDiagnostic(recoveryError, (value) => String(redact(value))),
7418
+ safeErrorDiagnostic(recoveryError),
7326
7419
  );
7327
7420
  throw recoveryError;
7328
7421
  }
@@ -7354,6 +7447,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7354
7447
  turnMetricOutcome = "cancelled";
7355
7448
  return claimedResult({ status: "cancelled" });
7356
7449
  }
7450
+ acknowledgeRecoveryQuiescence();
7357
7451
  await publishDurableSessionEvents(
7358
7452
  bus,
7359
7453
  input.workspaceId,
@@ -7369,7 +7463,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7369
7463
  } catch (recoveryError) {
7370
7464
  console.error(
7371
7465
  "sandbox deadline rotation recovery failed",
7372
- safeErrorDiagnostic(recoveryError, (value) => String(redact(value))),
7466
+ safeErrorDiagnostic(recoveryError),
7373
7467
  );
7374
7468
  throw recoveryError;
7375
7469
  }
@@ -7403,6 +7497,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7403
7497
  turnMetricOutcome = "cancelled";
7404
7498
  return claimedResult({ status: "cancelled" });
7405
7499
  }
7500
+ acknowledgeRecoveryQuiescence();
7406
7501
  await publishDurableSessionEvents(
7407
7502
  bus,
7408
7503
  input.workspaceId,
@@ -7418,7 +7513,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7418
7513
  // the turn through a second cancellation path.
7419
7514
  console.error(
7420
7515
  "worker-shutdown recovery checkpoint failed",
7421
- safeErrorDiagnostic(recoveryError, (value) => String(redact(value))),
7516
+ safeErrorDiagnostic(recoveryError),
7422
7517
  );
7423
7518
  throw recoveryError;
7424
7519
  }
@@ -7517,11 +7612,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7517
7612
  await reconcileConversationTruth({ requireDurable: true });
7518
7613
  }
7519
7614
  checkpointDurable = true;
7520
- } catch (checkpointError) {
7615
+ } catch {
7521
7616
  observability.warn("Codex lease-loss checkpoint failed; refusing automatic turn replay", {
7522
- workspaceId: input.workspaceId,
7523
- turnId: lostTurnId,
7524
- errorName: checkpointError instanceof Error ? checkpointError.name : "unknown",
7617
+ errorClass: "CodexCheckpointOperationError",
7618
+ errorCode: "codex_lease_loss_checkpoint_failed",
7619
+ origin: "worker",
7525
7620
  });
7526
7621
  }
7527
7622
 
@@ -7628,16 +7723,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7628
7723
  await flushRuntimeBatcher();
7629
7724
  await reconcileConversationTruth({ requireDurable: true });
7630
7725
  checkpointDurable = true;
7631
- } catch (checkpointError) {
7726
+ } catch {
7632
7727
  observability.incrementCounter({
7633
7728
  name: "opengeni_codex_failover_checkpoints_total",
7634
7729
  help: "Durable Codex failover checkpoint attempts by outcome.",
7635
7730
  labels: { workspace_key: codexWorkspaceKey, outcome: "failed" },
7636
7731
  });
7637
7732
  observability.warn("Codex failover checkpoint failed; refusing automatic replay", {
7638
- workspaceId: input.workspaceId,
7639
- turnId,
7640
- errorName: checkpointError instanceof Error ? checkpointError.name : "unknown",
7733
+ errorClass: "CodexCheckpointOperationError",
7734
+ errorCode: "codex_failover_checkpoint_failed",
7735
+ origin: "worker",
7641
7736
  });
7642
7737
  }
7643
7738
 
@@ -8190,6 +8285,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8190
8285
  return claimedResult({ status: "cancelled" });
8191
8286
  }
8192
8287
  if (recovery.action === "recovering") {
8288
+ acknowledgeRecoveryQuiescence();
8193
8289
  await publishDurableSessionEvents(
8194
8290
  bus,
8195
8291
  input.workspaceId,
@@ -8208,15 +8304,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8208
8304
  // truth, recover this SAME accepted turn, then let the workflow re-claim
8209
8305
  // it after a pacing delay. This is independent of goal state and never
8210
8306
  // relies on a synthetic continuation prompt.
8211
- const failure = redact(agentRunFailurePayload(error, { isCodexTurn })) as ReturnType<
8307
+ const failure = agentRunFailurePayload(error, { isCodexTurn }) as ReturnType<
8212
8308
  typeof agentRunFailurePayload
8213
8309
  >;
8214
8310
  if (isSessionEventPersistenceError(error)) {
8215
- // Never pass the original Drizzle/postgres-js error to telemetry: its
8216
- // nested cause may contain raw SQL and bound parameters. The typed DB
8217
- // boundary retains only SQLSTATE, stage, correlation, and safe catalog
8218
- // identifiers. Provider inference has already happened and is NOT
8219
- // retried by this terminal classification.
8311
+ // Preserve the exact source message in the internal runtime diagnostic;
8312
+ // SQLSTATE/catalog facts remain separate classification attributes.
8220
8313
  observability.error("session event persistence failed", {
8221
8314
  accountId: input.accountId,
8222
8315
  workspaceId: input.workspaceId,
@@ -8237,6 +8330,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8237
8330
  dbDataType: error.details.database.dataType,
8238
8331
  dbConstraint: error.details.database.constraint,
8239
8332
  dbRoutine: error.details.database.routine,
8333
+ error: error.message,
8240
8334
  });
8241
8335
  }
8242
8336
  if (failure.retryable && publish && turnId && turnStartedPublished) {
@@ -8268,6 +8362,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8268
8362
  turnMetricOutcome = "cancelled";
8269
8363
  return claimedResult({ status: "cancelled" });
8270
8364
  }
8365
+ acknowledgeRecoveryQuiescence();
8271
8366
  await publishDurableSessionEvents(
8272
8367
  bus,
8273
8368
  input.workspaceId,
@@ -8373,18 +8468,38 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8373
8468
  const credentialSessionToClear = runCredentialSession;
8374
8469
  runCredentialSession = null;
8375
8470
  if (credentialSessionToClear) {
8376
- await runWorkspaceMutationForSandbox(
8377
- requireResolvedSandboxForMutation(
8378
- "Run credential cleanup has no exact sandbox lease target",
8379
- ),
8380
- "runCredentialAttemptClear",
8381
- async () =>
8382
- await clearRunCredentialsForAttempt(credentialSessionToClear, {
8383
- sessionId: input.sessionId,
8384
- attemptId: input.attemptId,
8385
- executionGeneration,
8386
- }),
8387
- );
8471
+ const clearAttemptCredentials = async (): Promise<void> =>
8472
+ await clearRunCredentialsForAttempt(credentialSessionToClear, {
8473
+ sessionId: input.sessionId,
8474
+ attemptId: input.attemptId,
8475
+ executionGeneration,
8476
+ });
8477
+ await clearAttemptCredentialsWithSettledFence({
8478
+ activityStatus,
8479
+ runWorkspaceFencedClear: async () =>
8480
+ await runWorkspaceMutationForSandbox(
8481
+ requireResolvedSandboxForMutation(
8482
+ "Run credential cleanup has no exact sandbox lease target",
8483
+ ),
8484
+ "runCredentialAttemptClear",
8485
+ clearAttemptCredentials,
8486
+ ),
8487
+ clearExactAttempt: clearAttemptCredentials,
8488
+ onSettledAttemptFence: () => {
8489
+ // Terminal settlement closes the active attempt before this finally
8490
+ // runs, so the ordinary workspace admission correctly rejects it.
8491
+ // The attempt-qualified delete is nevertheless successor-safe: it
8492
+ // removes only this attempt/generation and clears the pointer only
8493
+ // when it still names that exact generation. Finish that cleanup
8494
+ // directly so a successful turn can continue into workspace capture,
8495
+ // tool teardown, and lease release.
8496
+ observability.info("retrying exact run credential cleanup after turn settlement", {
8497
+ "opengeni.session_id": input.sessionId,
8498
+ "opengeni.turn_id": turnId ?? "",
8499
+ "opengeni.attempt_id": input.attemptId,
8500
+ });
8501
+ },
8502
+ });
8388
8503
  }
8389
8504
  await drainAttemptOwnedSandboxWriters({
8390
8505
  // Normal turn completion owns the same process boundary as
@@ -8452,18 +8567,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8452
8567
  onReceiptFailure: (error) => {
8453
8568
  console.error(
8454
8569
  "agent turn quiescence receipt exhausted; signalling proof",
8455
- safeErrorDiagnostic(error, (value) => String(redact(value))),
8570
+ safeErrorDiagnostic(error),
8456
8571
  );
8457
8572
  },
8458
8573
  onPublishFailure: (error) => {
8459
8574
  console.error(
8460
8575
  "agent turn quiescence event fanout failed",
8461
- safeErrorDiagnostic(error, (value) => String(redact(value))),
8576
+ safeErrorDiagnostic(error),
8462
8577
  );
8463
8578
  },
8464
8579
  onSignalFailure: (error, attempt, retryMs) => {
8465
8580
  console.error("agent turn quiescence proof signal failed; retrying", {
8466
- error: safeErrorDiagnostic(error, (value) => String(redact(value))),
8581
+ error: safeErrorDiagnostic(error),
8467
8582
  attempt,
8468
8583
  retryMs,
8469
8584
  });
@@ -8711,7 +8826,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8711
8826
  sandboxToRelease.release().catch((releaseError) => {
8712
8827
  console.error(
8713
8828
  "sandbox lease release failed (turn outcome unaffected)",
8714
- safeErrorDiagnostic(releaseError, (value) => String(redact(value))),
8829
+ safeErrorDiagnostic(releaseError),
8715
8830
  );
8716
8831
  }),
8717
8832
  finalizerSignal,
@@ -8721,7 +8836,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8721
8836
  finalizationError ??= error;
8722
8837
  console.error(
8723
8838
  "agent turn finalization failed (turn outcome unaffected)",
8724
- safeErrorDiagnostic(error, (value) => String(redact(value))),
8839
+ safeErrorDiagnostic(error),
8725
8840
  );
8726
8841
  } finally {
8727
8842
  // If fallible finalization exited before the ordinary exact-holder
@@ -8782,9 +8897,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8782
8897
  },
8783
8898
  error:
8784
8899
  finalizationError || activityError
8785
- ? safeErrorForTelemetry(finalizationError ?? activityError, (value) =>
8786
- String(redact(value)),
8787
- )
8900
+ ? safeErrorForTelemetry(finalizationError ?? activityError)
8788
8901
  : undefined,
8789
8902
  });
8790
8903
  assertPhysicalToolQuiescenceForCancellation({
@@ -8916,7 +9029,19 @@ export function agentRunFailurePayload(
8916
9029
  attempts?: number;
8917
9030
  retryOutcome?: string;
8918
9031
  database?: Record<string, string>;
9032
+ historyPersistenceStage?: MandatoryHistoryPersistenceStage;
8919
9033
  } {
9034
+ if (error instanceof MandatoryHistoryPersistenceError) {
9035
+ const underlying = isSessionEventPersistenceError(error.cause)
9036
+ ? agentRunFailurePayload(error.cause, options)
9037
+ : {
9038
+ error: error.cause instanceof Error ? error.cause.message : String(error.cause),
9039
+ };
9040
+ return {
9041
+ ...underlying,
9042
+ historyPersistenceStage: error.stage,
9043
+ };
9044
+ }
8920
9045
  const message = error instanceof Error ? error.message : String(error);
8921
9046
  const status =
8922
9047
  typeof error === "object" && error !== null
@@ -8942,15 +9067,8 @@ export function agentRunFailurePayload(
8942
9067
  }
8943
9068
  if (isSessionEventPersistenceError(error)) {
8944
9069
  const { details } = error;
8945
- const eventLabel = details.eventTypes.join(", ") || "session events";
8946
- const failureLabel =
8947
- details.code === "db_deadlock"
8948
- ? "Database deadlock"
8949
- : details.code === "db_serialization_failure"
8950
- ? "Database serialization failure"
8951
- : "Database failure";
8952
9070
  return {
8953
- error: `${failureLabel} while persisting ${eventLabel}. The completed provider call and external effects were not retried.`,
9071
+ error: error.message,
8954
9072
  code: details.code,
8955
9073
  detail:
8956
9074
  details.retryOutcome === "exhausted"
@@ -9012,6 +9130,7 @@ export function agentRunFailurePayload(
9012
9130
  "A required MCP server was temporarily unreachable. The same turn will retry after a short delay.",
9013
9131
  code: "mcp_transport_unavailable",
9014
9132
  retryable: true,
9133
+ detail: message,
9015
9134
  };
9016
9135
  }
9017
9136
  if (
@@ -9229,9 +9348,10 @@ export function codexUsageLimitFailurePayload(
9229
9348
  // goal re-evaluates at most this far out (it will re-pause if still capped).
9230
9349
  const CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 60_000; // 1h
9231
9350
 
9232
- function pendingToolCallFromSdkEvent(event: unknown): {
9351
+ export function pendingToolCallFromSdkEvent(event: unknown): {
9233
9352
  callId: string;
9234
9353
  callType: string;
9354
+ callName: string | null;
9235
9355
  callItem: Record<string, unknown>;
9236
9356
  } | null {
9237
9357
  if (!event || typeof event !== "object") return null;
@@ -9252,10 +9372,16 @@ function pendingToolCallFromSdkEvent(event: unknown): {
9252
9372
  if (typeof callId !== "string" || callId.length === 0 || typeof callType !== "string") {
9253
9373
  return null;
9254
9374
  }
9255
- return { callId, callType, callItem: raw };
9375
+ const callItem = normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]');
9376
+ return {
9377
+ callId,
9378
+ callType,
9379
+ callName: typeof raw.name === "string" ? raw.name : null,
9380
+ callItem,
9381
+ };
9256
9382
  }
9257
9383
 
9258
- function completedToolCallFromSdkEvent(event: unknown): {
9384
+ export function completedToolCallFromSdkEvent(event: unknown): {
9259
9385
  callId: string;
9260
9386
  resultItem: Record<string, unknown>;
9261
9387
  } | null {
@@ -9270,7 +9396,11 @@ function completedToolCallFromSdkEvent(event: unknown): {
9270
9396
  ? (item.rawItem as Record<string, unknown>)
9271
9397
  : {};
9272
9398
  const callId = raw.callId ?? raw.call_id ?? item.id;
9273
- return typeof callId === "string" && callId.length > 0 ? { callId, resultItem: raw } : null;
9399
+ if (typeof callId !== "string" || callId.length === 0) return null;
9400
+ return {
9401
+ callId,
9402
+ resultItem: normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]'),
9403
+ };
9274
9404
  }
9275
9405
 
9276
9406
  /**
@@ -9571,12 +9701,7 @@ export async function recordAuthoritativeModelCallFact(input: {
9571
9701
  });
9572
9702
  } catch (error) {
9573
9703
  input.observability.warn("model call fact persist failed", {
9574
- accountId: input.accountId,
9575
- workspaceId: input.workspaceId,
9576
- sessionId: input.sessionId,
9577
- turnId: input.turnId,
9578
- sourceKey: input.sourceKey,
9579
- error: error instanceof Error ? error.message : String(error),
9704
+ ...safeErrorDiagnostic(error),
9580
9705
  });
9581
9706
  }
9582
9707
  }