@opengeni/worker-bundle 0.16.21 → 0.16.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/activities/agent-turn.d.ts +61 -14
  2. package/dist/activities/context-compaction.d.ts +2 -0
  3. package/dist/activities/environment.d.ts +1 -0
  4. package/dist/activities/expiring-material-renewal.d.ts +5 -3
  5. package/dist/activities/git-credential-renewal.d.ts +1 -1
  6. package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
  7. package/dist/activities/retained-screenshots.d.ts +93 -0
  8. package/dist/activities/run-credential-renewal.d.ts +1 -1
  9. package/dist/activities/run-input.d.ts +2 -0
  10. package/dist/activities/toolspace-token-renewal.d.ts +1 -1
  11. package/dist/activities/workspace-capture.d.ts +1 -0
  12. package/dist/activities.d.ts +3 -0
  13. package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
  14. package/dist/chunk-RAMNYSAW.js.map +1 -0
  15. package/dist/host-export.js +1 -1
  16. package/dist/index.js +1317 -390
  17. package/dist/index.js.map +1 -1
  18. package/dist/sandbox-resume.d.ts +6 -0
  19. package/dist/sandbox-routing.d.ts +11 -6
  20. package/dist/workflow-bundle.js +2 -1
  21. package/package.json +12 -12
  22. package/src/activities/agent-turn.ts +482 -360
  23. package/src/activities/codex-capacity.ts +10 -1
  24. package/src/activities/codex-rotation.ts +19 -5
  25. package/src/activities/context-compaction.ts +9 -3
  26. package/src/activities/environment.ts +13 -24
  27. package/src/activities/expiring-material-renewal.ts +16 -7
  28. package/src/activities/git-credential-renewal.ts +3 -3
  29. package/src/activities/recording.ts +4 -5
  30. package/src/activities/retained-screenshot-reaper.ts +202 -0
  31. package/src/activities/retained-screenshots.ts +892 -0
  32. package/src/activities/rig-verification.ts +13 -34
  33. package/src/activities/run-credential-renewal.ts +9 -2
  34. package/src/activities/run-input.ts +18 -4
  35. package/src/activities/scheduled-tasks.ts +19 -3
  36. package/src/activities/toolspace-token-renewal.ts +9 -2
  37. package/src/activities/workspace-capture.ts +33 -3
  38. package/src/activities.ts +10 -2
  39. package/src/host-export-pump.ts +28 -11
  40. package/src/http.ts +3 -6
  41. package/src/observability-metrics.ts +3 -3
  42. package/src/sandbox-resume.ts +45 -12
  43. package/src/sandbox-routing.ts +30 -15
  44. package/src/worker-service-lifecycle.ts +8 -4
  45. package/src/workflows/file-upload-reaper.ts +4 -1
  46. package/dist/activities/redaction.d.ts +0 -1
  47. package/dist/chunk-BJUW5LCM.js.map +0 -1
  48. package/src/activities/redaction.ts +0 -9
@@ -27,6 +27,7 @@ import {
27
27
  getCodexRotationSettings,
28
28
  getWorkspaceModelPolicy,
29
29
  getWorkspaceGrant,
30
+ getWorkspace,
30
31
  listCodexAccountStatuses,
31
32
  fetchCodexUsageForAccount,
32
33
  getSessionCodexState,
@@ -92,6 +93,7 @@ import {
92
93
  modelTerminalResponseFromSdkEvent,
93
94
  normalizeModelCallUsage,
94
95
  normalizeSdkEvent,
96
+ normalizeProtocolJsonValue,
95
97
  projectHistoryForProvider,
96
98
  sanitizeHistoryItemsForModel,
97
99
  projectModelInputForCapabilities,
@@ -137,7 +139,6 @@ import {
137
139
  type ModelCallUsageNormalization,
138
140
  type BuildAgentOptions,
139
141
  type ConnectorActionPolicyHooks,
140
- type RunAgentStreamOptions,
141
142
  type TurnToolCancellationFence,
142
143
  type BackendUnresolvableCode,
143
144
  type EstablishedSandboxSession,
@@ -263,19 +264,12 @@ import {
263
264
  import { withFirstPartyTools } from "./goals";
264
265
  import {
265
266
  mergeRigDefaultVariableSetEnvironment,
266
- resolveWorkspaceAgentInstructions,
267
267
  resolveWorkspacePackRuntime,
268
268
  resolveWorkspaceSkillLibraryRuntime,
269
269
  settingsWithPackSandboxImage,
270
270
  settingsWithRigImage,
271
271
  } from "./packs";
272
272
  import { deliverFailedChildTurnToParent } from "./parent-wake";
273
- import {
274
- createSecretRedactor,
275
- isCredentialHeaderName,
276
- redactSensitiveText,
277
- type SecretForRedaction,
278
- } from "./redaction";
279
273
  import {
280
274
  createModelHistoryAttachmentProjector,
281
275
  turnInput,
@@ -323,6 +317,8 @@ import {
323
317
  recordCreditMicros,
324
318
  recordModelCacheTokens,
325
319
  recordModelInputTokens,
320
+ recordSessionEventAppendLatency,
321
+ recordSessionEventPublishLatency,
326
322
  runtimeMetricsHooksForObservability,
327
323
  StreamTimingMetrics,
328
324
  turnLifecycleMetricsFor,
@@ -334,6 +330,16 @@ import {
334
330
  prepareRecordingForSettlement,
335
331
  type ActiveRecording,
336
332
  } from "./recording";
333
+ import {
334
+ collectRetainedScreenshotReceipts,
335
+ collectRetainedScreenshotRunStateReceipts,
336
+ compactRetainedScreenshotHistory,
337
+ compactRetainedScreenshotRunState,
338
+ materializeRetainedScreenshotHistory,
339
+ materializeRetainedScreenshotRunState,
340
+ retainComputerScreenshot,
341
+ typedScreenshotFromSdkEvent,
342
+ } from "./retained-screenshots";
337
343
  import { captureWorkspaceRevision, openFreshWorkspaceCaptureSession } from "./workspace-capture";
338
344
  import type { ChannelASession } from "@opengeni/runtime/sandbox";
339
345
  import { createObjectStorage, type ObjectStorage } from "@opengeni/storage";
@@ -347,9 +353,11 @@ import {
347
353
  DEFAULT_FIRST_PARTY_MCP_TOOLS,
348
354
  evaluateWorkspaceModelPolicy,
349
355
  readTurnExecutionPolicyV1,
356
+ resolveWorkspaceAgentHumanInputEnabled,
350
357
  resourceMountPath,
351
358
  type LatencyMode,
352
359
  type ResourceRef,
360
+ type RetainedArtifactMetadata,
353
361
  type SessionEvent,
354
362
  type SessionEventType,
355
363
  type SessionStatus,
@@ -364,6 +372,21 @@ import { createHash, randomUUID } from "node:crypto";
364
372
  export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
365
373
  const PROVIDER_CONNECTIVITY_BACKOFF_MS = [2_000, 5_000, 15_000, 30_000, 60_000] as const;
366
374
 
375
+ export class WorkspaceHumanInputDisabledError extends Error {
376
+ constructor(kind: "resume" | "interruption") {
377
+ super(`Workspace policy rejects structured human-input ${kind}`);
378
+ this.name = "WorkspaceHumanInputDisabledError";
379
+ }
380
+ }
381
+
382
+ export function assertWorkspaceHumanInputAllowed(
383
+ enabled: boolean,
384
+ kind: "resume" | "interruption",
385
+ attempted: boolean,
386
+ ): void {
387
+ if (!enabled && attempted) throw new WorkspaceHumanInputDisabledError(kind);
388
+ }
389
+
367
390
  /** Broad personal lookup is allowed only for a direct human/API command. */
368
391
  export function credentialSubjectIdForTurnInitiator(
369
392
  turn: Pick<SessionTurn, "source" | "initiator" | "initiatorContext">,
@@ -831,6 +854,33 @@ export async function waitForTurnStreamCleanup(
831
854
  await waitForTurnFinalizerStep(providerCompleted, signal);
832
855
  }
833
856
 
857
+ /**
858
+ * Terminal settlement closes the active attempt before turn finalization. The
859
+ * ordinary workspace admission therefore correctly returns `attempt_fenced`
860
+ * for the final attempt-qualified credential deletion. Retry only that exact
861
+ * settled-attempt case directly; every other fence/status and the direct
862
+ * deletion itself remain fail-closed.
863
+ */
864
+ export async function clearAttemptCredentialsWithSettledFence(input: {
865
+ activityStatus: RunAgentTurnResult["status"] | "unknown";
866
+ runWorkspaceFencedClear: () => Promise<void>;
867
+ clearExactAttempt: () => Promise<void>;
868
+ onSettledAttemptFence: () => void;
869
+ }): Promise<void> {
870
+ try {
871
+ await input.runWorkspaceFencedClear();
872
+ } catch (error) {
873
+ const settledAttemptFence =
874
+ error instanceof Error &&
875
+ error.name === "SandboxWorkspaceMutationFencedError" &&
876
+ (error as Error & { code?: unknown }).code === "attempt_fenced" &&
877
+ (input.activityStatus === "idle" || input.activityStatus === "failed");
878
+ if (!settledAttemptFence) throw error;
879
+ input.onSettledAttemptFence();
880
+ await input.clearExactAttempt();
881
+ }
882
+ }
883
+
834
884
  function turnFinalizerCancellationSignal(
835
885
  temporalSignal: AbortSignal | undefined,
836
886
  activityStatus: RunAgentTurnResult["status"] | "unknown",
@@ -847,84 +897,94 @@ function compactionFailureReason(reason: string): string {
847
897
  : `compaction summarization failed: ${reason}`;
848
898
  }
849
899
 
900
+ export type MandatoryHistoryPersistenceStage = "history_append" | "sandbox_envelope";
901
+
902
+ /**
903
+ * Preserve the exact storage failure for the permission-controlled turn event
904
+ * while attaching only a stable, non-secret operation token to public worker
905
+ * diagnostics. Mandatory history failures remain terminal and never inherit a
906
+ * provider-retry classification from coincidental source text or status.
907
+ */
908
+ export class MandatoryHistoryPersistenceError extends Error {
909
+ readonly name = "MandatoryHistoryPersistenceError";
910
+ readonly cause: unknown;
911
+
912
+ constructor(
913
+ readonly stage: MandatoryHistoryPersistenceStage,
914
+ cause: unknown,
915
+ ) {
916
+ super(cause instanceof Error ? cause.message : String(cause));
917
+ this.cause = cause;
918
+ }
919
+ }
920
+
921
+ export async function runMandatoryHistoryPersistenceStep<T>(
922
+ stage: MandatoryHistoryPersistenceStage,
923
+ operation: () => Promise<T>,
924
+ ): Promise<T> {
925
+ try {
926
+ return await operation();
927
+ } catch (cause) {
928
+ throw new MandatoryHistoryPersistenceError(stage, cause);
929
+ }
930
+ }
931
+
850
932
  export type SafeErrorDiagnostic = {
851
- name: string;
852
- message: string;
933
+ errorClass: "WorkerOperationError";
934
+ errorCode: "worker_operation_failed";
853
935
  status?: number;
854
- code?: string;
936
+ origin: "worker";
937
+ historyPersistenceStage?: MandatoryHistoryPersistenceStage;
855
938
  };
856
939
 
857
940
  /**
858
941
  * Produce the only exception shape allowed in worker logs. It deliberately
859
- * excludes stack, cause, response/request bodies, and arbitrary enumerable
860
- * properties while retaining bounded, redacted diagnostics.
942
+ * excludes the arbitrary source message, stack, cause, response/request
943
+ * bodies, and enumerable properties. Exact failure content belongs in the
944
+ * permission-controlled session event, not stdout or telemetry.
861
945
  */
862
- export function safeErrorDiagnostic(
863
- error: unknown,
864
- redactText: (value: string) => string = redactSensitiveText,
865
- ): SafeErrorDiagnostic {
866
- const rawName = error instanceof Error ? error.name : "Error";
867
- const name = /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error";
868
- const rawMessage = error instanceof Error ? error.message : String(error);
946
+ export function safeErrorDiagnostic(error: unknown): SafeErrorDiagnostic {
869
947
  const diagnostic: SafeErrorDiagnostic = {
870
- name,
871
- message: redactText(rawMessage).slice(0, 2_048),
948
+ errorClass: "WorkerOperationError",
949
+ errorCode: "worker_operation_failed",
950
+ origin: "worker",
872
951
  };
873
- if (error && typeof error === "object") {
874
- const status = Number(
875
- (error as { status?: unknown; statusCode?: unknown }).status ??
876
- (error as { statusCode?: unknown }).statusCode,
877
- );
878
- if (Number.isInteger(status) && status >= 100 && status <= 599) {
879
- diagnostic.status = status;
952
+ try {
953
+ let statusSource = error;
954
+ if (error instanceof MandatoryHistoryPersistenceError) {
955
+ diagnostic.historyPersistenceStage = error.stage;
956
+ statusSource = error.cause;
880
957
  }
881
- const rawCode = (error as { code?: unknown }).code;
882
- if (typeof rawCode === "string" || typeof rawCode === "number") {
883
- const code = redactText(String(rawCode)).slice(0, 80);
884
- if (/^[A-Za-z0-9_.:-]+$/.test(code)) diagnostic.code = code;
958
+ if (statusSource && typeof statusSource === "object") {
959
+ const status = Number(
960
+ (statusSource as { status?: unknown; statusCode?: unknown }).status ??
961
+ (statusSource as { statusCode?: unknown }).statusCode,
962
+ );
963
+ if (Number.isInteger(status) && status >= 100 && status <= 599) {
964
+ diagnostic.status = status;
965
+ }
885
966
  }
967
+ } catch {
968
+ // Public diagnostics are best-effort and must never replace the exact
969
+ // internal worker failure.
886
970
  }
887
971
  return diagnostic;
888
972
  }
889
973
 
890
- function safeErrorForTelemetry(error: unknown, redactText: (value: string) => string): Error {
891
- const diagnostic = safeErrorDiagnostic(error, redactText);
892
- const safe = new Error(diagnostic.message);
893
- safe.name = diagnostic.name;
974
+ function safeErrorForTelemetry(error: unknown): Error {
975
+ const diagnostic = safeErrorDiagnostic(error);
976
+ const safe = new Error("worker operation failed") as Error & {
977
+ code?: string;
978
+ status?: number;
979
+ origin?: string;
980
+ };
981
+ safe.name = "WorkerOperationError";
982
+ safe.code = diagnostic.errorCode;
983
+ if (diagnostic.status !== undefined) safe.status = diagnostic.status;
984
+ safe.origin = diagnostic.origin;
894
985
  return safe;
895
986
  }
896
987
 
897
- export function headerSecretRedactions(
898
- prefix: string,
899
- headers: Readonly<Record<string, string>> | undefined,
900
- credentialHeaderNames?: Iterable<string>,
901
- ): SecretForRedaction[] {
902
- const discovered: SecretForRedaction[] = [];
903
- const explicitCredentialHeaders = credentialHeaderNames
904
- ? new Set([...credentialHeaderNames].map((name) => name.toLowerCase()))
905
- : null;
906
- for (const [headerName, value] of Object.entries(headers ?? {})) {
907
- if (
908
- !explicitCredentialHeaders?.has(headerName.toLowerCase()) &&
909
- !isCredentialHeaderName(headerName)
910
- ) {
911
- continue;
912
- }
913
- const safeHeaderName = headerName.toUpperCase().replace(/[^A-Z0-9]+/g, "_");
914
- discovered.push({ name: `${prefix}_${safeHeaderName || "HEADER"}`, value });
915
- if (/^(?:proxy-)?authorization$/i.test(headerName)) {
916
- const credential = value.match(/^\s*[A-Za-z][A-Za-z0-9_-]*\s+(.+)\s*$/)?.[1];
917
- if (credential) {
918
- discovered.push({
919
- name: `${prefix}_${safeHeaderName || "AUTHORIZATION"}_CREDENTIAL`,
920
- value: credential,
921
- });
922
- }
923
- }
924
- }
925
- return discovered;
926
- }
927
-
928
988
  function compactionFailureReasonFromError(error: unknown): string {
929
989
  if (
930
990
  error instanceof CompactionProviderResponseError ||
@@ -1044,14 +1104,14 @@ function collectErrorStrings(value: unknown, seen = new WeakSet<object>()): stri
1044
1104
  * an earlier slice — the orphaned tool output that 400s the Responses API and
1045
1105
  * bricks the session on every replay.
1046
1106
  *
1047
- * Defending: sanitize the full current history into an API-valid sequence (the
1107
+ * Defending: structurally repair the full current history into an API-valid sequence (the
1048
1108
  * same pure rules the read path uses), then append only the new tail beyond the
1049
1109
  * watermark. A trailing dangling call is dropped here and re-evaluated next
1050
1110
  * pass once its result lands, so a call and its result are written together at
1051
1111
  * consecutive positions and a result is never persisted without its call. The
1052
- * watermark advances to the sanitized length — never past anything unwritten —
1112
+ * watermark advances to the repaired length — never past anything unwritten —
1053
1113
  * so a non-monotonic history can never desync it. When previously-persisted
1054
- * rows already exceed the sanitized length (e.g. legacy orphans written before
1114
+ * rows already exceed the repaired length (e.g. legacy orphans written before
1055
1115
  * this fix), nothing new is appended and the watermark holds steady.
1056
1116
  */
1057
1117
  /**
@@ -1518,14 +1578,9 @@ export async function emitModelCallUsage(input: {
1518
1578
  if (!authoritative) return false;
1519
1579
  try {
1520
1580
  input.observability.info("model call usage", {
1521
- accountId: input.accountId,
1522
- workspaceId: input.workspaceId,
1523
- sessionId: input.sessionId,
1524
- turnId: input.turnId,
1525
1581
  provider: input.provider,
1526
1582
  providerApi: input.providerApi,
1527
1583
  model: input.model,
1528
- sourceKey: input.sourceKey,
1529
1584
  inputTokens: telemetry.inputTokens,
1530
1585
  outputTokens: telemetry.outputTokens,
1531
1586
  cachedTokens: telemetry.cachedTokens,
@@ -1543,7 +1598,6 @@ export async function emitModelCallUsage(input: {
1543
1598
  provider: input.provider,
1544
1599
  providerApi: input.providerApi,
1545
1600
  model: input.model,
1546
- sourceKey: input.sourceKey,
1547
1601
  rejectedFields: normalizedUsage.rejectedFields.join(","),
1548
1602
  });
1549
1603
  }
@@ -1590,16 +1644,13 @@ export function historyRowsToAppend(
1590
1644
  // positions from 0) when callers do not pass an explicit next position.
1591
1645
  nextPosition: number = persistedHistoryCount,
1592
1646
  toolOutputTruncationTokens?: number,
1593
- redactValue: (value: unknown) => unknown = (value) => value,
1594
1647
  ): {
1595
1648
  rows: Array<{ position: number; item: Record<string, unknown> }>;
1596
1649
  nextWatermark: number;
1597
1650
  nextPosition: number;
1598
1651
  } {
1599
- const sanitized = sanitizeHistoryItemsForModel(rawHistory, toolOutputTruncationTokens).map(
1600
- (item) => redactValue(item) as Record<string, unknown>,
1601
- );
1602
- if (sanitized.length <= persistedHistoryCount) {
1652
+ const modelReady = sanitizeHistoryItemsForModel(rawHistory, toolOutputTruncationTokens);
1653
+ if (modelReady.length <= persistedHistoryCount) {
1603
1654
  return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
1604
1655
  }
1605
1656
  // Canonical model-facing inputs (including machine-input batches) are
@@ -1609,16 +1660,22 @@ export function historyRowsToAppend(
1609
1660
  // notes—must not accidentally become conversation memory during reconciliation.
1610
1661
  // Advance the in-memory watermark past them, but only allocate durable
1611
1662
  // positions to actual model/tool output.
1612
- const rows = sanitized
1613
- .slice(persistedHistoryCount)
1614
- .filter((item) => !(item.type === "message" && item.role === "system"))
1615
- .map((item, offset) => ({
1616
- position: nextPosition + offset,
1617
- item: item as Record<string, unknown>,
1618
- }));
1663
+ const rows: Array<{ position: number; item: Record<string, unknown> }> = [];
1664
+ for (const [offset, item] of modelReady.slice(persistedHistoryCount).entries()) {
1665
+ if (item.type === "message" && item.role === "system") {
1666
+ continue;
1667
+ }
1668
+ rows.push({
1669
+ position: nextPosition + rows.length,
1670
+ item: normalizeProtocolJsonValue(
1671
+ item as Record<string, unknown>,
1672
+ `$[${persistedHistoryCount + offset}]`,
1673
+ ),
1674
+ });
1675
+ }
1619
1676
  return {
1620
1677
  rows,
1621
- nextWatermark: sanitized.length,
1678
+ nextWatermark: modelReady.length,
1622
1679
  nextPosition: nextPosition + rows.length,
1623
1680
  };
1624
1681
  }
@@ -1660,16 +1717,7 @@ export function selectRejectedProviderArtifactHistoryIds(
1660
1717
  }
1661
1718
 
1662
1719
  /** Literal final per-call boundary for replayed and current-turn model input. */
1663
- export function secretRedactionModelInputFilter(
1664
- redactValue: (value: unknown) => unknown,
1665
- ): NonNullable<RunAgentStreamOptions["callModelInputFilter"]> {
1666
- return ({ modelData }) => ({
1667
- ...modelData,
1668
- input: redactValue(modelData.input) as typeof modelData.input,
1669
- });
1670
- }
1671
-
1672
- function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
1720
+ export function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
1673
1721
  if (item.type === "message") {
1674
1722
  return item.role === "assistant";
1675
1723
  }
@@ -2371,6 +2419,34 @@ export function codexCredentialLeaseDeadlineExpired(
2371
2419
  );
2372
2420
  }
2373
2421
 
2422
+ type OpStreamFinalizer = {
2423
+ finalizeOpStreamOps(): Promise<void>;
2424
+ };
2425
+
2426
+ /** Release retained Connected Machine output only after the turn is durable. */
2427
+ export async function finalizeDurableTurnOpStreams(
2428
+ sessions: readonly unknown[],
2429
+ fallback: OpStreamFinalizer | null,
2430
+ ): Promise<void> {
2431
+ const candidates = new Set<OpStreamFinalizer>();
2432
+ for (const session of sessions) {
2433
+ const candidate = session as Partial<OpStreamFinalizer> | null;
2434
+ if (typeof candidate?.finalizeOpStreamOps === "function") {
2435
+ candidates.add(candidate as OpStreamFinalizer);
2436
+ }
2437
+ }
2438
+ if (candidates.size === 0 && fallback) {
2439
+ candidates.add(fallback);
2440
+ }
2441
+ for (const candidate of candidates) {
2442
+ try {
2443
+ await candidate.finalizeOpStreamOps();
2444
+ } catch {
2445
+ // The runner's retention TTL owns the fallback.
2446
+ }
2447
+ }
2448
+ }
2449
+
2374
2450
  export function createRunAgentTurnActivity(services: () => Promise<ActivityServices>) {
2375
2451
  return async function runAgentTurn(input: RunAgentTurnInput): Promise<RunAgentTurnResult> {
2376
2452
  const {
@@ -2429,6 +2505,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2429
2505
  acknowledgeQuiescence = true;
2430
2506
  noteCancellationRequested();
2431
2507
  };
2508
+ const acknowledgeRecoveryQuiescence = (): void => {
2509
+ // requestSessionTurnRecovery closed this exact attempt and durably
2510
+ // recorded the replacement cause. The activity still owns the physical
2511
+ // tool/credential boundary until finally drains it and publishes the
2512
+ // exact quiescence receipt; a workflow result alone is never that proof.
2513
+ acknowledgeQuiescence = true;
2514
+ noteCancellationRequested();
2515
+ };
2432
2516
  let turnId: string | undefined;
2433
2517
  let triggerEventId: string | undefined;
2434
2518
  const claimedResult = (
@@ -2562,7 +2646,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2562
2646
  workspaceId: input.workspaceId,
2563
2647
  turnId,
2564
2648
  reason,
2565
- errorName: error instanceof Error ? error.name : "unknown",
2649
+ ...safeErrorDiagnostic(error),
2566
2650
  });
2567
2651
  observability.incrementCounter({
2568
2652
  name: "opengeni_codex_lease_renewals_total",
@@ -2594,8 +2678,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2594
2678
  return resolvedSandbox;
2595
2679
  };
2596
2680
  // The machine-primary SelfhostedSession (the UNWRAPPED backend, not the
2597
- // routing proxy): held so the turn's completion can final-ack this turn's
2598
- // settled op-stream ops AFTER the results are durably persisted.
2681
+ // routing proxy). Kept as a fallback finalizer; the routing proxy normally
2682
+ // aggregates it together with every machine reached after a mid-turn swap.
2599
2683
  let machinePrimarySession: import("@opengeni/runtime").SelfhostedSession | null = null;
2600
2684
  let lazyOwnedSandbox: EstablishedSandboxSession | null = null;
2601
2685
  let turnSandboxProvisioner: TurnSandboxProvisioner<ResumedTurnSandbox> | null = null;
@@ -2604,6 +2688,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2604
2688
  // THIS handle so a mid-turn sandbox_swap can never re-route those execs onto a
2605
2689
  // connected machine (the user's real computer).
2606
2690
  let setupBoxSession: unknown = null;
2691
+ const finalizeTurnOpStreamOps = async (): Promise<void> => {
2692
+ await finalizeDurableTurnOpStreams(
2693
+ [lazyOwnedSandbox?.session, resolvedSandbox?.established.session],
2694
+ machinePrimarySession,
2695
+ );
2696
+ };
2607
2697
  // A same-target API repair can replace the home provider while this turn is
2608
2698
  // alive. Keep setup/snapshot persistence on the rebound raw session while
2609
2699
  // preserving the SDK-owned routing proxy for eager turns. Lazy turns hold the
@@ -2951,7 +3041,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2951
3041
  sessionId: input.sessionId,
2952
3042
  sandboxGroupId: heartbeatGroupId,
2953
3043
  leaseEpoch: heartbeatEpoch,
2954
- error: error instanceof Error ? error.message : String(error),
3044
+ ...safeErrorDiagnostic(error),
2955
3045
  });
2956
3046
  })
2957
3047
  .finally(() => {
@@ -3104,6 +3194,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3104
3194
  // stable and this scalar append watermark is valid again. The sanitizer
3105
3195
  // remains the final call/result pairing guard for every other reconcile.
3106
3196
  let persistedHistoryCount = 0;
3197
+ const retainedScreenshotReceiptsByCallId = new Map<string, RetainedArtifactMetadata>();
3198
+ const computerScreenshotCallIds = new Set<string>();
3199
+ const materializeScreenshotHistory = async (history: Array<Record<string, unknown>>) => {
3200
+ collectRetainedScreenshotReceipts(history, retainedScreenshotReceiptsByCallId);
3201
+ return await materializeRetainedScreenshotHistory({
3202
+ db,
3203
+ objectStorage,
3204
+ workspaceId: input.workspaceId,
3205
+ sessionId: input.sessionId,
3206
+ history,
3207
+ });
3208
+ };
3209
+ const materializeScreenshotRunState = async (serialized: string) => {
3210
+ collectRetainedScreenshotRunStateReceipts(serialized, retainedScreenshotReceiptsByCallId);
3211
+ return await materializeRetainedScreenshotRunState({
3212
+ db,
3213
+ objectStorage,
3214
+ workspaceId: input.workspaceId,
3215
+ sessionId: input.sessionId,
3216
+ serialized,
3217
+ });
3218
+ };
3107
3219
  let providerArtifactCandidates: Awaited<
3108
3220
  ReturnType<typeof turnInput>
3109
3221
  >["providerArtifactCandidates"] = {
@@ -3121,15 +3233,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3121
3233
  if (!stream || !turnId) {
3122
3234
  return;
3123
3235
  }
3236
+ const durableTurnId = turnId;
3124
3237
  try {
3125
3238
  const rawHistory = (stream.state as { history?: unknown[] }).history;
3126
3239
  if (Array.isArray(rawHistory)) {
3127
- const { rows, nextWatermark, nextPosition } = historyRowsToAppend(
3240
+ const durableHistory = compactRetainedScreenshotHistory(
3128
3241
  rawHistory as Array<Record<string, unknown>>,
3242
+ retainedScreenshotReceiptsByCallId,
3243
+ );
3244
+ const { rows, nextWatermark, nextPosition } = historyRowsToAppend(
3245
+ durableHistory,
3129
3246
  persistedHistoryCount,
3130
3247
  nextHistoryPosition,
3131
3248
  modelRunSettings.modelToolOutputTruncationTokens,
3132
- (value) => redact(value),
3133
3249
  );
3134
3250
  const hasModelOrToolProgress = rows.some((row) =>
3135
3251
  isModelOrToolProgressHistoryItem(row.item),
@@ -3137,21 +3253,23 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3137
3253
  const shouldAppendRows =
3138
3254
  rows.length > 0 && (!options.skipInputOnlyRows || hasModelOrToolProgress);
3139
3255
  if (shouldAppendRows) {
3140
- const appended = await appendSessionHistoryItems(db, {
3141
- accountId: input.accountId,
3142
- workspaceId: input.workspaceId,
3143
- sessionId: input.sessionId,
3144
- turnId,
3145
- expectedExecutionGeneration: executionGeneration,
3146
- expectedAttemptId: input.attemptId,
3147
- modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
3148
- items: rows,
3256
+ await runMandatoryHistoryPersistenceStep("history_append", async () => {
3257
+ const appended = await appendSessionHistoryItems(db, {
3258
+ accountId: input.accountId,
3259
+ workspaceId: input.workspaceId,
3260
+ sessionId: input.sessionId,
3261
+ turnId: durableTurnId,
3262
+ expectedExecutionGeneration: executionGeneration,
3263
+ expectedAttemptId: input.attemptId,
3264
+ modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
3265
+ items: rows,
3266
+ });
3267
+ if (!appended) {
3268
+ throw new TurnAttemptFencedError(
3269
+ "turn execution generation was fenced while saving conversation history",
3270
+ );
3271
+ }
3149
3272
  });
3150
- if (!appended) {
3151
- throw new TurnAttemptFencedError(
3152
- "turn execution generation was fenced while saving conversation history",
3153
- );
3154
- }
3155
3273
  }
3156
3274
  if (shouldAppendRows || !options.skipInputOnlyRows) {
3157
3275
  persistedHistoryCount = nextWatermark;
@@ -3160,37 +3278,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3160
3278
  }
3161
3279
  const envelope = sandboxStateEntryFromRunState(stream.state);
3162
3280
  if (envelope) {
3163
- await upsertSandboxSessionEnvelope(db, {
3164
- accountId: input.accountId,
3165
- workspaceId: input.workspaceId,
3166
- sessionId: input.sessionId,
3167
- envelope,
3168
- });
3281
+ await runMandatoryHistoryPersistenceStep("sandbox_envelope", () =>
3282
+ upsertSandboxSessionEnvelope(db, {
3283
+ accountId: input.accountId,
3284
+ workspaceId: input.workspaceId,
3285
+ sessionId: input.sessionId,
3286
+ envelope,
3287
+ }),
3288
+ );
3169
3289
  }
3170
3290
  } catch (persistError) {
3171
3291
  console.error(
3172
3292
  "session history dual-write failed (run unaffected)",
3173
- safeErrorDiagnostic(persistError, (value) => String(redact(value))),
3293
+ safeErrorDiagnostic(persistError),
3174
3294
  );
3175
3295
  if (options.requireDurable) throw persistError;
3176
3296
  }
3177
3297
  };
3178
- // Reassigned after the variable set loads; the publish closure is
3179
- // created (and used for turn.started) before the variableSet is available.
3180
- // Generic auth/cookie/signed-URL/token classification is active before any
3181
- // exact run-secret provenance is available. Registered values strengthen
3182
- // this boundary as credentials are resolved and renewed during the turn.
3183
- let redact: (payload: unknown) => unknown = createSecretRedactor([]);
3184
- const secretRedactions = new Map<string, string>();
3185
3298
  const publishedRunCredentialNotices = new Set<string>();
3186
- const registerSecretRedactions = (secrets: SecretForRedaction[]): void => {
3187
- for (const secret of secrets) {
3188
- if (!secretRedactions.has(secret.value)) secretRedactions.set(secret.value, secret.name);
3189
- }
3190
- redact = createSecretRedactor(
3191
- [...secretRedactions].map(([value, name]) => ({ name, value })),
3192
- );
3193
- };
3299
+
3194
3300
  let variableSetId = "";
3195
3301
  // Rig telemetry (M3): set once the session loads; empty string for a rig-less
3196
3302
  // turn (mirrors variableSetId). Read by the activity span's finally block.
@@ -3377,7 +3483,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3377
3483
  ) => {
3378
3484
  const inputs = events.map((event) => ({
3379
3485
  ...event,
3380
- payload: redact(event.payload),
3486
+ payload: event.payload,
3381
3487
  turnId: turnId!,
3382
3488
  producerId,
3383
3489
  producerSeq: ++producerSeq,
@@ -3391,6 +3497,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3391
3497
  executionGeneration,
3392
3498
  input.attemptId,
3393
3499
  inputs,
3500
+ {
3501
+ onAppend: ({ durationSeconds }) =>
3502
+ recordSessionEventAppendLatency(observability, { durationSeconds }),
3503
+ onPublish: ({ durationSeconds }) =>
3504
+ recordSessionEventPublishLatency(observability, { durationSeconds }),
3505
+ },
3394
3506
  );
3395
3507
  if (inputs.length > 0 && !appended.accepted) {
3396
3508
  throw new TurnAttemptFencedError("turn execution generation was fenced");
@@ -3445,13 +3557,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3445
3557
  : undefined;
3446
3558
  const inputs = inputSettlement.events.map((event) => ({
3447
3559
  ...event,
3448
- payload: redact(event.payload),
3560
+ payload: event.payload,
3449
3561
  turnId: turnId!,
3450
3562
  producerId,
3451
3563
  producerSeq: ++producerSeq,
3452
3564
  }));
3453
- const redactedRunState = inputSettlement.runState
3454
- ? (redact(inputSettlement.runState) as typeof inputSettlement.runState)
3565
+ const runState = inputSettlement.runState
3566
+ ? {
3567
+ ...inputSettlement.runState,
3568
+ serializedRunState: compactRetainedScreenshotRunState(
3569
+ inputSettlement.runState.serializedRunState,
3570
+ retainedScreenshotReceiptsByCallId,
3571
+ ),
3572
+ }
3455
3573
  : undefined;
3456
3574
  const result = await applySessionTurnSettlement(db, input.workspaceId, {
3457
3575
  sessionId: input.sessionId,
@@ -3462,7 +3580,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3462
3580
  sessionStatus: inputSettlement.sessionStatus,
3463
3581
  activeTurnId: inputSettlement.activeTurnId,
3464
3582
  events: inputs,
3465
- ...(redactedRunState ? { runState: redactedRunState } : {}),
3583
+ ...(runState ? { runState } : {}),
3466
3584
  ...(recordingMutation ? { recording: recordingMutation } : {}),
3467
3585
  ...(compactionRequestFailure ? { compactionRequestFailure } : {}),
3468
3586
  });
@@ -3820,10 +3938,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3820
3938
  labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
3821
3939
  });
3822
3940
  observability.warn("Codex adaptive fleet shadow decision failed open", {
3823
- workspaceId: input.workspaceId,
3824
3941
  stage: shadowResult.stage,
3825
3942
  reason: shadowResult.reason,
3826
- errorName: shadowResult.errorName,
3943
+ errorClass: "CodexFleetShadowOperationError",
3944
+ errorCode: "codex_fleet_shadow_failed",
3945
+ origin: "worker",
3827
3946
  payloadBytes: shadowResult.payloadBytes,
3828
3947
  });
3829
3948
  }
@@ -4173,15 +4292,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4173
4292
  attemptId: input.attemptId,
4174
4293
  executionGeneration: turn.executionGeneration,
4175
4294
  };
4176
- const [workspaceAgentInstructions, instructionPolicySnapshot, preferenceSnapshot] =
4177
- await Promise.all([
4178
- resolveWorkspaceAgentInstructions(db, input.workspaceId),
4179
- getOrCreateWorkspaceInstructionPolicySnapshot(db, governanceClaims),
4180
- getOrCreatePreferenceRegistrySnapshot(db, governanceClaims).catch((error) => {
4181
- if (error instanceof PreferenceRegistryInitiatorError) return null;
4182
- throw error;
4183
- }),
4184
- ]);
4295
+ const [workspace, instructionPolicySnapshot, preferenceSnapshot] = await Promise.all([
4296
+ getWorkspace(db, input.workspaceId),
4297
+ getOrCreateWorkspaceInstructionPolicySnapshot(db, governanceClaims),
4298
+ getOrCreatePreferenceRegistrySnapshot(db, governanceClaims).catch((error) => {
4299
+ if (error instanceof PreferenceRegistryInitiatorError) return null;
4300
+ throw error;
4301
+ }),
4302
+ ]);
4303
+ if (!workspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
4304
+ const workspaceAgentInstructions = workspace.agentInstructions;
4305
+ const agentHumanInputEnabled = resolveWorkspaceAgentHumanInputEnabled(workspace.settings);
4306
+ assertWorkspaceHumanInputAllowed(agentHumanInputEnabled, "resume", humanInputResume !== null);
4185
4307
  const workspaceGovernance = renderWorkspaceGovernanceContext({
4186
4308
  instructionPolicy: instructionPolicySnapshot,
4187
4309
  preferences: preferenceSnapshot,
@@ -4206,33 +4328,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4206
4328
  openaiReasoningEffort: turn.reasoningEffort,
4207
4329
  sandboxBackend: turn.sandboxBackend,
4208
4330
  };
4209
- const sessionMcpHeaderNames = new Map<string, readonly string[]>();
4210
4331
  const runSettings = await settingsWithSessionMcpServersForRun(
4211
4332
  db,
4212
4333
  input.workspaceId,
4213
4334
  input.sessionId,
4214
4335
  input.attemptId,
4215
4336
  baseRunSettings,
4216
- {
4217
- onResolvedServers: (servers) => {
4218
- for (const server of servers) {
4219
- sessionMcpHeaderNames.set(server.id, server.headerNames);
4220
- }
4221
- },
4222
- },
4223
4337
  );
4224
- registerSecretRedactions([
4225
- ...(runSettings.accessKey
4226
- ? [{ name: "OPENGENI_ACCESS_KEY", value: runSettings.accessKey }]
4227
- : []),
4228
- ...runSettings.mcpServers.flatMap((server) => {
4229
- return headerSecretRedactions(
4230
- `MCP_${server.id.toUpperCase()}_STATIC`,
4231
- server.headers,
4232
- sessionMcpHeaderNames.get(server.id),
4233
- );
4234
- }),
4235
- ]);
4338
+
4236
4339
  // Multi-provider per-turn routing → the provider gating (compaction mode,
4237
4340
  // hosted web search, encrypted reasoning, context window) the agent and
4238
4341
  // compaction summarizer must use; null falls back to the legacy global
@@ -4263,49 +4366,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4263
4366
  projectModelInputForCapabilities(await attachmentProjector(items), modelInputPolicy);
4264
4367
  const compactionModelHistoryProjector = async (items: Array<Record<string, unknown>>) =>
4265
4368
  await modelHistoryProjector(projectHistoryForProvider(items, providerApi));
4266
- const selectedProvider = resolvedModel?.provider;
4267
- const publicProviderHeaders = new Set(
4268
- selectedProvider?.publicDefaultHeaderNames?.map((name) => name.toLowerCase()) ?? [],
4269
- );
4270
- const publicProviderQueries = new Set(
4271
- selectedProvider?.publicDefaultQueryNames?.map((name) => name.toLowerCase()) ?? [],
4272
- );
4273
- const fallbackProviderApiKey =
4274
- capabilitySettings.openaiProvider === "azure"
4275
- ? (capabilitySettings.azureOpenaiApiKey ?? capabilitySettings.azureOpenaiAdToken)
4276
- : capabilitySettings.openaiApiKey;
4277
- registerSecretRedactions([
4278
- ...(selectedProvider?.apiKey
4279
- ? [
4280
- {
4281
- name: `MODEL_${selectedProvider.id.toUpperCase()}_API_KEY`,
4282
- value: selectedProvider.apiKey,
4283
- },
4284
- ]
4285
- : !resolvedModel && fallbackProviderApiKey
4286
- ? [
4287
- {
4288
- name: "MODEL_PROVIDER_API_KEY",
4289
- value: fallbackProviderApiKey,
4290
- },
4291
- ]
4292
- : []),
4293
- ...Object.entries(selectedProvider?.defaultHeaders ?? {}).flatMap(([name, value]) =>
4294
- publicProviderHeaders.has(name.toLowerCase())
4295
- ? []
4296
- : [
4297
- {
4298
- name: `MODEL_PROVIDER_HEADER_${name.toUpperCase()}`,
4299
- value,
4300
- },
4301
- ],
4302
- ),
4303
- ...Object.entries(selectedProvider?.defaultQuery ?? {}).flatMap(([name, value]) =>
4304
- publicProviderQueries.has(name.toLowerCase())
4305
- ? []
4306
- : [{ name: `MODEL_PROVIDER_QUERY_${name.toUpperCase()}`, value }],
4307
- ),
4308
- ]);
4309
4369
  // Bind the provider/model catalog's context policy to every model-facing
4310
4370
  // path for this turn. In particular, Codex subscription turns must not
4311
4371
  // inherit the deployment's OpenAI/Azure mode or 1.05M context defaults:
@@ -4360,15 +4420,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4360
4420
  input.workspaceId,
4361
4421
  effectiveCodexCredentialId ?? "",
4362
4422
  );
4363
- const registerSnapshot = async (
4364
- snapshotPromise: ReturnType<typeof resolver.getToken>,
4365
- ) => {
4366
- const snapshot = await snapshotPromise;
4367
- registerSecretRedactions([
4368
- { name: "CODEX_ACCESS_TOKEN", value: snapshot.accessToken },
4369
- ]);
4370
- return snapshot;
4371
- };
4372
4423
  return {
4373
4424
  clientVersion: CODEX_CLIENT_VERSION,
4374
4425
  // Backend sticky cache-routing key — the SAME id as the body's
@@ -4378,8 +4429,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4378
4429
  // (per-request shard lottery = prod's measured 48.6% on sol);
4379
4430
  // with it, resends pin to the warm shard (Codex CLI parity).
4380
4431
  sessionId: input.sessionId,
4381
- getToken: () => registerSnapshot(resolver.getToken()),
4382
- refresh: () => registerSnapshot(resolver.refresh()),
4432
+ getToken: () => resolver.getToken(),
4433
+ refresh: () => resolver.refresh(),
4383
4434
  resolveModel: buildModelResolver(
4384
4435
  CODEX_FALLBACK_MODEL_SLUGS,
4385
4436
  CODEX_FALLBACK_MODEL_SLUGS[0],
@@ -4622,6 +4673,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4622
4673
  force: true,
4623
4674
  clearRequestedCompaction: true,
4624
4675
  trigger: "operator",
4676
+ materializeHistory: materializeScreenshotHistory,
4625
4677
  projectModelInput: compactionModelHistoryProjector,
4626
4678
  ...compactionModeOptions,
4627
4679
  },
@@ -4654,7 +4706,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4654
4706
  },
4655
4707
  );
4656
4708
  if (!isCompactionSummaryFailure(error)) throw error;
4657
- const errorMessage = String(redact(compactionFailureReasonFromError(error)));
4709
+ const errorMessage = String(compactionFailureReasonFromError(error));
4658
4710
  if (
4659
4711
  !(await settle!({
4660
4712
  events: [
@@ -4787,11 +4839,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4787
4839
  rigDefaultEnvironmentValues,
4788
4840
  workspaceVariableSet?.values ?? {},
4789
4841
  );
4790
- // Redact EVERY exported secret value (rig defaults + session set) from turn
4791
- // output, not just the session set's.
4792
- registerSecretRedactions(
4793
- Object.entries(sandboxWorkspaceEnvironmentValues).map(([name, value]) => ({ name, value })),
4794
- );
4795
4842
  // EFFECTIVE compute backend, resolved ONCE at turn start (Case B + Stage D
4796
4843
  // D1-lite) and reused for EVERY downstream decision: the env mint (skip
4797
4844
  // inert platform git tokens for a machine turn), the establish path (no phantom Modal
@@ -4897,7 +4944,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4897
4944
  )
4898
4945
  : null;
4899
4946
  if (initialRunCredentialMaterial) {
4900
- registerSecretRedactions(initialRunCredentialMaterial.redactions);
4901
4947
  for (const payload of runCredentialAuthNeededPayloads(initialRunCredentialMaterial)) {
4902
4948
  publishedRunCredentialNotices.add(JSON.stringify(payload));
4903
4949
  await publish!([{ type: "credential.auth_needed", payload }], true);
@@ -4913,7 +4959,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4913
4959
  undefined,
4914
4960
  );
4915
4961
  const establishPolicy: "eager" | "on-demand" =
4916
- lazyProvisionEnabled(settings) && !machinePrimary && runSettings.sandboxBackend !== "none"
4962
+ lazyProvisionEnabled(settings) &&
4963
+ !machinePrimary &&
4964
+ runSettings.sandboxBackend !== "none" &&
4965
+ // Resolved run credentials must be written to one exact leased sandbox
4966
+ // before agent execution. A warm active pointer can bypass the lazy
4967
+ // provisioner entirely, which previously skipped materialization.
4968
+ !initialRunCredentialMaterial
4917
4969
  ? "on-demand"
4918
4970
  : "eager";
4919
4971
  // Computed exactly ONCE per turn and reused for BOTH the box manifest
@@ -4987,6 +5039,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4987
5039
  sandboxWorkspaceEnvironmentValues,
4988
5040
  {
4989
5041
  skipGitHubToken: activeSandboxBackend === "selfhosted",
5042
+ skipToolspace: activeSandboxBackend === "selfhosted",
4990
5043
  deferGitHubToken:
4991
5044
  activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
4992
5045
  scope: connectionScope,
@@ -4999,19 +5052,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4999
5052
  cancellationSignal,
5000
5053
  undefined,
5001
5054
  );
5002
- registerSecretRedactions([
5003
- ...Object.entries(sandboxGitTokens ?? {}).flatMap(([provider, value]) =>
5004
- value ? [{ name: `GIT_${provider.toUpperCase()}_TOKEN`, value }] : [],
5005
- ),
5006
- ...(sandboxGitToken ? [{ name: "GIT_TOKEN", value: sandboxGitToken }] : []),
5007
- ...((sandboxGitCredentialBindings ?? []).map((binding) => ({
5008
- name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
5009
- value: binding.token,
5010
- })) satisfies SecretForRedaction[]),
5011
- ...(sandboxToolspaceToken
5012
- ? [{ name: "TOOLSPACE_TOKEN", value: sandboxToolspaceToken }]
5013
- : []),
5014
- ]);
5055
+
5015
5056
  const sandboxToolspaceTokenFile = sandboxToolspaceToken
5016
5057
  ? toolspaceTokenFileFromEnvironment(sandboxEnvironment, input.sessionId)
5017
5058
  : undefined;
@@ -5057,12 +5098,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5057
5098
  );
5058
5099
  if (binding) {
5059
5100
  assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
5060
- registerSecretRedactions([
5061
- {
5062
- name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
5063
- value: binding.token,
5064
- },
5065
- ]);
5066
5101
  }
5067
5102
  pendingBinding = binding;
5068
5103
  return binding
@@ -5150,7 +5185,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5150
5185
  toolspaceAuthority,
5151
5186
  );
5152
5187
  if (material) {
5153
- registerSecretRedactions([{ name: "TOOLSPACE_TOKEN", value: material.token }]);
5154
5188
  }
5155
5189
  return material;
5156
5190
  };
@@ -5277,7 +5311,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5277
5311
  );
5278
5312
  return;
5279
5313
  }
5280
- registerSecretRedactions(material.redactions);
5314
+
5281
5315
  await runWorkspaceMutationForSandbox(
5282
5316
  requireTargetSandbox(),
5283
5317
  "runCredentialMaterialization",
@@ -5435,6 +5469,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5435
5469
  db,
5436
5470
  settings,
5437
5471
  bus,
5472
+ opJournal,
5438
5473
  onOp: machineOpObserver.observer,
5439
5474
  onSandboxOperation: sandboxOperationObserver,
5440
5475
  onHomeSandboxRebound,
@@ -5544,6 +5579,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5544
5579
  db,
5545
5580
  settings,
5546
5581
  bus,
5582
+ opJournal,
5547
5583
  onSandboxOperation: sandboxOperationObserver,
5548
5584
  onHomeSandboxLost: publishSandboxLost,
5549
5585
  onHomeSandboxRebound,
@@ -5628,9 +5664,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5628
5664
  const resolveCredential: typeof rawResolveCredential = async (request) => {
5629
5665
  const result = await resolveFrozenCredential(request);
5630
5666
  if (result.status === "ok") {
5631
- registerSecretRedactions(
5632
- headerSecretRedactions("MCP", result.headers, Object.keys(result.headers)),
5633
- );
5634
5667
  }
5635
5668
  return result;
5636
5669
  };
@@ -5652,9 +5685,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5652
5685
  }) => Promise<T>,
5653
5686
  ): Promise<T> => {
5654
5687
  const snapshot = await resolver.getToken();
5655
- registerSecretRedactions([
5656
- { name: "CODEX_APPS_ACCESS_TOKEN", value: snapshot.accessToken },
5657
- ]);
5688
+
5658
5689
  return await withCodexAppsRequestAuthorization(
5659
5690
  db,
5660
5691
  { workspaceId: input.workspaceId, credentialId: codexAppsCredentialId },
@@ -5769,6 +5800,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5769
5800
  latencyMode: turnExecutionPolicy.latencyMode,
5770
5801
  ...(serviceTier ? { serviceTier } : {}),
5771
5802
  ...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
5803
+ humanInputEnabled: agentHumanInputEnabled,
5772
5804
  genesisTitleHint: isGenesisTurn,
5773
5805
  persistentSessionSettings: {
5774
5806
  titleIsSet: Boolean(session.title?.trim()),
@@ -5793,14 +5825,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5793
5825
  ...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
5794
5826
  ? { gitTokenSeed: sandboxGitToken }
5795
5827
  : {}),
5796
- // Toolspace is delivered on EVERY backend including selfhosted. The git-
5797
- // token skip does NOT transfer: that token is inert on a connected
5798
- // machine (it uses its own git creds), but the toolspace token is the
5799
- // machine's ONLY path to programmatic tool calling and grants no more
5800
- // than toolspace:call for its own session (own-session-bound,
5801
- // short-lived and renewable, per-active-turn budgeted, approval-tools
5802
- // excluded). The runtime seeds it to the box's token file over the same
5803
- // exec channel, off-manifest, on every backend.
5828
+ // Toolspace delivery is managed-sandbox-only. Connected Machines own any
5829
+ // manually configured API credentials and must never be contacted during
5830
+ // turn admission merely to seed OpenGeni tooling.
5804
5831
  ...(sandboxToolspaceToken
5805
5832
  ? {
5806
5833
  toolspaceTokenSeed: sandboxToolspaceToken,
@@ -6108,6 +6135,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6108
6135
  db,
6109
6136
  settings,
6110
6137
  bus,
6138
+ opJournal,
6111
6139
  onOp: machineOpObserver.observer,
6112
6140
  onSandboxOperation: sandboxOperationObserver,
6113
6141
  onHomeSandboxLost: publishSandboxLost,
@@ -6206,7 +6234,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6206
6234
  },
6207
6235
  );
6208
6236
  if (!isCompactionSummaryFailure(error)) throw error;
6209
- const errorMessage = String(redact(compactionFailureReasonFromError(error)));
6237
+ const errorMessage = String(compactionFailureReasonFromError(error));
6210
6238
  if (
6211
6239
  !(await settle!({
6212
6240
  events: [
@@ -6308,11 +6336,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6308
6336
  force: true,
6309
6337
  clearRequestedCompaction: true,
6310
6338
  trigger: "operator",
6339
+ materializeHistory: materializeScreenshotHistory,
6311
6340
  projectModelInput: compactionModelHistoryProjector,
6312
6341
  ...compactionModeOptions,
6313
6342
  }
6314
6343
  : {
6315
6344
  trigger: "auto",
6345
+ materializeHistory: materializeScreenshotHistory,
6316
6346
  projectModelInput: compactionModelHistoryProjector,
6317
6347
  ...compactionModeOptions,
6318
6348
  },
@@ -6346,11 +6376,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6346
6376
  },
6347
6377
  );
6348
6378
  if (!isCompactionSummaryFailure(compactError)) throw compactError;
6349
- const errorMessage = String(redact(compactionFailureReasonFromError(compactError)));
6379
+ const errorMessage = String(compactionFailureReasonFromError(compactError));
6350
6380
  observability.error("context compaction failed", {
6351
6381
  sessionId: input.sessionId,
6352
6382
  turnId,
6353
- error: errorMessage,
6383
+ ...safeErrorDiagnostic(compactError),
6354
6384
  });
6355
6385
  if (
6356
6386
  !(await settle!({
@@ -6465,6 +6495,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6465
6495
  ...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
6466
6496
  ...(runCredentialsNote ? { runCredentialsNote } : {}),
6467
6497
  providerApi,
6498
+ materializeModelHistory: materializeScreenshotHistory,
6499
+ materializeSerializedRunState: materializeScreenshotRunState,
6468
6500
  projectModelHistory: modelHistoryProjector,
6469
6501
  });
6470
6502
  runInput = prepared.input;
@@ -6478,16 +6510,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6478
6510
  // total rows no longer equal max(position)+1. Pre-compaction both reduce to
6479
6511
  // the old total-count value, so the common path is unchanged.
6480
6512
  //
6481
- // CRITICAL: seed from the SANITIZED active-row length, not the raw active
6513
+ // CRITICAL: seed from the structurally repaired active-row length, not the raw active
6482
6514
  // count. `prepareRunInput` builds `state.history` from
6483
6515
  // `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
6484
6516
  // rows (a legacy orphan/dangling pair), the in-memory history this turn
6485
6517
  // starts from is K shorter than the raw row count. The reconcile slices the
6486
- // re-sanitized `state.history` off `persistedHistoryCount`; seeding it from
6518
+ // repaired `state.history` off `persistedHistoryCount`; seeding it from
6487
6519
  // the raw count (K too high) skips K genuinely-new items, and a
6488
6520
  // `function_call` left in that skipped region can later have its
6489
6521
  // `function_call_result` persisted alone — the orphan that 400s on replay
6490
- // and bricks the session (issue-61). The sanitized seed is already
6522
+ // and bricks the session (issue-61). The repaired seed is already
6491
6523
  // orphan-free, so it is a stable prefix of the re-sanitized history and the
6492
6524
  // slice begins exactly at the first genuinely-new item.
6493
6525
  persistedHistoryCount = prepared.persistedHistoryCount;
@@ -6525,6 +6557,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6525
6557
  force: true,
6526
6558
  ...(triggerLabel === "operator" ? { clearRequestedCompaction: true } : {}),
6527
6559
  trigger: triggerLabel,
6560
+ materializeHistory: materializeScreenshotHistory,
6528
6561
  projectModelInput: compactionModelHistoryProjector,
6529
6562
  ...compactionModeOptions,
6530
6563
  },
@@ -6573,6 +6606,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6573
6606
  // its first-operation provisioner above.
6574
6607
  if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
6575
6608
  const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
6609
+ // `deferredSetup: true` below tells the runtime that the worker owns
6610
+ // platform setup, so the runtime intentionally skips its credential
6611
+ // session callback. Materialize host-managed run credentials here
6612
+ // before any setup command (including provider login hooks), then
6613
+ // decorate setup commands so they source the active generation.
6614
+ await attachRunCredentialRenewal(
6615
+ eagerSetupSession as RunCredentialCommandSession,
6616
+ resolvedSandbox,
6617
+ );
6618
+ const eagerCredentialSetupSession = initialRunCredentialMaterial
6619
+ ? withRunCredentialsSession(eagerSetupSession as object, input.sessionId)
6620
+ : eagerSetupSession;
6576
6621
  await runWorkspaceMutationForSandbox(
6577
6622
  resolvedSandbox,
6578
6623
  "eagerOwnedSandboxSetup",
@@ -6580,7 +6625,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6580
6625
  await runOwnedSandboxSetup(
6581
6626
  agent,
6582
6627
  ownedEstablished.session as never,
6583
- eagerSetupSession as never,
6628
+ eagerCredentialSetupSession as never,
6584
6629
  {
6585
6630
  settings: runSettings,
6586
6631
  environment: sandboxEnvironment,
@@ -6652,9 +6697,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6652
6697
  tokenSession: ToolspaceTokenWriterSession,
6653
6698
  ) => {
6654
6699
  const renewalSession =
6655
- activeSandboxBackend === "selfhosted"
6656
- ? tokenSession
6657
- : ((setupBoxSession as ToolspaceTokenWriterSession | null) ?? tokenSession);
6700
+ (setupBoxSession as ToolspaceTokenWriterSession | null) ?? tokenSession;
6658
6701
  await attachToolspaceTokenRenewal(renewalSession);
6659
6702
  },
6660
6703
  }
@@ -6679,7 +6722,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6679
6722
  contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
6680
6723
  contextCompactionRequested: () =>
6681
6724
  isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
6682
- callModelInputFilter: secretRedactionModelInputFilter((value) => redact(value)),
6683
6725
  ...(toolCancellationFenceRef.current
6684
6726
  ? { turnToolCancellationFence: toolCancellationFenceRef.current }
6685
6727
  : {}),
@@ -6717,6 +6759,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6717
6759
  }
6718
6760
  let stableToolCallIdsToClear: string[] | null = null;
6719
6761
  let completedCurrentToolBatch = false;
6762
+ let retainedScreenshotMetadata: RetainedArtifactMetadata | null = null;
6720
6763
  const responseResult = await processModelResponseTerminalEvent({
6721
6764
  event: next.value,
6722
6765
  state: modelResponseState,
@@ -6770,7 +6813,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6770
6813
  // conversation context preserved for the post-top-up resume.
6771
6814
  let serializedRunState: string | null = null;
6772
6815
  try {
6773
- serializedRunState = String(redact(stream.state.toString()));
6816
+ serializedRunState = String(stream.state.toString());
6774
6817
  } catch {
6775
6818
  serializedRunState = null;
6776
6819
  }
@@ -6792,7 +6835,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6792
6835
  modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
6793
6836
  callId: pendingToolCall.callId,
6794
6837
  callType: pendingToolCall.callType,
6795
- callItem: redact(pendingToolCall.callItem) as Record<string, unknown>,
6838
+ callItem: pendingToolCall.callItem as Record<string, unknown>,
6796
6839
  });
6797
6840
  if (!registered.accepted) {
6798
6841
  throw new TurnAttemptFencedError(
@@ -6800,9 +6843,39 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6800
6843
  );
6801
6844
  }
6802
6845
  currentToolBatchCallIds.add(pendingToolCall.callId);
6846
+ if (pendingToolCall.callName === "computer_screenshot") {
6847
+ computerScreenshotCallIds.add(pendingToolCall.callId);
6848
+ }
6803
6849
  }
6804
6850
  const completedToolCall = completedToolCallFromSdkEvent(next.value);
6805
6851
  if (completedToolCall) {
6852
+ const typedScreenshot = typedScreenshotFromSdkEvent(next.value);
6853
+ if (
6854
+ typedScreenshot &&
6855
+ typedScreenshot.callId === completedToolCall.callId &&
6856
+ computerScreenshotCallIds.has(completedToolCall.callId)
6857
+ ) {
6858
+ retainedScreenshotMetadata = await retainComputerScreenshot({
6859
+ db,
6860
+ objectStorage,
6861
+ accountId: input.accountId,
6862
+ workspaceId: input.workspaceId,
6863
+ sessionId: input.sessionId,
6864
+ turnId: activeTurnId,
6865
+ attemptId: input.attemptId,
6866
+ output: typedScreenshot,
6867
+ });
6868
+ retainedScreenshotReceiptsByCallId.set(
6869
+ completedToolCall.callId,
6870
+ retainedScreenshotMetadata,
6871
+ );
6872
+ }
6873
+ const durableResultItem = retainedScreenshotMetadata
6874
+ ? (compactRetainedScreenshotHistory(
6875
+ [completedToolCall.resultItem],
6876
+ retainedScreenshotReceiptsByCallId,
6877
+ )[0] ?? completedToolCall.resultItem)
6878
+ : completedToolCall.resultItem;
6806
6879
  // Keep every parallel result in the attempt ledger until the full
6807
6880
  // call batch has settled. The SDK's computed history is
6808
6881
  // non-monotonic while parallel calls complete (a later call may
@@ -6817,7 +6890,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6817
6890
  attemptId: input.attemptId,
6818
6891
  callId: completedToolCall.callId,
6819
6892
  modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
6820
- resultItem: redact(completedToolCall.resultItem) as Record<string, unknown>,
6893
+ resultItem: durableResultItem as Record<string, unknown>,
6821
6894
  });
6822
6895
  if (!recorded.accepted) {
6823
6896
  throw new TurnAttemptFencedError(
@@ -6846,7 +6919,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6846
6919
  completedCurrentToolBatch = currentBatchIsStable;
6847
6920
  }
6848
6921
  }
6849
- const normalized = normalizeSdkEvent(next.value);
6922
+ const normalized = normalizeSdkEvent(
6923
+ next.value,
6924
+ retainedScreenshotMetadata
6925
+ ? {
6926
+ toolOutputOverride: retainedScreenshotMetadata,
6927
+ retainedOutputEvidence: retainedScreenshotMetadata.available
6928
+ ? retainedScreenshotMetadata
6929
+ : {
6930
+ available: false,
6931
+ reason: retainedScreenshotMetadata.reason,
6932
+ },
6933
+ }
6934
+ : {},
6935
+ );
6850
6936
  for (const event of normalized) {
6851
6937
  streamTiming.onEvent(event.type);
6852
6938
  await batcher.push(event);
@@ -6870,6 +6956,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6870
6956
  currentToolBatchCallIds = new Set<string>();
6871
6957
  currentToolBatchCompletedCallIds = new Set<string>();
6872
6958
  }
6959
+ for (const callId of stableToolCallIdsToClear) {
6960
+ computerScreenshotCallIds.delete(callId);
6961
+ }
6873
6962
  }
6874
6963
  }
6875
6964
  } finally {
@@ -6983,6 +7072,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6983
7072
  const approvals = runtime.serializeApprovals(stream.interruptions);
6984
7073
  const humanInputInterruptions =
6985
7074
  runtime.serializeHumanInputRequests?.(stream.interruptions) ?? [];
7075
+ const latestWorkspace = await getWorkspace(db, input.workspaceId);
7076
+ if (!latestWorkspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
7077
+ assertWorkspaceHumanInputAllowed(
7078
+ resolveWorkspaceAgentHumanInputEnabled(latestWorkspace.settings),
7079
+ "interruption",
7080
+ humanInputInterruptions.length > 0,
7081
+ );
6986
7082
  const humanInputRequests = await Promise.all(
6987
7083
  humanInputInterruptions.map(async (interruption) => {
6988
7084
  const id = stableHumanInputRequestId(
@@ -7058,25 +7154,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7058
7154
  ) {
7059
7155
  return claimedResult({ status: "cancelled" });
7060
7156
  }
7157
+ // The interruption and its preceding tool results are now durable.
7158
+ await finalizeTurnOpStreamOps();
7061
7159
  activityStatus = "requires_action";
7062
7160
  return claimedResult({ status: "requires_action" });
7063
7161
  }
7064
7162
 
7065
- const finalOutput = String(redact(String(stream.finalOutput ?? "")));
7066
- await reconcileConversationTruth();
7163
+ const finalOutput = String(stream.finalOutput ?? "");
7164
+ await reconcileConversationTruth({ requireDurable: true });
7067
7165
  // Op-stream durability fence: the tool outputs are now durably in the
7068
7166
  // history store (a redispatch would NOT re-execute them), so this
7069
7167
  // turn's settled ops may advance their acked frontier — journal persist
7070
7168
  // then wire final ack (licensing the runner to GC its retained
7071
7169
  // frames). Best-effort: a miss leaves the runner's retention TTL to
7072
7170
  // reap, never fails a completed turn.
7073
- if (machinePrimarySession) {
7074
- try {
7075
- await machinePrimarySession.finalizeOpStreamOps();
7076
- } catch {
7077
- // The runner's retention TTL owns the fallback.
7078
- }
7079
- }
7171
+ await finalizeTurnOpStreamOps();
7080
7172
  if (
7081
7173
  !(await settle!({
7082
7174
  events: [
@@ -7143,8 +7235,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7143
7235
  sessionId: input.sessionId,
7144
7236
  turnId: activeTurnId,
7145
7237
  reason: recoveryKind,
7146
- code: overflow?.code ? String(redact(overflow.code)) : undefined,
7147
- error: String(redact(overflow?.message ?? compactionNeeded?.message ?? "")),
7238
+ ...safeErrorDiagnostic(attemptError),
7148
7239
  signalTokens: compactionNeeded?.signalTokens,
7149
7240
  thresholdTokens: compactionNeeded?.thresholdTokens,
7150
7241
  });
@@ -7189,13 +7280,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7189
7280
  );
7190
7281
  compactionRequestCleared = landmark.requestConsumed;
7191
7282
  if (!isCompactionSummaryFailure(compactError)) throw compactError;
7192
- compactionFailureMessage = String(
7193
- redact(compactionFailureReasonFromError(compactError)),
7194
- );
7283
+ compactionFailureMessage = String(compactionFailureReasonFromError(compactError));
7195
7284
  observability.warn("context compaction recovery compaction failed", {
7196
7285
  sessionId: input.sessionId,
7197
7286
  turnId: activeTurnId,
7198
- error: compactionFailureMessage,
7287
+ ...safeErrorDiagnostic(compactError),
7199
7288
  });
7200
7289
  }
7201
7290
  if (!compactionHandled) {
@@ -7306,6 +7395,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7306
7395
  turnMetricOutcome = "cancelled";
7307
7396
  return claimedResult({ status: "cancelled" });
7308
7397
  }
7398
+ acknowledgeRecoveryQuiescence();
7309
7399
  await publishDurableSessionEvents(
7310
7400
  bus,
7311
7401
  input.workspaceId,
@@ -7325,7 +7415,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7325
7415
  } catch (recoveryError) {
7326
7416
  console.error(
7327
7417
  "sandbox lease supersession recovery failed",
7328
- safeErrorDiagnostic(recoveryError, (value) => String(redact(value))),
7418
+ safeErrorDiagnostic(recoveryError),
7329
7419
  );
7330
7420
  throw recoveryError;
7331
7421
  }
@@ -7357,6 +7447,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7357
7447
  turnMetricOutcome = "cancelled";
7358
7448
  return claimedResult({ status: "cancelled" });
7359
7449
  }
7450
+ acknowledgeRecoveryQuiescence();
7360
7451
  await publishDurableSessionEvents(
7361
7452
  bus,
7362
7453
  input.workspaceId,
@@ -7372,7 +7463,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7372
7463
  } catch (recoveryError) {
7373
7464
  console.error(
7374
7465
  "sandbox deadline rotation recovery failed",
7375
- safeErrorDiagnostic(recoveryError, (value) => String(redact(value))),
7466
+ safeErrorDiagnostic(recoveryError),
7376
7467
  );
7377
7468
  throw recoveryError;
7378
7469
  }
@@ -7406,6 +7497,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7406
7497
  turnMetricOutcome = "cancelled";
7407
7498
  return claimedResult({ status: "cancelled" });
7408
7499
  }
7500
+ acknowledgeRecoveryQuiescence();
7409
7501
  await publishDurableSessionEvents(
7410
7502
  bus,
7411
7503
  input.workspaceId,
@@ -7421,7 +7513,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7421
7513
  // the turn through a second cancellation path.
7422
7514
  console.error(
7423
7515
  "worker-shutdown recovery checkpoint failed",
7424
- safeErrorDiagnostic(recoveryError, (value) => String(redact(value))),
7516
+ safeErrorDiagnostic(recoveryError),
7425
7517
  );
7426
7518
  throw recoveryError;
7427
7519
  }
@@ -7520,11 +7612,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7520
7612
  await reconcileConversationTruth({ requireDurable: true });
7521
7613
  }
7522
7614
  checkpointDurable = true;
7523
- } catch (checkpointError) {
7615
+ } catch {
7524
7616
  observability.warn("Codex lease-loss checkpoint failed; refusing automatic turn replay", {
7525
- workspaceId: input.workspaceId,
7526
- turnId: lostTurnId,
7527
- errorName: checkpointError instanceof Error ? checkpointError.name : "unknown",
7617
+ errorClass: "CodexCheckpointOperationError",
7618
+ errorCode: "codex_lease_loss_checkpoint_failed",
7619
+ origin: "worker",
7528
7620
  });
7529
7621
  }
7530
7622
 
@@ -7631,16 +7723,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7631
7723
  await flushRuntimeBatcher();
7632
7724
  await reconcileConversationTruth({ requireDurable: true });
7633
7725
  checkpointDurable = true;
7634
- } catch (checkpointError) {
7726
+ } catch {
7635
7727
  observability.incrementCounter({
7636
7728
  name: "opengeni_codex_failover_checkpoints_total",
7637
7729
  help: "Durable Codex failover checkpoint attempts by outcome.",
7638
7730
  labels: { workspace_key: codexWorkspaceKey, outcome: "failed" },
7639
7731
  });
7640
7732
  observability.warn("Codex failover checkpoint failed; refusing automatic replay", {
7641
- workspaceId: input.workspaceId,
7642
- turnId,
7643
- errorName: checkpointError instanceof Error ? checkpointError.name : "unknown",
7733
+ errorClass: "CodexCheckpointOperationError",
7734
+ errorCode: "codex_failover_checkpoint_failed",
7735
+ origin: "worker",
7644
7736
  });
7645
7737
  }
7646
7738
 
@@ -8193,6 +8285,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8193
8285
  return claimedResult({ status: "cancelled" });
8194
8286
  }
8195
8287
  if (recovery.action === "recovering") {
8288
+ acknowledgeRecoveryQuiescence();
8196
8289
  await publishDurableSessionEvents(
8197
8290
  bus,
8198
8291
  input.workspaceId,
@@ -8211,15 +8304,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8211
8304
  // truth, recover this SAME accepted turn, then let the workflow re-claim
8212
8305
  // it after a pacing delay. This is independent of goal state and never
8213
8306
  // relies on a synthetic continuation prompt.
8214
- const failure = redact(agentRunFailurePayload(error, { isCodexTurn })) as ReturnType<
8307
+ const failure = agentRunFailurePayload(error, { isCodexTurn }) as ReturnType<
8215
8308
  typeof agentRunFailurePayload
8216
8309
  >;
8217
8310
  if (isSessionEventPersistenceError(error)) {
8218
- // Never pass the original Drizzle/postgres-js error to telemetry: its
8219
- // nested cause may contain raw SQL and bound parameters. The typed DB
8220
- // boundary retains only SQLSTATE, stage, correlation, and safe catalog
8221
- // identifiers. Provider inference has already happened and is NOT
8222
- // retried by this terminal classification.
8311
+ // Preserve the exact source message in the internal runtime diagnostic;
8312
+ // SQLSTATE/catalog facts remain separate classification attributes.
8223
8313
  observability.error("session event persistence failed", {
8224
8314
  accountId: input.accountId,
8225
8315
  workspaceId: input.workspaceId,
@@ -8240,6 +8330,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8240
8330
  dbDataType: error.details.database.dataType,
8241
8331
  dbConstraint: error.details.database.constraint,
8242
8332
  dbRoutine: error.details.database.routine,
8333
+ error: error.message,
8243
8334
  });
8244
8335
  }
8245
8336
  if (failure.retryable && publish && turnId && turnStartedPublished) {
@@ -8271,6 +8362,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8271
8362
  turnMetricOutcome = "cancelled";
8272
8363
  return claimedResult({ status: "cancelled" });
8273
8364
  }
8365
+ acknowledgeRecoveryQuiescence();
8274
8366
  await publishDurableSessionEvents(
8275
8367
  bus,
8276
8368
  input.workspaceId,
@@ -8376,18 +8468,38 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8376
8468
  const credentialSessionToClear = runCredentialSession;
8377
8469
  runCredentialSession = null;
8378
8470
  if (credentialSessionToClear) {
8379
- await runWorkspaceMutationForSandbox(
8380
- requireResolvedSandboxForMutation(
8381
- "Run credential cleanup has no exact sandbox lease target",
8382
- ),
8383
- "runCredentialAttemptClear",
8384
- async () =>
8385
- await clearRunCredentialsForAttempt(credentialSessionToClear, {
8386
- sessionId: input.sessionId,
8387
- attemptId: input.attemptId,
8388
- executionGeneration,
8389
- }),
8390
- );
8471
+ const clearAttemptCredentials = async (): Promise<void> =>
8472
+ await clearRunCredentialsForAttempt(credentialSessionToClear, {
8473
+ sessionId: input.sessionId,
8474
+ attemptId: input.attemptId,
8475
+ executionGeneration,
8476
+ });
8477
+ await clearAttemptCredentialsWithSettledFence({
8478
+ activityStatus,
8479
+ runWorkspaceFencedClear: async () =>
8480
+ await runWorkspaceMutationForSandbox(
8481
+ requireResolvedSandboxForMutation(
8482
+ "Run credential cleanup has no exact sandbox lease target",
8483
+ ),
8484
+ "runCredentialAttemptClear",
8485
+ clearAttemptCredentials,
8486
+ ),
8487
+ clearExactAttempt: clearAttemptCredentials,
8488
+ onSettledAttemptFence: () => {
8489
+ // Terminal settlement closes the active attempt before this finally
8490
+ // runs, so the ordinary workspace admission correctly rejects it.
8491
+ // The attempt-qualified delete is nevertheless successor-safe: it
8492
+ // removes only this attempt/generation and clears the pointer only
8493
+ // when it still names that exact generation. Finish that cleanup
8494
+ // directly so a successful turn can continue into workspace capture,
8495
+ // tool teardown, and lease release.
8496
+ observability.info("retrying exact run credential cleanup after turn settlement", {
8497
+ "opengeni.session_id": input.sessionId,
8498
+ "opengeni.turn_id": turnId ?? "",
8499
+ "opengeni.attempt_id": input.attemptId,
8500
+ });
8501
+ },
8502
+ });
8391
8503
  }
8392
8504
  await drainAttemptOwnedSandboxWriters({
8393
8505
  // Normal turn completion owns the same process boundary as
@@ -8455,18 +8567,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8455
8567
  onReceiptFailure: (error) => {
8456
8568
  console.error(
8457
8569
  "agent turn quiescence receipt exhausted; signalling proof",
8458
- safeErrorDiagnostic(error, (value) => String(redact(value))),
8570
+ safeErrorDiagnostic(error),
8459
8571
  );
8460
8572
  },
8461
8573
  onPublishFailure: (error) => {
8462
8574
  console.error(
8463
8575
  "agent turn quiescence event fanout failed",
8464
- safeErrorDiagnostic(error, (value) => String(redact(value))),
8576
+ safeErrorDiagnostic(error),
8465
8577
  );
8466
8578
  },
8467
8579
  onSignalFailure: (error, attempt, retryMs) => {
8468
8580
  console.error("agent turn quiescence proof signal failed; retrying", {
8469
- error: safeErrorDiagnostic(error, (value) => String(redact(value))),
8581
+ error: safeErrorDiagnostic(error),
8470
8582
  attempt,
8471
8583
  retryMs,
8472
8584
  });
@@ -8714,7 +8826,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8714
8826
  sandboxToRelease.release().catch((releaseError) => {
8715
8827
  console.error(
8716
8828
  "sandbox lease release failed (turn outcome unaffected)",
8717
- safeErrorDiagnostic(releaseError, (value) => String(redact(value))),
8829
+ safeErrorDiagnostic(releaseError),
8718
8830
  );
8719
8831
  }),
8720
8832
  finalizerSignal,
@@ -8724,7 +8836,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8724
8836
  finalizationError ??= error;
8725
8837
  console.error(
8726
8838
  "agent turn finalization failed (turn outcome unaffected)",
8727
- safeErrorDiagnostic(error, (value) => String(redact(value))),
8839
+ safeErrorDiagnostic(error),
8728
8840
  );
8729
8841
  } finally {
8730
8842
  // If fallible finalization exited before the ordinary exact-holder
@@ -8785,9 +8897,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8785
8897
  },
8786
8898
  error:
8787
8899
  finalizationError || activityError
8788
- ? safeErrorForTelemetry(finalizationError ?? activityError, (value) =>
8789
- String(redact(value)),
8790
- )
8900
+ ? safeErrorForTelemetry(finalizationError ?? activityError)
8791
8901
  : undefined,
8792
8902
  });
8793
8903
  assertPhysicalToolQuiescenceForCancellation({
@@ -8919,7 +9029,19 @@ export function agentRunFailurePayload(
8919
9029
  attempts?: number;
8920
9030
  retryOutcome?: string;
8921
9031
  database?: Record<string, string>;
9032
+ historyPersistenceStage?: MandatoryHistoryPersistenceStage;
8922
9033
  } {
9034
+ if (error instanceof MandatoryHistoryPersistenceError) {
9035
+ const underlying = isSessionEventPersistenceError(error.cause)
9036
+ ? agentRunFailurePayload(error.cause, options)
9037
+ : {
9038
+ error: error.cause instanceof Error ? error.cause.message : String(error.cause),
9039
+ };
9040
+ return {
9041
+ ...underlying,
9042
+ historyPersistenceStage: error.stage,
9043
+ };
9044
+ }
8923
9045
  const message = error instanceof Error ? error.message : String(error);
8924
9046
  const status =
8925
9047
  typeof error === "object" && error !== null
@@ -8945,15 +9067,8 @@ export function agentRunFailurePayload(
8945
9067
  }
8946
9068
  if (isSessionEventPersistenceError(error)) {
8947
9069
  const { details } = error;
8948
- const eventLabel = details.eventTypes.join(", ") || "session events";
8949
- const failureLabel =
8950
- details.code === "db_deadlock"
8951
- ? "Database deadlock"
8952
- : details.code === "db_serialization_failure"
8953
- ? "Database serialization failure"
8954
- : "Database failure";
8955
9070
  return {
8956
- error: `${failureLabel} while persisting ${eventLabel}. The completed provider call and external effects were not retried.`,
9071
+ error: error.message,
8957
9072
  code: details.code,
8958
9073
  detail:
8959
9074
  details.retryOutcome === "exhausted"
@@ -9015,6 +9130,7 @@ export function agentRunFailurePayload(
9015
9130
  "A required MCP server was temporarily unreachable. The same turn will retry after a short delay.",
9016
9131
  code: "mcp_transport_unavailable",
9017
9132
  retryable: true,
9133
+ detail: message,
9018
9134
  };
9019
9135
  }
9020
9136
  if (
@@ -9232,9 +9348,10 @@ export function codexUsageLimitFailurePayload(
9232
9348
  // goal re-evaluates at most this far out (it will re-pause if still capped).
9233
9349
  const CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 60_000; // 1h
9234
9350
 
9235
- function pendingToolCallFromSdkEvent(event: unknown): {
9351
+ export function pendingToolCallFromSdkEvent(event: unknown): {
9236
9352
  callId: string;
9237
9353
  callType: string;
9354
+ callName: string | null;
9238
9355
  callItem: Record<string, unknown>;
9239
9356
  } | null {
9240
9357
  if (!event || typeof event !== "object") return null;
@@ -9255,10 +9372,16 @@ function pendingToolCallFromSdkEvent(event: unknown): {
9255
9372
  if (typeof callId !== "string" || callId.length === 0 || typeof callType !== "string") {
9256
9373
  return null;
9257
9374
  }
9258
- return { callId, callType, callItem: raw };
9375
+ const callItem = normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]');
9376
+ return {
9377
+ callId,
9378
+ callType,
9379
+ callName: typeof raw.name === "string" ? raw.name : null,
9380
+ callItem,
9381
+ };
9259
9382
  }
9260
9383
 
9261
- function completedToolCallFromSdkEvent(event: unknown): {
9384
+ export function completedToolCallFromSdkEvent(event: unknown): {
9262
9385
  callId: string;
9263
9386
  resultItem: Record<string, unknown>;
9264
9387
  } | null {
@@ -9273,7 +9396,11 @@ function completedToolCallFromSdkEvent(event: unknown): {
9273
9396
  ? (item.rawItem as Record<string, unknown>)
9274
9397
  : {};
9275
9398
  const callId = raw.callId ?? raw.call_id ?? item.id;
9276
- return typeof callId === "string" && callId.length > 0 ? { callId, resultItem: raw } : null;
9399
+ if (typeof callId !== "string" || callId.length === 0) return null;
9400
+ return {
9401
+ callId,
9402
+ resultItem: normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]'),
9403
+ };
9277
9404
  }
9278
9405
 
9279
9406
  /**
@@ -9574,12 +9701,7 @@ export async function recordAuthoritativeModelCallFact(input: {
9574
9701
  });
9575
9702
  } catch (error) {
9576
9703
  input.observability.warn("model call fact persist failed", {
9577
- accountId: input.accountId,
9578
- workspaceId: input.workspaceId,
9579
- sessionId: input.sessionId,
9580
- turnId: input.turnId,
9581
- sourceKey: input.sourceKey,
9582
- error: error instanceof Error ? error.message : String(error),
9704
+ ...safeErrorDiagnostic(error),
9583
9705
  });
9584
9706
  }
9585
9707
  }