@opengeni/worker-bundle 0.16.21 → 0.16.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/activities/agent-turn.d.ts +61 -14
  2. package/dist/activities/context-compaction.d.ts +2 -0
  3. package/dist/activities/environment.d.ts +1 -0
  4. package/dist/activities/expiring-material-renewal.d.ts +5 -3
  5. package/dist/activities/git-credential-renewal.d.ts +1 -1
  6. package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
  7. package/dist/activities/retained-screenshots.d.ts +93 -0
  8. package/dist/activities/run-credential-renewal.d.ts +1 -1
  9. package/dist/activities/run-input.d.ts +2 -0
  10. package/dist/activities/toolspace-token-renewal.d.ts +1 -1
  11. package/dist/activities/workspace-capture.d.ts +1 -0
  12. package/dist/activities.d.ts +3 -0
  13. package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
  14. package/dist/chunk-RAMNYSAW.js.map +1 -0
  15. package/dist/host-export.js +1 -1
  16. package/dist/index.js +1317 -390
  17. package/dist/index.js.map +1 -1
  18. package/dist/sandbox-resume.d.ts +6 -0
  19. package/dist/sandbox-routing.d.ts +11 -6
  20. package/dist/workflow-bundle.js +2 -1
  21. package/package.json +12 -12
  22. package/src/activities/agent-turn.ts +482 -360
  23. package/src/activities/codex-capacity.ts +10 -1
  24. package/src/activities/codex-rotation.ts +19 -5
  25. package/src/activities/context-compaction.ts +9 -3
  26. package/src/activities/environment.ts +13 -24
  27. package/src/activities/expiring-material-renewal.ts +16 -7
  28. package/src/activities/git-credential-renewal.ts +3 -3
  29. package/src/activities/recording.ts +4 -5
  30. package/src/activities/retained-screenshot-reaper.ts +202 -0
  31. package/src/activities/retained-screenshots.ts +892 -0
  32. package/src/activities/rig-verification.ts +13 -34
  33. package/src/activities/run-credential-renewal.ts +9 -2
  34. package/src/activities/run-input.ts +18 -4
  35. package/src/activities/scheduled-tasks.ts +19 -3
  36. package/src/activities/toolspace-token-renewal.ts +9 -2
  37. package/src/activities/workspace-capture.ts +33 -3
  38. package/src/activities.ts +10 -2
  39. package/src/host-export-pump.ts +28 -11
  40. package/src/http.ts +3 -6
  41. package/src/observability-metrics.ts +3 -3
  42. package/src/sandbox-resume.ts +45 -12
  43. package/src/sandbox-routing.ts +30 -15
  44. package/src/worker-service-lifecycle.ts +8 -4
  45. package/src/workflows/file-upload-reaper.ts +4 -1
  46. package/dist/activities/redaction.d.ts +0 -1
  47. package/dist/chunk-BJUW5LCM.js.map +0 -1
  48. package/src/activities/redaction.ts +0 -9
package/dist/index.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  createHostExportPump
3
- } from "./chunk-BJUW5LCM.js";
3
+ } from "./chunk-RAMNYSAW.js";
4
4
 
5
5
  // src/index.ts
6
6
  import {
@@ -72,6 +72,7 @@ import {
72
72
  getCodexRotationSettings,
73
73
  getWorkspaceModelPolicy as getWorkspaceModelPolicy2,
74
74
  getWorkspaceGrant,
75
+ getWorkspace as getWorkspace2,
75
76
  listCodexAccountStatuses as listCodexAccountStatuses2,
76
77
  fetchCodexUsageForAccount as fetchCodexUsageForAccount2,
77
78
  getSessionCodexState,
@@ -110,7 +111,7 @@ import {
110
111
  SandboxLeaseSupersededError as SandboxLeaseSupersededError2,
111
112
  SandboxImageConflictError,
112
113
  isSessionEventPersistenceError,
113
- getEnrollment,
114
+ getEnrollment as getEnrollment2,
114
115
  abandonRecordingForTurnAttempt,
115
116
  markSessionAttemptQuiesced,
116
117
  getOrCreatePreferenceRegistrySnapshot,
@@ -130,6 +131,7 @@ import {
130
131
  modelTerminalResponseFromSdkEvent,
131
132
  normalizeModelCallUsage,
132
133
  normalizeSdkEvent,
134
+ normalizeProtocolJsonValue,
133
135
  projectHistoryForProvider as projectHistoryForProvider2,
134
136
  sanitizeHistoryItemsForModel as sanitizeHistoryItemsForModel2,
135
137
  projectModelInputForCapabilities,
@@ -591,10 +593,21 @@ function selectCodexCredentialLeaseForTurn(args) {
591
593
  };
592
594
  }
593
595
  if (!rotationEnabled) {
594
- const credentialId = activeCredentialId && connectedIds.has(activeCredentialId) ? activeCredentialId : null;
596
+ const active = activeCredentialId ? accounts.find((account) => account.id === activeCredentialId) : void 0;
597
+ if (!active?.allocatorEnabled) {
598
+ return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
599
+ }
600
+ if (!isCodexCredentialHealthy(active, args.now)) {
601
+ return {
602
+ credentialId: null,
603
+ decision: { kind: "allCapped", earliestResetAt: availableAt(active, args.now) },
604
+ advanceActivePointer: false
605
+ };
606
+ }
595
607
  return {
596
- credentialId,
597
- decision: credentialId ? { kind: "active", credentialId, moved: false } : { kind: "none" }
608
+ credentialId: active.id,
609
+ decision: { kind: "active", credentialId: active.id, moved: false },
610
+ advanceActivePointer: false
598
611
  };
599
612
  }
600
613
  if (!leasingEnabled && pinDisposition !== "clearStale" && args.sessionPinnedCredentialId) {
@@ -872,7 +885,9 @@ function codexCapacityDecision(context, settings) {
872
885
  sessionLastCredentialId: context.sessionLastCredentialId,
873
886
  now
874
887
  });
875
- if (selected.credentialId) {
888
+ const selectedAccount = selected.credentialId ? context.accounts.find((account) => account.id === selected.credentialId) : void 0;
889
+ const selectedIsAvailable = selectedAccount !== void 0 && (selectedAccount.id === context.existingCredentialId ? isCodexCredentialHealthy(selectedAccount, now) : isCodexCredentialEligible(selectedAccount, now));
890
+ if (selected.credentialId && selectedIsAvailable) {
876
891
  return {
877
892
  kind: "available",
878
893
  credentialId: selected.credentialId,
@@ -1327,6 +1342,7 @@ async function turnInput(db, runtime, agent, trigger, options) {
1327
1342
  joinInternalContext(internalContext, attachmentContext),
1328
1343
  fileAttachments.map((attachment) => attachment.resource),
1329
1344
  options.providerApi,
1345
+ options.materializeModelHistory,
1330
1346
  options.projectModelHistory
1331
1347
  );
1332
1348
  }
@@ -1343,6 +1359,7 @@ async function turnInput(db, runtime, agent, trigger, options) {
1343
1359
  internalContext,
1344
1360
  [],
1345
1361
  options.providerApi,
1362
+ options.materializeModelHistory,
1346
1363
  options.projectModelHistory
1347
1364
  );
1348
1365
  }
@@ -1352,9 +1369,10 @@ async function turnInput(db, runtime, agent, trigger, options) {
1352
1369
  if (!state) {
1353
1370
  throw new Error("No saved run state is available for approval decision");
1354
1371
  }
1372
+ const serializedRunState = resumeRunState(state);
1355
1373
  const prepared = await runtime.prepareInput(agent, {
1356
1374
  kind: "approval",
1357
- serializedRunState: resumeRunState(state),
1375
+ serializedRunState: options.materializeSerializedRunState ? await options.materializeSerializedRunState(serializedRunState) : serializedRunState,
1358
1376
  approvalId: String(payload.approvalId ?? ""),
1359
1377
  decision: payload.decision === "approve" ? "approve" : "reject",
1360
1378
  ...typeof payload.message === "string" ? { message: payload.message } : {}
@@ -1380,9 +1398,10 @@ async function turnInput(db, runtime, agent, trigger, options) {
1380
1398
  if (!resume) {
1381
1399
  throw new Error("Human-input response does not resolve to a durable request");
1382
1400
  }
1401
+ const serializedRunState = resumeRunState(state);
1383
1402
  const prepared = await runtime.prepareInput(agent, {
1384
1403
  kind: "human_input",
1385
- serializedRunState: resumeRunState(state),
1404
+ serializedRunState: options.materializeSerializedRunState ? await options.materializeSerializedRunState(serializedRunState) : serializedRunState,
1386
1405
  toolCallId: resume.toolCallId
1387
1406
  });
1388
1407
  return {
@@ -1401,13 +1420,14 @@ function joinInternalContext(...parts) {
1401
1420
  const content = parts.map((part) => part?.trim()).filter((part) => Boolean(part));
1402
1421
  return content.length > 0 ? content.join("\n\n") : void 0;
1403
1422
  }
1404
- async function messageInput(db, runtime, agent, trigger, text, internalContext, currentAttachmentRefs = [], providerApi = "responses", projectModelHistory) {
1423
+ async function messageInput(db, runtime, agent, trigger, text, internalContext, currentAttachmentRefs = [], providerApi = "responses", materializeModelHistory, projectModelHistory) {
1405
1424
  const stored = await getActiveSessionHistoryItems(db, trigger.workspaceId, trigger.sessionId);
1406
1425
  const envelope = await getSandboxSessionEnvelope(db, trigger.workspaceId, trigger.sessionId);
1407
1426
  const canonicalView = projectRejectedProviderArtifacts(stored);
1408
1427
  const providerView = projectHistoryForProvider(canonicalView, providerApi);
1409
1428
  const referencedHistory = withCurrentUserAttachmentRefs(providerView, currentAttachmentRefs);
1410
- const historyItems = projectModelHistory ? await projectModelHistory(referencedHistory) : referencedHistory;
1429
+ const materializedHistory = materializeModelHistory ? await materializeModelHistory(referencedHistory) : referencedHistory;
1430
+ const historyItems = projectModelHistory ? await projectModelHistory(materializedHistory) : materializedHistory;
1411
1431
  const prepared = await runtime.prepareInput(agent, {
1412
1432
  kind: "message",
1413
1433
  ...text ? { text } : {},
@@ -1491,10 +1511,13 @@ async function maybeCompactContext(db, settings, scope, lastInputTokens, summari
1491
1511
  };
1492
1512
  }
1493
1513
  const canonicalItems = projectRejectedProviderArtifacts(active);
1494
- const projectForWire = async (input) => sanitizeHistoryItemsForModel(
1495
- options.projectModelInput ? await options.projectModelInput(input) : input,
1496
- settings.modelToolOutputTruncationTokens
1497
- );
1514
+ const projectForWire = async (input) => {
1515
+ const materialized = options.materializeHistory ? await options.materializeHistory(input) : input;
1516
+ return sanitizeHistoryItemsForModel(
1517
+ options.projectModelInput ? await options.projectModelInput(materialized) : materialized,
1518
+ settings.modelToolOutputTruncationTokens
1519
+ );
1520
+ };
1498
1521
  const items = await projectForWire(canonicalItems);
1499
1522
  const decision = decideCompaction({
1500
1523
  items,
@@ -1744,6 +1767,7 @@ import {
1744
1767
  import {
1745
1768
  gitCredentialBindingIdForRepository,
1746
1769
  gitCredentialProviderForRepository,
1770
+ gitRemoteIdentity,
1747
1771
  signDelegatedAccessToken
1748
1772
  } from "@opengeni/contracts";
1749
1773
  import {
@@ -1809,8 +1833,13 @@ function assertWorkspaceEcho(kind, scope, echoedWorkspaceId) {
1809
1833
  async function sandboxEnvironmentForRun(settings, resources, workspaceEnvironment = {}, options = {}) {
1810
1834
  const stableOptions = options.scope ? { workspaceId: options.scope.workspaceId } : {};
1811
1835
  const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment, stableOptions);
1836
+ if (options.skipToolspace) {
1837
+ delete environment.OPENGENI_TOOLSPACE_URL;
1838
+ delete environment.OPENGENI_TOOLSPACE_TOKEN_FILE;
1839
+ delete environment.OPENGENI_OGTOOL_PACKAGE_SPEC;
1840
+ }
1812
1841
  const toolspaceScope = options.scope;
1813
- const toolspaceToken = toolspaceScope && options.toolspaceAuthority ? await mintSandboxToolspaceToken(settings, toolspaceScope, options.toolspaceAuthority) : void 0;
1842
+ const toolspaceToken = !options.skipToolspace && toolspaceScope && options.toolspaceAuthority ? await mintSandboxToolspaceToken(settings, toolspaceScope, options.toolspaceAuthority) : void 0;
1814
1843
  if (toolspaceToken && toolspaceScope) {
1815
1844
  environment.OPENGENI_TOOLSPACE_URL ??= firstPartyMcpWorkspaceUrl(
1816
1845
  settings,
@@ -2136,7 +2165,7 @@ function gitCredentialSelections(resources) {
2136
2165
  );
2137
2166
  }
2138
2167
  bindingProviders.set(credentialBindingId, provider);
2139
- const normalizedRemote = normalizedGitRemote(resource.uri);
2168
+ const normalizedRemote = gitRemoteIdentity(resource.uri, provider);
2140
2169
  const claimedBinding = remoteBindings.get(normalizedRemote);
2141
2170
  if (claimedBinding && claimedBinding !== bindingKey) {
2142
2171
  throw new Error(
@@ -2192,14 +2221,6 @@ function normalizedGitHost(uri) {
2192
2221
  return "";
2193
2222
  }
2194
2223
  }
2195
- function normalizedGitRemote(uri) {
2196
- try {
2197
- const url = new URL(uri);
2198
- return `${url.protocol.toLowerCase()}//${url.host.toLowerCase()}${url.pathname.replace(/\/+$/, "").replace(/\.git$/, "")}`;
2199
- } catch {
2200
- return uri.trim().replace(/\/+$/, "").replace(/\.git$/, "").toLowerCase();
2201
- }
2202
- }
2203
2224
  function repositoryCredentialProvider(resource) {
2204
2225
  return gitCredentialProviderForRepository(resource);
2205
2226
  }
@@ -2302,7 +2323,7 @@ function startGitCredentialRenewalLoop(options) {
2302
2323
  } catch {
2303
2324
  }
2304
2325
  scheduleRefresh(nextDelayMs);
2305
- } catch (error) {
2326
+ } catch {
2306
2327
  if (stopped) return;
2307
2328
  const scheduledRetryMs = retryDelayMs;
2308
2329
  retryDelayMs = Math.min(retryDelayMs * 2, GIT_CREDENTIAL_MAX_RETRY_MS);
@@ -2310,7 +2331,7 @@ function startGitCredentialRenewalLoop(options) {
2310
2331
  options.onFailure?.({
2311
2332
  providers,
2312
2333
  retryDelayMs: scheduledRetryMs,
2313
- errorClass: error instanceof Error ? error.name : "UnknownError"
2334
+ errorClass: "GitCredentialRenewalOperationError"
2314
2335
  });
2315
2336
  } catch {
2316
2337
  }
@@ -2401,14 +2422,14 @@ function startExpiringMaterialRenewalLoop(options) {
2401
2422
  } catch {
2402
2423
  }
2403
2424
  scheduleRefresh(nextDelayMs);
2404
- } catch (error) {
2425
+ } catch {
2405
2426
  if (stopped) return;
2406
2427
  const scheduledRetryMs = retryDelayMs;
2407
2428
  retryDelayMs = Math.min(retryDelayMs * 2, options.policy.maxRetryMs);
2408
2429
  try {
2409
2430
  options.onFailure?.({
2410
2431
  retryDelayMs: scheduledRetryMs,
2411
- errorClass: error instanceof Error ? error.name : "UnknownError"
2432
+ errorClass: options.publicErrorClass
2412
2433
  });
2413
2434
  } catch {
2414
2435
  }
@@ -2450,6 +2471,7 @@ function startRunCredentialRenewalLoop(options) {
2450
2471
  resolve: options.resolve,
2451
2472
  write: options.write,
2452
2473
  expiresAt: (material) => material?.expiresAt ?? null,
2474
+ publicErrorClass: "RunCredentialRenewalOperationError",
2453
2475
  policy: {
2454
2476
  defaultRefreshMs: RUN_CREDENTIAL_DEFAULT_REFRESH_MS,
2455
2477
  expiryLeadMs: RUN_CREDENTIAL_EXPIRY_LEAD_MS,
@@ -2489,6 +2511,7 @@ function startToolspaceTokenRenewalLoop(options) {
2489
2511
  },
2490
2512
  write: options.write,
2491
2513
  expiresAt: (material) => material.expiresAt,
2514
+ publicErrorClass: "ToolspaceTokenRenewalOperationError",
2492
2515
  policy: {
2493
2516
  defaultRefreshMs: TOOLSPACE_TOKEN_DEFAULT_REFRESH_MS,
2494
2517
  expiryLeadMs: TOOLSPACE_TOKEN_EXPIRY_LEAD_MS,
@@ -2891,10 +2914,6 @@ function workspacePackRuntimeFromPacks(packs) {
2891
2914
  skills
2892
2915
  };
2893
2916
  }
2894
- async function resolveWorkspaceAgentInstructions(db, workspaceId) {
2895
- const workspace = await getWorkspace(db, workspaceId);
2896
- return workspace?.agentInstructions ?? null;
2897
- }
2898
2917
  function settingsWithPackSandboxImage(settings, sandboxImage, sandboxProviderImages = null) {
2899
2918
  if (!sandboxImage) {
2900
2919
  return settings;
@@ -3149,16 +3168,6 @@ function childCompletionSummary(child, goal, terminalStatus) {
3149
3168
  return lines.join("\n");
3150
3169
  }
3151
3170
 
3152
- // src/activities/redaction.ts
3153
- import {
3154
- createSecretRedactor,
3155
- identityRedactor,
3156
- isCredentialHeaderName,
3157
- redactSensitiveData,
3158
- redactSensitiveText,
3159
- redactSerializedJson
3160
- } from "@opengeni/contracts";
3161
-
3162
3171
  // src/activities/streaming.ts
3163
3172
  import { Context } from "@temporalio/activity";
3164
3173
  var TRAILING_FLUSH_MS = 33;
@@ -3309,7 +3318,6 @@ var ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE = "MCP request timeout recovery
3309
3318
 
3310
3319
  // src/sandbox-resume.ts
3311
3320
  import { sandboxArchiveCaptureTimeoutMs } from "@opengeni/config";
3312
- import { redactSensitiveText as redactSensitiveText2 } from "@opengeni/contracts";
3313
3321
  import { randomUUID } from "crypto";
3314
3322
  import {
3315
3323
  acquireLease,
@@ -3397,14 +3405,18 @@ var SnapshotTimeoutError = class extends Error {
3397
3405
  }
3398
3406
  };
3399
3407
  function safeSnapshotError(error) {
3400
- const rawName = error instanceof Error ? error.name : "Error";
3401
- return {
3402
- name: /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error",
3403
- message: redactSensitiveText2(error instanceof Error ? error.message : String(error)).slice(
3404
- 0,
3405
- 2048
3406
- )
3408
+ const fields = {
3409
+ errorClass: "SnapshotOperationError",
3410
+ errorCode: "snapshot_operation_failed",
3411
+ origin: "sandbox-resume"
3407
3412
  };
3413
+ try {
3414
+ const rawStatus = error && typeof error === "object" ? error.status ?? error.statusCode : void 0;
3415
+ const status = Number(rawStatus);
3416
+ if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
3417
+ } catch {
3418
+ }
3419
+ return fields;
3408
3420
  }
3409
3421
  async function waitForWarmSnapshot(snapshot, timeoutMs, signal) {
3410
3422
  if (signal?.aborted) return false;
@@ -3571,6 +3583,10 @@ async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEp
3571
3583
  if (typeof persistable.persistWorkspace !== "function") {
3572
3584
  return false;
3573
3585
  }
3586
+ const workspacePersistence = persistable.state?.workspacePersistence ?? persistable.state?.providerState?.workspacePersistence;
3587
+ if (!force && workspacePersistence === "snapshot_filesystem") {
3588
+ return false;
3589
+ }
3574
3590
  if (signal?.aborted) {
3575
3591
  return false;
3576
3592
  }
@@ -3580,7 +3596,7 @@ async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEp
3580
3596
  return false;
3581
3597
  }
3582
3598
  const instanceId = lease.instanceId;
3583
- const nativeModalPersistence = persistable.state?.workspacePersistence === "snapshot_filesystem" || persistable.state?.workspacePersistence === "snapshot_directory";
3599
+ const nativeModalPersistence = workspacePersistence === "snapshot_filesystem" || workspacePersistence === "snapshot_directory";
3584
3600
  const checkpointBinding = nativeModalPersistence ? await resolveModalCheckpointBindingBeforeCapture(settings, persistable, signal) : null;
3585
3601
  if (signal?.aborted) return false;
3586
3602
  const captureId = randomUUID();
@@ -4198,6 +4214,7 @@ import {
4198
4214
  verifyRetainedProcessMutationSettlement,
4199
4215
  verifyWorkspaceMutationSettlement,
4200
4216
  getSandbox,
4217
+ getEnrollment,
4201
4218
  markWarmLeaseInstanceLost as markWarmLeaseInstanceLost2,
4202
4219
  readLease as readLease2,
4203
4220
  readActiveSandbox,
@@ -4541,6 +4558,11 @@ function wrapTurnBoxWithRouting(services, ids, established) {
4541
4558
  } : null;
4542
4559
  },
4543
4560
  controlRpcFactory: controlRpcFactory(bus),
4561
+ resolveSelfhostedOpStream: async (sandbox) => {
4562
+ if (!sandbox.enrollmentId) return void 0;
4563
+ const enrollment = await getEnrollment(db, ids.workspaceId, sandbox.enrollmentId);
4564
+ return opStreamDepsFor(services, enrollment?.opStream === true);
4565
+ },
4544
4566
  relay: relayConfigFromSettings(settings),
4545
4567
  // A selfhosted swap target runs real commands too, so give it the same split
4546
4568
  // deadlines the machine-primary establish path uses (short control, long exec).
@@ -4676,6 +4698,11 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
4676
4698
  } : null;
4677
4699
  },
4678
4700
  controlRpcFactory: controlRpcFactory(bus),
4701
+ resolveSelfhostedOpStream: async (sandbox) => {
4702
+ if (!sandbox.enrollmentId) return void 0;
4703
+ const enrollment = await getEnrollment(db, ids.workspaceId, sandbox.enrollmentId);
4704
+ return opStreamDepsFor(services, enrollment?.opStream === true);
4705
+ },
4679
4706
  relay: relayConfigFromSettings(settings),
4680
4707
  ...selfhostedResolverTimeouts(settings),
4681
4708
  ...onOp !== void 0 ? { selfhostedOnOp: onOp } : {},
@@ -4780,7 +4807,7 @@ function opStreamDepsFor(services, machineAdvertisesOpStream) {
4780
4807
  async function establishSelfhostedTurnSession(services, args) {
4781
4808
  const { settings, bus, onOp } = services;
4782
4809
  const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
4783
- const opStream = opStreamDepsFor(services, "opStream" in args && args.opStream === true);
4810
+ const opStream = opStreamDepsFor(services, args.opStream);
4784
4811
  const { client, session } = await buildSelfhostedBackendSession({
4785
4812
  workspaceId: args.workspaceId,
4786
4813
  agentId: args.agentId,
@@ -4796,8 +4823,8 @@ async function establishSelfhostedTurnSession(services, args) {
4796
4823
  // Meter every control op (out-of-band telemetry) — no-op when unwired.
4797
4824
  ...onOp !== void 0 ? { onOp } : {},
4798
4825
  // The streaming exec transport — present iff the machine advertised the
4799
- // capability AND the server flag is on (latched per-op at OpStart; the
4800
- // legacy exec stays the permanent fallback wire form).
4826
+ // capability AND the server flag is on. It is required when execTimeoutMs=0;
4827
+ // legacy request/reply remains available only for explicitly bounded exec.
4801
4828
  ...opStream !== void 0 ? { opStream } : {}
4802
4829
  });
4803
4830
  return {
@@ -4848,11 +4875,11 @@ function eventAttributes(attributes) {
4848
4875
  if (!attributes) {
4849
4876
  return void 0;
4850
4877
  }
4851
- const sanitized = {};
4878
+ const projected = {};
4852
4879
  for (const [key, value] of Object.entries(attributes)) {
4853
- sanitized[key] = eventAttributeValue(value);
4880
+ projected[key] = eventAttributeValue(value);
4854
4881
  }
4855
- return sanitized;
4882
+ return projected;
4856
4883
  }
4857
4884
  function eventAttributeValue(value) {
4858
4885
  if (value === null || value === void 0 || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
@@ -5345,6 +5372,22 @@ function recordBatchFlush(observability, input) {
5345
5372
  value: input.durationSeconds
5346
5373
  });
5347
5374
  }
5375
+ function recordSessionEventAppendLatency(observability, input) {
5376
+ observability.observeHistogram({
5377
+ name: "opengeni_session_event_append_seconds",
5378
+ help: "Duration in seconds of an appendSessionEvents DB write (the durable write path).",
5379
+ buckets: STREAM_IO_BUCKETS,
5380
+ value: input.durationSeconds
5381
+ });
5382
+ }
5383
+ function recordSessionEventPublishLatency(observability, input) {
5384
+ observability.observeHistogram({
5385
+ name: "opengeni_session_event_publish_seconds",
5386
+ help: "Duration in seconds of the best-effort NATS live fan-out publish (the delivery path).",
5387
+ buckets: STREAM_IO_BUCKETS,
5388
+ value: input.durationSeconds
5389
+ });
5390
+ }
5348
5391
  var MODEL_INPUT_TOKENS_BUCKETS = [
5349
5392
  1e3,
5350
5393
  5e3,
@@ -5630,12 +5673,667 @@ async function discardUnpublishedRecording(args) {
5630
5673
  }
5631
5674
  function scrubFreeText(value) {
5632
5675
  if (value === null || value === void 0) return null;
5633
- const trimmed = String(value).slice(0, 2e3);
5634
- return trimmed.length === 0 ? null : trimmed;
5676
+ const exact = String(value);
5677
+ return exact.length === 0 ? null : exact;
5635
5678
  }
5636
5679
 
5637
- // src/activities/workspace-capture.ts
5680
+ // src/activities/retained-screenshots.ts
5681
+ import {
5682
+ COMPUTER_SCREENSHOT_MAX_BYTES,
5683
+ COMPUTER_SCREENSHOT_MAX_DIMENSION,
5684
+ COMPUTER_SCREENSHOT_MAX_PIXELS,
5685
+ COMPUTER_SCREENSHOT_RETENTION_MS,
5686
+ COMPUTER_SCREENSHOT_WORKSPACE_QUOTA_BYTES,
5687
+ RetainedArtifactMetadataSchema,
5688
+ retainedScreenshotReferenceFromFile
5689
+ } from "@opengeni/contracts";
5690
+ import {
5691
+ RetainedScreenshotQuotaExceededError,
5692
+ getRetainedScreenshotArtifact,
5693
+ prepareRetainedScreenshotArtifact,
5694
+ recordRetainedScreenshotArtifactError,
5695
+ settleRetainedScreenshotArtifactReady
5696
+ } from "@opengeni/db";
5638
5697
  import { createHash as createHash3 } from "crypto";
5698
+ var PNG_SIGNATURE = Uint8Array.of(137, 80, 78, 71, 13, 10, 26, 10);
5699
+ var PNG_ALLOWED_CRITICAL_CHUNKS = /* @__PURE__ */ new Set(["IHDR", "PLTE", "IDAT", "IEND"]);
5700
+ var PNG_MAX_CHUNKS = 16384;
5701
+ var RETAINED_IMAGE_MARKER = "retained_artifact";
5702
+ var ScreenshotValidationError = class extends Error {
5703
+ constructor(reason, message) {
5704
+ super(message);
5705
+ this.reason = reason;
5706
+ this.name = "ScreenshotValidationError";
5707
+ }
5708
+ };
5709
+ var RetainedScreenshotUnavailableError = class extends Error {
5710
+ constructor(artifactId, reason) {
5711
+ super(`Retained screenshot ${artifactId} is unavailable: ${reason}`);
5712
+ this.artifactId = artifactId;
5713
+ this.reason = reason;
5714
+ this.name = "RetainedScreenshotUnavailableError";
5715
+ }
5716
+ };
5717
+ function typedScreenshotFromSdkEvent(event) {
5718
+ if (!event || typeof event !== "object") return null;
5719
+ const streamEvent = event;
5720
+ if (streamEvent.type !== "run_item_stream_event") return null;
5721
+ const item = streamEvent.item;
5722
+ if (!item || item.type !== "tool_call_output_item") return null;
5723
+ const output = item.output;
5724
+ if (!output || typeof output !== "object" || Array.isArray(output)) return null;
5725
+ const outputRecord = output;
5726
+ if (outputRecord.type !== "image" || !outputRecord.image || typeof outputRecord.image !== "object") {
5727
+ return null;
5728
+ }
5729
+ const image = outputRecord.image;
5730
+ if (!(image.data instanceof Uint8Array) || typeof image.mediaType !== "string") return null;
5731
+ const raw = item.rawItem && typeof item.rawItem === "object" && !Array.isArray(item.rawItem) ? item.rawItem : {};
5732
+ const callId = raw.callId ?? raw.call_id;
5733
+ if (typeof callId !== "string" || callId.length === 0) return null;
5734
+ const toolOutputId = raw.id ?? item.id ?? callId;
5735
+ if (typeof toolOutputId !== "string" || toolOutputId.length === 0) return null;
5736
+ return {
5737
+ callId,
5738
+ toolOutputId,
5739
+ bytes: image.data,
5740
+ mediaType: image.mediaType
5741
+ };
5742
+ }
5743
+ function validateComputerScreenshot(input, limits = {}) {
5744
+ const maxBytes = limits.maxBytes ?? COMPUTER_SCREENSHOT_MAX_BYTES;
5745
+ const maxDimension = limits.maxDimension ?? COMPUTER_SCREENSHOT_MAX_DIMENSION;
5746
+ const maxPixels = limits.maxPixels ?? COMPUTER_SCREENSHOT_MAX_PIXELS;
5747
+ if (input.mediaType !== "image/png") {
5748
+ throw new ScreenshotValidationError(
5749
+ "unsupported",
5750
+ "computer screenshot MIME must be image/png"
5751
+ );
5752
+ }
5753
+ if (input.bytes.byteLength === 0) {
5754
+ throw new ScreenshotValidationError("invalid_content", "computer screenshot is empty");
5755
+ }
5756
+ if (input.bytes.byteLength > maxBytes) {
5757
+ throw new ScreenshotValidationError(
5758
+ "oversized",
5759
+ `computer screenshot exceeds ${maxBytes} bytes`
5760
+ );
5761
+ }
5762
+ if (input.bytes.byteLength < PNG_SIGNATURE.byteLength + 25) {
5763
+ throw new ScreenshotValidationError("invalid_content", "computer screenshot PNG is truncated");
5764
+ }
5765
+ for (let index = 0; index < PNG_SIGNATURE.byteLength; index += 1) {
5766
+ if (input.bytes[index] !== PNG_SIGNATURE[index]) {
5767
+ throw new ScreenshotValidationError(
5768
+ "invalid_content",
5769
+ "computer screenshot PNG signature is invalid"
5770
+ );
5771
+ }
5772
+ }
5773
+ let offset = PNG_SIGNATURE.byteLength;
5774
+ let chunks = 0;
5775
+ let width = 0;
5776
+ let height = 0;
5777
+ let sawIhdr = false;
5778
+ let sawIdat = false;
5779
+ let endedIdat = false;
5780
+ let sawIend = false;
5781
+ while (offset < input.bytes.byteLength) {
5782
+ chunks += 1;
5783
+ if (chunks > PNG_MAX_CHUNKS || offset + 12 > input.bytes.byteLength) {
5784
+ throw new ScreenshotValidationError(
5785
+ "invalid_content",
5786
+ "computer screenshot PNG chunk table is invalid"
5787
+ );
5788
+ }
5789
+ const length = readUint32(input.bytes, offset);
5790
+ const typeOffset = offset + 4;
5791
+ const dataOffset = typeOffset + 4;
5792
+ const dataEnd = dataOffset + length;
5793
+ const crcOffset = dataEnd;
5794
+ const nextOffset = crcOffset + 4;
5795
+ if (!Number.isSafeInteger(nextOffset) || nextOffset > input.bytes.byteLength) {
5796
+ throw new ScreenshotValidationError(
5797
+ "invalid_content",
5798
+ "computer screenshot PNG chunk is truncated"
5799
+ );
5800
+ }
5801
+ const type = asciiChunkType(input.bytes, typeOffset);
5802
+ const expectedCrc = readUint32(input.bytes, crcOffset);
5803
+ const actualCrc = crc32(input.bytes.subarray(typeOffset, dataEnd));
5804
+ if (expectedCrc !== actualCrc) {
5805
+ throw new ScreenshotValidationError(
5806
+ "invalid_content",
5807
+ `computer screenshot PNG ${type} CRC is invalid`
5808
+ );
5809
+ }
5810
+ if (chunks === 1 && type !== "IHDR") {
5811
+ throw new ScreenshotValidationError(
5812
+ "invalid_content",
5813
+ "computer screenshot PNG must begin with IHDR"
5814
+ );
5815
+ }
5816
+ if (isCriticalChunk(type) && !PNG_ALLOWED_CRITICAL_CHUNKS.has(type)) {
5817
+ throw new ScreenshotValidationError(
5818
+ "invalid_content",
5819
+ `computer screenshot PNG has unsupported critical chunk ${type}`
5820
+ );
5821
+ }
5822
+ if (type === "IHDR") {
5823
+ if (sawIhdr || length !== 13) {
5824
+ throw new ScreenshotValidationError(
5825
+ "invalid_content",
5826
+ "computer screenshot PNG IHDR is invalid"
5827
+ );
5828
+ }
5829
+ sawIhdr = true;
5830
+ width = readUint32(input.bytes, dataOffset);
5831
+ height = readUint32(input.bytes, dataOffset + 4);
5832
+ const bitDepth = input.bytes[dataOffset + 8];
5833
+ const colorType = input.bytes[dataOffset + 9];
5834
+ const compression = input.bytes[dataOffset + 10];
5835
+ const filter = input.bytes[dataOffset + 11];
5836
+ const interlace = input.bytes[dataOffset + 12];
5837
+ if (!validPngColorMode(bitDepth, colorType) || compression !== 0 || filter !== 0 || interlace !== 0 && interlace !== 1) {
5838
+ throw new ScreenshotValidationError(
5839
+ "invalid_content",
5840
+ "computer screenshot PNG IHDR fields are invalid"
5841
+ );
5842
+ }
5843
+ if (width <= 0 || height <= 0 || width > maxDimension || height > maxDimension || width * height > maxPixels) {
5844
+ throw new ScreenshotValidationError(
5845
+ "oversized",
5846
+ "computer screenshot dimensions exceed policy"
5847
+ );
5848
+ }
5849
+ } else if (type === "IDAT") {
5850
+ if (!sawIhdr || sawIend || endedIdat || length === 0) {
5851
+ throw new ScreenshotValidationError(
5852
+ "invalid_content",
5853
+ "computer screenshot PNG IDAT ordering is invalid"
5854
+ );
5855
+ }
5856
+ sawIdat = true;
5857
+ } else if (sawIdat && type !== "IEND") {
5858
+ endedIdat = true;
5859
+ }
5860
+ if (type === "IEND") {
5861
+ if (!sawIhdr || !sawIdat || sawIend || length !== 0) {
5862
+ throw new ScreenshotValidationError(
5863
+ "invalid_content",
5864
+ "computer screenshot PNG IEND is invalid"
5865
+ );
5866
+ }
5867
+ sawIend = true;
5868
+ if (nextOffset !== input.bytes.byteLength) {
5869
+ throw new ScreenshotValidationError(
5870
+ "invalid_content",
5871
+ "computer screenshot PNG has trailing/polyglot bytes"
5872
+ );
5873
+ }
5874
+ }
5875
+ offset = nextOffset;
5876
+ }
5877
+ if (!sawIhdr || !sawIdat || !sawIend || offset !== input.bytes.byteLength) {
5878
+ throw new ScreenshotValidationError("invalid_content", "computer screenshot PNG is incomplete");
5879
+ }
5880
+ return {
5881
+ bytes: input.bytes,
5882
+ mediaType: "image/png",
5883
+ sizeBytes: input.bytes.byteLength,
5884
+ sha256: createHash3("sha256").update(input.bytes).digest("hex"),
5885
+ width,
5886
+ height
5887
+ };
5888
+ }
5889
+ function retainedScreenshotIdentity(input) {
5890
+ const settlementKey = createHash3("sha256").update("opengeni:computer-screenshot:v1\0").update(input.sessionId).update("\0").update(input.turnId).update("\0").update(input.attemptId).update("\0").update(input.toolCallId).update("\0").update(input.toolOutputId).digest("hex");
5891
+ const uuidBytes = Uint8Array.from(Buffer.from(settlementKey.slice(0, 32), "hex"));
5892
+ uuidBytes[6] = uuidBytes[6] & 15 | 128;
5893
+ uuidBytes[8] = uuidBytes[8] & 63 | 128;
5894
+ const hex = Buffer.from(uuidBytes).toString("hex");
5895
+ return {
5896
+ artifactId: `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`,
5897
+ settlementKey
5898
+ };
5899
+ }
5900
+ async function retainComputerScreenshot(input) {
5901
+ const identity = retainedScreenshotIdentity({
5902
+ sessionId: input.sessionId,
5903
+ turnId: input.turnId,
5904
+ attemptId: input.attemptId,
5905
+ toolCallId: input.output.callId,
5906
+ toolOutputId: input.output.toolOutputId
5907
+ });
5908
+ let screenshot;
5909
+ try {
5910
+ screenshot = validateComputerScreenshot(input.output);
5911
+ } catch (error) {
5912
+ if (error instanceof ScreenshotValidationError) {
5913
+ return unavailable(identity.artifactId, error.reason);
5914
+ }
5915
+ throw error;
5916
+ }
5917
+ if (!input.objectStorage) return unavailable(identity.artifactId, "missing_storage");
5918
+ const now = input.now ?? /* @__PURE__ */ new Date();
5919
+ const retentionExpiresAt = new Date(
5920
+ now.getTime() + (input.retentionMs ?? COMPUTER_SCREENSHOT_RETENTION_MS)
5921
+ );
5922
+ const objectKey = `workspaces/${input.workspaceId}/files/${identity.artifactId}/retained/computer-screenshot.png`;
5923
+ let prepared;
5924
+ try {
5925
+ prepared = (await prepareRetainedScreenshotArtifact(input.db, {
5926
+ artifactId: identity.artifactId,
5927
+ accountId: input.accountId,
5928
+ workspaceId: input.workspaceId,
5929
+ sessionId: input.sessionId,
5930
+ turnId: input.turnId,
5931
+ attemptId: input.attemptId,
5932
+ settlementKey: identity.settlementKey,
5933
+ toolCallId: input.output.callId,
5934
+ toolOutputId: input.output.toolOutputId,
5935
+ mediaType: screenshot.mediaType,
5936
+ sizeBytes: screenshot.sizeBytes,
5937
+ sha256: screenshot.sha256,
5938
+ width: screenshot.width,
5939
+ height: screenshot.height,
5940
+ retentionExpiresAt,
5941
+ bucket: input.objectStorage.bucket,
5942
+ objectKey,
5943
+ workspaceQuotaBytes: input.workspaceQuotaBytes ?? COMPUTER_SCREENSHOT_WORKSPACE_QUOTA_BYTES
5944
+ })).artifact;
5945
+ } catch (error) {
5946
+ if (error instanceof RetainedScreenshotQuotaExceededError) {
5947
+ return unavailable(identity.artifactId, "quota_exceeded");
5948
+ }
5949
+ throw error;
5950
+ }
5951
+ if (prepared.status === "ready") {
5952
+ return await verifyReadyArtifact(input.objectStorage, prepared) ? reference(prepared) : unavailable(identity.artifactId, "missing_storage");
5953
+ }
5954
+ if (prepared.status !== "pending" && prepared.status !== "reconciling") {
5955
+ return unavailable(identity.artifactId, unavailableReasonForStatus(prepared.status));
5956
+ }
5957
+ try {
5958
+ await input.objectStorage.putObject({
5959
+ key: objectKey,
5960
+ contentType: screenshot.mediaType,
5961
+ body: screenshot.bytes,
5962
+ sha256: screenshot.sha256
5963
+ });
5964
+ const head = await input.objectStorage.headFile(prepared.file);
5965
+ assertStoredScreenshotHead(head, screenshot);
5966
+ } catch (error) {
5967
+ await recordRetainedScreenshotArtifactError(input.db, {
5968
+ accountId: input.accountId,
5969
+ workspaceId: input.workspaceId,
5970
+ artifactId: identity.artifactId,
5971
+ settlementKey: identity.settlementKey,
5972
+ error: error instanceof Error ? error.message : String(error)
5973
+ }).catch(() => void 0);
5974
+ return unavailable(identity.artifactId, "storage_write_failed");
5975
+ }
5976
+ try {
5977
+ const settled = await settleRetainedScreenshotArtifactReady(input.db, {
5978
+ accountId: input.accountId,
5979
+ workspaceId: input.workspaceId,
5980
+ artifactId: identity.artifactId,
5981
+ settlementKey: identity.settlementKey
5982
+ });
5983
+ return reference(settled);
5984
+ } catch (error) {
5985
+ const stillReferenced = await getRetainedScreenshotArtifact(
5986
+ input.db,
5987
+ input.workspaceId,
5988
+ input.sessionId,
5989
+ identity.artifactId
5990
+ ).catch(() => void 0);
5991
+ if (stillReferenced?.status === "ready") {
5992
+ return await verifyReadyArtifact(input.objectStorage, stillReferenced) ? reference(stillReferenced) : unavailable(identity.artifactId, "missing_storage");
5993
+ }
5994
+ if (stillReferenced === null || stillReferenced?.status === "cleanup_queued" || stillReferenced?.status === "cleanup_pending" || stillReferenced?.status === "failed" || stillReferenced?.status === "expired" || stillReferenced?.status === "deleted") {
5995
+ await input.objectStorage.deleteObject(objectKey).catch(() => void 0);
5996
+ } else if (stillReferenced) {
5997
+ await recordRetainedScreenshotArtifactError(input.db, {
5998
+ accountId: input.accountId,
5999
+ workspaceId: input.workspaceId,
6000
+ artifactId: identity.artifactId,
6001
+ settlementKey: identity.settlementKey,
6002
+ error: error instanceof Error ? error.message : String(error)
6003
+ }).catch(() => void 0);
6004
+ }
6005
+ return unavailable(identity.artifactId, "pending");
6006
+ }
6007
+ }
6008
+ function compactRetainedScreenshotHistory(history, receiptsByCallId) {
6009
+ if (receiptsByCallId.size === 0) return history;
6010
+ return history.map((item) => {
6011
+ const callId = historyCallId(item);
6012
+ const receipt = callId ? receiptsByCallId.get(callId) : void 0;
6013
+ if (!receipt || !Array.isArray(item.output)) return item;
6014
+ let changed = false;
6015
+ const output = item.output.map((entry) => {
6016
+ if (!isInlineImageContent(entry)) return entry;
6017
+ changed = true;
6018
+ return {
6019
+ ...entry,
6020
+ image: { type: RETAINED_IMAGE_MARKER, artifact: receipt }
6021
+ };
6022
+ });
6023
+ return changed ? { ...item, output } : item;
6024
+ });
6025
+ }
6026
+ function collectRetainedScreenshotReceipts(history, target = /* @__PURE__ */ new Map()) {
6027
+ for (const item of history) {
6028
+ const callId = historyCallId(item);
6029
+ if (!callId || !Array.isArray(item.output)) continue;
6030
+ for (const entry of item.output) {
6031
+ const receipt = retainedReceiptFromImageContent(entry);
6032
+ if (receipt) {
6033
+ target.set(callId, receipt);
6034
+ break;
6035
+ }
6036
+ }
6037
+ }
6038
+ return target;
6039
+ }
6040
+ function compactRetainedScreenshotRunState(serialized, receiptsByCallId) {
6041
+ if (receiptsByCallId.size === 0) return serialized;
6042
+ const parsed = parseSerializedRunState(serialized);
6043
+ if (!parsed) return serialized;
6044
+ let changed = false;
6045
+ visitSerializedRunStateItemArrays(parsed, (items) => {
6046
+ const compacted = compactRetainedScreenshotHistory(items, receiptsByCallId);
6047
+ if (compacted !== items && compacted.some((item, index) => item !== items[index])) {
6048
+ items.splice(0, items.length, ...compacted);
6049
+ changed = true;
6050
+ }
6051
+ });
6052
+ return changed ? JSON.stringify(parsed) : serialized;
6053
+ }
6054
+ function collectRetainedScreenshotRunStateReceipts(serialized, target = /* @__PURE__ */ new Map()) {
6055
+ const parsed = parseSerializedRunState(serialized);
6056
+ if (!parsed) return target;
6057
+ visitSerializedRunStateItemArrays(parsed, (items) => {
6058
+ collectRetainedScreenshotReceipts(items, target);
6059
+ });
6060
+ return target;
6061
+ }
6062
+ async function materializeRetainedScreenshotHistory(input) {
6063
+ return await materializeRetainedScreenshotHistoryWithCache(input, /* @__PURE__ */ new Map());
6064
+ }
6065
+ async function materializeRetainedScreenshotRunState(input) {
6066
+ const parsed = parseSerializedRunState(input.serialized);
6067
+ if (!parsed) return input.serialized;
6068
+ const cache = /* @__PURE__ */ new Map();
6069
+ let changed = false;
6070
+ await visitSerializedRunStateItemArraysAsync(parsed, async (items) => {
6071
+ const materialized = await materializeRetainedScreenshotHistoryWithCache(
6072
+ { ...input, history: items },
6073
+ cache
6074
+ );
6075
+ if (materialized.some((item, index) => item !== items[index])) {
6076
+ items.splice(0, items.length, ...materialized);
6077
+ changed = true;
6078
+ }
6079
+ });
6080
+ return changed ? JSON.stringify(parsed) : input.serialized;
6081
+ }
6082
+ async function materializeRetainedScreenshotHistoryWithCache(input, cache) {
6083
+ const now = input.now ?? /* @__PURE__ */ new Date();
6084
+ const materializeEntry = async (entry) => {
6085
+ const receipt = retainedReceiptFromImageContent(entry);
6086
+ if (!receipt) return entry;
6087
+ if (!receipt.available) {
6088
+ throw new RetainedScreenshotUnavailableError(receipt.artifactId, receipt.reason);
6089
+ }
6090
+ let dataUrl = cache.get(receipt.artifactId);
6091
+ if (!dataUrl) {
6092
+ if (!input.objectStorage) {
6093
+ throw new RetainedScreenshotUnavailableError(receipt.artifactId, "missing_storage");
6094
+ }
6095
+ const artifact = await getRetainedScreenshotArtifact(
6096
+ input.db,
6097
+ input.workspaceId,
6098
+ input.sessionId,
6099
+ receipt.artifactId
6100
+ );
6101
+ if (!artifact) throw new RetainedScreenshotUnavailableError(receipt.artifactId, "deleted");
6102
+ if (artifact.status !== "ready") {
6103
+ throw new RetainedScreenshotUnavailableError(
6104
+ receipt.artifactId,
6105
+ unavailableReasonForStatus(artifact.status)
6106
+ );
6107
+ }
6108
+ if (artifact.retentionExpiresAt.getTime() <= now.getTime()) {
6109
+ throw new RetainedScreenshotUnavailableError(receipt.artifactId, "expired");
6110
+ }
6111
+ const bytes = await input.objectStorage.getFileBytes(artifact.file).catch(() => null);
6112
+ if (!bytes)
6113
+ throw new RetainedScreenshotUnavailableError(receipt.artifactId, "missing_storage");
6114
+ const validated = validateComputerScreenshot({
6115
+ bytes,
6116
+ mediaType: artifact.mediaType
6117
+ });
6118
+ if (validated.sizeBytes !== artifact.sizeBytes || validated.sha256 !== artifact.sha256 || validated.width !== artifact.width || validated.height !== artifact.height || receipt.sha256 !== artifact.sha256 || receipt.originalBytes !== artifact.sizeBytes) {
6119
+ throw new RetainedScreenshotUnavailableError(receipt.artifactId, "invalid_content");
6120
+ }
6121
+ dataUrl = `data:${validated.mediaType};base64,${Buffer.from(validated.bytes).toString("base64")}`;
6122
+ cache.set(receipt.artifactId, dataUrl);
6123
+ }
6124
+ return { ...entry, image: dataUrl };
6125
+ };
6126
+ const materialized = [];
6127
+ for (const item of input.history) {
6128
+ if (!Array.isArray(item.output)) {
6129
+ materialized.push(item);
6130
+ continue;
6131
+ }
6132
+ let changed = false;
6133
+ const output = [];
6134
+ for (const entry of item.output) {
6135
+ const next = await materializeEntry(entry);
6136
+ changed ||= next !== entry;
6137
+ output.push(next);
6138
+ }
6139
+ materialized.push(changed ? { ...item, output } : item);
6140
+ }
6141
+ return materialized;
6142
+ }
6143
+ function parseSerializedRunState(serialized) {
6144
+ try {
6145
+ const parsed = JSON.parse(serialized);
6146
+ return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
6147
+ } catch {
6148
+ return null;
6149
+ }
6150
+ }
6151
+ function visitSerializedRunStateItemArrays(root, visitor) {
6152
+ const originalInput = root.originalInput;
6153
+ if (Array.isArray(originalInput)) {
6154
+ visitor(originalInput);
6155
+ }
6156
+ if (Array.isArray(root.generatedItems)) {
6157
+ for (const wrapper of root.generatedItems) {
6158
+ if (!wrapper || typeof wrapper !== "object" || Array.isArray(wrapper)) continue;
6159
+ const record = wrapper;
6160
+ const rawItem = record.rawItem;
6161
+ if (!rawItem || typeof rawItem !== "object" || Array.isArray(rawItem)) continue;
6162
+ const items = [rawItem];
6163
+ visitor(items);
6164
+ if (items[0] !== rawItem) record.rawItem = items[0];
6165
+ }
6166
+ }
6167
+ if (Array.isArray(root.modelResponses)) {
6168
+ for (const response of root.modelResponses) {
6169
+ if (!response || typeof response !== "object" || Array.isArray(response)) continue;
6170
+ const output = response.output;
6171
+ if (Array.isArray(output)) visitor(output);
6172
+ }
6173
+ }
6174
+ const lastModelResponse = root.lastModelResponse;
6175
+ if (lastModelResponse && typeof lastModelResponse === "object" && !Array.isArray(lastModelResponse)) {
6176
+ const output = lastModelResponse.output;
6177
+ if (Array.isArray(output)) visitor(output);
6178
+ }
6179
+ }
6180
+ async function visitSerializedRunStateItemArraysAsync(root, visitor) {
6181
+ const originalInput = root.originalInput;
6182
+ if (Array.isArray(originalInput)) {
6183
+ await visitor(originalInput);
6184
+ }
6185
+ if (Array.isArray(root.generatedItems)) {
6186
+ for (const wrapper of root.generatedItems) {
6187
+ if (!wrapper || typeof wrapper !== "object" || Array.isArray(wrapper)) continue;
6188
+ const record = wrapper;
6189
+ const rawItem = record.rawItem;
6190
+ if (!rawItem || typeof rawItem !== "object" || Array.isArray(rawItem)) continue;
6191
+ const items = [rawItem];
6192
+ await visitor(items);
6193
+ if (items[0] !== rawItem) record.rawItem = items[0];
6194
+ }
6195
+ }
6196
+ if (Array.isArray(root.modelResponses)) {
6197
+ for (const response of root.modelResponses) {
6198
+ if (!response || typeof response !== "object" || Array.isArray(response)) continue;
6199
+ const output = response.output;
6200
+ if (Array.isArray(output)) await visitor(output);
6201
+ }
6202
+ }
6203
+ const lastModelResponse = root.lastModelResponse;
6204
+ if (lastModelResponse && typeof lastModelResponse === "object" && !Array.isArray(lastModelResponse)) {
6205
+ const output = lastModelResponse.output;
6206
+ if (Array.isArray(output)) await visitor(output);
6207
+ }
6208
+ }
6209
+ function reference(artifact) {
6210
+ if (!artifact.sessionId) {
6211
+ throw new Error(`Ready retained screenshot is detached: ${artifact.artifactId}`);
6212
+ }
6213
+ const value = retainedScreenshotReferenceFromFile({
6214
+ ...artifact.file,
6215
+ sessionId: artifact.sessionId,
6216
+ width: artifact.width,
6217
+ height: artifact.height,
6218
+ expiresAt: artifact.retentionExpiresAt.toISOString()
6219
+ });
6220
+ if (!value)
6221
+ throw new Error(`Ready retained screenshot receipt is invalid: ${artifact.artifactId}`);
6222
+ return value;
6223
+ }
6224
+ function unavailable(artifactId, reason) {
6225
+ return RetainedArtifactMetadataSchema.parse({
6226
+ available: false,
6227
+ artifactId,
6228
+ reason
6229
+ });
6230
+ }
6231
+ function unavailableReasonForStatus(status) {
6232
+ switch (status) {
6233
+ case "pending":
6234
+ case "reconciling":
6235
+ return "pending";
6236
+ case "cleanup_queued":
6237
+ case "cleanup_pending":
6238
+ return "failed";
6239
+ case "failed":
6240
+ case "expired":
6241
+ case "deleted":
6242
+ return status;
6243
+ case "ready":
6244
+ return "missing_storage";
6245
+ }
6246
+ }
6247
+ async function verifyReadyArtifact(storage, artifact) {
6248
+ try {
6249
+ assertStoredScreenshotHead(await storage.headFile(artifact.file), {
6250
+ sizeBytes: artifact.sizeBytes,
6251
+ mediaType: "image/png",
6252
+ sha256: artifact.sha256
6253
+ });
6254
+ return true;
6255
+ } catch {
6256
+ return false;
6257
+ }
6258
+ }
6259
+ function assertStoredScreenshotHead(head, expected) {
6260
+ if (head.ContentLength !== expected.sizeBytes) {
6261
+ throw new Error("retained screenshot object size mismatch");
6262
+ }
6263
+ if (head.ContentType !== expected.mediaType) {
6264
+ throw new Error("retained screenshot object MIME mismatch");
6265
+ }
6266
+ if (head.Metadata?.sha256 !== expected.sha256) {
6267
+ throw new Error("retained screenshot object SHA-256 metadata mismatch");
6268
+ }
6269
+ }
6270
+ function retainedReceiptFromImageContent(entry) {
6271
+ if (!entry || typeof entry !== "object" || Array.isArray(entry)) return null;
6272
+ const image = entry.image;
6273
+ if (!image || typeof image !== "object" || Array.isArray(image)) return null;
6274
+ const marker = image;
6275
+ if (marker.type !== RETAINED_IMAGE_MARKER) return null;
6276
+ const parsed = RetainedArtifactMetadataSchema.safeParse(marker.artifact);
6277
+ if (!parsed.success) return null;
6278
+ return parsed.data.available && parsed.data.kind !== "computer_screenshot" ? null : parsed.data;
6279
+ }
6280
+ function isInlineImageContent(entry) {
6281
+ if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
6282
+ const record = entry;
6283
+ return record.type === "input_image" && typeof record.image === "string" && record.image.startsWith("data:image/png;base64,");
6284
+ }
6285
+ function historyCallId(item) {
6286
+ const value = item.callId ?? item.call_id;
6287
+ return typeof value === "string" && value.length > 0 ? value : null;
6288
+ }
6289
+ function readUint32(bytes, offset) {
6290
+ return (bytes[offset] << 24 | bytes[offset + 1] << 16 | bytes[offset + 2] << 8 | bytes[offset + 3]) >>> 0;
6291
+ }
6292
+ function asciiChunkType(bytes, offset) {
6293
+ const chars = [bytes[offset], bytes[offset + 1], bytes[offset + 2], bytes[offset + 3]];
6294
+ if (chars.some(
6295
+ (value) => value === void 0 || value < 65 || value > 90 && value < 97 || value > 122
6296
+ )) {
6297
+ throw new ScreenshotValidationError(
6298
+ "invalid_content",
6299
+ "computer screenshot PNG chunk type is invalid"
6300
+ );
6301
+ }
6302
+ return String.fromCharCode(...chars);
6303
+ }
6304
+ function isCriticalChunk(type) {
6305
+ return type.charCodeAt(0) >= 65 && type.charCodeAt(0) <= 90;
6306
+ }
6307
+ function validPngColorMode(bitDepth, colorType) {
6308
+ if (bitDepth === void 0 || colorType === void 0) return false;
6309
+ switch (colorType) {
6310
+ case 0:
6311
+ return [1, 2, 4, 8, 16].includes(bitDepth);
6312
+ case 2:
6313
+ case 4:
6314
+ case 6:
6315
+ return bitDepth === 8 || bitDepth === 16;
6316
+ case 3:
6317
+ return [1, 2, 4, 8].includes(bitDepth);
6318
+ default:
6319
+ return false;
6320
+ }
6321
+ }
6322
+ var CRC32_TABLE = Uint32Array.from({ length: 256 }, (_, value) => {
6323
+ let current = value;
6324
+ for (let bit = 0; bit < 8; bit += 1) {
6325
+ current = (current & 1) !== 0 ? 3988292384 ^ current >>> 1 : current >>> 1;
6326
+ }
6327
+ return current >>> 0;
6328
+ });
6329
+ function crc32(bytes) {
6330
+ let crc = 4294967295;
6331
+ for (const byte of bytes) crc = CRC32_TABLE[(crc ^ byte) & 255] ^ crc >>> 8;
6332
+ return (crc ^ 4294967295) >>> 0;
6333
+ }
6334
+
6335
+ // src/activities/workspace-capture.ts
6336
+ import { createHash as createHash4 } from "crypto";
5639
6337
  import {
5640
6338
  deleteWorkspaceCaptureRows,
5641
6339
  insertFailedWorkspaceCapture,
@@ -5753,7 +6451,7 @@ async function readCaptureRepository(svc, root) {
5753
6451
  return { complete: false, degradedReason: "repository_read_unavailable" };
5754
6452
  }
5755
6453
  try {
5756
- const diff = await svc.gitDiff({
6454
+ const workingRequest = svc.gitDiff({
5757
6455
  path: root,
5758
6456
  staged: false,
5759
6457
  includeUntracked: true,
@@ -5762,7 +6460,24 @@ async function readCaptureRepository(svc, root) {
5762
6460
  contextLines: 3,
5763
6461
  maxBytesPerFile: PER_FILE_DIFF_GUARD_BYTES
5764
6462
  });
5765
- return { complete: true, status, diff };
6463
+ const branchRequest = Promise.resolve().then(
6464
+ () => svc.gitDiff({
6465
+ path: root,
6466
+ staged: false,
6467
+ includeUntracked: true,
6468
+ fromRef: "origin/HEAD",
6469
+ pathspec: [],
6470
+ contextLines: 3,
6471
+ maxBytesPerFile: PER_FILE_DIFF_GUARD_BYTES
6472
+ })
6473
+ ).then((value) => value).catch(() => null);
6474
+ const [diff, branchDiff] = await Promise.all([workingRequest, branchRequest]);
6475
+ return {
6476
+ complete: true,
6477
+ status,
6478
+ diff,
6479
+ ...branchDiff ? { branchDiff } : {}
6480
+ };
5766
6481
  } catch (error) {
5767
6482
  const boxExiting = classifyCaptureEntryError(error);
5768
6483
  if (boxExiting) throw boxExiting;
@@ -5915,7 +6630,7 @@ async function runCapture(input, ctx, startedAt, signal, stage) {
5915
6630
  );
5916
6631
  return;
5917
6632
  }
5918
- const { status, diff } = repository;
6633
+ const { status, diff, branchDiff } = repository;
5919
6634
  const diffFiles = diff.files.filter((f) => !isUnderResidueDir(joinRepoPath(root, f.path)));
5920
6635
  const statusFiles = status.files.filter((f) => !isUnderResidueDir(joinRepoPath(root, f.path)));
5921
6636
  repos.push({
@@ -5926,7 +6641,12 @@ async function runCapture(input, ctx, startedAt, signal, stage) {
5926
6641
  ahead: status.ahead,
5927
6642
  behind: status.behind,
5928
6643
  status: statusFiles,
5929
- diff: diffFiles
6644
+ diff: diffFiles,
6645
+ ...branchDiff ? {
6646
+ branchDiff: branchDiff.files.filter(
6647
+ (f) => !isUnderResidueDir(joinRepoPath(root, f.path))
6648
+ )
6649
+ } : {}
5930
6650
  });
5931
6651
  for (const f of diffFiles) {
5932
6652
  additions += f.additions;
@@ -6337,7 +7057,7 @@ function joinRepoPath(repoRoot, repoRelPath) {
6337
7057
  return `${repoRoot.replace(/\/+$/, "")}/${repoRelPath}`;
6338
7058
  }
6339
7059
  function sha256(bytes) {
6340
- return createHash3("sha256").update(bytes).digest("hex");
7060
+ return createHash4("sha256").update(bytes).digest("hex");
6341
7061
  }
6342
7062
  function utf8(s) {
6343
7063
  return new TextEncoder().encode(s);
@@ -6417,11 +7137,21 @@ import {
6417
7137
  DEFAULT_FIRST_PARTY_MCP_TOOLS,
6418
7138
  evaluateWorkspaceModelPolicy as evaluateWorkspaceModelPolicy2,
6419
7139
  readTurnExecutionPolicyV1,
7140
+ resolveWorkspaceAgentHumanInputEnabled,
6420
7141
  resourceMountPath as resourceMountPath2
6421
7142
  } from "@opengeni/contracts";
6422
- import { createHash as createHash4, randomUUID as randomUUID2 } from "crypto";
7143
+ import { createHash as createHash5, randomUUID as randomUUID2 } from "crypto";
6423
7144
  var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
6424
7145
  var PROVIDER_CONNECTIVITY_BACKOFF_MS = [2e3, 5e3, 15e3, 3e4, 6e4];
7146
+ var WorkspaceHumanInputDisabledError = class extends Error {
7147
+ constructor(kind) {
7148
+ super(`Workspace policy rejects structured human-input ${kind}`);
7149
+ this.name = "WorkspaceHumanInputDisabledError";
7150
+ }
7151
+ };
7152
+ function assertWorkspaceHumanInputAllowed(enabled, kind, attempted) {
7153
+ if (!enabled && attempted) throw new WorkspaceHumanInputDisabledError(kind);
7154
+ }
6425
7155
  function credentialSubjectIdForTurnInitiator(turn) {
6426
7156
  return directPersonalConnectionSubjectId(turn);
6427
7157
  }
@@ -6541,10 +7271,10 @@ function filterUnmaterializedSandboxFileDownloads(downloads, materializedFileIds
6541
7271
  return downloads.filter((download) => !materializedFileIds.has(download.fileId));
6542
7272
  }
6543
7273
  function codexWorkspaceMetricKey(workspaceId) {
6544
- return createHash4("sha256").update(workspaceId).digest("hex").slice(0, 12);
7274
+ return createHash5("sha256").update(workspaceId).digest("hex").slice(0, 12);
6545
7275
  }
6546
7276
  function stableHumanInputRequestId(sessionId, turnId, toolCallId) {
6547
- const hex = createHash4("sha256").update("opengeni-human-input-v1\0").update(sessionId).update("\0").update(turnId).update("\0").update(toolCallId).digest("hex").slice(0, 32).split("");
7277
+ const hex = createHash5("sha256").update("opengeni-human-input-v1\0").update(sessionId).update("\0").update(turnId).update("\0").update(toolCallId).digest("hex").slice(0, 32).split("");
6548
7278
  hex[12] = "5";
6549
7279
  hex[16] = ["8", "9", "a", "b"][Number.parseInt(hex[16] ?? "0", 16) % 4] ?? "8";
6550
7280
  const value = hex.join("");
@@ -6647,6 +7377,16 @@ async function waitForTurnStreamCleanup(batcherFlush, providerCompleted, signal)
6647
7377
  await waitForTurnFinalizerStep(batcherFlush, signal);
6648
7378
  await waitForTurnFinalizerStep(providerCompleted, signal);
6649
7379
  }
7380
+ async function clearAttemptCredentialsWithSettledFence(input) {
7381
+ try {
7382
+ await input.runWorkspaceFencedClear();
7383
+ } catch (error) {
7384
+ const settledAttemptFence = error instanceof Error && error.name === "SandboxWorkspaceMutationFencedError" && error.code === "attempt_fenced" && (input.activityStatus === "idle" || input.activityStatus === "failed");
7385
+ if (!settledAttemptFence) throw error;
7386
+ input.onSettledAttemptFence();
7387
+ await input.clearExactAttempt();
7388
+ }
7389
+ }
6650
7390
  function turnFinalizerCancellationSignal(temporalSignal, activityStatus) {
6651
7391
  if (activityStatus !== "cancelled" || temporalSignal?.aborted) return temporalSignal;
6652
7392
  const fenced = new AbortController();
@@ -6656,56 +7396,55 @@ function turnFinalizerCancellationSignal(temporalSignal, activityStatus) {
6656
7396
  function compactionFailureReason(reason) {
6657
7397
  return reason.startsWith("compaction summarization failed:") ? reason : `compaction summarization failed: ${reason}`;
6658
7398
  }
6659
- function safeErrorDiagnostic(error, redactText = redactSensitiveText) {
6660
- const rawName = error instanceof Error ? error.name : "Error";
6661
- const name = /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error";
6662
- const rawMessage = error instanceof Error ? error.message : String(error);
7399
+ var MandatoryHistoryPersistenceError = class extends Error {
7400
+ constructor(stage, cause) {
7401
+ super(cause instanceof Error ? cause.message : String(cause));
7402
+ this.stage = stage;
7403
+ this.cause = cause;
7404
+ }
7405
+ name = "MandatoryHistoryPersistenceError";
7406
+ cause;
7407
+ };
7408
+ async function runMandatoryHistoryPersistenceStep(stage, operation) {
7409
+ try {
7410
+ return await operation();
7411
+ } catch (cause) {
7412
+ throw new MandatoryHistoryPersistenceError(stage, cause);
7413
+ }
7414
+ }
7415
+ function safeErrorDiagnostic(error) {
6663
7416
  const diagnostic = {
6664
- name,
6665
- message: redactText(rawMessage).slice(0, 2048)
7417
+ errorClass: "WorkerOperationError",
7418
+ errorCode: "worker_operation_failed",
7419
+ origin: "worker"
6666
7420
  };
6667
- if (error && typeof error === "object") {
6668
- const status = Number(
6669
- error.status ?? error.statusCode
6670
- );
6671
- if (Number.isInteger(status) && status >= 100 && status <= 599) {
6672
- diagnostic.status = status;
6673
- }
6674
- const rawCode = error.code;
6675
- if (typeof rawCode === "string" || typeof rawCode === "number") {
6676
- const code = redactText(String(rawCode)).slice(0, 80);
6677
- if (/^[A-Za-z0-9_.:-]+$/.test(code)) diagnostic.code = code;
7421
+ try {
7422
+ let statusSource = error;
7423
+ if (error instanceof MandatoryHistoryPersistenceError) {
7424
+ diagnostic.historyPersistenceStage = error.stage;
7425
+ statusSource = error.cause;
7426
+ }
7427
+ if (statusSource && typeof statusSource === "object") {
7428
+ const status = Number(
7429
+ statusSource.status ?? statusSource.statusCode
7430
+ );
7431
+ if (Number.isInteger(status) && status >= 100 && status <= 599) {
7432
+ diagnostic.status = status;
7433
+ }
6678
7434
  }
7435
+ } catch {
6679
7436
  }
6680
7437
  return diagnostic;
6681
7438
  }
6682
- function safeErrorForTelemetry(error, redactText) {
6683
- const diagnostic = safeErrorDiagnostic(error, redactText);
6684
- const safe = new Error(diagnostic.message);
6685
- safe.name = diagnostic.name;
7439
+ function safeErrorForTelemetry(error) {
7440
+ const diagnostic = safeErrorDiagnostic(error);
7441
+ const safe = new Error("worker operation failed");
7442
+ safe.name = "WorkerOperationError";
7443
+ safe.code = diagnostic.errorCode;
7444
+ if (diagnostic.status !== void 0) safe.status = diagnostic.status;
7445
+ safe.origin = diagnostic.origin;
6686
7446
  return safe;
6687
7447
  }
6688
- function headerSecretRedactions(prefix, headers, credentialHeaderNames) {
6689
- const discovered = [];
6690
- const explicitCredentialHeaders = credentialHeaderNames ? new Set([...credentialHeaderNames].map((name) => name.toLowerCase())) : null;
6691
- for (const [headerName, value] of Object.entries(headers ?? {})) {
6692
- if (!explicitCredentialHeaders?.has(headerName.toLowerCase()) && !isCredentialHeaderName(headerName)) {
6693
- continue;
6694
- }
6695
- const safeHeaderName = headerName.toUpperCase().replace(/[^A-Z0-9]+/g, "_");
6696
- discovered.push({ name: `${prefix}_${safeHeaderName || "HEADER"}`, value });
6697
- if (/^(?:proxy-)?authorization$/i.test(headerName)) {
6698
- const credential = value.match(/^\s*[A-Za-z][A-Za-z0-9_-]*\s+(.+)\s*$/)?.[1];
6699
- if (credential) {
6700
- discovered.push({
6701
- name: `${prefix}_${safeHeaderName || "AUTHORIZATION"}_CREDENTIAL`,
6702
- value: credential
6703
- });
6704
- }
6705
- }
6706
- }
6707
- return discovered;
6708
- }
6709
7448
  function compactionFailureReasonFromError(error) {
6710
7449
  if (error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError2) {
6711
7450
  return compactionFailureReason(error.message);
@@ -7034,14 +7773,9 @@ async function emitModelCallUsage(input) {
7034
7773
  if (!authoritative) return false;
7035
7774
  try {
7036
7775
  input.observability.info("model call usage", {
7037
- accountId: input.accountId,
7038
- workspaceId: input.workspaceId,
7039
- sessionId: input.sessionId,
7040
- turnId: input.turnId,
7041
7776
  provider: input.provider,
7042
7777
  providerApi: input.providerApi,
7043
7778
  model: input.model,
7044
- sourceKey: input.sourceKey,
7045
7779
  inputTokens: telemetry.inputTokens,
7046
7780
  outputTokens: telemetry.outputTokens,
7047
7781
  cachedTokens: telemetry.cachedTokens,
@@ -7055,7 +7789,6 @@ async function emitModelCallUsage(input) {
7055
7789
  provider: input.provider,
7056
7790
  providerApi: input.providerApi,
7057
7791
  model: input.model,
7058
- sourceKey: input.sourceKey,
7059
7792
  rejectedFields: normalizedUsage.rejectedFields.join(",")
7060
7793
  });
7061
7794
  }
@@ -7074,20 +7807,27 @@ function applyCodexCacheTelemetry(observability, provider, normalizedUsage) {
7074
7807
  promptTokens: normalizedUsage.telemetry.inputTokens
7075
7808
  });
7076
7809
  }
7077
- function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = persistedHistoryCount, toolOutputTruncationTokens, redactValue = (value) => value) {
7078
- const sanitized = sanitizeHistoryItemsForModel2(rawHistory, toolOutputTruncationTokens).map(
7079
- (item) => redactValue(item)
7080
- );
7081
- if (sanitized.length <= persistedHistoryCount) {
7810
+ function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = persistedHistoryCount, toolOutputTruncationTokens) {
7811
+ const modelReady = sanitizeHistoryItemsForModel2(rawHistory, toolOutputTruncationTokens);
7812
+ if (modelReady.length <= persistedHistoryCount) {
7082
7813
  return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
7083
7814
  }
7084
- const rows = sanitized.slice(persistedHistoryCount).filter((item) => !(item.type === "message" && item.role === "system")).map((item, offset) => ({
7085
- position: nextPosition + offset,
7086
- item
7087
- }));
7815
+ const rows = [];
7816
+ for (const [offset, item] of modelReady.slice(persistedHistoryCount).entries()) {
7817
+ if (item.type === "message" && item.role === "system") {
7818
+ continue;
7819
+ }
7820
+ rows.push({
7821
+ position: nextPosition + rows.length,
7822
+ item: normalizeProtocolJsonValue(
7823
+ item,
7824
+ `$[${persistedHistoryCount + offset}]`
7825
+ )
7826
+ });
7827
+ }
7088
7828
  return {
7089
7829
  rows,
7090
- nextWatermark: sanitized.length,
7830
+ nextWatermark: modelReady.length,
7091
7831
  nextPosition: nextPosition + rows.length
7092
7832
  };
7093
7833
  }
@@ -7111,12 +7851,6 @@ function selectRejectedProviderArtifactHistoryIds(activeHistory, candidates, wir
7111
7851
  }
7112
7852
  return selected;
7113
7853
  }
7114
- function secretRedactionModelInputFilter(redactValue) {
7115
- return ({ modelData }) => ({
7116
- ...modelData,
7117
- input: redactValue(modelData.input)
7118
- });
7119
- }
7120
7854
  function isModelOrToolProgressHistoryItem(item) {
7121
7855
  if (item.type === "message") {
7122
7856
  return item.role === "assistant";
@@ -7456,6 +8190,24 @@ async function refreshCappedCodexUsageRows(db, settings, workspaceId, accounts,
7456
8190
  function codexCredentialLeaseDeadlineExpired(confirmedUntilMs, nowMs = performance.now()) {
7457
8191
  return confirmedUntilMs === null || !Number.isFinite(confirmedUntilMs) || confirmedUntilMs <= nowMs;
7458
8192
  }
8193
+ async function finalizeDurableTurnOpStreams(sessions, fallback) {
8194
+ const candidates = /* @__PURE__ */ new Set();
8195
+ for (const session of sessions) {
8196
+ const candidate = session;
8197
+ if (typeof candidate?.finalizeOpStreamOps === "function") {
8198
+ candidates.add(candidate);
8199
+ }
8200
+ }
8201
+ if (candidates.size === 0 && fallback) {
8202
+ candidates.add(fallback);
8203
+ }
8204
+ for (const candidate of candidates) {
8205
+ try {
8206
+ await candidate.finalizeOpStreamOps();
8207
+ } catch {
8208
+ }
8209
+ }
8210
+ }
7459
8211
  function createRunAgentTurnActivity(services) {
7460
8212
  return async function runAgentTurn2(input) {
7461
8213
  const {
@@ -7499,6 +8251,10 @@ function createRunAgentTurnActivity(services) {
7499
8251
  acknowledgeQuiescence = true;
7500
8252
  noteCancellationRequested();
7501
8253
  };
8254
+ const acknowledgeRecoveryQuiescence = () => {
8255
+ acknowledgeQuiescence = true;
8256
+ noteCancellationRequested();
8257
+ };
7502
8258
  let turnId;
7503
8259
  let triggerEventId;
7504
8260
  const claimedResult = (result) => {
@@ -7600,7 +8356,7 @@ function createRunAgentTurnActivity(services) {
7600
8356
  workspaceId: input.workspaceId,
7601
8357
  turnId,
7602
8358
  reason,
7603
- errorName: error instanceof Error ? error.name : "unknown"
8359
+ ...safeErrorDiagnostic(error)
7604
8360
  });
7605
8361
  observability.incrementCounter({
7606
8362
  name: "opengeni_codex_lease_renewals_total",
@@ -7631,6 +8387,12 @@ function createRunAgentTurnActivity(services) {
7631
8387
  let lazyOwnedSandbox = null;
7632
8388
  let turnSandboxProvisioner = null;
7633
8389
  let setupBoxSession = null;
8390
+ const finalizeTurnOpStreamOps = async () => {
8391
+ await finalizeDurableTurnOpStreams(
8392
+ [lazyOwnedSandbox?.session, resolvedSandbox?.established.session],
8393
+ machinePrimarySession
8394
+ );
8395
+ };
7634
8396
  const onHomeSandboxRebound = (rebound) => {
7635
8397
  const current = resolvedSandbox;
7636
8398
  const previousSession = current?.established.session;
@@ -7861,7 +8623,7 @@ function createRunAgentTurnActivity(services) {
7861
8623
  sessionId: input.sessionId,
7862
8624
  sandboxGroupId: heartbeatGroupId,
7863
8625
  leaseEpoch: heartbeatEpoch,
7864
- error: error instanceof Error ? error.message : String(error)
8626
+ ...safeErrorDiagnostic(error)
7865
8627
  });
7866
8628
  }).finally(() => {
7867
8629
  rotationInFlight = null;
@@ -7969,6 +8731,28 @@ function createRunAgentTurnActivity(services) {
7969
8731
  }
7970
8732
  };
7971
8733
  let persistedHistoryCount = 0;
8734
+ const retainedScreenshotReceiptsByCallId = /* @__PURE__ */ new Map();
8735
+ const computerScreenshotCallIds = /* @__PURE__ */ new Set();
8736
+ const materializeScreenshotHistory = async (history) => {
8737
+ collectRetainedScreenshotReceipts(history, retainedScreenshotReceiptsByCallId);
8738
+ return await materializeRetainedScreenshotHistory({
8739
+ db,
8740
+ objectStorage,
8741
+ workspaceId: input.workspaceId,
8742
+ sessionId: input.sessionId,
8743
+ history
8744
+ });
8745
+ };
8746
+ const materializeScreenshotRunState = async (serialized) => {
8747
+ collectRetainedScreenshotRunStateReceipts(serialized, retainedScreenshotReceiptsByCallId);
8748
+ return await materializeRetainedScreenshotRunState({
8749
+ db,
8750
+ objectStorage,
8751
+ workspaceId: input.workspaceId,
8752
+ sessionId: input.sessionId,
8753
+ serialized
8754
+ });
8755
+ };
7972
8756
  let providerArtifactCandidates = {
7973
8757
  knownHistoryItemIds: [],
7974
8758
  historyItemIds: []
@@ -7978,36 +8762,42 @@ function createRunAgentTurnActivity(services) {
7978
8762
  if (!stream || !turnId) {
7979
8763
  return;
7980
8764
  }
8765
+ const durableTurnId = turnId;
7981
8766
  try {
7982
8767
  const rawHistory = stream.state.history;
7983
8768
  if (Array.isArray(rawHistory)) {
7984
- const { rows, nextWatermark, nextPosition } = historyRowsToAppend(
8769
+ const durableHistory = compactRetainedScreenshotHistory(
7985
8770
  rawHistory,
8771
+ retainedScreenshotReceiptsByCallId
8772
+ );
8773
+ const { rows, nextWatermark, nextPosition } = historyRowsToAppend(
8774
+ durableHistory,
7986
8775
  persistedHistoryCount,
7987
8776
  nextHistoryPosition,
7988
- modelRunSettings.modelToolOutputTruncationTokens,
7989
- (value) => redact(value)
8777
+ modelRunSettings.modelToolOutputTruncationTokens
7990
8778
  );
7991
8779
  const hasModelOrToolProgress = rows.some(
7992
8780
  (row) => isModelOrToolProgressHistoryItem(row.item)
7993
8781
  );
7994
8782
  const shouldAppendRows = rows.length > 0 && (!options.skipInputOnlyRows || hasModelOrToolProgress);
7995
8783
  if (shouldAppendRows) {
7996
- const appended = await appendSessionHistoryItems(db, {
7997
- accountId: input.accountId,
7998
- workspaceId: input.workspaceId,
7999
- sessionId: input.sessionId,
8000
- turnId,
8001
- expectedExecutionGeneration: executionGeneration,
8002
- expectedAttemptId: input.attemptId,
8003
- modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
8004
- items: rows
8784
+ await runMandatoryHistoryPersistenceStep("history_append", async () => {
8785
+ const appended = await appendSessionHistoryItems(db, {
8786
+ accountId: input.accountId,
8787
+ workspaceId: input.workspaceId,
8788
+ sessionId: input.sessionId,
8789
+ turnId: durableTurnId,
8790
+ expectedExecutionGeneration: executionGeneration,
8791
+ expectedAttemptId: input.attemptId,
8792
+ modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
8793
+ items: rows
8794
+ });
8795
+ if (!appended) {
8796
+ throw new TurnAttemptFencedError(
8797
+ "turn execution generation was fenced while saving conversation history"
8798
+ );
8799
+ }
8005
8800
  });
8006
- if (!appended) {
8007
- throw new TurnAttemptFencedError(
8008
- "turn execution generation was fenced while saving conversation history"
8009
- );
8010
- }
8011
8801
  }
8012
8802
  if (shouldAppendRows || !options.skipInputOnlyRows) {
8013
8803
  persistedHistoryCount = nextWatermark;
@@ -8016,32 +8806,25 @@ function createRunAgentTurnActivity(services) {
8016
8806
  }
8017
8807
  const envelope = sandboxStateEntryFromRunState(stream.state);
8018
8808
  if (envelope) {
8019
- await upsertSandboxSessionEnvelope(db, {
8020
- accountId: input.accountId,
8021
- workspaceId: input.workspaceId,
8022
- sessionId: input.sessionId,
8023
- envelope
8024
- });
8809
+ await runMandatoryHistoryPersistenceStep(
8810
+ "sandbox_envelope",
8811
+ () => upsertSandboxSessionEnvelope(db, {
8812
+ accountId: input.accountId,
8813
+ workspaceId: input.workspaceId,
8814
+ sessionId: input.sessionId,
8815
+ envelope
8816
+ })
8817
+ );
8025
8818
  }
8026
8819
  } catch (persistError) {
8027
8820
  console.error(
8028
8821
  "session history dual-write failed (run unaffected)",
8029
- safeErrorDiagnostic(persistError, (value) => String(redact(value)))
8822
+ safeErrorDiagnostic(persistError)
8030
8823
  );
8031
8824
  if (options.requireDurable) throw persistError;
8032
8825
  }
8033
8826
  };
8034
- let redact = createSecretRedactor([]);
8035
- const secretRedactions = /* @__PURE__ */ new Map();
8036
8827
  const publishedRunCredentialNotices = /* @__PURE__ */ new Set();
8037
- const registerSecretRedactions = (secrets) => {
8038
- for (const secret of secrets) {
8039
- if (!secretRedactions.has(secret.value)) secretRedactions.set(secret.value, secret.name);
8040
- }
8041
- redact = createSecretRedactor(
8042
- [...secretRedactions].map(([value, name]) => ({ name, value }))
8043
- );
8044
- };
8045
8828
  let variableSetId = "";
8046
8829
  let rigId = "";
8047
8830
  let rigVersionId = "";
@@ -8173,7 +8956,7 @@ function createRunAgentTurnActivity(services) {
8173
8956
  publish = async (events, immediate = false) => {
8174
8957
  const inputs = events.map((event) => ({
8175
8958
  ...event,
8176
- payload: redact(event.payload),
8959
+ payload: event.payload,
8177
8960
  turnId,
8178
8961
  producerId,
8179
8962
  producerSeq: ++producerSeq
@@ -8186,7 +8969,11 @@ function createRunAgentTurnActivity(services) {
8186
8969
  turnId,
8187
8970
  executionGeneration,
8188
8971
  input.attemptId,
8189
- inputs
8972
+ inputs,
8973
+ {
8974
+ onAppend: ({ durationSeconds }) => recordSessionEventAppendLatency(observability, { durationSeconds }),
8975
+ onPublish: ({ durationSeconds }) => recordSessionEventPublishLatency(observability, { durationSeconds })
8976
+ }
8190
8977
  );
8191
8978
  if (inputs.length > 0 && !appended.accepted) {
8192
8979
  throw new TurnAttemptFencedError("turn execution generation was fenced");
@@ -8231,12 +9018,18 @@ function createRunAgentTurnActivity(services) {
8231
9018
  } : void 0;
8232
9019
  const inputs = inputSettlement.events.map((event) => ({
8233
9020
  ...event,
8234
- payload: redact(event.payload),
9021
+ payload: event.payload,
8235
9022
  turnId,
8236
9023
  producerId,
8237
9024
  producerSeq: ++producerSeq
8238
9025
  }));
8239
- const redactedRunState = inputSettlement.runState ? redact(inputSettlement.runState) : void 0;
9026
+ const runState = inputSettlement.runState ? {
9027
+ ...inputSettlement.runState,
9028
+ serializedRunState: compactRetainedScreenshotRunState(
9029
+ inputSettlement.runState.serializedRunState,
9030
+ retainedScreenshotReceiptsByCallId
9031
+ )
9032
+ } : void 0;
8240
9033
  const result = await applySessionTurnSettlement(db, input.workspaceId, {
8241
9034
  sessionId: input.sessionId,
8242
9035
  turnId,
@@ -8246,7 +9039,7 @@ function createRunAgentTurnActivity(services) {
8246
9039
  sessionStatus: inputSettlement.sessionStatus,
8247
9040
  activeTurnId: inputSettlement.activeTurnId,
8248
9041
  events: inputs,
8249
- ...redactedRunState ? { runState: redactedRunState } : {},
9042
+ ...runState ? { runState } : {},
8250
9043
  ...recordingMutation ? { recording: recordingMutation } : {},
8251
9044
  ...compactionRequestFailure ? { compactionRequestFailure } : {}
8252
9045
  });
@@ -8538,10 +9331,11 @@ function createRunAgentTurnActivity(services) {
8538
9331
  labels: codexFleetShadowErrorMetricLabelsV1(shadowResult)
8539
9332
  });
8540
9333
  observability.warn("Codex adaptive fleet shadow decision failed open", {
8541
- workspaceId: input.workspaceId,
8542
9334
  stage: shadowResult.stage,
8543
9335
  reason: shadowResult.reason,
8544
- errorName: shadowResult.errorName,
9336
+ errorClass: "CodexFleetShadowOperationError",
9337
+ errorCode: "codex_fleet_shadow_failed",
9338
+ origin: "worker",
8545
9339
  payloadBytes: shadowResult.payloadBytes
8546
9340
  });
8547
9341
  }
@@ -8829,14 +9623,18 @@ function createRunAgentTurnActivity(services) {
8829
9623
  attemptId: input.attemptId,
8830
9624
  executionGeneration: turn.executionGeneration
8831
9625
  };
8832
- const [workspaceAgentInstructions, instructionPolicySnapshot, preferenceSnapshot] = await Promise.all([
8833
- resolveWorkspaceAgentInstructions(db, input.workspaceId),
9626
+ const [workspace, instructionPolicySnapshot, preferenceSnapshot] = await Promise.all([
9627
+ getWorkspace2(db, input.workspaceId),
8834
9628
  getOrCreateWorkspaceInstructionPolicySnapshot(db, governanceClaims),
8835
9629
  getOrCreatePreferenceRegistrySnapshot(db, governanceClaims).catch((error) => {
8836
9630
  if (error instanceof PreferenceRegistryInitiatorError) return null;
8837
9631
  throw error;
8838
9632
  })
8839
9633
  ]);
9634
+ if (!workspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
9635
+ const workspaceAgentInstructions = workspace.agentInstructions;
9636
+ const agentHumanInputEnabled = resolveWorkspaceAgentHumanInputEnabled(workspace.settings);
9637
+ assertWorkspaceHumanInputAllowed(agentHumanInputEnabled, "resume", humanInputResume !== null);
8840
9638
  const workspaceGovernance = renderWorkspaceGovernanceContext({
8841
9639
  instructionPolicy: instructionPolicySnapshot,
8842
9640
  preferences: preferenceSnapshot
@@ -8860,31 +9658,13 @@ function createRunAgentTurnActivity(services) {
8860
9658
  openaiReasoningEffort: turn.reasoningEffort,
8861
9659
  sandboxBackend: turn.sandboxBackend
8862
9660
  };
8863
- const sessionMcpHeaderNames = /* @__PURE__ */ new Map();
8864
9661
  const runSettings = await settingsWithSessionMcpServersForRun(
8865
9662
  db,
8866
9663
  input.workspaceId,
8867
9664
  input.sessionId,
8868
9665
  input.attemptId,
8869
- baseRunSettings,
8870
- {
8871
- onResolvedServers: (servers) => {
8872
- for (const server of servers) {
8873
- sessionMcpHeaderNames.set(server.id, server.headerNames);
8874
- }
8875
- }
8876
- }
9666
+ baseRunSettings
8877
9667
  );
8878
- registerSecretRedactions([
8879
- ...runSettings.accessKey ? [{ name: "OPENGENI_ACCESS_KEY", value: runSettings.accessKey }] : [],
8880
- ...runSettings.mcpServers.flatMap((server) => {
8881
- return headerSecretRedactions(
8882
- `MCP_${server.id.toUpperCase()}_STATIC`,
8883
- server.headers,
8884
- sessionMcpHeaderNames.get(server.id)
8885
- );
8886
- })
8887
- ]);
8888
9668
  const resolvedModel = runtime.resolveTurnModel(
8889
9669
  capabilitySettings,
8890
9670
  turnExecutionPolicy.productModelId
@@ -8900,38 +9680,6 @@ function createRunAgentTurnActivity(services) {
8900
9680
  );
8901
9681
  const modelHistoryProjector = async (items) => projectModelInputForCapabilities(await attachmentProjector(items), modelInputPolicy);
8902
9682
  const compactionModelHistoryProjector = async (items) => await modelHistoryProjector(projectHistoryForProvider2(items, providerApi));
8903
- const selectedProvider = resolvedModel?.provider;
8904
- const publicProviderHeaders = new Set(
8905
- selectedProvider?.publicDefaultHeaderNames?.map((name) => name.toLowerCase()) ?? []
8906
- );
8907
- const publicProviderQueries = new Set(
8908
- selectedProvider?.publicDefaultQueryNames?.map((name) => name.toLowerCase()) ?? []
8909
- );
8910
- const fallbackProviderApiKey = capabilitySettings.openaiProvider === "azure" ? capabilitySettings.azureOpenaiApiKey ?? capabilitySettings.azureOpenaiAdToken : capabilitySettings.openaiApiKey;
8911
- registerSecretRedactions([
8912
- ...selectedProvider?.apiKey ? [
8913
- {
8914
- name: `MODEL_${selectedProvider.id.toUpperCase()}_API_KEY`,
8915
- value: selectedProvider.apiKey
8916
- }
8917
- ] : !resolvedModel && fallbackProviderApiKey ? [
8918
- {
8919
- name: "MODEL_PROVIDER_API_KEY",
8920
- value: fallbackProviderApiKey
8921
- }
8922
- ] : [],
8923
- ...Object.entries(selectedProvider?.defaultHeaders ?? {}).flatMap(
8924
- ([name, value]) => publicProviderHeaders.has(name.toLowerCase()) ? [] : [
8925
- {
8926
- name: `MODEL_PROVIDER_HEADER_${name.toUpperCase()}`,
8927
- value
8928
- }
8929
- ]
8930
- ),
8931
- ...Object.entries(selectedProvider?.defaultQuery ?? {}).flatMap(
8932
- ([name, value]) => publicProviderQueries.has(name.toLowerCase()) ? [] : [{ name: `MODEL_PROVIDER_QUERY_${name.toUpperCase()}`, value }]
8933
- )
8934
- ]);
8935
9683
  modelRunSettings = resolvedModel ? settingsWithResolvedModelContext(runSettings, resolvedModel.configured) : runSettings;
8936
9684
  {
8937
9685
  const workspaceModelPolicy = await getWorkspaceModelPolicy2(db, input.workspaceId);
@@ -8957,13 +9705,6 @@ function createRunAgentTurnActivity(services) {
8957
9705
  input.workspaceId,
8958
9706
  effectiveCodexCredentialId ?? ""
8959
9707
  );
8960
- const registerSnapshot = async (snapshotPromise) => {
8961
- const snapshot = await snapshotPromise;
8962
- registerSecretRedactions([
8963
- { name: "CODEX_ACCESS_TOKEN", value: snapshot.accessToken }
8964
- ]);
8965
- return snapshot;
8966
- };
8967
9708
  return {
8968
9709
  clientVersion: CODEX_CLIENT_VERSION,
8969
9710
  // Backend sticky cache-routing key — the SAME id as the body's
@@ -8973,8 +9714,8 @@ function createRunAgentTurnActivity(services) {
8973
9714
  // (per-request shard lottery = prod's measured 48.6% on sol);
8974
9715
  // with it, resends pin to the warm shard (Codex CLI parity).
8975
9716
  sessionId: input.sessionId,
8976
- getToken: () => registerSnapshot(resolver.getToken()),
8977
- refresh: () => registerSnapshot(resolver.refresh()),
9717
+ getToken: () => resolver.getToken(),
9718
+ refresh: () => resolver.refresh(),
8978
9719
  resolveModel: buildModelResolver(
8979
9720
  CODEX_FALLBACK_MODEL_SLUGS,
8980
9721
  CODEX_FALLBACK_MODEL_SLUGS[0]
@@ -9174,6 +9915,7 @@ function createRunAgentTurnActivity(services) {
9174
9915
  force: true,
9175
9916
  clearRequestedCompaction: true,
9176
9917
  trigger: "operator",
9918
+ materializeHistory: materializeScreenshotHistory,
9177
9919
  projectModelInput: compactionModelHistoryProjector,
9178
9920
  ...compactionModeOptions
9179
9921
  }
@@ -9200,7 +9942,7 @@ function createRunAgentTurnActivity(services) {
9200
9942
  }
9201
9943
  );
9202
9944
  if (!isCompactionSummaryFailure(error)) throw error;
9203
- const errorMessage = String(redact(compactionFailureReasonFromError(error)));
9945
+ const errorMessage = String(compactionFailureReasonFromError(error));
9204
9946
  if (!await settle({
9205
9947
  events: [
9206
9948
  {
@@ -9306,9 +10048,6 @@ function createRunAgentTurnActivity(services) {
9306
10048
  rigDefaultEnvironmentValues,
9307
10049
  workspaceVariableSet?.values ?? {}
9308
10050
  );
9309
- registerSecretRedactions(
9310
- Object.entries(sandboxWorkspaceEnvironmentValues).map(([name, value]) => ({ name, value }))
9311
- );
9312
10051
  const routingOn = routingEnabled(settings);
9313
10052
  let activeSandboxPointer = routingOn ? await readActiveSandbox2(db, input.workspaceId, input.sessionId).catch(() => null) : null;
9314
10053
  let activeSandboxRecord = null;
@@ -9365,7 +10104,6 @@ function createRunAgentTurnActivity(services) {
9365
10104
  void 0
9366
10105
  ) : null;
9367
10106
  if (initialRunCredentialMaterial) {
9368
- registerSecretRedactions(initialRunCredentialMaterial.redactions);
9369
10107
  for (const payload of runCredentialAuthNeededPayloads(initialRunCredentialMaterial)) {
9370
10108
  publishedRunCredentialNotices.add(JSON.stringify(payload));
9371
10109
  await publish([{ type: "credential.auth_needed", payload }], true);
@@ -9378,7 +10116,10 @@ function createRunAgentTurnActivity(services) {
9378
10116
  cancellationSignal,
9379
10117
  void 0
9380
10118
  );
9381
- const establishPolicy = lazyProvisionEnabled(settings) && !machinePrimary && runSettings.sandboxBackend !== "none" ? "on-demand" : "eager";
10119
+ const establishPolicy = lazyProvisionEnabled(settings) && !machinePrimary && runSettings.sandboxBackend !== "none" && // Resolved run credentials must be written to one exact leased sandbox
10120
+ // before agent execution. A warm active pointer can bypass the lazy
10121
+ // provisioner entirely, which previously skipped materialization.
10122
+ !initialRunCredentialMaterial ? "on-demand" : "eager";
9382
10123
  if (activeSandboxBackend !== "selfhosted") {
9383
10124
  await assertGitHubResourcesRemainAuthorized(db, input.workspaceId, turnResources);
9384
10125
  }
@@ -9423,6 +10164,7 @@ function createRunAgentTurnActivity(services) {
9423
10164
  sandboxWorkspaceEnvironmentValues,
9424
10165
  {
9425
10166
  skipGitHubToken: activeSandboxBackend === "selfhosted",
10167
+ skipToolspace: activeSandboxBackend === "selfhosted",
9426
10168
  deferGitHubToken: activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
9427
10169
  scope: connectionScope,
9428
10170
  ...gitCredentialAuthority ? { authority: gitCredentialAuthority } : {},
@@ -9434,17 +10176,6 @@ function createRunAgentTurnActivity(services) {
9434
10176
  cancellationSignal,
9435
10177
  void 0
9436
10178
  );
9437
- registerSecretRedactions([
9438
- ...Object.entries(sandboxGitTokens ?? {}).flatMap(
9439
- ([provider, value]) => value ? [{ name: `GIT_${provider.toUpperCase()}_TOKEN`, value }] : []
9440
- ),
9441
- ...sandboxGitToken ? [{ name: "GIT_TOKEN", value: sandboxGitToken }] : [],
9442
- ...(sandboxGitCredentialBindings ?? []).map((binding) => ({
9443
- name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
9444
- value: binding.token
9445
- })),
9446
- ...sandboxToolspaceToken ? [{ name: "TOOLSPACE_TOKEN", value: sandboxToolspaceToken }] : []
9447
- ]);
9448
10179
  const sandboxToolspaceTokenFile = sandboxToolspaceToken ? toolspaceTokenFileFromEnvironment(sandboxEnvironment, input.sessionId) : void 0;
9449
10180
  const initialGitCredentials = sandboxGitCredentialBindings ? {
9450
10181
  bindings: sandboxGitCredentialBindings,
@@ -9477,12 +10208,6 @@ function createRunAgentTurnActivity(services) {
9477
10208
  );
9478
10209
  if (binding) {
9479
10210
  assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
9480
- registerSecretRedactions([
9481
- {
9482
- name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
9483
- value: binding.token
9484
- }
9485
- ]);
9486
10211
  }
9487
10212
  pendingBinding = binding;
9488
10213
  return binding ? {
@@ -9559,7 +10284,6 @@ function createRunAgentTurnActivity(services) {
9559
10284
  toolspaceAuthority
9560
10285
  );
9561
10286
  if (material) {
9562
- registerSecretRedactions([{ name: "TOOLSPACE_TOKEN", value: material.token }]);
9563
10287
  }
9564
10288
  return material;
9565
10289
  };
@@ -9667,7 +10391,6 @@ function createRunAgentTurnActivity(services) {
9667
10391
  );
9668
10392
  return;
9669
10393
  }
9670
- registerSecretRedactions(material.redactions);
9671
10394
  await runWorkspaceMutationForSandbox(
9672
10395
  requireTargetSandbox(),
9673
10396
  "runCredentialMaterialization",
@@ -9742,7 +10465,7 @@ function createRunAgentTurnActivity(services) {
9742
10465
  sandboxHolderId = managedOwnership?.holderId ?? null;
9743
10466
  sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
9744
10467
  if (machinePrimary) {
9745
- const machineOpStream = settings.agentOpStreamEnabled === true ? (await getEnrollment(db, input.workspaceId, activeSandboxRecord.enrollmentId))?.opStream === true : false;
10468
+ const machineOpStream = settings.agentOpStreamEnabled === true ? (await getEnrollment2(db, input.workspaceId, activeSandboxRecord.enrollmentId))?.opStream === true : false;
9746
10469
  const established = await establishSelfhostedTurnSession(
9747
10470
  {
9748
10471
  db,
@@ -9774,6 +10497,7 @@ function createRunAgentTurnActivity(services) {
9774
10497
  db,
9775
10498
  settings,
9776
10499
  bus,
10500
+ opJournal,
9777
10501
  onOp: machineOpObserver.observer,
9778
10502
  onSandboxOperation: sandboxOperationObserver,
9779
10503
  onHomeSandboxRebound
@@ -9864,6 +10588,7 @@ function createRunAgentTurnActivity(services) {
9864
10588
  db,
9865
10589
  settings,
9866
10590
  bus,
10591
+ opJournal,
9867
10592
  onSandboxOperation: sandboxOperationObserver,
9868
10593
  onHomeSandboxLost: publishSandboxLost,
9869
10594
  onHomeSandboxRebound
@@ -9942,9 +10667,6 @@ function createRunAgentTurnActivity(services) {
9942
10667
  const resolveCredential = async (request) => {
9943
10668
  const result = await resolveFrozenCredential(request);
9944
10669
  if (result.status === "ok") {
9945
- registerSecretRedactions(
9946
- headerSecretRedactions("MCP", result.headers, Object.keys(result.headers))
9947
- );
9948
10670
  }
9949
10671
  return result;
9950
10672
  };
@@ -9960,9 +10682,6 @@ function createRunAgentTurnActivity(services) {
9960
10682
  clientVersion: CODEX_CLIENT_VERSION,
9961
10683
  withAuthorization: async (use) => {
9962
10684
  const snapshot = await resolver.getToken();
9963
- registerSecretRedactions([
9964
- { name: "CODEX_APPS_ACCESS_TOKEN", value: snapshot.accessToken }
9965
- ]);
9966
10685
  return await withCodexAppsRequestAuthorization(
9967
10686
  db,
9968
10687
  { workspaceId: input.workspaceId, credentialId: codexAppsCredentialId },
@@ -10042,6 +10761,7 @@ function createRunAgentTurnActivity(services) {
10042
10761
  latencyMode: turnExecutionPolicy.latencyMode,
10043
10762
  ...serviceTier ? { serviceTier } : {},
10044
10763
  ...humanInputResume ? { humanInputResponse: humanInputResume } : {},
10764
+ humanInputEnabled: agentHumanInputEnabled,
10045
10765
  genesisTitleHint: isGenesisTurn,
10046
10766
  persistentSessionSettings: {
10047
10767
  titleIsSet: Boolean(session.title?.trim())
@@ -10060,14 +10780,9 @@ function createRunAgentTurnActivity(services) {
10060
10780
  ...activeSandboxBackend !== "selfhosted" && sandboxGitTokens ? { gitTokenSeeds: sandboxGitTokens } : {},
10061
10781
  ...activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings ? { gitCredentialBindings: sandboxGitCredentialBindings } : {},
10062
10782
  ...activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken ? { gitTokenSeed: sandboxGitToken } : {},
10063
- // Toolspace is delivered on EVERY backend including selfhosted. The git-
10064
- // token skip does NOT transfer: that token is inert on a connected
10065
- // machine (it uses its own git creds), but the toolspace token is the
10066
- // machine's ONLY path to programmatic tool calling and grants no more
10067
- // than toolspace:call for its own session (own-session-bound,
10068
- // short-lived and renewable, per-active-turn budgeted, approval-tools
10069
- // excluded). The runtime seeds it to the box's token file over the same
10070
- // exec channel, off-manifest, on every backend.
10783
+ // Toolspace delivery is managed-sandbox-only. Connected Machines own any
10784
+ // manually configured API credentials and must never be contacted during
10785
+ // turn admission merely to seed OpenGeni tooling.
10071
10786
  ...sandboxToolspaceToken ? {
10072
10787
  toolspaceTokenSeed: sandboxToolspaceToken,
10073
10788
  toolspaceTokenSessionId: input.sessionId
@@ -10327,6 +11042,7 @@ function createRunAgentTurnActivity(services) {
10327
11042
  db,
10328
11043
  settings,
10329
11044
  bus,
11045
+ opJournal,
10330
11046
  onOp: machineOpObserver.observer,
10331
11047
  onSandboxOperation: sandboxOperationObserver,
10332
11048
  onHomeSandboxLost: publishSandboxLost,
@@ -10417,7 +11133,7 @@ function createRunAgentTurnActivity(services) {
10417
11133
  }
10418
11134
  );
10419
11135
  if (!isCompactionSummaryFailure(error)) throw error;
10420
- const errorMessage = String(redact(compactionFailureReasonFromError(error)));
11136
+ const errorMessage = String(compactionFailureReasonFromError(error));
10421
11137
  if (!await settle({
10422
11138
  events: [
10423
11139
  {
@@ -10502,10 +11218,12 @@ function createRunAgentTurnActivity(services) {
10502
11218
  force: true,
10503
11219
  clearRequestedCompaction: true,
10504
11220
  trigger: "operator",
11221
+ materializeHistory: materializeScreenshotHistory,
10505
11222
  projectModelInput: compactionModelHistoryProjector,
10506
11223
  ...compactionModeOptions
10507
11224
  } : {
10508
11225
  trigger: "auto",
11226
+ materializeHistory: materializeScreenshotHistory,
10509
11227
  projectModelInput: compactionModelHistoryProjector,
10510
11228
  ...compactionModeOptions
10511
11229
  }
@@ -10539,11 +11257,11 @@ function createRunAgentTurnActivity(services) {
10539
11257
  }
10540
11258
  );
10541
11259
  if (!isCompactionSummaryFailure(compactError)) throw compactError;
10542
- const errorMessage = String(redact(compactionFailureReasonFromError(compactError)));
11260
+ const errorMessage = String(compactionFailureReasonFromError(compactError));
10543
11261
  observability.error("context compaction failed", {
10544
11262
  sessionId: input.sessionId,
10545
11263
  turnId,
10546
- error: errorMessage
11264
+ ...safeErrorDiagnostic(compactError)
10547
11265
  });
10548
11266
  if (!await settle({
10549
11267
  events: [
@@ -10640,6 +11358,8 @@ function createRunAgentTurnActivity(services) {
10640
11358
  ...unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {},
10641
11359
  ...runCredentialsNote ? { runCredentialsNote } : {},
10642
11360
  providerApi,
11361
+ materializeModelHistory: materializeScreenshotHistory,
11362
+ materializeSerializedRunState: materializeScreenshotRunState,
10643
11363
  projectModelHistory: modelHistoryProjector
10644
11364
  });
10645
11365
  runInput = prepared.input;
@@ -10675,6 +11395,7 @@ function createRunAgentTurnActivity(services) {
10675
11395
  force: true,
10676
11396
  ...triggerLabel === "operator" ? { clearRequestedCompaction: true } : {},
10677
11397
  trigger: triggerLabel,
11398
+ materializeHistory: materializeScreenshotHistory,
10678
11399
  projectModelInput: compactionModelHistoryProjector,
10679
11400
  ...compactionModeOptions
10680
11401
  }
@@ -10706,13 +11427,18 @@ function createRunAgentTurnActivity(services) {
10706
11427
  const runStreamOnce = async () => {
10707
11428
  if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
10708
11429
  const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
11430
+ await attachRunCredentialRenewal(
11431
+ eagerSetupSession,
11432
+ resolvedSandbox
11433
+ );
11434
+ const eagerCredentialSetupSession = initialRunCredentialMaterial ? withRunCredentialsSession(eagerSetupSession, input.sessionId) : eagerSetupSession;
10709
11435
  await runWorkspaceMutationForSandbox(
10710
11436
  resolvedSandbox,
10711
11437
  "eagerOwnedSandboxSetup",
10712
11438
  async () => await runOwnedSandboxSetup(
10713
11439
  agent,
10714
11440
  ownedEstablished.session,
10715
- eagerSetupSession,
11441
+ eagerCredentialSetupSession,
10716
11442
  {
10717
11443
  settings: runSettings,
10718
11444
  environment: sandboxEnvironment,
@@ -10770,7 +11496,7 @@ function createRunAgentTurnActivity(services) {
10770
11496
  } : {},
10771
11497
  ...sandboxToolspaceToken && sandboxToolspaceTokenExpiresAt && !lazyOwnedSandbox ? {
10772
11498
  onToolspaceTokenSessionReady: async (tokenSession) => {
10773
- const renewalSession = activeSandboxBackend === "selfhosted" ? tokenSession : setupBoxSession ?? tokenSession;
11499
+ const renewalSession = setupBoxSession ?? tokenSession;
10774
11500
  await attachToolspaceTokenRenewal(renewalSession);
10775
11501
  }
10776
11502
  } : {},
@@ -10785,7 +11511,6 @@ function createRunAgentTurnActivity(services) {
10785
11511
  } : {},
10786
11512
  contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
10787
11513
  contextCompactionRequested: () => isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
10788
- callModelInputFilter: secretRedactionModelInputFilter((value) => redact(value)),
10789
11514
  ...toolCancellationFenceRef.current ? { turnToolCancellationFence: toolCancellationFenceRef.current } : {}
10790
11515
  });
10791
11516
  };
@@ -10816,6 +11541,7 @@ function createRunAgentTurnActivity(services) {
10816
11541
  }
10817
11542
  let stableToolCallIdsToClear = null;
10818
11543
  let completedCurrentToolBatch = false;
11544
+ let retainedScreenshotMetadata = null;
10819
11545
  const responseResult = await processModelResponseTerminalEvent({
10820
11546
  event: next.value,
10821
11547
  state: modelResponseState,
@@ -10866,7 +11592,7 @@ function createRunAgentTurnActivity(services) {
10866
11592
  } catch (limitError) {
10867
11593
  let serializedRunState = null;
10868
11594
  try {
10869
- serializedRunState = String(redact(stream.state.toString()));
11595
+ serializedRunState = String(stream.state.toString());
10870
11596
  } catch {
10871
11597
  serializedRunState = null;
10872
11598
  }
@@ -10888,7 +11614,7 @@ function createRunAgentTurnActivity(services) {
10888
11614
  modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
10889
11615
  callId: pendingToolCall.callId,
10890
11616
  callType: pendingToolCall.callType,
10891
- callItem: redact(pendingToolCall.callItem)
11617
+ callItem: pendingToolCall.callItem
10892
11618
  });
10893
11619
  if (!registered.accepted) {
10894
11620
  throw new TurnAttemptFencedError(
@@ -10896,9 +11622,33 @@ function createRunAgentTurnActivity(services) {
10896
11622
  );
10897
11623
  }
10898
11624
  currentToolBatchCallIds.add(pendingToolCall.callId);
11625
+ if (pendingToolCall.callName === "computer_screenshot") {
11626
+ computerScreenshotCallIds.add(pendingToolCall.callId);
11627
+ }
10899
11628
  }
10900
11629
  const completedToolCall = completedToolCallFromSdkEvent(next.value);
10901
11630
  if (completedToolCall) {
11631
+ const typedScreenshot = typedScreenshotFromSdkEvent(next.value);
11632
+ if (typedScreenshot && typedScreenshot.callId === completedToolCall.callId && computerScreenshotCallIds.has(completedToolCall.callId)) {
11633
+ retainedScreenshotMetadata = await retainComputerScreenshot({
11634
+ db,
11635
+ objectStorage,
11636
+ accountId: input.accountId,
11637
+ workspaceId: input.workspaceId,
11638
+ sessionId: input.sessionId,
11639
+ turnId: activeTurnId,
11640
+ attemptId: input.attemptId,
11641
+ output: typedScreenshot
11642
+ });
11643
+ retainedScreenshotReceiptsByCallId.set(
11644
+ completedToolCall.callId,
11645
+ retainedScreenshotMetadata
11646
+ );
11647
+ }
11648
+ const durableResultItem = retainedScreenshotMetadata ? compactRetainedScreenshotHistory(
11649
+ [completedToolCall.resultItem],
11650
+ retainedScreenshotReceiptsByCallId
11651
+ )[0] ?? completedToolCall.resultItem : completedToolCall.resultItem;
10902
11652
  const recorded = await recordPendingSessionToolCallResult(db, {
10903
11653
  accountId: input.accountId,
10904
11654
  workspaceId: input.workspaceId,
@@ -10908,7 +11658,7 @@ function createRunAgentTurnActivity(services) {
10908
11658
  attemptId: input.attemptId,
10909
11659
  callId: completedToolCall.callId,
10910
11660
  modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
10911
- resultItem: redact(completedToolCall.resultItem)
11661
+ resultItem: durableResultItem
10912
11662
  });
10913
11663
  if (!recorded.accepted) {
10914
11664
  throw new TurnAttemptFencedError(
@@ -10927,7 +11677,16 @@ function createRunAgentTurnActivity(services) {
10927
11677
  completedCurrentToolBatch = currentBatchIsStable;
10928
11678
  }
10929
11679
  }
10930
- const normalized = normalizeSdkEvent(next.value);
11680
+ const normalized = normalizeSdkEvent(
11681
+ next.value,
11682
+ retainedScreenshotMetadata ? {
11683
+ toolOutputOverride: retainedScreenshotMetadata,
11684
+ retainedOutputEvidence: retainedScreenshotMetadata.available ? retainedScreenshotMetadata : {
11685
+ available: false,
11686
+ reason: retainedScreenshotMetadata.reason
11687
+ }
11688
+ } : {}
11689
+ );
10931
11690
  for (const event of normalized) {
10932
11691
  streamTiming.onEvent(event.type);
10933
11692
  await batcher.push(event);
@@ -10951,6 +11710,9 @@ function createRunAgentTurnActivity(services) {
10951
11710
  currentToolBatchCallIds = /* @__PURE__ */ new Set();
10952
11711
  currentToolBatchCompletedCallIds = /* @__PURE__ */ new Set();
10953
11712
  }
11713
+ for (const callId of stableToolCallIdsToClear) {
11714
+ computerScreenshotCallIds.delete(callId);
11715
+ }
10954
11716
  }
10955
11717
  }
10956
11718
  } finally {
@@ -11050,6 +11812,13 @@ function createRunAgentTurnActivity(services) {
11050
11812
  await reconcileConversationTruth();
11051
11813
  const approvals = runtime.serializeApprovals(stream.interruptions);
11052
11814
  const humanInputInterruptions = runtime.serializeHumanInputRequests?.(stream.interruptions) ?? [];
11815
+ const latestWorkspace = await getWorkspace2(db, input.workspaceId);
11816
+ if (!latestWorkspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
11817
+ assertWorkspaceHumanInputAllowed(
11818
+ resolveWorkspaceAgentHumanInputEnabled(latestWorkspace.settings),
11819
+ "interruption",
11820
+ humanInputInterruptions.length > 0
11821
+ );
11053
11822
  const humanInputRequests = await Promise.all(
11054
11823
  humanInputInterruptions.map(async (interruption) => {
11055
11824
  const id = stableHumanInputRequestId(
@@ -11115,17 +11884,13 @@ function createRunAgentTurnActivity(services) {
11115
11884
  })) {
11116
11885
  return claimedResult({ status: "cancelled" });
11117
11886
  }
11887
+ await finalizeTurnOpStreamOps();
11118
11888
  activityStatus = "requires_action";
11119
11889
  return claimedResult({ status: "requires_action" });
11120
11890
  }
11121
- const finalOutput = String(redact(String(stream.finalOutput ?? "")));
11122
- await reconcileConversationTruth();
11123
- if (machinePrimarySession) {
11124
- try {
11125
- await machinePrimarySession.finalizeOpStreamOps();
11126
- } catch {
11127
- }
11128
- }
11891
+ const finalOutput = String(stream.finalOutput ?? "");
11892
+ await reconcileConversationTruth({ requireDurable: true });
11893
+ await finalizeTurnOpStreamOps();
11129
11894
  if (!await settle({
11130
11895
  events: [
11131
11896
  {
@@ -11183,8 +11948,7 @@ function createRunAgentTurnActivity(services) {
11183
11948
  sessionId: input.sessionId,
11184
11949
  turnId: activeTurnId,
11185
11950
  reason: recoveryKind,
11186
- code: overflow?.code ? String(redact(overflow.code)) : void 0,
11187
- error: String(redact(overflow?.message ?? compactionNeeded?.message ?? "")),
11951
+ ...safeErrorDiagnostic(attemptError),
11188
11952
  signalTokens: compactionNeeded?.signalTokens,
11189
11953
  thresholdTokens: compactionNeeded?.thresholdTokens
11190
11954
  });
@@ -11226,13 +11990,11 @@ function createRunAgentTurnActivity(services) {
11226
11990
  );
11227
11991
  compactionRequestCleared = landmark.requestConsumed;
11228
11992
  if (!isCompactionSummaryFailure(compactError)) throw compactError;
11229
- compactionFailureMessage = String(
11230
- redact(compactionFailureReasonFromError(compactError))
11231
- );
11993
+ compactionFailureMessage = String(compactionFailureReasonFromError(compactError));
11232
11994
  observability.warn("context compaction recovery compaction failed", {
11233
11995
  sessionId: input.sessionId,
11234
11996
  turnId: activeTurnId,
11235
- error: compactionFailureMessage
11997
+ ...safeErrorDiagnostic(compactError)
11236
11998
  });
11237
11999
  }
11238
12000
  if (!compactionHandled) {
@@ -11305,6 +12067,7 @@ function createRunAgentTurnActivity(services) {
11305
12067
  turnMetricOutcome = "cancelled";
11306
12068
  return claimedResult({ status: "cancelled" });
11307
12069
  }
12070
+ acknowledgeRecoveryQuiescence();
11308
12071
  await publishDurableSessionEvents(
11309
12072
  bus,
11310
12073
  input.workspaceId,
@@ -11322,7 +12085,7 @@ function createRunAgentTurnActivity(services) {
11322
12085
  } catch (recoveryError) {
11323
12086
  console.error(
11324
12087
  "sandbox lease supersession recovery failed",
11325
- safeErrorDiagnostic(recoveryError, (value) => String(redact(value)))
12088
+ safeErrorDiagnostic(recoveryError)
11326
12089
  );
11327
12090
  throw recoveryError;
11328
12091
  }
@@ -11349,6 +12112,7 @@ function createRunAgentTurnActivity(services) {
11349
12112
  turnMetricOutcome = "cancelled";
11350
12113
  return claimedResult({ status: "cancelled" });
11351
12114
  }
12115
+ acknowledgeRecoveryQuiescence();
11352
12116
  await publishDurableSessionEvents(
11353
12117
  bus,
11354
12118
  input.workspaceId,
@@ -11364,7 +12128,7 @@ function createRunAgentTurnActivity(services) {
11364
12128
  } catch (recoveryError) {
11365
12129
  console.error(
11366
12130
  "sandbox deadline rotation recovery failed",
11367
- safeErrorDiagnostic(recoveryError, (value) => String(redact(value)))
12131
+ safeErrorDiagnostic(recoveryError)
11368
12132
  );
11369
12133
  throw recoveryError;
11370
12134
  }
@@ -11387,6 +12151,7 @@ function createRunAgentTurnActivity(services) {
11387
12151
  turnMetricOutcome = "cancelled";
11388
12152
  return claimedResult({ status: "cancelled" });
11389
12153
  }
12154
+ acknowledgeRecoveryQuiescence();
11390
12155
  await publishDurableSessionEvents(
11391
12156
  bus,
11392
12157
  input.workspaceId,
@@ -11399,7 +12164,7 @@ function createRunAgentTurnActivity(services) {
11399
12164
  } catch (recoveryError) {
11400
12165
  console.error(
11401
12166
  "worker-shutdown recovery checkpoint failed",
11402
- safeErrorDiagnostic(recoveryError, (value) => String(redact(value)))
12167
+ safeErrorDiagnostic(recoveryError)
11403
12168
  );
11404
12169
  throw recoveryError;
11405
12170
  }
@@ -11471,11 +12236,11 @@ function createRunAgentTurnActivity(services) {
11471
12236
  await reconcileConversationTruth({ requireDurable: true });
11472
12237
  }
11473
12238
  checkpointDurable = true;
11474
- } catch (checkpointError) {
12239
+ } catch {
11475
12240
  observability.warn("Codex lease-loss checkpoint failed; refusing automatic turn replay", {
11476
- workspaceId: input.workspaceId,
11477
- turnId: lostTurnId,
11478
- errorName: checkpointError instanceof Error ? checkpointError.name : "unknown"
12241
+ errorClass: "CodexCheckpointOperationError",
12242
+ errorCode: "codex_lease_loss_checkpoint_failed",
12243
+ origin: "worker"
11479
12244
  });
11480
12245
  }
11481
12246
  const settlement = await settleCodexCredentialLeaseLoss(db, {
@@ -11550,16 +12315,16 @@ function createRunAgentTurnActivity(services) {
11550
12315
  await flushRuntimeBatcher();
11551
12316
  await reconcileConversationTruth({ requireDurable: true });
11552
12317
  checkpointDurable = true;
11553
- } catch (checkpointError) {
12318
+ } catch {
11554
12319
  observability.incrementCounter({
11555
12320
  name: "opengeni_codex_failover_checkpoints_total",
11556
12321
  help: "Durable Codex failover checkpoint attempts by outcome.",
11557
12322
  labels: { workspace_key: codexWorkspaceKey, outcome: "failed" }
11558
12323
  });
11559
12324
  observability.warn("Codex failover checkpoint failed; refusing automatic replay", {
11560
- workspaceId: input.workspaceId,
11561
- turnId,
11562
- errorName: checkpointError instanceof Error ? checkpointError.name : "unknown"
12325
+ errorClass: "CodexCheckpointOperationError",
12326
+ errorCode: "codex_failover_checkpoint_failed",
12327
+ origin: "worker"
11563
12328
  });
11564
12329
  }
11565
12330
  if (checkpointDurable) {
@@ -11984,6 +12749,7 @@ function createRunAgentTurnActivity(services) {
11984
12749
  return claimedResult({ status: "cancelled" });
11985
12750
  }
11986
12751
  if (recovery.action === "recovering") {
12752
+ acknowledgeRecoveryQuiescence();
11987
12753
  await publishDurableSessionEvents(
11988
12754
  bus,
11989
12755
  input.workspaceId,
@@ -11996,7 +12762,7 @@ function createRunAgentTurnActivity(services) {
11996
12762
  return claimedResult({ status: "recovering" });
11997
12763
  }
11998
12764
  }
11999
- const failure = redact(agentRunFailurePayload(error, { isCodexTurn }));
12765
+ const failure = agentRunFailurePayload(error, { isCodexTurn });
12000
12766
  if (isSessionEventPersistenceError(error)) {
12001
12767
  observability.error("session event persistence failed", {
12002
12768
  accountId: input.accountId,
@@ -12017,7 +12783,8 @@ function createRunAgentTurnActivity(services) {
12017
12783
  dbColumn: error.details.database.column,
12018
12784
  dbDataType: error.details.database.dataType,
12019
12785
  dbConstraint: error.details.database.constraint,
12020
- dbRoutine: error.details.database.routine
12786
+ dbRoutine: error.details.database.routine,
12787
+ error: error.message
12021
12788
  });
12022
12789
  }
12023
12790
  if (failure.retryable && publish && turnId && turnStartedPublished) {
@@ -12049,6 +12816,7 @@ function createRunAgentTurnActivity(services) {
12049
12816
  turnMetricOutcome = "cancelled";
12050
12817
  return claimedResult({ status: "cancelled" });
12051
12818
  }
12819
+ acknowledgeRecoveryQuiescence();
12052
12820
  await publishDurableSessionEvents(
12053
12821
  bus,
12054
12822
  input.workspaceId,
@@ -12129,17 +12897,29 @@ function createRunAgentTurnActivity(services) {
12129
12897
  const credentialSessionToClear = runCredentialSession;
12130
12898
  runCredentialSession = null;
12131
12899
  if (credentialSessionToClear) {
12132
- await runWorkspaceMutationForSandbox(
12133
- requireResolvedSandboxForMutation(
12134
- "Run credential cleanup has no exact sandbox lease target"
12900
+ const clearAttemptCredentials = async () => await clearRunCredentialsForAttempt(credentialSessionToClear, {
12901
+ sessionId: input.sessionId,
12902
+ attemptId: input.attemptId,
12903
+ executionGeneration
12904
+ });
12905
+ await clearAttemptCredentialsWithSettledFence({
12906
+ activityStatus,
12907
+ runWorkspaceFencedClear: async () => await runWorkspaceMutationForSandbox(
12908
+ requireResolvedSandboxForMutation(
12909
+ "Run credential cleanup has no exact sandbox lease target"
12910
+ ),
12911
+ "runCredentialAttemptClear",
12912
+ clearAttemptCredentials
12135
12913
  ),
12136
- "runCredentialAttemptClear",
12137
- async () => await clearRunCredentialsForAttempt(credentialSessionToClear, {
12138
- sessionId: input.sessionId,
12139
- attemptId: input.attemptId,
12140
- executionGeneration
12141
- })
12142
- );
12914
+ clearExactAttempt: clearAttemptCredentials,
12915
+ onSettledAttemptFence: () => {
12916
+ observability.info("retrying exact run credential cleanup after turn settlement", {
12917
+ "opengeni.session_id": input.sessionId,
12918
+ "opengeni.turn_id": turnId ?? "",
12919
+ "opengeni.attempt_id": input.attemptId
12920
+ });
12921
+ }
12922
+ });
12143
12923
  }
12144
12924
  await drainAttemptOwnedSandboxWriters({
12145
12925
  // Normal turn completion owns the same process boundary as
@@ -12197,18 +12977,18 @@ function createRunAgentTurnActivity(services) {
12197
12977
  onReceiptFailure: (error) => {
12198
12978
  console.error(
12199
12979
  "agent turn quiescence receipt exhausted; signalling proof",
12200
- safeErrorDiagnostic(error, (value) => String(redact(value)))
12980
+ safeErrorDiagnostic(error)
12201
12981
  );
12202
12982
  },
12203
12983
  onPublishFailure: (error) => {
12204
12984
  console.error(
12205
12985
  "agent turn quiescence event fanout failed",
12206
- safeErrorDiagnostic(error, (value) => String(redact(value)))
12986
+ safeErrorDiagnostic(error)
12207
12987
  );
12208
12988
  },
12209
12989
  onSignalFailure: (error, attempt, retryMs) => {
12210
12990
  console.error("agent turn quiescence proof signal failed; retrying", {
12211
- error: safeErrorDiagnostic(error, (value) => String(redact(value))),
12991
+ error: safeErrorDiagnostic(error),
12212
12992
  attempt,
12213
12993
  retryMs
12214
12994
  });
@@ -12394,7 +13174,7 @@ function createRunAgentTurnActivity(services) {
12394
13174
  sandboxToRelease.release().catch((releaseError) => {
12395
13175
  console.error(
12396
13176
  "sandbox lease release failed (turn outcome unaffected)",
12397
- safeErrorDiagnostic(releaseError, (value) => String(redact(value)))
13177
+ safeErrorDiagnostic(releaseError)
12398
13178
  );
12399
13179
  }),
12400
13180
  finalizerSignal
@@ -12404,7 +13184,7 @@ function createRunAgentTurnActivity(services) {
12404
13184
  finalizationError ??= error;
12405
13185
  console.error(
12406
13186
  "agent turn finalization failed (turn outcome unaffected)",
12407
- safeErrorDiagnostic(error, (value) => String(redact(value)))
13187
+ safeErrorDiagnostic(error)
12408
13188
  );
12409
13189
  } finally {
12410
13190
  if (!sandboxResumeController.signal.aborted) {
@@ -12459,10 +13239,7 @@ function createRunAgentTurnActivity(services) {
12459
13239
  "opengeni.duration_ms": Math.round(durationSeconds * 1e3),
12460
13240
  "opengeni.finalization_duration_ms": Math.round(finalizationDurationSeconds * 1e3)
12461
13241
  },
12462
- error: finalizationError || activityError ? safeErrorForTelemetry(
12463
- finalizationError ?? activityError,
12464
- (value) => String(redact(value))
12465
- ) : void 0
13242
+ error: finalizationError || activityError ? safeErrorForTelemetry(finalizationError ?? activityError) : void 0
12466
13243
  });
12467
13244
  assertPhysicalToolQuiescenceForCancellation({
12468
13245
  acknowledgeQuiescence,
@@ -12525,6 +13302,15 @@ function isTransientProviderError(error) {
12525
13302
  );
12526
13303
  }
12527
13304
  function agentRunFailurePayload(error, options = {}) {
13305
+ if (error instanceof MandatoryHistoryPersistenceError) {
13306
+ const underlying = isSessionEventPersistenceError(error.cause) ? agentRunFailurePayload(error.cause, options) : {
13307
+ error: error.cause instanceof Error ? error.cause.message : String(error.cause)
13308
+ };
13309
+ return {
13310
+ ...underlying,
13311
+ historyPersistenceStage: error.stage
13312
+ };
13313
+ }
12528
13314
  const message = error instanceof Error ? error.message : String(error);
12529
13315
  const status = typeof error === "object" && error !== null ? Number(
12530
13316
  error.status ?? error.statusCode
@@ -12539,10 +13325,8 @@ function agentRunFailurePayload(error, options = {}) {
12539
13325
  }
12540
13326
  if (isSessionEventPersistenceError(error)) {
12541
13327
  const { details } = error;
12542
- const eventLabel = details.eventTypes.join(", ") || "session events";
12543
- const failureLabel = details.code === "db_deadlock" ? "Database deadlock" : details.code === "db_serialization_failure" ? "Database serialization failure" : "Database failure";
12544
13328
  return {
12545
- error: `${failureLabel} while persisting ${eventLabel}. The completed provider call and external effects were not retried.`,
13329
+ error: error.message,
12546
13330
  code: details.code,
12547
13331
  detail: details.retryOutcome === "exhausted" ? `The idempotent persistence transaction failed after ${details.attempts} attempts.` : "The database rejected the idempotent persistence transaction.",
12548
13332
  correlationId: details.correlationId,
@@ -12584,7 +13368,8 @@ function agentRunFailurePayload(error, options = {}) {
12584
13368
  return {
12585
13369
  error: "A required MCP server was temporarily unreachable. The same turn will retry after a short delay.",
12586
13370
  code: "mcp_transport_unavailable",
12587
- retryable: true
13371
+ retryable: true,
13372
+ detail: message
12588
13373
  };
12589
13374
  }
12590
13375
  if (status === 429 || code === "rate_limit_exceeded" || /(?:too many requests|rate.?limit|\b429\b)/i.test(message)) {
@@ -12713,7 +13498,13 @@ function pendingToolCallFromSdkEvent(event) {
12713
13498
  if (typeof callId !== "string" || callId.length === 0 || typeof callType !== "string") {
12714
13499
  return null;
12715
13500
  }
12716
- return { callId, callType, callItem: raw };
13501
+ const callItem = normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]');
13502
+ return {
13503
+ callId,
13504
+ callType,
13505
+ callName: typeof raw.name === "string" ? raw.name : null,
13506
+ callItem
13507
+ };
12717
13508
  }
12718
13509
  function completedToolCallFromSdkEvent(event) {
12719
13510
  if (!event || typeof event !== "object") return null;
@@ -12724,7 +13515,11 @@ function completedToolCallFromSdkEvent(event) {
12724
13515
  }
12725
13516
  const raw = item.rawItem && typeof item.rawItem === "object" && !Array.isArray(item.rawItem) ? item.rawItem : {};
12726
13517
  const callId = raw.callId ?? raw.call_id ?? item.id;
12727
- return typeof callId === "string" && callId.length > 0 ? { callId, resultItem: raw } : null;
13518
+ if (typeof callId !== "string" || callId.length === 0) return null;
13519
+ return {
13520
+ callId,
13521
+ resultItem: normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]')
13522
+ };
12728
13523
  }
12729
13524
  var BudgetExhaustedError = class extends Error {
12730
13525
  constructor(message, serializedRunState) {
@@ -12919,12 +13714,7 @@ async function recordAuthoritativeModelCallFact(input) {
12919
13714
  });
12920
13715
  } catch (error) {
12921
13716
  input.observability.warn("model call fact persist failed", {
12922
- accountId: input.accountId,
12923
- workspaceId: input.workspaceId,
12924
- sessionId: input.sessionId,
12925
- turnId: input.turnId,
12926
- sourceKey: input.sourceKey,
12927
- error: error instanceof Error ? error.message : String(error)
13717
+ ...safeErrorDiagnostic(error)
12928
13718
  });
12929
13719
  }
12930
13720
  }
@@ -14626,6 +15416,7 @@ import {
14626
15416
  isCodexBilledTurn as isCodexBilledTurn2,
14627
15417
  markScheduledTaskRunFailedIfQueued,
14628
15418
  recordUsageEvent as recordUsageEvent3,
15419
+ requireScheduledTaskTargetInTransaction,
14629
15420
  requireSession as requireSession3,
14630
15421
  setTemporalWorkflowId,
14631
15422
  settleScheduledTaskRunInTransaction,
@@ -14748,7 +15539,11 @@ function createScheduledTaskActivities(services) {
14748
15539
  const sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
14749
15540
  const goalSpec = task.agentConfig.goal ?? null;
14750
15541
  const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
14751
- if (task.runMode === "new_session_per_run" || !task.reusableSessionId) {
15542
+ const existingSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
15543
+ if (task.runMode === "existing_session" && !existingSessionId) {
15544
+ throw new Error("scheduled task target session is unavailable");
15545
+ }
15546
+ if (task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && !existingSessionId) {
14752
15547
  const variableSet = task.variableSetId ? await getVariableSet(db, task.workspaceId, task.variableSetId) : null;
14753
15548
  if (task.variableSetId && !variableSet) {
14754
15549
  throw new Error(`variable set not found: ${task.variableSetId}`);
@@ -14915,7 +15710,7 @@ function createScheduledTaskActivities(services) {
14915
15710
  workflowWakeRevision: scheduledUpdate.workflowWakeRevision
14916
15711
  };
14917
15712
  } else {
14918
- const session = await requireSession3(db, task.workspaceId, task.reusableSessionId);
15713
+ const session = await requireSession3(db, task.workspaceId, existingSessionId);
14919
15714
  assertReusableSessionRevivable(session.status);
14920
15715
  if ((session.variableSetId ?? null) !== (task.variableSetId ?? null)) {
14921
15716
  throw new Error(
@@ -14927,7 +15722,7 @@ function createScheduledTaskActivities(services) {
14927
15722
  "scheduled task OpenGeni Slack bot binding does not match its reusable session"
14928
15723
  );
14929
15724
  }
14930
- const reusableGoal = goalSpec && run.status === "queued" ? await upsertSessionGoal(db, {
15725
+ const reusableGoal = task.runMode === "reusable_session" && goalSpec && run.status === "queued" ? await upsertSessionGoal(db, {
14931
15726
  accountId: task.accountId,
14932
15727
  workspaceId: task.workspaceId,
14933
15728
  sessionId: session.id,
@@ -14984,6 +15779,13 @@ function createScheduledTaskActivities(services) {
14984
15779
  personalConnectionDelegations: taskPersonalConnectionDelegations
14985
15780
  },
14986
15781
  async (tx, wakeEventId) => {
15782
+ if (task.runMode === "existing_session") {
15783
+ await requireScheduledTaskTargetInTransaction(tx, {
15784
+ workspaceId: task.workspaceId,
15785
+ taskId: task.id,
15786
+ targetSessionId: session.id
15787
+ });
15788
+ }
14987
15789
  if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");
14988
15790
  await settleScheduledTaskRunInTransaction(tx, {
14989
15791
  workspaceId: task.workspaceId,
@@ -15414,9 +16216,6 @@ import {
15414
16216
  markWarmLeaseInstanceLost as markWarmLeaseInstanceLost3,
15415
16217
  recordWarmingSandboxCreated as recordWarmingSandboxCreated2,
15416
16218
  releaseLeaseHolder as releaseLeaseHolder2,
15417
- sanitizeEventPayload,
15418
- sanitizeEventString,
15419
- sanitizeMemoryText,
15420
16219
  touchLeaseHolder as touchLeaseHolder2,
15421
16220
  updateRigChangeStatus
15422
16221
  } from "@opengeni/db";
@@ -15429,7 +16228,6 @@ import {
15429
16228
  serializeEstablishedSandboxEnvelope,
15430
16229
  tagModalSandbox as tagModalSandbox2
15431
16230
  } from "@opengeni/runtime";
15432
- var OUTPUT_TAIL_LIMIT = 64 * 1024;
15433
16231
  var RIG_VERIFICATION_CLEANUP_RESERVE_MAX_MS = 2 * 6e4;
15434
16232
  var RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS = 6e4;
15435
16233
  var RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS = 1e4;
@@ -15564,15 +16362,6 @@ function cleanupWaitMs(lifecycle) {
15564
16362
  Math.min(RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS, lifecycle.cleanupDeadlineAtMs - Date.now())
15565
16363
  );
15566
16364
  }
15567
- function tail(value, limit = OUTPUT_TAIL_LIMIT) {
15568
- return value.length > limit ? value.slice(-limit) : value;
15569
- }
15570
- function scrubVerificationOutput(value) {
15571
- return sanitizeMemoryText(sanitizeEventString(value)).text;
15572
- }
15573
- function scrubVerificationPayload(value) {
15574
- return sanitizeEventPayload(value);
15575
- }
15576
16365
  function systemGrant(rig) {
15577
16366
  return {
15578
16367
  accountId: rig.accountId,
@@ -15919,7 +16708,7 @@ async function runCommand(session, command, timeoutMs, commandRunner) {
15919
16708
  const result = await commandRunner(session, args);
15920
16709
  return {
15921
16710
  exitCode: sandboxCommandExitCode2(result),
15922
- output: scrubVerificationOutput(tail(sandboxCommandOutput(result)))
16711
+ output: sandboxCommandOutput(result)
15923
16712
  };
15924
16713
  }
15925
16714
  function setupAppendCommand(change) {
@@ -16050,7 +16839,7 @@ function createRigVerificationActivities(services) {
16050
16839
  verification.passed = false;
16051
16840
  const updated2 = await updateRigChangeStatus(db, input.workspaceId, change.id, {
16052
16841
  status: "rejected",
16053
- verification: scrubVerificationPayload(verification)
16842
+ verification
16054
16843
  });
16055
16844
  await recordRigAuditEvent(db, {
16056
16845
  grant,
@@ -16077,7 +16866,7 @@ function createRigVerificationActivities(services) {
16077
16866
  );
16078
16867
  checkResults.push({
16079
16868
  name: check.name,
16080
- command: scrubVerificationOutput(check.command),
16869
+ command: check.command,
16081
16870
  ...result
16082
16871
  });
16083
16872
  }
@@ -16089,7 +16878,7 @@ function createRigVerificationActivities(services) {
16089
16878
  if (classified.action === "auto_promote") {
16090
16879
  await updateRigChangeStatus(db, input.workspaceId, change.id, {
16091
16880
  status: "verifying",
16092
- verification: scrubVerificationPayload(verification)
16881
+ verification
16093
16882
  });
16094
16883
  const { change: merged } = await promoteSetupAppendChange({ db }, grant, rig, {
16095
16884
  ...change,
@@ -16105,7 +16894,7 @@ function createRigVerificationActivities(services) {
16105
16894
  }
16106
16895
  const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
16107
16896
  status: classified.status,
16108
- verification: scrubVerificationPayload(verification)
16897
+ verification
16109
16898
  });
16110
16899
  await recordRigAuditEvent(db, {
16111
16900
  grant,
@@ -16127,12 +16916,10 @@ function createRigVerificationActivities(services) {
16127
16916
  } catch (error) {
16128
16917
  verification.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
16129
16918
  verification.passed = false;
16130
- verification.error = scrubVerificationOutput(
16131
- error instanceof Error ? error.message : String(error)
16132
- );
16919
+ verification.error = error instanceof Error ? error.message : String(error);
16133
16920
  const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
16134
16921
  status: "failed",
16135
- verification: scrubVerificationPayload(verification)
16922
+ verification
16136
16923
  });
16137
16924
  await recordRigAuditEvent(db, {
16138
16925
  grant,
@@ -16195,7 +16982,7 @@ function createRigVerificationActivities(services) {
16195
16982
  for (const check of version.checks) {
16196
16983
  checkResults.push({
16197
16984
  name: check.name,
16198
- command: scrubVerificationOutput(check.command),
16985
+ command: check.command,
16199
16986
  ...await runCommand(
16200
16987
  established.session,
16201
16988
  check.command,
@@ -16209,33 +16996,30 @@ function createRigVerificationActivities(services) {
16209
16996
  grant,
16210
16997
  action: passed ? "rig.verification.passed" : "rig.verification.failed",
16211
16998
  rigId: rig.id,
16212
- metadata: scrubVerificationPayload({
16999
+ metadata: {
16213
17000
  versionId: version.id,
16214
17001
  startedAt,
16215
17002
  finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
16216
17003
  passed,
16217
17004
  checkResults
16218
- })
17005
+ }
16219
17006
  });
16220
17007
  return { versionId: version.id, passed, checkResults };
16221
17008
  }
16222
17009
  );
16223
17010
  } catch (error) {
16224
- const detail = tail(
16225
- scrubVerificationOutput(error instanceof Error ? error.message : String(error)),
16226
- 4096
16227
- );
17011
+ const detail = error instanceof Error ? error.message : String(error);
16228
17012
  await recordRigAuditEvent(db, {
16229
17013
  grant,
16230
17014
  action: "rig.verification.failed",
16231
17015
  rigId: rig.id,
16232
- metadata: scrubVerificationPayload({
17016
+ metadata: {
16233
17017
  versionId: version.id,
16234
17018
  startedAt,
16235
17019
  finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
16236
17020
  passed: false,
16237
17021
  error: detail
16238
- })
17022
+ }
16239
17023
  });
16240
17024
  throw error;
16241
17025
  }
@@ -16243,6 +17027,144 @@ function createRigVerificationActivities(services) {
16243
17027
  };
16244
17028
  }
16245
17029
 
17030
+ // src/activities/retained-screenshot-reaper.ts
17031
+ import {
17032
+ claimRetainedScreenshotMaintenance,
17033
+ completeRetainedScreenshotMaintenance,
17034
+ promoteRetainedScreenshotMaintenanceCleanup
17035
+ } from "@opengeni/db";
17036
+ var RETAINED_SCREENSHOT_PENDING_GRACE_MS = 15 * 60 * 1e3;
17037
+ var RETAINED_SCREENSHOT_CLAIM_TIMEOUT_MS = 10 * 60 * 1e3;
17038
+ var RETAINED_SCREENSHOT_MAINTENANCE_BATCH_SIZE = 100;
17039
+ function createRetainedScreenshotMaintenanceActivities(services, options = {}) {
17040
+ const pendingGraceMs = options.pendingGraceMs ?? RETAINED_SCREENSHOT_PENDING_GRACE_MS;
17041
+ const claimTimeoutMs = options.claimTimeoutMs ?? RETAINED_SCREENSHOT_CLAIM_TIMEOUT_MS;
17042
+ const batchSize = options.batchSize ?? RETAINED_SCREENSHOT_MAINTENANCE_BATCH_SIZE;
17043
+ const fileExists = options.fileExists ?? ((storage, file) => storage.fileExists(file));
17044
+ const headFile = options.headFile ?? ((storage, file) => storage.headFile(file));
17045
+ const deleteObject = options.deleteObject ?? ((storage, key) => storage.deleteObject(key));
17046
+ async function maintainRetainedScreenshots2() {
17047
+ const { db, objectStorage, observability } = await services();
17048
+ if (!objectStorage) {
17049
+ return { claimed: 0, ready: 0, deleted: 0, failed: 0, retryable: 0 };
17050
+ }
17051
+ const claims = await claimRetainedScreenshotMaintenance(db, {
17052
+ pendingGraceMs,
17053
+ claimTimeoutMs,
17054
+ limit: batchSize
17055
+ });
17056
+ const result = {
17057
+ claimed: claims.length,
17058
+ ready: 0,
17059
+ deleted: 0,
17060
+ failed: 0,
17061
+ retryable: 0
17062
+ };
17063
+ for (const claim of claims) {
17064
+ try {
17065
+ if (claim.action === "reconcile") {
17066
+ const outcome2 = await reconcileClaim(db, objectStorage, claim, {
17067
+ fileExists,
17068
+ headFile,
17069
+ deleteObject
17070
+ });
17071
+ if (outcome2 === "ready") {
17072
+ result.ready += 1;
17073
+ } else {
17074
+ result.failed += 1;
17075
+ }
17076
+ continue;
17077
+ }
17078
+ await deleteObject(objectStorage, claim.objectKey);
17079
+ const outcome = terminalDeleteOutcome(claim.cleanupReason);
17080
+ const settled = await completeRetainedScreenshotMaintenance(db, {
17081
+ accountId: claim.accountId,
17082
+ workspaceId: claim.workspaceId,
17083
+ artifactId: claim.artifactId,
17084
+ claimId: claim.claimId,
17085
+ outcome
17086
+ });
17087
+ if (!settled) throw new Error("retained screenshot cleanup claim was superseded");
17088
+ result.deleted += 1;
17089
+ } catch (error) {
17090
+ result.retryable += 1;
17091
+ observability.warn("retained screenshot maintenance failed; claim remains reclaimable", {
17092
+ workspaceId: claim.workspaceId,
17093
+ sessionId: claim.sessionId,
17094
+ artifactId: claim.artifactId,
17095
+ action: claim.action,
17096
+ error: error instanceof Error ? error.message : String(error)
17097
+ });
17098
+ }
17099
+ }
17100
+ if (claims.length > 0) {
17101
+ observability.info("retained screenshot maintenance swept", result);
17102
+ }
17103
+ return result;
17104
+ }
17105
+ return { maintainRetainedScreenshots: maintainRetainedScreenshots2 };
17106
+ }
17107
+ async function reconcileClaim(db, storage, claim, provider) {
17108
+ const observation = await observeClaimObject(storage, claim, provider);
17109
+ if (observation === "matches") {
17110
+ const settled2 = await completeRetainedScreenshotMaintenance(db, {
17111
+ accountId: claim.accountId,
17112
+ workspaceId: claim.workspaceId,
17113
+ artifactId: claim.artifactId,
17114
+ claimId: claim.claimId,
17115
+ outcome: "ready"
17116
+ });
17117
+ if (!settled2) throw new Error("retained screenshot reconcile claim was superseded");
17118
+ return "ready";
17119
+ }
17120
+ const promoted = await promoteRetainedScreenshotMaintenanceCleanup(db, {
17121
+ accountId: claim.accountId,
17122
+ workspaceId: claim.workspaceId,
17123
+ artifactId: claim.artifactId,
17124
+ claimId: claim.claimId,
17125
+ cleanupReason: "failed"
17126
+ });
17127
+ if (!promoted) throw new Error("retained screenshot reconcile claim was superseded");
17128
+ await provider.deleteObject(storage, claim.objectKey);
17129
+ const settled = await completeRetainedScreenshotMaintenance(db, {
17130
+ accountId: claim.accountId,
17131
+ workspaceId: claim.workspaceId,
17132
+ artifactId: claim.artifactId,
17133
+ claimId: claim.claimId,
17134
+ outcome: "failed"
17135
+ });
17136
+ if (!settled) throw new Error("retained screenshot reconcile claim was superseded");
17137
+ return "failed";
17138
+ }
17139
+ async function observeClaimObject(storage, claim, provider) {
17140
+ const file = claimFile(storage, claim);
17141
+ if (!await provider.fileExists(storage, file)) return "missing";
17142
+ const head = await provider.headFile(storage, file);
17143
+ return head.ContentLength === claim.sizeBytes && head.ContentType === claim.mediaType && head.Metadata?.sha256 === claim.sha256 ? "matches" : "mismatch";
17144
+ }
17145
+ function claimFile(storage, claim) {
17146
+ const timestamp = (/* @__PURE__ */ new Date(0)).toISOString();
17147
+ return {
17148
+ id: claim.artifactId,
17149
+ workspaceId: claim.workspaceId,
17150
+ status: "pending_upload",
17151
+ filename: `computer-screenshot-${claim.artifactId}.png`,
17152
+ safeFilename: `computer-screenshot-${claim.artifactId}.png`,
17153
+ contentType: claim.mediaType,
17154
+ sizeBytes: claim.sizeBytes,
17155
+ sha256: claim.sha256,
17156
+ bucket: storage.bucket,
17157
+ objectKey: claim.objectKey,
17158
+ createdAt: timestamp,
17159
+ updatedAt: timestamp
17160
+ };
17161
+ }
17162
+ function terminalDeleteOutcome(reason) {
17163
+ if (reason === "expired") return "expired";
17164
+ if (reason === "failed") return "failed";
17165
+ return "deleted";
17166
+ }
17167
+
16246
17168
  // src/activities.ts
16247
17169
  function createActivityServices(dependencies) {
16248
17170
  let servicesPromise = null;
@@ -16313,6 +17235,7 @@ function controlActivities(services) {
16313
17235
  ...createCodexCapacityActivities(services),
16314
17236
  ...createRigVerificationActivities(services),
16315
17237
  ...createFileUploadReaperActivities(services),
17238
+ ...createRetainedScreenshotMaintenanceActivities(services),
16316
17239
  ...createWorkflowWakeActivities(services),
16317
17240
  // P1.3: the SOLE liveness/GC/cost-stop driver. Only reapSandboxLeases — no
16318
17241
  // *ForViewer activities, no ownerHeartbeat, no resolveOwnerTaskQueue.
@@ -16323,7 +17246,9 @@ function createControlActivities(dependencies = {}) {
16323
17246
  return controlActivities(createActivityServices(dependencies));
16324
17247
  }
16325
17248
  function createTurnActivities(dependencies = {}) {
16326
- return { runAgentTurn: createRunAgentTurnActivity(createActivityServices(dependencies)) };
17249
+ return {
17250
+ runAgentTurn: createRunAgentTurnActivity(createActivityServices(dependencies))
17251
+ };
16327
17252
  }
16328
17253
  var defaultControlActivities = createControlActivities();
16329
17254
  var defaultTurnActivities = createTurnActivities();
@@ -16345,6 +17270,7 @@ var getCodexCapacityWait = defaultControlActivities.getCodexCapacityWait;
16345
17270
  var reconcileCodexCapacityWait = defaultControlActivities.reconcileCodexCapacityWait;
16346
17271
  var reapSandboxLeases = defaultControlActivities.reapSandboxLeases;
16347
17272
  var reapExpiredFileUploads = defaultControlActivities.reapExpiredFileUploads;
17273
+ var maintainRetainedScreenshots = defaultControlActivities.maintainRetainedScreenshots;
16348
17274
  var dispatchSessionWorkflowWakes = defaultControlActivities.dispatchSessionWorkflowWakes;
16349
17275
  var verifyRigChange = defaultControlActivities.verifyRigChange;
16350
17276
  var verifyRigVersion = defaultControlActivities.verifyRigVersion;
@@ -16426,7 +17352,7 @@ function createWorkerHttpHandler(input) {
16426
17352
  checks: Object.fromEntries(
16427
17353
  Object.keys(checks).map((name) => [
16428
17354
  name,
16429
- { ok: false, error: `worker is ${state}` }
17355
+ { ok: false, error: "worker_not_ready" }
16430
17356
  ])
16431
17357
  )
16432
17358
  } : { ...await runReadinessChecks(checks, timeoutMs), ...state ? { state } : {} };
@@ -16449,11 +17375,8 @@ async function runReadinessChecks(checks, timeoutMs = 2e3) {
16449
17375
  try {
16450
17376
  await withTimeout(Promise.resolve().then(check), timeoutMs);
16451
17377
  return [name, { ok: true }];
16452
- } catch (error) {
16453
- return [
16454
- name,
16455
- { ok: false, error: error instanceof Error ? error.message : String(error) }
16456
- ];
17378
+ } catch {
17379
+ return [name, { ok: false, error: "dependency_unavailable" }];
16457
17380
  }
16458
17381
  }
16459
17382
  )
@@ -16595,20 +17518,24 @@ function createWorkerServiceLifecycle(input) {
16595
17518
  })();
16596
17519
  return runPromise;
16597
17520
  },
16598
- drain: (reason = "host request") => {
17521
+ drain: (_reason = "host request") => {
16599
17522
  if (state === "draining" || state === "stopped" || state === "failed") {
16600
17523
  return;
16601
17524
  }
16602
17525
  state = "draining";
16603
17526
  input.observability.info("OpenGeni worker draining (graceful shutdown)", {
16604
17527
  role: input.role,
16605
- reason
17528
+ errorClass: "WorkerLifecycleOperation",
17529
+ errorCode: "worker_draining",
17530
+ origin: "worker-lifecycle"
16606
17531
  });
16607
17532
  try {
16608
17533
  input.worker.shutdown();
16609
- } catch (error) {
17534
+ } catch {
16610
17535
  input.observability.warn("worker shutdown request failed", {
16611
- error: error instanceof Error ? error.message : String(error)
17536
+ errorClass: "WorkerLifecycleOperationError",
17537
+ errorCode: "worker_shutdown_request_failed",
17538
+ origin: "worker-lifecycle"
16612
17539
  });
16613
17540
  }
16614
17541
  },