@opengeni/worker-bundle 0.11.2 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -104,6 +104,7 @@ import {
104
104
  SUMMARY_BUFFER_TOKENS,
105
105
  runOwnedSandboxSetup,
106
106
  RoutingMutationOutcomeUnknownError,
107
+ WorkspaceArchiveIntegrityError,
107
108
  swapTargetEstablishability,
108
109
  type SandboxFileDownload,
109
110
  type SandboxFileDownloadFailure,
@@ -113,6 +114,7 @@ import {
113
114
  type ModelCallUsageInput,
114
115
  type ModelCallUsageNormalization,
115
116
  type BuildAgentOptions,
117
+ type RunAgentStreamOptions,
116
118
  type TurnToolCancellationFence,
117
119
  type BackendUnresolvableCode,
118
120
  type EstablishedSandboxSession,
@@ -182,7 +184,11 @@ import {
182
184
  type CodexUsageHeaderSnapshot,
183
185
  } from "@opengeni/codex";
184
186
  import { mergeResourceRefs } from "./common";
185
- import { enabledCapabilityMcpToolRefs, resolveSessionToolPolicy } from "@opengeni/core";
187
+ import {
188
+ enabledCapabilityMcpToolRefs,
189
+ resolveSessionToolPolicy,
190
+ sessionToolPolicyAllowsDefaultNativeTools,
191
+ } from "@opengeni/core";
186
192
  import { maybeCompactContext } from "./context-compaction";
187
193
  import { TurnAttemptFencedError } from "./turn-attempt-fenced";
188
194
  import {
@@ -226,7 +232,12 @@ import {
226
232
  settingsWithRigImage,
227
233
  } from "./packs";
228
234
  import { deliverFailedChildTurnToParent } from "./parent-wake";
229
- import { createSecretRedactor, identityRedactor, type SecretForRedaction } from "./redaction";
235
+ import {
236
+ createSecretRedactor,
237
+ isCredentialHeaderName,
238
+ redactSensitiveText,
239
+ type SecretForRedaction,
240
+ } from "./redaction";
230
241
  import { applyCodexHistoryStrip, turnInput, type TurnCodexAccount } from "./run-input";
231
242
  import {
232
243
  createRuntimeBatcher,
@@ -242,7 +253,6 @@ import type {
242
253
  } from "./types";
243
254
  import {
244
255
  resumeBoxForTurn,
245
- acquireSelfhostedLeaseForTurn,
246
256
  sandboxLeaseHolderIdForAttempt,
247
257
  maybePersistWarmWorkspaceSnapshot,
248
258
  waitForWarmSnapshot,
@@ -613,6 +623,84 @@ function compactionFailureReason(reason: string): string {
613
623
  : `compaction summarization failed: ${reason}`;
614
624
  }
615
625
 
626
+ export type SafeErrorDiagnostic = {
627
+ name: string;
628
+ message: string;
629
+ status?: number;
630
+ code?: string;
631
+ };
632
+
633
+ /**
634
+ * Produce the only exception shape allowed in worker logs. It deliberately
635
+ * excludes stack, cause, response/request bodies, and arbitrary enumerable
636
+ * properties while retaining bounded, redacted diagnostics.
637
+ */
638
+ export function safeErrorDiagnostic(
639
+ error: unknown,
640
+ redactText: (value: string) => string = redactSensitiveText,
641
+ ): SafeErrorDiagnostic {
642
+ const rawName = error instanceof Error ? error.name : "Error";
643
+ const name = /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error";
644
+ const rawMessage = error instanceof Error ? error.message : String(error);
645
+ const diagnostic: SafeErrorDiagnostic = {
646
+ name,
647
+ message: redactText(rawMessage).slice(0, 2_048),
648
+ };
649
+ if (error && typeof error === "object") {
650
+ const status = Number(
651
+ (error as { status?: unknown; statusCode?: unknown }).status ??
652
+ (error as { statusCode?: unknown }).statusCode,
653
+ );
654
+ if (Number.isInteger(status) && status >= 100 && status <= 599) {
655
+ diagnostic.status = status;
656
+ }
657
+ const rawCode = (error as { code?: unknown }).code;
658
+ if (typeof rawCode === "string" || typeof rawCode === "number") {
659
+ const code = redactText(String(rawCode)).slice(0, 80);
660
+ if (/^[A-Za-z0-9_.:-]+$/.test(code)) diagnostic.code = code;
661
+ }
662
+ }
663
+ return diagnostic;
664
+ }
665
+
666
+ function safeErrorForTelemetry(error: unknown, redactText: (value: string) => string): Error {
667
+ const diagnostic = safeErrorDiagnostic(error, redactText);
668
+ const safe = new Error(diagnostic.message);
669
+ safe.name = diagnostic.name;
670
+ return safe;
671
+ }
672
+
673
+ export function headerSecretRedactions(
674
+ prefix: string,
675
+ headers: Readonly<Record<string, string>> | undefined,
676
+ credentialHeaderNames?: Iterable<string>,
677
+ ): SecretForRedaction[] {
678
+ const discovered: SecretForRedaction[] = [];
679
+ const explicitCredentialHeaders = credentialHeaderNames
680
+ ? new Set([...credentialHeaderNames].map((name) => name.toLowerCase()))
681
+ : null;
682
+ for (const [headerName, value] of Object.entries(headers ?? {})) {
683
+ if (
684
+ !explicitCredentialHeaders?.has(headerName.toLowerCase()) &&
685
+ !isCredentialHeaderName(headerName)
686
+ ) {
687
+ continue;
688
+ }
689
+ const safeHeaderName = headerName.toUpperCase().replace(/[^A-Z0-9]+/g, "_");
690
+ discovered.push({ name: `${prefix}_${safeHeaderName || "HEADER"}`, value });
691
+ if (/^(?:proxy-)?authorization$/i.test(headerName)) {
692
+ const credential = value.match(/^\s*[A-Za-z][A-Za-z0-9_-]*\s+(.+)\s*$/)?.[1];
693
+ if (credential) {
694
+ discovered.push({
695
+ name: `${prefix}_${safeHeaderName || "AUTHORIZATION"}_CREDENTIAL`,
696
+ value: credential,
697
+ });
698
+ }
699
+ }
700
+ }
701
+ return discovered;
702
+ }
703
+
616
704
  function compactionFailureReasonFromError(error: unknown): string {
617
705
  if (
618
706
  error instanceof CompactionProviderResponseError ||
@@ -1198,14 +1286,15 @@ export function historyRowsToAppend(
1198
1286
  // positions from 0) when callers do not pass an explicit next position.
1199
1287
  nextPosition: number = persistedHistoryCount,
1200
1288
  toolOutputTruncationTokens?: number,
1289
+ redactValue: (value: unknown) => unknown = (value) => value,
1201
1290
  ): {
1202
1291
  rows: Array<{ position: number; item: Record<string, unknown> }>;
1203
1292
  nextWatermark: number;
1204
1293
  nextPosition: number;
1205
1294
  } {
1206
- const sanitized = sanitizeHistoryItemsForModel(rawHistory, toolOutputTruncationTokens).filter(
1207
- (item) => !isEphemeralInternalContext(item),
1208
- );
1295
+ const sanitized = sanitizeHistoryItemsForModel(rawHistory, toolOutputTruncationTokens)
1296
+ .filter((item) => !isEphemeralInternalContext(item))
1297
+ .map((item) => redactValue(item) as Record<string, unknown>);
1209
1298
  if (sanitized.length <= persistedHistoryCount) {
1210
1299
  return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
1211
1300
  }
@@ -1220,6 +1309,16 @@ export function historyRowsToAppend(
1220
1309
  };
1221
1310
  }
1222
1311
 
1312
+ /** Literal final per-call boundary for replayed and current-turn model input. */
1313
+ export function secretRedactionModelInputFilter(
1314
+ redactValue: (value: unknown) => unknown,
1315
+ ): NonNullable<RunAgentStreamOptions["callModelInputFilter"]> {
1316
+ return ({ modelData }) => ({
1317
+ ...modelData,
1318
+ input: redactValue(modelData.input) as typeof modelData.input,
1319
+ });
1320
+ }
1321
+
1223
1322
  function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
1224
1323
  if (item.type === "message") {
1225
1324
  return item.role === "assistant";
@@ -1322,12 +1421,29 @@ export async function resolveActiveSandboxBackend(
1322
1421
  } catch (error) {
1323
1422
  console.error(
1324
1423
  "active sandbox backend resolution failed (turn proceeds on home backend)",
1325
- error,
1424
+ safeErrorDiagnostic(error),
1326
1425
  );
1327
1426
  return undefined;
1328
1427
  }
1329
1428
  }
1330
1429
 
1430
+ /**
1431
+ * Managed-sandbox ownership exists only when this turn actually uses the
1432
+ * managed home. A Connected Machine is fenced by its active-route epoch and
1433
+ * must remain independent from cloud recovery state for the same session.
1434
+ */
1435
+ export function managedSandboxOwnershipForTurn(
1436
+ machinePrimary: boolean,
1437
+ attemptId: string,
1438
+ sandboxGroupId: string,
1439
+ ): { holderId: TurnSandboxLeaseHolderId; sandboxGroupId: string } | null {
1440
+ if (machinePrimary) return null;
1441
+ return {
1442
+ holderId: sandboxLeaseHolderIdForAttempt(attemptId),
1443
+ sandboxGroupId,
1444
+ };
1445
+ }
1446
+
1331
1447
  /**
1332
1448
  * Classify a persisted active-sandbox pointer for TURN-START RECONCILE (issue #341
1333
1449
  * invariant B). Returns the typed reason to RESET the pointer to the session HOME,
@@ -1543,6 +1659,22 @@ export function computerToolModeForTurn(
1543
1659
  return "hosted";
1544
1660
  }
1545
1661
 
1662
+ /**
1663
+ * Native web search requires two independent grants: the resolved provider
1664
+ * must advertise runnable support, and the immutable session/turn policy must
1665
+ * still track workspace defaults. The null model is the legacy built-in
1666
+ * Responses path, whose deployment flag is its provider capability gate.
1667
+ * There is deliberately no cross-provider or sandbox/curl fallback.
1668
+ */
1669
+ export function hostedWebSearchForTurn(
1670
+ resolvedModel: { configured: { hostedWebSearch: boolean } } | null,
1671
+ deploymentWebSearchEnabled: boolean,
1672
+ defaultNativeToolsAllowed: boolean,
1673
+ ): boolean {
1674
+ if (!defaultNativeToolsAllowed) return false;
1675
+ return resolvedModel?.configured.hostedWebSearch ?? deploymentWebSearchEnabled;
1676
+ }
1677
+
1546
1678
  /**
1547
1679
  * Decide whether this turn's provider accepts typed file/image content in a
1548
1680
  * user message. The legacy built-in client and registry Responses clients use
@@ -1650,6 +1782,9 @@ export function isLazySandboxProvisionRetryable(error: unknown): boolean {
1650
1782
  if (error instanceof SandboxImageConflictError) {
1651
1783
  return false;
1652
1784
  }
1785
+ if (error instanceof WorkspaceArchiveIntegrityError) {
1786
+ return error.retryable;
1787
+ }
1653
1788
  if (error instanceof SandboxLeaseSupersededError || error instanceof SandboxWarmingTimeoutError) {
1654
1789
  return true;
1655
1790
  }
@@ -2225,6 +2360,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2225
2360
  await current?.flush().catch(() => undefined);
2226
2361
  };
2227
2362
  let preparedTools: Awaited<ReturnType<OpenGeniRuntime["prepareTools"]>> | null = null;
2363
+ // Resolved with the MCP policy at the immutable turn boundary. Provider
2364
+ // capability is checked separately when buildAgent is called; this flag is
2365
+ // only the durable omission/narrowing entitlement.
2366
+ let defaultNativeToolsAllowed = false;
2228
2367
  const toolCancellationFenceRef: {
2229
2368
  current: TurnToolCancellationFence | null;
2230
2369
  } = {
@@ -2286,7 +2425,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2286
2425
  ]).catch((publishError) => {
2287
2426
  // The lease transition is already authoritative. A fenced/failed audit
2288
2427
  // append must not prevent the same logical turn from recovering.
2289
- console.error("sandbox box lost event publish failed", publishError);
2428
+ console.error("sandbox box lost event publish failed", safeErrorDiagnostic(publishError));
2290
2429
  });
2291
2430
  };
2292
2431
  const startLeaseHeartbeat = (
@@ -2447,7 +2586,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2447
2586
  }
2448
2587
  console.error(
2449
2588
  "computer-use recording start failed (action outcome unaffected)",
2450
- recordingError,
2589
+ safeErrorDiagnostic(recordingError),
2451
2590
  );
2452
2591
  }
2453
2592
  })();
@@ -2495,6 +2634,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2495
2634
  persistedHistoryCount,
2496
2635
  nextHistoryPosition,
2497
2636
  modelRunSettings.modelToolOutputTruncationTokens,
2637
+ (value) => redact(value),
2498
2638
  );
2499
2639
  const hasModelOrToolProgress = rows.some((row) =>
2500
2640
  isModelOrToolProgressHistoryItem(row.item),
@@ -2538,13 +2678,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2538
2678
  });
2539
2679
  }
2540
2680
  } catch (persistError) {
2541
- console.error("session history dual-write failed (run unaffected)", persistError);
2681
+ console.error(
2682
+ "session history dual-write failed (run unaffected)",
2683
+ safeErrorDiagnostic(persistError, (value) => String(redact(value))),
2684
+ );
2542
2685
  if (options.requireDurable) throw persistError;
2543
2686
  }
2544
2687
  };
2545
2688
  // Reassigned after the variable set loads; the publish closure is
2546
2689
  // created (and used for turn.started) before the variableSet is available.
2547
- let redact: (payload: unknown) => unknown = identityRedactor;
2690
+ // Generic auth/cookie/signed-URL/token classification is active before any
2691
+ // exact run-secret provenance is available. Registered values strengthen
2692
+ // this boundary as credentials are resolved and renewed during the turn.
2693
+ let redact: (payload: unknown) => unknown = createSecretRedactor([]);
2548
2694
  const secretRedactions = new Map<string, string>();
2549
2695
  const publishedRunCredentialNotices = new Set<string>();
2550
2696
  const registerSecretRedactions = (secrets: SecretForRedaction[]): void => {
@@ -2805,6 +2951,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2805
2951
  producerId,
2806
2952
  producerSeq: ++producerSeq,
2807
2953
  }));
2954
+ const redactedRunState = inputSettlement.runState
2955
+ ? (redact(inputSettlement.runState) as typeof inputSettlement.runState)
2956
+ : undefined;
2808
2957
  const result = await applySessionTurnSettlement(db, input.workspaceId, {
2809
2958
  sessionId: input.sessionId,
2810
2959
  turnId: turnId!,
@@ -2814,7 +2963,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2814
2963
  sessionStatus: inputSettlement.sessionStatus,
2815
2964
  activeTurnId: inputSettlement.activeTurnId,
2816
2965
  events: inputs,
2817
- ...(inputSettlement.runState ? { runState: inputSettlement.runState } : {}),
2966
+ ...(redactedRunState ? { runState: redactedRunState } : {}),
2818
2967
  ...(recordingMutation ? { recording: recordingMutation } : {}),
2819
2968
  ...(compactionRequestFailure ? { compactionRequestFailure } : {}),
2820
2969
  });
@@ -3246,43 +3395,42 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3246
3395
  const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
3247
3396
  () => null,
3248
3397
  );
3249
- if (goal?.status === "active") {
3250
- const armed = await armCodexCapacityWait(db, {
3251
- accountId: input.accountId,
3252
- workspaceId: input.workspaceId,
3253
- sessionId: input.sessionId,
3254
- turnId,
3255
- attemptId: input.attemptId,
3256
- workflowId: input.workflowId,
3257
- goalId: goal.id,
3258
- goalVersion: goal.version,
3259
- earliestResetAt: null,
3260
- resetKind: "bounded_refresh",
3261
- failurePayload: {
3262
- error: "All connected Codex subscriptions are disabled for new allocations.",
3263
- code: "codex_allocator_disabled",
3264
- detail: "waiting for a credential to be re-enabled, reconnected, or added",
3398
+ const activeGoal = goal?.status === "active" ? goal : null;
3399
+ const armed = await armCodexCapacityWait(db, {
3400
+ accountId: input.accountId,
3401
+ workspaceId: input.workspaceId,
3402
+ sessionId: input.sessionId,
3403
+ turnId,
3404
+ attemptId: input.attemptId,
3405
+ workflowId: input.workflowId,
3406
+ goalId: activeGoal?.id ?? null,
3407
+ goalVersion: activeGoal?.version ?? null,
3408
+ earliestResetAt: null,
3409
+ resetKind: "bounded_refresh",
3410
+ failurePayload: {
3411
+ error: "All connected Codex subscriptions are disabled for new allocations.",
3412
+ code: "codex_allocator_disabled",
3413
+ detail: "waiting for a credential to be re-enabled, reconnected, or added",
3414
+ },
3415
+ });
3416
+ if (armed.action === "waiting") {
3417
+ await publishDurableSessionEvents(
3418
+ bus,
3419
+ input.workspaceId,
3420
+ input.sessionId,
3421
+ armed.events,
3422
+ );
3423
+ turnMetricOutcome = "recovering";
3424
+ activityStatus = "waiting_capacity";
3425
+ return claimedResult({
3426
+ status: "waiting_capacity",
3427
+ capacityWait: {
3428
+ waiterId: armed.waiter.id,
3429
+ generation: armed.waiter.generation,
3430
+ nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
3431
+ wakeRevision: armed.waiter.wakeRevision,
3265
3432
  },
3266
3433
  });
3267
- if (armed.action === "waiting") {
3268
- await publishDurableSessionEvents(
3269
- bus,
3270
- input.workspaceId,
3271
- input.sessionId,
3272
- armed.events,
3273
- );
3274
- turnMetricOutcome = "failed";
3275
- activityStatus = "idle";
3276
- return claimedResult({
3277
- status: "idle",
3278
- capacityWait: {
3279
- waiterId: armed.waiter.id,
3280
- generation: armed.waiter.generation,
3281
- nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
3282
- wakeRevision: armed.waiter.wakeRevision,
3283
- },
3284
- });
3285
- }
3286
3434
  }
3287
3435
  if (
3288
3436
  !(await settle!({
@@ -3361,43 +3509,41 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3361
3509
  "all connected Codex subscriptions are rate-limited",
3362
3510
  { allAccounts: true },
3363
3511
  );
3364
- if (goalActive && goal) {
3365
- const authoritativeResetAt = authoritativeCodexCapacityResetAt(
3366
- leased.accounts,
3367
- new Date(),
3512
+ const authoritativeResetAt = authoritativeCodexCapacityResetAt(
3513
+ leased.accounts,
3514
+ new Date(),
3515
+ );
3516
+ const armed = await armCodexCapacityWait(db, {
3517
+ accountId: input.accountId,
3518
+ workspaceId: input.workspaceId,
3519
+ sessionId: input.sessionId,
3520
+ turnId,
3521
+ attemptId: input.attemptId,
3522
+ workflowId: input.workflowId,
3523
+ goalId: goalActive && goal ? goal.id : null,
3524
+ goalVersion: goalActive && goal ? goal.version : null,
3525
+ earliestResetAt: authoritativeResetAt,
3526
+ resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
3527
+ failurePayload,
3528
+ });
3529
+ if (armed.action === "waiting") {
3530
+ await publishDurableSessionEvents(
3531
+ bus,
3532
+ input.workspaceId,
3533
+ input.sessionId,
3534
+ armed.events,
3368
3535
  );
3369
- const armed = await armCodexCapacityWait(db, {
3370
- accountId: input.accountId,
3371
- workspaceId: input.workspaceId,
3372
- sessionId: input.sessionId,
3373
- turnId,
3374
- attemptId: input.attemptId,
3375
- workflowId: input.workflowId,
3376
- goalId: goal.id,
3377
- goalVersion: goal.version,
3378
- earliestResetAt: authoritativeResetAt,
3379
- resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
3380
- failurePayload,
3536
+ turnMetricOutcome = "recovering";
3537
+ activityStatus = "waiting_capacity";
3538
+ return claimedResult({
3539
+ status: "waiting_capacity",
3540
+ capacityWait: {
3541
+ waiterId: armed.waiter.id,
3542
+ generation: armed.waiter.generation,
3543
+ nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
3544
+ wakeRevision: armed.waiter.wakeRevision,
3545
+ },
3381
3546
  });
3382
- if (armed.action === "waiting") {
3383
- await publishDurableSessionEvents(
3384
- bus,
3385
- input.workspaceId,
3386
- input.sessionId,
3387
- armed.events,
3388
- );
3389
- turnMetricOutcome = "failed";
3390
- activityStatus = "idle";
3391
- return claimedResult({
3392
- status: "idle",
3393
- capacityWait: {
3394
- waiterId: armed.waiter.id,
3395
- generation: armed.waiter.generation,
3396
- nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
3397
- wakeRevision: armed.waiter.wakeRevision,
3398
- },
3399
- });
3400
- }
3401
3547
  }
3402
3548
  if (
3403
3549
  !(await settle!({
@@ -3552,13 +3698,33 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3552
3698
  openaiReasoningEffort: turn.reasoningEffort,
3553
3699
  sandboxBackend: turn.sandboxBackend,
3554
3700
  };
3701
+ const sessionMcpHeaderNames = new Map<string, readonly string[]>();
3555
3702
  const runSettings = await settingsWithSessionMcpServersForRun(
3556
3703
  db,
3557
3704
  input.workspaceId,
3558
3705
  input.sessionId,
3559
3706
  input.attemptId,
3560
3707
  baseRunSettings,
3708
+ {
3709
+ onResolvedServers: (servers) => {
3710
+ for (const server of servers) {
3711
+ sessionMcpHeaderNames.set(server.id, server.headerNames);
3712
+ }
3713
+ },
3714
+ },
3561
3715
  );
3716
+ registerSecretRedactions([
3717
+ ...(runSettings.accessKey
3718
+ ? [{ name: "OPENGENI_ACCESS_KEY", value: runSettings.accessKey }]
3719
+ : []),
3720
+ ...runSettings.mcpServers.flatMap((server) => {
3721
+ return headerSecretRedactions(
3722
+ `MCP_${server.id.toUpperCase()}_STATIC`,
3723
+ server.headers,
3724
+ sessionMcpHeaderNames.get(server.id),
3725
+ );
3726
+ }),
3727
+ ]);
3562
3728
  // Multi-provider per-turn routing → the provider gating (compaction mode,
3563
3729
  // hosted web search, encrypted reasoning, context window) the agent and
3564
3730
  // compaction summarizer must use; null falls back to the legacy global
@@ -3576,6 +3742,39 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3576
3742
  capabilitySettings,
3577
3743
  turnExecutionPolicy.productModelId,
3578
3744
  );
3745
+ const selectedProvider = resolvedModel?.provider;
3746
+ const publicProviderHeaders = new Set(
3747
+ selectedProvider?.publicDefaultHeaderNames?.map((name) => name.toLowerCase()) ?? [],
3748
+ );
3749
+ const publicProviderQueries = new Set(
3750
+ selectedProvider?.publicDefaultQueryNames?.map((name) => name.toLowerCase()) ?? [],
3751
+ );
3752
+ const fallbackProviderApiKey =
3753
+ capabilitySettings.openaiProvider === "azure"
3754
+ ? (capabilitySettings.azureOpenaiApiKey ?? capabilitySettings.azureOpenaiAdToken)
3755
+ : capabilitySettings.openaiApiKey;
3756
+ registerSecretRedactions([
3757
+ ...(selectedProvider?.apiKey
3758
+ ? [
3759
+ {
3760
+ name: `MODEL_${selectedProvider.id.toUpperCase()}_API_KEY`,
3761
+ value: selectedProvider.apiKey,
3762
+ },
3763
+ ]
3764
+ : !resolvedModel && fallbackProviderApiKey
3765
+ ? [{ name: "MODEL_PROVIDER_API_KEY", value: fallbackProviderApiKey }]
3766
+ : []),
3767
+ ...Object.entries(selectedProvider?.defaultHeaders ?? {}).flatMap(([name, value]) =>
3768
+ publicProviderHeaders.has(name.toLowerCase())
3769
+ ? []
3770
+ : [{ name: `MODEL_PROVIDER_HEADER_${name.toUpperCase()}`, value }],
3771
+ ),
3772
+ ...Object.entries(selectedProvider?.defaultQuery ?? {}).flatMap(([name, value]) =>
3773
+ publicProviderQueries.has(name.toLowerCase())
3774
+ ? []
3775
+ : [{ name: `MODEL_PROVIDER_QUERY_${name.toUpperCase()}`, value }],
3776
+ ),
3777
+ ]);
3579
3778
  // Bind the provider/model catalog's context policy to every model-facing
3580
3779
  // path for this turn. In particular, Codex subscription turns must not
3581
3780
  // inherit the deployment's OpenAI/Azure mode or 1.05M context defaults:
@@ -3630,6 +3829,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3630
3829
  input.workspaceId,
3631
3830
  effectiveCodexCredentialId ?? "",
3632
3831
  );
3832
+ const registerSnapshot = async (
3833
+ snapshotPromise: ReturnType<typeof resolver.getToken>,
3834
+ ) => {
3835
+ const snapshot = await snapshotPromise;
3836
+ registerSecretRedactions([
3837
+ { name: "CODEX_ACCESS_TOKEN", value: snapshot.accessToken },
3838
+ ]);
3839
+ return snapshot;
3840
+ };
3633
3841
  return {
3634
3842
  clientVersion: CODEX_CLIENT_VERSION,
3635
3843
  // Backend sticky cache-routing key — the SAME id as the body's
@@ -3639,8 +3847,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3639
3847
  // (per-request shard lottery = prod's measured 48.6% on sol);
3640
3848
  // with it, resends pin to the warm shard (Codex CLI parity).
3641
3849
  sessionId: input.sessionId,
3642
- getToken: resolver.getToken,
3643
- refresh: resolver.refresh,
3850
+ getToken: () => registerSnapshot(resolver.getToken()),
3851
+ refresh: () => registerSnapshot(resolver.refresh()),
3644
3852
  resolveModel: buildModelResolver(
3645
3853
  CODEX_FALLBACK_MODEL_SLUGS,
3646
3854
  CODEX_FALLBACK_MODEL_SLUGS[0],
@@ -3783,7 +3991,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3783
3991
  // recovery path re-dispatch this exact maintenance execution.
3784
3992
  if (shouldRecoverCompactionProviderFailure(error)) throw error;
3785
3993
  if (!isCompactionSummaryFailure(error)) throw error;
3786
- const errorMessage = compactionFailureReasonFromError(error);
3994
+ const errorMessage = String(redact(compactionFailureReasonFromError(error)));
3787
3995
  if (
3788
3996
  !(await settle!({
3789
3997
  events: [
@@ -3867,7 +4075,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3867
4075
  // materialized allow-list. `withCodexAppsTool` below keeps the existing
3868
4076
  // single Codex lazy router/connector overlay; it is not a second policy
3869
4077
  // or discovery path.
3870
- const effectivePolicyTools = resolveSessionToolPolicy({
4078
+ const resolvedToolPolicy = resolveSessionToolPolicy({
3871
4079
  ...(session.toolPolicy ? { toolPolicy: session.toolPolicy } : {}),
3872
4080
  sessionTools: session.tools,
3873
4081
  turnTools: turn.tools,
@@ -3876,7 +4084,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3876
4084
  defaultMcpServerIds: enabledCapabilityMcpToolRefs(settings, mcpSettings).map(
3877
4085
  (tool) => tool.id,
3878
4086
  ),
3879
- }).toolRefs;
4087
+ });
4088
+ const effectivePolicyTools = resolvedToolPolicy.toolRefs;
4089
+ defaultNativeToolsAllowed = sessionToolPolicyAllowsDefaultNativeTools(
4090
+ resolvedToolPolicy.effectivePolicy,
4091
+ );
3880
4092
  const turnTools = withCodexAppsTool(
3881
4093
  runSettings,
3882
4094
  withFirstPartyTools(runSettings, effectivePolicyTools),
@@ -4141,6 +4353,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4141
4353
  cancellationSignal,
4142
4354
  undefined,
4143
4355
  );
4356
+ registerSecretRedactions([
4357
+ ...Object.entries(sandboxGitTokens ?? {}).flatMap(([provider, value]) =>
4358
+ value ? [{ name: `GIT_${provider.toUpperCase()}_TOKEN`, value }] : [],
4359
+ ),
4360
+ ...(sandboxGitToken ? [{ name: "GIT_TOKEN", value: sandboxGitToken }] : []),
4361
+ ...((sandboxGitCredentialBindings ?? []).map((binding) => ({
4362
+ name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
4363
+ value: binding.token,
4364
+ })) satisfies SecretForRedaction[]),
4365
+ ...(sandboxToolspaceToken
4366
+ ? [{ name: "TOOLSPACE_TOKEN", value: sandboxToolspaceToken }]
4367
+ : []),
4368
+ ]);
4144
4369
  const sandboxToolspaceTokenFile = sandboxToolspaceToken
4145
4370
  ? toolspaceTokenFileFromEnvironment(sandboxEnvironment, input.sessionId)
4146
4371
  : undefined;
@@ -4186,6 +4411,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4186
4411
  );
4187
4412
  if (binding) {
4188
4413
  assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
4414
+ registerSecretRedactions([
4415
+ {
4416
+ name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
4417
+ value: binding.token,
4418
+ },
4419
+ ]);
4189
4420
  }
4190
4421
  pendingBinding = binding;
4191
4422
  return binding
@@ -4266,8 +4497,17 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4266
4497
  await previous?.stop();
4267
4498
  if (toolspaceTokenRenewalClosed) return;
4268
4499
 
4269
- const mint = async () =>
4270
- await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority);
4500
+ const mint = async () => {
4501
+ const material = await mintSandboxToolspaceToken(
4502
+ runSettings,
4503
+ connectionScope,
4504
+ toolspaceAuthority,
4505
+ );
4506
+ if (material) {
4507
+ registerSecretRedactions([{ name: "TOOLSPACE_TOKEN", value: material.token }]);
4508
+ }
4509
+ return material;
4510
+ };
4271
4511
  const write = async (material: NonNullable<Awaited<ReturnType<typeof mint>>>) => {
4272
4512
  const runAs = sandboxRunAs(runSettings);
4273
4513
  const targetSandbox = resolvedSandbox ?? initialSandbox;
@@ -4483,8 +4723,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4483
4723
  // variableSet delta (validateNoEnvironmentDelta). Passing sandboxEnvironment
4484
4724
  // here makes current==target so the delta is empty.
4485
4725
  if (settings.sandboxOwnershipEnabled && turn.sandboxBackend !== "none") {
4486
- sandboxHolderId = sandboxLeaseHolderIdForAttempt(input.attemptId);
4487
- sandboxGroupId = session.sandboxGroupId;
4726
+ const managedOwnership = managedSandboxOwnershipForTurn(
4727
+ machinePrimary,
4728
+ input.attemptId,
4729
+ session.sandboxGroupId,
4730
+ );
4731
+ sandboxHolderId = managedOwnership?.holderId ?? null;
4732
+ sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
4488
4733
  // STAGE D honest-label guard: a machine-home session carries
4489
4734
  // turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
4490
4735
  // when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
@@ -4497,13 +4742,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4497
4742
  // degrade to a genuine cloud box exactly like today (home=modal did).
4498
4743
  if (machinePrimary) {
4499
4744
  // STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
4500
- // establish or lease a phantom Modal home box (today's path leased + BILLED
4501
- // a cloud box the turn never touched). Build the SelfhostedSession DIRECTLY
4502
- // (no Modal box created) and take the group lease with backend "selfhosted"
4503
- // (refcount/idle bookkeeping; the reaper drains it cold with NO provider
4504
- // stop, and bills ZERO warm-seconds). The session is a harmless in-memory
4505
- // bind (no NATS round-trip), so build it FIRST; if the lease then fences,
4506
- // there is nothing to clean up.
4745
+ // establish OR lease the managed home box. The active-sandbox pointer is
4746
+ // the machine route's own epoch fence; the managed-home lease separately
4747
+ // owns cloud recovery, snapshots, billing, and reaping. Mixing those two
4748
+ // ownership domains lets an unrelated degraded cloud archive block a
4749
+ // healthy explicitly selected machine.
4507
4750
  // Whether the machine's latest Hello advertised the op-stream engine
4508
4751
  // (refreshed on every connect). Read only when the server flag is on —
4509
4752
  // one indexed lookup, and the flag off keeps this path byte-identical.
@@ -4533,21 +4776,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4533
4776
  // (buildSelfhostedBackendSession); EstablishedSandboxSession widens it.
4534
4777
  machinePrimarySession =
4535
4778
  established.session as import("@opengeni/runtime").SelfhostedSession;
4536
- const lease = await waitForTurnOperation(
4537
- acquireSelfhostedLeaseForTurn(
4538
- { db, settings },
4539
- {
4540
- accountId: input.accountId,
4541
- workspaceId: input.workspaceId,
4542
- sandboxGroupId: session.sandboxGroupId,
4543
- sessionId: input.sessionId,
4544
- },
4545
- "turn",
4546
- sandboxHolderId,
4547
- ),
4548
- cancellationSignal,
4549
- async (lateLease) => await lateLease.release(),
4550
- );
4551
4779
  setupBoxSession = established.session;
4552
4780
  resolvedSandbox = {
4553
4781
  // Wrap in the SAME routing proxy so a mid-turn swap (to another machine
@@ -4589,8 +4817,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4589
4817
  },
4590
4818
  established,
4591
4819
  ),
4592
- leaseEpoch: lease.leaseEpoch,
4593
- release: lease.release,
4820
+ // For a machine route this is the active-pointer epoch, not a cloud
4821
+ // lease epoch. Cloud-only heartbeat/snapshot/capture paths require
4822
+ // sandboxGroupId and remain disabled for this branch.
4823
+ leaseEpoch: activeSandboxPointer!.activeEpoch,
4824
+ release: async () => undefined,
4594
4825
  };
4595
4826
  } else if (establishPolicy === "on-demand") {
4596
4827
  // Lazy sandbox provisioning: holder/group ids are fixed at turn start,
@@ -4623,7 +4854,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4623
4854
  // running a DIFFERENT image (solo → recreate on the new image; N-holders →
4624
4855
  // SandboxImageConflictError surfaced as an actionable turn error). Prefer the
4625
4856
  // explicit Modal image ref, else the docker image. The selfhosted branch
4626
- // (establishSelfhostedTurnSession/acquireSelfhostedLeaseForTurn) NEVER passes
4857
+ // (establishSelfhostedTurnSession) NEVER passes
4627
4858
  // an image — B3 lives only on this Modal else-branch.
4628
4859
  ...((runSettings.modalImageRef ?? runSettings.dockerImage)
4629
4860
  ? {
@@ -4637,7 +4868,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4637
4868
  ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
4638
4869
  },
4639
4870
  "turn",
4640
- sandboxHolderId,
4871
+ managedOwnership!.holderId,
4641
4872
  ),
4642
4873
  cancellationSignal,
4643
4874
  async (lateSandbox) => await lateSandbox.release(),
@@ -4719,7 +4950,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4719
4950
  // (initialize + tools/list) can resolve the per-workspace bearer from
4720
4951
  // codexRequestStorage (runtime/codexAppsMcpRequestInit). withCodex is the
4721
4952
  // identity on every non-codex turn, so this is a no-op for existing paths.
4722
- const resolveCredential = connectionTokenResolverForTurn({
4953
+ const rawResolveCredential = connectionTokenResolverForTurn({
4723
4954
  db,
4724
4955
  settings: runSettings,
4725
4956
  connectionCredentials: connectionCredentials ?? null,
@@ -4730,6 +4961,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4730
4961
  attemptId: input.attemptId,
4731
4962
  turn,
4732
4963
  });
4964
+ const resolveCredential: typeof rawResolveCredential = async (request) => {
4965
+ const result = await rawResolveCredential(request);
4966
+ if (result.status === "ok") {
4967
+ registerSecretRedactions(
4968
+ headerSecretRedactions("MCP", result.headers, Object.keys(result.headers)),
4969
+ );
4970
+ }
4971
+ return result;
4972
+ };
4733
4973
  preparedTools = await waitForTurnOperation(
4734
4974
  withCodex(() =>
4735
4975
  runtime.prepareTools(runSettings, turnTools, {
@@ -4795,6 +5035,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4795
5035
  },
4796
5036
  }
4797
5037
  : {};
5038
+ const hostedWebSearch = hostedWebSearchForTurn(
5039
+ resolvedModel,
5040
+ runSettings.webSearchEnabled,
5041
+ defaultNativeToolsAllowed,
5042
+ );
4798
5043
  const agent = runtime.buildAgent(modelRunSettings, turnResources, {
4799
5044
  reasoningEffort: turn.reasoningEffort,
4800
5045
  ...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
@@ -4854,11 +5099,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4854
5099
  // store/compaction follow the provider's compaction mode (registry
4855
5100
  // providers resolve to "client"); encrypted reasoning is only
4856
5101
  // round-tripped on the Responses wire API; hosted web search is attached
4857
- // only when the model opts in; the effective context window drives the
4858
- // compaction threshold.
5102
+ // only when BOTH the model opts in and this immutable turn still tracks
5103
+ // workspace defaults (an explicit session/turn remains narrowed); the
5104
+ // effective context window drives the compaction threshold.
5105
+ hostedWebSearch,
4859
5106
  ...(resolvedModel
4860
5107
  ? {
4861
- hostedWebSearch: resolvedModel.configured.hostedWebSearch,
4862
5108
  encryptedReasoning:
4863
5109
  resolvedModel.provider.api === "responses" &&
4864
5110
  runSettings.openaiReasoningEncryptedContent,
@@ -5206,7 +5452,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5206
5452
  } catch (compactError) {
5207
5453
  if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
5208
5454
  if (!isCompactionSummaryFailure(compactError)) throw compactError;
5209
- const errorMessage = compactionFailureReasonFromError(compactError);
5455
+ const errorMessage = String(redact(compactionFailureReasonFromError(compactError)));
5210
5456
  observability.error("context compaction failed", {
5211
5457
  sessionId: input.sessionId,
5212
5458
  turnId,
@@ -5559,6 +5805,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5559
5805
  contextCompactionSignal: () => modelResponseUsageContextSignal(modelResponseUsageState),
5560
5806
  contextCompactionRequested: () =>
5561
5807
  isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
5808
+ callModelInputFilter: secretRedactionModelInputFilter((value) => redact(value)),
5562
5809
  ...(toolCancellationFenceRef.current
5563
5810
  ? { turnToolCancellationFence: toolCancellationFenceRef.current }
5564
5811
  : {}),
@@ -5641,7 +5888,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5641
5888
  // conversation context preserved for the post-top-up resume.
5642
5889
  let serializedRunState: string | null = null;
5643
5890
  try {
5644
- serializedRunState = stream.state.toString();
5891
+ serializedRunState = String(redact(stream.state.toString()));
5645
5892
  } catch {
5646
5893
  serializedRunState = null;
5647
5894
  }
@@ -5661,7 +5908,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5661
5908
  executionGeneration,
5662
5909
  attemptId: input.attemptId,
5663
5910
  modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
5664
- ...pendingToolCall,
5911
+ callId: pendingToolCall.callId,
5912
+ callType: pendingToolCall.callType,
5913
+ callItem: redact(pendingToolCall.callItem) as Record<string, unknown>,
5665
5914
  });
5666
5915
  if (!registered.accepted) {
5667
5916
  throw new TurnAttemptFencedError(
@@ -5686,7 +5935,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5686
5935
  attemptId: input.attemptId,
5687
5936
  callId: completedToolCall.callId,
5688
5937
  modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
5689
- resultItem: completedToolCall.resultItem,
5938
+ resultItem: redact(completedToolCall.resultItem) as Record<string, unknown>,
5690
5939
  });
5691
5940
  if (!recorded.accepted) {
5692
5941
  throw new TurnAttemptFencedError(
@@ -5911,7 +6160,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5911
6160
  return claimedResult({ status: "requires_action" });
5912
6161
  }
5913
6162
 
5914
- const finalOutput = String(stream.finalOutput ?? "");
6163
+ const finalOutput = String(redact(String(stream.finalOutput ?? "")));
5915
6164
  await reconcileConversationTruth();
5916
6165
  // Op-stream durability fence: the tool outputs are now durably in the
5917
6166
  // history store (a redispatch would NOT re-execute them), so this
@@ -5992,8 +6241,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5992
6241
  sessionId: input.sessionId,
5993
6242
  turnId: activeTurnId,
5994
6243
  reason: recoveryKind,
5995
- code: overflow?.code,
5996
- error: overflow?.message ?? compactionNeeded?.message,
6244
+ code: overflow?.code ? String(redact(overflow.code)) : undefined,
6245
+ error: String(redact(overflow?.message ?? compactionNeeded?.message ?? "")),
5997
6246
  signalTokens: compactionNeeded?.signalTokens,
5998
6247
  thresholdTokens: compactionNeeded?.thresholdTokens,
5999
6248
  });
@@ -6020,7 +6269,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6020
6269
  // empty summary and must not create a new goal continuation.
6021
6270
  if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
6022
6271
  if (!isCompactionSummaryFailure(compactError)) throw compactError;
6023
- compactionFailureMessage = compactionFailureReasonFromError(compactError);
6272
+ compactionFailureMessage = String(
6273
+ redact(compactionFailureReasonFromError(compactError)),
6274
+ );
6024
6275
  observability.warn("context compaction recovery compaction failed", {
6025
6276
  sessionId: input.sessionId,
6026
6277
  turnId: activeTurnId,
@@ -6126,7 +6377,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6126
6377
  turnMetricOutcome = "recovering";
6127
6378
  return claimedResult({ status: "recovering" });
6128
6379
  } catch (recoveryError) {
6129
- console.error("sandbox lease supersession recovery failed", recoveryError);
6380
+ console.error(
6381
+ "sandbox lease supersession recovery failed",
6382
+ safeErrorDiagnostic(recoveryError, (value) => String(redact(value))),
6383
+ );
6130
6384
  throw recoveryError;
6131
6385
  }
6132
6386
  }
@@ -6172,7 +6426,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6172
6426
  // The database transition is atomic. If it could not commit, surface
6173
6427
  // the failure so Temporal can retry on a healthy worker; never mutate
6174
6428
  // the turn through a second cancellation path.
6175
- console.error("worker-shutdown recovery checkpoint failed", recoveryError);
6429
+ console.error(
6430
+ "worker-shutdown recovery checkpoint failed",
6431
+ safeErrorDiagnostic(recoveryError, (value) => String(redact(value))),
6432
+ );
6176
6433
  throw recoveryError;
6177
6434
  }
6178
6435
  }
@@ -6729,7 +6986,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6729
6986
  // same-policy continuation path. When no alternate exists (all capped,
6730
6987
  // or a single non-rotating account), persist the native capacity wait
6731
6988
  // instead of an in-memory delay/user-message recovery.
6732
- if (goalActive && goal && rotationResumeMs === null) {
6989
+ if (rotationResumeMs === null) {
6733
6990
  const providerResetAt =
6734
6991
  capacityAuthoritativeResetAt ??
6735
6992
  (usageLimit.resetsInSeconds !== null &&
@@ -6744,8 +7001,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6744
7001
  turnId,
6745
7002
  attemptId: input.attemptId,
6746
7003
  workflowId: input.workflowId,
6747
- goalId: goal.id,
6748
- goalVersion: goal.version,
7004
+ goalId: goalActive && goal ? goal.id : null,
7005
+ goalVersion: goalActive && goal ? goal.version : null,
6749
7006
  earliestResetAt: providerResetAt,
6750
7007
  resetKind: providerResetAt ? "authoritative" : "bounded_refresh",
6751
7008
  failurePayload,
@@ -6766,11 +7023,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6766
7023
  input.sessionId,
6767
7024
  armed.events,
6768
7025
  );
6769
- turnMetricOutcome = "failed";
6770
- activityStatus = "idle";
7026
+ turnMetricOutcome = "recovering";
7027
+ activityStatus = "waiting_capacity";
6771
7028
  activityError = error;
6772
7029
  return claimedResult({
6773
- status: "idle",
7030
+ status: "waiting_capacity",
6774
7031
  capacityWait: {
6775
7032
  waiterId: armed.waiter.id,
6776
7033
  generation: armed.waiter.generation,
@@ -6893,7 +7150,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6893
7150
  // truth, recover this SAME accepted turn, then let the workflow re-claim
6894
7151
  // it after a pacing delay. This is independent of goal state and never
6895
7152
  // relies on a synthetic continuation prompt.
6896
- const failure = agentRunFailurePayload(error, { isCodexTurn });
7153
+ const failure = redact(agentRunFailurePayload(error, { isCodexTurn })) as ReturnType<
7154
+ typeof agentRunFailurePayload
7155
+ >;
6897
7156
  if (isSessionEventPersistenceError(error)) {
6898
7157
  // Never pass the original Drizzle/postgres-js error to telemetry: its
6899
7158
  // nested cause may contain raw SQL and bound parameters. The typed DB
@@ -7031,8 +7290,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7031
7290
  );
7032
7291
  }
7033
7292
  await drainAttemptOwnedSandboxWriters({
7034
- toolCancellationFence: acknowledgeQuiescence ? toolCancellationFence : null,
7035
- cancellationReason: cancellationSignal?.reason,
7293
+ // Normal turn completion owns the same process boundary as
7294
+ // Pause/Steer: yielded provider shells must be terminated, polled,
7295
+ // and durably settled before workspace capture. Only receipt
7296
+ // publication remains conditional on acknowledgeQuiescence.
7297
+ toolCancellationFence,
7298
+ cancellationReason: cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
7036
7299
  gitCredentialRenewals: gitRenewalsToStop,
7037
7300
  toolspaceTokenRenewal: toolspaceRenewalToStop,
7038
7301
  runCredentialRenewal: runRenewalToStop,
@@ -7090,14 +7353,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7090
7353
  });
7091
7354
  },
7092
7355
  onReceiptFailure: (error) => {
7093
- console.error("agent turn quiescence receipt exhausted; signalling proof", error);
7356
+ console.error(
7357
+ "agent turn quiescence receipt exhausted; signalling proof",
7358
+ safeErrorDiagnostic(error, (value) => String(redact(value))),
7359
+ );
7094
7360
  },
7095
7361
  onPublishFailure: (error) => {
7096
- console.error("agent turn quiescence event fanout failed", error);
7362
+ console.error(
7363
+ "agent turn quiescence event fanout failed",
7364
+ safeErrorDiagnostic(error, (value) => String(redact(value))),
7365
+ );
7097
7366
  },
7098
7367
  onSignalFailure: (error, attempt, retryMs) => {
7099
7368
  console.error("agent turn quiescence proof signal failed; retrying", {
7100
- error,
7369
+ error: safeErrorDiagnostic(error, (value) => String(redact(value))),
7101
7370
  attempt,
7102
7371
  retryMs,
7103
7372
  });
@@ -7348,14 +7617,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7348
7617
  resolvedSandbox = null; // drop ownership now; the exact-holder release may finish later
7349
7618
  await waitForTurnFinalizerStep(
7350
7619
  sandboxToRelease.release().catch((releaseError) => {
7351
- console.error("sandbox lease release failed (turn outcome unaffected)", releaseError);
7620
+ console.error(
7621
+ "sandbox lease release failed (turn outcome unaffected)",
7622
+ safeErrorDiagnostic(releaseError, (value) => String(redact(value))),
7623
+ );
7352
7624
  }),
7353
7625
  finalizerSignal,
7354
7626
  );
7355
7627
  }
7356
7628
  } catch (error) {
7357
7629
  finalizationError ??= error;
7358
- console.error("agent turn finalization failed (turn outcome unaffected)", error);
7630
+ console.error(
7631
+ "agent turn finalization failed (turn outcome unaffected)",
7632
+ safeErrorDiagnostic(error, (value) => String(redact(value))),
7633
+ );
7359
7634
  } finally {
7360
7635
  cancellationSignal?.removeEventListener("abort", noteCancellationRequested);
7361
7636
  const completedAt = performance.now();
@@ -7405,7 +7680,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7405
7680
  "opengeni.duration_ms": Math.round(durationSeconds * 1000),
7406
7681
  "opengeni.finalization_duration_ms": Math.round(finalizationDurationSeconds * 1000),
7407
7682
  },
7408
- error: finalizationError ?? activityError,
7683
+ error:
7684
+ finalizationError || activityError
7685
+ ? safeErrorForTelemetry(finalizationError ?? activityError, (value) =>
7686
+ String(redact(value)),
7687
+ )
7688
+ : undefined,
7409
7689
  });
7410
7690
  assertPhysicalToolQuiescenceForCancellation({
7411
7691
  acknowledgeQuiescence,
@@ -7480,6 +7760,13 @@ async function assertGitHubTokenMintSelectionAuthorized(
7480
7760
  * that will NOT clear on retry) is classified and returned BEFORE this in
7481
7761
  * agentRunFailurePayload, so it never reaches here.
7482
7762
  */
7763
+ const STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE =
7764
+ "unable to connect. is the computer able to access the url?";
7765
+
7766
+ function isExactStatuslessUpstreamConnectivityMessage(message: string): boolean {
7767
+ return message.trim().toLowerCase() === STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE;
7768
+ }
7769
+
7483
7770
  export function isTransientProviderError(error: unknown): boolean {
7484
7771
  const status =
7485
7772
  typeof error === "object" && error !== null && "status" in error
@@ -7501,6 +7788,9 @@ export function isTransientProviderError(error: unknown): boolean {
7501
7788
  return true;
7502
7789
  }
7503
7790
  const message = error instanceof Error ? error.message : String(error);
7791
+ if (isExactStatuslessUpstreamConnectivityMessage(message)) {
7792
+ return true;
7793
+ }
7504
7794
  return /overloaded|an error occurred while processing your request|connection error|service unavailable|bad gateway|gateway timeout/i.test(
7505
7795
  message,
7506
7796
  );
@@ -7627,6 +7917,14 @@ export function agentRunFailurePayload(
7627
7917
  // retryable so a goal-bearing session idles and auto-continues instead of going
7628
7918
  // terminal on a provider's bad minute. See isTransientProviderError.
7629
7919
  if (isTransientProviderError(error)) {
7920
+ if (isExactStatuslessUpstreamConnectivityMessage(message)) {
7921
+ return {
7922
+ error:
7923
+ "OpenGeni could not reach an upstream service. The same turn will retry after a short delay.",
7924
+ code: "upstream_connectivity_unavailable",
7925
+ retryable: true,
7926
+ };
7927
+ }
7630
7928
  return { error: message, code: "provider_unavailable", retryable: true };
7631
7929
  }
7632
7930
  return { error: message };