@opengeni/worker-bundle 0.11.2 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -11,7 +11,11 @@ import {
11
11
  startupRetryOptions,
12
12
  temporalConnectionOptions
13
13
  } from "@opengeni/config";
14
- import { createDb as createDb2, markSessionWorkflowWakeDelivered } from "@opengeni/db";
14
+ import {
15
+ assertRuntimeDatabasePosture as assertRuntimeDatabasePosture2,
16
+ createDb as createDb2,
17
+ markSessionWorkflowWakeDelivered
18
+ } from "@opengeni/db";
15
19
  import { createNatsEventBus as createNatsEventBus2 } from "@opengeni/events";
16
20
  import {
17
21
  createObservability as createObservability2,
@@ -137,6 +141,7 @@ import {
137
141
  SUMMARY_BUFFER_TOKENS as SUMMARY_BUFFER_TOKENS2,
138
142
  runOwnedSandboxSetup,
139
143
  RoutingMutationOutcomeUnknownError,
144
+ WorkspaceArchiveIntegrityError as WorkspaceArchiveIntegrityError2,
140
145
  swapTargetEstablishability,
141
146
  deleteRecordingArtifacts as deleteRecordingArtifacts2,
142
147
  stopRecording as stopRecordingOnBox2
@@ -181,22 +186,14 @@ import { CancelledFailure as CancelledFailure2 } from "@temporalio/activity";
181
186
  // src/activities/capabilities.ts
182
187
  import {
183
188
  environmentsEncryptionKeyBytes,
184
- parseModelProvidersJson
189
+ withCodexCatalogProvider
185
190
  } from "@opengeni/config";
186
191
  import { settingsWithMcpCapabilityServers } from "@opengeni/core";
187
192
  import {
188
193
  CODEX_APPS_MCP_SERVER_ID,
189
194
  CODEX_APPS_MCP_SERVER_NAME,
190
195
  CODEX_APPS_MCP_URL,
191
- CODEX_APPS_STARTUP_TIMEOUT_MS,
192
- CODEX_FALLBACK_MODEL_SLUGS,
193
- CODEX_MODEL_CONTEXT_WINDOW_TOKENS,
194
- CODEX_MODEL_EFFECTIVE_CONTEXT_WINDOW_TOKENS,
195
- CODEX_MODEL_AUTO_COMPACT_TOKEN_LIMIT,
196
- CODEX_MODEL_TOOL_OUTPUT_TRUNCATION_TOKENS,
197
- CODEX_MODEL_ID_PREFIX,
198
- CODEX_PROVIDER_BASE_URL,
199
- CODEX_PROVIDER_ID
196
+ CODEX_APPS_STARTUP_TIMEOUT_MS
200
197
  } from "@opengeni/codex";
201
198
  import {
202
199
  listEnabledMcpCapabilityServers,
@@ -208,7 +205,7 @@ async function settingsWithEnabledCapabilityMcpServers(db, workspaceId, settings
208
205
  const enabled = await listEnabledMcpCapabilityServers(db, workspaceId);
209
206
  return settingsWithMcpCapabilityServers(settings, enabled);
210
207
  }
211
- async function settingsWithSessionMcpServersForRun(db, workspaceId, sessionId, attemptId, settings) {
208
+ async function settingsWithSessionMcpServersForRun(db, workspaceId, sessionId, attemptId, settings, options) {
212
209
  const encryptionKey = environmentsEncryptionKeyBytes(settings);
213
210
  if (!encryptionKey) {
214
211
  const metadata = await listSessionMcpServerMetadata(db, workspaceId, sessionId);
@@ -221,10 +218,15 @@ async function settingsWithSessionMcpServersForRun(db, workspaceId, sessionId, a
221
218
  );
222
219
  }
223
220
  }
224
- return settingsWithSessionMcpServers(
225
- settings,
226
- await listSessionMcpServersForRun(db, workspaceId, sessionId, attemptId, encryptionKey ?? null)
221
+ const servers = await listSessionMcpServersForRun(
222
+ db,
223
+ workspaceId,
224
+ sessionId,
225
+ attemptId,
226
+ encryptionKey ?? null
227
227
  );
228
+ options?.onResolvedServers?.(servers);
229
+ return settingsWithSessionMcpServers(settings, servers);
228
230
  }
229
231
  function settingsWithSessionMcpServers(settings, servers) {
230
232
  if (servers.length === 0) {
@@ -255,6 +257,9 @@ async function settingsWithCodexCredential(db, workspaceId, settings, activeOver
255
257
  return settings;
256
258
  }
257
259
  const withProvider = withCodexProvider(settings);
260
+ if (!settings.codexConnectedAppsEnabled) {
261
+ return withProvider;
262
+ }
258
263
  return withCodexAppsMcpServer(withProvider);
259
264
  }
260
265
  function withCodexAppsMcpServer(settings) {
@@ -279,30 +284,7 @@ function withCodexAppsMcpServer(settings) {
279
284
  };
280
285
  }
281
286
  function withCodexProvider(settings) {
282
- const providers = parseModelProvidersJson(settings.modelProvidersJson);
283
- if (providers.some((provider) => provider.id === CODEX_PROVIDER_ID)) {
284
- return settings;
285
- }
286
- const codexProvider = {
287
- kind: "codex-subscription",
288
- id: CODEX_PROVIDER_ID,
289
- label: "Codex (ChatGPT subscription)",
290
- api: "responses",
291
- baseUrl: CODEX_PROVIDER_BASE_URL,
292
- models: CODEX_FALLBACK_MODEL_SLUGS.map((slug) => ({
293
- id: `${CODEX_MODEL_ID_PREFIX}${slug}`,
294
- upstreamModelId: slug,
295
- label: slug,
296
- reasoningEffort: true,
297
- // These three values are the live Codex CLI catalog policy, kept distinct
298
- // so proactive compaction and the effective input guard match Codex core.
299
- contextWindowTokens: CODEX_MODEL_CONTEXT_WINDOW_TOKENS,
300
- effectiveContextWindowTokens: CODEX_MODEL_EFFECTIVE_CONTEXT_WINDOW_TOKENS,
301
- autoCompactTokenLimit: CODEX_MODEL_AUTO_COMPACT_TOKEN_LIMIT,
302
- toolOutputTruncationTokens: CODEX_MODEL_TOOL_OUTPUT_TRUNCATION_TOKENS
303
- }))
304
- };
305
- return { ...settings, modelProvidersJson: JSON.stringify([...providers, codexProvider]) };
287
+ return withCodexCatalogProvider(settings);
306
288
  }
307
289
 
308
290
  // src/activities/codex-rotation.ts
@@ -1009,7 +991,7 @@ function createCodexCapacityActivities(services) {
1009
991
  }
1010
992
  }
1011
993
  if (result.action === "resumed") {
1012
- return { action: "resumed", updateId: result.update.id };
994
+ return { action: "resumed" };
1013
995
  }
1014
996
  if (result.action === "waiting") {
1015
997
  return {
@@ -1029,7 +1011,7 @@ function createCodexCapacityActivities(services) {
1029
1011
  import {
1030
1012
  buildModelResolver,
1031
1013
  CODEX_CLIENT_VERSION,
1032
- CODEX_FALLBACK_MODEL_SLUGS as CODEX_FALLBACK_MODEL_SLUGS2,
1014
+ CODEX_FALLBACK_MODEL_SLUGS,
1033
1015
  CodexReloginRequired,
1034
1016
  classifyCodexResponseTimeoutError,
1035
1017
  classifyCodexUsageLimitError,
@@ -1063,7 +1045,11 @@ function workflowIdForSession(sessionId) {
1063
1045
  }
1064
1046
 
1065
1047
  // src/activities/agent-turn.ts
1066
- import { enabledCapabilityMcpToolRefs, resolveSessionToolPolicy } from "@opengeni/core";
1048
+ import {
1049
+ enabledCapabilityMcpToolRefs,
1050
+ resolveSessionToolPolicy,
1051
+ sessionToolPolicyAllowsDefaultNativeTools
1052
+ } from "@opengeni/core";
1067
1053
 
1068
1054
  // src/activities/context-compaction.ts
1069
1055
  import {
@@ -2675,44 +2661,14 @@ function childCompletionSummary(child, goal, terminalStatus) {
2675
2661
  }
2676
2662
 
2677
2663
  // src/activities/redaction.ts
2678
- var MIN_REDACTABLE_VALUE_LENGTH = 6;
2679
- function createSecretRedactor(secrets) {
2680
- const usable = secrets.filter((secret) => secret.value.length >= MIN_REDACTABLE_VALUE_LENGTH).sort((a, b) => b.value.length - a.value.length);
2681
- if (usable.length === 0) {
2682
- return identityRedactor;
2683
- }
2684
- const redact = (payload) => {
2685
- if (typeof payload === "string") {
2686
- let out = payload;
2687
- for (const secret of usable) {
2688
- if (out.includes(secret.value)) {
2689
- out = out.split(secret.value).join(`[redacted:${secret.name}]`);
2690
- }
2691
- }
2692
- return out;
2693
- }
2694
- if (Array.isArray(payload)) {
2695
- return payload.map(redact);
2696
- }
2697
- if (isPlainObject(payload)) {
2698
- return Object.fromEntries(
2699
- Object.entries(payload).map(([key, value]) => [key, redact(value)])
2700
- );
2701
- }
2702
- return payload;
2703
- };
2704
- return redact;
2705
- }
2706
- function identityRedactor(payload) {
2707
- return payload;
2708
- }
2709
- function isPlainObject(value) {
2710
- if (!value || typeof value !== "object") {
2711
- return false;
2712
- }
2713
- const proto = Object.getPrototypeOf(value);
2714
- return proto === Object.prototype || proto === null;
2715
- }
2664
+ import {
2665
+ createSecretRedactor,
2666
+ identityRedactor,
2667
+ isCredentialHeaderName,
2668
+ redactSensitiveData,
2669
+ redactSensitiveText,
2670
+ redactSerializedJson
2671
+ } from "@opengeni/contracts";
2716
2672
 
2717
2673
  // src/activities/run-input.ts
2718
2674
  import { createHash } from "crypto";
@@ -3162,6 +3118,7 @@ async function nextStreamEvent(iterator, context) {
3162
3118
  }
3163
3119
 
3164
3120
  // src/sandbox-resume.ts
3121
+ import { redactSensitiveText as redactSensitiveText2 } from "@opengeni/contracts";
3165
3122
  import {
3166
3123
  acquireLease,
3167
3124
  beginSandboxRematerialization,
@@ -3198,6 +3155,9 @@ function sandboxLeaseHolderIdForAttempt(attemptId) {
3198
3155
  }
3199
3156
  return `turn-attempt:${normalized}`;
3200
3157
  }
3158
+ function isRetryableDegradedRestore(restore) {
3159
+ return restore.status === "degraded" && restore.retryable === true;
3160
+ }
3201
3161
  var SandboxWarmingTimeoutError = class extends Error {
3202
3162
  constructor(backend, timeoutMs) {
3203
3163
  super(
@@ -3237,6 +3197,16 @@ var SnapshotTimeoutError = class extends Error {
3237
3197
  this.name = "SnapshotTimeoutError";
3238
3198
  }
3239
3199
  };
3200
+ function safeSnapshotError(error) {
3201
+ const rawName = error instanceof Error ? error.name : "Error";
3202
+ return {
3203
+ name: /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error",
3204
+ message: redactSensitiveText2(error instanceof Error ? error.message : String(error)).slice(
3205
+ 0,
3206
+ 2048
3207
+ )
3208
+ };
3209
+ }
3240
3210
  async function waitForWarmSnapshot(snapshot, timeoutMs, signal) {
3241
3211
  if (signal?.aborted) return false;
3242
3212
  let timeout;
@@ -3262,7 +3232,10 @@ async function waitForWarmSnapshot(snapshot, timeoutMs, signal) {
3262
3232
  } catch (error) {
3263
3233
  if (signal?.aborted) return false;
3264
3234
  if (error instanceof SnapshotTimeoutError) {
3265
- console.error("mid-session workspace snapshot wait timed out (turn unaffected)", error);
3235
+ console.error(
3236
+ "mid-session workspace snapshot wait timed out (turn unaffected)",
3237
+ safeSnapshotError(error)
3238
+ );
3266
3239
  return false;
3267
3240
  }
3268
3241
  return true;
@@ -3438,7 +3411,10 @@ async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEp
3438
3411
  await deletePriorPersistedSnapshot(persistable, priorArchiveForGc);
3439
3412
  return true;
3440
3413
  } catch (error) {
3441
- console.error("mid-session workspace snapshot failed (turn unaffected)", error);
3414
+ console.error(
3415
+ "mid-session workspace snapshot failed (turn unaffected)",
3416
+ safeSnapshotError(error)
3417
+ );
3442
3418
  return false;
3443
3419
  }
3444
3420
  }
@@ -3759,70 +3735,6 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
3759
3735
  throw warmingError;
3760
3736
  }
3761
3737
  }
3762
- async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
3763
- const { db, settings } = services;
3764
- const leaseTtlMs = settings.sandboxLeaseTtlMs;
3765
- let released = false;
3766
- const release = async () => {
3767
- if (released) {
3768
- return;
3769
- }
3770
- released = true;
3771
- await releaseLeaseHolder(db, {
3772
- accountId: ids.accountId,
3773
- workspaceId: ids.workspaceId,
3774
- sandboxGroupId: ids.sandboxGroupId,
3775
- kind,
3776
- holderId,
3777
- idleGraceMs: settings.sandboxIdleGraceMs
3778
- });
3779
- };
3780
- const acquired = await acquireLease(db, {
3781
- accountId: ids.accountId,
3782
- workspaceId: ids.workspaceId,
3783
- sandboxGroupId: ids.sandboxGroupId,
3784
- kind,
3785
- holderId,
3786
- subjectId: ids.sessionId,
3787
- backend: "selfhosted",
3788
- os: "linux",
3789
- leaseTtlMs,
3790
- warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs
3791
- });
3792
- if (acquired.role === "blocked") {
3793
- await release();
3794
- throw new SandboxLeaseRecoveryBlockedError(
3795
- ids.sandboxGroupId,
3796
- acquired.lease.leaseEpoch,
3797
- acquired.code,
3798
- acquired.lease.recovery
3799
- );
3800
- }
3801
- if (acquired.role === "fenced") {
3802
- await release();
3803
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
3804
- }
3805
- if (acquired.role === "spawner") {
3806
- const expectedEpoch = acquired.lease.leaseEpoch;
3807
- const committed = await commitWarmingToWarm(db, {
3808
- accountId: ids.accountId,
3809
- workspaceId: ids.workspaceId,
3810
- sandboxGroupId: ids.sandboxGroupId,
3811
- expectedEpoch,
3812
- // No provider box id; stamp a clearly non-box marker for diagnostics.
3813
- instanceId: `selfhosted:${ids.sessionId}`,
3814
- resumeBackendId: "selfhosted",
3815
- resumeState: null,
3816
- leaseTtlMs
3817
- });
3818
- if (!committed.committed || !committed.lease) {
3819
- await release();
3820
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
3821
- }
3822
- return { leaseEpoch: committed.lease.leaseEpoch, release };
3823
- }
3824
- return { leaseEpoch: acquired.lease.leaseEpoch, release };
3825
- }
3826
3738
  async function waitForWarm(services, ids) {
3827
3739
  const { db, settings } = services;
3828
3740
  const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
@@ -3839,6 +3751,9 @@ async function waitForWarm(services, ids) {
3839
3751
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
3840
3752
  }
3841
3753
  if (lease.liveness === "cold") {
3754
+ if (isRetryableDegradedRestore(lease.recovery.restore)) {
3755
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
3756
+ }
3842
3757
  if (lease.recovery.restore.status === "degraded" || lease.recovery.restore.status === "unrecoverable") {
3843
3758
  throw new SandboxLeaseRecoveryBlockedError(
3844
3759
  ids.sandboxGroupId,
@@ -3857,14 +3772,17 @@ async function waitForWarm(services, ids) {
3857
3772
  import {
3858
3773
  advanceWorkspaceGenerationForRetainedProcess,
3859
3774
  advanceWorkspaceGeneration,
3775
+ getRetainedProcess,
3860
3776
  retainWorkspaceMutationProcess,
3777
+ retainedProcessSettlementIdentity,
3861
3778
  settleRetainedProcess,
3862
3779
  verifyRetainedProcessMutationSettlement,
3863
3780
  verifyWorkspaceMutationSettlement,
3864
3781
  getSandbox,
3865
3782
  markWarmLeaseInstanceLost as markWarmLeaseInstanceLost2,
3866
3783
  readLease as readLease2,
3867
- readActiveSandbox
3784
+ readActiveSandbox,
3785
+ SandboxRetainedProcessPromotionFencedError
3868
3786
  } from "@opengeni/db";
3869
3787
  import {
3870
3788
  buildSelfhostedBackendSession,
@@ -3900,7 +3818,10 @@ function selfhostedTimeoutsFromSettings(settings) {
3900
3818
  }
3901
3819
  function selfhostedResolverTimeouts(settings) {
3902
3820
  const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
3903
- return { selfhostedTimeoutMs: timeoutMs, selfhostedExecTimeoutMs: execTimeoutMs };
3821
+ return {
3822
+ selfhostedTimeoutMs: timeoutMs,
3823
+ selfhostedExecTimeoutMs: execTimeoutMs
3824
+ };
3904
3825
  }
3905
3826
  function homeRouteRecoveryDisposition(lease) {
3906
3827
  if (!lease) return "unrecoverable";
@@ -3945,7 +3866,10 @@ async function resolveCurrentHomeBackend(services, ids, established) {
3945
3866
  throw homeRouteRecoveryError(lease, fallbackEpoch);
3946
3867
  }
3947
3868
  if (lease.instanceId === established.instanceId) {
3948
- services.onHomeSandboxRebound?.({ established, leaseEpoch: lease.leaseEpoch });
3869
+ services.onHomeSandboxRebound?.({
3870
+ established,
3871
+ leaseEpoch: lease.leaseEpoch
3872
+ });
3949
3873
  return {
3950
3874
  session: established.session,
3951
3875
  sandboxId: null,
@@ -3999,7 +3923,10 @@ async function resolveCurrentHomeBackend(services, ids, established) {
3999
3923
  if (rebound.instanceId !== lease.instanceId || rebound.backendId !== resumeBackend) {
4000
3924
  throw homeRouteRecoveryError(lease, lease.leaseEpoch);
4001
3925
  }
4002
- services.onHomeSandboxRebound?.({ established: rebound, leaseEpoch: lease.leaseEpoch });
3926
+ services.onHomeSandboxRebound?.({
3927
+ established: rebound,
3928
+ leaseEpoch: lease.leaseEpoch
3929
+ });
4003
3930
  Object.assign(established, rebound);
4004
3931
  return {
4005
3932
  session: rebound.session,
@@ -4040,29 +3967,46 @@ function afterPersistableHomeMutation(services, ids, home) {
4040
3967
  }
4041
3968
  const exactAdmission = admission;
4042
3969
  if (outcome === "resolved" && retainedProcess) {
4043
- await retainWorkspaceMutationProcess(services.db, {
4044
- accountId: fence.accountId,
4045
- workspaceId: ids.workspaceId,
4046
- sessionId: ids.sessionId,
4047
- processId: retainedProcess.id,
4048
- providerSessionId: retainedProcess.providerSessionId,
4049
- admissionId: exactAdmission.id,
4050
- admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
4051
- operation: op,
4052
- owner: {
4053
- kind: "turn",
4054
- turnId: fence.turnId,
4055
- executionGeneration: fence.executionGeneration,
4056
- attemptId: fence.attemptId,
4057
- holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
4058
- sandboxGroupId: home.sandboxGroupId,
4059
- expectedEpoch: backend.leaseEpoch,
4060
- expectedInstanceId: backend.providerInstanceId,
4061
- routeKind: exactAdmission.routeKind,
4062
- routeTargetId: exactAdmission.routeTargetId,
4063
- routeEpoch: exactAdmission.routeEpoch
3970
+ try {
3971
+ await retainWorkspaceMutationProcess(services.db, {
3972
+ accountId: fence.accountId,
3973
+ workspaceId: ids.workspaceId,
3974
+ sessionId: ids.sessionId,
3975
+ processId: retainedProcess.id,
3976
+ providerSessionId: retainedProcess.providerSessionId,
3977
+ admissionId: exactAdmission.id,
3978
+ admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
3979
+ operation: op,
3980
+ owner: {
3981
+ kind: "turn",
3982
+ turnId: fence.turnId,
3983
+ executionGeneration: fence.executionGeneration,
3984
+ attemptId: fence.attemptId,
3985
+ holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
3986
+ sandboxGroupId: home.sandboxGroupId,
3987
+ expectedEpoch: backend.leaseEpoch,
3988
+ expectedInstanceId: backend.providerInstanceId,
3989
+ routeKind: exactAdmission.routeKind,
3990
+ routeTargetId: exactAdmission.routeTargetId,
3991
+ routeEpoch: exactAdmission.routeEpoch
3992
+ }
3993
+ });
3994
+ } catch (error) {
3995
+ if (!(error instanceof SandboxRetainedProcessPromotionFencedError)) throw error;
3996
+ const durable = error.process;
3997
+ if (durable.id !== retainedProcess.id || durable.providerSessionId !== retainedProcess.providerSessionId || durable.sessionId !== ids.sessionId || durable.sandboxGroupId !== home.sandboxGroupId || durable.parentAdmissionId !== exactAdmission.id || durable.leaseEpoch !== backend.leaseEpoch || durable.providerInstanceId !== backend.providerInstanceId || durable.routeKind !== exactAdmission.routeKind || durable.routeTargetId !== exactAdmission.routeTargetId || durable.routeEpoch !== exactAdmission.routeEpoch || durable.state !== "active") {
3998
+ throw new Error("Durable retained-process promotion returned a mismatched identity", {
3999
+ cause: error
4000
+ });
4064
4001
  }
4065
- });
4002
+ return {
4003
+ status: "retained_process_durable_output_rejected",
4004
+ retainedProcess: {
4005
+ id: durable.id,
4006
+ providerSessionId: durable.providerSessionId
4007
+ }
4008
+ };
4009
+ }
4066
4010
  return;
4067
4011
  }
4068
4012
  await verifyWorkspaceMutationSettlement(services.db, {
@@ -4114,12 +4058,21 @@ function afterRetainedProcessMutation(services, ids) {
4114
4058
  function settleRetainedProcessForTurn(services, ids) {
4115
4059
  const fence = ids.workspaceMutationFence;
4116
4060
  if (!fence) return void 0;
4117
- return async ({ process: process2, proof }) => {
4061
+ return async ({ backend, process: process2, proof }) => {
4062
+ const durable = await getRetainedProcess(services.db, {
4063
+ workspaceId: ids.workspaceId,
4064
+ sessionId: ids.sessionId,
4065
+ processId: process2.id
4066
+ });
4067
+ if (!durable || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== backend.kind || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
4068
+ throw new Error("Retained-process settlement lost its exact durable backend identity");
4069
+ }
4118
4070
  await settleRetainedProcess(services.db, {
4119
4071
  accountId: fence.accountId,
4120
4072
  workspaceId: ids.workspaceId,
4121
4073
  sessionId: ids.sessionId,
4122
4074
  processId: process2.id,
4075
+ expected: retainedProcessSettlementIdentity(durable),
4123
4076
  outcome: proof.outcome,
4124
4077
  exitCode: proof.exitCode,
4125
4078
  reason: proof.reason,
@@ -4449,6 +4402,7 @@ import {
4449
4402
  var turnTrackers = /* @__PURE__ */ new WeakMap();
4450
4403
  var creditBalanceGaugeAccounts = /* @__PURE__ */ new WeakMap();
4451
4404
  var modelCacheCounterTotals = /* @__PURE__ */ new WeakMap();
4405
+ var retainedProcessTerminalBacklogTotals = /* @__PURE__ */ new WeakMap();
4452
4406
  var MAX_MODEL_CACHE_COUNTER_TOTAL = 1e12;
4453
4407
  function observabilityEventLogger(observability) {
4454
4408
  return {
@@ -4734,6 +4688,103 @@ function recordSandboxOrphansTerminated(observability, count) {
4734
4688
  amount: count
4735
4689
  });
4736
4690
  }
4691
+ var RETAINED_PROCESS_OWNER_STATES = [
4692
+ "direct",
4693
+ "queued",
4694
+ "running",
4695
+ "requires_action",
4696
+ "recovering",
4697
+ "waiting_capacity",
4698
+ "completed",
4699
+ "failed",
4700
+ "cancelled",
4701
+ "superseded",
4702
+ "withdrawn_for_edit",
4703
+ "missing",
4704
+ "unknown"
4705
+ ];
4706
+ var RETAINED_PROCESS_OWNER_STATE_SET = new Set(RETAINED_PROCESS_OWNER_STATES);
4707
+ function recordRetainedProcessInventoryGauges(observability, counts) {
4708
+ const normalized = /* @__PURE__ */ new Map();
4709
+ for (const ownerState of RETAINED_PROCESS_OWNER_STATES) {
4710
+ normalized.set(ownerState, { active: 0, terminal: 0 });
4711
+ }
4712
+ for (const count of counts) {
4713
+ const ownerState = RETAINED_PROCESS_OWNER_STATE_SET.has(count.ownerState) ? count.ownerState : "unknown";
4714
+ const current = normalized.get(ownerState);
4715
+ current.active += Math.max(0, count.activeCount);
4716
+ current.terminal += Math.max(0, count.terminalOwnerCount);
4717
+ }
4718
+ let terminalTotal = 0;
4719
+ for (const [ownerState, count] of normalized) {
4720
+ terminalTotal += count.terminal;
4721
+ observability.setGauge({
4722
+ name: "opengeni_retained_processes_active",
4723
+ help: "Current active retained provider processes by durable owner state.",
4724
+ labels: { owner_state: ownerState },
4725
+ value: count.active
4726
+ });
4727
+ observability.setGauge({
4728
+ name: "opengeni_retained_processes_terminal_owner_backlog",
4729
+ help: "Current active retained processes whose exact owner attempt is terminal.",
4730
+ labels: { owner_state: ownerState },
4731
+ value: count.terminal
4732
+ });
4733
+ }
4734
+ const previous = retainedProcessTerminalBacklogTotals.get(observability);
4735
+ if (previous !== void 0 && terminalTotal > previous) {
4736
+ observability.incrementCounter({
4737
+ name: "opengeni_retained_process_terminal_owner_backlog_growth_total",
4738
+ help: "Positive growth in terminal-owner retained-process backlog between app samples.",
4739
+ amount: terminalTotal - previous
4740
+ });
4741
+ }
4742
+ retainedProcessTerminalBacklogTotals.set(observability, terminalTotal);
4743
+ }
4744
+ var EXPIRED_DRAINING_BACKENDS = [
4745
+ "none",
4746
+ "local",
4747
+ "docker",
4748
+ "modal",
4749
+ "e2b",
4750
+ "daytona",
4751
+ "blaxel",
4752
+ "runloop",
4753
+ "selfhosted",
4754
+ "unknown"
4755
+ ];
4756
+ var EXPIRED_DRAINING_BACKEND_SET = new Set(EXPIRED_DRAINING_BACKENDS);
4757
+ var EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"];
4758
+ function recordExpiredDrainingSandboxLeaseGauges(observability, counts) {
4759
+ const normalized = /* @__PURE__ */ new Map();
4760
+ for (const backend of EXPIRED_DRAINING_BACKENDS) {
4761
+ for (const ageBucket of EXPIRED_DRAINING_AGE_BUCKETS) {
4762
+ normalized.set(`${backend}:${ageBucket}`, 0);
4763
+ }
4764
+ }
4765
+ for (const count of counts) {
4766
+ const backend = EXPIRED_DRAINING_BACKEND_SET.has(count.backend) ? count.backend : "unknown";
4767
+ if (!EXPIRED_DRAINING_AGE_BUCKETS.includes(count.ageBucket)) continue;
4768
+ const key = `${backend}:${count.ageBucket}`;
4769
+ normalized.set(key, (normalized.get(key) ?? 0) + Math.max(0, count.count));
4770
+ }
4771
+ for (const [key, value] of normalized) {
4772
+ const [backend, ageBucket] = key.split(":");
4773
+ observability.setGauge({
4774
+ name: "opengeni_sandbox_leases_expired_draining",
4775
+ help: "Current expired draining sandbox leases by backend and fixed age bucket.",
4776
+ labels: { backend, age_bucket: ageBucket },
4777
+ value
4778
+ });
4779
+ }
4780
+ }
4781
+ function recordRetainedProcessReconciliation(observability, outcome) {
4782
+ observability.incrementCounter({
4783
+ name: "opengeni_retained_process_reconciliation_total",
4784
+ help: "Bounded retained-process reconciliation observations by fixed app outcome.",
4785
+ labels: { outcome }
4786
+ });
4787
+ }
4737
4788
  function recordCreditMicros(observability, kind, amountMicros) {
4738
4789
  if (!observability || amountMicros <= 0) {
4739
4790
  return;
@@ -5952,6 +6003,56 @@ function turnFinalizerCancellationSignal(temporalSignal, activityStatus) {
5952
6003
  function compactionFailureReason(reason) {
5953
6004
  return reason.startsWith("compaction summarization failed:") ? reason : `compaction summarization failed: ${reason}`;
5954
6005
  }
6006
+ function safeErrorDiagnostic(error, redactText = redactSensitiveText) {
6007
+ const rawName = error instanceof Error ? error.name : "Error";
6008
+ const name = /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error";
6009
+ const rawMessage = error instanceof Error ? error.message : String(error);
6010
+ const diagnostic = {
6011
+ name,
6012
+ message: redactText(rawMessage).slice(0, 2048)
6013
+ };
6014
+ if (error && typeof error === "object") {
6015
+ const status = Number(
6016
+ error.status ?? error.statusCode
6017
+ );
6018
+ if (Number.isInteger(status) && status >= 100 && status <= 599) {
6019
+ diagnostic.status = status;
6020
+ }
6021
+ const rawCode = error.code;
6022
+ if (typeof rawCode === "string" || typeof rawCode === "number") {
6023
+ const code = redactText(String(rawCode)).slice(0, 80);
6024
+ if (/^[A-Za-z0-9_.:-]+$/.test(code)) diagnostic.code = code;
6025
+ }
6026
+ }
6027
+ return diagnostic;
6028
+ }
6029
+ function safeErrorForTelemetry(error, redactText) {
6030
+ const diagnostic = safeErrorDiagnostic(error, redactText);
6031
+ const safe = new Error(diagnostic.message);
6032
+ safe.name = diagnostic.name;
6033
+ return safe;
6034
+ }
6035
+ function headerSecretRedactions(prefix, headers, credentialHeaderNames) {
6036
+ const discovered = [];
6037
+ const explicitCredentialHeaders = credentialHeaderNames ? new Set([...credentialHeaderNames].map((name) => name.toLowerCase())) : null;
6038
+ for (const [headerName, value] of Object.entries(headers ?? {})) {
6039
+ if (!explicitCredentialHeaders?.has(headerName.toLowerCase()) && !isCredentialHeaderName(headerName)) {
6040
+ continue;
6041
+ }
6042
+ const safeHeaderName = headerName.toUpperCase().replace(/[^A-Z0-9]+/g, "_");
6043
+ discovered.push({ name: `${prefix}_${safeHeaderName || "HEADER"}`, value });
6044
+ if (/^(?:proxy-)?authorization$/i.test(headerName)) {
6045
+ const credential = value.match(/^\s*[A-Za-z][A-Za-z0-9_-]*\s+(.+)\s*$/)?.[1];
6046
+ if (credential) {
6047
+ discovered.push({
6048
+ name: `${prefix}_${safeHeaderName || "AUTHORIZATION"}_CREDENTIAL`,
6049
+ value: credential
6050
+ });
6051
+ }
6052
+ }
6053
+ }
6054
+ return discovered;
6055
+ }
5955
6056
  function compactionFailureReasonFromError(error) {
5956
6057
  if (error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError) {
5957
6058
  return compactionFailureReason(error.message);
@@ -6264,10 +6365,8 @@ function applyCodexCacheTelemetry(observability, provider, normalizedUsage) {
6264
6365
  promptTokens: normalizedUsage.telemetry.inputTokens
6265
6366
  });
6266
6367
  }
6267
- function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = persistedHistoryCount, toolOutputTruncationTokens) {
6268
- const sanitized = sanitizeHistoryItemsForModel2(rawHistory, toolOutputTruncationTokens).filter(
6269
- (item) => !isEphemeralInternalContext(item)
6270
- );
6368
+ function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = persistedHistoryCount, toolOutputTruncationTokens, redactValue = (value) => value) {
6369
+ const sanitized = sanitizeHistoryItemsForModel2(rawHistory, toolOutputTruncationTokens).filter((item) => !isEphemeralInternalContext(item)).map((item) => redactValue(item));
6271
6370
  if (sanitized.length <= persistedHistoryCount) {
6272
6371
  return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
6273
6372
  }
@@ -6281,6 +6380,12 @@ function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = p
6281
6380
  nextPosition: nextPosition + rows.length
6282
6381
  };
6283
6382
  }
6383
+ function secretRedactionModelInputFilter(redactValue) {
6384
+ return ({ modelData }) => ({
6385
+ ...modelData,
6386
+ input: redactValue(modelData.input)
6387
+ });
6388
+ }
6284
6389
  function isModelOrToolProgressHistoryItem(item) {
6285
6390
  if (item.type === "message") {
6286
6391
  return item.role === "assistant";
@@ -6312,11 +6417,18 @@ async function resolveActiveSandboxBackend(routingOn, loadPointer, loadSandboxKi
6312
6417
  } catch (error) {
6313
6418
  console.error(
6314
6419
  "active sandbox backend resolution failed (turn proceeds on home backend)",
6315
- error
6420
+ safeErrorDiagnostic(error)
6316
6421
  );
6317
6422
  return void 0;
6318
6423
  }
6319
6424
  }
6425
+ function managedSandboxOwnershipForTurn(machinePrimary, attemptId, sandboxGroupId) {
6426
+ if (machinePrimary) return null;
6427
+ return {
6428
+ holderId: sandboxLeaseHolderIdForAttempt(attemptId),
6429
+ sandboxGroupId
6430
+ };
6431
+ }
6320
6432
  function pointerReconcileReason(record) {
6321
6433
  if (!record) {
6322
6434
  return "stale_pointer";
@@ -6406,6 +6518,10 @@ function computerToolModeForTurn(resolvedModel) {
6406
6518
  }
6407
6519
  return "hosted";
6408
6520
  }
6521
+ function hostedWebSearchForTurn(resolvedModel, deploymentWebSearchEnabled, defaultNativeToolsAllowed) {
6522
+ if (!defaultNativeToolsAllowed) return false;
6523
+ return resolvedModel?.configured.hostedWebSearch ?? deploymentWebSearchEnabled;
6524
+ }
6409
6525
  function modelAcceptsTypedAttachmentContentForTurn(resolvedModel) {
6410
6526
  return resolvedModel === null || resolvedModel.provider.api === "responses";
6411
6527
  }
@@ -6464,6 +6580,9 @@ function isLazySandboxProvisionRetryable(error) {
6464
6580
  if (error instanceof SandboxImageConflictError) {
6465
6581
  return false;
6466
6582
  }
6583
+ if (error instanceof WorkspaceArchiveIntegrityError2) {
6584
+ return error.retryable;
6585
+ }
6467
6586
  if (error instanceof SandboxLeaseSupersededError2 || error instanceof SandboxWarmingTimeoutError) {
6468
6587
  return true;
6469
6588
  }
@@ -6854,6 +6973,7 @@ function createRunAgentTurnActivity(services) {
6854
6973
  await current?.flush().catch(() => void 0);
6855
6974
  };
6856
6975
  let preparedTools = null;
6976
+ let defaultNativeToolsAllowed = false;
6857
6977
  const toolCancellationFenceRef = {
6858
6978
  current: null
6859
6979
  };
@@ -6890,7 +7010,7 @@ function createRunAgentTurnActivity(services) {
6890
7010
  payload: { sandboxId: lostSandbox.instanceId }
6891
7011
  }
6892
7012
  ]).catch((publishError) => {
6893
- console.error("sandbox box lost event publish failed", publishError);
7013
+ console.error("sandbox box lost event publish failed", safeErrorDiagnostic(publishError));
6894
7014
  });
6895
7015
  };
6896
7016
  const startLeaseHeartbeat = (sandbox, warmBackend) => {
@@ -7007,7 +7127,7 @@ function createRunAgentTurnActivity(services) {
7007
7127
  }
7008
7128
  console.error(
7009
7129
  "computer-use recording start failed (action outcome unaffected)",
7010
- recordingError
7130
+ safeErrorDiagnostic(recordingError)
7011
7131
  );
7012
7132
  }
7013
7133
  })();
@@ -7027,7 +7147,8 @@ function createRunAgentTurnActivity(services) {
7027
7147
  rawHistory,
7028
7148
  persistedHistoryCount,
7029
7149
  nextHistoryPosition,
7030
- modelRunSettings.modelToolOutputTruncationTokens
7150
+ modelRunSettings.modelToolOutputTruncationTokens,
7151
+ (value) => redact(value)
7031
7152
  );
7032
7153
  const hasModelOrToolProgress = rows.some(
7033
7154
  (row) => isModelOrToolProgressHistoryItem(row.item)
@@ -7070,11 +7191,14 @@ function createRunAgentTurnActivity(services) {
7070
7191
  });
7071
7192
  }
7072
7193
  } catch (persistError) {
7073
- console.error("session history dual-write failed (run unaffected)", persistError);
7194
+ console.error(
7195
+ "session history dual-write failed (run unaffected)",
7196
+ safeErrorDiagnostic(persistError, (value) => String(redact(value)))
7197
+ );
7074
7198
  if (options.requireDurable) throw persistError;
7075
7199
  }
7076
7200
  };
7077
- let redact = identityRedactor;
7201
+ let redact = createSecretRedactor([]);
7078
7202
  const secretRedactions = /* @__PURE__ */ new Map();
7079
7203
  const publishedRunCredentialNotices = /* @__PURE__ */ new Set();
7080
7204
  const registerSecretRedactions = (secrets) => {
@@ -7271,6 +7395,7 @@ function createRunAgentTurnActivity(services) {
7271
7395
  producerId,
7272
7396
  producerSeq: ++producerSeq
7273
7397
  }));
7398
+ const redactedRunState = inputSettlement.runState ? redact(inputSettlement.runState) : void 0;
7274
7399
  const result = await applySessionTurnSettlement(db, input.workspaceId, {
7275
7400
  sessionId: input.sessionId,
7276
7401
  turnId,
@@ -7280,7 +7405,7 @@ function createRunAgentTurnActivity(services) {
7280
7405
  sessionStatus: inputSettlement.sessionStatus,
7281
7406
  activeTurnId: inputSettlement.activeTurnId,
7282
7407
  events: inputs,
7283
- ...inputSettlement.runState ? { runState: inputSettlement.runState } : {},
7408
+ ...redactedRunState ? { runState: redactedRunState } : {},
7284
7409
  ...recordingMutation ? { recording: recordingMutation } : {},
7285
7410
  ...compactionRequestFailure ? { compactionRequestFailure } : {}
7286
7411
  });
@@ -7634,43 +7759,42 @@ function createRunAgentTurnActivity(services) {
7634
7759
  const goal = await getSessionGoal3(db, input.workspaceId, input.sessionId).catch(
7635
7760
  () => null
7636
7761
  );
7637
- if (goal?.status === "active") {
7638
- const armed = await armCodexCapacityWait(db, {
7639
- accountId: input.accountId,
7640
- workspaceId: input.workspaceId,
7641
- sessionId: input.sessionId,
7642
- turnId,
7643
- attemptId: input.attemptId,
7644
- workflowId: input.workflowId,
7645
- goalId: goal.id,
7646
- goalVersion: goal.version,
7647
- earliestResetAt: null,
7648
- resetKind: "bounded_refresh",
7649
- failurePayload: {
7650
- error: "All connected Codex subscriptions are disabled for new allocations.",
7651
- code: "codex_allocator_disabled",
7652
- detail: "waiting for a credential to be re-enabled, reconnected, or added"
7762
+ const activeGoal = goal?.status === "active" ? goal : null;
7763
+ const armed = await armCodexCapacityWait(db, {
7764
+ accountId: input.accountId,
7765
+ workspaceId: input.workspaceId,
7766
+ sessionId: input.sessionId,
7767
+ turnId,
7768
+ attemptId: input.attemptId,
7769
+ workflowId: input.workflowId,
7770
+ goalId: activeGoal?.id ?? null,
7771
+ goalVersion: activeGoal?.version ?? null,
7772
+ earliestResetAt: null,
7773
+ resetKind: "bounded_refresh",
7774
+ failurePayload: {
7775
+ error: "All connected Codex subscriptions are disabled for new allocations.",
7776
+ code: "codex_allocator_disabled",
7777
+ detail: "waiting for a credential to be re-enabled, reconnected, or added"
7778
+ }
7779
+ });
7780
+ if (armed.action === "waiting") {
7781
+ await publishDurableSessionEvents(
7782
+ bus,
7783
+ input.workspaceId,
7784
+ input.sessionId,
7785
+ armed.events
7786
+ );
7787
+ turnMetricOutcome = "recovering";
7788
+ activityStatus = "waiting_capacity";
7789
+ return claimedResult({
7790
+ status: "waiting_capacity",
7791
+ capacityWait: {
7792
+ waiterId: armed.waiter.id,
7793
+ generation: armed.waiter.generation,
7794
+ nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
7795
+ wakeRevision: armed.waiter.wakeRevision
7653
7796
  }
7654
7797
  });
7655
- if (armed.action === "waiting") {
7656
- await publishDurableSessionEvents(
7657
- bus,
7658
- input.workspaceId,
7659
- input.sessionId,
7660
- armed.events
7661
- );
7662
- turnMetricOutcome = "failed";
7663
- activityStatus = "idle";
7664
- return claimedResult({
7665
- status: "idle",
7666
- capacityWait: {
7667
- waiterId: armed.waiter.id,
7668
- generation: armed.waiter.generation,
7669
- nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
7670
- wakeRevision: armed.waiter.wakeRevision
7671
- }
7672
- });
7673
- }
7674
7798
  }
7675
7799
  if (!await settle({
7676
7800
  events: [
@@ -7736,43 +7860,41 @@ function createRunAgentTurnActivity(services) {
7736
7860
  "all connected Codex subscriptions are rate-limited",
7737
7861
  { allAccounts: true }
7738
7862
  );
7739
- if (goalActive && goal) {
7740
- const authoritativeResetAt = authoritativeCodexCapacityResetAt(
7741
- leased.accounts,
7742
- /* @__PURE__ */ new Date()
7863
+ const authoritativeResetAt = authoritativeCodexCapacityResetAt(
7864
+ leased.accounts,
7865
+ /* @__PURE__ */ new Date()
7866
+ );
7867
+ const armed = await armCodexCapacityWait(db, {
7868
+ accountId: input.accountId,
7869
+ workspaceId: input.workspaceId,
7870
+ sessionId: input.sessionId,
7871
+ turnId,
7872
+ attemptId: input.attemptId,
7873
+ workflowId: input.workflowId,
7874
+ goalId: goalActive && goal ? goal.id : null,
7875
+ goalVersion: goalActive && goal ? goal.version : null,
7876
+ earliestResetAt: authoritativeResetAt,
7877
+ resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
7878
+ failurePayload
7879
+ });
7880
+ if (armed.action === "waiting") {
7881
+ await publishDurableSessionEvents(
7882
+ bus,
7883
+ input.workspaceId,
7884
+ input.sessionId,
7885
+ armed.events
7743
7886
  );
7744
- const armed = await armCodexCapacityWait(db, {
7745
- accountId: input.accountId,
7746
- workspaceId: input.workspaceId,
7747
- sessionId: input.sessionId,
7748
- turnId,
7749
- attemptId: input.attemptId,
7750
- workflowId: input.workflowId,
7751
- goalId: goal.id,
7752
- goalVersion: goal.version,
7753
- earliestResetAt: authoritativeResetAt,
7754
- resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
7755
- failurePayload
7887
+ turnMetricOutcome = "recovering";
7888
+ activityStatus = "waiting_capacity";
7889
+ return claimedResult({
7890
+ status: "waiting_capacity",
7891
+ capacityWait: {
7892
+ waiterId: armed.waiter.id,
7893
+ generation: armed.waiter.generation,
7894
+ nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
7895
+ wakeRevision: armed.waiter.wakeRevision
7896
+ }
7756
7897
  });
7757
- if (armed.action === "waiting") {
7758
- await publishDurableSessionEvents(
7759
- bus,
7760
- input.workspaceId,
7761
- input.sessionId,
7762
- armed.events
7763
- );
7764
- turnMetricOutcome = "failed";
7765
- activityStatus = "idle";
7766
- return claimedResult({
7767
- status: "idle",
7768
- capacityWait: {
7769
- waiterId: armed.waiter.id,
7770
- generation: armed.waiter.generation,
7771
- nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
7772
- wakeRevision: armed.waiter.wakeRevision
7773
- }
7774
- });
7775
- }
7776
7898
  }
7777
7899
  if (!await settle({
7778
7900
  events: [
@@ -7881,17 +8003,57 @@ function createRunAgentTurnActivity(services) {
7881
8003
  openaiReasoningEffort: turn.reasoningEffort,
7882
8004
  sandboxBackend: turn.sandboxBackend
7883
8005
  };
8006
+ const sessionMcpHeaderNames = /* @__PURE__ */ new Map();
7884
8007
  const runSettings = await settingsWithSessionMcpServersForRun(
7885
8008
  db,
7886
8009
  input.workspaceId,
7887
8010
  input.sessionId,
7888
8011
  input.attemptId,
7889
- baseRunSettings
8012
+ baseRunSettings,
8013
+ {
8014
+ onResolvedServers: (servers) => {
8015
+ for (const server of servers) {
8016
+ sessionMcpHeaderNames.set(server.id, server.headerNames);
8017
+ }
8018
+ }
8019
+ }
7890
8020
  );
8021
+ registerSecretRedactions([
8022
+ ...runSettings.accessKey ? [{ name: "OPENGENI_ACCESS_KEY", value: runSettings.accessKey }] : [],
8023
+ ...runSettings.mcpServers.flatMap((server) => {
8024
+ return headerSecretRedactions(
8025
+ `MCP_${server.id.toUpperCase()}_STATIC`,
8026
+ server.headers,
8027
+ sessionMcpHeaderNames.get(server.id)
8028
+ );
8029
+ })
8030
+ ]);
7891
8031
  const resolvedModel = runtime.resolveTurnModel(
7892
8032
  capabilitySettings,
7893
8033
  turnExecutionPolicy.productModelId
7894
8034
  );
8035
+ const selectedProvider = resolvedModel?.provider;
8036
+ const publicProviderHeaders = new Set(
8037
+ selectedProvider?.publicDefaultHeaderNames?.map((name) => name.toLowerCase()) ?? []
8038
+ );
8039
+ const publicProviderQueries = new Set(
8040
+ selectedProvider?.publicDefaultQueryNames?.map((name) => name.toLowerCase()) ?? []
8041
+ );
8042
+ const fallbackProviderApiKey = capabilitySettings.openaiProvider === "azure" ? capabilitySettings.azureOpenaiApiKey ?? capabilitySettings.azureOpenaiAdToken : capabilitySettings.openaiApiKey;
8043
+ registerSecretRedactions([
8044
+ ...selectedProvider?.apiKey ? [
8045
+ {
8046
+ name: `MODEL_${selectedProvider.id.toUpperCase()}_API_KEY`,
8047
+ value: selectedProvider.apiKey
8048
+ }
8049
+ ] : !resolvedModel && fallbackProviderApiKey ? [{ name: "MODEL_PROVIDER_API_KEY", value: fallbackProviderApiKey }] : [],
8050
+ ...Object.entries(selectedProvider?.defaultHeaders ?? {}).flatMap(
8051
+ ([name, value]) => publicProviderHeaders.has(name.toLowerCase()) ? [] : [{ name: `MODEL_PROVIDER_HEADER_${name.toUpperCase()}`, value }]
8052
+ ),
8053
+ ...Object.entries(selectedProvider?.defaultQuery ?? {}).flatMap(
8054
+ ([name, value]) => publicProviderQueries.has(name.toLowerCase()) ? [] : [{ name: `MODEL_PROVIDER_QUERY_${name.toUpperCase()}`, value }]
8055
+ )
8056
+ ]);
7895
8057
  modelRunSettings = resolvedModel ? settingsWithResolvedModelContext(runSettings, resolvedModel.configured) : runSettings;
7896
8058
  {
7897
8059
  const workspaceModelPolicy = await getWorkspaceModelPolicy2(db, input.workspaceId);
@@ -7917,6 +8079,13 @@ function createRunAgentTurnActivity(services) {
7917
8079
  input.workspaceId,
7918
8080
  effectiveCodexCredentialId ?? ""
7919
8081
  );
8082
+ const registerSnapshot = async (snapshotPromise) => {
8083
+ const snapshot = await snapshotPromise;
8084
+ registerSecretRedactions([
8085
+ { name: "CODEX_ACCESS_TOKEN", value: snapshot.accessToken }
8086
+ ]);
8087
+ return snapshot;
8088
+ };
7920
8089
  return {
7921
8090
  clientVersion: CODEX_CLIENT_VERSION,
7922
8091
  // Backend sticky cache-routing key — the SAME id as the body's
@@ -7926,11 +8095,11 @@ function createRunAgentTurnActivity(services) {
7926
8095
  // (per-request shard lottery = prod's measured 48.6% on sol);
7927
8096
  // with it, resends pin to the warm shard (Codex CLI parity).
7928
8097
  sessionId: input.sessionId,
7929
- getToken: resolver.getToken,
7930
- refresh: resolver.refresh,
8098
+ getToken: () => registerSnapshot(resolver.getToken()),
8099
+ refresh: () => registerSnapshot(resolver.refresh()),
7931
8100
  resolveModel: buildModelResolver(
7932
- CODEX_FALLBACK_MODEL_SLUGS2,
7933
- CODEX_FALLBACK_MODEL_SLUGS2[0]
8101
+ CODEX_FALLBACK_MODEL_SLUGS,
8102
+ CODEX_FALLBACK_MODEL_SLUGS[0]
7934
8103
  ),
7935
8104
  onUsageHeaders: (snapshot) => {
7936
8105
  latestCodexUsage = snapshot;
@@ -8057,7 +8226,7 @@ function createRunAgentTurnActivity(services) {
8057
8226
  } catch (error) {
8058
8227
  if (shouldRecoverCompactionProviderFailure(error)) throw error;
8059
8228
  if (!isCompactionSummaryFailure(error)) throw error;
8060
- const errorMessage = compactionFailureReasonFromError(error);
8229
+ const errorMessage = String(redact(compactionFailureReasonFromError(error)));
8061
8230
  if (!await settle({
8062
8231
  events: [
8063
8232
  {
@@ -8121,7 +8290,7 @@ function createRunAgentTurnActivity(services) {
8121
8290
  return claimedResult({ status: "idle" });
8122
8291
  }
8123
8292
  const turnResources = mergeResourceRefs(session.resources, turn.resources);
8124
- const effectivePolicyTools = resolveSessionToolPolicy({
8293
+ const resolvedToolPolicy = resolveSessionToolPolicy({
8125
8294
  ...session.toolPolicy ? { toolPolicy: session.toolPolicy } : {},
8126
8295
  sessionTools: session.tools,
8127
8296
  turnTools: turn.tools,
@@ -8130,7 +8299,11 @@ function createRunAgentTurnActivity(services) {
8130
8299
  defaultMcpServerIds: enabledCapabilityMcpToolRefs(settings, mcpSettings).map(
8131
8300
  (tool) => tool.id
8132
8301
  )
8133
- }).toolRefs;
8302
+ });
8303
+ const effectivePolicyTools = resolvedToolPolicy.toolRefs;
8304
+ defaultNativeToolsAllowed = sessionToolPolicyAllowsDefaultNativeTools(
8305
+ resolvedToolPolicy.effectivePolicy
8306
+ );
8134
8307
  const turnTools = withCodexAppsTool(
8135
8308
  runSettings,
8136
8309
  withFirstPartyTools(runSettings, effectivePolicyTools)
@@ -8298,6 +8471,17 @@ function createRunAgentTurnActivity(services) {
8298
8471
  cancellationSignal,
8299
8472
  void 0
8300
8473
  );
8474
+ registerSecretRedactions([
8475
+ ...Object.entries(sandboxGitTokens ?? {}).flatMap(
8476
+ ([provider, value]) => value ? [{ name: `GIT_${provider.toUpperCase()}_TOKEN`, value }] : []
8477
+ ),
8478
+ ...sandboxGitToken ? [{ name: "GIT_TOKEN", value: sandboxGitToken }] : [],
8479
+ ...(sandboxGitCredentialBindings ?? []).map((binding) => ({
8480
+ name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
8481
+ value: binding.token
8482
+ })),
8483
+ ...sandboxToolspaceToken ? [{ name: "TOOLSPACE_TOKEN", value: sandboxToolspaceToken }] : []
8484
+ ]);
8301
8485
  const sandboxToolspaceTokenFile = sandboxToolspaceToken ? toolspaceTokenFileFromEnvironment(sandboxEnvironment, input.sessionId) : void 0;
8302
8486
  const initialGitCredentials = sandboxGitCredentialBindings ? {
8303
8487
  bindings: sandboxGitCredentialBindings,
@@ -8330,6 +8514,12 @@ function createRunAgentTurnActivity(services) {
8330
8514
  );
8331
8515
  if (binding) {
8332
8516
  assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
8517
+ registerSecretRedactions([
8518
+ {
8519
+ name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
8520
+ value: binding.token
8521
+ }
8522
+ ]);
8333
8523
  }
8334
8524
  pendingBinding = binding;
8335
8525
  return binding ? {
@@ -8399,7 +8589,17 @@ function createRunAgentTurnActivity(services) {
8399
8589
  toolspaceTokenRenewal = null;
8400
8590
  await previous?.stop();
8401
8591
  if (toolspaceTokenRenewalClosed) return;
8402
- const mint = async () => await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority);
8592
+ const mint = async () => {
8593
+ const material = await mintSandboxToolspaceToken(
8594
+ runSettings,
8595
+ connectionScope,
8596
+ toolspaceAuthority
8597
+ );
8598
+ if (material) {
8599
+ registerSecretRedactions([{ name: "TOOLSPACE_TOKEN", value: material.token }]);
8600
+ }
8601
+ return material;
8602
+ };
8403
8603
  const write = async (material) => {
8404
8604
  const runAs = sandboxRunAs(runSettings);
8405
8605
  const targetSandbox = resolvedSandbox ?? initialSandbox;
@@ -8571,8 +8771,13 @@ function createRunAgentTurnActivity(services) {
8571
8771
  runCredentialRenewal = controller;
8572
8772
  };
8573
8773
  if (settings.sandboxOwnershipEnabled && turn.sandboxBackend !== "none") {
8574
- sandboxHolderId = sandboxLeaseHolderIdForAttempt(input.attemptId);
8575
- sandboxGroupId = session.sandboxGroupId;
8774
+ const managedOwnership = managedSandboxOwnershipForTurn(
8775
+ machinePrimary,
8776
+ input.attemptId,
8777
+ session.sandboxGroupId
8778
+ );
8779
+ sandboxHolderId = managedOwnership?.holderId ?? null;
8780
+ sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
8576
8781
  if (machinePrimary) {
8577
8782
  const machineOpStream = settings.agentOpStreamEnabled === true ? (await getEnrollment(db, input.workspaceId, activeSandboxRecord.enrollmentId))?.opStream === true : false;
8578
8783
  const established = await establishSelfhostedTurnSession(
@@ -8593,21 +8798,6 @@ function createRunAgentTurnActivity(services) {
8593
8798
  }
8594
8799
  );
8595
8800
  machinePrimarySession = established.session;
8596
- const lease = await waitForTurnOperation(
8597
- acquireSelfhostedLeaseForTurn(
8598
- { db, settings },
8599
- {
8600
- accountId: input.accountId,
8601
- workspaceId: input.workspaceId,
8602
- sandboxGroupId: session.sandboxGroupId,
8603
- sessionId: input.sessionId
8604
- },
8605
- "turn",
8606
- sandboxHolderId
8607
- ),
8608
- cancellationSignal,
8609
- async (lateLease) => await lateLease.release()
8610
- );
8611
8801
  setupBoxSession = established.session;
8612
8802
  resolvedSandbox = {
8613
8803
  // Wrap in the SAME routing proxy so a mid-turn swap (to another machine
@@ -8649,8 +8839,11 @@ function createRunAgentTurnActivity(services) {
8649
8839
  },
8650
8840
  established
8651
8841
  ),
8652
- leaseEpoch: lease.leaseEpoch,
8653
- release: lease.release
8842
+ // For a machine route this is the active-pointer epoch, not a cloud
8843
+ // lease epoch. Cloud-only heartbeat/snapshot/capture paths require
8844
+ // sandboxGroupId and remain disabled for this branch.
8845
+ leaseEpoch: activeSandboxPointer.activeEpoch,
8846
+ release: async () => void 0
8654
8847
  };
8655
8848
  } else if (establishPolicy === "on-demand") {
8656
8849
  } else {
@@ -8679,7 +8872,7 @@ function createRunAgentTurnActivity(services) {
8679
8872
  // running a DIFFERENT image (solo → recreate on the new image; N-holders →
8680
8873
  // SandboxImageConflictError surfaced as an actionable turn error). Prefer the
8681
8874
  // explicit Modal image ref, else the docker image. The selfhosted branch
8682
- // (establishSelfhostedTurnSession/acquireSelfhostedLeaseForTurn) NEVER passes
8875
+ // (establishSelfhostedTurnSession) NEVER passes
8683
8876
  // an image — B3 lives only on this Modal else-branch.
8684
8877
  ...runSettings.modalImageRef ?? runSettings.dockerImage ? {
8685
8878
  image: runSettings.modalImageRef ?? runSettings.dockerImage
@@ -8691,7 +8884,7 @@ function createRunAgentTurnActivity(services) {
8691
8884
  ...rigVersion ? { rigVersionId: rigVersion.id } : {}
8692
8885
  },
8693
8886
  "turn",
8694
- sandboxHolderId
8887
+ managedOwnership.holderId
8695
8888
  ),
8696
8889
  cancellationSignal,
8697
8890
  async (lateSandbox) => await lateSandbox.release()
@@ -8752,7 +8945,7 @@ function createRunAgentTurnActivity(services) {
8752
8945
  throwIfWorkerShuttingDown();
8753
8946
  throwIfTurnCancelled();
8754
8947
  const mcpCredentialRootSessionId = connectionCredentials?.mcpCredentials && hostCredentialRootSessionId ? hostCredentialRootSessionId : input.sessionId;
8755
- const resolveCredential = connectionTokenResolverForTurn({
8948
+ const rawResolveCredential = connectionTokenResolverForTurn({
8756
8949
  db,
8757
8950
  settings: runSettings,
8758
8951
  connectionCredentials: connectionCredentials ?? null,
@@ -8763,6 +8956,15 @@ function createRunAgentTurnActivity(services) {
8763
8956
  attemptId: input.attemptId,
8764
8957
  turn
8765
8958
  });
8959
+ const resolveCredential = async (request) => {
8960
+ const result = await rawResolveCredential(request);
8961
+ if (result.status === "ok") {
8962
+ registerSecretRedactions(
8963
+ headerSecretRedactions("MCP", result.headers, Object.keys(result.headers))
8964
+ );
8965
+ }
8966
+ return result;
8967
+ };
8766
8968
  preparedTools = await waitForTurnOperation(
8767
8969
  withCodex(
8768
8970
  () => runtime.prepareTools(runSettings, turnTools, {
@@ -8797,6 +8999,11 @@ function createRunAgentTurnActivity(services) {
8797
8999
  variableNames: Object.keys(workspaceVariableSet.values)
8798
9000
  }
8799
9001
  } : {};
9002
+ const hostedWebSearch = hostedWebSearchForTurn(
9003
+ resolvedModel,
9004
+ runSettings.webSearchEnabled,
9005
+ defaultNativeToolsAllowed
9006
+ );
8800
9007
  const agent = runtime.buildAgent(modelRunSettings, turnResources, {
8801
9008
  reasoningEffort: turn.reasoningEffort,
8802
9009
  ...humanInputResume ? { humanInputResponse: humanInputResume } : {},
@@ -8848,10 +9055,11 @@ function createRunAgentTurnActivity(services) {
8848
9055
  // store/compaction follow the provider's compaction mode (registry
8849
9056
  // providers resolve to "client"); encrypted reasoning is only
8850
9057
  // round-tripped on the Responses wire API; hosted web search is attached
8851
- // only when the model opts in; the effective context window drives the
8852
- // compaction threshold.
9058
+ // only when BOTH the model opts in and this immutable turn still tracks
9059
+ // workspace defaults (an explicit session/turn remains narrowed); the
9060
+ // effective context window drives the compaction threshold.
9061
+ hostedWebSearch,
8853
9062
  ...resolvedModel ? {
8854
- hostedWebSearch: resolvedModel.configured.hostedWebSearch,
8855
9063
  encryptedReasoning: resolvedModel.provider.api === "responses" && runSettings.openaiReasoningEncryptedContent,
8856
9064
  contextWindowTokens: resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
8857
9065
  // The ChatGPT/Codex backend rejects the SDK's HOSTED sandbox tools —
@@ -9147,7 +9355,7 @@ function createRunAgentTurnActivity(services) {
9147
9355
  } catch (compactError) {
9148
9356
  if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
9149
9357
  if (!isCompactionSummaryFailure(compactError)) throw compactError;
9150
- const errorMessage = compactionFailureReasonFromError(compactError);
9358
+ const errorMessage = String(redact(compactionFailureReasonFromError(compactError)));
9151
9359
  observability.error("context compaction failed", {
9152
9360
  sessionId: input.sessionId,
9153
9361
  turnId,
@@ -9409,6 +9617,7 @@ function createRunAgentTurnActivity(services) {
9409
9617
  } : {},
9410
9618
  contextCompactionSignal: () => modelResponseUsageContextSignal(modelResponseUsageState),
9411
9619
  contextCompactionRequested: () => isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
9620
+ callModelInputFilter: secretRedactionModelInputFilter((value) => redact(value)),
9412
9621
  ...toolCancellationFenceRef.current ? { turnToolCancellationFence: toolCancellationFenceRef.current } : {}
9413
9622
  });
9414
9623
  };
@@ -9481,7 +9690,7 @@ function createRunAgentTurnActivity(services) {
9481
9690
  } catch (limitError) {
9482
9691
  let serializedRunState = null;
9483
9692
  try {
9484
- serializedRunState = stream.state.toString();
9693
+ serializedRunState = String(redact(stream.state.toString()));
9485
9694
  } catch {
9486
9695
  serializedRunState = null;
9487
9696
  }
@@ -9501,7 +9710,9 @@ function createRunAgentTurnActivity(services) {
9501
9710
  executionGeneration,
9502
9711
  attemptId: input.attemptId,
9503
9712
  modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
9504
- ...pendingToolCall
9713
+ callId: pendingToolCall.callId,
9714
+ callType: pendingToolCall.callType,
9715
+ callItem: redact(pendingToolCall.callItem)
9505
9716
  });
9506
9717
  if (!registered.accepted) {
9507
9718
  throw new TurnAttemptFencedError(
@@ -9521,7 +9732,7 @@ function createRunAgentTurnActivity(services) {
9521
9732
  attemptId: input.attemptId,
9522
9733
  callId: completedToolCall.callId,
9523
9734
  modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
9524
- resultItem: completedToolCall.resultItem
9735
+ resultItem: redact(completedToolCall.resultItem)
9525
9736
  });
9526
9737
  if (!recorded.accepted) {
9527
9738
  throw new TurnAttemptFencedError(
@@ -9713,7 +9924,7 @@ function createRunAgentTurnActivity(services) {
9713
9924
  activityStatus = "requires_action";
9714
9925
  return claimedResult({ status: "requires_action" });
9715
9926
  }
9716
- const finalOutput = String(stream.finalOutput ?? "");
9927
+ const finalOutput = String(redact(String(stream.finalOutput ?? "")));
9717
9928
  await reconcileConversationTruth();
9718
9929
  if (machinePrimarySession) {
9719
9930
  try {
@@ -9778,8 +9989,8 @@ function createRunAgentTurnActivity(services) {
9778
9989
  sessionId: input.sessionId,
9779
9990
  turnId: activeTurnId,
9780
9991
  reason: recoveryKind,
9781
- code: overflow?.code,
9782
- error: overflow?.message ?? compactionNeeded?.message,
9992
+ code: overflow?.code ? String(redact(overflow.code)) : void 0,
9993
+ error: String(redact(overflow?.message ?? compactionNeeded?.message ?? "")),
9783
9994
  signalTokens: compactionNeeded?.signalTokens,
9784
9995
  thresholdTokens: compactionNeeded?.thresholdTokens
9785
9996
  });
@@ -9803,7 +10014,9 @@ function createRunAgentTurnActivity(services) {
9803
10014
  } catch (compactError) {
9804
10015
  if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
9805
10016
  if (!isCompactionSummaryFailure(compactError)) throw compactError;
9806
- compactionFailureMessage = compactionFailureReasonFromError(compactError);
10017
+ compactionFailureMessage = String(
10018
+ redact(compactionFailureReasonFromError(compactError))
10019
+ );
9807
10020
  observability.warn("context compaction recovery compaction failed", {
9808
10021
  sessionId: input.sessionId,
9809
10022
  turnId: activeTurnId,
@@ -9878,7 +10091,10 @@ function createRunAgentTurnActivity(services) {
9878
10091
  turnMetricOutcome = "recovering";
9879
10092
  return claimedResult({ status: "recovering" });
9880
10093
  } catch (recoveryError) {
9881
- console.error("sandbox lease supersession recovery failed", recoveryError);
10094
+ console.error(
10095
+ "sandbox lease supersession recovery failed",
10096
+ safeErrorDiagnostic(recoveryError, (value) => String(redact(value)))
10097
+ );
9882
10098
  throw recoveryError;
9883
10099
  }
9884
10100
  }
@@ -9910,7 +10126,10 @@ function createRunAgentTurnActivity(services) {
9910
10126
  turnMetricOutcome = "recovering";
9911
10127
  return claimedResult({ status: "recovering" });
9912
10128
  } catch (recoveryError) {
9913
- console.error("worker-shutdown recovery checkpoint failed", recoveryError);
10129
+ console.error(
10130
+ "worker-shutdown recovery checkpoint failed",
10131
+ safeErrorDiagnostic(recoveryError, (value) => String(redact(value)))
10132
+ );
9914
10133
  throw recoveryError;
9915
10134
  }
9916
10135
  }
@@ -10336,7 +10555,7 @@ function createRunAgentTurnActivity(services) {
10336
10555
  usageLimit,
10337
10556
  error instanceof Error ? error.message : String(error)
10338
10557
  );
10339
- if (goalActive && goal && rotationResumeMs === null) {
10558
+ if (rotationResumeMs === null) {
10340
10559
  const providerResetAt = capacityAuthoritativeResetAt ?? (usageLimit.resetsInSeconds !== null && Number.isFinite(usageLimit.resetsInSeconds) && usageLimit.resetsInSeconds > 0 ? new Date(Date.now() + Math.ceil(usageLimit.resetsInSeconds) * 1e3) : null);
10341
10560
  const armed = await armCodexCapacityWait(db, {
10342
10561
  accountId: input.accountId,
@@ -10345,8 +10564,8 @@ function createRunAgentTurnActivity(services) {
10345
10564
  turnId,
10346
10565
  attemptId: input.attemptId,
10347
10566
  workflowId: input.workflowId,
10348
- goalId: goal.id,
10349
- goalVersion: goal.version,
10567
+ goalId: goalActive && goal ? goal.id : null,
10568
+ goalVersion: goalActive && goal ? goal.version : null,
10350
10569
  earliestResetAt: providerResetAt,
10351
10570
  resetKind: providerResetAt ? "authoritative" : "bounded_refresh",
10352
10571
  failurePayload,
@@ -10365,11 +10584,11 @@ function createRunAgentTurnActivity(services) {
10365
10584
  input.sessionId,
10366
10585
  armed.events
10367
10586
  );
10368
- turnMetricOutcome = "failed";
10369
- activityStatus = "idle";
10587
+ turnMetricOutcome = "recovering";
10588
+ activityStatus = "waiting_capacity";
10370
10589
  activityError = error;
10371
10590
  return claimedResult({
10372
- status: "idle",
10591
+ status: "waiting_capacity",
10373
10592
  capacityWait: {
10374
10593
  waiterId: armed.waiter.id,
10375
10594
  generation: armed.waiter.generation,
@@ -10460,7 +10679,7 @@ function createRunAgentTurnActivity(services) {
10460
10679
  activityStatus = "idle";
10461
10680
  return claimedResult({ status: "idle" });
10462
10681
  }
10463
- const failure = agentRunFailurePayload(error, { isCodexTurn });
10682
+ const failure = redact(agentRunFailurePayload(error, { isCodexTurn }));
10464
10683
  if (isSessionEventPersistenceError(error)) {
10465
10684
  observability.error("session event persistence failed", {
10466
10685
  accountId: input.accountId,
@@ -10570,8 +10789,12 @@ function createRunAgentTurnActivity(services) {
10570
10789
  );
10571
10790
  }
10572
10791
  await drainAttemptOwnedSandboxWriters({
10573
- toolCancellationFence: acknowledgeQuiescence ? toolCancellationFence : null,
10574
- cancellationReason: cancellationSignal?.reason,
10792
+ // Normal turn completion owns the same process boundary as
10793
+ // Pause/Steer: yielded provider shells must be terminated, polled,
10794
+ // and durably settled before workspace capture. Only receipt
10795
+ // publication remains conditional on acknowledgeQuiescence.
10796
+ toolCancellationFence,
10797
+ cancellationReason: cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
10575
10798
  gitCredentialRenewals: gitRenewalsToStop,
10576
10799
  toolspaceTokenRenewal: toolspaceRenewalToStop,
10577
10800
  runCredentialRenewal: runRenewalToStop
@@ -10619,14 +10842,20 @@ function createRunAgentTurnActivity(services) {
10619
10842
  });
10620
10843
  },
10621
10844
  onReceiptFailure: (error) => {
10622
- console.error("agent turn quiescence receipt exhausted; signalling proof", error);
10845
+ console.error(
10846
+ "agent turn quiescence receipt exhausted; signalling proof",
10847
+ safeErrorDiagnostic(error, (value) => String(redact(value)))
10848
+ );
10623
10849
  },
10624
10850
  onPublishFailure: (error) => {
10625
- console.error("agent turn quiescence event fanout failed", error);
10851
+ console.error(
10852
+ "agent turn quiescence event fanout failed",
10853
+ safeErrorDiagnostic(error, (value) => String(redact(value)))
10854
+ );
10626
10855
  },
10627
10856
  onSignalFailure: (error, attempt, retryMs) => {
10628
10857
  console.error("agent turn quiescence proof signal failed; retrying", {
10629
- error,
10858
+ error: safeErrorDiagnostic(error, (value) => String(redact(value))),
10630
10859
  attempt,
10631
10860
  retryMs
10632
10861
  });
@@ -10812,14 +11041,20 @@ function createRunAgentTurnActivity(services) {
10812
11041
  resolvedSandbox = null;
10813
11042
  await waitForTurnFinalizerStep(
10814
11043
  sandboxToRelease.release().catch((releaseError) => {
10815
- console.error("sandbox lease release failed (turn outcome unaffected)", releaseError);
11044
+ console.error(
11045
+ "sandbox lease release failed (turn outcome unaffected)",
11046
+ safeErrorDiagnostic(releaseError, (value) => String(redact(value)))
11047
+ );
10816
11048
  }),
10817
11049
  finalizerSignal
10818
11050
  );
10819
11051
  }
10820
11052
  } catch (error) {
10821
11053
  finalizationError ??= error;
10822
- console.error("agent turn finalization failed (turn outcome unaffected)", error);
11054
+ console.error(
11055
+ "agent turn finalization failed (turn outcome unaffected)",
11056
+ safeErrorDiagnostic(error, (value) => String(redact(value)))
11057
+ );
10823
11058
  } finally {
10824
11059
  cancellationSignal?.removeEventListener("abort", noteCancellationRequested);
10825
11060
  const completedAt = performance.now();
@@ -10868,7 +11103,10 @@ function createRunAgentTurnActivity(services) {
10868
11103
  "opengeni.duration_ms": Math.round(durationSeconds * 1e3),
10869
11104
  "opengeni.finalization_duration_ms": Math.round(finalizationDurationSeconds * 1e3)
10870
11105
  },
10871
- error: finalizationError ?? activityError
11106
+ error: finalizationError || activityError ? safeErrorForTelemetry(
11107
+ finalizationError ?? activityError,
11108
+ (value) => String(redact(value))
11109
+ ) : void 0
10872
11110
  });
10873
11111
  assertPhysicalToolQuiescenceForCancellation({
10874
11112
  acknowledgeQuiescence,
@@ -10907,6 +11145,10 @@ async function assertGitHubTokenMintSelectionAuthorized(db, workspaceId, install
10907
11145
  );
10908
11146
  }
10909
11147
  }
11148
+ var STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE = "unable to connect. is the computer able to access the url?";
11149
+ function isExactStatuslessUpstreamConnectivityMessage(message) {
11150
+ return message.trim().toLowerCase() === STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE;
11151
+ }
10910
11152
  function isTransientProviderError(error) {
10911
11153
  const status = typeof error === "object" && error !== null && "status" in error ? Number(error.status) : void 0;
10912
11154
  if (status !== void 0 && Number.isFinite(status)) {
@@ -10917,6 +11159,9 @@ function isTransientProviderError(error) {
10917
11159
  return true;
10918
11160
  }
10919
11161
  const message = error instanceof Error ? error.message : String(error);
11162
+ if (isExactStatuslessUpstreamConnectivityMessage(message)) {
11163
+ return true;
11164
+ }
10920
11165
  return /overloaded|an error occurred while processing your request|connection error|service unavailable|bad gateway|gateway timeout/i.test(
10921
11166
  message
10922
11167
  );
@@ -10984,6 +11229,13 @@ function agentRunFailurePayload(error, options = {}) {
10984
11229
  };
10985
11230
  }
10986
11231
  if (isTransientProviderError(error)) {
11232
+ if (isExactStatuslessUpstreamConnectivityMessage(message)) {
11233
+ return {
11234
+ error: "OpenGeni could not reach an upstream service. The same turn will retry after a short delay.",
11235
+ code: "upstream_connectivity_unavailable",
11236
+ retryable: true
11237
+ };
11238
+ }
10987
11239
  return { error: message, code: "provider_unavailable", retryable: true };
10988
11240
  }
10989
11241
  return { error: message };
@@ -11432,19 +11684,27 @@ import {
11432
11684
  accrueWarmSeconds as accrueWarmSeconds2,
11433
11685
  confirmDrainCold,
11434
11686
  appendSessionEventToSandboxGroup,
11687
+ claimTerminalRetainedProcesses,
11688
+ countActiveRetainedProcessesByOwnerState,
11689
+ countExpiredDrainingSandboxLeases,
11435
11690
  countQueuedTurns,
11436
11691
  countSandboxLeasesByLiveness,
11692
+ deferRetainedProcessReconciliation,
11437
11693
  forceDrainOverLimitViewerOnlyBoxes,
11438
11694
  getBillingBalance as getBillingBalance4,
11439
11695
  listCreditBalancesByAccount,
11440
11696
  listLiveModalSandboxLeaseAttributions,
11441
11697
  listMeterableWarmLeases,
11442
11698
  persistDrainSnapshot,
11699
+ recordRetainedProcessReconciliationProof,
11443
11700
  readWorkspaceArchiveCapturePreflight as readWorkspaceArchiveCapturePreflight2,
11444
11701
  readLease as readLease3,
11445
11702
  reapExpiredSessionListSnapshots,
11446
11703
  reapStaleLeaseHoldersGlobal,
11447
- rlsContextForWorkspace
11704
+ retainedProcessReconciliationProof,
11705
+ retainedProcessSettlementIdentity as retainedProcessSettlementIdentity2,
11706
+ rlsContextForWorkspace,
11707
+ settleRetainedProcess as settleRetainedProcess2
11448
11708
  } from "@opengeni/db";
11449
11709
  import { sandboxWarmRateMicrosPerSecond as sandboxWarmRateMicrosPerSecond2 } from "@opengeni/config";
11450
11710
  import {
@@ -11452,13 +11712,19 @@ import {
11452
11712
  createSandboxClientForBackend,
11453
11713
  deletePriorPersistedSnapshot as deletePriorPersistedSnapshot2,
11454
11714
  deserializeSandboxSessionStateEnvelope,
11715
+ isExecSessionLostBanner,
11455
11716
  isProviderSandboxNotFoundError as isProviderSandboxNotFoundError3,
11717
+ parseExecBannerExitCode,
11456
11718
  sweepModalOrphanSandboxes,
11457
11719
  terminateModalSandboxById
11458
11720
  } from "@opengeni/runtime";
11721
+ var RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
11722
+ var RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
11723
+ var RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5e3;
11459
11724
  function createSandboxLeaseActivities(services, options = {}) {
11460
11725
  const terminateBox = options.terminateBox ?? terminateProviderBox;
11461
11726
  const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
11727
+ const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
11462
11728
  async function reapSandboxLeases2() {
11463
11729
  const service = await services();
11464
11730
  const { db, settings, observability } = service;
@@ -11502,6 +11768,7 @@ function createSandboxLeaseActivities(services, options = {}) {
11502
11768
  metered.workspaceIds,
11503
11769
  observability
11504
11770
  );
11771
+ await reconcileTerminalRetainedProcesses(db, settings, observability, probeRetainedProcess);
11505
11772
  const drainable = await reapStaleLeaseHoldersGlobal(db, {
11506
11773
  viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
11507
11774
  // Dead-worker turn holders: a live holder is touched every 10s from the
@@ -11573,6 +11840,276 @@ function createSandboxLeaseActivities(services, options = {}) {
11573
11840
  }
11574
11841
  return { reapSandboxLeases: reapSandboxLeases2 };
11575
11842
  }
11843
+ async function reconcileTerminalRetainedProcesses(db, settings, observability, probe) {
11844
+ const claimId = crypto.randomUUID();
11845
+ let claims;
11846
+ try {
11847
+ claims = await claimTerminalRetainedProcesses(db, {
11848
+ claimId,
11849
+ limit: RETAINED_PROCESS_RECONCILIATION_LIMIT,
11850
+ claimTtlMs: RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS
11851
+ });
11852
+ } catch (error) {
11853
+ recordRetainedProcessReconciliation(observability, "claim_failed");
11854
+ observability.warn("sandbox reaper: retained-process claim failed", {
11855
+ error: error instanceof Error ? error.message : String(error)
11856
+ });
11857
+ return;
11858
+ }
11859
+ for (const claim of claims) {
11860
+ let process2 = claim.process;
11861
+ const expected = retainedProcessSettlementIdentity2(process2);
11862
+ let proof = retainedProcessReconciliationProof(process2);
11863
+ if (!proof) {
11864
+ const lease = await readLease3(db, process2.workspaceId, process2.sandboxGroupId).catch(
11865
+ () => null
11866
+ );
11867
+ if (!lease || lease.id !== process2.leaseId || lease.leaseEpoch !== process2.leaseEpoch || lease.backend !== process2.providerBackend || lease.instanceId !== process2.providerInstanceId || process2.routeTargetId !== null) {
11868
+ await deferRetainedProcessClaim(
11869
+ db,
11870
+ settings,
11871
+ observability,
11872
+ process2,
11873
+ expected,
11874
+ claim.claimId,
11875
+ "identity_mismatch"
11876
+ );
11877
+ continue;
11878
+ }
11879
+ let observation;
11880
+ try {
11881
+ observation = await probe(settings, lease, process2);
11882
+ } catch (error) {
11883
+ observability.warn("sandbox reaper: retained-process provider probe failed", {
11884
+ processId: process2.id,
11885
+ providerBackend: process2.providerBackend,
11886
+ error: error instanceof Error ? error.message : String(error)
11887
+ });
11888
+ observation = { status: "deferred", reason: "provider_error" };
11889
+ }
11890
+ if (observation.status === "deferred") {
11891
+ await deferRetainedProcessClaim(
11892
+ db,
11893
+ settings,
11894
+ observability,
11895
+ process2,
11896
+ expected,
11897
+ claim.claimId,
11898
+ observation.reason
11899
+ );
11900
+ continue;
11901
+ }
11902
+ try {
11903
+ process2 = await recordRetainedProcessReconciliationProof(db, {
11904
+ accountId: process2.accountId,
11905
+ workspaceId: process2.workspaceId,
11906
+ sessionId: process2.sessionId,
11907
+ processId: process2.id,
11908
+ expected,
11909
+ claimId: claim.claimId,
11910
+ proof: observation.proof
11911
+ });
11912
+ proof = observation.proof;
11913
+ recordRetainedProcessReconciliation(observability, `proof_${proof.outcome}`);
11914
+ } catch (error) {
11915
+ recordRetainedProcessReconciliation(observability, "proof_checkpoint_failed");
11916
+ observability.warn("sandbox reaper: retained-process proof checkpoint failed", {
11917
+ processId: process2.id,
11918
+ error: error instanceof Error ? error.message : String(error)
11919
+ });
11920
+ continue;
11921
+ }
11922
+ }
11923
+ try {
11924
+ await settleRetainedProcess2(db, {
11925
+ accountId: process2.accountId,
11926
+ workspaceId: process2.workspaceId,
11927
+ sessionId: process2.sessionId,
11928
+ processId: process2.id,
11929
+ expected,
11930
+ reconciliationClaimId: claim.claimId,
11931
+ outcome: proof.outcome,
11932
+ exitCode: proof.exitCode,
11933
+ reason: proof.reason,
11934
+ idleGraceMs: settings.sandboxIdleGraceMs
11935
+ });
11936
+ recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
11937
+ } catch (error) {
11938
+ recordRetainedProcessReconciliation(observability, "settlement_failed");
11939
+ observability.warn("sandbox reaper: retained-process settlement failed", {
11940
+ processId: process2.id,
11941
+ error: error instanceof Error ? error.message : String(error)
11942
+ });
11943
+ await deferRetainedProcessClaim(
11944
+ db,
11945
+ settings,
11946
+ observability,
11947
+ process2,
11948
+ expected,
11949
+ claim.claimId,
11950
+ "settlement_failed"
11951
+ );
11952
+ }
11953
+ }
11954
+ }
11955
+ async function deferRetainedProcessClaim(db, settings, observability, process2, expected, claimId, outcome) {
11956
+ const exponent = Math.min(4, Math.max(0, process2.reconcileAttempts - 1));
11957
+ const retryAfterMs = Math.min(
11958
+ 5 * 6e4,
11959
+ Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4) * 2 ** exponent
11960
+ );
11961
+ try {
11962
+ await deferRetainedProcessReconciliation(db, {
11963
+ accountId: process2.accountId,
11964
+ workspaceId: process2.workspaceId,
11965
+ sessionId: process2.sessionId,
11966
+ processId: process2.id,
11967
+ expected,
11968
+ claimId,
11969
+ outcome,
11970
+ retryAfterMs
11971
+ });
11972
+ recordRetainedProcessReconciliation(observability, outcome);
11973
+ } catch (error) {
11974
+ recordRetainedProcessReconciliation(observability, "defer_failed");
11975
+ observability.warn("sandbox reaper: retained-process defer failed", {
11976
+ processId: process2.id,
11977
+ error: error instanceof Error ? error.message : String(error)
11978
+ });
11979
+ }
11980
+ }
11981
+ var RETAINED_PROCESS_PROBE_TIMEOUT = Symbol("retained-process-provider-probe-timeout");
11982
+ async function withRetainedProcessProbeTimeout(promise) {
11983
+ let timer;
11984
+ try {
11985
+ return await Promise.race([
11986
+ promise,
11987
+ new Promise((_resolve, reject) => {
11988
+ timer = setTimeout(
11989
+ () => reject(RETAINED_PROCESS_PROBE_TIMEOUT),
11990
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
11991
+ );
11992
+ timer.unref?.();
11993
+ })
11994
+ ]);
11995
+ } finally {
11996
+ if (timer) clearTimeout(timer);
11997
+ }
11998
+ }
11999
+ async function probeRetainedProcessAtProvider(settings, lease, process2) {
12000
+ if (lease.id !== process2.leaseId || lease.sandboxGroupId !== process2.sandboxGroupId || lease.leaseEpoch !== process2.leaseEpoch || lease.backend !== process2.providerBackend || lease.instanceId !== process2.providerInstanceId || lease.resumeBackendId !== process2.providerBackend || process2.routeTargetId !== null) {
12001
+ return { status: "deferred", reason: "identity_mismatch" };
12002
+ }
12003
+ if (!lease.resumeState) {
12004
+ return { status: "deferred", reason: "resume_state_missing" };
12005
+ }
12006
+ const envelopeBackend = lease.resumeState.backendId;
12007
+ if (envelopeBackend !== void 0 && envelopeBackend !== process2.providerBackend) {
12008
+ return { status: "deferred", reason: "identity_mismatch" };
12009
+ }
12010
+ if (providerIdentityFromResumeState(lease.resumeState) !== process2.providerInstanceId) {
12011
+ return { status: "deferred", reason: "identity_mismatch" };
12012
+ }
12013
+ let client;
12014
+ try {
12015
+ client = createSandboxClientForBackend(process2.providerBackend, settings);
12016
+ } catch {
12017
+ return { status: "deferred", reason: "provider_error" };
12018
+ }
12019
+ if (!client || client.backendId !== process2.providerBackend || !client.resume) {
12020
+ return { status: "deferred", reason: "backend_unsupported" };
12021
+ }
12022
+ const envelopeSessionState = lease.resumeState.sessionState ?? lease.resumeState;
12023
+ let resumedState;
12024
+ try {
12025
+ resumedState = await deserializeSandboxSessionStateEnvelope(
12026
+ client,
12027
+ envelopeSessionState
12028
+ );
12029
+ } catch {
12030
+ return { status: "deferred", reason: "provider_error" };
12031
+ }
12032
+ if (resumedState === void 0) {
12033
+ return { status: "deferred", reason: "resume_state_missing" };
12034
+ }
12035
+ let session;
12036
+ try {
12037
+ session = await withRetainedProcessProbeTimeout(
12038
+ client.resume(resumedState)
12039
+ );
12040
+ } catch (error) {
12041
+ if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
12042
+ return { status: "deferred", reason: "provider_timeout" };
12043
+ }
12044
+ if (isProviderSandboxNotFoundError3(client.backendId, error)) {
12045
+ return {
12046
+ status: "proved",
12047
+ proof: {
12048
+ outcome: "lost",
12049
+ exitCode: null,
12050
+ reason: "provider_instance_not_found"
12051
+ }
12052
+ };
12053
+ }
12054
+ return { status: "deferred", reason: "provider_error" };
12055
+ }
12056
+ if (typeof session.writeStdin !== "function") {
12057
+ return { status: "deferred", reason: "backend_unsupported" };
12058
+ }
12059
+ let result;
12060
+ try {
12061
+ result = await withRetainedProcessProbeTimeout(
12062
+ session.writeStdin({
12063
+ sessionId: process2.providerSessionId,
12064
+ chars: "",
12065
+ yieldTimeMs: 1e3,
12066
+ maxOutputTokens: 2e3
12067
+ })
12068
+ );
12069
+ } catch (error) {
12070
+ if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
12071
+ return { status: "deferred", reason: "provider_timeout" };
12072
+ }
12073
+ if (isProviderSandboxNotFoundError3(client.backendId, error)) {
12074
+ return {
12075
+ status: "proved",
12076
+ proof: {
12077
+ outcome: "lost",
12078
+ exitCode: null,
12079
+ reason: "provider_instance_not_found"
12080
+ }
12081
+ };
12082
+ }
12083
+ return { status: "deferred", reason: "provider_error" };
12084
+ }
12085
+ return classifyRetainedProcessPollResult(result, process2.providerSessionId);
12086
+ }
12087
+ function classifyRetainedProcessPollResult(result, providerSessionId) {
12088
+ if (typeof result !== "string") {
12089
+ return { status: "deferred", reason: "provider_unknown" };
12090
+ }
12091
+ if (isExecSessionLostBanner(result, providerSessionId)) {
12092
+ return {
12093
+ status: "proved",
12094
+ proof: {
12095
+ outcome: "lost",
12096
+ exitCode: null,
12097
+ reason: "provider_session_lost_banner"
12098
+ }
12099
+ };
12100
+ }
12101
+ const exitCode = parseExecBannerExitCode(result);
12102
+ if (exitCode !== null) {
12103
+ return {
12104
+ status: "proved",
12105
+ proof: { outcome: "exited", exitCode, reason: "provider_exit_banner" }
12106
+ };
12107
+ }
12108
+ return {
12109
+ status: "deferred",
12110
+ reason: result.includes("Process running with session ID") ? "provider_running" : "provider_unknown"
12111
+ };
12112
+ }
11576
12113
  async function accrueWarmTick(db, settings, observability) {
11577
12114
  const workspaceIds = /* @__PURE__ */ new Set();
11578
12115
  let accrued = 0;
@@ -11626,6 +12163,26 @@ async function refreshQueueLeaseAndCreditGauges(db, observability) {
11626
12163
  error: error instanceof Error ? error.message : String(error)
11627
12164
  });
11628
12165
  }
12166
+ try {
12167
+ recordRetainedProcessInventoryGauges(
12168
+ observability,
12169
+ await countActiveRetainedProcessesByOwnerState(db)
12170
+ );
12171
+ } catch (error) {
12172
+ observability.warn("sandbox reaper: retained-process gauge refresh failed", {
12173
+ error: error instanceof Error ? error.message : String(error)
12174
+ });
12175
+ }
12176
+ try {
12177
+ recordExpiredDrainingSandboxLeaseGauges(
12178
+ observability,
12179
+ await countExpiredDrainingSandboxLeases(db)
12180
+ );
12181
+ } catch (error) {
12182
+ observability.warn("sandbox reaper: expired-draining gauge refresh failed", {
12183
+ error: error instanceof Error ? error.message : String(error)
12184
+ });
12185
+ }
11629
12186
  try {
11630
12187
  recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
11631
12188
  } catch (error) {
@@ -11954,6 +12511,16 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
11954
12511
  }
11955
12512
 
11956
12513
  // src/activities/scheduled-tasks.ts
12514
+ import {
12515
+ OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
12516
+ OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
12517
+ OPENGENI_SLACK_BOT_SESSION_METADATA_KEY
12518
+ } from "@opengeni/contracts";
12519
+ import {
12520
+ openGeniSlackBotMetadata,
12521
+ requireOpenGeniSlackBotConnection,
12522
+ scheduledSlackBotConnectionId
12523
+ } from "@opengeni/core";
11957
12524
  import {
11958
12525
  appendSessionEventsWithLockedSessionUpdate,
11959
12526
  addSessionSystemUpdateWithSourceMutation as addSessionSystemUpdateWithSourceMutation2,
@@ -11983,6 +12550,15 @@ function createScheduledTaskActivities(services) {
11983
12550
  dispatchScheduledTaskRun: async (input) => {
11984
12551
  const { settings, db, bus, wakeSessionWorkflow } = await services();
11985
12552
  const task = await requireScheduledTask(db, input.workspaceId, input.taskId);
12553
+ const slackBotConnection = task.agentConfig.slackBotConnectionId ? await requireOpenGeniSlackBotConnection(
12554
+ db,
12555
+ task.workspaceId,
12556
+ task.agentConfig.slackBotConnectionId
12557
+ ) : null;
12558
+ const slackBotMetadata = slackBotConnection ? openGeniSlackBotMetadata(slackBotConnection.metadata) : null;
12559
+ if (slackBotConnection && !slackBotMetadata) {
12560
+ throw new Error("OpenGeni Slack bot connection metadata is invalid");
12561
+ }
11986
12562
  const model = task.agentConfig.model ?? settings.openaiModel;
11987
12563
  const isCodexRun = await isCodexBilledTurn2({
11988
12564
  db,
@@ -12083,6 +12659,8 @@ function createScheduledTaskActivities(services) {
12083
12659
  frozenRigVersionId = rig.activeVersion.id;
12084
12660
  }
12085
12661
  let session;
12662
+ const taskMetadata = { ...task.agentConfig.metadata };
12663
+ delete taskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
12086
12664
  try {
12087
12665
  session = await createSession(db, {
12088
12666
  accountId: task.accountId,
@@ -12091,11 +12669,14 @@ function createScheduledTaskActivities(services) {
12091
12669
  resources: task.agentConfig.resources,
12092
12670
  tools: taskTools,
12093
12671
  metadata: {
12094
- ...task.agentConfig.metadata,
12672
+ ...taskMetadata,
12095
12673
  model,
12096
12674
  reasoningEffort,
12097
12675
  scheduledTaskId: task.id,
12098
- scheduledTaskRunId: run.id
12676
+ scheduledTaskRunId: run.id,
12677
+ ...slackBotConnection ? {
12678
+ [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: slackBotConnection.id
12679
+ } : {}
12099
12680
  },
12100
12681
  createdBy: {
12101
12682
  kind: "service",
@@ -12149,7 +12730,15 @@ function createScheduledTaskActivities(services) {
12149
12730
  scheduledTaskId: task.id,
12150
12731
  scheduledTaskRunId: run.id,
12151
12732
  // Names/ids only; never values.
12152
- ...variableSet ? { variableSetId: variableSet.id, variableSetName: variableSet.name } : {}
12733
+ ...variableSet ? { variableSetId: variableSet.id, variableSetName: variableSet.name } : {},
12734
+ ...slackBotConnection && slackBotMetadata ? {
12735
+ slackBotConnection: {
12736
+ credentialRole: OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
12737
+ credentialLabel: OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
12738
+ connectionId: slackBotConnection.id,
12739
+ slackTeamId: slackBotMetadata.slackTeamId
12740
+ }
12741
+ } : {}
12153
12742
  }
12154
12743
  },
12155
12744
  ...goal ? [
@@ -12226,6 +12815,11 @@ function createScheduledTaskActivities(services) {
12226
12815
  "scheduled task variableSet attachment does not match its reusable session"
12227
12816
  );
12228
12817
  }
12818
+ if (scheduledSlackBotConnectionId(session.metadata) !== (task.agentConfig.slackBotConnectionId ?? null)) {
12819
+ throw new Error(
12820
+ "scheduled task OpenGeni Slack bot binding does not match its reusable session"
12821
+ );
12822
+ }
12229
12823
  const reusableGoal = goalSpec && run.status === "queued" ? await upsertSessionGoal(db, {
12230
12824
  accountId: task.accountId,
12231
12825
  workspaceId: task.workspaceId,
@@ -13600,7 +14194,10 @@ var documentActivity = proxyActivities({
13600
14194
  });
13601
14195
 
13602
14196
  // src/http.ts
13603
- import { dbSql } from "@opengeni/db";
14197
+ import {
14198
+ assertRuntimeDatabasePosture,
14199
+ dbSql
14200
+ } from "@opengeni/db";
13604
14201
  function createWorkerHttpHandler(input) {
13605
14202
  const { settings, observability, checks, lifecycle } = input;
13606
14203
  const timeoutMs = input.timeoutMs ?? 2e3;
@@ -13675,8 +14272,12 @@ async function runReadinessChecks(checks, timeoutMs = 2e3) {
13675
14272
  checks: result
13676
14273
  };
13677
14274
  }
13678
- function dbReadyCheck(db) {
14275
+ function dbReadyCheck(db, posture) {
13679
14276
  return async () => {
14277
+ if (posture) {
14278
+ await assertRuntimeDatabasePosture(db, posture);
14279
+ return;
14280
+ }
13680
14281
  await db.execute(dbSql`select 1`);
13681
14282
  };
13682
14283
  }
@@ -13722,7 +14323,35 @@ import {
13722
14323
  } from "@opengeni/core";
13723
14324
 
13724
14325
  // src/concurrency.ts
13725
- var TURN_WORKER_MAX_CONCURRENT_TURNS = 16;
14326
+ function turnWorkerConcurrencyOptions(settings) {
14327
+ if (settings.turnWorkerConcurrencyMode === "fixed") {
14328
+ return {
14329
+ maxConcurrentActivityTaskExecutions: settings.turnWorkerMaxConcurrentTurns
14330
+ };
14331
+ }
14332
+ return {
14333
+ tuner: {
14334
+ tunerOptions: {
14335
+ targetCpuUsage: settings.turnWorkerTargetCpuUsage,
14336
+ targetMemoryUsage: settings.turnWorkerTargetMemoryUsage
14337
+ },
14338
+ activityTaskSlotOptions: {
14339
+ minimumSlots: 1,
14340
+ maximumSlots: settings.turnWorkerMaxConcurrentTurns,
14341
+ rampThrottle: "50ms"
14342
+ }
14343
+ }
14344
+ };
14345
+ }
14346
+ function turnWorkerConcurrencyLogFields(settings) {
14347
+ const resourceBased = settings.turnWorkerConcurrencyMode === "resource-based";
14348
+ return {
14349
+ concurrencyMode: settings.turnWorkerConcurrencyMode,
14350
+ maxConcurrentTurns: settings.turnWorkerMaxConcurrentTurns,
14351
+ targetCpuUsage: resourceBased ? settings.turnWorkerTargetCpuUsage : null,
14352
+ targetMemoryUsage: resourceBased ? settings.turnWorkerTargetMemoryUsage : null
14353
+ };
14354
+ }
13726
14355
  var CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES = 32;
13727
14356
  var CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS = 40;
13728
14357
 
@@ -13858,10 +14487,10 @@ async function createOpenGeniWorker(options) {
13858
14487
  taskQueue: options.role === "control" ? settings.temporalTaskQueue : turnTaskQueue(settings.temporalTaskQueue),
13859
14488
  ...options.role === "control" ? {
13860
14489
  ...options.workflowBundle ? { workflowBundle: options.workflowBundle } : resolveOpenGeniWorkflowDefinition(),
13861
- maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS
13862
- } : {},
14490
+ maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
14491
+ maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES
14492
+ } : turnWorkerConcurrencyOptions(settings),
13863
14493
  activities,
13864
- maxConcurrentActivityTaskExecutions: options.role === "turn" ? TURN_WORKER_MAX_CONCURRENT_TURNS : CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
13865
14494
  // Cancellation is delivered through an activity heartbeat. The SDK would
13866
14495
  // otherwise throttle a two-minute heartbeat timeout to its 60-second cap,
13867
14496
  // making Pause/Steer take roughly a minute even though runAgentTurn emits a
@@ -14155,7 +14784,7 @@ async function createOpenGeniWorkerService(options) {
14155
14784
  settings,
14156
14785
  observability,
14157
14786
  checks: {
14158
- db: dbReadyCheck(options.activityDependencies.db),
14787
+ db: dbReadyCheck(options.activityDependencies.db, options.databasePosture),
14159
14788
  nats: natsReadyCheck(options.activityDependencies.bus),
14160
14789
  temporal: temporalReadyCheck(workerBundle.connection)
14161
14790
  },
@@ -14193,7 +14822,12 @@ async function createOpenGeniWorkerService(options) {
14193
14822
  observability.info("OpenGeni worker listening", {
14194
14823
  role: options.role,
14195
14824
  temporalTaskQueue: options.role === "control" ? settings.temporalTaskQueue : turnTaskQueue(settings.temporalTaskQueue),
14196
- maxConcurrentActivityTaskExecutions: options.role === "turn" ? TURN_WORKER_MAX_CONCURRENT_TURNS : CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
14825
+ ...options.role === "turn" ? turnWorkerConcurrencyLogFields(settings) : {
14826
+ concurrencyMode: "fixed",
14827
+ maxConcurrentTurns: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
14828
+ targetCpuUsage: null,
14829
+ targetMemoryUsage: null
14830
+ },
14197
14831
  maxConcurrentWorkflowTaskExecutions: options.role === "control" ? CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS : 0,
14198
14832
  httpPort: options.http === false ? null : settings.workerHttpPort
14199
14833
  });
@@ -14240,9 +14874,19 @@ async function startWorker() {
14240
14874
  ...searchPath ? { searchPath } : {},
14241
14875
  rlsStrategy: settings.rlsStrategy
14242
14876
  });
14877
+ const databasePosture = {
14878
+ rlsStrategy: settings.rlsStrategy,
14879
+ expectedRole: settings.runtimeDatabaseRole,
14880
+ targetSchema: settings.dbSchema.trim() || "public"
14881
+ };
14243
14882
  const controlPlaneAuth = resolveNatsControlPlaneAuth2(settings);
14244
14883
  let bus;
14245
14884
  try {
14885
+ await retryStartupDependency(
14886
+ "PostgreSQL runtime posture",
14887
+ () => assertRuntimeDatabasePosture2(dbClient.db, databasePosture),
14888
+ { ...retryOptions, onRetry }
14889
+ );
14246
14890
  bus = await retryStartupDependency(
14247
14891
  "NATS",
14248
14892
  () => createNatsEventBus2(
@@ -14255,6 +14899,7 @@ async function startWorker() {
14255
14899
  await runOpenGeniWorker({
14256
14900
  role,
14257
14901
  settings,
14902
+ databasePosture,
14258
14903
  activityDependencies: {
14259
14904
  observability,
14260
14905
  db: dbClient.db,