@opengeni/worker-bundle 0.11.2 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,3 +1,13 @@
1
+ import {
2
+ OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
3
+ OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
4
+ OPENGENI_SLACK_BOT_SESSION_METADATA_KEY,
5
+ } from "@opengeni/contracts";
6
+ import {
7
+ openGeniSlackBotMetadata,
8
+ requireOpenGeniSlackBotConnection,
9
+ scheduledSlackBotConnectionId,
10
+ } from "@opengeni/core";
1
11
  import {
2
12
  appendSessionEventsWithLockedSessionUpdate,
3
13
  addSessionSystemUpdateWithSourceMutation,
@@ -41,6 +51,23 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
41
51
  ): Promise<DispatchScheduledTaskRunResult> => {
42
52
  const { settings, db, bus, wakeSessionWorkflow } = await services();
43
53
  const task = await requireScheduledTask(db, input.workspaceId, input.taskId);
54
+ // A scheduled bot selection was authorized when the task was written,
55
+ // but connection status and tenant/role binding are mutable. Revalidate
56
+ // before any session/model cost and never fall back to a personal Slack
57
+ // OAuth grant when the selected bot has been revoked or changed.
58
+ const slackBotConnection = task.agentConfig.slackBotConnectionId
59
+ ? await requireOpenGeniSlackBotConnection(
60
+ db,
61
+ task.workspaceId,
62
+ task.agentConfig.slackBotConnectionId,
63
+ )
64
+ : null;
65
+ const slackBotMetadata = slackBotConnection
66
+ ? openGeniSlackBotMetadata(slackBotConnection.metadata)
67
+ : null;
68
+ if (slackBotConnection && !slackBotMetadata) {
69
+ throw new Error("OpenGeni Slack bot connection metadata is invalid");
70
+ }
44
71
  // The scheduled task's model can be codex/<slug>; resolve it here so the
45
72
  // admission gate can skip the credit/cost gates for a codex-billed run
46
73
  // (paid by the user's ChatGPT/Codex plan). This file uses BASE settings (no
@@ -160,6 +187,8 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
160
187
  frozenRigVersionId = rig.activeVersion.id;
161
188
  }
162
189
  let session: Awaited<ReturnType<typeof createSession>>;
190
+ const taskMetadata = { ...task.agentConfig.metadata };
191
+ delete taskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
163
192
  try {
164
193
  session = await createSession(db, {
165
194
  accountId: task.accountId,
@@ -168,11 +197,16 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
168
197
  resources: task.agentConfig.resources,
169
198
  tools: taskTools,
170
199
  metadata: {
171
- ...task.agentConfig.metadata,
200
+ ...taskMetadata,
172
201
  model,
173
202
  reasoningEffort,
174
203
  scheduledTaskId: task.id,
175
204
  scheduledTaskRunId: run.id,
205
+ ...(slackBotConnection
206
+ ? {
207
+ [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: slackBotConnection.id,
208
+ }
209
+ : {}),
176
210
  },
177
211
  createdBy: {
178
212
  kind: "service",
@@ -231,6 +265,16 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
231
265
  ...(variableSet
232
266
  ? { variableSetId: variableSet.id, variableSetName: variableSet.name }
233
267
  : {}),
268
+ ...(slackBotConnection && slackBotMetadata
269
+ ? {
270
+ slackBotConnection: {
271
+ credentialRole: OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
272
+ credentialLabel: OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
273
+ connectionId: slackBotConnection.id,
274
+ slackTeamId: slackBotMetadata.slackTeamId,
275
+ },
276
+ }
277
+ : {}),
234
278
  },
235
279
  },
236
280
  ...(goal
@@ -316,6 +360,14 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
316
360
  "scheduled task variableSet attachment does not match its reusable session",
317
361
  );
318
362
  }
363
+ if (
364
+ scheduledSlackBotConnectionId(session.metadata) !==
365
+ (task.agentConfig.slackBotConnectionId ?? null)
366
+ ) {
367
+ throw new Error(
368
+ "scheduled task OpenGeni Slack bot binding does not match its reusable session",
369
+ );
370
+ }
319
371
  // A recurring "maintain X" task re-establishes its objective on every
320
372
  // fire: replace the goal text, reactivate it, and reset the counters.
321
373
  const reusableGoal =
@@ -120,8 +120,7 @@ export type ReconcileCodexCapacityWaitInput = {
120
120
 
121
121
  export type ReconcileCodexCapacityWaitResult =
122
122
  | ({ action: "waiting" } & CodexCapacityWaitRef)
123
- | { action: "resumed"; updateId: string }
124
- | { action: "superseded" | "stale" };
123
+ | { action: "resumed" | "paused" | "superseded" | "stale" };
125
124
 
126
125
  export type ActivityDependencies = Partial<ActivityServices>;
127
126
 
@@ -241,7 +240,7 @@ export type IndexDocumentInput = {
241
240
  type ClaimedRunAgentTurnResult = {
242
241
  // "recovering": this attempt ended after durably preserving the same current
243
242
  // inference for a new attempt. Recovery is not prompt queue work.
244
- status: "idle" | "requires_action" | "failed" | "cancelled" | "recovering";
243
+ status: "idle" | "requires_action" | "failed" | "cancelled" | "recovering" | "waiting_capacity";
245
244
  turnId: string;
246
245
  attemptId: string;
247
246
  // Provider backpressure pacing: when set on an idle or recovering result, the
@@ -253,9 +252,10 @@ type ClaimedRunAgentTurnResult = {
253
252
  // "continue now" (invariant 4: NO THRASH). Distinct from a normal continueDelayMs:0
254
253
  // which legitimately means "a rotation candidate is ready, re-dispatch immediately".
255
254
  idleUntilReset?: boolean;
256
- // Durable native zero-pool wait. Unlike continueDelayMs, this reference is
257
- // persisted in Postgres and reconstructed after workflow/worker restart.
258
- // The workflow must not call maybeContinueGoal while this waiter is active.
255
+ // Durable native zero-pool wait for this same nonterminal logical turn.
256
+ // Unlike continueDelayMs, this reference is persisted in Postgres and
257
+ // reconstructed after workflow/worker restart. The workflow must not call
258
+ // maybeContinueGoal or manufacture queue/input work while it is active.
259
259
  capacityWait?: CodexCapacityWaitRef;
260
260
  // This execution reached a durable terminal-for-now boundary (for example,
261
261
  // maintenance could not run or same-turn context recovery failed). End this
@@ -1,11 +1,63 @@
1
+ import type { Settings } from "@opengeni/config";
2
+ import type { WorkerOptions } from "@temporalio/worker";
3
+
4
+ export type TurnWorkerConcurrencySettings = Pick<
5
+ Settings,
6
+ | "turnWorkerConcurrencyMode"
7
+ | "turnWorkerMaxConcurrentTurns"
8
+ | "turnWorkerTargetCpuUsage"
9
+ | "turnWorkerTargetMemoryUsage"
10
+ >;
11
+
12
+ export type TurnWorkerConcurrencyOptions = Pick<
13
+ WorkerOptions,
14
+ "maxConcurrentActivityTaskExecutions" | "tuner"
15
+ >;
16
+
1
17
  /**
2
- * Process-local Temporal execution ceilings.
3
- *
4
- * Turn workers host only runAgentTurn, so this is the exact number of agent
5
- * turns one worker process may own concurrently. Keep the value centralized so
6
- * production and the real Temporal integration harness exercise one topology.
18
+ * Ordinary multi-worker deployments keep a fixed per-process ceiling so HPA or
19
+ * an operator owns horizontal capacity. The single-machine profile selects the
20
+ * resource tuner instead: Temporal admits activity slots while whole-system CPU
21
+ * and memory remain below the configured targets, then stops polling more work.
7
22
  */
8
- export const TURN_WORKER_MAX_CONCURRENT_TURNS = 16;
23
+ export function turnWorkerConcurrencyOptions(
24
+ settings: TurnWorkerConcurrencySettings,
25
+ ): TurnWorkerConcurrencyOptions {
26
+ if (settings.turnWorkerConcurrencyMode === "fixed") {
27
+ return {
28
+ maxConcurrentActivityTaskExecutions: settings.turnWorkerMaxConcurrentTurns,
29
+ };
30
+ }
31
+
32
+ return {
33
+ tuner: {
34
+ tunerOptions: {
35
+ targetCpuUsage: settings.turnWorkerTargetCpuUsage,
36
+ targetMemoryUsage: settings.turnWorkerTargetMemoryUsage,
37
+ },
38
+ activityTaskSlotOptions: {
39
+ minimumSlots: 1,
40
+ maximumSlots: settings.turnWorkerMaxConcurrentTurns,
41
+ rampThrottle: "50ms",
42
+ },
43
+ },
44
+ };
45
+ }
46
+
47
+ export function turnWorkerConcurrencyLogFields(settings: TurnWorkerConcurrencySettings): {
48
+ concurrencyMode: "fixed" | "resource-based";
49
+ maxConcurrentTurns: number;
50
+ targetCpuUsage: number | null;
51
+ targetMemoryUsage: number | null;
52
+ } {
53
+ const resourceBased = settings.turnWorkerConcurrencyMode === "resource-based";
54
+ return {
55
+ concurrencyMode: settings.turnWorkerConcurrencyMode,
56
+ maxConcurrentTurns: settings.turnWorkerMaxConcurrentTurns,
57
+ targetCpuUsage: resourceBased ? settings.turnWorkerTargetCpuUsage : null,
58
+ targetMemoryUsage: resourceBased ? settings.turnWorkerTargetMemoryUsage : null,
59
+ };
60
+ }
9
61
 
10
62
  export const CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES = 32;
11
63
  export const CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS = 40;
package/src/http.ts CHANGED
@@ -1,5 +1,10 @@
1
1
  import type { Settings } from "@opengeni/config";
2
- import { dbSql, type Database } from "@opengeni/db";
2
+ import {
3
+ assertRuntimeDatabasePosture,
4
+ dbSql,
5
+ type Database,
6
+ type RuntimeDatabasePostureOptions,
7
+ } from "@opengeni/db";
3
8
  import type { EventBus } from "@opengeni/events";
4
9
  import type { Observability } from "@opengeni/observability";
5
10
 
@@ -112,8 +117,15 @@ export async function runReadinessChecks(
112
117
  };
113
118
  }
114
119
 
115
- export function dbReadyCheck(db: Database): () => Promise<void> {
120
+ export function dbReadyCheck(
121
+ db: Database,
122
+ posture?: RuntimeDatabasePostureOptions,
123
+ ): () => Promise<void> {
116
124
  return async () => {
125
+ if (posture) {
126
+ await assertRuntimeDatabasePosture(db, posture);
127
+ return;
128
+ }
117
129
  await db.execute(dbSql`select 1`);
118
130
  };
119
131
  }
package/src/index.ts CHANGED
@@ -7,7 +7,13 @@ import {
7
7
  temporalConnectionOptions,
8
8
  type Settings,
9
9
  } from "@opengeni/config";
10
- import { createDb, markSessionWorkflowWakeDelivered, type Database } from "@opengeni/db";
10
+ import {
11
+ assertRuntimeDatabasePosture,
12
+ createDb,
13
+ markSessionWorkflowWakeDelivered,
14
+ type Database,
15
+ type RuntimeDatabasePostureOptions,
16
+ } from "@opengeni/db";
11
17
  import { createNatsEventBus, type EventBus } from "@opengeni/events";
12
18
  import {
13
19
  createObservability,
@@ -51,7 +57,8 @@ import {
51
57
  import {
52
58
  CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
53
59
  CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
54
- TURN_WORKER_MAX_CONCURRENT_TURNS,
60
+ turnWorkerConcurrencyLogFields,
61
+ turnWorkerConcurrencyOptions,
55
62
  } from "./concurrency";
56
63
  import {
57
64
  constructWithOwnedConnection,
@@ -173,13 +180,10 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
173
180
  ? { workflowBundle: options.workflowBundle }
174
181
  : resolveOpenGeniWorkflowDefinition()),
175
182
  maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
183
+ maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
176
184
  }
177
- : {}),
185
+ : turnWorkerConcurrencyOptions(settings)),
178
186
  activities,
179
- maxConcurrentActivityTaskExecutions:
180
- options.role === "turn"
181
- ? TURN_WORKER_MAX_CONCURRENT_TURNS
182
- : CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
183
187
  // Cancellation is delivered through an activity heartbeat. The SDK would
184
188
  // otherwise throttle a two-minute heartbeat timeout to its 60-second cap,
185
189
  // making Pause/Steer take roughly a minute even though runAgentTurn emits a
@@ -471,6 +475,11 @@ export async function registerSessionWorkflowWakeDispatcherSchedule(
471
475
 
472
476
  export type OpenGeniWorkerServiceOptions = Omit<WorkerOptions, "activityDependencies"> & {
473
477
  activityDependencies: ActivityDependencies & { db: Database; bus: EventBus };
478
+ /**
479
+ * Exact catalog posture required by the standalone runtime. Embedded hosts
480
+ * may omit this when they own an equivalent database isolation contract.
481
+ */
482
+ databasePosture?: RuntimeDatabasePostureOptions;
474
483
  /**
475
484
  * `role-default` registers OpenGeni's internal maintenance schedules on a
476
485
  * control worker only. These are engine maintenance schedules, not a host's
@@ -588,7 +597,7 @@ export async function createOpenGeniWorkerService(
588
597
  settings,
589
598
  observability,
590
599
  checks: {
591
- db: dbReadyCheck(options.activityDependencies.db),
600
+ db: dbReadyCheck(options.activityDependencies.db, options.databasePosture),
592
601
  nats: natsReadyCheck(options.activityDependencies.bus),
593
602
  temporal: temporalReadyCheck(workerBundle.connection),
594
603
  },
@@ -631,10 +640,14 @@ export async function createOpenGeniWorkerService(
631
640
  options.role === "control"
632
641
  ? settings.temporalTaskQueue
633
642
  : turnTaskQueue(settings.temporalTaskQueue),
634
- maxConcurrentActivityTaskExecutions:
635
- options.role === "turn"
636
- ? TURN_WORKER_MAX_CONCURRENT_TURNS
637
- : CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
643
+ ...(options.role === "turn"
644
+ ? turnWorkerConcurrencyLogFields(settings)
645
+ : {
646
+ concurrencyMode: "fixed",
647
+ maxConcurrentTurns: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
648
+ targetCpuUsage: null,
649
+ targetMemoryUsage: null,
650
+ }),
638
651
  maxConcurrentWorkflowTaskExecutions:
639
652
  options.role === "control" ? CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS : 0,
640
653
  httpPort: options.http === false ? null : settings.workerHttpPort,
@@ -693,9 +706,19 @@ export async function startWorker() {
693
706
  ...(searchPath ? { searchPath } : {}),
694
707
  rlsStrategy: settings.rlsStrategy,
695
708
  });
709
+ const databasePosture = {
710
+ rlsStrategy: settings.rlsStrategy,
711
+ expectedRole: settings.runtimeDatabaseRole,
712
+ targetSchema: settings.dbSchema.trim() || "public",
713
+ } as const;
696
714
  const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
697
715
  let bus: Awaited<ReturnType<typeof createNatsEventBus>> | undefined;
698
716
  try {
717
+ await retryStartupDependency(
718
+ "PostgreSQL runtime posture",
719
+ () => assertRuntimeDatabasePosture(dbClient.db, databasePosture),
720
+ { ...retryOptions, onRetry },
721
+ );
699
722
  bus = await retryStartupDependency(
700
723
  "NATS",
701
724
  () =>
@@ -711,6 +734,7 @@ export async function startWorker() {
711
734
  await runOpenGeniWorker({
712
735
  role,
713
736
  settings,
737
+ databasePosture,
714
738
  activityDependencies: {
715
739
  observability,
716
740
  db: dbClient.db,
@@ -19,6 +19,7 @@ export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
19
19
  const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
20
20
  const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
21
21
  const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
22
+ const retainedProcessTerminalBacklogTotals = new WeakMap<Observability, number>();
22
23
 
23
24
  // A worker process reaching one trillion tokens in one provider/cache counter is
24
25
  // already far outside an ordinary scrape lifetime. Refuse further increments
@@ -383,6 +384,142 @@ export function recordSandboxOrphansTerminated(observability: Observability, cou
383
384
  });
384
385
  }
385
386
 
387
+ const RETAINED_PROCESS_OWNER_STATES = [
388
+ "direct",
389
+ "queued",
390
+ "running",
391
+ "requires_action",
392
+ "recovering",
393
+ "waiting_capacity",
394
+ "completed",
395
+ "failed",
396
+ "cancelled",
397
+ "superseded",
398
+ "withdrawn_for_edit",
399
+ "missing",
400
+ "unknown",
401
+ ] as const;
402
+
403
+ const RETAINED_PROCESS_OWNER_STATE_SET = new Set<string>(RETAINED_PROCESS_OWNER_STATES);
404
+
405
+ export function recordRetainedProcessInventoryGauges(
406
+ observability: Observability,
407
+ counts: Array<{ ownerState: string; activeCount: number; terminalOwnerCount: number }>,
408
+ ): void {
409
+ const normalized = new Map<string, { active: number; terminal: number }>();
410
+ for (const ownerState of RETAINED_PROCESS_OWNER_STATES) {
411
+ normalized.set(ownerState, { active: 0, terminal: 0 });
412
+ }
413
+ for (const count of counts) {
414
+ const ownerState = RETAINED_PROCESS_OWNER_STATE_SET.has(count.ownerState)
415
+ ? count.ownerState
416
+ : "unknown";
417
+ const current = normalized.get(ownerState)!;
418
+ current.active += Math.max(0, count.activeCount);
419
+ current.terminal += Math.max(0, count.terminalOwnerCount);
420
+ }
421
+
422
+ let terminalTotal = 0;
423
+ for (const [ownerState, count] of normalized) {
424
+ terminalTotal += count.terminal;
425
+ observability.setGauge({
426
+ name: "opengeni_retained_processes_active",
427
+ help: "Current active retained provider processes by durable owner state.",
428
+ labels: { owner_state: ownerState },
429
+ value: count.active,
430
+ });
431
+ observability.setGauge({
432
+ name: "opengeni_retained_processes_terminal_owner_backlog",
433
+ help: "Current active retained processes whose exact owner attempt is terminal.",
434
+ labels: { owner_state: ownerState },
435
+ value: count.terminal,
436
+ });
437
+ }
438
+ const previous = retainedProcessTerminalBacklogTotals.get(observability);
439
+ if (previous !== undefined && terminalTotal > previous) {
440
+ observability.incrementCounter({
441
+ name: "opengeni_retained_process_terminal_owner_backlog_growth_total",
442
+ help: "Positive growth in terminal-owner retained-process backlog between app samples.",
443
+ amount: terminalTotal - previous,
444
+ });
445
+ }
446
+ retainedProcessTerminalBacklogTotals.set(observability, terminalTotal);
447
+ }
448
+
449
+ const EXPIRED_DRAINING_BACKENDS = [
450
+ "none",
451
+ "local",
452
+ "docker",
453
+ "modal",
454
+ "e2b",
455
+ "daytona",
456
+ "blaxel",
457
+ "runloop",
458
+ "selfhosted",
459
+ "unknown",
460
+ ] as const;
461
+ const EXPIRED_DRAINING_BACKEND_SET = new Set<string>(EXPIRED_DRAINING_BACKENDS);
462
+ const EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"] as const;
463
+
464
+ export function recordExpiredDrainingSandboxLeaseGauges(
465
+ observability: Observability,
466
+ counts: Array<{ backend: string; ageBucket: string; count: number }>,
467
+ ): void {
468
+ const normalized = new Map<string, number>();
469
+ for (const backend of EXPIRED_DRAINING_BACKENDS) {
470
+ for (const ageBucket of EXPIRED_DRAINING_AGE_BUCKETS) {
471
+ normalized.set(`${backend}:${ageBucket}`, 0);
472
+ }
473
+ }
474
+ for (const count of counts) {
475
+ const backend = EXPIRED_DRAINING_BACKEND_SET.has(count.backend) ? count.backend : "unknown";
476
+ if (!EXPIRED_DRAINING_AGE_BUCKETS.includes(count.ageBucket as never)) continue;
477
+ const key = `${backend}:${count.ageBucket}`;
478
+ normalized.set(key, (normalized.get(key) ?? 0) + Math.max(0, count.count));
479
+ }
480
+ for (const [key, value] of normalized) {
481
+ const [backend, ageBucket] = key.split(":") as [string, string];
482
+ observability.setGauge({
483
+ name: "opengeni_sandbox_leases_expired_draining",
484
+ help: "Current expired draining sandbox leases by backend and fixed age bucket.",
485
+ labels: { backend, age_bucket: ageBucket },
486
+ value,
487
+ });
488
+ }
489
+ }
490
+
491
+ export const RETAINED_PROCESS_RECONCILIATION_OUTCOMES = [
492
+ "claim_failed",
493
+ "proof_exited",
494
+ "proof_lost",
495
+ "proof_checkpoint_failed",
496
+ "settled_exited",
497
+ "settled_lost",
498
+ "settlement_failed",
499
+ "identity_mismatch",
500
+ "resume_state_missing",
501
+ "backend_unsupported",
502
+ "provider_running",
503
+ "provider_unknown",
504
+ "provider_timeout",
505
+ "provider_error",
506
+ "defer_failed",
507
+ ] as const;
508
+
509
+ export type RetainedProcessReconciliationOutcome =
510
+ (typeof RETAINED_PROCESS_RECONCILIATION_OUTCOMES)[number];
511
+
512
+ export function recordRetainedProcessReconciliation(
513
+ observability: Observability,
514
+ outcome: RetainedProcessReconciliationOutcome,
515
+ ): void {
516
+ observability.incrementCounter({
517
+ name: "opengeni_retained_process_reconciliation_total",
518
+ help: "Bounded retained-process reconciliation observations by fixed app outcome.",
519
+ labels: { outcome },
520
+ });
521
+ }
522
+
386
523
  export function recordCreditMicros(
387
524
  observability: Observability | undefined,
388
525
  kind: CreditMicrosKind,
@@ -21,6 +21,7 @@
21
21
  // Liveness between turns is the lease refcount; there is no keepalive loop.
22
22
 
23
23
  import type { Settings } from "@opengeni/config";
24
+ import { redactSensitiveText } from "@opengeni/contracts";
24
25
  import {
25
26
  acquireLease,
26
27
  beginSandboxRematerialization,
@@ -75,6 +76,13 @@ export function sandboxLeaseHolderIdForAttempt(attemptId: string): TurnSandboxLe
75
76
  return `turn-attempt:${normalized}`;
76
77
  }
77
78
 
79
+ export function isRetryableDegradedRestore(restore: {
80
+ status: string;
81
+ retryable?: boolean;
82
+ }): boolean {
83
+ return restore.status === "degraded" && restore.retryable === true;
84
+ }
85
+
78
86
  /** The minimal services surface resumeBoxForTurn needs. A subset of
79
87
  * ActivityServices so a test (and the API later) can pass a lean bag. */
80
88
  export type SandboxResumeServices = {
@@ -207,6 +215,17 @@ class SnapshotTimeoutError extends Error {
207
215
  }
208
216
  }
209
217
 
218
+ function safeSnapshotError(error: unknown): { name: string; message: string } {
219
+ const rawName = error instanceof Error ? error.name : "Error";
220
+ return {
221
+ name: /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error",
222
+ message: redactSensitiveText(error instanceof Error ? error.message : String(error)).slice(
223
+ 0,
224
+ 2_048,
225
+ ),
226
+ };
227
+ }
228
+
210
229
  export async function waitForWarmSnapshot(
211
230
  snapshot: Promise<unknown>,
212
231
  timeoutMs: number,
@@ -239,7 +258,10 @@ export async function waitForWarmSnapshot(
239
258
  } catch (error) {
240
259
  if (signal?.aborted) return false;
241
260
  if (error instanceof SnapshotTimeoutError) {
242
- console.error("mid-session workspace snapshot wait timed out (turn unaffected)", error);
261
+ console.error(
262
+ "mid-session workspace snapshot wait timed out (turn unaffected)",
263
+ safeSnapshotError(error),
264
+ );
243
265
  return false;
244
266
  }
245
267
  return true;
@@ -521,7 +543,10 @@ export async function maybePersistWarmWorkspaceSnapshot(
521
543
  } catch (error) {
522
544
  // Protection, not a dependency: a failed snapshot must never fail (or slow
523
545
  // down retrying) the turn. The next heartbeat/turn-end tick retries.
524
- console.error("mid-session workspace snapshot failed (turn unaffected)", error);
546
+ console.error(
547
+ "mid-session workspace snapshot failed (turn unaffected)",
548
+ safeSnapshotError(error),
549
+ );
525
550
  return false;
526
551
  }
527
552
  }
@@ -963,118 +988,6 @@ export async function resumeBoxForTurn(
963
988
  }
964
989
  }
965
990
 
966
- /**
967
- * Stage D machine-primary: acquire the group lease for a turn whose ACTIVE sandbox
968
- * is a connected machine (selfhosted) WITHOUT establishing a provider box. The
969
- * "box" is the user's physical machine reached over NATS; there is NOTHING to
970
- * spawn, snapshot, expose, or serialize — so we take the lease (the SAME group-keyed
971
- * refcount/idle/epoch bookkeeping resumeBoxForTurn uses) but skip every box step.
972
- *
973
- * The lease is keyed backend "selfhosted": the reaper's terminateProviderBox
974
- * short-circuits a selfhosted lease (it is the user's machine — NEVER provider-stop),
975
- * draining it to cold with no stop; and the reaper's warm-meter rate for "selfhosted"
976
- * is 0 (no cloud seconds billed for a box that does not exist). Returns the lease
977
- * epoch + an idempotent release so the turn's lease-heartbeat + `finally` release are
978
- * identical to the cloud path.
979
- *
980
- * holderId is the globally unique durable turn-attempt id. It must not be a
981
- * Temporal activity id, because activity ids are only workflow-local and
982
- * collide when sibling sessions share one sandbox group.
983
- */
984
- export async function acquireSelfhostedLeaseForTurn(
985
- services: SandboxResumeServices,
986
- ids: {
987
- accountId: string;
988
- workspaceId: string;
989
- sandboxGroupId: string;
990
- sessionId: string;
991
- },
992
- kind: "turn",
993
- holderId: TurnSandboxLeaseHolderId,
994
- ): Promise<{ leaseEpoch: number; release: () => Promise<void> }> {
995
- const { db, settings } = services;
996
- const leaseTtlMs = settings.sandboxLeaseTtlMs;
997
-
998
- let released = false;
999
- const release = async (): Promise<void> => {
1000
- if (released) {
1001
- return;
1002
- }
1003
- released = true;
1004
- await releaseLeaseHolder(db, {
1005
- accountId: ids.accountId,
1006
- workspaceId: ids.workspaceId,
1007
- sandboxGroupId: ids.sandboxGroupId,
1008
- kind,
1009
- holderId,
1010
- idleGraceMs: settings.sandboxIdleGraceMs,
1011
- });
1012
- };
1013
-
1014
- const acquired = await acquireLease(db, {
1015
- accountId: ids.accountId,
1016
- workspaceId: ids.workspaceId,
1017
- sandboxGroupId: ids.sandboxGroupId,
1018
- kind,
1019
- holderId,
1020
- subjectId: ids.sessionId,
1021
- backend: "selfhosted",
1022
- os: "linux",
1023
- leaseTtlMs,
1024
- warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
1025
- });
1026
-
1027
- // FENCED: a newer epoch re-established the group concurrently. Release our
1028
- // just-registered holder + surface the supersession (the outer turn catch
1029
- // requeues, mirroring resumeBoxForTurn).
1030
- if (acquired.role === "blocked") {
1031
- await release();
1032
- throw new SandboxLeaseRecoveryBlockedError(
1033
- ids.sandboxGroupId,
1034
- acquired.lease.leaseEpoch,
1035
- acquired.code,
1036
- acquired.lease.recovery,
1037
- );
1038
- }
1039
- if (acquired.role === "fenced") {
1040
- await release();
1041
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
1042
- }
1043
-
1044
- // SPAWNER: we won the cold->warming CAS. There is NO box to establish — commit
1045
- // warm immediately so the lease does not linger in 'warming' (no box id, no
1046
- // resume_state; selfhosted is re-addressed by enrollment via the active pointer,
1047
- // never resumed from an envelope). resume_backend_id "selfhosted" keeps the
1048
- // reaper's never-provider-stop short-circuit correct.
1049
- if (acquired.role === "spawner") {
1050
- const expectedEpoch = acquired.lease.leaseEpoch;
1051
- const committed = await commitWarmingToWarm(db, {
1052
- accountId: ids.accountId,
1053
- workspaceId: ids.workspaceId,
1054
- sandboxGroupId: ids.sandboxGroupId,
1055
- expectedEpoch,
1056
- // No provider box id; stamp a clearly non-box marker for diagnostics.
1057
- instanceId: `selfhosted:${ids.sessionId}`,
1058
- resumeBackendId: "selfhosted",
1059
- resumeState: null,
1060
- leaseTtlMs,
1061
- });
1062
- if (!committed.committed || !committed.lease) {
1063
- // A reaper reset our warming row, or a sibling re-established + bumped the
1064
- // epoch. Release our holder; surface supersession (NEVER touch any box).
1065
- await release();
1066
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1067
- }
1068
- return { leaseEpoch: committed.lease.leaseEpoch, release };
1069
- }
1070
-
1071
- // ATTACHED / REARMED: the lease is live. There is no box to resume — just carry
1072
- // the lease epoch (same-session turns are serialized, so a concurrent 'warming'
1073
- // spawner on this group is not expected; the machine session is independent of the
1074
- // lease state regardless).
1075
- return { leaseEpoch: acquired.lease.leaseEpoch, release };
1076
- }
1077
-
1078
991
  /**
1079
992
  * Poll a warming lease until the spawner commits warm. If the warming row is
1080
993
  * reset to cold (the spawner died and the reaper reset it), surface supersession
@@ -1104,6 +1017,12 @@ async function waitForWarm(
1104
1017
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
1105
1018
  }
1106
1019
  if (lease.liveness === "cold") {
1020
+ if (isRetryableDegradedRestore(lease.recovery.restore)) {
1021
+ // The elected spawner hit a transient archive verification failure.
1022
+ // Re-enter admission so one caller can become the next fenced spawner;
1023
+ // the archive remains authoritative and must not become terminal.
1024
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
1025
+ }
1107
1026
  if (
1108
1027
  lease.recovery.restore.status === "degraded" ||
1109
1028
  lease.recovery.restore.status === "unrecoverable"