@opengeni/worker-bundle 0.11.2 → 0.11.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/http.ts CHANGED
@@ -1,5 +1,10 @@
1
1
  import type { Settings } from "@opengeni/config";
2
- import { dbSql, type Database } from "@opengeni/db";
2
+ import {
3
+ assertRuntimeDatabasePosture,
4
+ dbSql,
5
+ type Database,
6
+ type RuntimeDatabasePostureOptions,
7
+ } from "@opengeni/db";
3
8
  import type { EventBus } from "@opengeni/events";
4
9
  import type { Observability } from "@opengeni/observability";
5
10
 
@@ -112,8 +117,15 @@ export async function runReadinessChecks(
112
117
  };
113
118
  }
114
119
 
115
- export function dbReadyCheck(db: Database): () => Promise<void> {
120
+ export function dbReadyCheck(
121
+ db: Database,
122
+ posture?: RuntimeDatabasePostureOptions,
123
+ ): () => Promise<void> {
116
124
  return async () => {
125
+ if (posture) {
126
+ await assertRuntimeDatabasePosture(db, posture);
127
+ return;
128
+ }
117
129
  await db.execute(dbSql`select 1`);
118
130
  };
119
131
  }
package/src/index.ts CHANGED
@@ -7,7 +7,13 @@ import {
7
7
  temporalConnectionOptions,
8
8
  type Settings,
9
9
  } from "@opengeni/config";
10
- import { createDb, markSessionWorkflowWakeDelivered, type Database } from "@opengeni/db";
10
+ import {
11
+ assertRuntimeDatabasePosture,
12
+ createDb,
13
+ markSessionWorkflowWakeDelivered,
14
+ type Database,
15
+ type RuntimeDatabasePostureOptions,
16
+ } from "@opengeni/db";
11
17
  import { createNatsEventBus, type EventBus } from "@opengeni/events";
12
18
  import {
13
19
  createObservability,
@@ -471,6 +477,11 @@ export async function registerSessionWorkflowWakeDispatcherSchedule(
471
477
 
472
478
  export type OpenGeniWorkerServiceOptions = Omit<WorkerOptions, "activityDependencies"> & {
473
479
  activityDependencies: ActivityDependencies & { db: Database; bus: EventBus };
480
+ /**
481
+ * Exact catalog posture required by the standalone runtime. Embedded hosts
482
+ * may omit this when they own an equivalent database isolation contract.
483
+ */
484
+ databasePosture?: RuntimeDatabasePostureOptions;
474
485
  /**
475
486
  * `role-default` registers OpenGeni's internal maintenance schedules on a
476
487
  * control worker only. These are engine maintenance schedules, not a host's
@@ -588,7 +599,7 @@ export async function createOpenGeniWorkerService(
588
599
  settings,
589
600
  observability,
590
601
  checks: {
591
- db: dbReadyCheck(options.activityDependencies.db),
602
+ db: dbReadyCheck(options.activityDependencies.db, options.databasePosture),
592
603
  nats: natsReadyCheck(options.activityDependencies.bus),
593
604
  temporal: temporalReadyCheck(workerBundle.connection),
594
605
  },
@@ -693,9 +704,19 @@ export async function startWorker() {
693
704
  ...(searchPath ? { searchPath } : {}),
694
705
  rlsStrategy: settings.rlsStrategy,
695
706
  });
707
+ const databasePosture = {
708
+ rlsStrategy: settings.rlsStrategy,
709
+ expectedRole: settings.runtimeDatabaseRole,
710
+ targetSchema: settings.dbSchema.trim() || "public",
711
+ } as const;
696
712
  const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
697
713
  let bus: Awaited<ReturnType<typeof createNatsEventBus>> | undefined;
698
714
  try {
715
+ await retryStartupDependency(
716
+ "PostgreSQL runtime posture",
717
+ () => assertRuntimeDatabasePosture(dbClient.db, databasePosture),
718
+ { ...retryOptions, onRetry },
719
+ );
699
720
  bus = await retryStartupDependency(
700
721
  "NATS",
701
722
  () =>
@@ -711,6 +732,7 @@ export async function startWorker() {
711
732
  await runOpenGeniWorker({
712
733
  role,
713
734
  settings,
735
+ databasePosture,
714
736
  activityDependencies: {
715
737
  observability,
716
738
  db: dbClient.db,
@@ -19,6 +19,7 @@ export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
19
19
  const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
20
20
  const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
21
21
  const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
22
+ const retainedProcessTerminalBacklogTotals = new WeakMap<Observability, number>();
22
23
 
23
24
  // A worker process reaching one trillion tokens in one provider/cache counter is
24
25
  // already far outside an ordinary scrape lifetime. Refuse further increments
@@ -383,6 +384,142 @@ export function recordSandboxOrphansTerminated(observability: Observability, cou
383
384
  });
384
385
  }
385
386
 
387
+ const RETAINED_PROCESS_OWNER_STATES = [
388
+ "direct",
389
+ "queued",
390
+ "running",
391
+ "requires_action",
392
+ "recovering",
393
+ "waiting_capacity",
394
+ "completed",
395
+ "failed",
396
+ "cancelled",
397
+ "superseded",
398
+ "withdrawn_for_edit",
399
+ "missing",
400
+ "unknown",
401
+ ] as const;
402
+
403
+ const RETAINED_PROCESS_OWNER_STATE_SET = new Set<string>(RETAINED_PROCESS_OWNER_STATES);
404
+
405
+ export function recordRetainedProcessInventoryGauges(
406
+ observability: Observability,
407
+ counts: Array<{ ownerState: string; activeCount: number; terminalOwnerCount: number }>,
408
+ ): void {
409
+ const normalized = new Map<string, { active: number; terminal: number }>();
410
+ for (const ownerState of RETAINED_PROCESS_OWNER_STATES) {
411
+ normalized.set(ownerState, { active: 0, terminal: 0 });
412
+ }
413
+ for (const count of counts) {
414
+ const ownerState = RETAINED_PROCESS_OWNER_STATE_SET.has(count.ownerState)
415
+ ? count.ownerState
416
+ : "unknown";
417
+ const current = normalized.get(ownerState)!;
418
+ current.active += Math.max(0, count.activeCount);
419
+ current.terminal += Math.max(0, count.terminalOwnerCount);
420
+ }
421
+
422
+ let terminalTotal = 0;
423
+ for (const [ownerState, count] of normalized) {
424
+ terminalTotal += count.terminal;
425
+ observability.setGauge({
426
+ name: "opengeni_retained_processes_active",
427
+ help: "Current active retained provider processes by durable owner state.",
428
+ labels: { owner_state: ownerState },
429
+ value: count.active,
430
+ });
431
+ observability.setGauge({
432
+ name: "opengeni_retained_processes_terminal_owner_backlog",
433
+ help: "Current active retained processes whose exact owner attempt is terminal.",
434
+ labels: { owner_state: ownerState },
435
+ value: count.terminal,
436
+ });
437
+ }
438
+ const previous = retainedProcessTerminalBacklogTotals.get(observability);
439
+ if (previous !== undefined && terminalTotal > previous) {
440
+ observability.incrementCounter({
441
+ name: "opengeni_retained_process_terminal_owner_backlog_growth_total",
442
+ help: "Positive growth in terminal-owner retained-process backlog between app samples.",
443
+ amount: terminalTotal - previous,
444
+ });
445
+ }
446
+ retainedProcessTerminalBacklogTotals.set(observability, terminalTotal);
447
+ }
448
+
449
+ const EXPIRED_DRAINING_BACKENDS = [
450
+ "none",
451
+ "local",
452
+ "docker",
453
+ "modal",
454
+ "e2b",
455
+ "daytona",
456
+ "blaxel",
457
+ "runloop",
458
+ "selfhosted",
459
+ "unknown",
460
+ ] as const;
461
+ const EXPIRED_DRAINING_BACKEND_SET = new Set<string>(EXPIRED_DRAINING_BACKENDS);
462
+ const EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"] as const;
463
+
464
+ export function recordExpiredDrainingSandboxLeaseGauges(
465
+ observability: Observability,
466
+ counts: Array<{ backend: string; ageBucket: string; count: number }>,
467
+ ): void {
468
+ const normalized = new Map<string, number>();
469
+ for (const backend of EXPIRED_DRAINING_BACKENDS) {
470
+ for (const ageBucket of EXPIRED_DRAINING_AGE_BUCKETS) {
471
+ normalized.set(`${backend}:${ageBucket}`, 0);
472
+ }
473
+ }
474
+ for (const count of counts) {
475
+ const backend = EXPIRED_DRAINING_BACKEND_SET.has(count.backend) ? count.backend : "unknown";
476
+ if (!EXPIRED_DRAINING_AGE_BUCKETS.includes(count.ageBucket as never)) continue;
477
+ const key = `${backend}:${count.ageBucket}`;
478
+ normalized.set(key, (normalized.get(key) ?? 0) + Math.max(0, count.count));
479
+ }
480
+ for (const [key, value] of normalized) {
481
+ const [backend, ageBucket] = key.split(":") as [string, string];
482
+ observability.setGauge({
483
+ name: "opengeni_sandbox_leases_expired_draining",
484
+ help: "Current expired draining sandbox leases by backend and fixed age bucket.",
485
+ labels: { backend, age_bucket: ageBucket },
486
+ value,
487
+ });
488
+ }
489
+ }
490
+
491
+ export const RETAINED_PROCESS_RECONCILIATION_OUTCOMES = [
492
+ "claim_failed",
493
+ "proof_exited",
494
+ "proof_lost",
495
+ "proof_checkpoint_failed",
496
+ "settled_exited",
497
+ "settled_lost",
498
+ "settlement_failed",
499
+ "identity_mismatch",
500
+ "resume_state_missing",
501
+ "backend_unsupported",
502
+ "provider_running",
503
+ "provider_unknown",
504
+ "provider_timeout",
505
+ "provider_error",
506
+ "defer_failed",
507
+ ] as const;
508
+
509
+ export type RetainedProcessReconciliationOutcome =
510
+ (typeof RETAINED_PROCESS_RECONCILIATION_OUTCOMES)[number];
511
+
512
+ export function recordRetainedProcessReconciliation(
513
+ observability: Observability,
514
+ outcome: RetainedProcessReconciliationOutcome,
515
+ ): void {
516
+ observability.incrementCounter({
517
+ name: "opengeni_retained_process_reconciliation_total",
518
+ help: "Bounded retained-process reconciliation observations by fixed app outcome.",
519
+ labels: { outcome },
520
+ });
521
+ }
522
+
386
523
  export function recordCreditMicros(
387
524
  observability: Observability | undefined,
388
525
  kind: CreditMicrosKind,
@@ -21,6 +21,7 @@
21
21
  // Liveness between turns is the lease refcount; there is no keepalive loop.
22
22
 
23
23
  import type { Settings } from "@opengeni/config";
24
+ import { redactSensitiveText } from "@opengeni/contracts";
24
25
  import {
25
26
  acquireLease,
26
27
  beginSandboxRematerialization,
@@ -75,6 +76,13 @@ export function sandboxLeaseHolderIdForAttempt(attemptId: string): TurnSandboxLe
75
76
  return `turn-attempt:${normalized}`;
76
77
  }
77
78
 
79
+ export function isRetryableDegradedRestore(restore: {
80
+ status: string;
81
+ retryable?: boolean;
82
+ }): boolean {
83
+ return restore.status === "degraded" && restore.retryable === true;
84
+ }
85
+
78
86
  /** The minimal services surface resumeBoxForTurn needs. A subset of
79
87
  * ActivityServices so a test (and the API later) can pass a lean bag. */
80
88
  export type SandboxResumeServices = {
@@ -207,6 +215,17 @@ class SnapshotTimeoutError extends Error {
207
215
  }
208
216
  }
209
217
 
218
+ function safeSnapshotError(error: unknown): { name: string; message: string } {
219
+ const rawName = error instanceof Error ? error.name : "Error";
220
+ return {
221
+ name: /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error",
222
+ message: redactSensitiveText(error instanceof Error ? error.message : String(error)).slice(
223
+ 0,
224
+ 2_048,
225
+ ),
226
+ };
227
+ }
228
+
210
229
  export async function waitForWarmSnapshot(
211
230
  snapshot: Promise<unknown>,
212
231
  timeoutMs: number,
@@ -239,7 +258,10 @@ export async function waitForWarmSnapshot(
239
258
  } catch (error) {
240
259
  if (signal?.aborted) return false;
241
260
  if (error instanceof SnapshotTimeoutError) {
242
- console.error("mid-session workspace snapshot wait timed out (turn unaffected)", error);
261
+ console.error(
262
+ "mid-session workspace snapshot wait timed out (turn unaffected)",
263
+ safeSnapshotError(error),
264
+ );
243
265
  return false;
244
266
  }
245
267
  return true;
@@ -521,7 +543,10 @@ export async function maybePersistWarmWorkspaceSnapshot(
521
543
  } catch (error) {
522
544
  // Protection, not a dependency: a failed snapshot must never fail (or slow
523
545
  // down retrying) the turn. The next heartbeat/turn-end tick retries.
524
- console.error("mid-session workspace snapshot failed (turn unaffected)", error);
546
+ console.error(
547
+ "mid-session workspace snapshot failed (turn unaffected)",
548
+ safeSnapshotError(error),
549
+ );
525
550
  return false;
526
551
  }
527
552
  }
@@ -963,118 +988,6 @@ export async function resumeBoxForTurn(
963
988
  }
964
989
  }
965
990
 
966
- /**
967
- * Stage D machine-primary: acquire the group lease for a turn whose ACTIVE sandbox
968
- * is a connected machine (selfhosted) WITHOUT establishing a provider box. The
969
- * "box" is the user's physical machine reached over NATS; there is NOTHING to
970
- * spawn, snapshot, expose, or serialize — so we take the lease (the SAME group-keyed
971
- * refcount/idle/epoch bookkeeping resumeBoxForTurn uses) but skip every box step.
972
- *
973
- * The lease is keyed backend "selfhosted": the reaper's terminateProviderBox
974
- * short-circuits a selfhosted lease (it is the user's machine — NEVER provider-stop),
975
- * draining it to cold with no stop; and the reaper's warm-meter rate for "selfhosted"
976
- * is 0 (no cloud seconds billed for a box that does not exist). Returns the lease
977
- * epoch + an idempotent release so the turn's lease-heartbeat + `finally` release are
978
- * identical to the cloud path.
979
- *
980
- * holderId is the globally unique durable turn-attempt id. It must not be a
981
- * Temporal activity id, because activity ids are only workflow-local and
982
- * collide when sibling sessions share one sandbox group.
983
- */
984
- export async function acquireSelfhostedLeaseForTurn(
985
- services: SandboxResumeServices,
986
- ids: {
987
- accountId: string;
988
- workspaceId: string;
989
- sandboxGroupId: string;
990
- sessionId: string;
991
- },
992
- kind: "turn",
993
- holderId: TurnSandboxLeaseHolderId,
994
- ): Promise<{ leaseEpoch: number; release: () => Promise<void> }> {
995
- const { db, settings } = services;
996
- const leaseTtlMs = settings.sandboxLeaseTtlMs;
997
-
998
- let released = false;
999
- const release = async (): Promise<void> => {
1000
- if (released) {
1001
- return;
1002
- }
1003
- released = true;
1004
- await releaseLeaseHolder(db, {
1005
- accountId: ids.accountId,
1006
- workspaceId: ids.workspaceId,
1007
- sandboxGroupId: ids.sandboxGroupId,
1008
- kind,
1009
- holderId,
1010
- idleGraceMs: settings.sandboxIdleGraceMs,
1011
- });
1012
- };
1013
-
1014
- const acquired = await acquireLease(db, {
1015
- accountId: ids.accountId,
1016
- workspaceId: ids.workspaceId,
1017
- sandboxGroupId: ids.sandboxGroupId,
1018
- kind,
1019
- holderId,
1020
- subjectId: ids.sessionId,
1021
- backend: "selfhosted",
1022
- os: "linux",
1023
- leaseTtlMs,
1024
- warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
1025
- });
1026
-
1027
- // FENCED: a newer epoch re-established the group concurrently. Release our
1028
- // just-registered holder + surface the supersession (the outer turn catch
1029
- // requeues, mirroring resumeBoxForTurn).
1030
- if (acquired.role === "blocked") {
1031
- await release();
1032
- throw new SandboxLeaseRecoveryBlockedError(
1033
- ids.sandboxGroupId,
1034
- acquired.lease.leaseEpoch,
1035
- acquired.code,
1036
- acquired.lease.recovery,
1037
- );
1038
- }
1039
- if (acquired.role === "fenced") {
1040
- await release();
1041
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
1042
- }
1043
-
1044
- // SPAWNER: we won the cold->warming CAS. There is NO box to establish — commit
1045
- // warm immediately so the lease does not linger in 'warming' (no box id, no
1046
- // resume_state; selfhosted is re-addressed by enrollment via the active pointer,
1047
- // never resumed from an envelope). resume_backend_id "selfhosted" keeps the
1048
- // reaper's never-provider-stop short-circuit correct.
1049
- if (acquired.role === "spawner") {
1050
- const expectedEpoch = acquired.lease.leaseEpoch;
1051
- const committed = await commitWarmingToWarm(db, {
1052
- accountId: ids.accountId,
1053
- workspaceId: ids.workspaceId,
1054
- sandboxGroupId: ids.sandboxGroupId,
1055
- expectedEpoch,
1056
- // No provider box id; stamp a clearly non-box marker for diagnostics.
1057
- instanceId: `selfhosted:${ids.sessionId}`,
1058
- resumeBackendId: "selfhosted",
1059
- resumeState: null,
1060
- leaseTtlMs,
1061
- });
1062
- if (!committed.committed || !committed.lease) {
1063
- // A reaper reset our warming row, or a sibling re-established + bumped the
1064
- // epoch. Release our holder; surface supersession (NEVER touch any box).
1065
- await release();
1066
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1067
- }
1068
- return { leaseEpoch: committed.lease.leaseEpoch, release };
1069
- }
1070
-
1071
- // ATTACHED / REARMED: the lease is live. There is no box to resume — just carry
1072
- // the lease epoch (same-session turns are serialized, so a concurrent 'warming'
1073
- // spawner on this group is not expected; the machine session is independent of the
1074
- // lease state regardless).
1075
- return { leaseEpoch: acquired.lease.leaseEpoch, release };
1076
- }
1077
-
1078
991
  /**
1079
992
  * Poll a warming lease until the spawner commits warm. If the warming row is
1080
993
  * reset to cold (the spawner died and the reaper reset it), surface supersession
@@ -1104,6 +1017,12 @@ async function waitForWarm(
1104
1017
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
1105
1018
  }
1106
1019
  if (lease.liveness === "cold") {
1020
+ if (isRetryableDegradedRestore(lease.recovery.restore)) {
1021
+ // The elected spawner hit a transient archive verification failure.
1022
+ // Re-enter admission so one caller can become the next fenced spawner;
1023
+ // the archive remains authoritative and must not become terminal.
1024
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
1025
+ }
1107
1026
  if (
1108
1027
  lease.recovery.restore.status === "degraded" ||
1109
1028
  lease.recovery.restore.status === "unrecoverable"
@@ -18,7 +18,9 @@ import type { Settings } from "@opengeni/config";
18
18
  import {
19
19
  advanceWorkspaceGenerationForRetainedProcess,
20
20
  advanceWorkspaceGeneration,
21
+ getRetainedProcess,
21
22
  retainWorkspaceMutationProcess,
23
+ retainedProcessSettlementIdentity,
22
24
  settleRetainedProcess,
23
25
  verifyRetainedProcessMutationSettlement,
24
26
  verifyWorkspaceMutationSettlement,
@@ -26,6 +28,7 @@ import {
26
28
  markWarmLeaseInstanceLost,
27
29
  readLease,
28
30
  readActiveSandbox,
31
+ SandboxRetainedProcessPromotionFencedError,
29
32
  type Database,
30
33
  type SandboxWorkspaceMutationAdmission,
31
34
  } from "@opengeni/db";
@@ -47,6 +50,7 @@ import {
47
50
  type RoutableBackendSession,
48
51
  type RoutableSandbox,
49
52
  type ResolvedActiveBackend,
53
+ type RoutingMutationSettlementResult,
50
54
  type RoutingRetainedProcess,
51
55
  type RoutingRetainedProcessTerminalProof,
52
56
  type SelfhostedOpObserver,
@@ -175,7 +179,10 @@ function selfhostedResolverTimeouts(settings: Settings): {
175
179
  selfhostedExecTimeoutMs: number;
176
180
  } {
177
181
  const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
178
- return { selfhostedTimeoutMs: timeoutMs, selfhostedExecTimeoutMs: execTimeoutMs };
182
+ return {
183
+ selfhostedTimeoutMs: timeoutMs,
184
+ selfhostedExecTimeoutMs: execTimeoutMs,
185
+ };
179
186
  }
180
187
 
181
188
  type HomeRouteLeaseIdentity = {
@@ -232,7 +239,9 @@ function homeRouteRecoveryError(
232
239
  );
233
240
  }
234
241
 
235
- function providerIdentityFromResumeState(resumeState: Record<string, unknown>): string | null {
242
+ export function providerIdentityFromResumeState(
243
+ resumeState: Record<string, unknown>,
244
+ ): string | null {
236
245
  const sessionState =
237
246
  resumeState.sessionState && typeof resumeState.sessionState === "object"
238
247
  ? (resumeState.sessionState as Record<string, unknown>)
@@ -288,7 +297,10 @@ async function resolveCurrentHomeBackend(
288
297
  // A route epoch can advance without a provider replacement. Reuse the exact
289
298
  // established handle only when its identity still equals the durable one.
290
299
  if (lease.instanceId === established.instanceId) {
291
- services.onHomeSandboxRebound?.({ established, leaseEpoch: lease.leaseEpoch });
300
+ services.onHomeSandboxRebound?.({
301
+ established,
302
+ leaseEpoch: lease.leaseEpoch,
303
+ });
292
304
  return {
293
305
  session: established.session as RoutableBackendSession,
294
306
  sandboxId: null,
@@ -357,7 +369,10 @@ async function resolveCurrentHomeBackend(
357
369
  if (rebound.instanceId !== lease.instanceId || rebound.backendId !== resumeBackend) {
358
370
  throw homeRouteRecoveryError(lease, lease.leaseEpoch);
359
371
  }
360
- services.onHomeSandboxRebound?.({ established: rebound, leaseEpoch: lease.leaseEpoch });
372
+ services.onHomeSandboxRebound?.({
373
+ established: rebound,
374
+ leaseEpoch: lease.leaseEpoch,
375
+ });
361
376
  // Keep the resolver's mutable home reference on the latest verified handle so
362
377
  // a later route epoch does not resume the same replacement again. This is only
363
378
  // a worker-side handle update; the SDK-facing RoutingSandboxSession remains the
@@ -423,7 +438,7 @@ function afterPersistableHomeMutation(
423
438
  outcome: "resolved" | "rejected";
424
439
  result?: unknown;
425
440
  retainedProcess?: RoutingRetainedProcess;
426
- }) => Promise<void>)
441
+ }) => Promise<void | RoutingMutationSettlementResult>)
427
442
  | undefined {
428
443
  const fence = ids.workspaceMutationFence;
429
444
  if (!home || !fence) return undefined;
@@ -442,29 +457,58 @@ function afterPersistableHomeMutation(
442
457
  }
443
458
  const exactAdmission = admission as SandboxWorkspaceMutationAdmission;
444
459
  if (outcome === "resolved" && retainedProcess) {
445
- await retainWorkspaceMutationProcess(services.db, {
446
- accountId: fence.accountId,
447
- workspaceId: ids.workspaceId,
448
- sessionId: ids.sessionId,
449
- processId: retainedProcess.id,
450
- providerSessionId: retainedProcess.providerSessionId,
451
- admissionId: exactAdmission.id,
452
- admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
453
- operation: op,
454
- owner: {
455
- kind: "turn",
456
- turnId: fence.turnId,
457
- executionGeneration: fence.executionGeneration,
458
- attemptId: fence.attemptId,
459
- holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
460
- sandboxGroupId: home.sandboxGroupId,
461
- expectedEpoch: backend.leaseEpoch,
462
- expectedInstanceId: backend.providerInstanceId,
463
- routeKind: exactAdmission.routeKind,
464
- routeTargetId: exactAdmission.routeTargetId,
465
- routeEpoch: exactAdmission.routeEpoch,
466
- },
467
- });
460
+ try {
461
+ await retainWorkspaceMutationProcess(services.db, {
462
+ accountId: fence.accountId,
463
+ workspaceId: ids.workspaceId,
464
+ sessionId: ids.sessionId,
465
+ processId: retainedProcess.id,
466
+ providerSessionId: retainedProcess.providerSessionId,
467
+ admissionId: exactAdmission.id,
468
+ admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
469
+ operation: op,
470
+ owner: {
471
+ kind: "turn",
472
+ turnId: fence.turnId,
473
+ executionGeneration: fence.executionGeneration,
474
+ attemptId: fence.attemptId,
475
+ holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
476
+ sandboxGroupId: home.sandboxGroupId,
477
+ expectedEpoch: backend.leaseEpoch,
478
+ expectedInstanceId: backend.providerInstanceId,
479
+ routeKind: exactAdmission.routeKind,
480
+ routeTargetId: exactAdmission.routeTargetId,
481
+ routeEpoch: exactAdmission.routeEpoch,
482
+ },
483
+ });
484
+ } catch (error) {
485
+ if (!(error instanceof SandboxRetainedProcessPromotionFencedError)) throw error;
486
+ const durable = error.process;
487
+ if (
488
+ durable.id !== retainedProcess.id ||
489
+ durable.providerSessionId !== retainedProcess.providerSessionId ||
490
+ durable.sessionId !== ids.sessionId ||
491
+ durable.sandboxGroupId !== home.sandboxGroupId ||
492
+ durable.parentAdmissionId !== exactAdmission.id ||
493
+ durable.leaseEpoch !== backend.leaseEpoch ||
494
+ durable.providerInstanceId !== backend.providerInstanceId ||
495
+ durable.routeKind !== exactAdmission.routeKind ||
496
+ durable.routeTargetId !== exactAdmission.routeTargetId ||
497
+ durable.routeEpoch !== exactAdmission.routeEpoch ||
498
+ durable.state !== "active"
499
+ ) {
500
+ throw new Error("Durable retained-process promotion returned a mismatched identity", {
501
+ cause: error,
502
+ });
503
+ }
504
+ return {
505
+ status: "retained_process_durable_output_rejected",
506
+ retainedProcess: {
507
+ id: durable.id,
508
+ providerSessionId: durable.providerSessionId,
509
+ },
510
+ };
511
+ }
468
512
  return;
469
513
  }
470
514
  await verifyWorkspaceMutationSettlement(services.db, {
@@ -556,12 +600,30 @@ function settleRetainedProcessForTurn(
556
600
  | undefined {
557
601
  const fence = ids.workspaceMutationFence;
558
602
  if (!fence) return undefined;
559
- return async ({ process, proof }) => {
603
+ return async ({ backend, process, proof }) => {
604
+ const durable = await getRetainedProcess(services.db, {
605
+ workspaceId: ids.workspaceId,
606
+ sessionId: ids.sessionId,
607
+ processId: process.id,
608
+ });
609
+ if (
610
+ !durable ||
611
+ durable.providerSessionId !== process.providerSessionId ||
612
+ durable.providerBackend !== backend.kind ||
613
+ durable.providerInstanceId !== backend.providerInstanceId ||
614
+ durable.leaseEpoch !== backend.leaseEpoch ||
615
+ durable.routeKind !== (backend.sandboxId === null ? "home" : "active") ||
616
+ durable.routeTargetId !== backend.sandboxId ||
617
+ durable.routeEpoch !== backend.activeEpoch
618
+ ) {
619
+ throw new Error("Retained-process settlement lost its exact durable backend identity");
620
+ }
560
621
  await settleRetainedProcess(services.db, {
561
622
  accountId: fence.accountId,
562
623
  workspaceId: ids.workspaceId,
563
624
  sessionId: ids.sessionId,
564
625
  processId: process.id,
626
+ expected: retainedProcessSettlementIdentity(durable),
565
627
  outcome: proof.outcome,
566
628
  exitCode: proof.exitCode,
567
629
  reason: proof.reason,
@@ -762,7 +824,10 @@ export function wrapLazyTurnBoxWithRouting(
762
824
  backendId: string;
763
825
  agentDefaultManifest: unknown;
764
826
  provisioner: {
765
- get(): Promise<{ established: EstablishedSandboxSession; leaseEpoch?: number }>;
827
+ get(): Promise<{
828
+ established: EstablishedSandboxSession;
829
+ leaseEpoch?: number;
830
+ }>;
766
831
  };
767
832
  homeLeaseIdentity?: {
768
833
  accountId: string;