@opengeni/worker-bundle 0.10.8 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,24 +15,46 @@
15
15
  // request/reply connection.
16
16
 
17
17
  import type { Settings } from "@opengeni/config";
18
- import { getSandbox, readActiveSandbox, type Database } from "@opengeni/db";
18
+ import {
19
+ advanceWorkspaceGenerationForRetainedProcess,
20
+ advanceWorkspaceGeneration,
21
+ retainWorkspaceMutationProcess,
22
+ settleRetainedProcess,
23
+ verifyRetainedProcessMutationSettlement,
24
+ verifyWorkspaceMutationSettlement,
25
+ getSandbox,
26
+ markWarmLeaseInstanceLost,
27
+ readLease,
28
+ readActiveSandbox,
29
+ type Database,
30
+ type SandboxWorkspaceMutationAdmission,
31
+ } from "@opengeni/db";
19
32
  import type { EventBus } from "@opengeni/events";
20
33
  import {
21
34
  buildSelfhostedBackendSession,
35
+ establishSandboxSessionFromEnvelope,
36
+ isProviderSandboxGoneDuringRoutedOperation,
37
+ isProviderSandboxNotFoundError,
22
38
  makeActiveBackendResolver,
23
39
  NatsControlRpc,
24
40
  NatsOpStreamTransport,
41
+ RoutingBackendRecoveryRequiredError,
25
42
  RoutingSandboxSession,
43
+ verifySandboxExecReadiness,
26
44
  type ControlRpc,
27
45
  type EstablishedSandboxSession,
28
46
  type NatsRequestConnection,
29
47
  type RoutableBackendSession,
30
48
  type RoutableSandbox,
49
+ type ResolvedActiveBackend,
50
+ type RoutingRetainedProcess,
51
+ type RoutingRetainedProcessTerminalProof,
31
52
  type SelfhostedOpObserver,
32
53
  type SelfhostedRelayConfig,
33
54
  type OpStreamJournal,
34
55
  type SelfhostedOpStreamDeps,
35
56
  } from "@opengeni/runtime";
57
+ import { sandboxLeaseHolderIdForAttempt } from "./sandbox-resume";
36
58
 
37
59
  export type RoutingWiringServices = {
38
60
  db: Database;
@@ -49,11 +71,38 @@ export type RoutingWiringServices = {
49
71
  * the runtime defaults (generation "1", no persistence) — tests / non-turn
50
72
  * callers. Only consulted when op-stream is actually enabled for the turn. */
51
73
  opJournal?: OpStreamJournal;
74
+ /** Durable lifecycle notification emitted only by the observer that wins the
75
+ * exact warm-instance loss CAS. */
76
+ onHomeSandboxLost?: (input: {
77
+ sandboxGroupId: string;
78
+ instanceId: string;
79
+ leaseEpoch: number;
80
+ }) => Promise<void>;
81
+ /** Called when a route repair replaces the worker's original home handle. */
82
+ onHomeSandboxRebound?: (input: {
83
+ established: EstablishedSandboxSession;
84
+ leaseEpoch: number;
85
+ }) => void;
52
86
  };
53
87
 
54
88
  export type RoutingWiringIds = {
55
89
  workspaceId: string;
56
90
  sessionId: string;
91
+ /** Canonical turn-attempt fence used to admit persistable-home mutations.
92
+ * Omit outside a running worker attempt; mutation hooks then remain disabled. */
93
+ workspaceMutationFence?: {
94
+ accountId: string;
95
+ turnId: string;
96
+ executionGeneration: number;
97
+ attemptId: string;
98
+ };
99
+ homeLease?: {
100
+ accountId: string;
101
+ sandboxGroupId: string;
102
+ leaseEpoch: number;
103
+ instanceId: string;
104
+ backend: string;
105
+ };
57
106
  /**
58
107
  * The run's declared sandbox environment — the SAME object the turn passes to
59
108
  * `runtime.buildAgent`'s `sandboxEnvironment` and to `resumeBoxForTurn` (so the
@@ -129,6 +178,398 @@ function selfhostedResolverTimeouts(settings: Settings): {
129
178
  return { selfhostedTimeoutMs: timeoutMs, selfhostedExecTimeoutMs: execTimeoutMs };
130
179
  }
131
180
 
181
+ type HomeRouteLeaseIdentity = {
182
+ accountId: string;
183
+ sandboxGroupId: string;
184
+ backend: string;
185
+ };
186
+
187
+ type HomeRouteResolutionIds = HomeRouteLeaseIdentity & {
188
+ workspaceId: string;
189
+ sessionId: string;
190
+ };
191
+
192
+ type HomeRouteRecovery = "pending" | "degraded" | "unrecoverable" | "superseded";
193
+
194
+ /**
195
+ * Translate the durable lease/recovery state into the typed disposition the
196
+ * routing proxy exposes when the home cannot be safely resumed. A route repair
197
+ * must never fall back to the old in-memory provider handle when the lease is
198
+ * warming, restoring, unverifiable, or otherwise inconsistent.
199
+ */
200
+ function homeRouteRecoveryDisposition(
201
+ lease: Awaited<ReturnType<typeof readLease>>,
202
+ ): HomeRouteRecovery {
203
+ if (!lease) return "unrecoverable";
204
+ if (lease.liveness === "warming") return "pending";
205
+ if (lease.liveness === "draining") return "superseded";
206
+ if (
207
+ lease.recovery.restore.status === "pending" ||
208
+ lease.recovery.restore.status === "restoring" ||
209
+ lease.recovery.restore.status === "verifying"
210
+ ) {
211
+ return "pending";
212
+ }
213
+ if (
214
+ lease.recovery.restore.status === "degraded" ||
215
+ lease.recovery.workspace.status === "degraded" ||
216
+ lease.recovery.archive.status === "unverified" ||
217
+ lease.recovery.archive.status === "invalid"
218
+ ) {
219
+ return "degraded";
220
+ }
221
+ return "unrecoverable";
222
+ }
223
+
224
+ function homeRouteRecoveryError(
225
+ lease: Awaited<ReturnType<typeof readLease>>,
226
+ fallbackEpoch: number,
227
+ ): RoutingBackendRecoveryRequiredError {
228
+ return new RoutingBackendRecoveryRequiredError(
229
+ "resolve_home_backend",
230
+ lease?.leaseEpoch ?? fallbackEpoch,
231
+ homeRouteRecoveryDisposition(lease),
232
+ );
233
+ }
234
+
235
+ function providerIdentityFromResumeState(resumeState: Record<string, unknown>): string | null {
236
+ const sessionState =
237
+ resumeState.sessionState && typeof resumeState.sessionState === "object"
238
+ ? (resumeState.sessionState as Record<string, unknown>)
239
+ : null;
240
+ const providerState =
241
+ sessionState?.providerState && typeof sessionState.providerState === "object"
242
+ ? (sessionState.providerState as Record<string, unknown>)
243
+ : null;
244
+ for (const field of [
245
+ "sandboxId",
246
+ "instanceId",
247
+ "id",
248
+ "hostId",
249
+ "containerId",
250
+ "workspaceRootPath",
251
+ "agentId",
252
+ ]) {
253
+ const value = providerState?.[field];
254
+ if (typeof value === "string" && value.length > 0) return value;
255
+ }
256
+ return null;
257
+ }
258
+
259
+ /**
260
+ * Resolve the current durable home identity after the route epoch changes.
261
+ *
262
+ * Same-target attach/repair can replace the provider instance while a worker
263
+ * turn still holds the stable routing proxy. The proxy must therefore resume
264
+ * the lease's CURRENT identity, not reuse the object captured at turn start.
265
+ * This seam is deliberately resume-only: lease election/rematerialization is
266
+ * owned by the API/lease state machine, and a routed tool call must never create
267
+ * a rival provider or replay an ambiguous mutation.
268
+ */
269
+ async function resolveCurrentHomeBackend(
270
+ services: RoutingWiringServices,
271
+ ids: HomeRouteResolutionIds,
272
+ established: EstablishedSandboxSession,
273
+ ): Promise<ResolvedActiveBackend> {
274
+ const lease = await readLease(services.db, ids.workspaceId, ids.sandboxGroupId);
275
+ const fallbackEpoch = lease?.leaseEpoch ?? 0;
276
+ if (
277
+ !lease ||
278
+ lease.liveness !== "warm" ||
279
+ lease.recovery.provider.status !== "exists" ||
280
+ lease.instanceId === null ||
281
+ lease.recovery.provider.instanceId !== lease.instanceId ||
282
+ lease.recovery.workspace.status !== "ready" ||
283
+ (lease.recovery.restore.status !== "not_required" && lease.recovery.restore.status !== "ready")
284
+ ) {
285
+ throw homeRouteRecoveryError(lease, fallbackEpoch);
286
+ }
287
+
288
+ // A route epoch can advance without a provider replacement. Reuse the exact
289
+ // established handle only when its identity still equals the durable one.
290
+ if (lease.instanceId === established.instanceId) {
291
+ services.onHomeSandboxRebound?.({ established, leaseEpoch: lease.leaseEpoch });
292
+ return {
293
+ session: established.session as RoutableBackendSession,
294
+ sandboxId: null,
295
+ kind: established.backendId,
296
+ leaseEpoch: lease.leaseEpoch,
297
+ providerInstanceId: lease.instanceId,
298
+ };
299
+ }
300
+
301
+ const resumeState = lease.resumeState;
302
+ const resumeBackend = lease.resumeBackendId ?? lease.backend ?? ids.backend;
303
+ if (
304
+ !resumeState ||
305
+ resumeBackend !== ids.backend ||
306
+ providerIdentityFromResumeState(resumeState) !== lease.instanceId
307
+ ) {
308
+ throw homeRouteRecoveryError(lease, lease.leaseEpoch);
309
+ }
310
+
311
+ let rebound: EstablishedSandboxSession;
312
+ try {
313
+ rebound = await establishSandboxSessionFromEnvelope(services.settings, resumeState, {
314
+ sessionId: ids.sessionId,
315
+ recovery: "resume-only",
316
+ backendOverride: resumeBackend as never,
317
+ });
318
+ // A resumed handle is not enough evidence for the route to use it. Keep the
319
+ // same bounded readiness gate used before publishing a Modal lease warm.
320
+ await verifySandboxExecReadiness(rebound);
321
+ } catch (error) {
322
+ if (!isProviderSandboxNotFoundError(resumeBackend, error)) {
323
+ // The durable identity is present, but a transient resume/provider error
324
+ // did not prove the box gone. Keep the lease untouched and make the next
325
+ // independent route operation retry the resume; never use the old handle.
326
+ throw new RoutingBackendRecoveryRequiredError(
327
+ "resolve_home_backend",
328
+ lease.leaseEpoch,
329
+ "pending",
330
+ );
331
+ }
332
+ const marked = await markWarmLeaseInstanceLost(services.db, {
333
+ accountId: ids.accountId,
334
+ workspaceId: ids.workspaceId,
335
+ sandboxGroupId: ids.sandboxGroupId,
336
+ expectedEpoch: lease.leaseEpoch,
337
+ expectedInstanceId: lease.instanceId,
338
+ diagnostic: "provider_not_found_during_home_route_rebind",
339
+ });
340
+ if (marked.status === "marked") {
341
+ await services.onHomeSandboxLost?.({
342
+ sandboxGroupId: ids.sandboxGroupId,
343
+ instanceId: lease.instanceId,
344
+ leaseEpoch: marked.lease.leaseEpoch,
345
+ });
346
+ }
347
+ throw new RoutingBackendRecoveryRequiredError(
348
+ "resolve_home_backend",
349
+ marked.lease?.leaseEpoch ?? lease.leaseEpoch,
350
+ marked.status === "stale" ? "superseded" : homeRouteRecoveryDisposition(marked.lease),
351
+ );
352
+ }
353
+
354
+ // The provider identity must agree with the exact durable lease row before
355
+ // any caller can publish or route through this rebound handle. A mismatch is
356
+ // unverifiable local state, not permission to try the old handle.
357
+ if (rebound.instanceId !== lease.instanceId || rebound.backendId !== resumeBackend) {
358
+ throw homeRouteRecoveryError(lease, lease.leaseEpoch);
359
+ }
360
+ services.onHomeSandboxRebound?.({ established: rebound, leaseEpoch: lease.leaseEpoch });
361
+ // Keep the resolver's mutable home reference on the latest verified handle so
362
+ // a later route epoch does not resume the same replacement again. This is only
363
+ // a worker-side handle update; the SDK-facing RoutingSandboxSession remains the
364
+ // stable object returned below and is never replaced.
365
+ Object.assign(established, rebound);
366
+ return {
367
+ session: rebound.session as RoutableBackendSession,
368
+ sandboxId: null,
369
+ kind: rebound.backendId,
370
+ leaseEpoch: lease.leaseEpoch,
371
+ providerInstanceId: lease.instanceId,
372
+ };
373
+ }
374
+
375
+ function beforePersistableHomeMutation(
376
+ services: RoutingWiringServices,
377
+ ids: RoutingWiringIds,
378
+ home: { accountId: string; sandboxGroupId: string } | undefined,
379
+ ):
380
+ | ((input: {
381
+ op: string;
382
+ backend: ResolvedActiveBackend;
383
+ }) => Promise<SandboxWorkspaceMutationAdmission | null>)
384
+ | undefined {
385
+ const fence = ids.workspaceMutationFence;
386
+ if (!home || !fence) return undefined;
387
+ return async ({ op, backend }) => {
388
+ // A connected-machine target is intentionally non-persistable: its writes
389
+ // do not dirty the session home's archive. Exact home identity metadata is
390
+ // present only on a verified durable home route.
391
+ if (
392
+ backend.sandboxId !== null ||
393
+ backend.leaseEpoch === undefined ||
394
+ backend.providerInstanceId === undefined
395
+ ) {
396
+ return null;
397
+ }
398
+ return await advanceWorkspaceGeneration(services.db, {
399
+ accountId: fence.accountId,
400
+ workspaceId: ids.workspaceId,
401
+ sessionId: ids.sessionId,
402
+ turnId: fence.turnId,
403
+ executionGeneration: fence.executionGeneration,
404
+ attemptId: fence.attemptId,
405
+ holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
406
+ sandboxGroupId: home.sandboxGroupId,
407
+ expectedEpoch: backend.leaseEpoch,
408
+ expectedInstanceId: backend.providerInstanceId,
409
+ operation: op,
410
+ });
411
+ };
412
+ }
413
+
414
+ function afterPersistableHomeMutation(
415
+ services: RoutingWiringServices,
416
+ ids: RoutingWiringIds,
417
+ home: { accountId: string; sandboxGroupId: string } | undefined,
418
+ ):
419
+ | ((input: {
420
+ op: string;
421
+ backend: ResolvedActiveBackend;
422
+ admission: unknown;
423
+ outcome: "resolved" | "rejected";
424
+ result?: unknown;
425
+ retainedProcess?: RoutingRetainedProcess;
426
+ }) => Promise<void>)
427
+ | undefined {
428
+ const fence = ids.workspaceMutationFence;
429
+ if (!home || !fence) return undefined;
430
+ return async ({ op, backend, admission, outcome, retainedProcess }) => {
431
+ if (
432
+ backend.sandboxId !== null ||
433
+ backend.leaseEpoch === undefined ||
434
+ backend.providerInstanceId === undefined ||
435
+ !admission ||
436
+ typeof admission !== "object" ||
437
+ typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).id !== "string" ||
438
+ typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).workspaceGeneration !==
439
+ "number"
440
+ ) {
441
+ return;
442
+ }
443
+ const exactAdmission = admission as SandboxWorkspaceMutationAdmission;
444
+ if (outcome === "resolved" && retainedProcess) {
445
+ await retainWorkspaceMutationProcess(services.db, {
446
+ accountId: fence.accountId,
447
+ workspaceId: ids.workspaceId,
448
+ sessionId: ids.sessionId,
449
+ processId: retainedProcess.id,
450
+ providerSessionId: retainedProcess.providerSessionId,
451
+ admissionId: exactAdmission.id,
452
+ admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
453
+ operation: op,
454
+ owner: {
455
+ kind: "turn",
456
+ turnId: fence.turnId,
457
+ executionGeneration: fence.executionGeneration,
458
+ attemptId: fence.attemptId,
459
+ holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
460
+ sandboxGroupId: home.sandboxGroupId,
461
+ expectedEpoch: backend.leaseEpoch,
462
+ expectedInstanceId: backend.providerInstanceId,
463
+ routeKind: exactAdmission.routeKind,
464
+ routeTargetId: exactAdmission.routeTargetId,
465
+ routeEpoch: exactAdmission.routeEpoch,
466
+ },
467
+ });
468
+ return;
469
+ }
470
+ await verifyWorkspaceMutationSettlement(services.db, {
471
+ accountId: fence.accountId,
472
+ workspaceId: ids.workspaceId,
473
+ sessionId: ids.sessionId,
474
+ turnId: fence.turnId,
475
+ executionGeneration: fence.executionGeneration,
476
+ attemptId: fence.attemptId,
477
+ holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
478
+ sandboxGroupId: home.sandboxGroupId,
479
+ expectedEpoch: backend.leaseEpoch,
480
+ expectedInstanceId: backend.providerInstanceId,
481
+ admission: exactAdmission,
482
+ operation: op,
483
+ outcome,
484
+ });
485
+ };
486
+ }
487
+
488
+ function beforeRetainedProcessMutation(
489
+ services: RoutingWiringServices,
490
+ ids: RoutingWiringIds,
491
+ ):
492
+ | ((input: {
493
+ op: string;
494
+ backend: ResolvedActiveBackend;
495
+ process: RoutingRetainedProcess;
496
+ }) => Promise<SandboxWorkspaceMutationAdmission>)
497
+ | undefined {
498
+ const fence = ids.workspaceMutationFence;
499
+ if (!fence) return undefined;
500
+ return async ({ op, process }) =>
501
+ await advanceWorkspaceGenerationForRetainedProcess(services.db, {
502
+ accountId: fence.accountId,
503
+ workspaceId: ids.workspaceId,
504
+ sessionId: ids.sessionId,
505
+ processId: process.id,
506
+ operation: op,
507
+ });
508
+ }
509
+
510
+ function afterRetainedProcessMutation(
511
+ services: RoutingWiringServices,
512
+ ids: RoutingWiringIds,
513
+ ):
514
+ | ((input: {
515
+ op: string;
516
+ backend: ResolvedActiveBackend;
517
+ process: RoutingRetainedProcess;
518
+ admission: unknown;
519
+ outcome: "resolved" | "rejected";
520
+ result?: unknown;
521
+ }) => Promise<void>)
522
+ | undefined {
523
+ const fence = ids.workspaceMutationFence;
524
+ if (!fence) return undefined;
525
+ return async ({ op, process, admission, outcome }) => {
526
+ if (
527
+ !admission ||
528
+ typeof admission !== "object" ||
529
+ typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).id !== "string" ||
530
+ typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).workspaceGeneration !==
531
+ "number"
532
+ ) {
533
+ throw new Error("Retained-process mutation settlement lacked its exact admission");
534
+ }
535
+ await verifyRetainedProcessMutationSettlement(services.db, {
536
+ accountId: fence.accountId,
537
+ workspaceId: ids.workspaceId,
538
+ sessionId: ids.sessionId,
539
+ processId: process.id,
540
+ admission: admission as SandboxWorkspaceMutationAdmission,
541
+ operation: op,
542
+ outcome,
543
+ });
544
+ };
545
+ }
546
+
547
+ function settleRetainedProcessForTurn(
548
+ services: RoutingWiringServices,
549
+ ids: RoutingWiringIds,
550
+ ):
551
+ | ((input: {
552
+ backend: ResolvedActiveBackend;
553
+ process: RoutingRetainedProcess;
554
+ proof: RoutingRetainedProcessTerminalProof;
555
+ }) => Promise<void>)
556
+ | undefined {
557
+ const fence = ids.workspaceMutationFence;
558
+ if (!fence) return undefined;
559
+ return async ({ process, proof }) => {
560
+ await settleRetainedProcess(services.db, {
561
+ accountId: fence.accountId,
562
+ workspaceId: ids.workspaceId,
563
+ sessionId: ids.sessionId,
564
+ processId: process.id,
565
+ outcome: proof.outcome,
566
+ exitCode: proof.exitCode,
567
+ reason: proof.reason,
568
+ idleGraceMs: services.settings.sandboxIdleGraceMs,
569
+ });
570
+ };
571
+ }
572
+
132
573
  /** Build the selfhosted `ControlRpc` over the events bus's request/reply
133
574
  * connection. A null bus / unconfigured NATS yields a NatsControlRpc whose
134
575
  * connection factory returns null → agent_offline on every op (never a throw). */
@@ -160,6 +601,11 @@ export function wrapTurnBoxWithRouting(
160
601
  established: EstablishedSandboxSession,
161
602
  ): EstablishedSandboxSession {
162
603
  const { db, settings, bus, onOp } = services;
604
+ const beforeMutation = beforePersistableHomeMutation(services, ids, ids.homeLease);
605
+ const afterMutation = afterPersistableHomeMutation(services, ids, ids.homeLease);
606
+ const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
607
+ const afterProcessMutation = afterRetainedProcessMutation(services, ids);
608
+ const settleProcess = settleRetainedProcessForTurn(services, ids);
163
609
  const resolver = makeActiveBackendResolver({
164
610
  workspaceId: ids.workspaceId,
165
611
  defaultBackend: established.session as RoutableBackendSession,
@@ -210,6 +656,22 @@ export function wrapTurnBoxWithRouting(
210
656
  // the pinned machine — passing defaultIsHome:false makes the null branch throw typed
211
657
  // `home_unavailable_this_turn` instead. Forward the explicit boolean (including false).
212
658
  ...(ids.defaultIsHome !== undefined ? { defaultIsHome: ids.defaultIsHome } : {}),
659
+ ...(ids.homeLease
660
+ ? {
661
+ resolveDefaultBackend: () =>
662
+ resolveCurrentHomeBackend(
663
+ services,
664
+ {
665
+ workspaceId: ids.workspaceId,
666
+ sessionId: ids.sessionId,
667
+ accountId: ids.homeLease!.accountId,
668
+ sandboxGroupId: ids.homeLease!.sandboxGroupId,
669
+ backend: ids.homeLease!.backend,
670
+ },
671
+ established,
672
+ ),
673
+ }
674
+ : {}),
213
675
  });
214
676
 
215
677
  const proxy = new RoutingSandboxSession({
@@ -223,12 +685,70 @@ export function wrapTurnBoxWithRouting(
223
685
  session: established.session as RoutableBackendSession,
224
686
  sandboxId: null,
225
687
  kind: established.backendId,
688
+ ...(ids.homeLease
689
+ ? {
690
+ leaseEpoch: ids.homeLease.leaseEpoch,
691
+ providerInstanceId: ids.homeLease.instanceId,
692
+ }
693
+ : {}),
226
694
  },
227
695
  readPointer: async () => {
696
+ if (!routingEnabled(settings)) {
697
+ return { activeSandboxId: null, activeEpoch: 0 };
698
+ }
228
699
  const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
229
700
  return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
230
701
  },
231
702
  resolveActiveBackend: resolver,
703
+ ...(beforeMutation ? { beforeMutation } : {}),
704
+ ...(afterMutation ? { afterMutation } : {}),
705
+ ...(beforeProcessMutation ? { beforeProcessMutation } : {}),
706
+ ...(afterProcessMutation ? { afterProcessMutation } : {}),
707
+ ...(settleProcess ? { settleProcess } : {}),
708
+ ...(ids.homeLease
709
+ ? {
710
+ onDefaultBackendError: async ({
711
+ error,
712
+ backend,
713
+ }: {
714
+ error: unknown;
715
+ backend: ResolvedActiveBackend;
716
+ }) => {
717
+ const home = ids.homeLease!;
718
+ if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
719
+ const expectedEpoch = backend.leaseEpoch ?? home.leaseEpoch;
720
+ const expectedInstanceId = backend.providerInstanceId ?? home.instanceId;
721
+ const marked = await markWarmLeaseInstanceLost(db, {
722
+ accountId: home.accountId,
723
+ workspaceId: ids.workspaceId,
724
+ sandboxGroupId: home.sandboxGroupId,
725
+ expectedEpoch,
726
+ expectedInstanceId,
727
+ diagnostic: "provider_not_found_during_routed_operation",
728
+ });
729
+ if (marked.status === "marked") {
730
+ await services.onHomeSandboxLost?.({
731
+ sandboxGroupId: home.sandboxGroupId,
732
+ instanceId: expectedInstanceId,
733
+ leaseEpoch: marked.lease.leaseEpoch,
734
+ });
735
+ }
736
+ const lease = marked.lease;
737
+ const restore = lease?.recovery.restore.status;
738
+ return {
739
+ leaseEpoch: lease?.leaseEpoch ?? expectedEpoch,
740
+ recovery:
741
+ marked.status === "stale"
742
+ ? ("superseded" as const)
743
+ : restore === "pending"
744
+ ? ("pending" as const)
745
+ : restore === "degraded"
746
+ ? ("degraded" as const)
747
+ : ("unrecoverable" as const),
748
+ };
749
+ },
750
+ }
751
+ : {}),
232
752
  });
233
753
 
234
754
  return { ...established, session: proxy };
@@ -241,10 +761,22 @@ export function wrapLazyTurnBoxWithRouting(
241
761
  client: EstablishedSandboxSession["client"];
242
762
  backendId: string;
243
763
  agentDefaultManifest: unknown;
244
- provisioner: { get(): Promise<{ established: EstablishedSandboxSession }> };
764
+ provisioner: {
765
+ get(): Promise<{ established: EstablishedSandboxSession; leaseEpoch?: number }>;
766
+ };
767
+ homeLeaseIdentity?: {
768
+ accountId: string;
769
+ sandboxGroupId: string;
770
+ backend: string;
771
+ };
245
772
  },
246
773
  ): EstablishedSandboxSession {
247
774
  const { db, settings, bus, onOp } = services;
775
+ const beforeMutation = beforePersistableHomeMutation(services, ids, args.homeLeaseIdentity);
776
+ const afterMutation = afterPersistableHomeMutation(services, ids, args.homeLeaseIdentity);
777
+ const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
778
+ const afterProcessMutation = afterRetainedProcessMutation(services, ids);
779
+ const settleProcess = settleRetainedProcessForTurn(services, ids);
248
780
  const syntheticSession: RoutableBackendSession = {
249
781
  state: { manifest: args.agentDefaultManifest },
250
782
  };
@@ -287,6 +819,19 @@ export function wrapLazyTurnBoxWithRouting(
287
819
  resolveActiveBackend: async (pointer) => {
288
820
  if (pointer.activeSandboxId === null || !routingEnabled(settings)) {
289
821
  const provisioned = await args.provisioner.get();
822
+ if (args.homeLeaseIdentity && provisioned.leaseEpoch !== undefined) {
823
+ return resolveCurrentHomeBackend(
824
+ services,
825
+ {
826
+ workspaceId: ids.workspaceId,
827
+ sessionId: ids.sessionId,
828
+ accountId: args.homeLeaseIdentity.accountId,
829
+ sandboxGroupId: args.homeLeaseIdentity.sandboxGroupId,
830
+ backend: args.homeLeaseIdentity.backend,
831
+ },
832
+ provisioned.established,
833
+ );
834
+ }
290
835
  return {
291
836
  session: provisioned.established.session as RoutableBackendSession,
292
837
  sandboxId: null,
@@ -295,6 +840,56 @@ export function wrapLazyTurnBoxWithRouting(
295
840
  }
296
841
  return routedResolver(pointer);
297
842
  },
843
+ ...(beforeMutation ? { beforeMutation } : {}),
844
+ ...(afterMutation ? { afterMutation } : {}),
845
+ ...(beforeProcessMutation ? { beforeProcessMutation } : {}),
846
+ ...(afterProcessMutation ? { afterProcessMutation } : {}),
847
+ ...(settleProcess ? { settleProcess } : {}),
848
+ ...(args.homeLeaseIdentity
849
+ ? {
850
+ onDefaultBackendError: async ({
851
+ error,
852
+ backend,
853
+ }: {
854
+ error: unknown;
855
+ backend: ResolvedActiveBackend;
856
+ }) => {
857
+ const home = args.homeLeaseIdentity!;
858
+ if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
859
+ if (backend.leaseEpoch === undefined || backend.providerInstanceId === undefined) {
860
+ return null;
861
+ }
862
+ const marked = await markWarmLeaseInstanceLost(db, {
863
+ accountId: home.accountId,
864
+ workspaceId: ids.workspaceId,
865
+ sandboxGroupId: home.sandboxGroupId,
866
+ expectedEpoch: backend.leaseEpoch,
867
+ expectedInstanceId: backend.providerInstanceId,
868
+ diagnostic: "provider_not_found_during_routed_operation",
869
+ });
870
+ if (marked.status === "marked") {
871
+ await services.onHomeSandboxLost?.({
872
+ sandboxGroupId: home.sandboxGroupId,
873
+ instanceId: backend.providerInstanceId,
874
+ leaseEpoch: marked.lease.leaseEpoch,
875
+ });
876
+ }
877
+ const lease = marked.lease;
878
+ const restore = lease?.recovery.restore.status;
879
+ return {
880
+ leaseEpoch: lease?.leaseEpoch ?? backend.leaseEpoch,
881
+ recovery:
882
+ marked.status === "stale"
883
+ ? ("superseded" as const)
884
+ : restore === "pending"
885
+ ? ("pending" as const)
886
+ : restore === "degraded"
887
+ ? ("degraded" as const)
888
+ : ("unrecoverable" as const),
889
+ };
890
+ },
891
+ }
892
+ : {}),
298
893
  });
299
894
 
300
895
  return {
@@ -34,9 +34,10 @@ export const activity = proxyActivities<WorkflowControlActivities>({
34
34
  },
35
35
  });
36
36
 
37
- /** Goal continuation is advisory at an idle boundary. A transient failure gets
38
- * a short retry window, then records an explicit delayed outbox wake instead
39
- * of relying on an unrelated future mutation. */
37
+ /** Goal continuation evaluates a durable Postgres obligation at an idle
38
+ * boundary. A transient failure gets a short retry window, then records an
39
+ * explicit delayed outbox wake instead of relying on an unrelated mutation or
40
+ * keeping workflow history alive with polling. */
40
41
  export const goalActivity = proxyActivities<Pick<typeof activities, "maybeContinueGoal">>({
41
42
  startToCloseTimeout: "30 seconds",
42
43
  retry: {