@opengeni/worker-bundle 0.11.2 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -18,7 +18,9 @@ import type { Settings } from "@opengeni/config";
18
18
  import {
19
19
  advanceWorkspaceGenerationForRetainedProcess,
20
20
  advanceWorkspaceGeneration,
21
+ getRetainedProcess,
21
22
  retainWorkspaceMutationProcess,
23
+ retainedProcessSettlementIdentity,
22
24
  settleRetainedProcess,
23
25
  verifyRetainedProcessMutationSettlement,
24
26
  verifyWorkspaceMutationSettlement,
@@ -26,6 +28,7 @@ import {
26
28
  markWarmLeaseInstanceLost,
27
29
  readLease,
28
30
  readActiveSandbox,
31
+ SandboxRetainedProcessPromotionFencedError,
29
32
  type Database,
30
33
  type SandboxWorkspaceMutationAdmission,
31
34
  } from "@opengeni/db";
@@ -47,6 +50,7 @@ import {
47
50
  type RoutableBackendSession,
48
51
  type RoutableSandbox,
49
52
  type ResolvedActiveBackend,
53
+ type RoutingMutationSettlementResult,
50
54
  type RoutingRetainedProcess,
51
55
  type RoutingRetainedProcessTerminalProof,
52
56
  type SelfhostedOpObserver,
@@ -175,7 +179,10 @@ function selfhostedResolverTimeouts(settings: Settings): {
175
179
  selfhostedExecTimeoutMs: number;
176
180
  } {
177
181
  const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
178
- return { selfhostedTimeoutMs: timeoutMs, selfhostedExecTimeoutMs: execTimeoutMs };
182
+ return {
183
+ selfhostedTimeoutMs: timeoutMs,
184
+ selfhostedExecTimeoutMs: execTimeoutMs,
185
+ };
179
186
  }
180
187
 
181
188
  type HomeRouteLeaseIdentity = {
@@ -232,7 +239,9 @@ function homeRouteRecoveryError(
232
239
  );
233
240
  }
234
241
 
235
- function providerIdentityFromResumeState(resumeState: Record<string, unknown>): string | null {
242
+ export function providerIdentityFromResumeState(
243
+ resumeState: Record<string, unknown>,
244
+ ): string | null {
236
245
  const sessionState =
237
246
  resumeState.sessionState && typeof resumeState.sessionState === "object"
238
247
  ? (resumeState.sessionState as Record<string, unknown>)
@@ -288,7 +297,10 @@ async function resolveCurrentHomeBackend(
288
297
  // A route epoch can advance without a provider replacement. Reuse the exact
289
298
  // established handle only when its identity still equals the durable one.
290
299
  if (lease.instanceId === established.instanceId) {
291
- services.onHomeSandboxRebound?.({ established, leaseEpoch: lease.leaseEpoch });
300
+ services.onHomeSandboxRebound?.({
301
+ established,
302
+ leaseEpoch: lease.leaseEpoch,
303
+ });
292
304
  return {
293
305
  session: established.session as RoutableBackendSession,
294
306
  sandboxId: null,
@@ -357,7 +369,10 @@ async function resolveCurrentHomeBackend(
357
369
  if (rebound.instanceId !== lease.instanceId || rebound.backendId !== resumeBackend) {
358
370
  throw homeRouteRecoveryError(lease, lease.leaseEpoch);
359
371
  }
360
- services.onHomeSandboxRebound?.({ established: rebound, leaseEpoch: lease.leaseEpoch });
372
+ services.onHomeSandboxRebound?.({
373
+ established: rebound,
374
+ leaseEpoch: lease.leaseEpoch,
375
+ });
361
376
  // Keep the resolver's mutable home reference on the latest verified handle so
362
377
  // a later route epoch does not resume the same replacement again. This is only
363
378
  // a worker-side handle update; the SDK-facing RoutingSandboxSession remains the
@@ -423,7 +438,7 @@ function afterPersistableHomeMutation(
423
438
  outcome: "resolved" | "rejected";
424
439
  result?: unknown;
425
440
  retainedProcess?: RoutingRetainedProcess;
426
- }) => Promise<void>)
441
+ }) => Promise<void | RoutingMutationSettlementResult>)
427
442
  | undefined {
428
443
  const fence = ids.workspaceMutationFence;
429
444
  if (!home || !fence) return undefined;
@@ -442,29 +457,58 @@ function afterPersistableHomeMutation(
442
457
  }
443
458
  const exactAdmission = admission as SandboxWorkspaceMutationAdmission;
444
459
  if (outcome === "resolved" && retainedProcess) {
445
- await retainWorkspaceMutationProcess(services.db, {
446
- accountId: fence.accountId,
447
- workspaceId: ids.workspaceId,
448
- sessionId: ids.sessionId,
449
- processId: retainedProcess.id,
450
- providerSessionId: retainedProcess.providerSessionId,
451
- admissionId: exactAdmission.id,
452
- admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
453
- operation: op,
454
- owner: {
455
- kind: "turn",
456
- turnId: fence.turnId,
457
- executionGeneration: fence.executionGeneration,
458
- attemptId: fence.attemptId,
459
- holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
460
- sandboxGroupId: home.sandboxGroupId,
461
- expectedEpoch: backend.leaseEpoch,
462
- expectedInstanceId: backend.providerInstanceId,
463
- routeKind: exactAdmission.routeKind,
464
- routeTargetId: exactAdmission.routeTargetId,
465
- routeEpoch: exactAdmission.routeEpoch,
466
- },
467
- });
460
+ try {
461
+ await retainWorkspaceMutationProcess(services.db, {
462
+ accountId: fence.accountId,
463
+ workspaceId: ids.workspaceId,
464
+ sessionId: ids.sessionId,
465
+ processId: retainedProcess.id,
466
+ providerSessionId: retainedProcess.providerSessionId,
467
+ admissionId: exactAdmission.id,
468
+ admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
469
+ operation: op,
470
+ owner: {
471
+ kind: "turn",
472
+ turnId: fence.turnId,
473
+ executionGeneration: fence.executionGeneration,
474
+ attemptId: fence.attemptId,
475
+ holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
476
+ sandboxGroupId: home.sandboxGroupId,
477
+ expectedEpoch: backend.leaseEpoch,
478
+ expectedInstanceId: backend.providerInstanceId,
479
+ routeKind: exactAdmission.routeKind,
480
+ routeTargetId: exactAdmission.routeTargetId,
481
+ routeEpoch: exactAdmission.routeEpoch,
482
+ },
483
+ });
484
+ } catch (error) {
485
+ if (!(error instanceof SandboxRetainedProcessPromotionFencedError)) throw error;
486
+ const durable = error.process;
487
+ if (
488
+ durable.id !== retainedProcess.id ||
489
+ durable.providerSessionId !== retainedProcess.providerSessionId ||
490
+ durable.sessionId !== ids.sessionId ||
491
+ durable.sandboxGroupId !== home.sandboxGroupId ||
492
+ durable.parentAdmissionId !== exactAdmission.id ||
493
+ durable.leaseEpoch !== backend.leaseEpoch ||
494
+ durable.providerInstanceId !== backend.providerInstanceId ||
495
+ durable.routeKind !== exactAdmission.routeKind ||
496
+ durable.routeTargetId !== exactAdmission.routeTargetId ||
497
+ durable.routeEpoch !== exactAdmission.routeEpoch ||
498
+ durable.state !== "active"
499
+ ) {
500
+ throw new Error("Durable retained-process promotion returned a mismatched identity", {
501
+ cause: error,
502
+ });
503
+ }
504
+ return {
505
+ status: "retained_process_durable_output_rejected",
506
+ retainedProcess: {
507
+ id: durable.id,
508
+ providerSessionId: durable.providerSessionId,
509
+ },
510
+ };
511
+ }
468
512
  return;
469
513
  }
470
514
  await verifyWorkspaceMutationSettlement(services.db, {
@@ -556,12 +600,30 @@ function settleRetainedProcessForTurn(
556
600
  | undefined {
557
601
  const fence = ids.workspaceMutationFence;
558
602
  if (!fence) return undefined;
559
- return async ({ process, proof }) => {
603
+ return async ({ backend, process, proof }) => {
604
+ const durable = await getRetainedProcess(services.db, {
605
+ workspaceId: ids.workspaceId,
606
+ sessionId: ids.sessionId,
607
+ processId: process.id,
608
+ });
609
+ if (
610
+ !durable ||
611
+ durable.providerSessionId !== process.providerSessionId ||
612
+ durable.providerBackend !== backend.kind ||
613
+ durable.providerInstanceId !== backend.providerInstanceId ||
614
+ durable.leaseEpoch !== backend.leaseEpoch ||
615
+ durable.routeKind !== (backend.sandboxId === null ? "home" : "active") ||
616
+ durable.routeTargetId !== backend.sandboxId ||
617
+ durable.routeEpoch !== backend.activeEpoch
618
+ ) {
619
+ throw new Error("Retained-process settlement lost its exact durable backend identity");
620
+ }
560
621
  await settleRetainedProcess(services.db, {
561
622
  accountId: fence.accountId,
562
623
  workspaceId: ids.workspaceId,
563
624
  sessionId: ids.sessionId,
564
625
  processId: process.id,
626
+ expected: retainedProcessSettlementIdentity(durable),
565
627
  outcome: proof.outcome,
566
628
  exitCode: proof.exitCode,
567
629
  reason: proof.reason,
@@ -762,7 +824,10 @@ export function wrapLazyTurnBoxWithRouting(
762
824
  backendId: string;
763
825
  agentDefaultManifest: unknown;
764
826
  provisioner: {
765
- get(): Promise<{ established: EstablishedSandboxSession; leaseEpoch?: number }>;
827
+ get(): Promise<{
828
+ established: EstablishedSandboxSession;
829
+ leaseEpoch?: number;
830
+ }>;
766
831
  };
767
832
  homeLeaseIdentity?: {
768
833
  accountId: string;
@@ -296,9 +296,11 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
296
296
  const capacityCheckBackstop =
297
297
  input.maxCapacityChecksPerRun ?? CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP;
298
298
  if (workflowInfo().continueAsNewSuggested || capacityChecksThisRun >= capacityCheckBackstop) {
299
- // The waiter/outbox is durable in Postgres. A fresh workflow run reads
300
- // it before goal continuation, reconstructs its timer, and turns any
301
- // unobserved wake revision into an immediate evaluation.
299
+ // The waiter and exact nonterminal turn are durable in Postgres, and
300
+ // the old attempt is already closed. A fresh workflow run reads the
301
+ // waiter before goal continuation, reconstructs its timer, and turns
302
+ // any unobserved wake revision into an immediate evaluation. This is
303
+ // an ownerless-attempt boundary, not a settled-turn boundary.
302
304
  // A quiescence proof is the one signal that is not merely a replaceable
303
305
  // wake hint. It may have arrived while the reconciliation activity was
304
306
  // running, so commit it before crossing this nested continue-as-new
@@ -324,11 +326,12 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
324
326
  // decision so no accepted physical fact can be dropped with run history.
325
327
  await persistPendingQuiescenceProofs();
326
328
  // History-overflow guard. The top of the loop is the only safe
327
- // continueAsNew boundary: no turn is mid-flight (every path that reaches a
328
- // new iteration settled or recovered its turn first). Every interruption
329
- // is already durable in Postgres, so ordinary Temporal signals are
330
- // replaceable wake hints and none must be carried into the next workflow
331
- // run. A
329
+ // continueAsNew boundary: no activity attempt owns a turn. A same logical
330
+ // turn may still be nonterminal (`recovering` or `waiting_capacity`), but
331
+ // its dispatch attempt has closed and all recovery/wait truth is durable
332
+ // in Postgres. Every interruption is already durable there too, so ordinary
333
+ // Temporal signals are replaceable wake hints and none must be carried into
334
+ // the next workflow run. A
332
335
  // buffered userMessage/queueChanged signal only
333
336
  // bumps `wakeups`, and its turn was written to Postgres BEFORE the signal
334
337
  // was sent, so the fresh run observes it on its first durable work peek
@@ -604,9 +607,9 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
604
607
  }
605
608
 
606
609
  if (outcome.kind === "failure") {
607
- // A capacity settlement may have committed just before the activity
608
- // transport/worker failed. Recover that durable boundary before generic
609
- // failSession can overwrite the capacity-idle session.
610
+ // A capacity wait may have committed just before the activity
611
+ // transport/worker failed. Recover that durable same-turn boundary
612
+ // before generic failSession can overwrite the nonterminal turn.
610
613
  {
611
614
  const capacityWait = await activity.getCodexCapacityWait({
612
615
  workspaceId,