@opengeni/worker-bundle 0.13.10 → 0.14.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,10 +10,10 @@ import { type SecretForRedaction } from "./redaction";
10
10
  import { type TurnCodexAccount } from "./run-input";
11
11
  import type { ActivityServices, EscapedMcpTimeoutRecoveryDetail, RunAgentTurnInput, RunAgentTurnResult, SessionAttemptQuiescenceProof } from "./types";
12
12
  import { type TurnSandboxLeaseHolderId } from "../sandbox-resume";
13
- import { type LatencyMode, type SessionEvent, type SessionEventType, type SessionTurn, type ToolAuthNeededPayload, type TurnExecutionPolicyV1, type TurnInitiator } from "@opengeni/contracts";
13
+ import { type LatencyMode, type SessionEvent, type SessionEventType, type SessionTurn, type ToolAuthNeededPayload, type TurnExecutionPolicyV1 } from "@opengeni/contracts";
14
14
  export declare const PROVIDER_BACKPRESSURE_DELAY_MS = 60000;
15
- /** Personal connection authority follows the immutable human turn initiator, never the worker. */
16
- export declare function credentialSubjectIdForTurnInitiator(initiator: Pick<TurnInitiator, "kind" | "subjectId">): string | undefined;
15
+ /** Broad personal lookup is allowed only for a direct human/API command. */
16
+ export declare function credentialSubjectIdForTurnInitiator(turn: Pick<SessionTurn, "source" | "initiator" | "initiatorContext">): string | undefined;
17
17
  /**
18
18
  * Direct authenticated-human work is the persisted human source with subject
19
19
  * authority and no inherited agent or legacy provenance. Use this complete
package/dist/index.d.ts CHANGED
@@ -62,8 +62,8 @@ export declare function createWorkerWorkflowSignaler(settings: Settings, db: Dat
62
62
  /**
63
63
  * Register the ONE global reaper Temporal Schedule (the sole liveness/GC/cost-stop
64
64
  * driver — P1.3 / OD-3) and durable system-update outbox repair cadence. With
65
- * sandbox ownership off the activity performs only bounded DB outbox repair;
66
- * it never reads/terminates sandbox leases.
65
+ * sandbox ownership off the activity performs bounded DB outbox repair and
66
+ * read-only observability projections; it never mutates or terminates sandbox leases.
67
67
  *
68
68
  * The Schedule fires sandboxReaperWorkflow on the worker's global task queue
69
69
  * every settings.sandboxLeaseReaperPeriodMs (the SAME cadence the boot invariant
package/dist/index.js CHANGED
@@ -71,6 +71,7 @@ import {
71
71
  CODEX_CREDENTIAL_LEASE_TTL_MS,
72
72
  getCodexRotationSettings,
73
73
  getWorkspaceModelPolicy as getWorkspaceModelPolicy2,
74
+ getWorkspaceGrant,
74
75
  listCodexAccountStatuses as listCodexAccountStatuses2,
75
76
  fetchCodexUsageForAccount as fetchCodexUsageForAccount2,
76
77
  getSessionCodexState,
@@ -115,6 +116,7 @@ import {
115
116
  markSessionAttemptQuiesced
116
117
  } from "@opengeni/db";
117
118
  import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
119
+ import { sandboxOperationMetricObserver } from "@opengeni/observability";
118
120
  import {
119
121
  sandboxStateEntryFromRunState,
120
122
  maxTurnsExceededRunState,
@@ -1026,6 +1028,8 @@ function workflowIdForSession(sessionId) {
1026
1028
  import {
1027
1029
  assertSessionAllowsProductModel,
1028
1030
  defaultSessionMcpServerIds,
1031
+ directPersonalConnectionSubjectId,
1032
+ withFrozenPersonalConnectionDelegations,
1029
1033
  resolveSessionToolPolicy
1030
1034
  } from "@opengeni/core";
1031
1035
 
@@ -2802,6 +2806,7 @@ function mergeRigDefaultVariableSetEnvironment(rigDefaultValues, sessionValues)
2802
2806
  import {
2803
2807
  getSession,
2804
2808
  getSessionGoal as getSessionGoal2,
2809
+ getSessionParentPersonalConnectionDelegations,
2805
2810
  addSessionSystemUpdateWithSourceMutation,
2806
2811
  claimPendingSessionSystemUpdateOutbox,
2807
2812
  claimPendingSessionWorkflowWakes,
@@ -2820,6 +2825,11 @@ async function notifyParentOfChildIdle(svc, workspaceId, childSessionId, episode
2820
2825
  const goal = await getSessionGoal2(svc.db, workspaceId, childSessionId);
2821
2826
  const clientEventId = `child-completion:${childSessionId}:${episodeKey}`;
2822
2827
  const payload = childCompletionPayload(child, goal);
2828
+ const personalConnectionDelegations = await getSessionParentPersonalConnectionDelegations(
2829
+ svc.db,
2830
+ workspaceId,
2831
+ childSessionId
2832
+ );
2823
2833
  const outbox = await getOrCreateSessionSystemUpdateOutbox(svc.db, {
2824
2834
  accountId: child.accountId,
2825
2835
  workspaceId,
@@ -2831,7 +2841,8 @@ async function notifyParentOfChildIdle(svc, workspaceId, childSessionId, episode
2831
2841
  dedupeKey: clientEventId,
2832
2842
  summary: childCompletionSummary(child, goal, "idle"),
2833
2843
  payload,
2834
- lineage: { childSessionId: child.id, parentSessionId: child.parentSessionId }
2844
+ lineage: { childSessionId: child.id, parentSessionId: child.parentSessionId },
2845
+ personalConnectionDelegations
2835
2846
  });
2836
2847
  if (outbox.status === "delivered") {
2837
2848
  return;
@@ -2881,7 +2892,8 @@ async function deliverParentSystemUpdateOutbox(svc, outbox) {
2881
2892
  dedupeKey: outbox.dedupeKey,
2882
2893
  summary: outbox.summary,
2883
2894
  payload: outbox.payload,
2884
- lineage: outbox.lineage
2895
+ lineage: outbox.lineage,
2896
+ personalConnectionDelegations: outbox.personalConnectionDelegations
2885
2897
  },
2886
2898
  async (tx) => {
2887
2899
  await markSessionSystemUpdateOutboxDeliveredInTransaction(tx, outbox);
@@ -4474,6 +4486,7 @@ function wrapTurnBoxWithRouting(services, ids, established) {
4474
4486
  return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
4475
4487
  },
4476
4488
  resolveActiveBackend: resolver,
4489
+ ...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
4477
4490
  ...beforeMutation ? { beforeMutation } : {},
4478
4491
  ...afterMutation ? { afterMutation } : {},
4479
4492
  ...beforeProcessMutation ? { beforeProcessMutation } : {},
@@ -4581,6 +4594,7 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
4581
4594
  }
4582
4595
  return routedResolver(pointer);
4583
4596
  },
4597
+ ...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
4584
4598
  ...beforeMutation ? { beforeMutation } : {},
4585
4599
  ...afterMutation ? { afterMutation } : {},
4586
4600
  ...beforeProcessMutation ? { beforeProcessMutation } : {},
@@ -4690,6 +4704,7 @@ function makeTurnOpJournal(context, details) {
4690
4704
  // src/observability-metrics.ts
4691
4705
  import { createHash as createHash2 } from "crypto";
4692
4706
  import { errorCodeToJSON } from "@opengeni/agent-proto";
4707
+ import { SandboxBackend } from "@opengeni/contracts";
4693
4708
  import {
4694
4709
  SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
4695
4710
  modelUsageTokenCountOrNull
@@ -4697,7 +4712,6 @@ import {
4697
4712
  var turnTrackers = /* @__PURE__ */ new WeakMap();
4698
4713
  var creditBalanceGaugeAccounts = /* @__PURE__ */ new WeakMap();
4699
4714
  var modelCacheCounterTotals = /* @__PURE__ */ new WeakMap();
4700
- var retainedProcessTerminalBacklogTotals = /* @__PURE__ */ new WeakMap();
4701
4715
  var MAX_MODEL_CACHE_COUNTER_TOTAL = 1e12;
4702
4716
  function observabilityEventLogger(observability) {
4703
4717
  return {
@@ -4949,6 +4963,21 @@ function recordSandboxLeaseGauges(observability, counts) {
4949
4963
  });
4950
4964
  }
4951
4965
  }
4966
+ function recordSandboxInventoryProjectionSuccess(observability, domain, timestampSeconds = Date.now() / 1e3) {
4967
+ observability.setGauge({
4968
+ name: "opengeni_sandbox_inventory_refresh_timestamp_seconds",
4969
+ help: "Unix timestamp of the last complete successful sandbox inventory projection by domain.",
4970
+ labels: { domain },
4971
+ value: timestampSeconds
4972
+ });
4973
+ }
4974
+ function recordSandboxInventoryProjectionFailure(observability, domain) {
4975
+ observability.incrementCounter({
4976
+ name: "opengeni_sandbox_inventory_refresh_failures_total",
4977
+ help: "Failed sandbox inventory projection refreshes by domain.",
4978
+ labels: { domain }
4979
+ });
4980
+ }
4952
4981
  function recordCreditBalanceGauges(observability, balances) {
4953
4982
  const previous = creditBalanceGaugeAccounts.get(observability) ?? /* @__PURE__ */ new Set();
4954
4983
  const current = /* @__PURE__ */ new Set();
@@ -5062,9 +5091,7 @@ function recordRetainedProcessInventoryGauges(observability, counts) {
5062
5091
  current.active += Math.max(0, count.activeCount);
5063
5092
  current.terminal += Math.max(0, count.terminalOwnerCount);
5064
5093
  }
5065
- let terminalTotal = 0;
5066
5094
  for (const [ownerState, count] of normalized) {
5067
- terminalTotal += count.terminal;
5068
5095
  observability.setGauge({
5069
5096
  name: "opengeni_retained_processes_active",
5070
5097
  help: "Current active retained provider processes by durable owner state.",
@@ -5078,28 +5105,8 @@ function recordRetainedProcessInventoryGauges(observability, counts) {
5078
5105
  value: count.terminal
5079
5106
  });
5080
5107
  }
5081
- const previous = retainedProcessTerminalBacklogTotals.get(observability);
5082
- if (previous !== void 0 && terminalTotal > previous) {
5083
- observability.incrementCounter({
5084
- name: "opengeni_retained_process_terminal_owner_backlog_growth_total",
5085
- help: "Positive growth in terminal-owner retained-process backlog between app samples.",
5086
- amount: terminalTotal - previous
5087
- });
5088
- }
5089
- retainedProcessTerminalBacklogTotals.set(observability, terminalTotal);
5090
- }
5091
- var EXPIRED_DRAINING_BACKENDS = [
5092
- "none",
5093
- "local",
5094
- "docker",
5095
- "modal",
5096
- "e2b",
5097
- "daytona",
5098
- "blaxel",
5099
- "runloop",
5100
- "selfhosted",
5101
- "unknown"
5102
- ];
5108
+ }
5109
+ var EXPIRED_DRAINING_BACKENDS = [...SandboxBackend.options, "unknown"];
5103
5110
  var EXPIRED_DRAINING_BACKEND_SET = new Set(EXPIRED_DRAINING_BACKENDS);
5104
5111
  var EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"];
5105
5112
  function recordExpiredDrainingSandboxLeaseGauges(observability, counts) {
@@ -6288,11 +6295,11 @@ import {
6288
6295
  } from "@opengeni/contracts";
6289
6296
  import { createHash as createHash4, randomUUID as randomUUID2 } from "crypto";
6290
6297
  var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
6291
- function credentialSubjectIdForTurnInitiator(initiator) {
6292
- return initiator.kind === "subject" ? initiator.subjectId : void 0;
6298
+ function credentialSubjectIdForTurnInitiator(turn) {
6299
+ return directPersonalConnectionSubjectId(turn);
6293
6300
  }
6294
6301
  function isDirectHumanTurnInitiation(turn) {
6295
- if (turn.source !== "user" || turn.initiator.kind !== "subject") {
6302
+ if (turn.source !== "user" && turn.source !== "api" || turn.initiator.kind !== "subject") {
6296
6303
  return false;
6297
6304
  }
6298
6305
  return !["via", "viaTruncated", "provenanceError", "backfill"].some(
@@ -7291,6 +7298,7 @@ function createRunAgentTurnActivity(services) {
7291
7298
  const machineOpObserver = makeMachineOpObserver(
7292
7299
  runtimeMetricsHooksForObservability(observability)
7293
7300
  );
7301
+ const sandboxOperationObserver = sandboxOperationMetricObserver(observability);
7294
7302
  let isCodexTurn = false;
7295
7303
  let isExternallyBilledTurn = false;
7296
7304
  let executionGeneration = 0;
@@ -9459,6 +9467,7 @@ function createRunAgentTurnActivity(services) {
9459
9467
  settings,
9460
9468
  bus,
9461
9469
  onOp: machineOpObserver.observer,
9470
+ onSandboxOperation: sandboxOperationObserver,
9462
9471
  opJournal
9463
9472
  },
9464
9473
  {
@@ -9484,6 +9493,7 @@ function createRunAgentTurnActivity(services) {
9484
9493
  settings,
9485
9494
  bus,
9486
9495
  onOp: machineOpObserver.observer,
9496
+ onSandboxOperation: sandboxOperationObserver,
9487
9497
  onHomeSandboxRebound
9488
9498
  },
9489
9499
  {
@@ -9572,6 +9582,7 @@ function createRunAgentTurnActivity(services) {
9572
9582
  db,
9573
9583
  settings,
9574
9584
  bus,
9585
+ onSandboxOperation: sandboxOperationObserver,
9575
9586
  onHomeSandboxLost: publishSandboxLost,
9576
9587
  onHomeSandboxRebound
9577
9588
  },
@@ -9630,8 +9641,23 @@ function createRunAgentTurnActivity(services) {
9630
9641
  attemptId: input.attemptId,
9631
9642
  turn
9632
9643
  });
9644
+ const personalConnectionDelegations = turn.personalConnectionDelegations;
9645
+ const delegatedMembershipChecks = /* @__PURE__ */ new Map();
9646
+ const delegatedOwnerHasMembership = async (subjectId) => {
9647
+ const existing = delegatedMembershipChecks.get(subjectId);
9648
+ if (existing) return await existing;
9649
+ const check = getWorkspaceGrant(db, subjectId, input.workspaceId).then(Boolean);
9650
+ delegatedMembershipChecks.set(subjectId, check);
9651
+ return await check;
9652
+ };
9653
+ const resolveFrozenCredential = withFrozenPersonalConnectionDelegations({
9654
+ resolveCredential: rawResolveCredential,
9655
+ settings: runSettings,
9656
+ personalConnectionDelegations,
9657
+ ownerHasWorkspaceMembership: delegatedOwnerHasMembership
9658
+ });
9633
9659
  const resolveCredential = async (request) => {
9634
- const result = await rawResolveCredential(request);
9660
+ const result = await resolveFrozenCredential(request);
9635
9661
  if (result.status === "ok") {
9636
9662
  registerSecretRedactions(
9637
9663
  headerSecretRedactions("MCP", result.headers, Object.keys(result.headers))
@@ -9639,7 +9665,7 @@ function createRunAgentTurnActivity(services) {
9639
9665
  }
9640
9666
  return result;
9641
9667
  };
9642
- const credentialSubjectId = credentialSubjectIdForTurnInitiator(turn.initiator);
9668
+ const credentialSubjectId = credentialSubjectIdForTurnInitiator(turn);
9643
9669
  preparedTools = await waitForTurnOperation(
9644
9670
  withCodex(
9645
9671
  () => runtime.prepareTools(runSettings, turnTools, {
@@ -9970,6 +9996,7 @@ function createRunAgentTurnActivity(services) {
9970
9996
  settings,
9971
9997
  bus,
9972
9998
  onOp: machineOpObserver.observer,
9999
+ onSandboxOperation: sandboxOperationObserver,
9973
10000
  onHomeSandboxLost: publishSandboxLost,
9974
10001
  onHomeSandboxRebound
9975
10002
  },
@@ -12724,6 +12751,7 @@ function createSandboxLeaseActivities(services, options = {}) {
12724
12751
  deleted: expiredSessionListSnapshots
12725
12752
  });
12726
12753
  }
12754
+ await refreshQueueLeaseAndCreditGauges(db, observability);
12727
12755
  if (!settings.sandboxOwnershipEnabled) {
12728
12756
  if (parentUpdates.claimed > 0) {
12729
12757
  observability.info("system-update outbox reconciled", parentUpdates);
@@ -12737,7 +12765,6 @@ function createSandboxLeaseActivities(services, options = {}) {
12737
12765
  modalOrphansTerminated: 0
12738
12766
  };
12739
12767
  }
12740
- await refreshQueueLeaseAndCreditGauges(db, observability);
12741
12768
  const rotationsRequested = await requestDueSandboxRotationsGlobal(
12742
12769
  db,
12743
12770
  settings.sandboxRotationLeadMs,
@@ -13453,54 +13480,65 @@ async function refreshQueueLeaseAndCreditGauges(db, observability) {
13453
13480
  error: error instanceof Error ? error.message : String(error)
13454
13481
  });
13455
13482
  }
13456
- try {
13483
+ await refreshSandboxInventoryGauge(observability, "leases", "sandbox-lease", async () => {
13457
13484
  recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
13458
- } catch (error) {
13459
- observability.warn("sandbox reaper: sandbox-lease gauge refresh failed", {
13460
- error: error instanceof Error ? error.message : String(error)
13461
- });
13462
- }
13463
- try {
13464
- recordSandboxCheckpointArtifactGauges(
13465
- observability,
13466
- await countSandboxCheckpointArtifactsByState(db)
13467
- );
13468
- } catch (error) {
13469
- observability.warn("sandbox reaper: checkpoint-artifact gauge refresh failed", {
13470
- error: error instanceof Error ? error.message : String(error)
13471
- });
13472
- }
13473
- try {
13474
- recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
13475
- } catch (error) {
13476
- observability.warn("sandbox reaper: rotation-backlog gauge refresh failed", {
13477
- error: error instanceof Error ? error.message : String(error)
13478
- });
13479
- }
13480
- try {
13481
- recordRetainedProcessInventoryGauges(
13482
- observability,
13483
- await countActiveRetainedProcessesByOwnerState(db)
13484
- );
13485
- } catch (error) {
13486
- observability.warn("sandbox reaper: retained-process gauge refresh failed", {
13487
- error: error instanceof Error ? error.message : String(error)
13488
- });
13489
- }
13485
+ });
13486
+ await refreshSandboxInventoryGauge(
13487
+ observability,
13488
+ "checkpoint_artifacts",
13489
+ "checkpoint-artifact",
13490
+ async () => {
13491
+ recordSandboxCheckpointArtifactGauges(
13492
+ observability,
13493
+ await countSandboxCheckpointArtifactsByState(db)
13494
+ );
13495
+ }
13496
+ );
13497
+ await refreshSandboxInventoryGauge(
13498
+ observability,
13499
+ "rotation_backlog",
13500
+ "rotation-backlog",
13501
+ async () => {
13502
+ recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
13503
+ }
13504
+ );
13505
+ await refreshSandboxInventoryGauge(
13506
+ observability,
13507
+ "retained_processes",
13508
+ "retained-process",
13509
+ async () => {
13510
+ recordRetainedProcessInventoryGauges(
13511
+ observability,
13512
+ await countActiveRetainedProcessesByOwnerState(db)
13513
+ );
13514
+ }
13515
+ );
13516
+ await refreshSandboxInventoryGauge(
13517
+ observability,
13518
+ "expired_drains",
13519
+ "expired-draining",
13520
+ async () => {
13521
+ recordExpiredDrainingSandboxLeaseGauges(
13522
+ observability,
13523
+ await countExpiredDrainingSandboxLeases(db)
13524
+ );
13525
+ }
13526
+ );
13490
13527
  try {
13491
- recordExpiredDrainingSandboxLeaseGauges(
13492
- observability,
13493
- await countExpiredDrainingSandboxLeases(db)
13494
- );
13528
+ recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
13495
13529
  } catch (error) {
13496
- observability.warn("sandbox reaper: expired-draining gauge refresh failed", {
13530
+ observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
13497
13531
  error: error instanceof Error ? error.message : String(error)
13498
13532
  });
13499
13533
  }
13534
+ }
13535
+ async function refreshSandboxInventoryGauge(observability, domain, diagnosticName, refresh) {
13500
13536
  try {
13501
- recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
13537
+ await refresh();
13538
+ recordSandboxInventoryProjectionSuccess(observability, domain);
13502
13539
  } catch (error) {
13503
- observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
13540
+ recordSandboxInventoryProjectionFailure(observability, domain);
13541
+ observability.warn(`sandbox reaper: ${diagnosticName} gauge refresh failed`, {
13504
13542
  error: error instanceof Error ? error.message : String(error)
13505
13543
  });
13506
13544
  }
@@ -13997,6 +14035,7 @@ import {
13997
14035
  enqueueSessionWorkflowWakeIfRunnable as enqueueSessionWorkflowWakeIfRunnable2,
13998
14036
  getBillingBalance as getBillingBalance5,
13999
14037
  getScheduledTask,
14038
+ getScheduledTaskPersonalConnectionDelegations,
14000
14039
  getRig,
14001
14040
  getVariableSet,
14002
14041
  isCodexBilledTurn as isCodexBilledTurn2,
@@ -14023,6 +14062,11 @@ function createScheduledTaskActivities(services) {
14023
14062
  if (!task) {
14024
14063
  return { action: "deleted" };
14025
14064
  }
14065
+ const taskPersonalConnectionDelegations = await getScheduledTaskPersonalConnectionDelegations(
14066
+ db,
14067
+ task.workspaceId,
14068
+ task.id
14069
+ );
14026
14070
  const slackBotConnection = task.agentConfig.slackBotConnectionId ? await requireOpenGeniSlackBotConnection(
14027
14071
  db,
14028
14072
  task.workspaceId,
@@ -14168,6 +14212,7 @@ function createScheduledTaskActivities(services) {
14168
14212
  variableSetId: task.variableSetId ?? null,
14169
14213
  rigId: frozenRigId,
14170
14214
  rigVersionId: frozenRigVersionId,
14215
+ personalConnectionDelegations: taskPersonalConnectionDelegations,
14171
14216
  maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
14172
14217
  // The durable agent config was privilege-checked when the task
14173
14218
  // was created/updated. Preserve that explicit policy if a broader
@@ -14250,7 +14295,8 @@ function createScheduledTaskActivities(services) {
14250
14295
  task.id,
14251
14296
  run.id
14252
14297
  ),
14253
- lineage: { scheduledTaskId: task.id, scheduledTaskRunId: run.id }
14298
+ lineage: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
14299
+ personalConnectionDelegations: taskPersonalConnectionDelegations
14254
14300
  },
14255
14301
  async (tx, wakeEventId) => {
14256
14302
  if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");
@@ -14349,7 +14395,8 @@ function createScheduledTaskActivities(services) {
14349
14395
  task.id,
14350
14396
  run.id
14351
14397
  ),
14352
- lineage: { scheduledTaskId: task.id, scheduledTaskRunId: run.id }
14398
+ lineage: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
14399
+ personalConnectionDelegations: taskPersonalConnectionDelegations
14353
14400
  },
14354
14401
  async (tx, wakeEventId) => {
14355
14402
  if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");