@capybara-hq/kernel 0.1.0-rc.23 → 0.1.0-rc.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/bootstrap.json +25 -25
  2. package/dist/admin/admin.module.d.ts +2 -1
  3. package/dist/admin/admin.module.js +7 -3
  4. package/dist/admin/outbox-dead.controller.d.ts +23 -0
  5. package/dist/admin/outbox-dead.controller.js +62 -0
  6. package/dist/admin/outbox-dead.schemas.d.ts +8 -0
  7. package/dist/admin/outbox-dead.schemas.js +18 -0
  8. package/dist/admin/outbox-dead.service.d.ts +66 -0
  9. package/dist/admin/outbox-dead.service.js +179 -0
  10. package/dist/admin/overview.service.d.ts +40 -5
  11. package/dist/admin/overview.service.js +41 -7
  12. package/dist/agents/from-template.js +1 -1
  13. package/dist/background/finish.d.ts +5 -2
  14. package/dist/background/finish.js +14 -3
  15. package/dist/background/launch.d.ts +2 -1
  16. package/dist/background/launch.js +5 -2
  17. package/dist/background/store.d.ts +8 -0
  18. package/dist/background/store.js +24 -0
  19. package/dist/catalog/admin-catalog.controller.d.ts +21 -2
  20. package/dist/catalog/admin-catalog.controller.js +45 -6
  21. package/dist/catalog/admin-catalog.schemas.d.ts +5 -0
  22. package/dist/catalog/admin-catalog.schemas.js +8 -0
  23. package/dist/catalog/catalog.module.d.ts +2 -1
  24. package/dist/catalog/catalog.module.js +5 -1
  25. package/dist/catalog/marketplaces.service.d.ts +1 -1
  26. package/dist/catalog/platform-import-update.d.ts +50 -0
  27. package/dist/catalog/platform-import-update.js +116 -0
  28. package/dist/catalog/plugin-import.d.ts +36 -1
  29. package/dist/catalog/plugin-import.js +23 -1
  30. package/dist/catalog/plugin-upgrade.d.ts +18 -0
  31. package/dist/catalog/plugin-upgrade.js +19 -0
  32. package/dist/channels/channels.module.d.ts +5 -2
  33. package/dist/channels/channels.module.js +9 -2
  34. package/dist/channels/deliver-job.d.ts +25 -0
  35. package/dist/channels/deliver-job.js +25 -0
  36. package/dist/channels/inbound.d.ts +1 -1
  37. package/dist/channels/inbound.js +2 -15
  38. package/dist/channels/jobs.d.ts +28 -18
  39. package/dist/channels/jobs.js +63 -26
  40. package/dist/config/schema.d.ts +0 -3
  41. package/dist/config/schema.js +11 -20
  42. package/dist/conversations/agent-tree.d.ts +16 -0
  43. package/dist/conversations/agent-tree.js +36 -0
  44. package/dist/conversations/conversations.service.d.ts +44 -14
  45. package/dist/conversations/conversations.service.js +52 -38
  46. package/dist/conversations/event-log.d.ts +7 -10
  47. package/dist/conversations/event-log.js +24 -15
  48. package/dist/conversations/events.js +2 -1
  49. package/dist/conversations/queue.service.d.ts +8 -3
  50. package/dist/conversations/queue.service.js +37 -13
  51. package/dist/conversations/titles.d.ts +21 -8
  52. package/dist/conversations/titles.js +42 -15
  53. package/dist/db/kernel-schema.d.ts +26 -8
  54. package/dist/devices/devices.service.d.ts +12 -3
  55. package/dist/devices/devices.service.js +18 -3
  56. package/dist/errors.d.ts +11 -0
  57. package/dist/errors.js +11 -0
  58. package/dist/evals/run-replay.js +14 -8
  59. package/dist/extensions/extensions.module.js +3 -0
  60. package/dist/extensions/lifecycle.d.ts +6 -1
  61. package/dist/extensions/lifecycle.js +9 -2
  62. package/dist/extensions/usage-subscribers.d.ts +43 -0
  63. package/dist/extensions/usage-subscribers.js +108 -0
  64. package/dist/http/api.module.js +2 -0
  65. package/dist/index.d.ts +1 -0
  66. package/dist/index.js +1 -0
  67. package/dist/infra/cli/kernel-cli.module.js +5 -0
  68. package/dist/infra/events/kernel-events.d.ts +9 -35
  69. package/dist/infra/outbox/outbox-relay.module.d.ts +6 -0
  70. package/dist/infra/outbox/outbox-relay.module.js +18 -0
  71. package/dist/infra/outbox/outbox.dispatcher.d.ts +103 -0
  72. package/dist/infra/outbox/outbox.dispatcher.js +310 -0
  73. package/dist/infra/outbox/outbox.module.d.ts +24 -0
  74. package/dist/infra/outbox/outbox.module.js +50 -0
  75. package/dist/infra/outbox/outbox.relay.d.ts +31 -0
  76. package/dist/infra/outbox/outbox.relay.js +76 -0
  77. package/dist/infra/outbox/outbox.service.d.ts +51 -0
  78. package/dist/infra/outbox/outbox.service.js +131 -0
  79. package/dist/infra/outbox/registry.d.ts +120 -0
  80. package/dist/infra/outbox/registry.js +108 -0
  81. package/dist/infra/queue/queue-names.d.ts +8 -0
  82. package/dist/infra/queue/queue-names.js +16 -0
  83. package/dist/kernel.module.js +3 -0
  84. package/dist/local-devices/device-policy.service.d.ts +4 -0
  85. package/dist/local-devices/device-policy.service.js +61 -3
  86. package/dist/local-devices/machines.d.ts +13 -5
  87. package/dist/local-devices/machines.js +45 -7
  88. package/dist/local-devices/policy-summary.d.ts +44 -1
  89. package/dist/local-devices/policy-summary.js +69 -0
  90. package/dist/models/embeddings.d.ts +9 -9
  91. package/dist/models/embeddings.js +11 -18
  92. package/dist/models/models.module.js +3 -0
  93. package/dist/orgs/orgs.service.js +7 -0
  94. package/dist/overrides.d.ts +20 -3
  95. package/dist/overrides.js +10 -2
  96. package/dist/platform/exclusions.js +5 -0
  97. package/dist/platform/ops/admin.ops.js +18 -2
  98. package/dist/platform/ops/catalog-admin.ops.d.ts +3 -1
  99. package/dist/platform/ops/catalog-admin.ops.js +32 -4
  100. package/dist/platform/ops/outbox.ops.d.ts +21 -0
  101. package/dist/platform/ops/outbox.ops.js +125 -0
  102. package/dist/platform/ops/schedules.ops.js +40 -1
  103. package/dist/platform/platform-ops.module.d.ts +2 -1
  104. package/dist/platform/platform-ops.module.js +3 -0
  105. package/dist/platform/route-schemas.js +3 -0
  106. package/dist/roles/worker.module.js +9 -4
  107. package/dist/runtime/langgraph-loop.js +13 -5
  108. package/dist/runtime/limits.d.ts +15 -0
  109. package/dist/runtime/limits.js +25 -0
  110. package/dist/runtime/loop.d.ts +8 -3
  111. package/dist/runtime/loop.js +2 -1
  112. package/dist/runtime/maintenance.d.ts +4 -2
  113. package/dist/runtime/maintenance.js +17 -2
  114. package/dist/runtime/model-call.d.ts +14 -16
  115. package/dist/runtime/model-call.js +14 -24
  116. package/dist/runtime/queue-lost-jobs.d.ts +24 -0
  117. package/dist/runtime/queue-lost-jobs.js +51 -0
  118. package/dist/runtime/run-control.d.ts +42 -5
  119. package/dist/runtime/run-control.js +66 -12
  120. package/dist/runtime/run-finish.d.ts +39 -12
  121. package/dist/runtime/run-finish.js +97 -14
  122. package/dist/runtime/run-jobs.d.ts +95 -13
  123. package/dist/runtime/run-jobs.js +199 -28
  124. package/dist/runtime/run-orphan-check.job.d.ts +142 -0
  125. package/dist/runtime/run-orphan-check.job.js +339 -0
  126. package/dist/runtime/run-processors.d.ts +16 -9
  127. package/dist/runtime/run-processors.js +1 -1
  128. package/dist/runtime/run-queue.d.ts +64 -55
  129. package/dist/runtime/run-queue.js +147 -284
  130. package/dist/runtime/run-steps.d.ts +1 -2
  131. package/dist/runtime/run-steps.js +8 -11
  132. package/dist/runtime/run-worker.d.ts +6 -17
  133. package/dist/runtime/run-worker.js +32 -34
  134. package/dist/runtime/runtime.module.d.ts +2 -3
  135. package/dist/runtime/runtime.module.js +5 -6
  136. package/dist/runtime/step-log.d.ts +16 -4
  137. package/dist/runtime/step-log.js +26 -6
  138. package/dist/runtime/stream.controller.d.ts +2 -4
  139. package/dist/runtime/stream.controller.js +5 -11
  140. package/dist/runtime/subagents/child.js +1 -1
  141. package/dist/runtime/subagents/definition.d.ts +2 -1
  142. package/dist/runtime/subagents/definition.js +4 -2
  143. package/dist/runtime/subagents/deliveries.d.ts +6 -3
  144. package/dist/runtime/subagents/deliveries.js +16 -7
  145. package/dist/runtime/subagents/dispatch.d.ts +12 -12
  146. package/dist/runtime/subagents/dispatch.js +13 -38
  147. package/dist/runtime/subagents/messages.d.ts +5 -1
  148. package/dist/runtime/subagents/messages.js +11 -3
  149. package/dist/runtime/subagents/skill-fork.d.ts +3 -3
  150. package/dist/runtime/subagents/skill-fork.js +5 -5
  151. package/dist/runtime/subagents/subagents.service.js +1 -1
  152. package/dist/runtime/subagents/tree-jobs.d.ts +61 -0
  153. package/dist/runtime/subagents/tree-jobs.js +86 -0
  154. package/dist/runtime/usage-report-jobs.d.ts +23 -48
  155. package/dist/runtime/usage-report-jobs.js +67 -119
  156. package/dist/runtime/usage-report.d.ts +28 -77
  157. package/dist/runtime/usage-report.js +35 -213
  158. package/dist/runtime/waits.d.ts +48 -198
  159. package/dist/runtime/waits.js +103 -805
  160. package/dist/sandbox/sandbox.service.d.ts +17 -4
  161. package/dist/sandbox/sandbox.service.js +55 -14
  162. package/dist/schedules/calendar.d.ts +134 -0
  163. package/dist/schedules/calendar.js +410 -0
  164. package/dist/schedules/fire.d.ts +1 -3
  165. package/dist/schedules/fire.js +5 -10
  166. package/dist/schedules/run-tracker.d.ts +0 -5
  167. package/dist/schedules/run-tracker.js +0 -13
  168. package/dist/schedules/scan.job.js +15 -20
  169. package/dist/schedules/schedule-math.d.ts +42 -0
  170. package/dist/schedules/schedule-math.js +52 -0
  171. package/dist/schedules/scheduled-tasks.service.d.ts +16 -0
  172. package/dist/schedules/scheduled-tasks.service.js +53 -0
  173. package/dist/schedules/schedules.controller.d.ts +5 -1
  174. package/dist/schedules/schedules.controller.js +15 -1
  175. package/dist/schedules/schedules.schemas.d.ts +35 -0
  176. package/dist/schedules/schedules.schemas.js +27 -0
  177. package/dist/schedules/store.d.ts +5 -0
  178. package/dist/schedules/store.js +20 -0
  179. package/dist/tools/approvals.service.d.ts +26 -7
  180. package/dist/tools/approvals.service.js +88 -10
  181. package/dist/tools/builtin-names.d.ts +5 -3
  182. package/dist/tools/builtin-names.js +12 -4
  183. package/dist/tools/builtins/builtin-tools.d.ts +1 -2
  184. package/dist/tools/builtins/builtin-tools.js +21 -17
  185. package/dist/tools/builtins/sandbox/bash.js +9 -3
  186. package/dist/tools/builtins/sandbox/common.d.ts +18 -1
  187. package/dist/tools/builtins/sandbox/common.js +14 -1
  188. package/dist/tools/builtins/sandbox/index.d.ts +2 -1
  189. package/dist/tools/builtins/sandbox/index.js +4 -1
  190. package/dist/tools/builtins/sandbox/task-list.d.ts +12 -0
  191. package/dist/tools/builtins/sandbox/task-list.js +130 -0
  192. package/dist/tools/builtins/sandbox/task-output.d.ts +8 -1
  193. package/dist/tools/builtins/sandbox/task-output.js +21 -3
  194. package/dist/tools/builtins/web/extract.d.ts +2 -4
  195. package/dist/tools/builtins/web/extract.js +2 -5
  196. package/dist/tools/pipeline.d.ts +1 -1
  197. package/dist/tools/pipeline.js +1 -1
  198. package/dist/tools/questions.service.d.ts +15 -6
  199. package/dist/tools/questions.service.js +20 -8
  200. package/dist/tools/registry.d.ts +4 -2
  201. package/dist/tools/registry.js +4 -3
  202. package/dist/tools/tools.module.js +4 -3
  203. package/package.json +3 -3
  204. package/src/migrations/0118_device_policy_ops.sql +6 -0
  205. package/src/migrations/0119_outbox.sql +22 -0
  206. package/src/migrations/0120_drop_usage_report_sweep.sql +7 -0
package/dist/index.d.ts CHANGED
@@ -70,6 +70,7 @@ export { ROUTE_SCHEMAS, type RouteSchemas } from "./platform/route-schemas.js";
70
70
  export { UNKNOWN_RESULT_TEXT } from "./runtime/context/pairing.js";
71
71
  export type { AgentLoop, LoopContext, LoopOutcome } from "./runtime/loop.js";
72
72
  export { MAINTENANCE_QUEUE_NAME } from "./runtime/maintenance-queue.js";
73
+ export { type OrphanCandidate, queuedRunsWithoutJob } from "./runtime/run-orphan-check.job.js";
73
74
  export { runJobId } from "./runtime/run-queue.js";
74
75
  export { SANDBOX_DESTROY_JOB, SANDBOX_SUSPEND_JOB, } from "./sandbox/jobs.js";
75
76
  export { SANDBOX_LOCK_TTL_MS, SandboxService, type SandboxUseInfo, } from "./sandbox/sandbox.service.js";
package/dist/index.js CHANGED
@@ -68,6 +68,7 @@ export { PlatformOperations } from "./platform/registry.js";
68
68
  export { ROUTE_SCHEMAS } from "./platform/route-schemas.js";
69
69
  export { UNKNOWN_RESULT_TEXT } from "./runtime/context/pairing.js";
70
70
  export { MAINTENANCE_QUEUE_NAME } from "./runtime/maintenance-queue.js";
71
+ export { queuedRunsWithoutJob } from "./runtime/run-orphan-check.job.js";
71
72
  export { runJobId } from "./runtime/run-queue.js";
72
73
  export { SANDBOX_DESTROY_JOB, SANDBOX_SUSPEND_JOB, } from "./sandbox/jobs.js";
73
74
  export { SANDBOX_LOCK_TTL_MS, SandboxService, } from "./sandbox/sandbox.service.js";
@@ -21,6 +21,7 @@ import { KernelEventsModule } from "../events/kernel-events.js";
21
21
  import { GracefulShutdownModule } from "../lifecycle/graceful-shutdown.module.js";
22
22
  import { LockModule } from "../lock/lock.module.js";
23
23
  import { LoggingModule } from "../logging/logging.module.js";
24
+ import { OutboxModule } from "../outbox/outbox.module.js";
24
25
  import { QueueModule } from "../queue/queue.module.js";
25
26
  import { KERNEL_BUS, KERNEL_LOGGER, REDIS, REDIS_SUBSCRIBER } from "../tokens.js";
26
27
  /** How long the CLI's Redis connection may take before a command fails (a publish only warns). */
@@ -110,6 +111,10 @@ let KernelCliModule = KernelCliModule_1 = class KernelCliModule {
110
111
  ConfigModule.forRoot(config, opts.env !== undefined ? { env: opts.env } : {}),
111
112
  GracefulShutdownModule,
112
113
  DatabaseModule.forRoot(),
114
+ // owed jobs a command's domain writes carry (an extension's embed call owes its usage
115
+ // report); delivered by the fast path when the command's context has the queue, else by a
116
+ // worker's relay
117
+ OutboxModule,
113
118
  // QueueModule's root gives the queues a command's modules register (the catalog's scope
114
119
  // sync and maintenance queues) the configured Redis; without it BullMQ would connect to
115
120
  // its default 127.0.0.1:6379
@@ -23,8 +23,9 @@ export interface KernelEventMap {
23
23
  };
24
24
  /**
25
25
  * A person decided a tool call approval (after the commit, exactly once: only the decision that
26
- * changed it from pending). The run's waits resume it and its IM delivery is enqueued; a lost
27
- * event is re-derived by the wait sweep (runs waiting on a decided approval are resumed).
26
+ * changed it from pending). The decision's transaction resumed the run and, for a conversation
27
+ * with a channel target, owed the IM delivery that updates the card (with its `approval_decided`
28
+ * append); the event itself is only the decision's announcement.
28
29
  */
29
30
  "approval.decided": {
30
31
  id: string;
@@ -36,8 +37,7 @@ export interface KernelEventMap {
36
37
  };
37
38
  /**
38
39
  * A person answered a tool call's input request, or skipped it with a message (after the commit,
39
- * exactly once). Same
40
- * consumers and the same re-derivation as `approval.decided`.
40
+ * exactly once). The same transaction contract as `approval.decided`.
41
41
  */
42
42
  "question.answered": {
43
43
  id: string;
@@ -47,38 +47,20 @@ export interface KernelEventMap {
47
47
  /** `skipped`: the run's starter sent a message instead (reply-while-question spec §6.3). */
48
48
  outcome: "answered" | "skipped";
49
49
  };
50
- /**
51
- * EventAppended: a conversation's event log advanced to `toSeq` (after the commit, from
52
- * `EventLog.notify`, right after the bus publish). The channel outbound delivery enqueues a job
53
- * for a conversation with an IM target; a lost event is re-derived by the next append's (and the
54
- * finished delivery job re-checks the log for content it has not delivered).
55
- */
56
- "conversation.event-appended": {
57
- conversationId: string;
58
- toSeq: number;
59
- };
60
50
  /**
61
51
  * A run started executing on this replica (its first start, a resume, or a takeover after a
62
52
  * stall), after it took the conversation lock and an org quota slot. The channel outbound
63
- * enqueues the conversation's IM delivery; a lost event is re-derived by the run's next append.
53
+ * enqueues the conversation's IM delivery at once (a hint: the run's appends owe it, transactional
54
+ * outbox spec §8.1).
64
55
  */
65
56
  "run.started": {
66
57
  conversationId: string;
67
58
  runId: string;
68
59
  };
69
60
  /**
70
- * A stop of a run waiting on a person was handled (`RunControl.abortWaiting`, whether or not it
71
- * still waited): its approval and question cards change, so the channel outbound enqueues the
72
- * conversation's IM delivery (re-derived by the next append when lost).
73
- */
74
- "run.wait-stopped": {
75
- conversationId: string;
76
- runId: string;
77
- };
78
- /**
79
- * `ensureRun` marked an orphaned run failed (after the commit): the waits stop its active fork
80
- * sub-agent at once (background sub-agents keep running, agent map spec §5.4). A lost event is
81
- * repaired by the wait sweep (fork children of finished runs).
61
+ * The `run-orphan-check` failed an orphaned run (after the commit): the waits stop its active
62
+ * fork sub-agent at once (background sub-agents keep running, agent map spec §5.4). A lost event
63
+ * is repaired by the `stop-tree` job the run's terminal transaction owed.
82
64
  */
83
65
  "run.orphaned": {
84
66
  runId: string;
@@ -99,14 +81,6 @@ export interface KernelEventMap {
99
81
  "background_task.finished": {
100
82
  conversationId: string;
101
83
  };
102
- /**
103
- * An extension's `ctx.models.embed` call wrote a `model_usage` row (after the commit, spec B
104
- * §5.4). The usage reporter enqueues its `usage.report` delivery; a lost event is re-derived by
105
- * the usage sweep (rows whose `reported_at` is still null).
106
- */
107
- "model_usage.recorded": {
108
- id: string;
109
- };
110
84
  }
111
85
  type Handler<K extends keyof KernelEventMap> = (payload: KernelEventMap[K]) => Promise<void> | void;
112
86
  /** Typed in-process event emitter; handlers run in order, a failing one is logged and skipped. */
@@ -0,0 +1,6 @@
1
+ /**
2
+ * The outbox relay (`@LockedInterval`), imported only by `WorkerModule`. Its dispatcher comes from
3
+ * the global `OutboxModule`, which every role has (the fast path runs everywhere).
4
+ */
5
+ export declare class OutboxRelayModule {
6
+ }
@@ -0,0 +1,18 @@
1
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
2
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
6
+ };
7
+ import { Module } from "@nestjs/common";
8
+ import { OutboxRelay } from "./outbox.relay.js";
9
+ /**
10
+ * The outbox relay (`@LockedInterval`), imported only by `WorkerModule`. Its dispatcher comes from
11
+ * the global `OutboxModule`, which every role has (the fast path runs everywhere).
12
+ */
13
+ let OutboxRelayModule = class OutboxRelayModule {
14
+ };
15
+ OutboxRelayModule = __decorate([
16
+ Module({ providers: [OutboxRelay] })
17
+ ], OutboxRelayModule);
18
+ export { OutboxRelayModule };
@@ -0,0 +1,103 @@
1
+ import { ModuleRef } from "@nestjs/core";
2
+ import type { Queue } from "bullmq";
3
+ import type { KernelLogger } from "../../log.js";
4
+ import { KernelDb } from "../database/kernel-db.js";
5
+ import { TransactionHost } from "../database/transaction-host.js";
6
+ import { type KernelQueueName } from "../queue/queue-names.js";
7
+ /**
8
+ * Retry schedule of a row whose add failed (spec §3.4): the next attempt waits
9
+ * `min(capMs, baseMs * 2^attempts)`, and the row is dead (kept, never retried) at `maxAttempts`.
10
+ */
11
+ export declare const OUTBOX_BACKOFF: {
12
+ readonly baseMs: 1000;
13
+ readonly capMs: 300000;
14
+ readonly maxAttempts: 20;
15
+ };
16
+ /**
17
+ * How long the queue adds of one claim pass may take in all: the claim transaction holds its rows'
18
+ * locks and a pool connection for at most about this long.
19
+ */
20
+ export declare const OUTBOX_REDIS_TIMEOUT_MS = 2000;
21
+ /**
22
+ * Most fast-path passes running at once in one process; more are skipped and left to the relay,
23
+ * so a slow Redis cannot take the whole pool (a local resource bound, no shared state).
24
+ */
25
+ export declare const OUTBOX_FAST_PATH_CONCURRENCY = 4;
26
+ /**
27
+ * Test seam (`KernelOverride`): returns true to make the adds to the given queue fail (a Redis
28
+ * outage) without calling Redis. The kernel never provides it; the dispatcher defaults to "never
29
+ * fails".
30
+ */
31
+ export declare const OUTBOX_DISPATCH_FAILS: unique symbol;
32
+ /** What one pass did: rows delivered, rows whose add failed, and of those the rows now dead. */
33
+ export interface DispatchResult {
34
+ sent: number;
35
+ failed: number;
36
+ dead: number;
37
+ }
38
+ /** The queue handle the dispatcher uses: adds, and the connection's state. */
39
+ export type OutboxQueue = Pick<Queue, "addBulk" | "getBackend">;
40
+ /**
41
+ * Delivers outbox rows to BullMQ (transactional outbox spec §3.3, §3.4). A pass claims rows
42
+ * `FOR UPDATE SKIP LOCKED` (so concurrent passes never take the same row), adds their jobs per
43
+ * queue with `addBulk` (the queue defaults overlaid by the registry's options of the job name, the
44
+ * row's job id, a future `run_at` as the remaining `delay`), and deletes
45
+ * the delivered rows in the same short transaction. A crash after the add and before the commit
46
+ * leaves the rows; the next pass adds again and BullMQ ignores the existing job id. A queue whose
47
+ * add fails keeps its rows, and a second short update records the attempt with exponential
48
+ * backoff, setting `dead_at` at the last one.
49
+ *
50
+ * The fast path never queues behind an outage: it skips the pass (the relay delivers the rows) when
51
+ * a queue's Redis connection is not ready or {@link OUTBOX_FAST_PATH_CONCURRENCY} passes already
52
+ * run. Within a pass, the first failed queue fails the queues after it without trying them.
53
+ *
54
+ * The queue handles are the instances the queues' owner modules registered (one per queue, no
55
+ * connection of the outbox's own), resolved by token at use time. A queue whose owner this
56
+ * process's roles do not load is not reachable here: no pass claims its rows, which stay for a
57
+ * process that has it.
58
+ */
59
+ export declare class OutboxDispatcher {
60
+ readonly txHost: TransactionHost;
61
+ private readonly db;
62
+ private readonly log;
63
+ private readonly moduleRef;
64
+ private readonly dispatchFails;
65
+ /** The owners' queue instances resolved so far (a per-process handle cache). */
66
+ private readonly queues;
67
+ /** Fast-path passes running in this process (bounded by OUTBOX_FAST_PATH_CONCURRENCY). */
68
+ private fastPaths;
69
+ constructor(txHost: TransactionHost, db: KernelDb, log: KernelLogger, moduleRef: Pick<ModuleRef, "get">, dispatchFails?: (queue: KernelQueueName) => boolean);
70
+ /** The queue instance its owner module registered, or undefined when no loaded module has it. */
71
+ private queue;
72
+ /** The kernel queues this process can add to, with their handles. */
73
+ private reachable;
74
+ /**
75
+ * The fast path: delivers the given rows (a committed transaction's own) right away. Never
76
+ * throws: whatever it does not deliver stays for the relay, and a failure is logged.
77
+ */
78
+ dispatchOwn(ids: readonly string[]): Promise<void>;
79
+ /**
80
+ * Every reachable queue's connection is ready (they reach the same Redis; the fast path does not
81
+ * read its rows' queues before claiming them).
82
+ */
83
+ private redisReady;
84
+ /**
85
+ * The relay: delivers up to `limit` due rows (not dead, `next_attempt_at` reached), oldest
86
+ * first.
87
+ */
88
+ dispatchDue(limit: number): Promise<DispatchResult>;
89
+ private dispatch;
90
+ /**
91
+ * One claim transaction: locks the rows (skipping rows another pass holds), adds their jobs and
92
+ * deletes the delivered rows. The only Redis calls made while a PG transaction is open in the
93
+ * kernel (spec §3.3): the locks held are outbox rows only, and the pass's adds together are
94
+ * bounded by {@link OUTBOX_REDIS_TIMEOUT_MS}.
95
+ */
96
+ private claimAndSend;
97
+ /**
98
+ * Records a failed add on its rows (after the claim transaction, so the attempt is kept whatever
99
+ * that transaction did): one more attempt, the error, the next attempt by {@link OUTBOX_BACKOFF},
100
+ * and `dead_at` at the last attempt. Returns how many rows it made dead.
101
+ */
102
+ private recordFailure;
103
+ }
@@ -0,0 +1,310 @@
1
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
2
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
6
+ };
7
+ var __param = (this && this.__param) || function (paramIndex, decorator) {
8
+ return function (target, key) { decorator(target, key, paramIndex); }
9
+ };
10
+ import { getQueueToken } from "@nestjs/bullmq";
11
+ import { Inject, Injectable, Optional } from "@nestjs/common";
12
+ import { ModuleRef } from "@nestjs/core";
13
+ import { sql } from "kysely";
14
+ import { KernelDb } from "../database/kernel-db.js";
15
+ import { TransactionHost } from "../database/transaction-host.js";
16
+ import { Transactional } from "../database/transactional.decorator.js";
17
+ import { QUEUE_JOB_DEFAULTS } from "../queue/queue.module.js";
18
+ import { KERNEL_QUEUES } from "../queue/queue-names.js";
19
+ import { KERNEL_LOGGER } from "../tokens.js";
20
+ import { outboxJobOptions } from "./registry.js";
21
+ /**
22
+ * Retry schedule of a row whose add failed (spec §3.4): the next attempt waits
23
+ * `min(capMs, baseMs * 2^attempts)`, and the row is dead (kept, never retried) at `maxAttempts`.
24
+ */
25
+ export const OUTBOX_BACKOFF = { baseMs: 1000, capMs: 300_000, maxAttempts: 20 };
26
+ /**
27
+ * How long the queue adds of one claim pass may take in all: the claim transaction holds its rows'
28
+ * locks and a pool connection for at most about this long.
29
+ */
30
+ export const OUTBOX_REDIS_TIMEOUT_MS = 2000;
31
+ /**
32
+ * Most fast-path passes running at once in one process; more are skipped and left to the relay,
33
+ * so a slow Redis cannot take the whole pool (a local resource bound, no shared state).
34
+ */
35
+ export const OUTBOX_FAST_PATH_CONCURRENCY = 4;
36
+ /**
37
+ * Test seam (`KernelOverride`): returns true to make the adds to the given queue fail (a Redis
38
+ * outage) without calling Redis. The kernel never provides it; the dispatcher defaults to "never
39
+ * fails".
40
+ */
41
+ export const OUTBOX_DISPATCH_FAILS = Symbol("OUTBOX_DISPATCH_FAILS");
42
+ /** Longest `last_error` kept on a row. */
43
+ const MAX_ERROR_LENGTH = 2000;
44
+ const errMsg = (e) => (e instanceof Error ? e.message : String(e));
45
+ const NOT_READY = Symbol("NOT_READY");
46
+ /**
47
+ * Whether the queue's Redis connection is ready now, without waiting: the backend's client
48
+ * promise only settles once connected, so a pending one counts as not ready.
49
+ */
50
+ async function connectionReady(queue) {
51
+ try {
52
+ // an already settled client promise wins the race (its callback is queued first)
53
+ const client = await Promise.race([queue.getBackend().client, Promise.resolve(NOT_READY)]);
54
+ return client !== NOT_READY && client.status === "ready";
55
+ }
56
+ catch {
57
+ return false;
58
+ }
59
+ }
60
+ /** Rejects when `p` takes longer than `ms`; a late rejection of `p` is swallowed. */
61
+ async function withTimeout(p, ms, what) {
62
+ p.catch(() => { });
63
+ let timer;
64
+ const timeout = new Promise((_, reject) => {
65
+ timer = setTimeout(() => reject(new Error(`${what} timed out after ${ms} ms`)), ms);
66
+ });
67
+ try {
68
+ return await Promise.race([p, timeout]);
69
+ }
70
+ finally {
71
+ clearTimeout(timer);
72
+ }
73
+ }
74
+ /**
75
+ * Delivers outbox rows to BullMQ (transactional outbox spec §3.3, §3.4). A pass claims rows
76
+ * `FOR UPDATE SKIP LOCKED` (so concurrent passes never take the same row), adds their jobs per
77
+ * queue with `addBulk` (the queue defaults overlaid by the registry's options of the job name, the
78
+ * row's job id, a future `run_at` as the remaining `delay`), and deletes
79
+ * the delivered rows in the same short transaction. A crash after the add and before the commit
80
+ * leaves the rows; the next pass adds again and BullMQ ignores the existing job id. A queue whose
81
+ * add fails keeps its rows, and a second short update records the attempt with exponential
82
+ * backoff, setting `dead_at` at the last one.
83
+ *
84
+ * The fast path never queues behind an outage: it skips the pass (the relay delivers the rows) when
85
+ * a queue's Redis connection is not ready or {@link OUTBOX_FAST_PATH_CONCURRENCY} passes already
86
+ * run. Within a pass, the first failed queue fails the queues after it without trying them.
87
+ *
88
+ * The queue handles are the instances the queues' owner modules registered (one per queue, no
89
+ * connection of the outbox's own), resolved by token at use time. A queue whose owner this
90
+ * process's roles do not load is not reachable here: no pass claims its rows, which stay for a
91
+ * process that has it.
92
+ */
93
+ let OutboxDispatcher = class OutboxDispatcher {
94
+ txHost;
95
+ db;
96
+ log;
97
+ moduleRef;
98
+ dispatchFails;
99
+ /** The owners' queue instances resolved so far (a per-process handle cache). */
100
+ queues = new Map();
101
+ /** Fast-path passes running in this process (bounded by OUTBOX_FAST_PATH_CONCURRENCY). */
102
+ fastPaths = 0;
103
+ constructor(txHost, db, log, moduleRef, dispatchFails = () => false) {
104
+ this.txHost = txHost;
105
+ this.db = db;
106
+ this.log = log;
107
+ this.moduleRef = moduleRef;
108
+ this.dispatchFails = dispatchFails;
109
+ }
110
+ /** The queue instance its owner module registered, or undefined when no loaded module has it. */
111
+ queue(name) {
112
+ const known = this.queues.get(name);
113
+ if (known)
114
+ return known;
115
+ let queue;
116
+ try {
117
+ queue = this.moduleRef.get(getQueueToken(name), { strict: false });
118
+ }
119
+ catch {
120
+ return undefined;
121
+ }
122
+ if (queue)
123
+ this.queues.set(name, queue);
124
+ return queue;
125
+ }
126
+ /** The kernel queues this process can add to, with their handles. */
127
+ reachable() {
128
+ const found = [];
129
+ for (const name of KERNEL_QUEUES) {
130
+ const queue = this.queue(name);
131
+ if (queue)
132
+ found.push([name, queue]);
133
+ }
134
+ return found;
135
+ }
136
+ /**
137
+ * The fast path: delivers the given rows (a committed transaction's own) right away. Never
138
+ * throws: whatever it does not deliver stays for the relay, and a failure is logged.
139
+ */
140
+ async dispatchOwn(ids) {
141
+ if (ids.length === 0)
142
+ return;
143
+ if (this.fastPaths >= OUTBOX_FAST_PATH_CONCURRENCY) {
144
+ this.log.debug("outbox fast path busy; the relay delivers the rows", { rows: ids.length });
145
+ return;
146
+ }
147
+ this.fastPaths++;
148
+ try {
149
+ if (!(await this.redisReady())) {
150
+ this.log.debug("outbox fast path: Redis not ready; the relay delivers the rows", {
151
+ rows: ids.length,
152
+ });
153
+ return;
154
+ }
155
+ await this.dispatch({ ids });
156
+ }
157
+ catch (e) {
158
+ this.log.error("outbox fast path failed; the relay delivers the rows", {
159
+ rows: ids.length,
160
+ error: errMsg(e),
161
+ });
162
+ }
163
+ finally {
164
+ this.fastPaths--;
165
+ }
166
+ }
167
+ /**
168
+ * Every reachable queue's connection is ready (they reach the same Redis; the fast path does not
169
+ * read its rows' queues before claiming them).
170
+ */
171
+ async redisReady() {
172
+ const ready = await Promise.all(this.reachable().map(([, q]) => connectionReady(q)));
173
+ return ready.every(Boolean);
174
+ }
175
+ /**
176
+ * The relay: delivers up to `limit` due rows (not dead, `next_attempt_at` reached), oldest
177
+ * first.
178
+ */
179
+ dispatchDue(limit) {
180
+ return this.dispatch({ limit });
181
+ }
182
+ async dispatch(claim) {
183
+ const { sent, failed } = await this.claimAndSend(claim);
184
+ let dead = 0;
185
+ for (const group of failed)
186
+ dead += await this.recordFailure(group);
187
+ return { sent, failed: failed.reduce((n, g) => n + g.ids.length, 0), dead };
188
+ }
189
+ /**
190
+ * One claim transaction: locks the rows (skipping rows another pass holds), adds their jobs and
191
+ * deletes the delivered rows. The only Redis calls made while a PG transaction is open in the
192
+ * kernel (spec §3.3): the locks held are outbox rows only, and the pass's adds together are
193
+ * bounded by {@link OUTBOX_REDIS_TIMEOUT_MS}.
194
+ */
195
+ async claimAndSend(claim) {
196
+ const queues = new Map(this.reachable());
197
+ if (queues.size === 0)
198
+ return { sent: 0, failed: [] };
199
+ // rows of a queue this process cannot reach are left for a process that can
200
+ let query = this.db
201
+ .selectFrom("outbox")
202
+ .select(["id", "queue", "job_name", "job_id", "data", "run_at"])
203
+ .where("dead_at", "is", null)
204
+ .where("queue", "in", [...queues.keys()]);
205
+ query =
206
+ "ids" in claim
207
+ ? query.where("id", "in", [...claim.ids])
208
+ : query.where("next_attempt_at", "<=", sql `now()`).limit(claim.limit);
209
+ const rows = await query.orderBy("id").forUpdate().skipLocked().execute();
210
+ if (rows.length === 0)
211
+ return { sent: 0, failed: [] };
212
+ const byQueue = new Map();
213
+ for (const row of rows) {
214
+ const group = byQueue.get(row.queue);
215
+ if (group)
216
+ group.push(row);
217
+ else
218
+ byQueue.set(row.queue, [row]);
219
+ }
220
+ const now = Date.now();
221
+ const deadline = now + OUTBOX_REDIS_TIMEOUT_MS;
222
+ const delivered = [];
223
+ const failed = [];
224
+ // after the first failure the queues left are failed without a try: Redis is likely down
225
+ let firstError;
226
+ for (const [name, group] of byQueue) {
227
+ const ids = group.map((r) => r.id);
228
+ if (firstError !== undefined) {
229
+ failed.push({
230
+ ids,
231
+ error: `not tried after an earlier queue failed: ${firstError}`.slice(0, MAX_ERROR_LENGTH),
232
+ });
233
+ continue;
234
+ }
235
+ try {
236
+ if (this.dispatchFails(name))
237
+ throw new Error("outbox dispatch failure (test seam)");
238
+ const queue = queues.get(name);
239
+ if (!queue)
240
+ throw new Error(`"${name}" is not a kernel queue`);
241
+ const left = deadline - Date.now();
242
+ if (left <= 0)
243
+ throw new Error(`no time left of the pass's ${OUTBOX_REDIS_TIMEOUT_MS} ms`);
244
+ await withTimeout(queue.addBulk(group.map((r) => ({
245
+ name: r.job_name,
246
+ data: r.data,
247
+ opts: {
248
+ ...QUEUE_JOB_DEFAULTS,
249
+ ...outboxJobOptions(r.job_name),
250
+ jobId: r.job_id,
251
+ delay: r.run_at ? Math.max(0, r.run_at.getTime() - now) : 0,
252
+ },
253
+ }))), left, `addBulk to ${name}`);
254
+ delivered.push(...ids);
255
+ }
256
+ catch (e) {
257
+ firstError = errMsg(e);
258
+ failed.push({ ids, error: firstError.slice(0, MAX_ERROR_LENGTH) });
259
+ }
260
+ }
261
+ if (delivered.length > 0)
262
+ await this.db.deleteFrom("outbox").where("id", "in", delivered).execute();
263
+ return { sent: delivered.length, failed };
264
+ }
265
+ /**
266
+ * Records a failed add on its rows (after the claim transaction, so the attempt is kept whatever
267
+ * that transaction did): one more attempt, the error, the next attempt by {@link OUTBOX_BACKOFF},
268
+ * and `dead_at` at the last attempt. Returns how many rows it made dead.
269
+ */
270
+ async recordFailure(group) {
271
+ const { baseMs, capMs, maxAttempts } = OUTBOX_BACKOFF;
272
+ // the right-hand sides read the row before the update: the first failure waits baseMs
273
+ const backoffMs = sql `least(${capMs}::float8, ${baseMs}::float8 * power(2, attempts))`;
274
+ const updated = await this.db
275
+ .updateTable("outbox")
276
+ .set({
277
+ attempts: sql `attempts + 1`,
278
+ last_error: group.error,
279
+ next_attempt_at: sql `now() + ${backoffMs} * interval '1 millisecond'`,
280
+ dead_at: sql `case when attempts + 1 >= ${maxAttempts}::int then now() end`,
281
+ })
282
+ .where("id", "in", group.ids)
283
+ .where("dead_at", "is", null)
284
+ .returning(["dead_at"])
285
+ .execute();
286
+ const dead = updated.filter((r) => r.dead_at !== null).length;
287
+ this.log.warn("outbox delivery failed; retrying with backoff", {
288
+ rows: group.ids.length,
289
+ dead,
290
+ error: group.error,
291
+ });
292
+ if (dead > 0) {
293
+ this.log.error("outbox rows dead after the last attempt", { rows: dead, error: group.error });
294
+ }
295
+ return dead;
296
+ }
297
+ };
298
+ __decorate([
299
+ Transactional({ propagation: "requires_new" })
300
+ ], OutboxDispatcher.prototype, "claimAndSend", null);
301
+ OutboxDispatcher = __decorate([
302
+ Injectable(),
303
+ __param(0, Inject(TransactionHost)),
304
+ __param(1, Inject(KernelDb)),
305
+ __param(2, Inject(KERNEL_LOGGER)),
306
+ __param(3, Inject(ModuleRef)),
307
+ __param(4, Optional()),
308
+ __param(4, Inject(OUTBOX_DISPATCH_FAILS))
309
+ ], OutboxDispatcher);
310
+ export { OutboxDispatcher };
@@ -0,0 +1,24 @@
1
+ import { OutboxDispatcher } from "./outbox.dispatcher.js";
2
+ import { type OutboxFastPath } from "./outbox.service.js";
3
+ /**
4
+ * Test seam (`KernelOverride`): receives each detached fast-path pass, so a test can await what a
5
+ * request started after its commit (the kernel never provides it).
6
+ */
7
+ export declare const OUTBOX_FAST_PATH_SETTLED: unique symbol;
8
+ /**
9
+ * The fast path (spec §3.3): starts the dispatcher on a committed transaction's rows and returns
10
+ * at once, so the commit's caller never waits on Redis (R9). `dispatchOwn` never throws; what it
11
+ * does not deliver stays for the relay.
12
+ */
13
+ export declare const outboxFastPath: (dispatcher: Pick<OutboxDispatcher, "dispatchOwn">, settled?: (pass: Promise<void>) => void) => OutboxFastPath;
14
+ /**
15
+ * Global: `OutboxService` (owed work written in the caller's transaction, transactional outbox
16
+ * spec §3.2), the `OutboxDispatcher` that delivers the rows (the relay drives it too) and the fast
17
+ * path `add` hands each committed transaction's rows to.
18
+ *
19
+ * It registers no queue: the dispatcher adds through the instances the queues' owner modules
20
+ * registered (one instance and connection per queue, as everywhere in the kernel), resolved at use
21
+ * time, and leaves the rows of a queue whose owner this process does not load to one that does.
22
+ */
23
+ export declare class OutboxModule {
24
+ }
@@ -0,0 +1,50 @@
1
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
2
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
6
+ };
7
+ import { Global, Module } from "@nestjs/common";
8
+ import { OutboxDispatcher } from "./outbox.dispatcher.js";
9
+ import { OUTBOX_FAST_PATH, OutboxService } from "./outbox.service.js";
10
+ /**
11
+ * Test seam (`KernelOverride`): receives each detached fast-path pass, so a test can await what a
12
+ * request started after its commit (the kernel never provides it).
13
+ */
14
+ export const OUTBOX_FAST_PATH_SETTLED = Symbol("OUTBOX_FAST_PATH_SETTLED");
15
+ /**
16
+ * The fast path (spec §3.3): starts the dispatcher on a committed transaction's rows and returns
17
+ * at once, so the commit's caller never waits on Redis (R9). `dispatchOwn` never throws; what it
18
+ * does not deliver stays for the relay.
19
+ */
20
+ export const outboxFastPath = (dispatcher, settled) => async (ids) => {
21
+ const pass = dispatcher.dispatchOwn(ids);
22
+ settled?.(pass);
23
+ };
24
+ /**
25
+ * Global: `OutboxService` (owed work written in the caller's transaction, transactional outbox
26
+ * spec §3.2), the `OutboxDispatcher` that delivers the rows (the relay drives it too) and the fast
27
+ * path `add` hands each committed transaction's rows to.
28
+ *
29
+ * It registers no queue: the dispatcher adds through the instances the queues' owner modules
30
+ * registered (one instance and connection per queue, as everywhere in the kernel), resolved at use
31
+ * time, and leaves the rows of a queue whose owner this process does not load to one that does.
32
+ */
33
+ let OutboxModule = class OutboxModule {
34
+ };
35
+ OutboxModule = __decorate([
36
+ Global(),
37
+ Module({
38
+ providers: [
39
+ OutboxService,
40
+ OutboxDispatcher,
41
+ {
42
+ provide: OUTBOX_FAST_PATH,
43
+ inject: [OutboxDispatcher, { token: OUTBOX_FAST_PATH_SETTLED, optional: true }],
44
+ useFactory: (dispatcher, settled) => outboxFastPath(dispatcher, settled ?? undefined),
45
+ },
46
+ ],
47
+ exports: [OutboxService, OutboxDispatcher, OUTBOX_FAST_PATH],
48
+ })
49
+ ], OutboxModule);
50
+ export { OutboxModule };