@capybara-hq/kernel 0.1.0-rc.22 → 0.1.0-rc.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bootstrap.json +25 -25
- package/dist/admin/admin.controller.d.ts +3 -3
- package/dist/admin/admin.controller.js +6 -2
- package/dist/admin/admin.module.d.ts +2 -1
- package/dist/admin/admin.module.js +7 -3
- package/dist/admin/admin.schemas.d.ts +13 -0
- package/dist/admin/outbox-dead.controller.d.ts +23 -0
- package/dist/admin/outbox-dead.controller.js +62 -0
- package/dist/admin/outbox-dead.schemas.d.ts +8 -0
- package/dist/admin/outbox-dead.schemas.js +18 -0
- package/dist/admin/outbox-dead.service.d.ts +66 -0
- package/dist/admin/outbox-dead.service.js +179 -0
- package/dist/admin/overview.service.d.ts +40 -5
- package/dist/admin/overview.service.js +41 -7
- package/dist/agents/from-template.js +1 -1
- package/dist/background/finish.d.ts +5 -2
- package/dist/background/finish.js +14 -3
- package/dist/background/launch.d.ts +44 -5
- package/dist/background/launch.js +92 -9
- package/dist/background/session.d.ts +14 -2
- package/dist/background/session.js +26 -4
- package/dist/background/status.d.ts +1 -1
- package/dist/background/status.js +3 -3
- package/dist/background/stop.d.ts +0 -2
- package/dist/background/stop.js +1 -21
- package/dist/background/store.d.ts +8 -0
- package/dist/background/store.js +24 -0
- package/dist/catalog/admin-catalog.controller.d.ts +21 -2
- package/dist/catalog/admin-catalog.controller.js +45 -6
- package/dist/catalog/admin-catalog.schemas.d.ts +5 -0
- package/dist/catalog/admin-catalog.schemas.js +8 -0
- package/dist/catalog/catalog-content.d.ts +5 -0
- package/dist/catalog/catalog-content.js +1 -0
- package/dist/catalog/catalog.module.d.ts +2 -1
- package/dist/catalog/catalog.module.js +5 -1
- package/dist/catalog/marketplaces.service.d.ts +1 -1
- package/dist/catalog/platform-import-update.d.ts +50 -0
- package/dist/catalog/platform-import-update.js +116 -0
- package/dist/catalog/plugin-import.d.ts +36 -1
- package/dist/catalog/plugin-import.js +23 -1
- package/dist/catalog/plugin-upgrade.d.ts +18 -0
- package/dist/catalog/plugin-upgrade.js +19 -0
- package/dist/catalog/requests.service.js +2 -2
- package/dist/channels/channels.module.d.ts +5 -2
- package/dist/channels/channels.module.js +9 -2
- package/dist/channels/deliver-job.d.ts +25 -0
- package/dist/channels/deliver-job.js +25 -0
- package/dist/channels/inbound.d.ts +1 -1
- package/dist/channels/inbound.js +2 -15
- package/dist/channels/jobs.d.ts +28 -18
- package/dist/channels/jobs.js +63 -26
- package/dist/config/schema.d.ts +3 -6
- package/dist/config/schema.js +20 -27
- package/dist/conversations/agent-tree.d.ts +16 -0
- package/dist/conversations/agent-tree.js +36 -0
- package/dist/conversations/conversations.service.d.ts +44 -14
- package/dist/conversations/conversations.service.js +52 -38
- package/dist/conversations/event-log.d.ts +7 -10
- package/dist/conversations/event-log.js +24 -15
- package/dist/conversations/events.js +2 -1
- package/dist/conversations/queue.service.d.ts +8 -3
- package/dist/conversations/queue.service.js +37 -13
- package/dist/conversations/titles.d.ts +21 -8
- package/dist/conversations/titles.js +42 -15
- package/dist/db/kernel-schema.d.ts +26 -8
- package/dist/devices/devices.service.d.ts +12 -3
- package/dist/devices/devices.service.js +30 -9
- package/dist/errors.d.ts +15 -0
- package/dist/errors.js +15 -0
- package/dist/evals/run-replay.js +14 -8
- package/dist/extensions/extensions.module.js +3 -0
- package/dist/extensions/lifecycle.d.ts +6 -1
- package/dist/extensions/lifecycle.js +9 -2
- package/dist/extensions/usage-subscribers.d.ts +43 -0
- package/dist/extensions/usage-subscribers.js +108 -0
- package/dist/http/api.module.js +2 -0
- package/dist/identity/settings.d.ts +53 -3
- package/dist/identity/settings.js +92 -9
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/infra/cli/kernel-cli.module.js +5 -0
- package/dist/infra/config/kernel-config.service.d.ts +7 -0
- package/dist/infra/config/kernel-config.service.js +20 -0
- package/dist/infra/events/kernel-events.d.ts +9 -35
- package/dist/infra/outbox/outbox-relay.module.d.ts +6 -0
- package/dist/infra/outbox/outbox-relay.module.js +18 -0
- package/dist/infra/outbox/outbox.dispatcher.d.ts +103 -0
- package/dist/infra/outbox/outbox.dispatcher.js +310 -0
- package/dist/infra/outbox/outbox.module.d.ts +24 -0
- package/dist/infra/outbox/outbox.module.js +50 -0
- package/dist/infra/outbox/outbox.relay.d.ts +31 -0
- package/dist/infra/outbox/outbox.relay.js +76 -0
- package/dist/infra/outbox/outbox.service.d.ts +51 -0
- package/dist/infra/outbox/outbox.service.js +131 -0
- package/dist/infra/outbox/registry.d.ts +120 -0
- package/dist/infra/outbox/registry.js +108 -0
- package/dist/infra/queue/queue-names.d.ts +8 -0
- package/dist/infra/queue/queue-names.js +16 -0
- package/dist/kernel.module.js +3 -0
- package/dist/local-devices/device-policy.service.d.ts +4 -0
- package/dist/local-devices/device-policy.service.js +61 -3
- package/dist/local-devices/local-device-gateway.d.ts +4 -2
- package/dist/local-devices/local-device-gateway.js +39 -14
- package/dist/local-devices/machines.d.ts +25 -1
- package/dist/local-devices/machines.js +91 -15
- package/dist/local-devices/policy-summary.d.ts +44 -1
- package/dist/local-devices/policy-summary.js +69 -0
- package/dist/models/embeddings.d.ts +9 -9
- package/dist/models/embeddings.js +11 -18
- package/dist/models/models.module.js +3 -0
- package/dist/orgs/orgs.service.js +7 -0
- package/dist/orgs/settings.controller.js +2 -2
- package/dist/orgs/settings.service.d.ts +11 -5
- package/dist/orgs/settings.service.js +28 -14
- package/dist/overrides.d.ts +20 -3
- package/dist/overrides.js +10 -2
- package/dist/platform/exclusions.js +5 -0
- package/dist/platform/ops/admin.ops.js +50 -9
- package/dist/platform/ops/catalog-admin.ops.d.ts +3 -1
- package/dist/platform/ops/catalog-admin.ops.js +32 -4
- package/dist/platform/ops/org-settings.ops.js +3 -3
- package/dist/platform/ops/orgs.ops.js +1 -0
- package/dist/platform/ops/outbox.ops.d.ts +21 -0
- package/dist/platform/ops/outbox.ops.js +125 -0
- package/dist/platform/ops/platform.ops.js +1 -1
- package/dist/platform/ops/schedules.ops.js +40 -1
- package/dist/platform/platform-ops.module.d.ts +2 -1
- package/dist/platform/platform-ops.module.js +3 -0
- package/dist/platform/route-schemas.js +3 -0
- package/dist/roles/worker.module.js +9 -4
- package/dist/runtime/langgraph-loop.js +13 -5
- package/dist/runtime/limits.d.ts +15 -0
- package/dist/runtime/limits.js +25 -0
- package/dist/runtime/loop.d.ts +8 -3
- package/dist/runtime/loop.js +2 -1
- package/dist/runtime/maintenance.d.ts +4 -2
- package/dist/runtime/maintenance.js +17 -2
- package/dist/runtime/model-call.d.ts +14 -16
- package/dist/runtime/model-call.js +14 -24
- package/dist/runtime/queue-lost-jobs.d.ts +24 -0
- package/dist/runtime/queue-lost-jobs.js +51 -0
- package/dist/runtime/run-control.d.ts +42 -5
- package/dist/runtime/run-control.js +66 -12
- package/dist/runtime/run-finish.d.ts +39 -12
- package/dist/runtime/run-finish.js +97 -14
- package/dist/runtime/run-jobs.d.ts +95 -13
- package/dist/runtime/run-jobs.js +199 -28
- package/dist/runtime/run-orphan-check.job.d.ts +142 -0
- package/dist/runtime/run-orphan-check.job.js +339 -0
- package/dist/runtime/run-processors.d.ts +16 -9
- package/dist/runtime/run-processors.js +1 -1
- package/dist/runtime/run-queue.d.ts +64 -55
- package/dist/runtime/run-queue.js +147 -284
- package/dist/runtime/run-steps.d.ts +1 -2
- package/dist/runtime/run-steps.js +8 -11
- package/dist/runtime/run-worker.d.ts +6 -17
- package/dist/runtime/run-worker.js +32 -34
- package/dist/runtime/runtime.module.d.ts +2 -3
- package/dist/runtime/runtime.module.js +5 -6
- package/dist/runtime/step-log.d.ts +16 -4
- package/dist/runtime/step-log.js +26 -6
- package/dist/runtime/stream.controller.d.ts +2 -4
- package/dist/runtime/stream.controller.js +5 -11
- package/dist/runtime/subagents/child.js +1 -1
- package/dist/runtime/subagents/definition.d.ts +2 -1
- package/dist/runtime/subagents/definition.js +4 -2
- package/dist/runtime/subagents/deliveries.d.ts +6 -3
- package/dist/runtime/subagents/deliveries.js +16 -7
- package/dist/runtime/subagents/dispatch.d.ts +12 -12
- package/dist/runtime/subagents/dispatch.js +13 -38
- package/dist/runtime/subagents/messages.d.ts +5 -1
- package/dist/runtime/subagents/messages.js +11 -3
- package/dist/runtime/subagents/skill-fork.d.ts +3 -3
- package/dist/runtime/subagents/skill-fork.js +5 -5
- package/dist/runtime/subagents/subagents.service.js +1 -1
- package/dist/runtime/subagents/tree-jobs.d.ts +61 -0
- package/dist/runtime/subagents/tree-jobs.js +86 -0
- package/dist/runtime/usage-report-jobs.d.ts +23 -48
- package/dist/runtime/usage-report-jobs.js +67 -119
- package/dist/runtime/usage-report.d.ts +28 -77
- package/dist/runtime/usage-report.js +35 -213
- package/dist/runtime/waits.d.ts +48 -198
- package/dist/runtime/waits.js +103 -805
- package/dist/sandbox/platform-tiers.d.ts +86 -0
- package/dist/sandbox/platform-tiers.js +97 -0
- package/dist/sandbox/sandbox.service.d.ts +18 -5
- package/dist/sandbox/sandbox.service.js +75 -20
- package/dist/sandbox/tiers.d.ts +14 -2
- package/dist/sandbox/tiers.js +8 -0
- package/dist/schedules/calendar.d.ts +134 -0
- package/dist/schedules/calendar.js +410 -0
- package/dist/schedules/fire.d.ts +1 -3
- package/dist/schedules/fire.js +5 -10
- package/dist/schedules/run-tracker.d.ts +0 -5
- package/dist/schedules/run-tracker.js +0 -13
- package/dist/schedules/scan.job.js +15 -20
- package/dist/schedules/schedule-math.d.ts +42 -0
- package/dist/schedules/schedule-math.js +52 -0
- package/dist/schedules/scheduled-tasks.service.d.ts +16 -0
- package/dist/schedules/scheduled-tasks.service.js +53 -0
- package/dist/schedules/schedules.controller.d.ts +5 -1
- package/dist/schedules/schedules.controller.js +15 -1
- package/dist/schedules/schedules.schemas.d.ts +35 -0
- package/dist/schedules/schedules.schemas.js +27 -0
- package/dist/schedules/store.d.ts +5 -0
- package/dist/schedules/store.js +20 -0
- package/dist/tools/approvals.service.d.ts +26 -7
- package/dist/tools/approvals.service.js +88 -10
- package/dist/tools/builtin-names.d.ts +5 -3
- package/dist/tools/builtin-names.js +12 -4
- package/dist/tools/builtins/builtin-tools.d.ts +1 -2
- package/dist/tools/builtins/builtin-tools.js +21 -17
- package/dist/tools/builtins/sandbox/bash.js +9 -3
- package/dist/tools/builtins/sandbox/common.d.ts +18 -1
- package/dist/tools/builtins/sandbox/common.js +14 -1
- package/dist/tools/builtins/sandbox/index.d.ts +2 -1
- package/dist/tools/builtins/sandbox/index.js +4 -1
- package/dist/tools/builtins/sandbox/task-list.d.ts +12 -0
- package/dist/tools/builtins/sandbox/task-list.js +130 -0
- package/dist/tools/builtins/sandbox/task-output.d.ts +8 -1
- package/dist/tools/builtins/sandbox/task-output.js +21 -3
- package/dist/tools/builtins/web/extract.d.ts +2 -4
- package/dist/tools/builtins/web/extract.js +2 -5
- package/dist/tools/pipeline.d.ts +1 -1
- package/dist/tools/pipeline.js +1 -1
- package/dist/tools/questions.service.d.ts +15 -6
- package/dist/tools/questions.service.js +20 -8
- package/dist/tools/registry.d.ts +4 -2
- package/dist/tools/registry.js +4 -3
- package/dist/tools/tools.module.js +4 -3
- package/package.json +3 -3
- package/src/migrations/0118_device_policy_ops.sql +6 -0
- package/src/migrations/0119_outbox.sql +22 -0
- package/src/migrations/0120_drop_usage_report_sweep.sql +7 -0
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
8
|
+
return function (target, key) { decorator(target, key, paramIndex); }
|
|
9
|
+
};
|
|
10
|
+
import { getQueueToken } from "@nestjs/bullmq";
|
|
11
|
+
import { Inject, Injectable, Optional } from "@nestjs/common";
|
|
12
|
+
import { ModuleRef } from "@nestjs/core";
|
|
13
|
+
import { sql } from "kysely";
|
|
14
|
+
import { KernelDb } from "../database/kernel-db.js";
|
|
15
|
+
import { TransactionHost } from "../database/transaction-host.js";
|
|
16
|
+
import { Transactional } from "../database/transactional.decorator.js";
|
|
17
|
+
import { QUEUE_JOB_DEFAULTS } from "../queue/queue.module.js";
|
|
18
|
+
import { KERNEL_QUEUES } from "../queue/queue-names.js";
|
|
19
|
+
import { KERNEL_LOGGER } from "../tokens.js";
|
|
20
|
+
import { outboxJobOptions } from "./registry.js";
|
|
21
|
+
/**
|
|
22
|
+
* Retry schedule of a row whose add failed (spec §3.4): the next attempt waits
|
|
23
|
+
* `min(capMs, baseMs * 2^attempts)`, and the row is dead (kept, never retried) at `maxAttempts`.
|
|
24
|
+
*/
|
|
25
|
+
export const OUTBOX_BACKOFF = { baseMs: 1000, capMs: 300_000, maxAttempts: 20 };
|
|
26
|
+
/**
|
|
27
|
+
* How long the queue adds of one claim pass may take in all: the claim transaction holds its rows'
|
|
28
|
+
* locks and a pool connection for at most about this long.
|
|
29
|
+
*/
|
|
30
|
+
export const OUTBOX_REDIS_TIMEOUT_MS = 2000;
|
|
31
|
+
/**
|
|
32
|
+
* Most fast-path passes running at once in one process; more are skipped and left to the relay,
|
|
33
|
+
* so a slow Redis cannot take the whole pool (a local resource bound, no shared state).
|
|
34
|
+
*/
|
|
35
|
+
export const OUTBOX_FAST_PATH_CONCURRENCY = 4;
|
|
36
|
+
/**
|
|
37
|
+
* Test seam (`KernelOverride`): returns true to make the adds to the given queue fail (a Redis
|
|
38
|
+
* outage) without calling Redis. The kernel never provides it; the dispatcher defaults to "never
|
|
39
|
+
* fails".
|
|
40
|
+
*/
|
|
41
|
+
export const OUTBOX_DISPATCH_FAILS = Symbol("OUTBOX_DISPATCH_FAILS");
|
|
42
|
+
/** Longest `last_error` kept on a row. */
|
|
43
|
+
const MAX_ERROR_LENGTH = 2000;
|
|
44
|
+
const errMsg = (e) => (e instanceof Error ? e.message : String(e));
|
|
45
|
+
const NOT_READY = Symbol("NOT_READY");
|
|
46
|
+
/**
|
|
47
|
+
* Whether the queue's Redis connection is ready now, without waiting: the backend's client
|
|
48
|
+
* promise only settles once connected, so a pending one counts as not ready.
|
|
49
|
+
*/
|
|
50
|
+
async function connectionReady(queue) {
|
|
51
|
+
try {
|
|
52
|
+
// an already settled client promise wins the race (its callback is queued first)
|
|
53
|
+
const client = await Promise.race([queue.getBackend().client, Promise.resolve(NOT_READY)]);
|
|
54
|
+
return client !== NOT_READY && client.status === "ready";
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/** Rejects when `p` takes longer than `ms`; a late rejection of `p` is swallowed. */
|
|
61
|
+
async function withTimeout(p, ms, what) {
|
|
62
|
+
p.catch(() => { });
|
|
63
|
+
let timer;
|
|
64
|
+
const timeout = new Promise((_, reject) => {
|
|
65
|
+
timer = setTimeout(() => reject(new Error(`${what} timed out after ${ms} ms`)), ms);
|
|
66
|
+
});
|
|
67
|
+
try {
|
|
68
|
+
return await Promise.race([p, timeout]);
|
|
69
|
+
}
|
|
70
|
+
finally {
|
|
71
|
+
clearTimeout(timer);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Delivers outbox rows to BullMQ (transactional outbox spec §3.3, §3.4). A pass claims rows
|
|
76
|
+
* `FOR UPDATE SKIP LOCKED` (so concurrent passes never take the same row), adds their jobs per
|
|
77
|
+
* queue with `addBulk` (the queue defaults overlaid by the registry's options of the job name, the
|
|
78
|
+
* row's job id, a future `run_at` as the remaining `delay`), and deletes
|
|
79
|
+
* the delivered rows in the same short transaction. A crash after the add and before the commit
|
|
80
|
+
* leaves the rows; the next pass adds again and BullMQ ignores the existing job id. A queue whose
|
|
81
|
+
* add fails keeps its rows, and a second short update records the attempt with exponential
|
|
82
|
+
* backoff, setting `dead_at` at the last one.
|
|
83
|
+
*
|
|
84
|
+
* The fast path never queues behind an outage: it skips the pass (the relay delivers the rows) when
|
|
85
|
+
* a queue's Redis connection is not ready or {@link OUTBOX_FAST_PATH_CONCURRENCY} passes already
|
|
86
|
+
* run. Within a pass, the first failed queue fails the queues after it without trying them.
|
|
87
|
+
*
|
|
88
|
+
* The queue handles are the instances the queues' owner modules registered (one per queue, no
|
|
89
|
+
* connection of the outbox's own), resolved by token at use time. A queue whose owner this
|
|
90
|
+
* process's roles do not load is not reachable here: no pass claims its rows, which stay for a
|
|
91
|
+
* process that has it.
|
|
92
|
+
*/
|
|
93
|
+
let OutboxDispatcher = class OutboxDispatcher {
|
|
94
|
+
txHost;
|
|
95
|
+
db;
|
|
96
|
+
log;
|
|
97
|
+
moduleRef;
|
|
98
|
+
dispatchFails;
|
|
99
|
+
/** The owners' queue instances resolved so far (a per-process handle cache). */
|
|
100
|
+
queues = new Map();
|
|
101
|
+
/** Fast-path passes running in this process (bounded by OUTBOX_FAST_PATH_CONCURRENCY). */
|
|
102
|
+
fastPaths = 0;
|
|
103
|
+
constructor(txHost, db, log, moduleRef, dispatchFails = () => false) {
|
|
104
|
+
this.txHost = txHost;
|
|
105
|
+
this.db = db;
|
|
106
|
+
this.log = log;
|
|
107
|
+
this.moduleRef = moduleRef;
|
|
108
|
+
this.dispatchFails = dispatchFails;
|
|
109
|
+
}
|
|
110
|
+
/** The queue instance its owner module registered, or undefined when no loaded module has it. */
|
|
111
|
+
queue(name) {
|
|
112
|
+
const known = this.queues.get(name);
|
|
113
|
+
if (known)
|
|
114
|
+
return known;
|
|
115
|
+
let queue;
|
|
116
|
+
try {
|
|
117
|
+
queue = this.moduleRef.get(getQueueToken(name), { strict: false });
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
return undefined;
|
|
121
|
+
}
|
|
122
|
+
if (queue)
|
|
123
|
+
this.queues.set(name, queue);
|
|
124
|
+
return queue;
|
|
125
|
+
}
|
|
126
|
+
/** The kernel queues this process can add to, with their handles. */
|
|
127
|
+
reachable() {
|
|
128
|
+
const found = [];
|
|
129
|
+
for (const name of KERNEL_QUEUES) {
|
|
130
|
+
const queue = this.queue(name);
|
|
131
|
+
if (queue)
|
|
132
|
+
found.push([name, queue]);
|
|
133
|
+
}
|
|
134
|
+
return found;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* The fast path: delivers the given rows (a committed transaction's own) right away. Never
|
|
138
|
+
* throws: whatever it does not deliver stays for the relay, and a failure is logged.
|
|
139
|
+
*/
|
|
140
|
+
async dispatchOwn(ids) {
|
|
141
|
+
if (ids.length === 0)
|
|
142
|
+
return;
|
|
143
|
+
if (this.fastPaths >= OUTBOX_FAST_PATH_CONCURRENCY) {
|
|
144
|
+
this.log.debug("outbox fast path busy; the relay delivers the rows", { rows: ids.length });
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
this.fastPaths++;
|
|
148
|
+
try {
|
|
149
|
+
if (!(await this.redisReady())) {
|
|
150
|
+
this.log.debug("outbox fast path: Redis not ready; the relay delivers the rows", {
|
|
151
|
+
rows: ids.length,
|
|
152
|
+
});
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
await this.dispatch({ ids });
|
|
156
|
+
}
|
|
157
|
+
catch (e) {
|
|
158
|
+
this.log.error("outbox fast path failed; the relay delivers the rows", {
|
|
159
|
+
rows: ids.length,
|
|
160
|
+
error: errMsg(e),
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
finally {
|
|
164
|
+
this.fastPaths--;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Every reachable queue's connection is ready (they reach the same Redis; the fast path does not
|
|
169
|
+
* read its rows' queues before claiming them).
|
|
170
|
+
*/
|
|
171
|
+
async redisReady() {
|
|
172
|
+
const ready = await Promise.all(this.reachable().map(([, q]) => connectionReady(q)));
|
|
173
|
+
return ready.every(Boolean);
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* The relay: delivers up to `limit` due rows (not dead, `next_attempt_at` reached), oldest
|
|
177
|
+
* first.
|
|
178
|
+
*/
|
|
179
|
+
dispatchDue(limit) {
|
|
180
|
+
return this.dispatch({ limit });
|
|
181
|
+
}
|
|
182
|
+
async dispatch(claim) {
|
|
183
|
+
const { sent, failed } = await this.claimAndSend(claim);
|
|
184
|
+
let dead = 0;
|
|
185
|
+
for (const group of failed)
|
|
186
|
+
dead += await this.recordFailure(group);
|
|
187
|
+
return { sent, failed: failed.reduce((n, g) => n + g.ids.length, 0), dead };
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* One claim transaction: locks the rows (skipping rows another pass holds), adds their jobs and
|
|
191
|
+
* deletes the delivered rows. The only Redis calls made while a PG transaction is open in the
|
|
192
|
+
* kernel (spec §3.3): the locks held are outbox rows only, and the pass's adds together are
|
|
193
|
+
* bounded by {@link OUTBOX_REDIS_TIMEOUT_MS}.
|
|
194
|
+
*/
|
|
195
|
+
async claimAndSend(claim) {
|
|
196
|
+
const queues = new Map(this.reachable());
|
|
197
|
+
if (queues.size === 0)
|
|
198
|
+
return { sent: 0, failed: [] };
|
|
199
|
+
// rows of a queue this process cannot reach are left for a process that can
|
|
200
|
+
let query = this.db
|
|
201
|
+
.selectFrom("outbox")
|
|
202
|
+
.select(["id", "queue", "job_name", "job_id", "data", "run_at"])
|
|
203
|
+
.where("dead_at", "is", null)
|
|
204
|
+
.where("queue", "in", [...queues.keys()]);
|
|
205
|
+
query =
|
|
206
|
+
"ids" in claim
|
|
207
|
+
? query.where("id", "in", [...claim.ids])
|
|
208
|
+
: query.where("next_attempt_at", "<=", sql `now()`).limit(claim.limit);
|
|
209
|
+
const rows = await query.orderBy("id").forUpdate().skipLocked().execute();
|
|
210
|
+
if (rows.length === 0)
|
|
211
|
+
return { sent: 0, failed: [] };
|
|
212
|
+
const byQueue = new Map();
|
|
213
|
+
for (const row of rows) {
|
|
214
|
+
const group = byQueue.get(row.queue);
|
|
215
|
+
if (group)
|
|
216
|
+
group.push(row);
|
|
217
|
+
else
|
|
218
|
+
byQueue.set(row.queue, [row]);
|
|
219
|
+
}
|
|
220
|
+
const now = Date.now();
|
|
221
|
+
const deadline = now + OUTBOX_REDIS_TIMEOUT_MS;
|
|
222
|
+
const delivered = [];
|
|
223
|
+
const failed = [];
|
|
224
|
+
// after the first failure the queues left are failed without a try: Redis is likely down
|
|
225
|
+
let firstError;
|
|
226
|
+
for (const [name, group] of byQueue) {
|
|
227
|
+
const ids = group.map((r) => r.id);
|
|
228
|
+
if (firstError !== undefined) {
|
|
229
|
+
failed.push({
|
|
230
|
+
ids,
|
|
231
|
+
error: `not tried after an earlier queue failed: ${firstError}`.slice(0, MAX_ERROR_LENGTH),
|
|
232
|
+
});
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
try {
|
|
236
|
+
if (this.dispatchFails(name))
|
|
237
|
+
throw new Error("outbox dispatch failure (test seam)");
|
|
238
|
+
const queue = queues.get(name);
|
|
239
|
+
if (!queue)
|
|
240
|
+
throw new Error(`"${name}" is not a kernel queue`);
|
|
241
|
+
const left = deadline - Date.now();
|
|
242
|
+
if (left <= 0)
|
|
243
|
+
throw new Error(`no time left of the pass's ${OUTBOX_REDIS_TIMEOUT_MS} ms`);
|
|
244
|
+
await withTimeout(queue.addBulk(group.map((r) => ({
|
|
245
|
+
name: r.job_name,
|
|
246
|
+
data: r.data,
|
|
247
|
+
opts: {
|
|
248
|
+
...QUEUE_JOB_DEFAULTS,
|
|
249
|
+
...outboxJobOptions(r.job_name),
|
|
250
|
+
jobId: r.job_id,
|
|
251
|
+
delay: r.run_at ? Math.max(0, r.run_at.getTime() - now) : 0,
|
|
252
|
+
},
|
|
253
|
+
}))), left, `addBulk to ${name}`);
|
|
254
|
+
delivered.push(...ids);
|
|
255
|
+
}
|
|
256
|
+
catch (e) {
|
|
257
|
+
firstError = errMsg(e);
|
|
258
|
+
failed.push({ ids, error: firstError.slice(0, MAX_ERROR_LENGTH) });
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
if (delivered.length > 0)
|
|
262
|
+
await this.db.deleteFrom("outbox").where("id", "in", delivered).execute();
|
|
263
|
+
return { sent: delivered.length, failed };
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Records a failed add on its rows (after the claim transaction, so the attempt is kept whatever
|
|
267
|
+
* that transaction did): one more attempt, the error, the next attempt by {@link OUTBOX_BACKOFF},
|
|
268
|
+
* and `dead_at` at the last attempt. Returns how many rows it made dead.
|
|
269
|
+
*/
|
|
270
|
+
async recordFailure(group) {
|
|
271
|
+
const { baseMs, capMs, maxAttempts } = OUTBOX_BACKOFF;
|
|
272
|
+
// the right-hand sides read the row before the update: the first failure waits baseMs
|
|
273
|
+
const backoffMs = sql `least(${capMs}::float8, ${baseMs}::float8 * power(2, attempts))`;
|
|
274
|
+
const updated = await this.db
|
|
275
|
+
.updateTable("outbox")
|
|
276
|
+
.set({
|
|
277
|
+
attempts: sql `attempts + 1`,
|
|
278
|
+
last_error: group.error,
|
|
279
|
+
next_attempt_at: sql `now() + ${backoffMs} * interval '1 millisecond'`,
|
|
280
|
+
dead_at: sql `case when attempts + 1 >= ${maxAttempts}::int then now() end`,
|
|
281
|
+
})
|
|
282
|
+
.where("id", "in", group.ids)
|
|
283
|
+
.where("dead_at", "is", null)
|
|
284
|
+
.returning(["dead_at"])
|
|
285
|
+
.execute();
|
|
286
|
+
const dead = updated.filter((r) => r.dead_at !== null).length;
|
|
287
|
+
this.log.warn("outbox delivery failed; retrying with backoff", {
|
|
288
|
+
rows: group.ids.length,
|
|
289
|
+
dead,
|
|
290
|
+
error: group.error,
|
|
291
|
+
});
|
|
292
|
+
if (dead > 0) {
|
|
293
|
+
this.log.error("outbox rows dead after the last attempt", { rows: dead, error: group.error });
|
|
294
|
+
}
|
|
295
|
+
return dead;
|
|
296
|
+
}
|
|
297
|
+
};
|
|
298
|
+
__decorate([
|
|
299
|
+
Transactional({ propagation: "requires_new" })
|
|
300
|
+
], OutboxDispatcher.prototype, "claimAndSend", null);
|
|
301
|
+
OutboxDispatcher = __decorate([
|
|
302
|
+
Injectable(),
|
|
303
|
+
__param(0, Inject(TransactionHost)),
|
|
304
|
+
__param(1, Inject(KernelDb)),
|
|
305
|
+
__param(2, Inject(KERNEL_LOGGER)),
|
|
306
|
+
__param(3, Inject(ModuleRef)),
|
|
307
|
+
__param(4, Optional()),
|
|
308
|
+
__param(4, Inject(OUTBOX_DISPATCH_FAILS))
|
|
309
|
+
], OutboxDispatcher);
|
|
310
|
+
export { OutboxDispatcher };
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { OutboxDispatcher } from "./outbox.dispatcher.js";
|
|
2
|
+
import { type OutboxFastPath } from "./outbox.service.js";
|
|
3
|
+
/**
|
|
4
|
+
* Test seam (`KernelOverride`): receives each detached fast-path pass, so a test can await what a
|
|
5
|
+
* request started after its commit (the kernel never provides it).
|
|
6
|
+
*/
|
|
7
|
+
export declare const OUTBOX_FAST_PATH_SETTLED: unique symbol;
|
|
8
|
+
/**
|
|
9
|
+
* The fast path (spec §3.3): starts the dispatcher on a committed transaction's rows and returns
|
|
10
|
+
* at once, so the commit's caller never waits on Redis (R9). `dispatchOwn` never throws; what it
|
|
11
|
+
* does not deliver stays for the relay.
|
|
12
|
+
*/
|
|
13
|
+
export declare const outboxFastPath: (dispatcher: Pick<OutboxDispatcher, "dispatchOwn">, settled?: (pass: Promise<void>) => void) => OutboxFastPath;
|
|
14
|
+
/**
|
|
15
|
+
* Global: `OutboxService` (owed work written in the caller's transaction, transactional outbox
|
|
16
|
+
* spec §3.2), the `OutboxDispatcher` that delivers the rows (the relay drives it too) and the fast
|
|
17
|
+
* path `add` hands each committed transaction's rows to.
|
|
18
|
+
*
|
|
19
|
+
* It registers no queue: the dispatcher adds through the instances the queues' owner modules
|
|
20
|
+
* registered (one instance and connection per queue, as everywhere in the kernel), resolved at use
|
|
21
|
+
* time, and leaves the rows of a queue whose owner this process does not load to one that does.
|
|
22
|
+
*/
|
|
23
|
+
export declare class OutboxModule {
|
|
24
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
import { Global, Module } from "@nestjs/common";
|
|
8
|
+
import { OutboxDispatcher } from "./outbox.dispatcher.js";
|
|
9
|
+
import { OUTBOX_FAST_PATH, OutboxService } from "./outbox.service.js";
|
|
10
|
+
/**
|
|
11
|
+
* Test seam (`KernelOverride`): receives each detached fast-path pass, so a test can await what a
|
|
12
|
+
* request started after its commit (the kernel never provides it).
|
|
13
|
+
*/
|
|
14
|
+
export const OUTBOX_FAST_PATH_SETTLED = Symbol("OUTBOX_FAST_PATH_SETTLED");
|
|
15
|
+
/**
|
|
16
|
+
* The fast path (spec §3.3): starts the dispatcher on a committed transaction's rows and returns
|
|
17
|
+
* at once, so the commit's caller never waits on Redis (R9). `dispatchOwn` never throws; what it
|
|
18
|
+
* does not deliver stays for the relay.
|
|
19
|
+
*/
|
|
20
|
+
export const outboxFastPath = (dispatcher, settled) => async (ids) => {
|
|
21
|
+
const pass = dispatcher.dispatchOwn(ids);
|
|
22
|
+
settled?.(pass);
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* Global: `OutboxService` (owed work written in the caller's transaction, transactional outbox
|
|
26
|
+
* spec §3.2), the `OutboxDispatcher` that delivers the rows (the relay drives it too) and the fast
|
|
27
|
+
* path `add` hands each committed transaction's rows to.
|
|
28
|
+
*
|
|
29
|
+
* It registers no queue: the dispatcher adds through the instances the queues' owner modules
|
|
30
|
+
* registered (one instance and connection per queue, as everywhere in the kernel), resolved at use
|
|
31
|
+
* time, and leaves the rows of a queue whose owner this process does not load to one that does.
|
|
32
|
+
*/
|
|
33
|
+
let OutboxModule = class OutboxModule {
|
|
34
|
+
};
|
|
35
|
+
OutboxModule = __decorate([
|
|
36
|
+
Global(),
|
|
37
|
+
Module({
|
|
38
|
+
providers: [
|
|
39
|
+
OutboxService,
|
|
40
|
+
OutboxDispatcher,
|
|
41
|
+
{
|
|
42
|
+
provide: OUTBOX_FAST_PATH,
|
|
43
|
+
inject: [OutboxDispatcher, { token: OUTBOX_FAST_PATH_SETTLED, optional: true }],
|
|
44
|
+
useFactory: (dispatcher, settled) => outboxFastPath(dispatcher, settled ?? undefined),
|
|
45
|
+
},
|
|
46
|
+
],
|
|
47
|
+
exports: [OutboxService, OutboxDispatcher, OUTBOX_FAST_PATH],
|
|
48
|
+
})
|
|
49
|
+
], OutboxModule);
|
|
50
|
+
export { OutboxModule };
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { KernelLogger } from "../../log.js";
|
|
2
|
+
import { KernelConfigService } from "../config/kernel-config.service.js";
|
|
3
|
+
import { LockService } from "../lock/lock.service.js";
|
|
4
|
+
import { OutboxDispatcher } from "./outbox.dispatcher.js";
|
|
5
|
+
/** The relay's `@LockedInterval` job name (its run lock key is the default `cron:outbox-relay`). */
|
|
6
|
+
export declare const OUTBOX_RELAY_JOB = "outbox-relay";
|
|
7
|
+
/** The relay's period: a fixed constant, not a config field. */
|
|
8
|
+
export declare const OUTBOX_RELAY_PERIOD_MS = 2000;
|
|
9
|
+
/** Rows one claim pass of the relay takes. */
|
|
10
|
+
export declare const OUTBOX_RELAY_BATCH = 500;
|
|
11
|
+
/** Most claim passes in one tick; what is left waits for the next tick. */
|
|
12
|
+
export declare const OUTBOX_RELAY_MAX_ROUNDS = 10;
|
|
13
|
+
/**
|
|
14
|
+
* The outbox relay (transactional outbox spec §3.3), on the worker role only: once per tick across
|
|
15
|
+
* replicas it delivers the due rows the fast path did not (a failed or skipped pass, a process that
|
|
16
|
+
* died after the commit, a retry whose backoff ran out). A tick runs passes of
|
|
17
|
+
* {@link OUTBOX_RELAY_BATCH} rows until one handles fewer (the backlog is drained) or
|
|
18
|
+
* {@link OUTBOX_RELAY_MAX_ROUNDS} have run. It also stops early when the run is aborted, and after
|
|
19
|
+
* a pass that delivered nothing but failed rows (Redis trouble: the next tick tries again, rather
|
|
20
|
+
* than spinning more passes that each hold a pool connection).
|
|
21
|
+
*/
|
|
22
|
+
export declare class OutboxRelay {
|
|
23
|
+
private readonly dispatcher;
|
|
24
|
+
readonly config: KernelConfigService;
|
|
25
|
+
readonly lockService: LockService;
|
|
26
|
+
readonly log: KernelLogger;
|
|
27
|
+
constructor(dispatcher: Pick<OutboxDispatcher, "dispatchDue">, config: KernelConfigService, lockService: LockService, log: KernelLogger);
|
|
28
|
+
relay(signal?: AbortSignal): Promise<void>;
|
|
29
|
+
/** One tick's passes (the body of {@link relay}); returns the rows delivered. */
|
|
30
|
+
relayOnce(signal?: AbortSignal): Promise<number>;
|
|
31
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
8
|
+
return function (target, key) { decorator(target, key, paramIndex); }
|
|
9
|
+
};
|
|
10
|
+
import { Inject, Injectable } from "@nestjs/common";
|
|
11
|
+
import { KernelConfigService } from "../config/kernel-config.service.js";
|
|
12
|
+
import { LockService } from "../lock/lock.service.js";
|
|
13
|
+
import { LockedInterval } from "../schedule/locked-interval.js";
|
|
14
|
+
import { KERNEL_LOGGER } from "../tokens.js";
|
|
15
|
+
import { OutboxDispatcher } from "./outbox.dispatcher.js";
|
|
16
|
+
/** The relay's `@LockedInterval` job name (its run lock key is the default `cron:outbox-relay`). */
|
|
17
|
+
export const OUTBOX_RELAY_JOB = "outbox-relay";
|
|
18
|
+
/** The relay's period: a fixed constant, not a config field. */
|
|
19
|
+
export const OUTBOX_RELAY_PERIOD_MS = 2000;
|
|
20
|
+
/** Rows one claim pass of the relay takes. */
|
|
21
|
+
export const OUTBOX_RELAY_BATCH = 500;
|
|
22
|
+
/** Most claim passes in one tick; what is left waits for the next tick. */
|
|
23
|
+
export const OUTBOX_RELAY_MAX_ROUNDS = 10;
|
|
24
|
+
/**
|
|
25
|
+
* The outbox relay (transactional outbox spec §3.3), on the worker role only: once per tick across
|
|
26
|
+
* replicas it delivers the due rows the fast path did not (a failed or skipped pass, a process that
|
|
27
|
+
* died after the commit, a retry whose backoff ran out). A tick runs passes of
|
|
28
|
+
* {@link OUTBOX_RELAY_BATCH} rows until one handles fewer (the backlog is drained) or
|
|
29
|
+
* {@link OUTBOX_RELAY_MAX_ROUNDS} have run. It also stops early when the run is aborted, and after
|
|
30
|
+
* a pass that delivered nothing but failed rows (Redis trouble: the next tick tries again, rather
|
|
31
|
+
* than spinning more passes that each hold a pool connection).
|
|
32
|
+
*/
|
|
33
|
+
let OutboxRelay = class OutboxRelay {
|
|
34
|
+
dispatcher;
|
|
35
|
+
config;
|
|
36
|
+
lockService;
|
|
37
|
+
log;
|
|
38
|
+
constructor(dispatcher, config, lockService, log) {
|
|
39
|
+
this.dispatcher = dispatcher;
|
|
40
|
+
this.config = config;
|
|
41
|
+
this.lockService = lockService;
|
|
42
|
+
this.log = log;
|
|
43
|
+
}
|
|
44
|
+
async relay(signal) {
|
|
45
|
+
await this.relayOnce(signal);
|
|
46
|
+
}
|
|
47
|
+
/** One tick's passes (the body of {@link relay}); returns the rows delivered. */
|
|
48
|
+
async relayOnce(signal) {
|
|
49
|
+
let sent = 0;
|
|
50
|
+
for (let round = 0; round < OUTBOX_RELAY_MAX_ROUNDS; round++) {
|
|
51
|
+
if (signal?.aborted)
|
|
52
|
+
break;
|
|
53
|
+
const r = await this.dispatcher.dispatchDue(OUTBOX_RELAY_BATCH);
|
|
54
|
+
sent += r.sent;
|
|
55
|
+
// rows handled: `dead` is a part of `failed`, not counted again
|
|
56
|
+
if (r.sent + r.failed < OUTBOX_RELAY_BATCH)
|
|
57
|
+
break;
|
|
58
|
+
if (r.sent === 0 && r.failed > 0)
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
if (sent > 0)
|
|
62
|
+
this.log.debug("outbox relay delivered rows", { rows: sent });
|
|
63
|
+
return sent;
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
__decorate([
|
|
67
|
+
LockedInterval(OUTBOX_RELAY_JOB, () => OUTBOX_RELAY_PERIOD_MS)
|
|
68
|
+
], OutboxRelay.prototype, "relay", null);
|
|
69
|
+
OutboxRelay = __decorate([
|
|
70
|
+
Injectable(),
|
|
71
|
+
__param(0, Inject(OutboxDispatcher)),
|
|
72
|
+
__param(1, Inject(KernelConfigService)),
|
|
73
|
+
__param(2, Inject(LockService)),
|
|
74
|
+
__param(3, Inject(KERNEL_LOGGER))
|
|
75
|
+
], OutboxRelay);
|
|
76
|
+
export { OutboxRelay };
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { KernelLogger } from "../../log.js";
|
|
2
|
+
import { KernelDb } from "../database/kernel-db.js";
|
|
3
|
+
import { TransactionHost } from "../database/transaction-host.js";
|
|
4
|
+
import type { KernelQueueName } from "../queue/queue-names.js";
|
|
5
|
+
import type { OutboxJobName } from "./registry.js";
|
|
6
|
+
/** A BullMQ job owed by the caller's transaction (transactional outbox spec §5). */
|
|
7
|
+
export interface OutboxJob<T> {
|
|
8
|
+
/** One of the kernel queues. */
|
|
9
|
+
queue: KernelQueueName;
|
|
10
|
+
/** A registered job name ({@link OutboxJobName}). */
|
|
11
|
+
name: OutboxJobName;
|
|
12
|
+
/** Deterministic, no `:`; one pending (not dead) row per (queue, jobId). */
|
|
13
|
+
jobId: string;
|
|
14
|
+
/** JSON data: stored as jsonb, so what JSON cannot carry (undefined, Date objects) is lost. */
|
|
15
|
+
data: T;
|
|
16
|
+
/** Delay target: delivered as a BullMQ `delay` computed at dispatch. */
|
|
17
|
+
runAt?: Date;
|
|
18
|
+
/** The org the work belongs to (its deletion removes the pending row); absent for platform work. */
|
|
19
|
+
orgId?: string;
|
|
20
|
+
}
|
|
21
|
+
/** Delivers the given outbox rows right after their transaction committed (the fast path, §3.3). */
|
|
22
|
+
export type OutboxFastPath = (ids: readonly string[]) => Promise<void>;
|
|
23
|
+
/** The fast path `add` hands each committed transaction's rows to (`OutboxModule` provides it). */
|
|
24
|
+
export declare const OUTBOX_FAST_PATH: unique symbol;
|
|
25
|
+
/**
|
|
26
|
+
* Test seam (`KernelOverride`): `true` makes a reused job id with different data throw instead of
|
|
27
|
+
* logging an error (spec §4, designer's call: "throws in tests, logs in production").
|
|
28
|
+
*/
|
|
29
|
+
export declare const OUTBOX_STRICT_CONFLICTS: unique symbol;
|
|
30
|
+
/**
|
|
31
|
+
* Writes owed work as a row in the caller's transaction (spec §3.2): the row commits or rolls back
|
|
32
|
+
* with the business write, and after the commit the fast path delivers it when it is due now (a
|
|
33
|
+
* row with a future `runAt` waits for the relay, spec §3.3).
|
|
34
|
+
*/
|
|
35
|
+
export declare class OutboxService {
|
|
36
|
+
readonly txHost: TransactionHost;
|
|
37
|
+
private readonly db;
|
|
38
|
+
private readonly log;
|
|
39
|
+
private readonly fastPath;
|
|
40
|
+
private readonly strictConflicts;
|
|
41
|
+
constructor(txHost: TransactionHost, db: KernelDb, log: KernelLogger, fastPath: OutboxFastPath, strictConflicts?: boolean);
|
|
42
|
+
/**
|
|
43
|
+
* Adds the job to the outbox in the transaction in context (throws outside one). Returns true when
|
|
44
|
+
* it inserted a row, false when the same (queue, jobId) is already owed (a no-op; a dead row is
|
|
45
|
+
* not owed). A conflict whose stored data differs is a bug in the job id: it throws under strict
|
|
46
|
+
* conflicts, else logs an error. Undefined data is stored as JSON null.
|
|
47
|
+
*/
|
|
48
|
+
add<T>(job: OutboxJob<T>): Promise<boolean>;
|
|
49
|
+
/** Collects `id` for its transaction; the first id registers the transaction's one hint. */
|
|
50
|
+
private deliverAfterCommit;
|
|
51
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
8
|
+
return function (target, key) { decorator(target, key, paramIndex); }
|
|
9
|
+
};
|
|
10
|
+
import { isDeepStrictEqual } from "node:util";
|
|
11
|
+
import { Inject, Injectable, Optional } from "@nestjs/common";
|
|
12
|
+
import { afterCommit } from "../database/after-commit.js";
|
|
13
|
+
import { KernelDb } from "../database/kernel-db.js";
|
|
14
|
+
import { currentStore, TransactionHost } from "../database/transaction-host.js";
|
|
15
|
+
import { Transactional } from "../database/transactional.decorator.js";
|
|
16
|
+
import { KERNEL_LOGGER } from "../tokens.js";
|
|
17
|
+
/** The fast path `add` hands each committed transaction's rows to (`OutboxModule` provides it). */
|
|
18
|
+
export const OUTBOX_FAST_PATH = Symbol("OUTBOX_FAST_PATH");
|
|
19
|
+
/**
|
|
20
|
+
* Test seam (`KernelOverride`): `true` makes a reused job id with different data throw instead of
|
|
21
|
+
* logging an error (spec §4, designer's call: "throws in tests, logs in production").
|
|
22
|
+
*/
|
|
23
|
+
export const OUTBOX_STRICT_CONFLICTS = Symbol("OUTBOX_STRICT_CONFLICTS");
|
|
24
|
+
/**
|
|
25
|
+
* The ids each open transaction inserted, keyed by its transaction store: lives only as long as
|
|
26
|
+
* that transaction (a WeakMap entry, no state shared across requests).
|
|
27
|
+
*/
|
|
28
|
+
const pendingIds = new WeakMap();
|
|
29
|
+
/**
|
|
30
|
+
* Writes owed work as a row in the caller's transaction (spec §3.2): the row commits or rolls back
|
|
31
|
+
* with the business write, and after the commit the fast path delivers it when it is due now (a
|
|
32
|
+
* row with a future `runAt` waits for the relay, spec §3.3).
|
|
33
|
+
*/
|
|
34
|
+
let OutboxService = class OutboxService {
|
|
35
|
+
txHost;
|
|
36
|
+
db;
|
|
37
|
+
log;
|
|
38
|
+
fastPath;
|
|
39
|
+
strictConflicts;
|
|
40
|
+
constructor(txHost, db, log, fastPath, strictConflicts = false) {
|
|
41
|
+
this.txHost = txHost;
|
|
42
|
+
this.db = db;
|
|
43
|
+
this.log = log;
|
|
44
|
+
this.fastPath = fastPath;
|
|
45
|
+
this.strictConflicts = strictConflicts;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Adds the job to the outbox in the transaction in context (throws outside one). Returns true when
|
|
49
|
+
* it inserted a row, false when the same (queue, jobId) is already owed (a no-op; a dead row is
|
|
50
|
+
* not owed). A conflict whose stored data differs is a bug in the job id: it throws under strict
|
|
51
|
+
* conflicts, else logs an error. Undefined data is stored as JSON null.
|
|
52
|
+
*/
|
|
53
|
+
async add(job) {
|
|
54
|
+
if (job.jobId.includes(":")) {
|
|
55
|
+
throw new Error(`outbox job id "${job.jobId}" must not contain ":" (BullMQ uses it as its key separator); ` +
|
|
56
|
+
'join the id\'s parts with "-" instead');
|
|
57
|
+
}
|
|
58
|
+
const json = JSON.stringify(job.data ?? null);
|
|
59
|
+
const row = await this.db
|
|
60
|
+
.insertInto("outbox")
|
|
61
|
+
.values({
|
|
62
|
+
queue: job.queue,
|
|
63
|
+
job_name: job.name,
|
|
64
|
+
job_id: job.jobId,
|
|
65
|
+
data: json,
|
|
66
|
+
run_at: job.runAt ?? null,
|
|
67
|
+
org_id: job.orgId ?? null,
|
|
68
|
+
})
|
|
69
|
+
.onConflict((oc) => oc.columns(["queue", "job_id"]).where("dead_at", "is", null).doNothing())
|
|
70
|
+
.returning("id")
|
|
71
|
+
.executeTakeFirst();
|
|
72
|
+
if (row) {
|
|
73
|
+
// only a job due now goes through the fast path (ruling R20): a delayed one (a backstop, a
|
|
74
|
+
// timeout) is claimed by the relay within its next tick and added with its remaining delay,
|
|
75
|
+
// so the commit's caller spends no claim transaction on work due in seconds or hours
|
|
76
|
+
if (job.runAt === undefined || job.runAt.getTime() <= Date.now()) {
|
|
77
|
+
this.deliverAfterCommit(row.id);
|
|
78
|
+
}
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
const stored = await this.db
|
|
82
|
+
.selectFrom("outbox")
|
|
83
|
+
.select(["job_name", "data"])
|
|
84
|
+
.where("queue", "=", job.queue)
|
|
85
|
+
.where("job_id", "=", job.jobId)
|
|
86
|
+
.where("dead_at", "is", null)
|
|
87
|
+
.executeTakeFirst();
|
|
88
|
+
// no row: the owed one was delivered (and deleted) since the conflict; BullMQ dedupes the id
|
|
89
|
+
if (stored && !isDeepStrictEqual(stored.data, JSON.parse(json))) {
|
|
90
|
+
const msg = "outbox job id reused with different data";
|
|
91
|
+
const fields = {
|
|
92
|
+
queue: job.queue,
|
|
93
|
+
jobId: job.jobId,
|
|
94
|
+
job: job.name,
|
|
95
|
+
owedJob: stored.job_name,
|
|
96
|
+
};
|
|
97
|
+
if (this.strictConflicts)
|
|
98
|
+
throw new Error(`${msg}: ${JSON.stringify(fields)}`);
|
|
99
|
+
this.log.error(msg, fields);
|
|
100
|
+
}
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
/** Collects `id` for its transaction; the first id registers the transaction's one hint. */
|
|
104
|
+
deliverAfterCommit(id) {
|
|
105
|
+
// `mandatory` guarantees a live store here
|
|
106
|
+
const store = currentStore();
|
|
107
|
+
if (!store)
|
|
108
|
+
throw new Error("outbox: no transaction in context");
|
|
109
|
+
const ids = pendingIds.get(store);
|
|
110
|
+
if (ids) {
|
|
111
|
+
ids.push(id);
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
const first = [id];
|
|
115
|
+
pendingIds.set(store, first);
|
|
116
|
+
afterCommit(() => this.fastPath(first));
|
|
117
|
+
}
|
|
118
|
+
};
|
|
119
|
+
__decorate([
|
|
120
|
+
Transactional({ propagation: "mandatory" })
|
|
121
|
+
], OutboxService.prototype, "add", null);
|
|
122
|
+
OutboxService = __decorate([
|
|
123
|
+
Injectable(),
|
|
124
|
+
__param(0, Inject(TransactionHost)),
|
|
125
|
+
__param(1, Inject(KernelDb)),
|
|
126
|
+
__param(2, Inject(KERNEL_LOGGER)),
|
|
127
|
+
__param(3, Inject(OUTBOX_FAST_PATH)),
|
|
128
|
+
__param(4, Optional()),
|
|
129
|
+
__param(4, Inject(OUTBOX_STRICT_CONFLICTS))
|
|
130
|
+
], OutboxService);
|
|
131
|
+
export { OutboxService };
|