@btravstack/outbox 0.0.0-stage โ 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +103 -2
- package/dist/index.cjs +213 -0
- package/dist/index.d.cts +63 -0
- package/dist/index.d.mts +63 -0
- package/dist/index.mjs +209 -0
- package/dist/outbox-Bn2wYUh6.d.cts +86 -0
- package/dist/outbox-Bn2wYUh6.d.mts +86 -0
- package/dist/prisma.cjs +85 -0
- package/dist/prisma.d.cts +52 -0
- package/dist/prisma.d.mts +52 -0
- package/dist/prisma.mjs +84 -0
- package/package.json +87 -3
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Benoit TRAVERS
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,104 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @btravstack/outbox
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
> The transactional outbox relay for [`@btravstack/core`](../core): a poll loop
|
|
4
|
+
> that publishes committed facts in outbox order, a per-tenant
|
|
5
|
+
> claim so replicas take turns rather than race for the same rows, and a Prisma
|
|
6
|
+
> 8 store.
|
|
7
|
+
|
|
8
|
+
๐ **[Documentation](https://btravstack.github.io/btravstack/reference/outbox)** ยท
|
|
9
|
+
[API Reference](https://btravstack.github.io/btravstack/api/outbox/)
|
|
10
|
+
|
|
11
|
+
```sh
|
|
12
|
+
pnpm add @btravstack/outbox @btravstack/core @btravstack/config @btravstack/di unthrown
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Four peer dependencies, plus `@prisma/orm-postgres` โ optional, and needed only
|
|
16
|
+
if you compose the Prisma store from `@btravstack/outbox/prisma`. Node `>=22`.
|
|
17
|
+
|
|
18
|
+
## A worked example
|
|
19
|
+
|
|
20
|
+
<!-- doctest: group=order-amqp-worker -->
|
|
21
|
+
<!-- doctest: prelude
|
|
22
|
+
import { Module, Port, Provider } from "@btravstack/di";
|
|
23
|
+
import type { OutboxDatabase } from "@btravstack/outbox/prisma";
|
|
24
|
+
import { TaggedError, type AsyncResult } from "unthrown";
|
|
25
|
+
|
|
26
|
+
// The application's own halves, declared here so this sample stands on the
|
|
27
|
+
// published packages alone: its database port, from `prismaDatabase`, and a
|
|
28
|
+
// transport client of its own contract.
|
|
29
|
+
class OrderDatabase extends Port("ReadmeOrderDatabase")<OutboxDatabase<unknown>> {}
|
|
30
|
+
class Refused extends TaggedError("Refused") {}
|
|
31
|
+
class Broker extends Port("ReadmeBroker")<{
|
|
32
|
+
readonly send: (topic: string, body: unknown) => AsyncResult<void, Refused>;
|
|
33
|
+
}> {}
|
|
34
|
+
declare const BrokerModule: Module<Broker, never, never>;
|
|
35
|
+
declare const OrderPersistenceModule: Module<OrderDatabase, never, never>;
|
|
36
|
+
-->
|
|
37
|
+
|
|
38
|
+
The application writes its outbox row in the same transaction as the row it
|
|
39
|
+
describes โ that stays its own. What it provides the relay is where the table
|
|
40
|
+
is, and what publishing a row means:
|
|
41
|
+
|
|
42
|
+
```ts
|
|
43
|
+
import { OutboxPublisher, OutboxStore, outbox } from "@btravstack/outbox";
|
|
44
|
+
import { prismaOutboxStore } from "@btravstack/outbox/prisma";
|
|
45
|
+
|
|
46
|
+
const store = Provider(OutboxStore)({
|
|
47
|
+
inject: { db: OrderDatabase },
|
|
48
|
+
sync: ({ db }) => prismaOutboxStore(db, { schema: "orders" }),
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
const publisher = Provider(OutboxPublisher)({
|
|
52
|
+
inject: { broker: Broker },
|
|
53
|
+
sync: ({ broker }) => ({
|
|
54
|
+
publish: (message) =>
|
|
55
|
+
broker.send(`${message.kind}.changed`, {
|
|
56
|
+
eventId: message.id,
|
|
57
|
+
tenantId: message.tenantId,
|
|
58
|
+
id: message.subjectId,
|
|
59
|
+
payload: message.payload === null ? null : JSON.parse(message.payload),
|
|
60
|
+
}),
|
|
61
|
+
}),
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
export const Relay = Module("Relay")({
|
|
65
|
+
imports: [OrderPersistenceModule, BrokerModule, outbox()],
|
|
66
|
+
provides: [store, publisher],
|
|
67
|
+
exports: [],
|
|
68
|
+
});
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Options
|
|
72
|
+
|
|
73
|
+
| Option | Where | What it is |
|
|
74
|
+
| ------------------- | ---------------------------- | -------------------------------------------------------------------------------- |
|
|
75
|
+
| `OUTBOX_TENANTS` | environment, or `tenants` | the tenants this relay serves, comma-separated โ required |
|
|
76
|
+
| `OUTBOX_POLL_MS` | environment, or `pollMs` | the idle sleep between sweeps (default `200`) |
|
|
77
|
+
| `OUTBOX_MAX_LAG_MS` | environment, or `maxLagMs` | the oldest pending age the `outbox` health check tolerates (default 60 s) |
|
|
78
|
+
| `clock` | `outbox({ clock })` | what the poll sleeps on and the lag is measured against (default: `systemClock`) |
|
|
79
|
+
| `schema`, `table` | `prismaOutboxStore(db, {โฆ})` | where the model lives (default `public`, `outboxMessage`) |
|
|
80
|
+
|
|
81
|
+
The full table โ defaults, semantics, the table's PSL and the reasoning โ lives
|
|
82
|
+
on [the reference page](https://btravstack.github.io/btravstack/reference/outbox),
|
|
83
|
+
which is this list's one detailed home.
|
|
84
|
+
|
|
85
|
+
## What it decides, and what it does not
|
|
86
|
+
|
|
87
|
+
**It decides** that delivery is at-least-once, that one relay publishes a
|
|
88
|
+
tenant at a time while its claiming session lives โ so replicas take turns
|
|
89
|
+
rather than race for a tenant's rows, and a tenant's committed facts go out in
|
|
90
|
+
outbox order (which is not commit order: the reference page says what that
|
|
91
|
+
means for a subject); a session lost mid-batch, like a crash between a publish
|
|
92
|
+
and its mark, re-publishes โ that a refused
|
|
93
|
+
publish stops its tenant's batch and backs that tenant off, and that a tenant
|
|
94
|
+
falling behind is unhealthy on `/healthz`. A subscriber deduplicates on the
|
|
95
|
+
outbox id, which the publisher puts on the wire.
|
|
96
|
+
|
|
97
|
+
**It does not decide** what a publish is, which transport it rides or how the
|
|
98
|
+
payload is encoded โ that is your `OutboxPublisher` โ nor the transaction that
|
|
99
|
+
writes the row, which is your adapter's. There is no exactly-once, no
|
|
100
|
+
retention and no second store; the reasons are in [`AGENTS.md`](./AGENTS.md).
|
|
101
|
+
|
|
102
|
+
## License
|
|
103
|
+
|
|
104
|
+
MIT
|
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
let _btravstack_di = require("@btravstack/di");
|
|
3
|
+
let _btravstack_core = require("@btravstack/core");
|
|
4
|
+
let unthrown = require("unthrown");
|
|
5
|
+
let _btravstack_config = require("@btravstack/config");
|
|
6
|
+
//#region src/outbox.ts
|
|
7
|
+
/**
|
|
8
|
+
* Where the relay reads from. An application provides it โ from
|
|
9
|
+
* `@btravstack/outbox/prisma`'s `prismaOutboxStore`, `memoryOutboxStore`, or
|
|
10
|
+
* an adapter of its own โ beside the transaction that writes the rows, which
|
|
11
|
+
* stays the application's.
|
|
12
|
+
*/
|
|
13
|
+
var OutboxStore = class extends (0, _btravstack_di.Port)("OutboxStore") {};
|
|
14
|
+
var OutboxPublisher = class extends (0, _btravstack_di.Port)("OutboxPublisher") {};
|
|
15
|
+
//#endregion
|
|
16
|
+
//#region src/memory.ts
|
|
17
|
+
/**
|
|
18
|
+
* An outbox held in the process: for tests, and for an application whose
|
|
19
|
+
* in-memory repositories need somewhere to record their facts. Its claim is a
|
|
20
|
+
* per-tenant flag, so two relays over ONE instance behave as two replicas over
|
|
21
|
+
* one table do โ which is the only sharing an in-process store can have.
|
|
22
|
+
*/
|
|
23
|
+
const memoryOutboxStore = (clock = _btravstack_core.systemClock) => {
|
|
24
|
+
const messages = [];
|
|
25
|
+
const published = /* @__PURE__ */ new Set();
|
|
26
|
+
const claimed = /* @__PURE__ */ new Set();
|
|
27
|
+
const pendingOf = (tenantId, limit) => messages.filter((message) => message.tenantId === tenantId && !published.has(message.id)).slice(0, limit);
|
|
28
|
+
return {
|
|
29
|
+
append: (message) => {
|
|
30
|
+
messages.push({
|
|
31
|
+
...message,
|
|
32
|
+
id: messages.length + 1,
|
|
33
|
+
occurredAt: new Date(clock.now())
|
|
34
|
+
});
|
|
35
|
+
},
|
|
36
|
+
pending: (tenantId, limit) => (0, unthrown.OkAsync)(pendingOf(tenantId, limit)),
|
|
37
|
+
oldestPending: (tenantIds) => (0, unthrown.OkAsync)(tenantIds.flatMap((tenantId) => pendingOf(tenantId, 1).map(({ occurredAt }) => ({
|
|
38
|
+
tenantId,
|
|
39
|
+
occurredAt
|
|
40
|
+
})))),
|
|
41
|
+
claim: (tenantId, limit, relay) => {
|
|
42
|
+
if (claimed.has(tenantId)) return (0, unthrown.OkAsync)();
|
|
43
|
+
claimed.add(tenantId);
|
|
44
|
+
return (0, unthrown.OkAsync)().flatMap(() => relay(pendingOf(tenantId, limit))).map((ids) => {
|
|
45
|
+
for (const id of ids) published.add(id);
|
|
46
|
+
}).tapFailure(() => claimed.delete(tenantId)).tap(() => claimed.delete(tenantId));
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
};
|
|
50
|
+
//#endregion
|
|
51
|
+
//#region src/relay.ts
|
|
52
|
+
/** How many messages one claim hands the publisher. */
|
|
53
|
+
const BATCH = 32;
|
|
54
|
+
/** The ceiling the back-off doubles towards, unless the poll interval is already longer. */
|
|
55
|
+
const MAX_BACKOFF_MS = 3e4;
|
|
56
|
+
var OutboxConfig = class extends (0, _btravstack_di.Port)("OutboxConfig") {};
|
|
57
|
+
/** The running relay. Nothing resolves it; it exists to be started and stopped. */
|
|
58
|
+
var OutboxRelay = class extends (0, _btravstack_di.Port)("OutboxRelay") {};
|
|
59
|
+
/** A tenant is bounded โ the relay is told its tenants โ so it rides the instruments; the subject does not. */
|
|
60
|
+
const publishing = (message) => ({
|
|
61
|
+
component: "outbox",
|
|
62
|
+
name: "publish",
|
|
63
|
+
attributes: {
|
|
64
|
+
operation: "publish",
|
|
65
|
+
kind: message.kind,
|
|
66
|
+
"btravstack.tenant_id": message.tenantId
|
|
67
|
+
},
|
|
68
|
+
details: {
|
|
69
|
+
"btravstack.outbox.id": message.id,
|
|
70
|
+
"btravstack.outbox.subject": message.subjectId
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
/** Not traced: one span per tenant per poll would bury the publishes it contains. */
|
|
74
|
+
const claiming = (tenantId) => ({
|
|
75
|
+
component: "outbox",
|
|
76
|
+
name: "claim",
|
|
77
|
+
attributes: {
|
|
78
|
+
operation: "claim",
|
|
79
|
+
"btravstack.tenant_id": tenantId
|
|
80
|
+
},
|
|
81
|
+
traced: false
|
|
82
|
+
});
|
|
83
|
+
const startRelay = (store, publisher, observers, clock, { tenants, pollMs }) => {
|
|
84
|
+
const stopping = new AbortController();
|
|
85
|
+
const { signal } = stopping;
|
|
86
|
+
const publishInOrder = async (batch) => {
|
|
87
|
+
const published = [];
|
|
88
|
+
for (const message of batch) {
|
|
89
|
+
if (!(await (0, _btravstack_core.observed)(observers, publishing(message), () => publisher.publish(message))).isOk()) break;
|
|
90
|
+
published.push(message.id);
|
|
91
|
+
}
|
|
92
|
+
return published;
|
|
93
|
+
};
|
|
94
|
+
const sweep = async (tenantId) => {
|
|
95
|
+
let swept = "idle";
|
|
96
|
+
return (await (0, _btravstack_core.observed)(observers, claiming(tenantId), () => store.claim(tenantId, BATCH, (batch) => (0, unthrown.fromSafePromise)(publishInOrder(batch).then((published) => {
|
|
97
|
+
swept = published.length < batch.length ? "failed" : batch.length === BATCH ? "full" : "idle";
|
|
98
|
+
return published;
|
|
99
|
+
}))))).isOk() ? swept : "failed";
|
|
100
|
+
};
|
|
101
|
+
const backoff = (failures) => Math.min(pollMs * 2 ** failures, Math.max(pollMs, MAX_BACKOFF_MS));
|
|
102
|
+
const running = (async () => {
|
|
103
|
+
const schedule = tenants.map((tenantId) => ({
|
|
104
|
+
tenantId,
|
|
105
|
+
due: clock.now(),
|
|
106
|
+
failures: 0
|
|
107
|
+
}));
|
|
108
|
+
while (!signal.aborted) {
|
|
109
|
+
for (const tenant of schedule) {
|
|
110
|
+
if (signal.aborted) break;
|
|
111
|
+
if (tenant.due > clock.now()) continue;
|
|
112
|
+
const swept = await sweep(tenant.tenantId);
|
|
113
|
+
tenant.failures = swept === "failed" ? tenant.failures + 1 : 0;
|
|
114
|
+
tenant.due = clock.now() + (swept === "full" ? 0 : backoff(tenant.failures));
|
|
115
|
+
}
|
|
116
|
+
const wait = Math.min(...schedule.map(({ due }) => due)) - clock.now();
|
|
117
|
+
if (wait > 0) await clock.sleep(wait, signal);
|
|
118
|
+
}
|
|
119
|
+
})();
|
|
120
|
+
return { stop: () => (0, unthrown.fromSafePromise)((async () => {
|
|
121
|
+
stopping.abort();
|
|
122
|
+
await running;
|
|
123
|
+
})()) };
|
|
124
|
+
};
|
|
125
|
+
/**
|
|
126
|
+
* The outbox relay: the other half of a write that recorded its fact in the
|
|
127
|
+
* same transaction as the row it describes. It claims each tenant's oldest
|
|
128
|
+
* pending messages, hands them to the application's {@link OutboxPublisher}
|
|
129
|
+
* in outbox order, marks published what was published, and sleeps.
|
|
130
|
+
*
|
|
131
|
+
* ```ts
|
|
132
|
+
* outbox({ tenants: ["acme"] });
|
|
133
|
+
* ```
|
|
134
|
+
*
|
|
135
|
+
* **At-least-once, with replicas taking turns.** A crash between a publish and
|
|
136
|
+
* its mark re-publishes on the next claim, and so does a claiming session the
|
|
137
|
+
* database ends mid-batch, so a subscriber must tolerate a repeat, keyed by the
|
|
138
|
+
* outbox id. What the claim rules out is replicas racing: a tenant is held by
|
|
139
|
+
* one relay while its claiming session lives and skipped by the rest, so its
|
|
140
|
+
* committed facts go out in outbox order โ which is not commit order: an id
|
|
141
|
+
* still in an open transaction surfaces after a higher one that committed.
|
|
142
|
+
*
|
|
143
|
+
* **A refused publish stops the tenant's batch** and backs that tenant off,
|
|
144
|
+
* doubling from the poll interval to 30 seconds, so a later fact never
|
|
145
|
+
* overtakes an earlier one and no other tenant waits. A message the publisher refuses forever therefore holds its tenant's
|
|
146
|
+
* outbox โ which is what the health check is for: it reports a tenant whose
|
|
147
|
+
* oldest pending message is older than `maxLagMs`.
|
|
148
|
+
*
|
|
149
|
+
* Started as the graph builds, before the runtime accepts anything, and
|
|
150
|
+
* stopped when the application scope closes, after the runtime has drained.
|
|
151
|
+
* Every claim and publish is reported to `Observers`; the module holds no
|
|
152
|
+
* logger of its own.
|
|
153
|
+
*/
|
|
154
|
+
const outbox = (options = {}) => {
|
|
155
|
+
const clock = options.clock ?? _btravstack_core.systemClock;
|
|
156
|
+
const config = _btravstack_config.Config.provider(OutboxConfig)(_btravstack_config.Config.object({
|
|
157
|
+
tenants: _btravstack_config.Config.pinned(options.tenants, _btravstack_config.Config.list("OUTBOX_TENANTS")),
|
|
158
|
+
pollMs: _btravstack_config.Config.pinned(options.pollMs, _btravstack_config.Config.integer("OUTBOX_POLL_MS", {
|
|
159
|
+
min: 1,
|
|
160
|
+
max: 6e4,
|
|
161
|
+
default: 200
|
|
162
|
+
})),
|
|
163
|
+
maxLagMs: _btravstack_config.Config.pinned(options.maxLagMs, _btravstack_config.Config.integer("OUTBOX_MAX_LAG_MS", {
|
|
164
|
+
min: 1,
|
|
165
|
+
default: 6e4
|
|
166
|
+
}))
|
|
167
|
+
}));
|
|
168
|
+
const relay = (0, _btravstack_di.Provider)(OutboxRelay)({
|
|
169
|
+
inject: {
|
|
170
|
+
store: OutboxStore,
|
|
171
|
+
publisher: OutboxPublisher,
|
|
172
|
+
observers: _btravstack_core.Observers,
|
|
173
|
+
config: OutboxConfig
|
|
174
|
+
},
|
|
175
|
+
acquire: ({ store, publisher, observers, config: bound }) => (0, unthrown.OkAsync)(startRelay(store, publisher, observers, clock, bound)),
|
|
176
|
+
release: (running) => running.stop().get()
|
|
177
|
+
});
|
|
178
|
+
const healthCheck = _btravstack_di.Provider.member(_btravstack_core.HealthChecks)({
|
|
179
|
+
inject: {
|
|
180
|
+
store: OutboxStore,
|
|
181
|
+
config: OutboxConfig
|
|
182
|
+
},
|
|
183
|
+
sync: ({ store, config: { tenants, maxLagMs } }) => ({
|
|
184
|
+
name: "outbox",
|
|
185
|
+
check: () => store.oldestPending(tenants).flatMap((oldest) => {
|
|
186
|
+
const behind = oldest.map(({ tenantId, occurredAt }) => ({
|
|
187
|
+
tenantId,
|
|
188
|
+
lagMs: clock.now() - occurredAt.getTime()
|
|
189
|
+
})).filter(({ lagMs }) => lagMs > maxLagMs);
|
|
190
|
+
return behind.length === 0 ? (0, unthrown.OkAsync)() : (0, unthrown.ErrAsync)(new _btravstack_core.HealthCheckFailed({ reason: behind.map(({ tenantId, lagMs }) => `${tenantId} is ${String(lagMs)} ms behind`).join(", ") }));
|
|
191
|
+
})
|
|
192
|
+
})
|
|
193
|
+
});
|
|
194
|
+
return (0, _btravstack_di.Module)("Outbox")({
|
|
195
|
+
needs: [
|
|
196
|
+
_btravstack_config.Env,
|
|
197
|
+
OutboxStore,
|
|
198
|
+
OutboxPublisher
|
|
199
|
+
],
|
|
200
|
+
provides: [
|
|
201
|
+
config,
|
|
202
|
+
_btravstack_core.noObserverMember,
|
|
203
|
+
relay,
|
|
204
|
+
healthCheck
|
|
205
|
+
],
|
|
206
|
+
exports: [_btravstack_core.HealthChecks]
|
|
207
|
+
});
|
|
208
|
+
};
|
|
209
|
+
//#endregion
|
|
210
|
+
exports.OutboxPublisher = OutboxPublisher;
|
|
211
|
+
exports.OutboxStore = OutboxStore;
|
|
212
|
+
exports.memoryOutboxStore = memoryOutboxStore;
|
|
213
|
+
exports.outbox = outbox;
|
package/dist/index.d.cts
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { a as OutboxStoreService, i as OutboxStore, n as OutboxPublisher, o as PublishRefused, r as OutboxPublisherService, t as OutboxMessage } from "./outbox-Bn2wYUh6.cjs";
|
|
2
|
+
import { Clock, HealthChecks } from "@btravstack/core";
|
|
3
|
+
import { ConfigInvalid, Env } from "@btravstack/config";
|
|
4
|
+
import { Module, Scope } from "@btravstack/di";
|
|
5
|
+
//#region src/memory.d.ts
|
|
6
|
+
/** What an in-memory write appends: the envelope, without the two fields the store assigns. */
|
|
7
|
+
type OutboxAppend = Omit<OutboxMessage, "id" | "occurredAt">;
|
|
8
|
+
/** The in-process store, plus the write half an application's in-memory adapter calls. */
|
|
9
|
+
type MemoryOutboxStore = OutboxStoreService & {
|
|
10
|
+
readonly append: (message: OutboxAppend) => void;
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* An outbox held in the process: for tests, and for an application whose
|
|
14
|
+
* in-memory repositories need somewhere to record their facts. Its claim is a
|
|
15
|
+
* per-tenant flag, so two relays over ONE instance behave as two replicas over
|
|
16
|
+
* one table do โ which is the only sharing an in-process store can have.
|
|
17
|
+
*/
|
|
18
|
+
export declare const memoryOutboxStore: (clock?: Clock) => MemoryOutboxStore;
|
|
19
|
+
//#endregion
|
|
20
|
+
//#region src/relay.d.ts
|
|
21
|
+
/** What {@link outbox} is handed. Each field pins the variable named beside it. */
|
|
22
|
+
type OutboxOptions = {
|
|
23
|
+
/** The tenants this relay serves โ `OUTBOX_TENANTS`, comma-separated, required. */
|
|
24
|
+
readonly tenants?: readonly string[];
|
|
25
|
+
/** The idle sleep between sweeps โ `OUTBOX_POLL_MS` (default `200`). */
|
|
26
|
+
readonly pollMs?: number;
|
|
27
|
+
/** The oldest pending age `/healthz` tolerates โ `OUTBOX_MAX_LAG_MS` (default `60_000`). */
|
|
28
|
+
readonly maxLagMs?: number;
|
|
29
|
+
/** What the poll sleeps on and the lag is measured against (default: the kernel's `systemClock`). */
|
|
30
|
+
readonly clock?: Clock;
|
|
31
|
+
};
|
|
32
|
+
/**
|
|
33
|
+
* The outbox relay: the other half of a write that recorded its fact in the
|
|
34
|
+
* same transaction as the row it describes. It claims each tenant's oldest
|
|
35
|
+
* pending messages, hands them to the application's {@link OutboxPublisher}
|
|
36
|
+
* in outbox order, marks published what was published, and sleeps.
|
|
37
|
+
*
|
|
38
|
+
* ```ts
|
|
39
|
+
* outbox({ tenants: ["acme"] });
|
|
40
|
+
* ```
|
|
41
|
+
*
|
|
42
|
+
* **At-least-once, with replicas taking turns.** A crash between a publish and
|
|
43
|
+
* its mark re-publishes on the next claim, and so does a claiming session the
|
|
44
|
+
* database ends mid-batch, so a subscriber must tolerate a repeat, keyed by the
|
|
45
|
+
* outbox id. What the claim rules out is replicas racing: a tenant is held by
|
|
46
|
+
* one relay while its claiming session lives and skipped by the rest, so its
|
|
47
|
+
* committed facts go out in outbox order โ which is not commit order: an id
|
|
48
|
+
* still in an open transaction surfaces after a higher one that committed.
|
|
49
|
+
*
|
|
50
|
+
* **A refused publish stops the tenant's batch** and backs that tenant off,
|
|
51
|
+
* doubling from the poll interval to 30 seconds, so a later fact never
|
|
52
|
+
* overtakes an earlier one and no other tenant waits. A message the publisher refuses forever therefore holds its tenant's
|
|
53
|
+
* outbox โ which is what the health check is for: it reports a tenant whose
|
|
54
|
+
* oldest pending message is older than `maxLagMs`.
|
|
55
|
+
*
|
|
56
|
+
* Started as the graph builds, before the runtime accepts anything, and
|
|
57
|
+
* stopped when the application scope closes, after the runtime has drained.
|
|
58
|
+
* Every claim and publish is reported to `Observers`; the module holds no
|
|
59
|
+
* logger of its own.
|
|
60
|
+
*/
|
|
61
|
+
export declare const outbox: (options?: OutboxOptions) => Module<HealthChecks, ConfigInvalid, Env | OutboxStore | OutboxPublisher | Scope>;
|
|
62
|
+
//#endregion
|
|
63
|
+
export { type MemoryOutboxStore, type OutboxAppend, type OutboxMessage, type OutboxOptions, OutboxPublisher, type OutboxPublisherService, OutboxStore, type OutboxStoreService, type PublishRefused };
|
package/dist/index.d.mts
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { a as OutboxStoreService, i as OutboxStore, n as OutboxPublisher, o as PublishRefused, r as OutboxPublisherService, t as OutboxMessage } from "./outbox-Bn2wYUh6.mjs";
|
|
2
|
+
import { Module, Scope } from "@btravstack/di";
|
|
3
|
+
import { Clock, HealthChecks } from "@btravstack/core";
|
|
4
|
+
import { ConfigInvalid, Env } from "@btravstack/config";
|
|
5
|
+
//#region src/memory.d.ts
|
|
6
|
+
/** What an in-memory write appends: the envelope, without the two fields the store assigns. */
|
|
7
|
+
type OutboxAppend = Omit<OutboxMessage, "id" | "occurredAt">;
|
|
8
|
+
/** The in-process store, plus the write half an application's in-memory adapter calls. */
|
|
9
|
+
type MemoryOutboxStore = OutboxStoreService & {
|
|
10
|
+
readonly append: (message: OutboxAppend) => void;
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* An outbox held in the process: for tests, and for an application whose
|
|
14
|
+
* in-memory repositories need somewhere to record their facts. Its claim is a
|
|
15
|
+
* per-tenant flag, so two relays over ONE instance behave as two replicas over
|
|
16
|
+
* one table do โ which is the only sharing an in-process store can have.
|
|
17
|
+
*/
|
|
18
|
+
export declare const memoryOutboxStore: (clock?: Clock) => MemoryOutboxStore;
|
|
19
|
+
//#endregion
|
|
20
|
+
//#region src/relay.d.ts
|
|
21
|
+
/** What {@link outbox} is handed. Each field pins the variable named beside it. */
|
|
22
|
+
type OutboxOptions = {
|
|
23
|
+
/** The tenants this relay serves โ `OUTBOX_TENANTS`, comma-separated, required. */
|
|
24
|
+
readonly tenants?: readonly string[];
|
|
25
|
+
/** The idle sleep between sweeps โ `OUTBOX_POLL_MS` (default `200`). */
|
|
26
|
+
readonly pollMs?: number;
|
|
27
|
+
/** The oldest pending age `/healthz` tolerates โ `OUTBOX_MAX_LAG_MS` (default `60_000`). */
|
|
28
|
+
readonly maxLagMs?: number;
|
|
29
|
+
/** What the poll sleeps on and the lag is measured against (default: the kernel's `systemClock`). */
|
|
30
|
+
readonly clock?: Clock;
|
|
31
|
+
};
|
|
32
|
+
/**
|
|
33
|
+
* The outbox relay: the other half of a write that recorded its fact in the
|
|
34
|
+
* same transaction as the row it describes. It claims each tenant's oldest
|
|
35
|
+
* pending messages, hands them to the application's {@link OutboxPublisher}
|
|
36
|
+
* in outbox order, marks published what was published, and sleeps.
|
|
37
|
+
*
|
|
38
|
+
* ```ts
|
|
39
|
+
* outbox({ tenants: ["acme"] });
|
|
40
|
+
* ```
|
|
41
|
+
*
|
|
42
|
+
* **At-least-once, with replicas taking turns.** A crash between a publish and
|
|
43
|
+
* its mark re-publishes on the next claim, and so does a claiming session the
|
|
44
|
+
* database ends mid-batch, so a subscriber must tolerate a repeat, keyed by the
|
|
45
|
+
* outbox id. What the claim rules out is replicas racing: a tenant is held by
|
|
46
|
+
* one relay while its claiming session lives and skipped by the rest, so its
|
|
47
|
+
* committed facts go out in outbox order โ which is not commit order: an id
|
|
48
|
+
* still in an open transaction surfaces after a higher one that committed.
|
|
49
|
+
*
|
|
50
|
+
* **A refused publish stops the tenant's batch** and backs that tenant off,
|
|
51
|
+
* doubling from the poll interval to 30 seconds, so a later fact never
|
|
52
|
+
* overtakes an earlier one and no other tenant waits. A message the publisher refuses forever therefore holds its tenant's
|
|
53
|
+
* outbox โ which is what the health check is for: it reports a tenant whose
|
|
54
|
+
* oldest pending message is older than `maxLagMs`.
|
|
55
|
+
*
|
|
56
|
+
* Started as the graph builds, before the runtime accepts anything, and
|
|
57
|
+
* stopped when the application scope closes, after the runtime has drained.
|
|
58
|
+
* Every claim and publish is reported to `Observers`; the module holds no
|
|
59
|
+
* logger of its own.
|
|
60
|
+
*/
|
|
61
|
+
export declare const outbox: (options?: OutboxOptions) => Module<HealthChecks, ConfigInvalid, Env | OutboxStore | OutboxPublisher | Scope>;
|
|
62
|
+
//#endregion
|
|
63
|
+
export { type MemoryOutboxStore, type OutboxAppend, type OutboxMessage, type OutboxOptions, OutboxPublisher, type OutboxPublisherService, OutboxStore, type OutboxStoreService, type PublishRefused };
|
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import { Module, Port, Provider } from "@btravstack/di";
|
|
2
|
+
import { HealthCheckFailed, HealthChecks, Observers, noObserverMember, observed, systemClock } from "@btravstack/core";
|
|
3
|
+
import { ErrAsync, OkAsync, fromSafePromise } from "unthrown";
|
|
4
|
+
import { Config, Env } from "@btravstack/config";
|
|
5
|
+
//#region src/outbox.ts
|
|
6
|
+
/**
|
|
7
|
+
* Where the relay reads from. An application provides it โ from
|
|
8
|
+
* `@btravstack/outbox/prisma`'s `prismaOutboxStore`, `memoryOutboxStore`, or
|
|
9
|
+
* an adapter of its own โ beside the transaction that writes the rows, which
|
|
10
|
+
* stays the application's.
|
|
11
|
+
*/
|
|
12
|
+
var OutboxStore = class extends Port("OutboxStore") {};
|
|
13
|
+
var OutboxPublisher = class extends Port("OutboxPublisher") {};
|
|
14
|
+
//#endregion
|
|
15
|
+
//#region src/memory.ts
|
|
16
|
+
/**
|
|
17
|
+
* An outbox held in the process: for tests, and for an application whose
|
|
18
|
+
* in-memory repositories need somewhere to record their facts. Its claim is a
|
|
19
|
+
* per-tenant flag, so two relays over ONE instance behave as two replicas over
|
|
20
|
+
* one table do โ which is the only sharing an in-process store can have.
|
|
21
|
+
*/
|
|
22
|
+
const memoryOutboxStore = (clock = systemClock) => {
|
|
23
|
+
const messages = [];
|
|
24
|
+
const published = /* @__PURE__ */ new Set();
|
|
25
|
+
const claimed = /* @__PURE__ */ new Set();
|
|
26
|
+
const pendingOf = (tenantId, limit) => messages.filter((message) => message.tenantId === tenantId && !published.has(message.id)).slice(0, limit);
|
|
27
|
+
return {
|
|
28
|
+
append: (message) => {
|
|
29
|
+
messages.push({
|
|
30
|
+
...message,
|
|
31
|
+
id: messages.length + 1,
|
|
32
|
+
occurredAt: new Date(clock.now())
|
|
33
|
+
});
|
|
34
|
+
},
|
|
35
|
+
pending: (tenantId, limit) => OkAsync(pendingOf(tenantId, limit)),
|
|
36
|
+
oldestPending: (tenantIds) => OkAsync(tenantIds.flatMap((tenantId) => pendingOf(tenantId, 1).map(({ occurredAt }) => ({
|
|
37
|
+
tenantId,
|
|
38
|
+
occurredAt
|
|
39
|
+
})))),
|
|
40
|
+
claim: (tenantId, limit, relay) => {
|
|
41
|
+
if (claimed.has(tenantId)) return OkAsync();
|
|
42
|
+
claimed.add(tenantId);
|
|
43
|
+
return OkAsync().flatMap(() => relay(pendingOf(tenantId, limit))).map((ids) => {
|
|
44
|
+
for (const id of ids) published.add(id);
|
|
45
|
+
}).tapFailure(() => claimed.delete(tenantId)).tap(() => claimed.delete(tenantId));
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
};
|
|
49
|
+
//#endregion
|
|
50
|
+
//#region src/relay.ts
|
|
51
|
+
/** How many messages one claim hands the publisher. */
|
|
52
|
+
const BATCH = 32;
|
|
53
|
+
/** The ceiling the back-off doubles towards, unless the poll interval is already longer. */
|
|
54
|
+
const MAX_BACKOFF_MS = 3e4;
|
|
55
|
+
var OutboxConfig = class extends Port("OutboxConfig") {};
|
|
56
|
+
/** The running relay. Nothing resolves it; it exists to be started and stopped. */
|
|
57
|
+
var OutboxRelay = class extends Port("OutboxRelay") {};
|
|
58
|
+
/** A tenant is bounded โ the relay is told its tenants โ so it rides the instruments; the subject does not. */
|
|
59
|
+
const publishing = (message) => ({
|
|
60
|
+
component: "outbox",
|
|
61
|
+
name: "publish",
|
|
62
|
+
attributes: {
|
|
63
|
+
operation: "publish",
|
|
64
|
+
kind: message.kind,
|
|
65
|
+
"btravstack.tenant_id": message.tenantId
|
|
66
|
+
},
|
|
67
|
+
details: {
|
|
68
|
+
"btravstack.outbox.id": message.id,
|
|
69
|
+
"btravstack.outbox.subject": message.subjectId
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
/** Not traced: one span per tenant per poll would bury the publishes it contains. */
|
|
73
|
+
const claiming = (tenantId) => ({
|
|
74
|
+
component: "outbox",
|
|
75
|
+
name: "claim",
|
|
76
|
+
attributes: {
|
|
77
|
+
operation: "claim",
|
|
78
|
+
"btravstack.tenant_id": tenantId
|
|
79
|
+
},
|
|
80
|
+
traced: false
|
|
81
|
+
});
|
|
82
|
+
const startRelay = (store, publisher, observers, clock, { tenants, pollMs }) => {
|
|
83
|
+
const stopping = new AbortController();
|
|
84
|
+
const { signal } = stopping;
|
|
85
|
+
const publishInOrder = async (batch) => {
|
|
86
|
+
const published = [];
|
|
87
|
+
for (const message of batch) {
|
|
88
|
+
if (!(await observed(observers, publishing(message), () => publisher.publish(message))).isOk()) break;
|
|
89
|
+
published.push(message.id);
|
|
90
|
+
}
|
|
91
|
+
return published;
|
|
92
|
+
};
|
|
93
|
+
const sweep = async (tenantId) => {
|
|
94
|
+
let swept = "idle";
|
|
95
|
+
return (await observed(observers, claiming(tenantId), () => store.claim(tenantId, BATCH, (batch) => fromSafePromise(publishInOrder(batch).then((published) => {
|
|
96
|
+
swept = published.length < batch.length ? "failed" : batch.length === BATCH ? "full" : "idle";
|
|
97
|
+
return published;
|
|
98
|
+
}))))).isOk() ? swept : "failed";
|
|
99
|
+
};
|
|
100
|
+
const backoff = (failures) => Math.min(pollMs * 2 ** failures, Math.max(pollMs, MAX_BACKOFF_MS));
|
|
101
|
+
const running = (async () => {
|
|
102
|
+
const schedule = tenants.map((tenantId) => ({
|
|
103
|
+
tenantId,
|
|
104
|
+
due: clock.now(),
|
|
105
|
+
failures: 0
|
|
106
|
+
}));
|
|
107
|
+
while (!signal.aborted) {
|
|
108
|
+
for (const tenant of schedule) {
|
|
109
|
+
if (signal.aborted) break;
|
|
110
|
+
if (tenant.due > clock.now()) continue;
|
|
111
|
+
const swept = await sweep(tenant.tenantId);
|
|
112
|
+
tenant.failures = swept === "failed" ? tenant.failures + 1 : 0;
|
|
113
|
+
tenant.due = clock.now() + (swept === "full" ? 0 : backoff(tenant.failures));
|
|
114
|
+
}
|
|
115
|
+
const wait = Math.min(...schedule.map(({ due }) => due)) - clock.now();
|
|
116
|
+
if (wait > 0) await clock.sleep(wait, signal);
|
|
117
|
+
}
|
|
118
|
+
})();
|
|
119
|
+
return { stop: () => fromSafePromise((async () => {
|
|
120
|
+
stopping.abort();
|
|
121
|
+
await running;
|
|
122
|
+
})()) };
|
|
123
|
+
};
|
|
124
|
+
/**
|
|
125
|
+
* The outbox relay: the other half of a write that recorded its fact in the
|
|
126
|
+
* same transaction as the row it describes. It claims each tenant's oldest
|
|
127
|
+
* pending messages, hands them to the application's {@link OutboxPublisher}
|
|
128
|
+
* in outbox order, marks published what was published, and sleeps.
|
|
129
|
+
*
|
|
130
|
+
* ```ts
|
|
131
|
+
* outbox({ tenants: ["acme"] });
|
|
132
|
+
* ```
|
|
133
|
+
*
|
|
134
|
+
* **At-least-once, with replicas taking turns.** A crash between a publish and
|
|
135
|
+
* its mark re-publishes on the next claim, and so does a claiming session the
|
|
136
|
+
* database ends mid-batch, so a subscriber must tolerate a repeat, keyed by the
|
|
137
|
+
* outbox id. What the claim rules out is replicas racing: a tenant is held by
|
|
138
|
+
* one relay while its claiming session lives and skipped by the rest, so its
|
|
139
|
+
* committed facts go out in outbox order โ which is not commit order: an id
|
|
140
|
+
* still in an open transaction surfaces after a higher one that committed.
|
|
141
|
+
*
|
|
142
|
+
* **A refused publish stops the tenant's batch** and backs that tenant off,
|
|
143
|
+
* doubling from the poll interval to 30 seconds, so a later fact never
|
|
144
|
+
* overtakes an earlier one and no other tenant waits. A message the publisher refuses forever therefore holds its tenant's
|
|
145
|
+
* outbox โ which is what the health check is for: it reports a tenant whose
|
|
146
|
+
* oldest pending message is older than `maxLagMs`.
|
|
147
|
+
*
|
|
148
|
+
* Started as the graph builds, before the runtime accepts anything, and
|
|
149
|
+
* stopped when the application scope closes, after the runtime has drained.
|
|
150
|
+
* Every claim and publish is reported to `Observers`; the module holds no
|
|
151
|
+
* logger of its own.
|
|
152
|
+
*/
|
|
153
|
+
const outbox = (options = {}) => {
|
|
154
|
+
const clock = options.clock ?? systemClock;
|
|
155
|
+
const config = Config.provider(OutboxConfig)(Config.object({
|
|
156
|
+
tenants: Config.pinned(options.tenants, Config.list("OUTBOX_TENANTS")),
|
|
157
|
+
pollMs: Config.pinned(options.pollMs, Config.integer("OUTBOX_POLL_MS", {
|
|
158
|
+
min: 1,
|
|
159
|
+
max: 6e4,
|
|
160
|
+
default: 200
|
|
161
|
+
})),
|
|
162
|
+
maxLagMs: Config.pinned(options.maxLagMs, Config.integer("OUTBOX_MAX_LAG_MS", {
|
|
163
|
+
min: 1,
|
|
164
|
+
default: 6e4
|
|
165
|
+
}))
|
|
166
|
+
}));
|
|
167
|
+
const relay = Provider(OutboxRelay)({
|
|
168
|
+
inject: {
|
|
169
|
+
store: OutboxStore,
|
|
170
|
+
publisher: OutboxPublisher,
|
|
171
|
+
observers: Observers,
|
|
172
|
+
config: OutboxConfig
|
|
173
|
+
},
|
|
174
|
+
acquire: ({ store, publisher, observers, config: bound }) => OkAsync(startRelay(store, publisher, observers, clock, bound)),
|
|
175
|
+
release: (running) => running.stop().get()
|
|
176
|
+
});
|
|
177
|
+
const healthCheck = Provider.member(HealthChecks)({
|
|
178
|
+
inject: {
|
|
179
|
+
store: OutboxStore,
|
|
180
|
+
config: OutboxConfig
|
|
181
|
+
},
|
|
182
|
+
sync: ({ store, config: { tenants, maxLagMs } }) => ({
|
|
183
|
+
name: "outbox",
|
|
184
|
+
check: () => store.oldestPending(tenants).flatMap((oldest) => {
|
|
185
|
+
const behind = oldest.map(({ tenantId, occurredAt }) => ({
|
|
186
|
+
tenantId,
|
|
187
|
+
lagMs: clock.now() - occurredAt.getTime()
|
|
188
|
+
})).filter(({ lagMs }) => lagMs > maxLagMs);
|
|
189
|
+
return behind.length === 0 ? OkAsync() : ErrAsync(new HealthCheckFailed({ reason: behind.map(({ tenantId, lagMs }) => `${tenantId} is ${String(lagMs)} ms behind`).join(", ") }));
|
|
190
|
+
})
|
|
191
|
+
})
|
|
192
|
+
});
|
|
193
|
+
return Module("Outbox")({
|
|
194
|
+
needs: [
|
|
195
|
+
Env,
|
|
196
|
+
OutboxStore,
|
|
197
|
+
OutboxPublisher
|
|
198
|
+
],
|
|
199
|
+
provides: [
|
|
200
|
+
config,
|
|
201
|
+
noObserverMember,
|
|
202
|
+
relay,
|
|
203
|
+
healthCheck
|
|
204
|
+
],
|
|
205
|
+
exports: [HealthChecks]
|
|
206
|
+
});
|
|
207
|
+
};
|
|
208
|
+
//#endregion
|
|
209
|
+
export { OutboxPublisher, OutboxStore, memoryOutboxStore, outbox };
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { AsyncResult } from "unthrown";
|
|
2
|
+
//#region src/outbox.d.ts
|
|
3
|
+
/**
|
|
4
|
+
* One committed fact awaiting broadcast: a row of the outbox table, as the
|
|
5
|
+
* relay reads it.
|
|
6
|
+
*
|
|
7
|
+
* The row IS the envelope. `tenantId` is whose fact it is, and the unit the
|
|
8
|
+
* relay claims by; `kind` is which sort of thing changed; `subjectId` is which
|
|
9
|
+
* one, and the key a reader compacts on; `payload` is the application's own
|
|
10
|
+
* encoding, handed to the publisher untouched โ a `null` payload is the
|
|
11
|
+
* **tombstone**, the last word about a subject. `id` is the outbox sequence,
|
|
12
|
+
* and the order the relay publishes a tenant's facts in.
|
|
13
|
+
*/
|
|
14
|
+
type OutboxMessage = {
|
|
15
|
+
readonly id: number;
|
|
16
|
+
readonly tenantId: string;
|
|
17
|
+
readonly kind: string;
|
|
18
|
+
readonly subjectId: string;
|
|
19
|
+
readonly payload: string | null;
|
|
20
|
+
readonly occurredAt: Date;
|
|
21
|
+
};
|
|
22
|
+
/**
|
|
23
|
+
* What a store answers for the relay. Both operations promise `never`: a
|
|
24
|
+
* database that will not answer is a defect the relay observes and retries,
|
|
25
|
+
* not a domain outcome.
|
|
26
|
+
*/
|
|
27
|
+
type OutboxStoreService = {
|
|
28
|
+
/**
|
|
29
|
+
* A tenant's oldest unpublished messages, in outbox order, at most `limit`.
|
|
30
|
+
* A read and nothing more โ it claims nothing, which is what lets the health
|
|
31
|
+
* check and a spec look without racing the relay.
|
|
32
|
+
*/
|
|
33
|
+
readonly pending: (tenantId: string, limit: number) => AsyncResult<readonly OutboxMessage[], never>;
|
|
34
|
+
/**
|
|
35
|
+
* When each of `tenantIds`' oldest unpublished message was written, for the
|
|
36
|
+
* tenants that have one โ in ONE round trip however many tenants are asked
|
|
37
|
+
* about, because the health check asks on every `/healthz`, and a query per
|
|
38
|
+
* tenant would queue the pool behind the probe meant to report it.
|
|
39
|
+
*/
|
|
40
|
+
readonly oldestPending: (tenantIds: readonly string[]) => AsyncResult<readonly {
|
|
41
|
+
readonly tenantId: string;
|
|
42
|
+
readonly occurredAt: Date;
|
|
43
|
+
}[], never>;
|
|
44
|
+
/**
|
|
45
|
+
* Claims a tenant's oldest unpublished messages, hands them to `relay`, and
|
|
46
|
+
* marks published exactly the ids `relay` answers โ all under one claim, so
|
|
47
|
+
* no other caller holding the same store is handed the same tenant until it
|
|
48
|
+
* is released.
|
|
49
|
+
*
|
|
50
|
+
* A tenant another caller has claimed is **skipped, not waited for**: the
|
|
51
|
+
* call answers `Ok` without running `relay`. A defect from `relay` releases
|
|
52
|
+
* the claim and marks nothing.
|
|
53
|
+
*/
|
|
54
|
+
readonly claim: (tenantId: string, limit: number, relay: (batch: readonly OutboxMessage[]) => AsyncResult<readonly number[], never>) => AsyncResult<void, never>;
|
|
55
|
+
};
|
|
56
|
+
declare const OutboxStore_base: import("@btravstack/di").PortClass<"OutboxStore">;
|
|
57
|
+
/**
|
|
58
|
+
* Where the relay reads from. An application provides it โ from
|
|
59
|
+
* `@btravstack/outbox/prisma`'s `prismaOutboxStore`, `memoryOutboxStore`, or
|
|
60
|
+
* an adapter of its own โ beside the transaction that writes the rows, which
|
|
61
|
+
* stays the application's.
|
|
62
|
+
*/
|
|
63
|
+
declare class OutboxStore extends OutboxStore_base<OutboxStoreService> {}
|
|
64
|
+
/**
|
|
65
|
+
* Whatever tagged error the publisher models โ a broker that refused, a
|
|
66
|
+
* message the contract rejects. The relay reports it and acts on none of it,
|
|
67
|
+
* so the type asks for a tag and nothing else.
|
|
68
|
+
*/
|
|
69
|
+
type PublishRefused = {
|
|
70
|
+
readonly _tag: string;
|
|
71
|
+
};
|
|
72
|
+
/**
|
|
73
|
+
* What "publish" means, which only the application knows: the transport, the
|
|
74
|
+
* contract and the decoding of `payload`.
|
|
75
|
+
*
|
|
76
|
+
* Any `Err` is a message left pending: the relay stops the tenant's batch
|
|
77
|
+
* there, so a later fact never overtakes it, and tries again after its
|
|
78
|
+
* back-off.
|
|
79
|
+
*/
|
|
80
|
+
type OutboxPublisherService = {
|
|
81
|
+
readonly publish: (message: OutboxMessage) => AsyncResult<void, PublishRefused>;
|
|
82
|
+
};
|
|
83
|
+
declare const OutboxPublisher_base: import("@btravstack/di").PortClass<"OutboxPublisher">;
|
|
84
|
+
declare class OutboxPublisher extends OutboxPublisher_base<OutboxPublisherService> {}
|
|
85
|
+
//#endregion
|
|
86
|
+
export { OutboxStoreService as a, OutboxStore as i, OutboxPublisher as n, PublishRefused as o, OutboxPublisherService as r, OutboxMessage as t };
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { AsyncResult } from "unthrown";
|
|
2
|
+
//#region src/outbox.d.ts
|
|
3
|
+
/**
|
|
4
|
+
* One committed fact awaiting broadcast: a row of the outbox table, as the
|
|
5
|
+
* relay reads it.
|
|
6
|
+
*
|
|
7
|
+
* The row IS the envelope. `tenantId` is whose fact it is, and the unit the
|
|
8
|
+
* relay claims by; `kind` is which sort of thing changed; `subjectId` is which
|
|
9
|
+
* one, and the key a reader compacts on; `payload` is the application's own
|
|
10
|
+
* encoding, handed to the publisher untouched โ a `null` payload is the
|
|
11
|
+
* **tombstone**, the last word about a subject. `id` is the outbox sequence,
|
|
12
|
+
* and the order the relay publishes a tenant's facts in.
|
|
13
|
+
*/
|
|
14
|
+
type OutboxMessage = {
|
|
15
|
+
readonly id: number;
|
|
16
|
+
readonly tenantId: string;
|
|
17
|
+
readonly kind: string;
|
|
18
|
+
readonly subjectId: string;
|
|
19
|
+
readonly payload: string | null;
|
|
20
|
+
readonly occurredAt: Date;
|
|
21
|
+
};
|
|
22
|
+
/**
|
|
23
|
+
* What a store answers for the relay. Both operations promise `never`: a
|
|
24
|
+
* database that will not answer is a defect the relay observes and retries,
|
|
25
|
+
* not a domain outcome.
|
|
26
|
+
*/
|
|
27
|
+
type OutboxStoreService = {
|
|
28
|
+
/**
|
|
29
|
+
* A tenant's oldest unpublished messages, in outbox order, at most `limit`.
|
|
30
|
+
* A read and nothing more โ it claims nothing, which is what lets the health
|
|
31
|
+
* check and a spec look without racing the relay.
|
|
32
|
+
*/
|
|
33
|
+
readonly pending: (tenantId: string, limit: number) => AsyncResult<readonly OutboxMessage[], never>;
|
|
34
|
+
/**
|
|
35
|
+
* When each of `tenantIds`' oldest unpublished message was written, for the
|
|
36
|
+
* tenants that have one โ in ONE round trip however many tenants are asked
|
|
37
|
+
* about, because the health check asks on every `/healthz`, and a query per
|
|
38
|
+
* tenant would queue the pool behind the probe meant to report it.
|
|
39
|
+
*/
|
|
40
|
+
readonly oldestPending: (tenantIds: readonly string[]) => AsyncResult<readonly {
|
|
41
|
+
readonly tenantId: string;
|
|
42
|
+
readonly occurredAt: Date;
|
|
43
|
+
}[], never>;
|
|
44
|
+
/**
|
|
45
|
+
* Claims a tenant's oldest unpublished messages, hands them to `relay`, and
|
|
46
|
+
* marks published exactly the ids `relay` answers โ all under one claim, so
|
|
47
|
+
* no other caller holding the same store is handed the same tenant until it
|
|
48
|
+
* is released.
|
|
49
|
+
*
|
|
50
|
+
* A tenant another caller has claimed is **skipped, not waited for**: the
|
|
51
|
+
* call answers `Ok` without running `relay`. A defect from `relay` releases
|
|
52
|
+
* the claim and marks nothing.
|
|
53
|
+
*/
|
|
54
|
+
readonly claim: (tenantId: string, limit: number, relay: (batch: readonly OutboxMessage[]) => AsyncResult<readonly number[], never>) => AsyncResult<void, never>;
|
|
55
|
+
};
|
|
56
|
+
declare const OutboxStore_base: import("@btravstack/di").PortClass<"OutboxStore">;
|
|
57
|
+
/**
|
|
58
|
+
* Where the relay reads from. An application provides it โ from
|
|
59
|
+
* `@btravstack/outbox/prisma`'s `prismaOutboxStore`, `memoryOutboxStore`, or
|
|
60
|
+
* an adapter of its own โ beside the transaction that writes the rows, which
|
|
61
|
+
* stays the application's.
|
|
62
|
+
*/
|
|
63
|
+
declare class OutboxStore extends OutboxStore_base<OutboxStoreService> {}
|
|
64
|
+
/**
|
|
65
|
+
* Whatever tagged error the publisher models โ a broker that refused, a
|
|
66
|
+
* message the contract rejects. The relay reports it and acts on none of it,
|
|
67
|
+
* so the type asks for a tag and nothing else.
|
|
68
|
+
*/
|
|
69
|
+
type PublishRefused = {
|
|
70
|
+
readonly _tag: string;
|
|
71
|
+
};
|
|
72
|
+
/**
|
|
73
|
+
* What "publish" means, which only the application knows: the transport, the
|
|
74
|
+
* contract and the decoding of `payload`.
|
|
75
|
+
*
|
|
76
|
+
* Any `Err` is a message left pending: the relay stops the tenant's batch
|
|
77
|
+
* there, so a later fact never overtakes it, and tries again after its
|
|
78
|
+
* back-off.
|
|
79
|
+
*/
|
|
80
|
+
type OutboxPublisherService = {
|
|
81
|
+
readonly publish: (message: OutboxMessage) => AsyncResult<void, PublishRefused>;
|
|
82
|
+
};
|
|
83
|
+
declare const OutboxPublisher_base: import("@btravstack/di").PortClass<"OutboxPublisher">;
|
|
84
|
+
declare class OutboxPublisher extends OutboxPublisher_base<OutboxPublisherService> {}
|
|
85
|
+
//#endregion
|
|
86
|
+
export { OutboxStoreService as a, OutboxStore as i, OutboxPublisher as n, PublishRefused as o, OutboxPublisherService as r, OutboxMessage as t };
|
package/dist/prisma.cjs
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
let unthrown = require("unthrown");
|
|
3
|
+
//#region src/prisma.ts
|
|
4
|
+
/**
|
|
5
|
+
* An `int8` id as the number the port speaks, refused rather than rounded past
|
|
6
|
+
* 2^53 โ a row whose id cannot be named exactly would be marked, and
|
|
7
|
+
* deduplicated on, as some other row.
|
|
8
|
+
*/
|
|
9
|
+
const safe = (id) => {
|
|
10
|
+
const value = Number(id);
|
|
11
|
+
if (!Number.isSafeInteger(value)) throw new RangeError(`outbox id ${String(id)} exceeds 2^53`);
|
|
12
|
+
return value;
|
|
13
|
+
};
|
|
14
|
+
const identifier = (name) => `"${name.replaceAll("\"", "\"\"")}"`;
|
|
15
|
+
/** A template whose text carries the table's identifier โ the one part a parameter cannot be. */
|
|
16
|
+
const statement = (...parts) => Object.assign([...parts], { raw: [...parts] });
|
|
17
|
+
/**
|
|
18
|
+
* The outbox store over a Prisma 8 client, in raw SQL against the documented
|
|
19
|
+
* table โ the model is the application's, declared in its own contract.
|
|
20
|
+
*
|
|
21
|
+
* **The claim is a transaction-scoped advisory lock per tenant**, taken with
|
|
22
|
+
* `pg_try_advisory_xact_lock` before the batch is read: a relay that does not
|
|
23
|
+
* get it skips the tenant, and the lock dies with the transaction, so a relay
|
|
24
|
+
* that crashes mid-batch releases it with its connection. The batch is read,
|
|
25
|
+
* published and marked inside that one transaction, so a mark that never
|
|
26
|
+
* commits leaves its rows pending rather than lost. The cost is a pooled
|
|
27
|
+
* connection held for the length of one batch's publishes.
|
|
28
|
+
*
|
|
29
|
+
* **One relay per tenant holds while its claiming session lives.** The claim
|
|
30
|
+
* lifts `idle_in_transaction_session_timeout` for its own transaction, so a
|
|
31
|
+
* configured timeout cannot end it mid-batch; a session the server ends any
|
|
32
|
+
* other way (a terminated backend, a failover) frees the lock while the relay
|
|
33
|
+
* is still publishing, and another relay may publish the same rows โ which is
|
|
34
|
+
* at-least-once delivery, deduplicated on the id.
|
|
35
|
+
*
|
|
36
|
+
* The table's `id` is a `BigInt` โ an `Int` runs out at 2^31 โ read through
|
|
37
|
+
* `pg/int8@1` and refused, as a defect, past 2^53 rather than rounded.
|
|
38
|
+
* `occurredAt` is read through `to_json`, which answers ISO 8601 whatever the
|
|
39
|
+
* column's codec or the server's `DateStyle`, so the store needs no codec
|
|
40
|
+
* beyond `pg/text@1` and `pg/int8@1` โ both of which the table itself uses.
|
|
41
|
+
*/
|
|
42
|
+
const prismaOutboxStore = (db, options = {}) => {
|
|
43
|
+
const schema = options.schema ?? "public";
|
|
44
|
+
const table = options.table ?? "outboxMessage";
|
|
45
|
+
const qualified = `${identifier(schema)}.${identifier(table)}`;
|
|
46
|
+
const { sql } = db.raw;
|
|
47
|
+
const lock = (tenantId) => sql`SELECT set_config('idle_in_transaction_session_timeout', '0', true) AS lifted, pg_try_advisory_xact_lock(hashtext(${`${schema}.${table}`}), hashtext(${tenantId}))::text AS locked`.returnsRow({
|
|
48
|
+
lifted: "pg/text@1",
|
|
49
|
+
locked: "pg/text@1"
|
|
50
|
+
}).build();
|
|
51
|
+
const select = (tenantId, limit) => sql(statement(`SELECT "id", "tenantId", "kind", "subjectId", "payload", to_json("occurredAt") #>> '{}' AS "occurredAt" FROM ${qualified} WHERE "tenantId" = `, ` AND "publishedAt" IS NULL ORDER BY "id" LIMIT `, "::int"), tenantId, String(limit)).returnsRow({
|
|
52
|
+
id: "pg/int8@1",
|
|
53
|
+
tenantId: "pg/text@1",
|
|
54
|
+
kind: "pg/text@1",
|
|
55
|
+
subjectId: "pg/text@1",
|
|
56
|
+
payload: "pg/text@1",
|
|
57
|
+
occurredAt: "pg/text@1"
|
|
58
|
+
}).build();
|
|
59
|
+
const oldest = (tenantIds) => sql(statement(`SELECT "tenantId", to_json(min("occurredAt")) #>> '{}' AS "occurredAt" FROM ${qualified} WHERE "publishedAt" IS NULL AND "tenantId" IN (SELECT json_array_elements_text(`, `::json)) GROUP BY "tenantId"`), JSON.stringify(tenantIds)).returnsRow({
|
|
60
|
+
tenantId: "pg/text@1",
|
|
61
|
+
occurredAt: "pg/text@1"
|
|
62
|
+
}).build();
|
|
63
|
+
const mark = (ids) => sql(statement(`UPDATE ${qualified} SET "publishedAt" = now() WHERE "id" = ANY(string_to_array(`, ", ',')::bigint[])"), ids.join(",")).affectedCount().build();
|
|
64
|
+
const read = async (tx, tenantId, limit) => (await tx.query(select(tenantId, limit))).map((row) => ({
|
|
65
|
+
...row,
|
|
66
|
+
id: safe(row.id),
|
|
67
|
+
occurredAt: new Date(row.occurredAt)
|
|
68
|
+
}));
|
|
69
|
+
const transaction = (work) => (0, unthrown.fromSafePromise)(Promise.resolve().then(() => db.transaction((tx) => work(tx))));
|
|
70
|
+
return {
|
|
71
|
+
pending: (tenantId, limit) => transaction((tx) => read(tx, tenantId, limit)),
|
|
72
|
+
oldestPending: (tenantIds) => transaction(async (tx) => (await tx.query(oldest(tenantIds))).map(({ tenantId, occurredAt }) => ({
|
|
73
|
+
tenantId,
|
|
74
|
+
occurredAt: new Date(occurredAt)
|
|
75
|
+
}))),
|
|
76
|
+
claim: (tenantId, limit, relay) => transaction(async (tx) => {
|
|
77
|
+
const [held] = await tx.query(lock(tenantId));
|
|
78
|
+
if (held?.locked !== "true") return;
|
|
79
|
+
const published = await relay(await read(tx, tenantId, limit)).get();
|
|
80
|
+
if (published.length > 0) await tx.query(mark(published));
|
|
81
|
+
})
|
|
82
|
+
};
|
|
83
|
+
};
|
|
84
|
+
//#endregion
|
|
85
|
+
exports.prismaOutboxStore = prismaOutboxStore;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { a as OutboxStoreService } from "./outbox-Bn2wYUh6.cjs";
|
|
2
|
+
//#region src/prisma.d.ts
|
|
3
|
+
/**
|
|
4
|
+
* The little of a Prisma 8 client this store needs: the raw lane to build its
|
|
5
|
+
* statements with, and `transaction` to run them in.
|
|
6
|
+
*/
|
|
7
|
+
export type OutboxDatabase<Tx> = {
|
|
8
|
+
/**
|
|
9
|
+
* Required to EXIST and not described further, for `@btravstack/prisma`'s
|
|
10
|
+
* reason: the real tag takes the contract's own interpolation and row-spec
|
|
11
|
+
* types, narrower than anything a package that cannot see a contract could
|
|
12
|
+
* name, and a parameter is contravariant.
|
|
13
|
+
*/
|
|
14
|
+
readonly raw: {
|
|
15
|
+
readonly sql: unknown;
|
|
16
|
+
};
|
|
17
|
+
readonly transaction: <R>(fn: (tx: Tx) => PromiseLike<R>) => Promise<R>;
|
|
18
|
+
};
|
|
19
|
+
/** Where the table lives. */
|
|
20
|
+
export type PrismaOutboxStoreOptions = {
|
|
21
|
+
/** The namespace the model is declared in (default `public`). */
|
|
22
|
+
readonly schema?: string;
|
|
23
|
+
/** The table Prisma maps the model to (default `outboxMessage`, the table of a model named `OutboxMessage`). */
|
|
24
|
+
readonly table?: string;
|
|
25
|
+
};
|
|
26
|
+
/**
|
|
27
|
+
* The outbox store over a Prisma 8 client, in raw SQL against the documented
|
|
28
|
+
* table โ the model is the application's, declared in its own contract.
|
|
29
|
+
*
|
|
30
|
+
* **The claim is a transaction-scoped advisory lock per tenant**, taken with
|
|
31
|
+
* `pg_try_advisory_xact_lock` before the batch is read: a relay that does not
|
|
32
|
+
* get it skips the tenant, and the lock dies with the transaction, so a relay
|
|
33
|
+
* that crashes mid-batch releases it with its connection. The batch is read,
|
|
34
|
+
* published and marked inside that one transaction, so a mark that never
|
|
35
|
+
* commits leaves its rows pending rather than lost. The cost is a pooled
|
|
36
|
+
* connection held for the length of one batch's publishes.
|
|
37
|
+
*
|
|
38
|
+
* **One relay per tenant holds while its claiming session lives.** The claim
|
|
39
|
+
* lifts `idle_in_transaction_session_timeout` for its own transaction, so a
|
|
40
|
+
* configured timeout cannot end it mid-batch; a session the server ends any
|
|
41
|
+
* other way (a terminated backend, a failover) frees the lock while the relay
|
|
42
|
+
* is still publishing, and another relay may publish the same rows โ which is
|
|
43
|
+
* at-least-once delivery, deduplicated on the id.
|
|
44
|
+
*
|
|
45
|
+
* The table's `id` is a `BigInt` โ an `Int` runs out at 2^31 โ read through
|
|
46
|
+
* `pg/int8@1` and refused, as a defect, past 2^53 rather than rounded.
|
|
47
|
+
* `occurredAt` is read through `to_json`, which answers ISO 8601 whatever the
|
|
48
|
+
* column's codec or the server's `DateStyle`, so the store needs no codec
|
|
49
|
+
* beyond `pg/text@1` and `pg/int8@1` โ both of which the table itself uses.
|
|
50
|
+
*/
|
|
51
|
+
export declare const prismaOutboxStore: <Tx>(db: OutboxDatabase<Tx>, options?: PrismaOutboxStoreOptions) => OutboxStoreService;
|
|
52
|
+
//#endregion
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { a as OutboxStoreService } from "./outbox-Bn2wYUh6.mjs";
|
|
2
|
+
//#region src/prisma.d.ts
|
|
3
|
+
/**
|
|
4
|
+
* The little of a Prisma 8 client this store needs: the raw lane to build its
|
|
5
|
+
* statements with, and `transaction` to run them in.
|
|
6
|
+
*/
|
|
7
|
+
export type OutboxDatabase<Tx> = {
|
|
8
|
+
/**
|
|
9
|
+
* Required to EXIST and not described further, for `@btravstack/prisma`'s
|
|
10
|
+
* reason: the real tag takes the contract's own interpolation and row-spec
|
|
11
|
+
* types, narrower than anything a package that cannot see a contract could
|
|
12
|
+
* name, and a parameter is contravariant.
|
|
13
|
+
*/
|
|
14
|
+
readonly raw: {
|
|
15
|
+
readonly sql: unknown;
|
|
16
|
+
};
|
|
17
|
+
readonly transaction: <R>(fn: (tx: Tx) => PromiseLike<R>) => Promise<R>;
|
|
18
|
+
};
|
|
19
|
+
/** Where the table lives. */
|
|
20
|
+
export type PrismaOutboxStoreOptions = {
|
|
21
|
+
/** The namespace the model is declared in (default `public`). */
|
|
22
|
+
readonly schema?: string;
|
|
23
|
+
/** The table Prisma maps the model to (default `outboxMessage`, the table of a model named `OutboxMessage`). */
|
|
24
|
+
readonly table?: string;
|
|
25
|
+
};
|
|
26
|
+
/**
|
|
27
|
+
* The outbox store over a Prisma 8 client, in raw SQL against the documented
|
|
28
|
+
* table โ the model is the application's, declared in its own contract.
|
|
29
|
+
*
|
|
30
|
+
* **The claim is a transaction-scoped advisory lock per tenant**, taken with
|
|
31
|
+
* `pg_try_advisory_xact_lock` before the batch is read: a relay that does not
|
|
32
|
+
* get it skips the tenant, and the lock dies with the transaction, so a relay
|
|
33
|
+
* that crashes mid-batch releases it with its connection. The batch is read,
|
|
34
|
+
* published and marked inside that one transaction, so a mark that never
|
|
35
|
+
* commits leaves its rows pending rather than lost. The cost is a pooled
|
|
36
|
+
* connection held for the length of one batch's publishes.
|
|
37
|
+
*
|
|
38
|
+
* **One relay per tenant holds while its claiming session lives.** The claim
|
|
39
|
+
* lifts `idle_in_transaction_session_timeout` for its own transaction, so a
|
|
40
|
+
* configured timeout cannot end it mid-batch; a session the server ends any
|
|
41
|
+
* other way (a terminated backend, a failover) frees the lock while the relay
|
|
42
|
+
* is still publishing, and another relay may publish the same rows โ which is
|
|
43
|
+
* at-least-once delivery, deduplicated on the id.
|
|
44
|
+
*
|
|
45
|
+
* The table's `id` is a `BigInt` โ an `Int` runs out at 2^31 โ read through
|
|
46
|
+
* `pg/int8@1` and refused, as a defect, past 2^53 rather than rounded.
|
|
47
|
+
* `occurredAt` is read through `to_json`, which answers ISO 8601 whatever the
|
|
48
|
+
* column's codec or the server's `DateStyle`, so the store needs no codec
|
|
49
|
+
* beyond `pg/text@1` and `pg/int8@1` โ both of which the table itself uses.
|
|
50
|
+
*/
|
|
51
|
+
export declare const prismaOutboxStore: <Tx>(db: OutboxDatabase<Tx>, options?: PrismaOutboxStoreOptions) => OutboxStoreService;
|
|
52
|
+
//#endregion
|
package/dist/prisma.mjs
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { fromSafePromise } from "unthrown";
|
|
2
|
+
//#region src/prisma.ts
|
|
3
|
+
/**
|
|
4
|
+
* An `int8` id as the number the port speaks, refused rather than rounded past
|
|
5
|
+
* 2^53 โ a row whose id cannot be named exactly would be marked, and
|
|
6
|
+
* deduplicated on, as some other row.
|
|
7
|
+
*/
|
|
8
|
+
const safe = (id) => {
|
|
9
|
+
const value = Number(id);
|
|
10
|
+
if (!Number.isSafeInteger(value)) throw new RangeError(`outbox id ${String(id)} exceeds 2^53`);
|
|
11
|
+
return value;
|
|
12
|
+
};
|
|
13
|
+
const identifier = (name) => `"${name.replaceAll("\"", "\"\"")}"`;
|
|
14
|
+
/** A template whose text carries the table's identifier โ the one part a parameter cannot be. */
|
|
15
|
+
const statement = (...parts) => Object.assign([...parts], { raw: [...parts] });
|
|
16
|
+
/**
|
|
17
|
+
* The outbox store over a Prisma 8 client, in raw SQL against the documented
|
|
18
|
+
* table โ the model is the application's, declared in its own contract.
|
|
19
|
+
*
|
|
20
|
+
* **The claim is a transaction-scoped advisory lock per tenant**, taken with
|
|
21
|
+
* `pg_try_advisory_xact_lock` before the batch is read: a relay that does not
|
|
22
|
+
* get it skips the tenant, and the lock dies with the transaction, so a relay
|
|
23
|
+
* that crashes mid-batch releases it with its connection. The batch is read,
|
|
24
|
+
* published and marked inside that one transaction, so a mark that never
|
|
25
|
+
* commits leaves its rows pending rather than lost. The cost is a pooled
|
|
26
|
+
* connection held for the length of one batch's publishes.
|
|
27
|
+
*
|
|
28
|
+
* **One relay per tenant holds while its claiming session lives.** The claim
|
|
29
|
+
* lifts `idle_in_transaction_session_timeout` for its own transaction, so a
|
|
30
|
+
* configured timeout cannot end it mid-batch; a session the server ends any
|
|
31
|
+
* other way (a terminated backend, a failover) frees the lock while the relay
|
|
32
|
+
* is still publishing, and another relay may publish the same rows โ which is
|
|
33
|
+
* at-least-once delivery, deduplicated on the id.
|
|
34
|
+
*
|
|
35
|
+
* The table's `id` is a `BigInt` โ an `Int` runs out at 2^31 โ read through
|
|
36
|
+
* `pg/int8@1` and refused, as a defect, past 2^53 rather than rounded.
|
|
37
|
+
* `occurredAt` is read through `to_json`, which answers ISO 8601 whatever the
|
|
38
|
+
* column's codec or the server's `DateStyle`, so the store needs no codec
|
|
39
|
+
* beyond `pg/text@1` and `pg/int8@1` โ both of which the table itself uses.
|
|
40
|
+
*/
|
|
41
|
+
const prismaOutboxStore = (db, options = {}) => {
|
|
42
|
+
const schema = options.schema ?? "public";
|
|
43
|
+
const table = options.table ?? "outboxMessage";
|
|
44
|
+
const qualified = `${identifier(schema)}.${identifier(table)}`;
|
|
45
|
+
const { sql } = db.raw;
|
|
46
|
+
const lock = (tenantId) => sql`SELECT set_config('idle_in_transaction_session_timeout', '0', true) AS lifted, pg_try_advisory_xact_lock(hashtext(${`${schema}.${table}`}), hashtext(${tenantId}))::text AS locked`.returnsRow({
|
|
47
|
+
lifted: "pg/text@1",
|
|
48
|
+
locked: "pg/text@1"
|
|
49
|
+
}).build();
|
|
50
|
+
const select = (tenantId, limit) => sql(statement(`SELECT "id", "tenantId", "kind", "subjectId", "payload", to_json("occurredAt") #>> '{}' AS "occurredAt" FROM ${qualified} WHERE "tenantId" = `, ` AND "publishedAt" IS NULL ORDER BY "id" LIMIT `, "::int"), tenantId, String(limit)).returnsRow({
|
|
51
|
+
id: "pg/int8@1",
|
|
52
|
+
tenantId: "pg/text@1",
|
|
53
|
+
kind: "pg/text@1",
|
|
54
|
+
subjectId: "pg/text@1",
|
|
55
|
+
payload: "pg/text@1",
|
|
56
|
+
occurredAt: "pg/text@1"
|
|
57
|
+
}).build();
|
|
58
|
+
const oldest = (tenantIds) => sql(statement(`SELECT "tenantId", to_json(min("occurredAt")) #>> '{}' AS "occurredAt" FROM ${qualified} WHERE "publishedAt" IS NULL AND "tenantId" IN (SELECT json_array_elements_text(`, `::json)) GROUP BY "tenantId"`), JSON.stringify(tenantIds)).returnsRow({
|
|
59
|
+
tenantId: "pg/text@1",
|
|
60
|
+
occurredAt: "pg/text@1"
|
|
61
|
+
}).build();
|
|
62
|
+
const mark = (ids) => sql(statement(`UPDATE ${qualified} SET "publishedAt" = now() WHERE "id" = ANY(string_to_array(`, ", ',')::bigint[])"), ids.join(",")).affectedCount().build();
|
|
63
|
+
const read = async (tx, tenantId, limit) => (await tx.query(select(tenantId, limit))).map((row) => ({
|
|
64
|
+
...row,
|
|
65
|
+
id: safe(row.id),
|
|
66
|
+
occurredAt: new Date(row.occurredAt)
|
|
67
|
+
}));
|
|
68
|
+
const transaction = (work) => fromSafePromise(Promise.resolve().then(() => db.transaction((tx) => work(tx))));
|
|
69
|
+
return {
|
|
70
|
+
pending: (tenantId, limit) => transaction((tx) => read(tx, tenantId, limit)),
|
|
71
|
+
oldestPending: (tenantIds) => transaction(async (tx) => (await tx.query(oldest(tenantIds))).map(({ tenantId, occurredAt }) => ({
|
|
72
|
+
tenantId,
|
|
73
|
+
occurredAt: new Date(occurredAt)
|
|
74
|
+
}))),
|
|
75
|
+
claim: (tenantId, limit, relay) => transaction(async (tx) => {
|
|
76
|
+
const [held] = await tx.query(lock(tenantId));
|
|
77
|
+
if (held?.locked !== "true") return;
|
|
78
|
+
const published = await relay(await read(tx, tenantId, limit)).get();
|
|
79
|
+
if (published.length > 0) await tx.query(mark(published));
|
|
80
|
+
})
|
|
81
|
+
};
|
|
82
|
+
};
|
|
83
|
+
//#endregion
|
|
84
|
+
export { prismaOutboxStore };
|
package/package.json
CHANGED
|
@@ -1,6 +1,90 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@btravstack/outbox",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
3
|
+
"version": "0.19.0",
|
|
4
|
+
"description": "The transactional outbox relay for @btravstack/core: a poll loop that publishes committed facts in outbox order, a per-tenant claim so replicas take turns rather than race for the same rows, and a Prisma 8 store",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"dependency-injection",
|
|
7
|
+
"outbox",
|
|
8
|
+
"postgres",
|
|
9
|
+
"prisma",
|
|
10
|
+
"typescript",
|
|
11
|
+
"unthrown"
|
|
12
|
+
],
|
|
13
|
+
"homepage": "https://github.com/btravstack/btravstack#readme",
|
|
14
|
+
"bugs": {
|
|
15
|
+
"url": "https://github.com/btravstack/btravstack/issues"
|
|
16
|
+
},
|
|
17
|
+
"license": "MIT",
|
|
18
|
+
"author": "Benoit TRAVERS <benoit.travers.fr@gmail.com>",
|
|
19
|
+
"repository": {
|
|
20
|
+
"type": "git",
|
|
21
|
+
"url": "https://github.com/btravstack/btravstack.git",
|
|
22
|
+
"directory": "packages/outbox"
|
|
23
|
+
},
|
|
24
|
+
"files": [
|
|
25
|
+
"dist"
|
|
26
|
+
],
|
|
27
|
+
"type": "module",
|
|
28
|
+
"sideEffects": false,
|
|
29
|
+
"main": "./dist/index.cjs",
|
|
30
|
+
"module": "./dist/index.mjs",
|
|
31
|
+
"types": "./dist/index.d.cts",
|
|
32
|
+
"exports": {
|
|
33
|
+
".": {
|
|
34
|
+
"import": {
|
|
35
|
+
"types": "./dist/index.d.mts",
|
|
36
|
+
"default": "./dist/index.mjs"
|
|
37
|
+
},
|
|
38
|
+
"require": {
|
|
39
|
+
"types": "./dist/index.d.cts",
|
|
40
|
+
"default": "./dist/index.cjs"
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
"./prisma": {
|
|
44
|
+
"import": {
|
|
45
|
+
"types": "./dist/prisma.d.mts",
|
|
46
|
+
"default": "./dist/prisma.mjs"
|
|
47
|
+
},
|
|
48
|
+
"require": {
|
|
49
|
+
"types": "./dist/prisma.d.cts",
|
|
50
|
+
"default": "./dist/prisma.cjs"
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
"./package.json": "./package.json"
|
|
54
|
+
},
|
|
55
|
+
"devDependencies": {
|
|
56
|
+
"@btravstack/config": "0.19.0",
|
|
57
|
+
"@btravstack/core": "0.19.0",
|
|
58
|
+
"@btravstack/di": "0.19.0",
|
|
59
|
+
"@btravstack/testing": "0.19.0",
|
|
60
|
+
"@btravstack/tsconfig": "0.4.0",
|
|
61
|
+
"@types/node": "26.6.4",
|
|
62
|
+
"@unthrown/vitest": "5.12.0",
|
|
63
|
+
"@vitest/coverage-v8": "5.0.3",
|
|
64
|
+
"tsdown": "0.23.0",
|
|
65
|
+
"typescript": "7.0.2",
|
|
66
|
+
"unthrown": "5.12.0",
|
|
67
|
+
"vitest": "5.0.3"
|
|
68
|
+
},
|
|
69
|
+
"peerDependencies": {
|
|
70
|
+
"@btravstack/config": "^0.19.0",
|
|
71
|
+
"@btravstack/core": "^0.19.0",
|
|
72
|
+
"@btravstack/di": "^0.19.0",
|
|
73
|
+
"@prisma/orm-postgres": "^8.0.0-rc.11",
|
|
74
|
+
"unthrown": "^5.0.0"
|
|
75
|
+
},
|
|
76
|
+
"peerDependenciesMeta": {
|
|
77
|
+
"@prisma/orm-postgres": {
|
|
78
|
+
"optional": true
|
|
79
|
+
}
|
|
80
|
+
},
|
|
81
|
+
"engines": {
|
|
82
|
+
"node": ">=22"
|
|
83
|
+
},
|
|
84
|
+
"scripts": {
|
|
85
|
+
"build": "tsdown src/index.ts src/prisma.ts --format cjs,esm --dts --clean",
|
|
86
|
+
"dev": "tsdown src/index.ts src/prisma.ts --format cjs,esm --dts --watch",
|
|
87
|
+
"test": "vitest run --coverage",
|
|
88
|
+
"typecheck": "tsc --noEmit && tsc --noEmit -p tsconfig.test-d.json"
|
|
89
|
+
}
|
|
6
90
|
}
|