@sguild/dispatcher 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +354 -0
- package/contracts/credit-reservation-funding-state/schema/payloads/reservation.funded-v1.json +59 -0
- package/contracts/credit-reservation-funding-state/schema/payloads/reservation.refunded-v1.json +80 -0
- package/contracts/credit-reservation-funding-state/schema/payloads/reservation.refunding-v1.json +74 -0
- package/contracts/credit-reservation-lock/schema/payloads/credit.consumed-v1.json +33 -0
- package/contracts/credit-reservation-lock/schema/payloads/credit.forfeited-v1.json +41 -0
- package/contracts/credit-reservation-lock/schema/payloads/credit.funded-v1.json +31 -0
- package/contracts/credit-reservation-lock/schema/payloads/credit.locked-v1.json +42 -0
- package/contracts/credit-reservation-lock/schema/payloads/credit.purchased-v1.json +39 -0
- package/contracts/credit-reservation-lock/schema/payloads/credit.released-v1.json +61 -0
- package/contracts/credit-reservation-lock/schema/payloads/credit.released-v2.json +77 -0
- package/contracts/credit-reservation-lock/schema/payloads/credit.reserved-v1.json +60 -0
- package/contracts/credit-reservation-lock/schema/payloads/customer.handoff-v1.json +35 -0
- package/contracts/event-envelope/schema/envelope-v1.json +79 -0
- package/contracts/event-types-registry.json +541 -0
- package/contracts/identity/schema/payloads/intake.amended-v1.json +124 -0
- package/contracts/identity/schema/payloads/intake.captured-v2.json +114 -0
- package/contracts/identity/schema/payloads/person.updated-v1.json +99 -0
- package/contracts/lead-lifecycle/schema/payloads/lead.attempt.exhausted-v1.json +36 -0
- package/contracts/lead-lifecycle/schema/payloads/lead.callback.scheduled-v1.json +39 -0
- package/contracts/lead-lifecycle/schema/payloads/lead.created-v1.json +50 -0
- package/contracts/lead-lifecycle/schema/payloads/lead.reached-v1.json +39 -0
- package/contracts/lead-lifecycle/schema/payloads/lead.stage.changed-v1.json +44 -0
- package/contracts/payment-flow/schema/payloads/payment.failed-v1.json +88 -0
- package/contracts/payment-flow/schema/payloads/payment.received-v1.json +69 -0
- package/contracts/refund-flow/schema/payloads/refund.completed-v1.json +75 -0
- package/contracts/refund-flow/schema/payloads/refund.initiated-v1.json +69 -0
- package/dist/config.d.ts +67 -0
- package/dist/config.js +81 -0
- package/dist/dispatcher-errors.d.ts +20 -0
- package/dist/dispatcher-errors.js +42 -0
- package/dist/dispatcher.d.ts +123 -0
- package/dist/dispatcher.js +171 -0
- package/dist/dlq.d.ts +173 -0
- package/dist/dlq.js +391 -0
- package/dist/fanout-drain.d.ts +11 -0
- package/dist/fanout-drain.js +31 -0
- package/dist/fanout.d.ts +144 -0
- package/dist/fanout.js +321 -0
- package/dist/inbox.d.ts +125 -0
- package/dist/inbox.js +120 -0
- package/dist/index.d.ts +36 -0
- package/dist/index.js +70 -0
- package/dist/internal/id.d.ts +38 -0
- package/dist/internal/id.js +78 -0
- package/dist/internal/pg-search-path.d.ts +34 -0
- package/dist/internal/pg-search-path.js +55 -0
- package/dist/internal/resolve-contract-path.d.ts +41 -0
- package/dist/internal/resolve-contract-path.js +65 -0
- package/dist/observability.d.ts +24 -0
- package/dist/observability.js +37 -0
- package/dist/postgres-consumer.d.ts +175 -0
- package/dist/postgres-consumer.js +561 -0
- package/dist/postgres-transport.d.ts +70 -0
- package/dist/postgres-transport.js +144 -0
- package/dist/producer-db.d.ts +80 -0
- package/dist/producer-db.js +115 -0
- package/dist/registry.d.ts +94 -0
- package/dist/registry.js +99 -0
- package/dist/signature.d.ts +44 -0
- package/dist/signature.js +79 -0
- package/dist/types.d.ts +107 -0
- package/dist/types.js +13 -0
- package/dist/validator.d.ts +60 -0
- package/dist/validator.js +171 -0
- package/package.json +48 -0
|
@@ -0,0 +1,561 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Postgres-queue transport: consumer-side polling worker.
|
|
4
|
+
*
|
|
5
|
+
* Implements the consume path of the bus dispatcher per ADR-0009
|
|
6
|
+
* §"Decision". Long-lived process (typically run from
|
|
7
|
+
* `<repo>/scripts/<consumer>-subscriber.ts` per the per-domain module
|
|
8
|
+
* pattern) that polls `dispatcher_event` past a per-(consumer,
|
|
9
|
+
* event_type) cursor, dedups against `dispatcher_dedup`, invokes the
|
|
10
|
+
* registered handler, retries on exception with exponential backoff and
|
|
11
|
+
* jitter, dead-letters on exhaustion, and advances the cursor atomically
|
|
12
|
+
* with the dedup write on success (or the dead-letter write on
|
|
13
|
+
* exhaustion).
|
|
14
|
+
*
|
|
15
|
+
* Phase 2 Slice 3 (this file): poll-only consumer. The LISTEN/NOTIFY
|
|
16
|
+
* wake-up primitive ADR-0009 §"Decision" calls for is the Slice 3b
|
|
17
|
+
* deliverable; Slice 3 ships poll-only because the polling loop is the
|
|
18
|
+
* durable fallback regardless of whether LISTEN/NOTIFY is wired, and
|
|
19
|
+
* shipping poll-only first gives consumers a working subscriber surface
|
|
20
|
+
* to scaffold against. Consumers cut over to the same call shape when
|
|
21
|
+
* LISTEN/NOTIFY lands; only the latency profile changes (sub-second on
|
|
22
|
+
* the happy path, polling cadence as the fallback under either model).
|
|
23
|
+
*
|
|
24
|
+
* Per-consumer scope: each consumer instance is identified by a
|
|
25
|
+
* stable string (e.g., "coaching-availability-subscriber") that goes
|
|
26
|
+
* onto the cursor and dedup rows. A given consumer string MUST be
|
|
27
|
+
* unique across deployments; running two processes against the same
|
|
28
|
+
* consumer string causes them to compete for cursor advancement and
|
|
29
|
+
* the dedup window collapses to per-consumer-id rather than
|
|
30
|
+
* per-process. Per the build plan §3, the typical shape is one
|
|
31
|
+
* consumer string per long-lived worker process per domain.
|
|
32
|
+
*
|
|
33
|
+
* Per-event-type fan-out: a single ConsumerLoop instance processes
|
|
34
|
+
* all subscribed event_types serially within a poll cycle (one
|
|
35
|
+
* batch per event_type, in registration order). For workloads where
|
|
36
|
+
* one event_type has substantially higher volume than others and the
|
|
37
|
+
* serial drain becomes a latency bottleneck, run multiple
|
|
38
|
+
* ConsumerLoop instances against the same database with different
|
|
39
|
+
* consumer strings — but typical Sguild volume per consumer fits
|
|
40
|
+
* comfortably inside a single loop's drain cadence.
|
|
41
|
+
*/
|
|
42
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
43
|
+
exports.ConsumerLoop = void 0;
|
|
44
|
+
const pg_1 = require("pg");
|
|
45
|
+
const pg_search_path_1 = require("./internal/pg-search-path");
|
|
46
|
+
const id_1 = require("./internal/id");
|
|
47
|
+
const observability_1 = require("./observability");
|
|
48
|
+
// =============================================================================
|
|
49
|
+
// Defaults aligned with ADR-0009 §"Action Items" and §"Trade-off Analysis".
|
|
50
|
+
// =============================================================================
|
|
51
|
+
/** Default poll interval; the LISTEN/NOTIFY wake-up shipping in Slice 3b will reduce typical wake-up latency to sub-second. */
|
|
52
|
+
const DEFAULT_POLL_INTERVAL_MS = 5000;
|
|
53
|
+
/** Default batch size per event_type per poll cycle. Bounded so a single consumer cannot starve other event_types within the same loop. */
|
|
54
|
+
const DEFAULT_BATCH_SIZE = 50;
|
|
55
|
+
/**
|
|
56
|
+
* Default retry-after delays in ms. Three retries past the initial
|
|
57
|
+
* attempt per ADR-0009 §"Decision"; exponential progression with the
|
|
58
|
+
* actual delay carrying multiplicative jitter (0-30 percent of base) so
|
|
59
|
+
* a thundering herd of failed events does not retry in lockstep.
|
|
60
|
+
*/
|
|
61
|
+
const DEFAULT_RETRY_DELAYS_MS = [1000, 5000, 15000];
|
|
62
|
+
/** ID prefix for dead-letter rows per the prefix convention in ADR-0002. */
|
|
63
|
+
const DEAD_LETTER_ID_PREFIX = "dlq_";
|
|
64
|
+
/** LISTEN channel name. Matches the trigger function in Slice 1's migration. */
|
|
65
|
+
const LISTEN_CHANNEL = "dispatcher_event_inserted";
|
|
66
|
+
/** Backoff delays when the LISTEN connection drops and needs to reconnect. */
|
|
67
|
+
const LISTEN_RECONNECT_DELAYS_MS = [1000, 5000, 15000, 60000];
|
|
68
|
+
// =============================================================================
|
|
69
|
+
// ConsumerLoop
|
|
70
|
+
// =============================================================================
|
|
71
|
+
class ConsumerLoop {
|
|
72
|
+
handlers;
|
|
73
|
+
running = false;
|
|
74
|
+
inflight = false;
|
|
75
|
+
options;
|
|
76
|
+
pool;
|
|
77
|
+
ownsPool;
|
|
78
|
+
databaseUrl;
|
|
79
|
+
/**
|
|
80
|
+
* Quoted table-name prefix. Empty string when no schema is configured;
|
|
81
|
+
* `"<schema>".` (with trailing dot) when a schema is set. Prepended to
|
|
82
|
+
* every dispatcher_X table reference in the SQL queries so they work
|
|
83
|
+
* even when Supabase's pooler drops the search_path option.
|
|
84
|
+
*/
|
|
85
|
+
tablePrefix;
|
|
86
|
+
subscribedEventTypes;
|
|
87
|
+
// LISTEN/NOTIFY state. The listenClient is a separate pg.Client (not
|
|
88
|
+
// the Prisma pool) because LISTEN ties up the connection for the
|
|
89
|
+
// duration; using a Prisma-pool connection would block other queries.
|
|
90
|
+
// Reconnect attempts run with backoff per LISTEN_RECONNECT_DELAYS_MS;
|
|
91
|
+
// polling stays as the durable fallback the whole time.
|
|
92
|
+
listenClient = null;
|
|
93
|
+
listenReconnectScheduled = false;
|
|
94
|
+
listenReconnectAttempt = 0;
|
|
95
|
+
// Wake-up controller. The polling loop's between-cycle sleep is
|
|
96
|
+
// interruptible: when a NOTIFY arrives for a subscribed event_type
|
|
97
|
+
// (or when stop() is called), `wakeUp()` aborts the sleep and the
|
|
98
|
+
// loop runs another cycle immediately.
|
|
99
|
+
sleepAbort = null;
|
|
100
|
+
constructor(handlers, options) {
|
|
101
|
+
this.handlers = handlers;
|
|
102
|
+
if (!options.consumer) {
|
|
103
|
+
throw new Error("ConsumerLoop: consumer name is required");
|
|
104
|
+
}
|
|
105
|
+
if (handlers.length === 0) {
|
|
106
|
+
throw new Error("ConsumerLoop: at least one handler must be registered");
|
|
107
|
+
}
|
|
108
|
+
this.options = {
|
|
109
|
+
consumer: options.consumer,
|
|
110
|
+
batchSize: options.batchSize ?? DEFAULT_BATCH_SIZE,
|
|
111
|
+
pollIntervalMs: options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS,
|
|
112
|
+
retryDelaysMs: options.retryDelaysMs ?? DEFAULT_RETRY_DELAYS_MS,
|
|
113
|
+
};
|
|
114
|
+
this.databaseUrl = options.databaseUrl ?? process.env.DATABASE_URL;
|
|
115
|
+
if (options.pool) {
|
|
116
|
+
this.pool = options.pool;
|
|
117
|
+
this.ownsPool = false;
|
|
118
|
+
}
|
|
119
|
+
else {
|
|
120
|
+
// Default pool from DATABASE_URL for the in-process Platform path.
|
|
121
|
+
// Owned by this loop so stop() can drain connections cleanly.
|
|
122
|
+
this.pool = new pg_1.Pool((0, pg_search_path_1.pgConfigWithSearchPath)(this.databaseUrl));
|
|
123
|
+
this.pool.on("error", (error) => {
|
|
124
|
+
console.warn(`ConsumerLoop[${this.options.consumer}]: pool error: ${error.message}`);
|
|
125
|
+
});
|
|
126
|
+
this.ownsPool = true;
|
|
127
|
+
}
|
|
128
|
+
// Schema-qualify the dispatcher tables if a schema is configured. Bypasses
|
|
129
|
+
// pgbouncer's transaction-mode pooling, which silently drops the libpq
|
|
130
|
+
// `options=-c search_path=...` startup parameter — the search_path config
|
|
131
|
+
// ends up not applying on pooler DSNs even though the helper sets it.
|
|
132
|
+
// Fall back to the DSN's `?schema=` parsed value when the caller did not
|
|
133
|
+
// pass an explicit `schema` option, so the in-process path (Platform's
|
|
134
|
+
// own DATABASE_URL with `?schema=platform`) gets the same treatment.
|
|
135
|
+
const resolvedSchema = options.schema ?? (0, pg_search_path_1.pgConfigWithSearchPath)(this.databaseUrl).schema;
|
|
136
|
+
if (resolvedSchema) {
|
|
137
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(resolvedSchema)) {
|
|
138
|
+
throw new Error(`ConsumerLoop[${options.consumer}]: invalid schema name "${resolvedSchema}"`);
|
|
139
|
+
}
|
|
140
|
+
this.tablePrefix = `"${resolvedSchema}".`;
|
|
141
|
+
}
|
|
142
|
+
else {
|
|
143
|
+
this.tablePrefix = "";
|
|
144
|
+
}
|
|
145
|
+
this.subscribedEventTypes = new Set(handlers.map((h) => h.eventType));
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Start the polling loop with LISTEN/NOTIFY wake-up. Resolves when
|
|
149
|
+
* `stop()` is called and the current in-flight batch (if any) finishes
|
|
150
|
+
* draining; otherwise runs forever.
|
|
151
|
+
*
|
|
152
|
+
* The LISTEN connection is best-effort: if it fails to set up or drops
|
|
153
|
+
* mid-flight, the polling loop still drains at the configured cadence
|
|
154
|
+
* and reconnect attempts run with backoff. Sub-second wake-up is the
|
|
155
|
+
* happy path; degraded poll-cadence wake-up is the durable fallback.
|
|
156
|
+
*/
|
|
157
|
+
async start() {
|
|
158
|
+
if (this.running) {
|
|
159
|
+
throw new Error(`ConsumerLoop[${this.options.consumer}]: already running`);
|
|
160
|
+
}
|
|
161
|
+
this.running = true;
|
|
162
|
+
// Kick off the LISTEN connection in the background; do not await so
|
|
163
|
+
// the polling loop starts immediately whether or not LISTEN comes
|
|
164
|
+
// up successfully.
|
|
165
|
+
void this.startListenConnection();
|
|
166
|
+
while (this.running) {
|
|
167
|
+
this.inflight = true;
|
|
168
|
+
try {
|
|
169
|
+
for (const reg of this.handlers) {
|
|
170
|
+
if (!this.running)
|
|
171
|
+
break;
|
|
172
|
+
await this.processBatchForEventType(reg);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
finally {
|
|
176
|
+
this.inflight = false;
|
|
177
|
+
}
|
|
178
|
+
if (!this.running)
|
|
179
|
+
break;
|
|
180
|
+
await this.sleepOrWakeUp(this.options.pollIntervalMs);
|
|
181
|
+
}
|
|
182
|
+
await this.stopListenConnection();
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Graceful shutdown. Sets the running flag false, wakes the polling
|
|
186
|
+
* loop early so it exits the current sleep without waiting for the
|
|
187
|
+
* full poll interval, then waits for the in-flight batch to drain
|
|
188
|
+
* before resolving. Closes the LISTEN connection on the way out.
|
|
189
|
+
*/
|
|
190
|
+
async stop() {
|
|
191
|
+
this.running = false;
|
|
192
|
+
this.wakeUp();
|
|
193
|
+
while (this.inflight) {
|
|
194
|
+
await this.sleep(50);
|
|
195
|
+
}
|
|
196
|
+
await this.stopListenConnection();
|
|
197
|
+
if (this.ownsPool) {
|
|
198
|
+
await this.pool.end().catch(() => {
|
|
199
|
+
// Pool may already be closing if Node is shutting down; the
|
|
200
|
+
// exit handler races with stop(). Swallow so SIGTERM drain
|
|
201
|
+
// doesn't crash.
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
// =============================================================================
|
|
206
|
+
// Internal: batch processing
|
|
207
|
+
// =============================================================================
|
|
208
|
+
// ===========================================================================
|
|
209
|
+
// Raw SQL queries. Unqualified table names so the connection's search_path
|
|
210
|
+
// (derived from the DSN's `?schema=<domain>` query param) resolves to the right
|
|
211
|
+
// schema. This is what makes the loop work against producer DBs whose
|
|
212
|
+
// dispatcher tables live in domain-specific schemas (Sales' `sales`,
|
|
213
|
+
// Growth's `growth`, etc.) — Platform's Prisma client always qualifies
|
|
214
|
+
// with `public`, which fails against those.
|
|
215
|
+
// ===========================================================================
|
|
216
|
+
async getCursorLastSeq(eventType) {
|
|
217
|
+
const result = await this.pool.query(`SELECT last_seq::text FROM ${this.tablePrefix}dispatcher_cursor
|
|
218
|
+
WHERE consumer = $1 AND event_type = $2`, [this.options.consumer, eventType]);
|
|
219
|
+
if (result.rows.length === 0)
|
|
220
|
+
return 0n;
|
|
221
|
+
return BigInt(result.rows[0].last_seq);
|
|
222
|
+
}
|
|
223
|
+
async findEventsAfter(eventType, afterSeq) {
|
|
224
|
+
const result = await this.pool.query(`SELECT id, seq::text, event_type, schema_version, tenant_id, producer,
|
|
225
|
+
subject, actor, occurred_at, payload, correlation_id
|
|
226
|
+
FROM ${this.tablePrefix}dispatcher_event
|
|
227
|
+
WHERE event_type = $1 AND seq > $2
|
|
228
|
+
ORDER BY seq ASC
|
|
229
|
+
LIMIT $3`, [eventType, afterSeq.toString(), this.options.batchSize]);
|
|
230
|
+
return result.rows.map((row) => ({
|
|
231
|
+
id: row.id,
|
|
232
|
+
seq: BigInt(row.seq),
|
|
233
|
+
eventType: row.event_type,
|
|
234
|
+
schemaVersion: row.schema_version,
|
|
235
|
+
tenantId: row.tenant_id,
|
|
236
|
+
producer: row.producer,
|
|
237
|
+
subject: row.subject,
|
|
238
|
+
actor: row.actor,
|
|
239
|
+
occurredAt: new Date(row.occurred_at),
|
|
240
|
+
payload: row.payload,
|
|
241
|
+
correlationId: row.correlation_id,
|
|
242
|
+
}));
|
|
243
|
+
}
|
|
244
|
+
async getDedupHit(eventId) {
|
|
245
|
+
const result = await this.pool.query(`SELECT 1 FROM ${this.tablePrefix}dispatcher_dedup
|
|
246
|
+
WHERE consumer = $1 AND event_id = $2 LIMIT 1`, [this.options.consumer, eventId]);
|
|
247
|
+
return result.rows.length > 0;
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Single-transaction dedup-insert + cursor-advance. The pg.PoolClient
|
|
251
|
+
* is checked out for the transaction so the BEGIN/COMMIT pair is
|
|
252
|
+
* scoped to one connection. Errors trigger a ROLLBACK and the loop
|
|
253
|
+
* retries via the normal retry path.
|
|
254
|
+
*/
|
|
255
|
+
async commitSuccessfulDelivery(eventId, eventType, seq) {
|
|
256
|
+
const client = await this.pool.connect();
|
|
257
|
+
try {
|
|
258
|
+
await client.query("BEGIN");
|
|
259
|
+
await client.query(`INSERT INTO ${this.tablePrefix}dispatcher_dedup (consumer, event_id)
|
|
260
|
+
VALUES ($1, $2)`, [this.options.consumer, eventId]);
|
|
261
|
+
await client.query(`INSERT INTO ${this.tablePrefix}dispatcher_cursor (consumer, event_type, last_seq, updated_at)
|
|
262
|
+
VALUES ($1, $2, $3, CURRENT_TIMESTAMP)
|
|
263
|
+
ON CONFLICT (consumer, event_type)
|
|
264
|
+
DO UPDATE SET last_seq = EXCLUDED.last_seq, updated_at = CURRENT_TIMESTAMP`, [this.options.consumer, eventType, seq.toString()]);
|
|
265
|
+
await client.query("COMMIT");
|
|
266
|
+
}
|
|
267
|
+
catch (e) {
|
|
268
|
+
await client.query("ROLLBACK").catch(() => { });
|
|
269
|
+
throw e;
|
|
270
|
+
}
|
|
271
|
+
finally {
|
|
272
|
+
client.release();
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
async commitDeadLetter(eventId, eventType, seq, attemptCount, lastError, envelope) {
|
|
276
|
+
const client = await this.pool.connect();
|
|
277
|
+
try {
|
|
278
|
+
await client.query("BEGIN");
|
|
279
|
+
await client.query(`INSERT INTO ${this.tablePrefix}dispatcher_dead_letter
|
|
280
|
+
(id, consumer, event_id, attempt_count, last_error, last_error_stack, envelope_snapshot)
|
|
281
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7)`, [
|
|
282
|
+
`${DEAD_LETTER_ID_PREFIX}${(0, id_1.uuidv7)()}`,
|
|
283
|
+
this.options.consumer,
|
|
284
|
+
eventId,
|
|
285
|
+
attemptCount,
|
|
286
|
+
lastError?.message ?? "unknown error",
|
|
287
|
+
lastError?.stack ?? null,
|
|
288
|
+
JSON.stringify(envelope),
|
|
289
|
+
]);
|
|
290
|
+
await client.query(`INSERT INTO ${this.tablePrefix}dispatcher_cursor (consumer, event_type, last_seq, updated_at)
|
|
291
|
+
VALUES ($1, $2, $3, CURRENT_TIMESTAMP)
|
|
292
|
+
ON CONFLICT (consumer, event_type)
|
|
293
|
+
DO UPDATE SET last_seq = EXCLUDED.last_seq, updated_at = CURRENT_TIMESTAMP`, [this.options.consumer, eventType, seq.toString()]);
|
|
294
|
+
await client.query("COMMIT");
|
|
295
|
+
}
|
|
296
|
+
catch (e) {
|
|
297
|
+
await client.query("ROLLBACK").catch(() => { });
|
|
298
|
+
throw e;
|
|
299
|
+
}
|
|
300
|
+
finally {
|
|
301
|
+
client.release();
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
// ===========================================================================
|
|
305
|
+
// Batch processing
|
|
306
|
+
// ===========================================================================
|
|
307
|
+
async processBatchForEventType(reg) {
|
|
308
|
+
const lastSeq = await this.getCursorLastSeq(reg.eventType);
|
|
309
|
+
const events = await this.findEventsAfter(reg.eventType, lastSeq);
|
|
310
|
+
for (const eventRow of events) {
|
|
311
|
+
if (!this.running)
|
|
312
|
+
break;
|
|
313
|
+
await this.deliverEvent(eventRow, reg.handler);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
async deliverEvent(eventRow, handler) {
|
|
317
|
+
// Dedup check: if the (consumer, event_id) tuple was already
|
|
318
|
+
// delivered, skip the handler invocation but still advance the
|
|
319
|
+
// cursor so a re-dispatched event does not block the queue.
|
|
320
|
+
if (await this.getDedupHit(eventRow.id)) {
|
|
321
|
+
(0, observability_1.recordDispatcherIncrement)("dispatcher.dedup_hit.count", {
|
|
322
|
+
event_type: eventRow.eventType,
|
|
323
|
+
consumer: this.options.consumer,
|
|
324
|
+
});
|
|
325
|
+
console.log(`[${this.options.consumer}] dedup-hit ${eventRow.eventType} ${eventRow.id}`);
|
|
326
|
+
await this.advanceCursor(eventRow.eventType, eventRow.seq);
|
|
327
|
+
return;
|
|
328
|
+
}
|
|
329
|
+
const envelope = rowToEnvelope(eventRow);
|
|
330
|
+
const totalAttempts = this.options.retryDelaysMs.length + 1;
|
|
331
|
+
let lastError = null;
|
|
332
|
+
let attemptCount = 0;
|
|
333
|
+
for (let attempt = 0; attempt < totalAttempts; attempt += 1) {
|
|
334
|
+
attemptCount = attempt + 1;
|
|
335
|
+
try {
|
|
336
|
+
const handlerStartedAt = Date.now();
|
|
337
|
+
await handler(envelope);
|
|
338
|
+
const handlerLatencyMs = Math.max(0, Date.now() - handlerStartedAt);
|
|
339
|
+
const endToEndLatencyMs = Math.max(0, Date.now() - eventRow.occurredAt.getTime());
|
|
340
|
+
await this.commitSuccessfulDelivery(eventRow.id, eventRow.eventType, eventRow.seq);
|
|
341
|
+
(0, observability_1.recordDispatcherIncrement)("dispatcher.consume.count", {
|
|
342
|
+
event_type: eventRow.eventType,
|
|
343
|
+
consumer: this.options.consumer,
|
|
344
|
+
});
|
|
345
|
+
(0, observability_1.recordDispatcherObservation)("dispatcher.end_to_end_latency_ms", endToEndLatencyMs, {
|
|
346
|
+
event_type: eventRow.eventType,
|
|
347
|
+
consumer: this.options.consumer,
|
|
348
|
+
});
|
|
349
|
+
(0, observability_1.recordDispatcherObservation)("dispatcher.handler_latency_ms", handlerLatencyMs, {
|
|
350
|
+
event_type: eventRow.eventType,
|
|
351
|
+
consumer: this.options.consumer,
|
|
352
|
+
});
|
|
353
|
+
console.log(`[${this.options.consumer}] delivered ${eventRow.eventType} ${eventRow.id} in ${handlerLatencyMs}ms (e2e ${endToEndLatencyMs}ms)`);
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
356
|
+
catch (e) {
|
|
357
|
+
lastError = e instanceof Error ? e : new Error(String(e));
|
|
358
|
+
if (attempt < this.options.retryDelaysMs.length) {
|
|
359
|
+
const baseDelay = this.options.retryDelaysMs[attempt];
|
|
360
|
+
const jitter = Math.random() * 0.3 * baseDelay;
|
|
361
|
+
console.warn(`[${this.options.consumer}] retry ${attemptCount}/${totalAttempts} for ${eventRow.eventType} ${eventRow.id} after ${baseDelay}ms: ${lastError.message}`);
|
|
362
|
+
await this.sleep(baseDelay + jitter);
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
// Retries exhausted: dead-letter the event and advance the cursor.
|
|
367
|
+
await this.commitDeadLetter(eventRow.id, eventRow.eventType, eventRow.seq, attemptCount, lastError, envelope);
|
|
368
|
+
(0, observability_1.recordDispatcherIncrement)("dispatcher.dead_letter.count", {
|
|
369
|
+
event_type: eventRow.eventType,
|
|
370
|
+
consumer: this.options.consumer,
|
|
371
|
+
});
|
|
372
|
+
console.error(`[${this.options.consumer}] DLQ ${eventRow.eventType} ${eventRow.id} after ${attemptCount} attempts: ${lastError?.message ?? "unknown error"}`);
|
|
373
|
+
}
|
|
374
|
+
async advanceCursor(eventType, seq) {
|
|
375
|
+
await this.pool.query(`INSERT INTO ${this.tablePrefix}dispatcher_cursor (consumer, event_type, last_seq, updated_at)
|
|
376
|
+
VALUES ($1, $2, $3, CURRENT_TIMESTAMP)
|
|
377
|
+
ON CONFLICT (consumer, event_type)
|
|
378
|
+
DO UPDATE SET last_seq = EXCLUDED.last_seq, updated_at = CURRENT_TIMESTAMP`, [this.options.consumer, eventType, seq.toString()]);
|
|
379
|
+
}
|
|
380
|
+
sleep(ms) {
|
|
381
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
382
|
+
}
|
|
383
|
+
// =============================================================================
|
|
384
|
+
// LISTEN/NOTIFY wake-up. The trigger function in Slice 1's migration
|
|
385
|
+
// fires `pg_notify('dispatcher_event_inserted', NEW.event_type)` on
|
|
386
|
+
// every insert into dispatcher_event. The consumer LISTENs on that
|
|
387
|
+
// channel, filters on event_type (only wakes up for event_types it
|
|
388
|
+
// subscribes to), and aborts the polling sleep so the next batch
|
|
389
|
+
// runs immediately. Polling cadence is the durable fallback for
|
|
390
|
+
// missed notifications (LISTEN connection drop, NOTIFY queue
|
|
391
|
+
// overflow, transient network blip, etc.).
|
|
392
|
+
// =============================================================================
|
|
393
|
+
/**
|
|
394
|
+
* Detect whether a DSN points at a transaction-mode pooler (Supabase's
|
|
395
|
+
* pgbouncer at port 6543). LISTEN holds a connection open waiting for
|
|
396
|
+
* NOTIFY; transaction-mode pooling reuses connections across queries
|
|
397
|
+
* and rejects LISTEN with ENOIDENTIFIER. The fix is to skip the LISTEN
|
|
398
|
+
* client entirely against pooler URLs and rely on polling. Operators
|
|
399
|
+
* who want sub-second wake-up can either provide a session-mode URL
|
|
400
|
+
* (port 5432 / direct connection) via a future databaseUrl override
|
|
401
|
+
* or run a non-pooler DSN.
|
|
402
|
+
*
|
|
403
|
+
* Signal: `pgbouncer=true` query param OR port 6543 anywhere in the
|
|
404
|
+
* URL. Both are conventions Supabase generates for pooled URLs.
|
|
405
|
+
*/
|
|
406
|
+
isPoolerDsn(dsn) {
|
|
407
|
+
return /pgbouncer=true/i.test(dsn) || /:6543\b/.test(dsn);
|
|
408
|
+
}
|
|
409
|
+
async startListenConnection() {
|
|
410
|
+
if (!this.running)
|
|
411
|
+
return;
|
|
412
|
+
if (this.listenClient)
|
|
413
|
+
return; // already connected
|
|
414
|
+
const databaseUrl = this.databaseUrl;
|
|
415
|
+
if (!databaseUrl) {
|
|
416
|
+
// No DSN provisioned means we cannot construct a pg.Client; the
|
|
417
|
+
// polling loop is the only path. Log and skip; do not throw, so
|
|
418
|
+
// a misconfigured env still gets degraded-but-correct delivery.
|
|
419
|
+
// Production deploys MUST set DATABASE_URL (or the producer-specific
|
|
420
|
+
// DISPATCHER_PRODUCER_DATABASE_URL_<PRODUCER> when this loop reads
|
|
421
|
+
// a non-self producer); this guard exists for CI and dev-loop edge
|
|
422
|
+
// cases.
|
|
423
|
+
console.warn(`ConsumerLoop[${this.options.consumer}]: no DSN provided; LISTEN/NOTIFY wake-up disabled, polling-only fallback active`);
|
|
424
|
+
return;
|
|
425
|
+
}
|
|
426
|
+
if (this.isPoolerDsn(databaseUrl)) {
|
|
427
|
+
// Transaction-mode pooler can't host a LISTEN socket. Log once at
|
|
428
|
+
// boot, never schedule a reconnect, drain via polling. The Slice 3
|
|
429
|
+
// ship memo treats polling as the durable fallback; this just makes
|
|
430
|
+
// explicit that there's no LISTEN path on this loop.
|
|
431
|
+
if (this.listenReconnectAttempt === 0) {
|
|
432
|
+
console.log(`ConsumerLoop[${this.options.consumer}]: DSN is a transaction-mode pooler; LISTEN/NOTIFY skipped, polling at ${this.options.pollIntervalMs}ms`);
|
|
433
|
+
this.listenReconnectAttempt = 1; // mark as "decided"; suppresses any later reconnect path
|
|
434
|
+
}
|
|
435
|
+
return;
|
|
436
|
+
}
|
|
437
|
+
const client = new pg_1.Client((0, pg_search_path_1.pgConfigWithSearchPath)(databaseUrl));
|
|
438
|
+
client.on("notification", (msg) => this.onNotification(msg));
|
|
439
|
+
client.on("error", (err) => {
|
|
440
|
+
console.warn(`ConsumerLoop[${this.options.consumer}]: LISTEN connection error: ${err.message}; will reconnect with backoff`);
|
|
441
|
+
void client.end().catch(() => undefined);
|
|
442
|
+
this.scheduleListenReconnect();
|
|
443
|
+
});
|
|
444
|
+
client.on("end", () => {
|
|
445
|
+
// Connection closed (server-side timeout, restart, etc.); reconnect
|
|
446
|
+
// unless we are shutting down.
|
|
447
|
+
if (this.running) {
|
|
448
|
+
this.scheduleListenReconnect();
|
|
449
|
+
}
|
|
450
|
+
});
|
|
451
|
+
try {
|
|
452
|
+
await client.connect();
|
|
453
|
+
await client.query(`LISTEN ${LISTEN_CHANNEL}`);
|
|
454
|
+
this.listenClient = client;
|
|
455
|
+
this.listenReconnectAttempt = 0; // success resets backoff
|
|
456
|
+
}
|
|
457
|
+
catch (e) {
|
|
458
|
+
console.warn(`ConsumerLoop[${this.options.consumer}]: LISTEN setup failed: ${e instanceof Error ? e.message : String(e)}; will reconnect with backoff`);
|
|
459
|
+
// Best-effort cleanup on the half-connected client
|
|
460
|
+
try {
|
|
461
|
+
await client.end();
|
|
462
|
+
}
|
|
463
|
+
catch {
|
|
464
|
+
// ignore
|
|
465
|
+
}
|
|
466
|
+
this.scheduleListenReconnect();
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
scheduleListenReconnect() {
|
|
470
|
+
if (!this.running)
|
|
471
|
+
return;
|
|
472
|
+
if (this.listenReconnectScheduled)
|
|
473
|
+
return;
|
|
474
|
+
this.listenReconnectScheduled = true;
|
|
475
|
+
this.listenClient = null;
|
|
476
|
+
const delay = LISTEN_RECONNECT_DELAYS_MS[Math.min(this.listenReconnectAttempt, LISTEN_RECONNECT_DELAYS_MS.length - 1)];
|
|
477
|
+
this.listenReconnectAttempt += 1;
|
|
478
|
+
setTimeout(() => {
|
|
479
|
+
this.listenReconnectScheduled = false;
|
|
480
|
+
void this.startListenConnection();
|
|
481
|
+
}, delay).unref();
|
|
482
|
+
}
|
|
483
|
+
async stopListenConnection() {
|
|
484
|
+
const client = this.listenClient;
|
|
485
|
+
if (!client)
|
|
486
|
+
return;
|
|
487
|
+
this.listenClient = null;
|
|
488
|
+
try {
|
|
489
|
+
// UNLISTEN is implicit on connection close, but explicit is
|
|
490
|
+
// clearer for the audit trail in Postgres logs.
|
|
491
|
+
await client.query(`UNLISTEN ${LISTEN_CHANNEL}`);
|
|
492
|
+
}
|
|
493
|
+
catch {
|
|
494
|
+
// ignore; about to close anyway
|
|
495
|
+
}
|
|
496
|
+
try {
|
|
497
|
+
await client.end();
|
|
498
|
+
}
|
|
499
|
+
catch {
|
|
500
|
+
// ignore
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
onNotification(msg) {
|
|
504
|
+
if (msg.channel !== LISTEN_CHANNEL)
|
|
505
|
+
return;
|
|
506
|
+
// Filter on event_type so we only wake up for events this consumer
|
|
507
|
+
// actually subscribes to. The trigger payload is the event_type
|
|
508
|
+
// string. NOTIFY payload limit is ~8000 bytes; event_type names are
|
|
509
|
+
// short, so truncation is not a concern.
|
|
510
|
+
const eventType = msg.payload ?? "";
|
|
511
|
+
if (!this.subscribedEventTypes.has(eventType))
|
|
512
|
+
return;
|
|
513
|
+
this.wakeUp();
|
|
514
|
+
}
|
|
515
|
+
wakeUp() {
|
|
516
|
+
if (this.sleepAbort) {
|
|
517
|
+
this.sleepAbort.abort();
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
async sleepOrWakeUp(ms) {
|
|
521
|
+
this.sleepAbort = new AbortController();
|
|
522
|
+
const signal = this.sleepAbort.signal;
|
|
523
|
+
try {
|
|
524
|
+
await new Promise((resolve) => {
|
|
525
|
+
const timeout = setTimeout(resolve, ms);
|
|
526
|
+
signal.addEventListener("abort", () => {
|
|
527
|
+
clearTimeout(timeout);
|
|
528
|
+
resolve();
|
|
529
|
+
}, { once: true });
|
|
530
|
+
});
|
|
531
|
+
}
|
|
532
|
+
finally {
|
|
533
|
+
this.sleepAbort = null;
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
exports.ConsumerLoop = ConsumerLoop;
|
|
538
|
+
function rowToEnvelope(row) {
|
|
539
|
+
// Reconstruct the envelope from the persisted row. The shape mirrors
|
|
540
|
+
// the producer's published envelope exactly so the handler sees the
|
|
541
|
+
// same object the producer intended.
|
|
542
|
+
const envelope = {
|
|
543
|
+
event_id: row.id,
|
|
544
|
+
event_type: row.eventType,
|
|
545
|
+
occurred_at: row.occurredAt.toISOString(),
|
|
546
|
+
tenant_id: row.tenantId,
|
|
547
|
+
producer: row.producer,
|
|
548
|
+
schema_version: row.schemaVersion,
|
|
549
|
+
payload: row.payload,
|
|
550
|
+
};
|
|
551
|
+
if (row.subject !== null) {
|
|
552
|
+
envelope.subject = row.subject;
|
|
553
|
+
}
|
|
554
|
+
if (row.actor) {
|
|
555
|
+
envelope.actor = row.actor;
|
|
556
|
+
}
|
|
557
|
+
if (row.correlationId !== null) {
|
|
558
|
+
envelope.correlation_id = row.correlationId;
|
|
559
|
+
}
|
|
560
|
+
return envelope;
|
|
561
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Postgres-queue transport for the dispatcher SDK.
|
|
3
|
+
*
|
|
4
|
+
* Implements the publish-side path of the bus dispatcher per ADR-0009
|
|
5
|
+
* §"Decision". The producer-side transactional guarantee is the
|
|
6
|
+
* load-bearing primitive: producers call `dispatcher.publish` inside a
|
|
7
|
+
* Prisma transaction, the SDK inserts the event row in the SAME
|
|
8
|
+
* transaction (via the `tx` argument), and either both the domain write
|
|
9
|
+
* and the event emit commit or both roll back. No outbox pattern, no
|
|
10
|
+
* staging table, no second state machine to debug; this is the property
|
|
11
|
+
* that breaks the tie among the four bus options ADR-0009 considered.
|
|
12
|
+
*
|
|
13
|
+
* Phase 2 Slice 2 (this file): publish path. Slice 3 lands the consumer
|
|
14
|
+
* polling worker plus LISTEN/NOTIFY listener; Slice 4 wires this into
|
|
15
|
+
* `dispatcher.ts`'s public surface (replacing the
|
|
16
|
+
* DispatcherNotImplementedError stubs); Slice 5 ships the per-consumer
|
|
17
|
+
* DLQ read API.
|
|
18
|
+
*
|
|
19
|
+
* Usage shape producers will adopt at their state-transition call sites:
|
|
20
|
+
*
|
|
21
|
+
* await prisma.$transaction(async (tx) => {
|
|
22
|
+
* // ... domain write that produces the event ...
|
|
23
|
+
* await tx.creditReservation.update({ where: { id }, data: { state: "locked" } });
|
|
24
|
+
*
|
|
25
|
+
* // ... emit the event in the SAME transaction ...
|
|
26
|
+
* await dispatcher.publish({
|
|
27
|
+
* event_type: "credit.locked",
|
|
28
|
+
* payload: { credit_reservation_id: id, ... },
|
|
29
|
+
* subject: personId,
|
|
30
|
+
* actor: "system:revenue",
|
|
31
|
+
* }, { tx });
|
|
32
|
+
* });
|
|
33
|
+
*
|
|
34
|
+
* Without the `tx` option, publish runs in its own transaction off the
|
|
35
|
+
* default Prisma client. That's the right shape for emit paths that have
|
|
36
|
+
* no domain write to coordinate with (e.g., a periodic job that emits a
|
|
37
|
+
* status event); it preserves the at-most-once-insert semantics but
|
|
38
|
+
* without coupling to a domain write.
|
|
39
|
+
*/
|
|
40
|
+
import type { Prisma } from "@prisma/client";
|
|
41
|
+
import type { EventEmit, PublishResult } from "./types";
|
|
42
|
+
/**
|
|
43
|
+
* ID prefix for envelope event_id values per envelope contract §4.1
|
|
44
|
+
* and ADR-0002. `evt_<UUID v7 canonical>` is the canonical shape.
|
|
45
|
+
*/
|
|
46
|
+
export declare const EVENT_ID_PREFIX = "evt_";
|
|
47
|
+
export type PublishOptions = {
|
|
48
|
+
/**
|
|
49
|
+
* Optional Prisma interactive-transaction client. When supplied, the
|
|
50
|
+
* SDK inserts the event row in the same transaction as the caller's
|
|
51
|
+
* domain write, giving the producer-transactional-guarantee shape
|
|
52
|
+
* ADR-0009 §"Producer transactional guarantee" describes. When
|
|
53
|
+
* omitted, the SDK uses the default Prisma client and the insert
|
|
54
|
+
* runs in its own transaction.
|
|
55
|
+
*/
|
|
56
|
+
tx?: Prisma.TransactionClient;
|
|
57
|
+
};
|
|
58
|
+
/**
|
|
59
|
+
* Publish an event to the Postgres-queue transport. Returns the canonical
|
|
60
|
+
* event_id so the producer can correlate retries per envelope contract
|
|
61
|
+
* §10.6 (producers SHALL reuse the same event_id on retry so consumer
|
|
62
|
+
* dedup works).
|
|
63
|
+
*
|
|
64
|
+
* Validation order: registry lookup → schema_version resolution →
|
|
65
|
+
* envelope construction → envelope JSON Schema validation → payload
|
|
66
|
+
* validation → INSERT. A failure at any step throws before the row
|
|
67
|
+
* lands, so the dispatcher_event table never carries unschema'd or
|
|
68
|
+
* unregistered events.
|
|
69
|
+
*/
|
|
70
|
+
export declare function publishToPostgres<TPayload>(emit: EventEmit<TPayload>, options?: PublishOptions): Promise<PublishResult>;
|