@nodii/telemetry 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/lambda/index.d.ts +77 -0
- package/dist/lambda/index.d.ts.map +1 -0
- package/dist/lambda/index.js +130 -0
- package/dist/lambda/index.js.map +1 -0
- package/dist/worker/checkpoint-store.d.ts +66 -0
- package/dist/worker/checkpoint-store.d.ts.map +1 -0
- package/dist/worker/checkpoint-store.js +131 -0
- package/dist/worker/checkpoint-store.js.map +1 -0
- package/dist/worker/dead-letter.d.ts +81 -0
- package/dist/worker/dead-letter.d.ts.map +1 -0
- package/dist/worker/dead-letter.js +165 -0
- package/dist/worker/dead-letter.js.map +1 -0
- package/dist/worker/dedupe-store.d.ts +46 -0
- package/dist/worker/dedupe-store.d.ts.map +1 -0
- package/dist/worker/dedupe-store.js +113 -0
- package/dist/worker/dedupe-store.js.map +1 -0
- package/dist/worker/index.d.ts +23 -0
- package/dist/worker/index.d.ts.map +1 -0
- package/dist/worker/index.js +46 -0
- package/dist/worker/index.js.map +1 -0
- package/dist/worker/observability-hooks.d.ts +99 -0
- package/dist/worker/observability-hooks.d.ts.map +1 -0
- package/dist/worker/observability-hooks.js +189 -0
- package/dist/worker/observability-hooks.js.map +1 -0
- package/dist/worker/outbox-dispatcher.d.ts +106 -0
- package/dist/worker/outbox-dispatcher.d.ts.map +1 -0
- package/dist/worker/outbox-dispatcher.js +285 -0
- package/dist/worker/outbox-dispatcher.js.map +1 -0
- package/dist/worker/replication-worker.d.ts +162 -0
- package/dist/worker/replication-worker.d.ts.map +1 -0
- package/dist/worker/replication-worker.js +513 -0
- package/dist/worker/replication-worker.js.map +1 -0
- package/dist/worker/signal-drain.d.ts +83 -0
- package/dist/worker/signal-drain.d.ts.map +1 -0
- package/dist/worker/signal-drain.js +131 -0
- package/dist/worker/signal-drain.js.map +1 -0
- package/dist/worker/single-active-lease.d.ts +180 -0
- package/dist/worker/single-active-lease.d.ts.map +1 -0
- package/dist/worker/single-active-lease.js +394 -0
- package/dist/worker/single-active-lease.js.map +1 -0
- package/dist/worker/streams-worker.d.ts +192 -0
- package/dist/worker/streams-worker.d.ts.map +1 -0
- package/dist/worker/streams-worker.js +488 -0
- package/dist/worker/streams-worker.js.map +1 -0
- package/dist/worker/sweeper-worker.d.ts +107 -0
- package/dist/worker/sweeper-worker.d.ts.map +1 -0
- package/dist/worker/sweeper-worker.js +245 -0
- package/dist/worker/sweeper-worker.js.map +1 -0
- package/dist/worker/tx.d.ts +39 -0
- package/dist/worker/tx.d.ts.map +1 -0
- package/dist/worker/tx.js +51 -0
- package/dist/worker/tx.js.map +1 -0
- package/dist/worker/worker-handle.d.ts +94 -0
- package/dist/worker/worker-handle.d.ts.map +1 -0
- package/dist/worker/worker-handle.js +15 -0
- package/dist/worker/worker-handle.js.map +1 -0
- package/package.json +10 -1
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
// D402 — DeadLetter: an INJECTED dead-letter sink with a fail-closed PII
|
|
2
|
+
// redaction hook.
|
|
3
|
+
//
|
|
4
|
+
// D402 (LOCKED):
|
|
5
|
+
// "DeadLetter — an injected sink (NEVER a default-on source-side `outbox_dlq`
|
|
6
|
+
// — see D-NEW / D401)."
|
|
7
|
+
//
|
|
8
|
+
// D401 forbids a default-on source-side DLQ table. So this primitive ships:
|
|
9
|
+
// 1. The `DeadLetterSink` INTERFACE — the caller injects the impl.
|
|
10
|
+
// 2. A fail-closed PII redaction wrapper AUTOMATICALLY applied by every worker
|
|
11
|
+
// constructor (via `ensurePiiRedaction`) to every dead item BEFORE it
|
|
12
|
+
// reaches the sink — fail-closed BY DEFAULT (D15: the SAFE state is the
|
|
13
|
+
// default; a consumer cannot leak raw PII by forgetting to wrap). If
|
|
14
|
+
// redaction throws, the payload is dropped to `[REDACTED]`, never raw. A
|
|
15
|
+
// consumer who wraps manually with `withPiiRedaction` is not double-wrapped.
|
|
16
|
+
// 3. A couple of REFERENCE impls the caller OPTS INTO:
|
|
17
|
+
// - `LoudNoopDeadLetterSink` — accepts the item but logs LOUDLY (never a
|
|
18
|
+
// silent drop). Only constructed when EXPLICITLY injected; never
|
|
19
|
+
// auto-wired anywhere.
|
|
20
|
+
// - `TableDeadLetterSink` — persists into a CONSUMER-OWNED table the
|
|
21
|
+
// caller opts into by passing its own `SqlExecutor` + table name. NOT a
|
|
22
|
+
// source-side `outbox_dlq`; it is the subscriber's own quarantine.
|
|
23
|
+
//
|
|
24
|
+
// There is NO default sink. A worker constructed without a DeadLetter has no
|
|
25
|
+
// dead-letter behavior; the caller must inject one to enable it. This is the
|
|
26
|
+
// D401/D402 "never a default-on" rule expressed in the type system.
|
|
27
|
+
import { attemptRedactString, redactFields } from "../redaction.js";
|
|
28
|
+
/** Strict SQL identifier whitelist (mirrors replica-consumer's quoteIdent). */
|
|
29
|
+
const IDENT_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
30
|
+
function quoteIdent(name) {
|
|
31
|
+
if (!IDENT_PATTERN.test(name)) {
|
|
32
|
+
throw new Error(`invalid SQL identifier: ${name}`);
|
|
33
|
+
}
|
|
34
|
+
return `"${name}"`;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Wrap a sink with the FAIL-CLOSED PII redaction hook. Both the item's
|
|
38
|
+
* `payload` AND its `errorMessage` are scrubbed before they reach the wrapped
|
|
39
|
+
* sink:
|
|
40
|
+
* - `payload` runs through the canonical D15 `redactFields` redactor; if it
|
|
41
|
+
* THROWS (e.g. a circular/non-serializable value) the whole payload is
|
|
42
|
+
* collapsed to `{ redaction: "[REDACTED]" }` rather than passed through raw.
|
|
43
|
+
* - `errorMessage` runs through the canonical D15 string redactor
|
|
44
|
+
* (`attemptRedactString`) — real apply errors embed PII routinely (a
|
|
45
|
+
* postgres `Key (email)=(user@example.com) already exists`, an HTTP error
|
|
46
|
+
* carrying a URL+querystring); if the scrub fails it collapses to
|
|
47
|
+
* `[REDACTED]`. Fail-closed, never fail-open, on BOTH fields.
|
|
48
|
+
*
|
|
49
|
+
* Every worker that injects a DeadLetterSink gets this wrapper applied so a
|
|
50
|
+
* dead item can never carry plaintext PII into a quarantine table or a log.
|
|
51
|
+
*/
|
|
52
|
+
/** Brand marking a sink as already PII-redacting (so it is not double-wrapped). */
|
|
53
|
+
const PII_REDACTING = Symbol.for("nodii.worker.piiRedactingSink");
|
|
54
|
+
export function withPiiRedaction(sink, opts = {}) {
|
|
55
|
+
const wrapped = {
|
|
56
|
+
async deadLetter(item) {
|
|
57
|
+
let safePayload;
|
|
58
|
+
try {
|
|
59
|
+
safePayload = redactFields(item.payload ?? {}, {
|
|
60
|
+
extraPaths: opts.extraPaths,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
// Fail CLOSED: redaction failed → drop the payload entirely rather than
|
|
65
|
+
// risk leaking raw PII into the sink.
|
|
66
|
+
safePayload = { redaction: "[REDACTED]" };
|
|
67
|
+
}
|
|
68
|
+
// Scrub the error string too — a raw apply error routinely quotes PII.
|
|
69
|
+
// attemptRedactString is itself fail-closed (returns ok:false on throw);
|
|
70
|
+
// on a non-ok result we drop the whole string to [REDACTED].
|
|
71
|
+
const scrubbed = attemptRedactString(item.errorMessage ?? "", {
|
|
72
|
+
extraPaths: opts.extraPaths,
|
|
73
|
+
});
|
|
74
|
+
const safeErrorMessage = scrubbed.ok ? scrubbed.redacted : "[REDACTED]";
|
|
75
|
+
await sink.deadLetter({
|
|
76
|
+
...item,
|
|
77
|
+
payload: safePayload,
|
|
78
|
+
errorMessage: safeErrorMessage,
|
|
79
|
+
});
|
|
80
|
+
},
|
|
81
|
+
};
|
|
82
|
+
wrapped[PII_REDACTING] = true;
|
|
83
|
+
return wrapped;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* The FAIL-CLOSED-BY-DEFAULT wrapper every worker applies to its injected
|
|
87
|
+
* DeadLetterSink: redaction runs on every dead item WITHOUT the consumer having
|
|
88
|
+
* to remember to wrap (D15: the SAFE state must be the default). An
|
|
89
|
+
* already-wrapped sink is returned as-is (no double-wrap). Used internally by the
|
|
90
|
+
* worker constructors.
|
|
91
|
+
*/
|
|
92
|
+
export function ensurePiiRedaction(sink) {
|
|
93
|
+
if (sink[PII_REDACTING])
|
|
94
|
+
return sink;
|
|
95
|
+
return withPiiRedaction(sink);
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Reference sink #1: accepts the dead item but logs it LOUDLY (error-level,
|
|
99
|
+
* with the failure cause + identity, never the raw payload — that's already
|
|
100
|
+
* redacted by `withPiiRedaction`). It does NOT persist. Useful as an explicit
|
|
101
|
+
* "I have NO durable DLQ yet but I refuse to silently drop" choice. NEVER
|
|
102
|
+
* auto-wired — the caller must construct + inject it.
|
|
103
|
+
*/
|
|
104
|
+
export class LoudNoopDeadLetterSink {
|
|
105
|
+
logger;
|
|
106
|
+
constructor(logger) {
|
|
107
|
+
this.logger = logger;
|
|
108
|
+
}
|
|
109
|
+
async deadLetter(item) {
|
|
110
|
+
this.logger.error("worker.dead_letter (no durable sink wired)", {
|
|
111
|
+
source_stream: item.sourceStream,
|
|
112
|
+
topic: item.topic,
|
|
113
|
+
event_id: item.eventId,
|
|
114
|
+
error_message: item.errorMessage,
|
|
115
|
+
retry_count: item.retryCount,
|
|
116
|
+
// payload is redacted by withPiiRedaction before reaching here.
|
|
117
|
+
payload: item.payload,
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Reference sink #2: persists the dead item into a CONSUMER-OWNED quarantine
|
|
123
|
+
* table the caller opts into (passing its OWN transaction/pool executor + table
|
|
124
|
+
* name). This is the SUBSCRIBER's own DLQ — NOT a source-side `outbox_dlq`
|
|
125
|
+
* (which D401 forbids). The caller controls the executor, so the write joins
|
|
126
|
+
* whatever transaction the caller chooses (or runs standalone on a pool conn).
|
|
127
|
+
*/
|
|
128
|
+
export class TableDeadLetterSink {
|
|
129
|
+
table;
|
|
130
|
+
insertSql;
|
|
131
|
+
exec;
|
|
132
|
+
constructor(opts) {
|
|
133
|
+
this.exec = opts.executor;
|
|
134
|
+
this.table = opts.table ?? "worker_dead_letter";
|
|
135
|
+
const t = quoteIdent(this.table);
|
|
136
|
+
this.insertSql = `INSERT INTO ${t}
|
|
137
|
+
(source_stream, topic, event_id, payload, error_message, retry_count, dead_lettered_at)
|
|
138
|
+
VALUES ($1, $2, $3, $4::jsonb, $5, $6, now())`;
|
|
139
|
+
}
|
|
140
|
+
/** Idempotent DDL for the consumer-owned quarantine table. */
|
|
141
|
+
static tableDDL(table = "worker_dead_letter") {
|
|
142
|
+
const t = quoteIdent(table);
|
|
143
|
+
return `CREATE TABLE IF NOT EXISTS ${t} (
|
|
144
|
+
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
145
|
+
source_stream TEXT NOT NULL,
|
|
146
|
+
topic TEXT NOT NULL,
|
|
147
|
+
event_id TEXT NOT NULL,
|
|
148
|
+
payload JSONB NOT NULL,
|
|
149
|
+
error_message TEXT NOT NULL,
|
|
150
|
+
retry_count INTEGER NOT NULL DEFAULT 0,
|
|
151
|
+
dead_lettered_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
152
|
+
)`;
|
|
153
|
+
}
|
|
154
|
+
async deadLetter(item) {
|
|
155
|
+
await this.exec.query(this.insertSql, [
|
|
156
|
+
item.sourceStream,
|
|
157
|
+
item.topic,
|
|
158
|
+
item.eventId,
|
|
159
|
+
JSON.stringify(item.payload ?? {}),
|
|
160
|
+
item.errorMessage,
|
|
161
|
+
item.retryCount,
|
|
162
|
+
]);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
//# sourceMappingURL=dead-letter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dead-letter.js","sourceRoot":"","sources":["../../src/worker/dead-letter.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,kBAAkB;AAClB,EAAE;AACF,iBAAiB;AACjB,gFAAgF;AAChF,2BAA2B;AAC3B,EAAE;AACF,4EAA4E;AAC5E,qEAAqE;AACrE,iFAAiF;AACjF,2EAA2E;AAC3E,6EAA6E;AAC7E,0EAA0E;AAC1E,8EAA8E;AAC9E,kFAAkF;AAClF,yDAAyD;AACzD,gFAAgF;AAChF,0EAA0E;AAC1E,gCAAgC;AAChC,4EAA4E;AAC5E,iFAAiF;AACjF,4EAA4E;AAC5E,EAAE;AACF,6EAA6E;AAC7E,6EAA6E;AAC7E,oEAAoE;AAEpE,OAAO,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAGpE,+EAA+E;AAC/E,MAAM,aAAa,GAAG,0BAA0B,CAAC;AAEjD,SAAS,UAAU,CAAC,IAAY;IAC9B,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,IAAI,IAAI,GAAG,CAAC;AACrB,CAAC;AA0CD;;;;;;;;;;;;;;;GAeG;AACH,mFAAmF;AACnF,MAAM,aAAa,GAAG,MAAM,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;AAElE,MAAM,UAAU,gBAAgB,CAC9B,IAAoB,EACpB,OAA2C,EAAE;IAE7C,MAAM,OAAO,GAAgD;QAC3D,KAAK,CAAC,UAAU,CAAC,IAAoB;YACnC,IAAI,WAAoC,CAAC;YACzC,IAAI,CAAC;gBACH,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,EAAE;oBAC7C,UAAU,EAAE,IAAI,CAAC,UAAU;iBAC5B,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,wEAAwE;gBACxE,sCAAsC;gBACtC,WAAW,GAAG,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC;YAC5C,CAAC;YACD,uEAAuE;YACvE,yEAAyE;YACzE,6DAA6D;YAC7D,MAAM,QAAQ,GAAG,mBAAmB,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,EAAE;gBAC5D,UAAU,EAAE,IAAI,CAAC,UAAU;aAC5B,CAAC,CAAC;YACH,MAAM,gBAAgB,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC;YACxE,MAAM,IAAI,CAAC,UAAU,CAAC;gBACpB,GAAG,IAAI;gBACP,OAAO,EAAE,WAAW;gBACpB,YAAY,EAAE,gBAAgB;aAC/B,CAAC,CAAC;QACL,CAAC;KACF,CAAC;IACF,OAAO,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC;IAC9B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAoB;IACrD,IAAK,IAAmC,CAAC,aAAa,CAAC;QAAE,OAAO,IAAI,CAAC;IACrE,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC;AAChC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,OAAO,sBAAsB;IACJ;IAA7B,YAA6B,MAAwB;QAAxB,WAAM,GAAN,MAAM,CAAkB;IAAG,CAAC;IAEzD,KAAK,CAAC,UAAU,CAAC,IAAoB;QACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4CAA4C,EAAE;YAC9D,aAAa,EAAE,IAAI,CAAC,YAAY;YAChC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE,IAAI,CAAC,OAAO;YACtB,aAAa,EAAE,IAAI,CAAC,YAAY;YAChC,WAAW,EAAE,IAAI,CAAC,UAAU;YAC5B,gEAAgE;YAChE,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;IACL,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,OAAO,mBAAmB;IACrB,KAAK,CAAS;IACN,SAAS,CAAS;IAClB,IAAI,CAAc;IAEnC,YAAY,IAA+C;QACzD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,oBAAoB,CAAC;QAChD,MAAM,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,eAAe,CAAC;;8CAES,CAAC;IAC7C,CAAC;IAED,8DAA8D;IAC9D,MAAM,CAAC,QAAQ,CAAC,KAAK,GAAG,oBAAoB;QAC1C,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC5B,OAAO,8BAA8B,CAAC;;;;;;;;;EASxC,CAAC;IACD,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAoB;QACnC,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE;YACpC,IAAI,CAAC,YAAY;YACjB,IAAI,CAAC,KAAK;YACV,IAAI,CAAC,OAAO;YACZ,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;YAClC,IAAI,CAAC,YAAY;YACjB,IAAI,CAAC,UAAU;SAChB,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { SqlExecutor } from "./tx.js";
|
|
2
|
+
/** The canonical un-prefixed dedupe table (per-service DB). */
|
|
3
|
+
export declare const DEFAULT_CONSUMED_EVENTS_TABLE = "consumed_events";
|
|
4
|
+
/** Default retention horizon for the sweep — 30d per 09-replication § 6. */
|
|
5
|
+
export declare const DEFAULT_RETENTION_DAYS = 30;
|
|
6
|
+
/** Idempotent DDL for the composite-PK dedupe table (D198). */
|
|
7
|
+
export declare function consumedEventsTableDDL(table?: string): string;
|
|
8
|
+
/**
|
|
9
|
+
* Dedupe over `consumed_events (event_id, topic)` (D198 composite PK). One
|
|
10
|
+
* instance serves a whole service DB (the table is per-service). All writes
|
|
11
|
+
* join the caller's apply transaction via the injected `tx`.
|
|
12
|
+
*/
|
|
13
|
+
export declare class DedupeStore {
|
|
14
|
+
readonly table: string;
|
|
15
|
+
private readonly selectSql;
|
|
16
|
+
private readonly insertSql;
|
|
17
|
+
private readonly sweepSql;
|
|
18
|
+
constructor(opts?: {
|
|
19
|
+
table?: string;
|
|
20
|
+
});
|
|
21
|
+
/**
|
|
22
|
+
* Has `(event_id, topic)` already been consumed? Call at step [1] of consume
|
|
23
|
+
* to skip an already-applied event before any work. Pass the apply `tx` (so
|
|
24
|
+
* the probe sees the tx's own uncommitted inserts) or a pooled connection for
|
|
25
|
+
* a cheap pre-tx probe.
|
|
26
|
+
*/
|
|
27
|
+
seen(eventId: string, topic: string, tx: SqlExecutor): Promise<boolean>;
|
|
28
|
+
/**
|
|
29
|
+
* Record `(event_id, topic)` as consumed INSIDE the caller's apply tx — the
|
|
30
|
+
* SAME `tx` the projection upsert + `checkpoint.advance(...)` run on, so the
|
|
31
|
+
* dedupe row commits together-or-none with them.
|
|
32
|
+
*
|
|
33
|
+
* @returns `true` if this call inserted the row (first time consumed), `false`
|
|
34
|
+
* if it was already present (ON CONFLICT DO NOTHING) — lets the
|
|
35
|
+
* caller detect a concurrent double-consume.
|
|
36
|
+
*/
|
|
37
|
+
record(eventId: string, topic: string, tx: SqlExecutor): Promise<boolean>;
|
|
38
|
+
/**
|
|
39
|
+
* Retention sweep — DELETE dedupe rows older than `retentionDays` (default
|
|
40
|
+
* 30d per 09-replication § 6). Run on a SweeperWorker tick. Returns the number
|
|
41
|
+
* of rows removed (surface as `nodii.worker.swept_total`). `tx` may be a
|
|
42
|
+
* pooled connection — the sweep is independent of any apply transaction.
|
|
43
|
+
*/
|
|
44
|
+
sweep(tx: SqlExecutor, retentionDays?: number): Promise<number>;
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=dedupe-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dedupe-store.d.ts","sourceRoot":"","sources":["../../src/worker/dedupe-store.ts"],"names":[],"mappings":"AA+BA,OAAO,KAAK,EAAE,WAAW,EAAU,MAAM,SAAS,CAAC;AAYnD,+DAA+D;AAC/D,eAAO,MAAM,6BAA6B,oBAAoB,CAAC;AAE/D,4EAA4E;AAC5E,eAAO,MAAM,sBAAsB,KAAK,CAAC;AAEzC,+DAA+D;AAC/D,wBAAgB,sBAAsB,CACpC,KAAK,SAAgC,GACpC,MAAM,CAQR;AAMD;;;;GAIG;AACH,qBAAa,WAAW;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;gBAEtB,IAAI,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAO;IAczC;;;;;OAKG;IACG,IAAI,CACR,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,EAAE,EAAE,WAAW,GACd,OAAO,CAAC,OAAO,CAAC;IAKnB;;;;;;;;OAQG;IACG,MAAM,CACV,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,EAAE,EAAE,WAAW,GACd,OAAO,CAAC,OAAO,CAAC;IAKnB;;;;;OAKG;IACG,KAAK,CACT,EAAE,EAAE,WAAW,EACf,aAAa,GAAE,MAA+B,GAC7C,OAAO,CAAC,MAAM,CAAC;CASnB"}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
// D402 — DedupeStore: the at-most-once dedupe primitive over the
|
|
2
|
+
// `consumed_events (event_id, topic)` composite-PK table.
|
|
3
|
+
//
|
|
4
|
+
// D402 (LOCKED):
|
|
5
|
+
// "DedupeStore — `consumed_events (event_id, topic)` composite-PK dedupe +
|
|
6
|
+
// retention sweep (default 30d per 09-replication-doctrine § 6)."
|
|
7
|
+
//
|
|
8
|
+
// D198 (LOCKED 2026-06-03) pins the COMPOSITE pk: the same event_id can
|
|
9
|
+
// legitimately recur across topics, so an event_id-only dedupe would wrongly
|
|
10
|
+
// short-circuit the second topic. Keying on `(event_id, topic)` is the
|
|
11
|
+
// cross-topic-correctness fix (see ts/replica-consumer/src/sql-builders.ts
|
|
12
|
+
// DEDUPE_SELECT_SQL / CONSUMED_EVENTS_INSERT_SQL — generalized here, NOT
|
|
13
|
+
// duplicated; same table, same column shape).
|
|
14
|
+
//
|
|
15
|
+
// The `consumed_events` table is intentionally UN-prefixed: each adopting
|
|
16
|
+
// service owns its own database, so the table is already per-service (operator-
|
|
17
|
+
// confirmed; see memory feedback_consumed_events_table).
|
|
18
|
+
//
|
|
19
|
+
// ── Atomic-in-apply-tx (the load-bearing property) ───────────────────────────
|
|
20
|
+
// `record(...)` runs on the CALLER's `tx` handle, the SAME one the projection
|
|
21
|
+
// upsert + `checkpoint.advance(...)` run on. The dedupe row is therefore
|
|
22
|
+
// committed together-or-none with the projection + the cursor: if apply fails
|
|
23
|
+
// after `record(...)`, the rollback removes the dedupe row too, so the event
|
|
24
|
+
// is NOT falsely marked consumed (a retry re-processes it). And because the
|
|
25
|
+
// cursor advance is in the same tx, a committed dedupe row always has a
|
|
26
|
+
// committed cursor that covers it.
|
|
27
|
+
//
|
|
28
|
+
// `seen(...)` is the read-side probe at step [1] of the consume sequence:
|
|
29
|
+
// call it (on the tx, or a pooled conn pre-tx) to skip an already-applied
|
|
30
|
+
// (event_id, topic) before doing any work.
|
|
31
|
+
/** Strict SQL identifier whitelist (mirrors replica-consumer's quoteIdent). */
|
|
32
|
+
const IDENT_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
33
|
+
function quoteIdent(name) {
|
|
34
|
+
if (!IDENT_PATTERN.test(name)) {
|
|
35
|
+
throw new Error(`invalid SQL identifier: ${name}`);
|
|
36
|
+
}
|
|
37
|
+
return `"${name}"`;
|
|
38
|
+
}
|
|
39
|
+
/** The canonical un-prefixed dedupe table (per-service DB). */
|
|
40
|
+
export const DEFAULT_CONSUMED_EVENTS_TABLE = "consumed_events";
|
|
41
|
+
/** Default retention horizon for the sweep — 30d per 09-replication § 6. */
|
|
42
|
+
export const DEFAULT_RETENTION_DAYS = 30;
|
|
43
|
+
/** Idempotent DDL for the composite-PK dedupe table (D198). */
|
|
44
|
+
export function consumedEventsTableDDL(table = DEFAULT_CONSUMED_EVENTS_TABLE) {
|
|
45
|
+
const t = quoteIdent(table);
|
|
46
|
+
return `CREATE TABLE IF NOT EXISTS ${t} (
|
|
47
|
+
event_id TEXT NOT NULL,
|
|
48
|
+
topic TEXT NOT NULL,
|
|
49
|
+
consumed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
50
|
+
PRIMARY KEY (event_id, topic)
|
|
51
|
+
)`;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Dedupe over `consumed_events (event_id, topic)` (D198 composite PK). One
|
|
55
|
+
* instance serves a whole service DB (the table is per-service). All writes
|
|
56
|
+
* join the caller's apply transaction via the injected `tx`.
|
|
57
|
+
*/
|
|
58
|
+
export class DedupeStore {
|
|
59
|
+
table;
|
|
60
|
+
selectSql;
|
|
61
|
+
insertSql;
|
|
62
|
+
sweepSql;
|
|
63
|
+
constructor(opts = {}) {
|
|
64
|
+
this.table = opts.table ?? DEFAULT_CONSUMED_EVENTS_TABLE;
|
|
65
|
+
const t = quoteIdent(this.table);
|
|
66
|
+
this.selectSql = `SELECT 1 AS one FROM ${t} WHERE event_id = $1 AND topic = $2 LIMIT 1`;
|
|
67
|
+
// ON CONFLICT DO NOTHING: a concurrent / replayed insert is a benign no-op
|
|
68
|
+
// (the row already proves consumption); rowCount distinguishes first-insert
|
|
69
|
+
// (1) from already-seen (0).
|
|
70
|
+
this.insertSql = `INSERT INTO ${t} (event_id, topic, consumed_at) VALUES ($1, $2, now())
|
|
71
|
+
ON CONFLICT (event_id, topic) DO NOTHING`;
|
|
72
|
+
// Retention sweep — parameterized day horizon, cast to an interval. Mirrors
|
|
73
|
+
// replica-consumer's SWEEP_CONSUMED_EVENTS_SQL.
|
|
74
|
+
this.sweepSql = `DELETE FROM ${t} WHERE consumed_at < now() - ($1::int || ' days')::interval`;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Has `(event_id, topic)` already been consumed? Call at step [1] of consume
|
|
78
|
+
* to skip an already-applied event before any work. Pass the apply `tx` (so
|
|
79
|
+
* the probe sees the tx's own uncommitted inserts) or a pooled connection for
|
|
80
|
+
* a cheap pre-tx probe.
|
|
81
|
+
*/
|
|
82
|
+
async seen(eventId, topic, tx) {
|
|
83
|
+
const res = await tx.query(this.selectSql, [eventId, topic]);
|
|
84
|
+
return res.rows.length > 0;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Record `(event_id, topic)` as consumed INSIDE the caller's apply tx — the
|
|
88
|
+
* SAME `tx` the projection upsert + `checkpoint.advance(...)` run on, so the
|
|
89
|
+
* dedupe row commits together-or-none with them.
|
|
90
|
+
*
|
|
91
|
+
* @returns `true` if this call inserted the row (first time consumed), `false`
|
|
92
|
+
* if it was already present (ON CONFLICT DO NOTHING) — lets the
|
|
93
|
+
* caller detect a concurrent double-consume.
|
|
94
|
+
*/
|
|
95
|
+
async record(eventId, topic, tx) {
|
|
96
|
+
const res = await tx.query(this.insertSql, [eventId, topic]);
|
|
97
|
+
return res.rowCount > 0;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Retention sweep — DELETE dedupe rows older than `retentionDays` (default
|
|
101
|
+
* 30d per 09-replication § 6). Run on a SweeperWorker tick. Returns the number
|
|
102
|
+
* of rows removed (surface as `nodii.worker.swept_total`). `tx` may be a
|
|
103
|
+
* pooled connection — the sweep is independent of any apply transaction.
|
|
104
|
+
*/
|
|
105
|
+
async sweep(tx, retentionDays = DEFAULT_RETENTION_DAYS) {
|
|
106
|
+
if (!Number.isInteger(retentionDays) || retentionDays < 0) {
|
|
107
|
+
throw new Error(`retentionDays must be a non-negative integer, got ${retentionDays}`);
|
|
108
|
+
}
|
|
109
|
+
const res = await tx.query(this.sweepSql, [retentionDays]);
|
|
110
|
+
return res.rowCount;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
//# sourceMappingURL=dedupe-store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dedupe-store.js","sourceRoot":"","sources":["../../src/worker/dedupe-store.ts"],"names":[],"mappings":"AAAA,iEAAiE;AACjE,0DAA0D;AAC1D,EAAE;AACF,iBAAiB;AACjB,6EAA6E;AAC7E,qEAAqE;AACrE,EAAE;AACF,wEAAwE;AACxE,6EAA6E;AAC7E,uEAAuE;AACvE,2EAA2E;AAC3E,yEAAyE;AACzE,8CAA8C;AAC9C,EAAE;AACF,0EAA0E;AAC1E,gFAAgF;AAChF,yDAAyD;AACzD,EAAE;AACF,gFAAgF;AAChF,gFAAgF;AAChF,2EAA2E;AAC3E,gFAAgF;AAChF,+EAA+E;AAC/E,8EAA8E;AAC9E,0EAA0E;AAC1E,qCAAqC;AACrC,EAAE;AACF,4EAA4E;AAC5E,4EAA4E;AAC5E,6CAA6C;AAI7C,+EAA+E;AAC/E,MAAM,aAAa,GAAG,0BAA0B,CAAC;AAEjD,SAAS,UAAU,CAAC,IAAY;IAC9B,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,IAAI,IAAI,GAAG,CAAC;AACrB,CAAC;AAED,+DAA+D;AAC/D,MAAM,CAAC,MAAM,6BAA6B,GAAG,iBAAiB,CAAC;AAE/D,4EAA4E;AAC5E,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAEzC,+DAA+D;AAC/D,MAAM,UAAU,sBAAsB,CACpC,KAAK,GAAG,6BAA6B;IAErC,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC5B,OAAO,8BAA8B,CAAC;;;;;EAKtC,CAAC;AACH,CAAC;AAMD;;;;GAIG;AACH,MAAM,OAAO,WAAW;IACb,KAAK,CAAS;IACN,SAAS,CAAS;IAClB,SAAS,CAAS;IAClB,QAAQ,CAAS;IAElC,YAAY,OAA2B,EAAE;QACvC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,6BAA6B,CAAC;QACzD,MAAM,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,wBAAwB,CAAC,6CAA6C,CAAC;QACxF,2EAA2E;QAC3E,4EAA4E;QAC5E,6BAA6B;QAC7B,IAAI,CAAC,SAAS,GAAG,eAAe,CAAC;yCACI,CAAC;QACtC,4EAA4E;QAC5E,gDAAgD;QAChD,IAAI,CAAC,QAAQ,GAAG,eAAe,CAAC,6DAA6D,CAAC;IAChG,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,IAAI,CACR,OAAe,EACf,KAAa,EACb,EAAe;QAEf,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,KAAK,CAAU,IAAI,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QACtE,OAAO,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,MAAM,CACV,OAAe,EACf,KAAa,EACb,EAAe;QAEf,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QAC7D,OAAO,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK,CACT,EAAe,EACf,gBAAwB,sBAAsB;QAE9C,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;YAC1D,MAAM,IAAI,KAAK,CACb,qDAAqD,aAAa,EAAE,CACrE,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC;QAC3D,OAAO,GAAG,CAAC,QAAQ,CAAC;IACtB,CAAC;CACF"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export { encodeLeaseValue, FenceCheckUnavailableError, FenceLostError, fenceKey, leaseKey, parseLeaseValue, SingleActiveLease, } from "./single-active-lease.js";
|
|
2
|
+
export type { LeaseRedisLike, LeaseTelemetry, SingleActiveLeaseOpts, } from "./single-active-lease.js";
|
|
3
|
+
export { postgresJsExecutor } from "./tx.js";
|
|
4
|
+
export type { SqlExecutor, SqlResult, SqlRow } from "./tx.js";
|
|
5
|
+
export { CheckpointStore, checkpointTableDDL, DEFAULT_CHECKPOINT_TABLE, } from "./checkpoint-store.js";
|
|
6
|
+
export type { Checkpoint, CheckpointAdvance } from "./checkpoint-store.js";
|
|
7
|
+
export { consumedEventsTableDDL, DedupeStore, DEFAULT_CONSUMED_EVENTS_TABLE, DEFAULT_RETENTION_DAYS, } from "./dedupe-store.js";
|
|
8
|
+
export { ensurePiiRedaction, LoudNoopDeadLetterSink, TableDeadLetterSink, withPiiRedaction, } from "./dead-letter.js";
|
|
9
|
+
export type { DeadLetterItem, DeadLetterLogger, DeadLetterSink, } from "./dead-letter.js";
|
|
10
|
+
export { hwmGaugeValue, ObservabilityHooks, WORKER_METRIC_NAMES, WORKER_SUBSTRATE_VOCABULARY, } from "./observability-hooks.js";
|
|
11
|
+
export type { ObservabilityHooksOpts, RecordMetricFn, WorkerMetricAttributes, } from "./observability-hooks.js";
|
|
12
|
+
export { SignalDrain } from "./signal-drain.js";
|
|
13
|
+
export type { DrainResult, SignalDrainOpts, StopHandler, } from "./signal-drain.js";
|
|
14
|
+
export type { WorkerHandle, WorkerHwm, WorkerStopOptions, WorkerStopResult, } from "./worker-handle.js";
|
|
15
|
+
export { SweeperWorker } from "./sweeper-worker.js";
|
|
16
|
+
export type { SweeperJobContext, SweeperJobDefinition, SweeperJobResult, SweeperWorkerOpts, } from "./sweeper-worker.js";
|
|
17
|
+
export { parseAutoClaimEntries, parseStreamEntries, StreamsWorker, streamsConsumerGroup, } from "./streams-worker.js";
|
|
18
|
+
export type { StreamApplyContext, StreamApplyHandler, StreamEvent, StreamsRedisLike, StreamsWorkerOpts, } from "./streams-worker.js";
|
|
19
|
+
export { OutboxDispatcher } from "./outbox-dispatcher.js";
|
|
20
|
+
export type { OutboxDispatcherOpts, OutboxPublisher, OutboxRow, } from "./outbox-dispatcher.js";
|
|
21
|
+
export { ReplicationWorker, replicationConsumerGroup, } from "./replication-worker.js";
|
|
22
|
+
export type { ReplicationApplyContext, ReplicationApplyHandler, ReplicationAuditHook, ReplicationEvent, ReplicationWorkerOpts, SnapshotBootstrap, } from "./replication-worker.js";
|
|
23
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/worker/index.ts"],"names":[],"mappings":"AAuBA,OAAO,EACL,gBAAgB,EAChB,0BAA0B,EAC1B,cAAc,EACd,QAAQ,EACR,QAAQ,EACR,eAAe,EACf,iBAAiB,GAClB,MAAM,0BAA0B,CAAC;AAClC,YAAY,EACV,cAAc,EACd,cAAc,EACd,qBAAqB,GACtB,MAAM,0BAA0B,CAAC;AAGlC,OAAO,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAG9D,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,wBAAwB,GACzB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAG3E,OAAO,EACL,sBAAsB,EACtB,WAAW,EACX,6BAA6B,EAC7B,sBAAsB,GACvB,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,cAAc,EACd,gBAAgB,EAChB,cAAc,GACf,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,mBAAmB,EACnB,2BAA2B,GAC5B,MAAM,0BAA0B,CAAC;AAClC,YAAY,EACV,sBAAsB,EACtB,cAAc,EACd,sBAAsB,GACvB,MAAM,0BAA0B,CAAC;AAGlC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EACV,WAAW,EACX,eAAe,EACf,WAAW,GACZ,MAAM,mBAAmB,CAAC;AAG3B,YAAY,EACV,YAAY,EACZ,SAAS,EACT,iBAAiB,EACjB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAM5B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,YAAY,EACV,iBAAiB,EACjB,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EACL,qBAAqB,EACrB,kBAAkB,EAClB,aAAa,EACb,oBAAoB,GACrB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,kBAAkB,EAClB,kBAAkB,EAClB,WAAW,EACX,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,YAAY,EACV,oBAAoB,EACpB,eAAe,EACf,SAAS,GACV,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EACL,iBAAiB,EACjB,wBAAwB,GACzB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,uBAAuB,EACvB,uBAAuB,EACvB,oBAAoB,EACpB,gBAAgB,EAChB,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,yBAAyB,CAAC"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// @nodii/telemetry/worker — the canonical background-worker substrate (D402).
|
|
2
|
+
//
|
|
3
|
+
// Every background worker on every Nodii service — stream consumer, sweeper,
|
|
4
|
+
// single-active leased loop, outbox drainer — is built from these canonical
|
|
5
|
+
// primitives. No service hand-rolls a lease, a checkpoint cursor, a dedupe
|
|
6
|
+
// store, or a worker lifecycle.
|
|
7
|
+
//
|
|
8
|
+
// D402 (LOCKED) primitives shipped here:
|
|
9
|
+
// - SingleActiveLease — Redis SET-NX-EX + Lua CAS-renew@ttl/2 + fencing token
|
|
10
|
+
// - CheckpointStore — durable HWM cursor, advanced atomically in the apply tx
|
|
11
|
+
// - DedupeStore — consumed_events (event_id, topic) composite-PK + sweep
|
|
12
|
+
// - DeadLetter — injected sink + fail-closed PII redaction (D401)
|
|
13
|
+
// - ObservabilityHooks — canonical worker metrics via the real recordMetric
|
|
14
|
+
// - SignalDrain — SIGTERM/SIGINT bounded graceful drain
|
|
15
|
+
// - WorkerHandle — the uniform handle the 4 worker types implement
|
|
16
|
+
//
|
|
17
|
+
// Resolve as a downstream consumer would:
|
|
18
|
+
// import {
|
|
19
|
+
// SingleActiveLease, CheckpointStore, DedupeStore,
|
|
20
|
+
// ObservabilityHooks, SignalDrain, type WorkerHandle,
|
|
21
|
+
// } from "@nodii/telemetry/worker";
|
|
22
|
+
// ── SingleActiveLease (already shipped) ──────────────────────────────────────
|
|
23
|
+
export { encodeLeaseValue, FenceCheckUnavailableError, FenceLostError, fenceKey, leaseKey, parseLeaseValue, SingleActiveLease, } from "./single-active-lease.js";
|
|
24
|
+
// ── Shared SQL-executor seam (the atomic-in-apply-tx handle) ─────────────────
|
|
25
|
+
export { postgresJsExecutor } from "./tx.js";
|
|
26
|
+
// ── CheckpointStore (HWM cursor) ─────────────────────────────────────────────
|
|
27
|
+
export { CheckpointStore, checkpointTableDDL, DEFAULT_CHECKPOINT_TABLE, } from "./checkpoint-store.js";
|
|
28
|
+
// ── DedupeStore ──────────────────────────────────────────────────────────────
|
|
29
|
+
export { consumedEventsTableDDL, DedupeStore, DEFAULT_CONSUMED_EVENTS_TABLE, DEFAULT_RETENTION_DAYS, } from "./dedupe-store.js";
|
|
30
|
+
// ── DeadLetter (injected sink + fail-closed PII redaction) ───────────────────
|
|
31
|
+
export { ensurePiiRedaction, LoudNoopDeadLetterSink, TableDeadLetterSink, withPiiRedaction, } from "./dead-letter.js";
|
|
32
|
+
// ── ObservabilityHooks ───────────────────────────────────────────────────────
|
|
33
|
+
export { hwmGaugeValue, ObservabilityHooks, WORKER_METRIC_NAMES, WORKER_SUBSTRATE_VOCABULARY, } from "./observability-hooks.js";
|
|
34
|
+
// ── SignalDrain ──────────────────────────────────────────────────────────────
|
|
35
|
+
export { SignalDrain } from "./signal-drain.js";
|
|
36
|
+
// ── Worker types (compose the primitives) ────────────────────────────────────
|
|
37
|
+
// The 4 D402 worker types, each implementing WorkerHandle.
|
|
38
|
+
// SweeperWorker — single-active, checkpoint-resumable interval job.
|
|
39
|
+
export { SweeperWorker } from "./sweeper-worker.js";
|
|
40
|
+
// StreamsWorker — generic (non-replication) Pattern-2 stream consumer.
|
|
41
|
+
export { parseAutoClaimEntries, parseStreamEntries, StreamsWorker, streamsConsumerGroup, } from "./streams-worker.js";
|
|
42
|
+
// OutboxDispatcher — Pattern-3 producer drain.
|
|
43
|
+
export { OutboxDispatcher } from "./outbox-dispatcher.js";
|
|
44
|
+
// ReplicationWorker — Pattern-3 subscriber consume (snapshot + LWW + audit).
|
|
45
|
+
export { ReplicationWorker, replicationConsumerGroup, } from "./replication-worker.js";
|
|
46
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/worker/index.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,EAAE;AACF,6EAA6E;AAC7E,4EAA4E;AAC5E,2EAA2E;AAC3E,gCAAgC;AAChC,EAAE;AACF,yCAAyC;AACzC,kFAAkF;AAClF,oFAAoF;AACpF,mFAAmF;AACnF,6EAA6E;AAC7E,+EAA+E;AAC/E,kEAAkE;AAClE,4EAA4E;AAC5E,EAAE;AACF,0CAA0C;AAC1C,aAAa;AACb,uDAAuD;AACvD,0DAA0D;AAC1D,sCAAsC;AAEtC,gFAAgF;AAChF,OAAO,EACL,gBAAgB,EAChB,0BAA0B,EAC1B,cAAc,EACd,QAAQ,EACR,QAAQ,EACR,eAAe,EACf,iBAAiB,GAClB,MAAM,0BAA0B,CAAC;AAOlC,gFAAgF;AAChF,OAAO,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAG7C,gFAAgF;AAChF,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,wBAAwB,GACzB,MAAM,uBAAuB,CAAC;AAG/B,gFAAgF;AAChF,OAAO,EACL,sBAAsB,EACtB,WAAW,EACX,6BAA6B,EAC7B,sBAAsB,GACvB,MAAM,mBAAmB,CAAC;AAE3B,gFAAgF;AAChF,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAO1B,gFAAgF;AAChF,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,mBAAmB,EACnB,2BAA2B,GAC5B,MAAM,0BAA0B,CAAC;AAOlC,gFAAgF;AAChF,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAehD,gFAAgF;AAChF,2DAA2D;AAE3D,oEAAoE;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAQpD,uEAAuE;AACvE,OAAO,EACL,qBAAqB,EACrB,kBAAkB,EAClB,aAAa,EACb,oBAAoB,GACrB,MAAM,qBAAqB,CAAC;AAS7B,+CAA+C;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAO1D,6EAA6E;AAC7E,OAAO,EACL,iBAAiB,EACjB,wBAAwB,GACzB,MAAM,yBAAyB,CAAC"}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { type RecordMetricArgs } from "../metrics.js";
|
|
2
|
+
import type { MetricVocabularyRegistry } from "../types.js";
|
|
3
|
+
/** Canonical worker-substrate metric names, pinned by D402 + the task. */
|
|
4
|
+
export declare const WORKER_METRIC_NAMES: {
|
|
5
|
+
readonly leaseAcquired: "nodii.worker.lease.acquired_total";
|
|
6
|
+
readonly leaseRenewed: "nodii.worker.lease.renewed_total";
|
|
7
|
+
readonly leaseLost: "nodii.worker.lease.lost_total";
|
|
8
|
+
readonly lag: "nodii.worker.lag";
|
|
9
|
+
readonly currentHwm: "nodii.worker.current_hwm";
|
|
10
|
+
readonly depth: "nodii.worker.depth";
|
|
11
|
+
readonly drainDurationMs: "nodii.worker.drain.duration_ms";
|
|
12
|
+
readonly swept: "nodii.worker.swept_total";
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* The locked worker-substrate metric vocabulary — merge into
|
|
16
|
+
* `initTelemetry({ vocabularyRegistry: { ...svc, ...WORKER_SUBSTRATE_VOCABULARY } })`
|
|
17
|
+
* so the canonical names register under family `async_worker` instead of
|
|
18
|
+
* tripping `MetricVocabularyError`. Mirrors how `REPLICA_CONSUMER_VOCABULARY`
|
|
19
|
+
* is consumed.
|
|
20
|
+
*/
|
|
21
|
+
export declare const WORKER_SUBSTRATE_VOCABULARY: MetricVocabularyRegistry;
|
|
22
|
+
/**
|
|
23
|
+
* Base attributes every worker-substrate metric carries so a service can slice
|
|
24
|
+
* the canonical series by worker. `service` + `scope` mirror the lease's
|
|
25
|
+
* `(service, scope)` key shape; `worker` is the worker-type label
|
|
26
|
+
* (`replication` | `sweeper` | `streams` | `outbox`).
|
|
27
|
+
*/
|
|
28
|
+
export interface WorkerMetricAttributes {
|
|
29
|
+
service: string;
|
|
30
|
+
scope: string;
|
|
31
|
+
worker?: string;
|
|
32
|
+
[key: string]: string | number | undefined;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The metric-recording seam. Defaults to telemetry's real `recordMetric`. A
|
|
36
|
+
* test may inject a capturing fn, but production ALWAYS gets the real path —
|
|
37
|
+
* there is no no-op default. (To assert emitted points in a test, prefer
|
|
38
|
+
* `_setMetricsBackend(...)` underneath the real `recordMetric`, exactly as
|
|
39
|
+
* `metrics.test.ts` does; the `record` seam here exists only for the rare case
|
|
40
|
+
* a caller wants to swap the whole function.)
|
|
41
|
+
*/
|
|
42
|
+
export type RecordMetricFn = (args: RecordMetricArgs) => void;
|
|
43
|
+
export interface ObservabilityHooksOpts {
|
|
44
|
+
/** Stable base attributes (service, scope, worker) for every emitted point. */
|
|
45
|
+
attributes: WorkerMetricAttributes;
|
|
46
|
+
/**
|
|
47
|
+
* The metric-recording fn. Defaults to the REAL telemetry `recordMetric`.
|
|
48
|
+
* Never a no-op. Injectable for direct-capture tests only.
|
|
49
|
+
*/
|
|
50
|
+
record?: RecordMetricFn;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* ObservabilityHooks emits the canonical worker-substrate metrics through the
|
|
54
|
+
* real telemetry metric API. One instance per worker; the worker types call
|
|
55
|
+
* the typed methods at the corresponding lifecycle points (lease acquire/renew/
|
|
56
|
+
* lose, per-tick lag/hwm/depth refresh, drain on stop, sweep count).
|
|
57
|
+
*
|
|
58
|
+
* Usage:
|
|
59
|
+
* const hooks = new ObservabilityHooks({
|
|
60
|
+
* attributes: { service: "billing", scope: "outbox", worker: "outbox" },
|
|
61
|
+
* });
|
|
62
|
+
* hooks.leaseAcquired();
|
|
63
|
+
* hooks.recordLag(12.5);
|
|
64
|
+
* hooks.recordDrainDuration(842);
|
|
65
|
+
*/
|
|
66
|
+
export declare class ObservabilityHooks {
|
|
67
|
+
private readonly attrs;
|
|
68
|
+
private readonly record;
|
|
69
|
+
constructor(opts: ObservabilityHooksOpts);
|
|
70
|
+
private emit;
|
|
71
|
+
/** The worker acquired its single-active lease. */
|
|
72
|
+
leaseAcquired(extra?: Record<string, string | number>): void;
|
|
73
|
+
/** The worker renewed its lease (CAS held) at ttl/2. */
|
|
74
|
+
leaseRenewed(extra?: Record<string, string | number>): void;
|
|
75
|
+
/** The worker lost its lease (renew CAS failed / taken over / errored). */
|
|
76
|
+
leaseLost(extra?: Record<string, string | number>): void;
|
|
77
|
+
/** Snapshot the current consumer lag in seconds. */
|
|
78
|
+
recordLag(seconds: number, extra?: Record<string, string | number>): void;
|
|
79
|
+
/**
|
|
80
|
+
* Snapshot the current durable HWM as a numeric position. A Redis-Streams id
|
|
81
|
+
* (`<ms>-<seq>`) is reduced to its leading ms component so the gauge is a
|
|
82
|
+
* monotonic-ish stall indicator (the same reduction outbox-dispatcher uses
|
|
83
|
+
* for `outbox.hwm`); callers with a numeric cursor pass it directly.
|
|
84
|
+
*/
|
|
85
|
+
recordHwm(position: number, extra?: Record<string, string | number>): void;
|
|
86
|
+
/** Snapshot the current backlog / stream depth (pending entries). */
|
|
87
|
+
recordDepth(depth: number, extra?: Record<string, string | number>): void;
|
|
88
|
+
/** Record the wall-clock ms a graceful drain (stop) took. */
|
|
89
|
+
recordDrainDuration(ms: number, extra?: Record<string, string | number>): void;
|
|
90
|
+
/** Count rows removed by a sweeper retention sweep. */
|
|
91
|
+
recordSwept(count: number, extra?: Record<string, string | number>): void;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Reduce a Redis-Streams id (`<ms>-<seq>`) to a numeric gauge position (its
|
|
95
|
+
* leading ms component). Mirrors outbox-dispatcher's `hwmGaugeValue`. A
|
|
96
|
+
* non-streams numeric string returns its Number; an unparseable value returns 0.
|
|
97
|
+
*/
|
|
98
|
+
export declare function hwmGaugeValue(streamId: string | null): number;
|
|
99
|
+
//# sourceMappingURL=observability-hooks.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"observability-hooks.d.ts","sourceRoot":"","sources":["../../src/worker/observability-hooks.ts"],"names":[],"mappings":"AA4BA,OAAO,EAAE,KAAK,gBAAgB,EAAgB,MAAM,eAAe,CAAC;AACpE,OAAO,KAAK,EAAc,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAExE,0EAA0E;AAC1E,eAAO,MAAM,mBAAmB;;;;;;;;;CAStB,CAAC;AAEX;;;;;;GAMG;AACH,eAAO,MAAM,2BAA2B,EAAE,wBAyDzC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;CAC5C;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,CAAC;AAE9D,MAAM,WAAW,sBAAsB;IACrC,+EAA+E;IAC/E,UAAU,EAAE,sBAAsB,CAAC;IACnC;;;OAGG;IACH,MAAM,CAAC,EAAE,cAAc,CAAC;CACzB;AAcD;;;;;;;;;;;;;GAaG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAyB;IAC/C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;gBAE5B,IAAI,EAAE,sBAAsB;IAMxC,OAAO,CAAC,IAAI;IAcZ,mDAAmD;IACnD,aAAa,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI;IAI5D,wDAAwD;IACxD,YAAY,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI;IAI3D,2EAA2E;IAC3E,SAAS,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI;IAIxD,oDAAoD;IACpD,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI;IAIzE;;;;;OAKG;IACH,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI;IAI1E,qEAAqE;IACrE,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI;IAIzE,6DAA6D;IAC7D,mBAAmB,CACjB,EAAE,EAAE,MAAM,EACV,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GACtC,IAAI;IAIP,uDAAuD;IACvD,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI;CAG1E;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAM7D"}
|