@nodii/telemetry 0.13.0 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/worker/checkpoint-store.d.ts +66 -0
- package/dist/worker/checkpoint-store.d.ts.map +1 -0
- package/dist/worker/checkpoint-store.js +131 -0
- package/dist/worker/checkpoint-store.js.map +1 -0
- package/dist/worker/dead-letter.d.ts +81 -0
- package/dist/worker/dead-letter.d.ts.map +1 -0
- package/dist/worker/dead-letter.js +165 -0
- package/dist/worker/dead-letter.js.map +1 -0
- package/dist/worker/dedupe-store.d.ts +46 -0
- package/dist/worker/dedupe-store.d.ts.map +1 -0
- package/dist/worker/dedupe-store.js +113 -0
- package/dist/worker/dedupe-store.js.map +1 -0
- package/dist/worker/index.d.ts +23 -0
- package/dist/worker/index.d.ts.map +1 -0
- package/dist/worker/index.js +46 -0
- package/dist/worker/index.js.map +1 -0
- package/dist/worker/observability-hooks.d.ts +99 -0
- package/dist/worker/observability-hooks.d.ts.map +1 -0
- package/dist/worker/observability-hooks.js +189 -0
- package/dist/worker/observability-hooks.js.map +1 -0
- package/dist/worker/outbox-dispatcher.d.ts +106 -0
- package/dist/worker/outbox-dispatcher.d.ts.map +1 -0
- package/dist/worker/outbox-dispatcher.js +285 -0
- package/dist/worker/outbox-dispatcher.js.map +1 -0
- package/dist/worker/replication-worker.d.ts +170 -0
- package/dist/worker/replication-worker.d.ts.map +1 -0
- package/dist/worker/replication-worker.js +576 -0
- package/dist/worker/replication-worker.js.map +1 -0
- package/dist/worker/signal-drain.d.ts +83 -0
- package/dist/worker/signal-drain.d.ts.map +1 -0
- package/dist/worker/signal-drain.js +131 -0
- package/dist/worker/signal-drain.js.map +1 -0
- package/dist/worker/single-active-lease.d.ts +180 -0
- package/dist/worker/single-active-lease.d.ts.map +1 -0
- package/dist/worker/single-active-lease.js +394 -0
- package/dist/worker/single-active-lease.js.map +1 -0
- package/dist/worker/streams-worker.d.ts +256 -0
- package/dist/worker/streams-worker.d.ts.map +1 -0
- package/dist/worker/streams-worker.js +621 -0
- package/dist/worker/streams-worker.js.map +1 -0
- package/dist/worker/sweeper-worker.d.ts +107 -0
- package/dist/worker/sweeper-worker.d.ts.map +1 -0
- package/dist/worker/sweeper-worker.js +245 -0
- package/dist/worker/sweeper-worker.js.map +1 -0
- package/dist/worker/tx.d.ts +39 -0
- package/dist/worker/tx.d.ts.map +1 -0
- package/dist/worker/tx.js +51 -0
- package/dist/worker/tx.js.map +1 -0
- package/dist/worker/worker-handle.d.ts +94 -0
- package/dist/worker/worker-handle.d.ts.map +1 -0
- package/dist/worker/worker-handle.js +15 -0
- package/dist/worker/worker-handle.js.map +1 -0
- package/package.json +6 -1
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export { encodeLeaseValue, FenceCheckUnavailableError, FenceLostError, fenceKey, leaseKey, parseLeaseValue, SingleActiveLease, } from "./single-active-lease.js";
|
|
2
|
+
export type { LeaseRedisLike, LeaseTelemetry, SingleActiveLeaseOpts, } from "./single-active-lease.js";
|
|
3
|
+
export { postgresJsExecutor } from "./tx.js";
|
|
4
|
+
export type { SqlExecutor, SqlResult, SqlRow } from "./tx.js";
|
|
5
|
+
export { CheckpointStore, checkpointTableDDL, DEFAULT_CHECKPOINT_TABLE, } from "./checkpoint-store.js";
|
|
6
|
+
export type { Checkpoint, CheckpointAdvance } from "./checkpoint-store.js";
|
|
7
|
+
export { consumedEventsTableDDL, DedupeStore, DEFAULT_CONSUMED_EVENTS_TABLE, DEFAULT_RETENTION_DAYS, } from "./dedupe-store.js";
|
|
8
|
+
export { ensurePiiRedaction, LoudNoopDeadLetterSink, TableDeadLetterSink, withPiiRedaction, } from "./dead-letter.js";
|
|
9
|
+
export type { DeadLetterItem, DeadLetterLogger, DeadLetterSink, } from "./dead-letter.js";
|
|
10
|
+
export { hwmGaugeValue, ObservabilityHooks, WORKER_METRIC_NAMES, WORKER_SUBSTRATE_VOCABULARY, } from "./observability-hooks.js";
|
|
11
|
+
export type { ObservabilityHooksOpts, RecordMetricFn, WorkerMetricAttributes, } from "./observability-hooks.js";
|
|
12
|
+
export { SignalDrain } from "./signal-drain.js";
|
|
13
|
+
export type { DrainResult, SignalDrainOpts, StopHandler, } from "./signal-drain.js";
|
|
14
|
+
export type { WorkerHandle, WorkerHwm, WorkerStopOptions, WorkerStopResult, } from "./worker-handle.js";
|
|
15
|
+
export { SweeperWorker } from "./sweeper-worker.js";
|
|
16
|
+
export type { SweeperJobContext, SweeperJobDefinition, SweeperJobResult, SweeperWorkerOpts, } from "./sweeper-worker.js";
|
|
17
|
+
export { parseAutoClaimEntries, parseAutoClaimEntriesPartitioned, parseStreamEntries, parseStreamEntriesPartitioned, PoisonApplyError, StreamsWorker, streamsConsumerGroup, } from "./streams-worker.js";
|
|
18
|
+
export type { MalformedStreamEntry, StreamApplyContext, StreamApplyHandler, StreamEvent, StreamsRedisLike, StreamsWorkerOpts, } from "./streams-worker.js";
|
|
19
|
+
export { OutboxDispatcher } from "./outbox-dispatcher.js";
|
|
20
|
+
export type { OutboxDispatcherOpts, OutboxPublisher, OutboxRow, } from "./outbox-dispatcher.js";
|
|
21
|
+
export { ReplicationWorker, replicationConsumerGroup, } from "./replication-worker.js";
|
|
22
|
+
export type { ReplicationApplyContext, ReplicationApplyHandler, ReplicationAuditHook, ReplicationEvent, ReplicationWorkerOpts, SnapshotBootstrap, } from "./replication-worker.js";
|
|
23
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/worker/index.ts"],"names":[],"mappings":"AAuBA,OAAO,EACL,gBAAgB,EAChB,0BAA0B,EAC1B,cAAc,EACd,QAAQ,EACR,QAAQ,EACR,eAAe,EACf,iBAAiB,GAClB,MAAM,0BAA0B,CAAC;AAClC,YAAY,EACV,cAAc,EACd,cAAc,EACd,qBAAqB,GACtB,MAAM,0BAA0B,CAAC;AAGlC,OAAO,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAG9D,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,wBAAwB,GACzB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAG3E,OAAO,EACL,sBAAsB,EACtB,WAAW,EACX,6BAA6B,EAC7B,sBAAsB,GACvB,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,cAAc,EACd,gBAAgB,EAChB,cAAc,GACf,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,mBAAmB,EACnB,2BAA2B,GAC5B,MAAM,0BAA0B,CAAC;AAClC,YAAY,EACV,sBAAsB,EACtB,cAAc,EACd,sBAAsB,GACvB,MAAM,0BAA0B,CAAC;AAGlC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EACV,WAAW,EACX,eAAe,EACf,WAAW,GACZ,MAAM,mBAAmB,CAAC;AAG3B,YAAY,EACV,YAAY,EACZ,SAAS,EACT,iBAAiB,EACjB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAM5B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,YAAY,EACV,iBAAiB,EACjB,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EACL,qBAAqB,EACrB,gCAAgC,EAChC,kBAAkB,EAClB,6BAA6B,EAC7B,gBAAgB,EAChB,aAAa,EACb,oBAAoB,GACrB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,oBAAoB,EACpB,kBAAkB,EAClB,kBAAkB,EAClB,WAAW,EACX,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,YAAY,EACV,oBAAoB,EACpB,eAAe,EACf,SAAS,GACV,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EACL,iBAAiB,EACjB,wBAAwB,GACzB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,uBAAuB,EACvB,uBAAuB,EACvB,oBAAoB,EACpB,gBAAgB,EAChB,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,yBAAyB,CAAC"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// @nodii/telemetry/worker — the canonical background-worker substrate (D402).
|
|
2
|
+
//
|
|
3
|
+
// Every background worker on every Nodii service — stream consumer, sweeper,
|
|
4
|
+
// single-active leased loop, outbox drainer — is built from these canonical
|
|
5
|
+
// primitives. No service hand-rolls a lease, a checkpoint cursor, a dedupe
|
|
6
|
+
// store, or a worker lifecycle.
|
|
7
|
+
//
|
|
8
|
+
// D402 (LOCKED) primitives shipped here:
|
|
9
|
+
// - SingleActiveLease — Redis SET-NX-EX + Lua CAS-renew@ttl/2 + fencing token
|
|
10
|
+
// - CheckpointStore — durable HWM cursor, advanced atomically in the apply tx
|
|
11
|
+
// - DedupeStore — consumed_events (event_id, topic) composite-PK + sweep
|
|
12
|
+
// - DeadLetter — injected sink + fail-closed PII redaction (D401)
|
|
13
|
+
// - ObservabilityHooks — canonical worker metrics via the real recordMetric
|
|
14
|
+
// - SignalDrain — SIGTERM/SIGINT bounded graceful drain
|
|
15
|
+
// - WorkerHandle — the uniform handle the 4 worker types implement
|
|
16
|
+
//
|
|
17
|
+
// Resolve as a downstream consumer would:
|
|
18
|
+
// import {
|
|
19
|
+
// SingleActiveLease, CheckpointStore, DedupeStore,
|
|
20
|
+
// ObservabilityHooks, SignalDrain, type WorkerHandle,
|
|
21
|
+
// } from "@nodii/telemetry/worker";
|
|
22
|
+
// ── SingleActiveLease (already shipped) ──────────────────────────────────────
|
|
23
|
+
export { encodeLeaseValue, FenceCheckUnavailableError, FenceLostError, fenceKey, leaseKey, parseLeaseValue, SingleActiveLease, } from "./single-active-lease.js";
|
|
24
|
+
// ── Shared SQL-executor seam (the atomic-in-apply-tx handle) ─────────────────
|
|
25
|
+
export { postgresJsExecutor } from "./tx.js";
|
|
26
|
+
// ── CheckpointStore (HWM cursor) ─────────────────────────────────────────────
|
|
27
|
+
export { CheckpointStore, checkpointTableDDL, DEFAULT_CHECKPOINT_TABLE, } from "./checkpoint-store.js";
|
|
28
|
+
// ── DedupeStore ──────────────────────────────────────────────────────────────
|
|
29
|
+
export { consumedEventsTableDDL, DedupeStore, DEFAULT_CONSUMED_EVENTS_TABLE, DEFAULT_RETENTION_DAYS, } from "./dedupe-store.js";
|
|
30
|
+
// ── DeadLetter (injected sink + fail-closed PII redaction) ───────────────────
|
|
31
|
+
export { ensurePiiRedaction, LoudNoopDeadLetterSink, TableDeadLetterSink, withPiiRedaction, } from "./dead-letter.js";
|
|
32
|
+
// ── ObservabilityHooks ───────────────────────────────────────────────────────
|
|
33
|
+
export { hwmGaugeValue, ObservabilityHooks, WORKER_METRIC_NAMES, WORKER_SUBSTRATE_VOCABULARY, } from "./observability-hooks.js";
|
|
34
|
+
// ── SignalDrain ──────────────────────────────────────────────────────────────
|
|
35
|
+
export { SignalDrain } from "./signal-drain.js";
|
|
36
|
+
// ── Worker types (compose the primitives) ────────────────────────────────────
|
|
37
|
+
// The 4 D402 worker types, each implementing WorkerHandle.
|
|
38
|
+
// SweeperWorker — single-active, checkpoint-resumable interval job.
|
|
39
|
+
export { SweeperWorker } from "./sweeper-worker.js";
|
|
40
|
+
// StreamsWorker — generic (non-replication) Pattern-2 stream consumer.
|
|
41
|
+
export { parseAutoClaimEntries, parseAutoClaimEntriesPartitioned, parseStreamEntries, parseStreamEntriesPartitioned, PoisonApplyError, StreamsWorker, streamsConsumerGroup, } from "./streams-worker.js";
|
|
42
|
+
// OutboxDispatcher — Pattern-3 producer drain.
|
|
43
|
+
export { OutboxDispatcher } from "./outbox-dispatcher.js";
|
|
44
|
+
// ReplicationWorker — Pattern-3 subscriber consume (snapshot + LWW + audit).
|
|
45
|
+
export { ReplicationWorker, replicationConsumerGroup, } from "./replication-worker.js";
|
|
46
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/worker/index.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,EAAE;AACF,6EAA6E;AAC7E,4EAA4E;AAC5E,2EAA2E;AAC3E,gCAAgC;AAChC,EAAE;AACF,yCAAyC;AACzC,kFAAkF;AAClF,oFAAoF;AACpF,mFAAmF;AACnF,6EAA6E;AAC7E,+EAA+E;AAC/E,kEAAkE;AAClE,4EAA4E;AAC5E,EAAE;AACF,0CAA0C;AAC1C,aAAa;AACb,uDAAuD;AACvD,0DAA0D;AAC1D,sCAAsC;AAEtC,gFAAgF;AAChF,OAAO,EACL,gBAAgB,EAChB,0BAA0B,EAC1B,cAAc,EACd,QAAQ,EACR,QAAQ,EACR,eAAe,EACf,iBAAiB,GAClB,MAAM,0BAA0B,CAAC;AAOlC,gFAAgF;AAChF,OAAO,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAG7C,gFAAgF;AAChF,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,wBAAwB,GACzB,MAAM,uBAAuB,CAAC;AAG/B,gFAAgF;AAChF,OAAO,EACL,sBAAsB,EACtB,WAAW,EACX,6BAA6B,EAC7B,sBAAsB,GACvB,MAAM,mBAAmB,CAAC;AAE3B,gFAAgF;AAChF,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAO1B,gFAAgF;AAChF,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,mBAAmB,EACnB,2BAA2B,GAC5B,MAAM,0BAA0B,CAAC;AAOlC,gFAAgF;AAChF,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAehD,gFAAgF;AAChF,2DAA2D;AAE3D,oEAAoE;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAQpD,uEAAuE;AACvE,OAAO,EACL,qBAAqB,EACrB,gCAAgC,EAChC,kBAAkB,EAClB,6BAA6B,EAC7B,gBAAgB,EAChB,aAAa,EACb,oBAAoB,GACrB,MAAM,qBAAqB,CAAC;AAU7B,+CAA+C;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAO1D,6EAA6E;AAC7E,OAAO,EACL,iBAAiB,EACjB,wBAAwB,GACzB,MAAM,yBAAyB,CAAC"}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { type RecordMetricArgs } from "../metrics.js";
|
|
2
|
+
import type { MetricVocabularyRegistry } from "../types.js";
|
|
3
|
+
/** Canonical worker-substrate metric names, pinned by D402 + the task. */
|
|
4
|
+
export declare const WORKER_METRIC_NAMES: {
|
|
5
|
+
readonly leaseAcquired: "nodii.worker.lease.acquired_total";
|
|
6
|
+
readonly leaseRenewed: "nodii.worker.lease.renewed_total";
|
|
7
|
+
readonly leaseLost: "nodii.worker.lease.lost_total";
|
|
8
|
+
readonly lag: "nodii.worker.lag";
|
|
9
|
+
readonly currentHwm: "nodii.worker.current_hwm";
|
|
10
|
+
readonly depth: "nodii.worker.depth";
|
|
11
|
+
readonly drainDurationMs: "nodii.worker.drain.duration_ms";
|
|
12
|
+
readonly swept: "nodii.worker.swept_total";
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* The locked worker-substrate metric vocabulary — merge into
|
|
16
|
+
* `initTelemetry({ vocabularyRegistry: { ...svc, ...WORKER_SUBSTRATE_VOCABULARY } })`
|
|
17
|
+
* so the canonical names register under family `async_worker` instead of
|
|
18
|
+
* tripping `MetricVocabularyError`. Mirrors how `REPLICA_CONSUMER_VOCABULARY`
|
|
19
|
+
* is consumed.
|
|
20
|
+
*/
|
|
21
|
+
export declare const WORKER_SUBSTRATE_VOCABULARY: MetricVocabularyRegistry;
|
|
22
|
+
/**
|
|
23
|
+
* Base attributes every worker-substrate metric carries so a service can slice
|
|
24
|
+
* the canonical series by worker. `service` + `scope` mirror the lease's
|
|
25
|
+
* `(service, scope)` key shape; `worker` is the worker-type label
|
|
26
|
+
* (`replication` | `sweeper` | `streams` | `outbox`).
|
|
27
|
+
*/
|
|
28
|
+
export interface WorkerMetricAttributes {
|
|
29
|
+
service: string;
|
|
30
|
+
scope: string;
|
|
31
|
+
worker?: string;
|
|
32
|
+
[key: string]: string | number | undefined;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The metric-recording seam. Defaults to telemetry's real `recordMetric`. A
|
|
36
|
+
* test may inject a capturing fn, but production ALWAYS gets the real path —
|
|
37
|
+
* there is no no-op default. (To assert emitted points in a test, prefer
|
|
38
|
+
* `_setMetricsBackend(...)` underneath the real `recordMetric`, exactly as
|
|
39
|
+
* `metrics.test.ts` does; the `record` seam here exists only for the rare case
|
|
40
|
+
* a caller wants to swap the whole function.)
|
|
41
|
+
*/
|
|
42
|
+
export type RecordMetricFn = (args: RecordMetricArgs) => void;
|
|
43
|
+
export interface ObservabilityHooksOpts {
|
|
44
|
+
/** Stable base attributes (service, scope, worker) for every emitted point. */
|
|
45
|
+
attributes: WorkerMetricAttributes;
|
|
46
|
+
/**
|
|
47
|
+
* The metric-recording fn. Defaults to the REAL telemetry `recordMetric`.
|
|
48
|
+
* Never a no-op. Injectable for direct-capture tests only.
|
|
49
|
+
*/
|
|
50
|
+
record?: RecordMetricFn;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* ObservabilityHooks emits the canonical worker-substrate metrics through the
|
|
54
|
+
* real telemetry metric API. One instance per worker; the worker types call
|
|
55
|
+
* the typed methods at the corresponding lifecycle points (lease acquire/renew/
|
|
56
|
+
* lose, per-tick lag/hwm/depth refresh, drain on stop, sweep count).
|
|
57
|
+
*
|
|
58
|
+
* Usage:
|
|
59
|
+
* const hooks = new ObservabilityHooks({
|
|
60
|
+
* attributes: { service: "billing", scope: "outbox", worker: "outbox" },
|
|
61
|
+
* });
|
|
62
|
+
* hooks.leaseAcquired();
|
|
63
|
+
* hooks.recordLag(12.5);
|
|
64
|
+
* hooks.recordDrainDuration(842);
|
|
65
|
+
*/
|
|
66
|
+
export declare class ObservabilityHooks {
|
|
67
|
+
private readonly attrs;
|
|
68
|
+
private readonly record;
|
|
69
|
+
constructor(opts: ObservabilityHooksOpts);
|
|
70
|
+
private emit;
|
|
71
|
+
/** The worker acquired its single-active lease. */
|
|
72
|
+
leaseAcquired(extra?: Record<string, string | number>): void;
|
|
73
|
+
/** The worker renewed its lease (CAS held) at ttl/2. */
|
|
74
|
+
leaseRenewed(extra?: Record<string, string | number>): void;
|
|
75
|
+
/** The worker lost its lease (renew CAS failed / taken over / errored). */
|
|
76
|
+
leaseLost(extra?: Record<string, string | number>): void;
|
|
77
|
+
/** Snapshot the current consumer lag in seconds. */
|
|
78
|
+
recordLag(seconds: number, extra?: Record<string, string | number>): void;
|
|
79
|
+
/**
|
|
80
|
+
* Snapshot the current durable HWM as a numeric position. A Redis-Streams id
|
|
81
|
+
* (`<ms>-<seq>`) is reduced to its leading ms component so the gauge is a
|
|
82
|
+
* monotonic-ish stall indicator (the same reduction outbox-dispatcher uses
|
|
83
|
+
* for `outbox.hwm`); callers with a numeric cursor pass it directly.
|
|
84
|
+
*/
|
|
85
|
+
recordHwm(position: number, extra?: Record<string, string | number>): void;
|
|
86
|
+
/** Snapshot the current backlog / stream depth (pending entries). */
|
|
87
|
+
recordDepth(depth: number, extra?: Record<string, string | number>): void;
|
|
88
|
+
/** Record the wall-clock ms a graceful drain (stop) took. */
|
|
89
|
+
recordDrainDuration(ms: number, extra?: Record<string, string | number>): void;
|
|
90
|
+
/** Count rows removed by a sweeper retention sweep. */
|
|
91
|
+
recordSwept(count: number, extra?: Record<string, string | number>): void;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Reduce a Redis-Streams id (`<ms>-<seq>`) to a numeric gauge position (its
|
|
95
|
+
* leading ms component). Mirrors outbox-dispatcher's `hwmGaugeValue`. A
|
|
96
|
+
* non-streams numeric string returns its Number; an unparseable value returns 0.
|
|
97
|
+
*/
|
|
98
|
+
export declare function hwmGaugeValue(streamId: string | null): number;
|
|
99
|
+
//# sourceMappingURL=observability-hooks.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"observability-hooks.d.ts","sourceRoot":"","sources":["../../src/worker/observability-hooks.ts"],"names":[],"mappings":"AA4BA,OAAO,EAAE,KAAK,gBAAgB,EAAgB,MAAM,eAAe,CAAC;AACpE,OAAO,KAAK,EAAc,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAExE,0EAA0E;AAC1E,eAAO,MAAM,mBAAmB;;;;;;;;;CAStB,CAAC;AAEX;;;;;;GAMG;AACH,eAAO,MAAM,2BAA2B,EAAE,wBAyDzC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;CAC5C;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,CAAC;AAE9D,MAAM,WAAW,sBAAsB;IACrC,+EAA+E;IAC/E,UAAU,EAAE,sBAAsB,CAAC;IACnC;;;OAGG;IACH,MAAM,CAAC,EAAE,cAAc,CAAC;CACzB;AAcD;;;;;;;;;;;;;GAaG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAyB;IAC/C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;gBAE5B,IAAI,EAAE,sBAAsB;IAMxC,OAAO,CAAC,IAAI;IAcZ,mDAAmD;IACnD,aAAa,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI;IAI5D,wDAAwD;IACxD,YAAY,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI;IAI3D,2EAA2E;IAC3E,SAAS,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI;IAIxD,oDAAoD;IACpD,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI;IAIzE;;;;;OAKG;IACH,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI;IAI1E,qEAAqE;IACrE,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI;IAIzE,6DAA6D;IAC7D,mBAAmB,CACjB,EAAE,EAAE,MAAM,EACV,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GACtC,IAAI;IAIP,uDAAuD;IACvD,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI;CAG1E;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAM7D"}
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
// D402 — ObservabilityHooks: the canonical worker-metric vocabulary, routed
|
|
2
|
+
// through telemetry's REAL `recordMetric` API (NOT a NOOP sink by default).
|
|
3
|
+
//
|
|
4
|
+
// D402 (LOCKED) — "ObservabilityHooks — lag / HWM / depth metrics with canonical
|
|
5
|
+
// names." The orchestrator's task pins the exact canonical names:
|
|
6
|
+
// - nodii.worker.lease.acquired_total (counter)
|
|
7
|
+
// - nodii.worker.lease.renewed_total (counter)
|
|
8
|
+
// - nodii.worker.lease.lost_total (counter)
|
|
9
|
+
// - nodii.worker.lag (gauge, seconds)
|
|
10
|
+
// - nodii.worker.current_hwm (gauge)
|
|
11
|
+
// - nodii.worker.depth (gauge — backlog/stream depth)
|
|
12
|
+
// - nodii.worker.drain.duration_ms (histogram, ms)
|
|
13
|
+
// - nodii.worker.swept_total (counter — sweepers only)
|
|
14
|
+
//
|
|
15
|
+
// These names live under the existing D18/D197 `async_worker` metric family
|
|
16
|
+
// (background workers are the async_worker family). A consuming service merges
|
|
17
|
+
// `WORKER_SUBSTRATE_VOCABULARY` (below) into `initTelemetry({ vocabularyRegistry
|
|
18
|
+
// })` exactly as it merges `REPLICA_CONSUMER_VOCABULARY` — otherwise the
|
|
19
|
+
// canonical names trip `MetricVocabularyError` (vocabulary is HARD per D24).
|
|
20
|
+
//
|
|
21
|
+
// ── Why this routes through `recordMetric`, never a default no-op ────────────
|
|
22
|
+
// R1 (never-defer) forbids a Noop sink wired as the default. The hooks bind to
|
|
23
|
+
// telemetry's own `recordMetric({ kind, name, value, attributes })`. That fn
|
|
24
|
+
// is the real metric path: it validates the name against the loaded vocabulary
|
|
25
|
+
// + the family + the kind, then records onto the live OTLP-backed metrics
|
|
26
|
+
// backend. The ONLY injection seam is for tests, which `_setMetricsBackend(...)`
|
|
27
|
+
// a capturing backend underneath — the hook itself always calls the real API.
|
|
28
|
+
import { recordMetric } from "../metrics.js";
|
|
29
|
+
/** Canonical worker-substrate metric names, pinned by D402 + the task. */
|
|
30
|
+
export const WORKER_METRIC_NAMES = {
|
|
31
|
+
leaseAcquired: "nodii.worker.lease.acquired_total",
|
|
32
|
+
leaseRenewed: "nodii.worker.lease.renewed_total",
|
|
33
|
+
leaseLost: "nodii.worker.lease.lost_total",
|
|
34
|
+
lag: "nodii.worker.lag",
|
|
35
|
+
currentHwm: "nodii.worker.current_hwm",
|
|
36
|
+
depth: "nodii.worker.depth",
|
|
37
|
+
drainDurationMs: "nodii.worker.drain.duration_ms",
|
|
38
|
+
swept: "nodii.worker.swept_total",
|
|
39
|
+
};
|
|
40
|
+
/**
|
|
41
|
+
* The locked worker-substrate metric vocabulary — merge into
|
|
42
|
+
* `initTelemetry({ vocabularyRegistry: { ...svc, ...WORKER_SUBSTRATE_VOCABULARY } })`
|
|
43
|
+
* so the canonical names register under family `async_worker` instead of
|
|
44
|
+
* tripping `MetricVocabularyError`. Mirrors how `REPLICA_CONSUMER_VOCABULARY`
|
|
45
|
+
* is consumed.
|
|
46
|
+
*/
|
|
47
|
+
export const WORKER_SUBSTRATE_VOCABULARY = {
|
|
48
|
+
[WORKER_METRIC_NAMES.leaseAcquired]: {
|
|
49
|
+
family: "async_worker",
|
|
50
|
+
kind: "counter",
|
|
51
|
+
description: "Single-active lease acquisitions (a worker took the lease for its scope).",
|
|
52
|
+
decision: "D402",
|
|
53
|
+
},
|
|
54
|
+
[WORKER_METRIC_NAMES.leaseRenewed]: {
|
|
55
|
+
family: "async_worker",
|
|
56
|
+
kind: "counter",
|
|
57
|
+
description: "Single-active lease renewals at ttl/2 (CAS held).",
|
|
58
|
+
decision: "D402",
|
|
59
|
+
},
|
|
60
|
+
[WORKER_METRIC_NAMES.leaseLost]: {
|
|
61
|
+
family: "async_worker",
|
|
62
|
+
kind: "counter",
|
|
63
|
+
description: "Single-active lease losses (renew CAS failed / lease taken over / errored).",
|
|
64
|
+
decision: "D402",
|
|
65
|
+
},
|
|
66
|
+
[WORKER_METRIC_NAMES.lag]: {
|
|
67
|
+
family: "async_worker",
|
|
68
|
+
kind: "gauge",
|
|
69
|
+
unit: "seconds",
|
|
70
|
+
description: "Wall-clock lag between source emission and consumer apply for the worker.",
|
|
71
|
+
decision: "D402",
|
|
72
|
+
},
|
|
73
|
+
[WORKER_METRIC_NAMES.currentHwm]: {
|
|
74
|
+
family: "async_worker",
|
|
75
|
+
kind: "gauge",
|
|
76
|
+
description: "Current durable high-water-mark (checkpoint cursor position) the worker has advanced to.",
|
|
77
|
+
decision: "D402",
|
|
78
|
+
},
|
|
79
|
+
[WORKER_METRIC_NAMES.depth]: {
|
|
80
|
+
family: "async_worker",
|
|
81
|
+
kind: "gauge",
|
|
82
|
+
description: "Backlog / stream depth pending for the worker (unprocessed entries).",
|
|
83
|
+
decision: "D402",
|
|
84
|
+
},
|
|
85
|
+
[WORKER_METRIC_NAMES.drainDurationMs]: {
|
|
86
|
+
family: "async_worker",
|
|
87
|
+
kind: "histogram",
|
|
88
|
+
unit: "ms",
|
|
89
|
+
description: "Graceful-drain wall-clock duration on stop() (flush in-flight + release leases).",
|
|
90
|
+
decision: "D402",
|
|
91
|
+
},
|
|
92
|
+
[WORKER_METRIC_NAMES.swept]: {
|
|
93
|
+
family: "async_worker",
|
|
94
|
+
kind: "counter",
|
|
95
|
+
description: "Rows removed by a SweeperWorker retention sweep.",
|
|
96
|
+
decision: "D402",
|
|
97
|
+
},
|
|
98
|
+
};
|
|
99
|
+
/** Strip `undefined`-valued attributes so the recordMetric label map is clean. */
|
|
100
|
+
function cleanAttributes(attrs, extra) {
|
|
101
|
+
const out = {};
|
|
102
|
+
for (const [k, v] of Object.entries({ ...attrs, ...extra })) {
|
|
103
|
+
if (v !== undefined)
|
|
104
|
+
out[k] = v;
|
|
105
|
+
}
|
|
106
|
+
return out;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* ObservabilityHooks emits the canonical worker-substrate metrics through the
|
|
110
|
+
* real telemetry metric API. One instance per worker; the worker types call
|
|
111
|
+
* the typed methods at the corresponding lifecycle points (lease acquire/renew/
|
|
112
|
+
* lose, per-tick lag/hwm/depth refresh, drain on stop, sweep count).
|
|
113
|
+
*
|
|
114
|
+
* Usage:
|
|
115
|
+
* const hooks = new ObservabilityHooks({
|
|
116
|
+
* attributes: { service: "billing", scope: "outbox", worker: "outbox" },
|
|
117
|
+
* });
|
|
118
|
+
* hooks.leaseAcquired();
|
|
119
|
+
* hooks.recordLag(12.5);
|
|
120
|
+
* hooks.recordDrainDuration(842);
|
|
121
|
+
*/
|
|
122
|
+
export class ObservabilityHooks {
|
|
123
|
+
attrs;
|
|
124
|
+
record;
|
|
125
|
+
constructor(opts) {
|
|
126
|
+
this.attrs = opts.attributes;
|
|
127
|
+
// Default = the REAL recordMetric. R1: never a no-op default.
|
|
128
|
+
this.record = opts.record ?? recordMetric;
|
|
129
|
+
}
|
|
130
|
+
emit(kind, name, value, extra) {
|
|
131
|
+
this.record({
|
|
132
|
+
kind,
|
|
133
|
+
name,
|
|
134
|
+
value,
|
|
135
|
+
attributes: cleanAttributes(this.attrs, extra),
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
/** The worker acquired its single-active lease. */
|
|
139
|
+
leaseAcquired(extra) {
|
|
140
|
+
this.emit("counter", WORKER_METRIC_NAMES.leaseAcquired, 1, extra);
|
|
141
|
+
}
|
|
142
|
+
/** The worker renewed its lease (CAS held) at ttl/2. */
|
|
143
|
+
leaseRenewed(extra) {
|
|
144
|
+
this.emit("counter", WORKER_METRIC_NAMES.leaseRenewed, 1, extra);
|
|
145
|
+
}
|
|
146
|
+
/** The worker lost its lease (renew CAS failed / taken over / errored). */
|
|
147
|
+
leaseLost(extra) {
|
|
148
|
+
this.emit("counter", WORKER_METRIC_NAMES.leaseLost, 1, extra);
|
|
149
|
+
}
|
|
150
|
+
/** Snapshot the current consumer lag in seconds. */
|
|
151
|
+
recordLag(seconds, extra) {
|
|
152
|
+
this.emit("gauge", WORKER_METRIC_NAMES.lag, seconds, extra);
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Snapshot the current durable HWM as a numeric position. A Redis-Streams id
|
|
156
|
+
* (`<ms>-<seq>`) is reduced to its leading ms component so the gauge is a
|
|
157
|
+
* monotonic-ish stall indicator (the same reduction outbox-dispatcher uses
|
|
158
|
+
* for `outbox.hwm`); callers with a numeric cursor pass it directly.
|
|
159
|
+
*/
|
|
160
|
+
recordHwm(position, extra) {
|
|
161
|
+
this.emit("gauge", WORKER_METRIC_NAMES.currentHwm, position, extra);
|
|
162
|
+
}
|
|
163
|
+
/** Snapshot the current backlog / stream depth (pending entries). */
|
|
164
|
+
recordDepth(depth, extra) {
|
|
165
|
+
this.emit("gauge", WORKER_METRIC_NAMES.depth, depth, extra);
|
|
166
|
+
}
|
|
167
|
+
/** Record the wall-clock ms a graceful drain (stop) took. */
|
|
168
|
+
recordDrainDuration(ms, extra) {
|
|
169
|
+
this.emit("histogram", WORKER_METRIC_NAMES.drainDurationMs, ms, extra);
|
|
170
|
+
}
|
|
171
|
+
/** Count rows removed by a sweeper retention sweep. */
|
|
172
|
+
recordSwept(count, extra) {
|
|
173
|
+
this.emit("counter", WORKER_METRIC_NAMES.swept, count, extra);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Reduce a Redis-Streams id (`<ms>-<seq>`) to a numeric gauge position (its
|
|
178
|
+
* leading ms component). Mirrors outbox-dispatcher's `hwmGaugeValue`. A
|
|
179
|
+
* non-streams numeric string returns its Number; an unparseable value returns 0.
|
|
180
|
+
*/
|
|
181
|
+
export function hwmGaugeValue(streamId) {
|
|
182
|
+
if (!streamId)
|
|
183
|
+
return 0;
|
|
184
|
+
const dash = streamId.indexOf("-");
|
|
185
|
+
const head = dash < 0 ? streamId : streamId.slice(0, dash);
|
|
186
|
+
const n = Number(head);
|
|
187
|
+
return Number.isFinite(n) ? n : 0;
|
|
188
|
+
}
|
|
189
|
+
//# sourceMappingURL=observability-hooks.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"observability-hooks.js","sourceRoot":"","sources":["../../src/worker/observability-hooks.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,4EAA4E;AAC5E,EAAE;AACF,iFAAiF;AACjF,kEAAkE;AAClE,oDAAoD;AACpD,oDAAoD;AACpD,oDAAoD;AACpD,6DAA6D;AAC7D,mDAAmD;AACnD,0EAA0E;AAC1E,2DAA2D;AAC3D,qEAAqE;AACrE,EAAE;AACF,4EAA4E;AAC5E,+EAA+E;AAC/E,iFAAiF;AACjF,yEAAyE;AACzE,6EAA6E;AAC7E,EAAE;AACF,gFAAgF;AAChF,+EAA+E;AAC/E,6EAA6E;AAC7E,+EAA+E;AAC/E,0EAA0E;AAC1E,iFAAiF;AACjF,8EAA8E;AAE9E,OAAO,EAAyB,YAAY,EAAE,MAAM,eAAe,CAAC;AAGpE,0EAA0E;AAC1E,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,aAAa,EAAE,mCAAmC;IAClD,YAAY,EAAE,kCAAkC;IAChD,SAAS,EAAE,+BAA+B;IAC1C,GAAG,EAAE,kBAAkB;IACvB,UAAU,EAAE,0BAA0B;IACtC,KAAK,EAAE,oBAAoB;IAC3B,eAAe,EAAE,gCAAgC;IACjD,KAAK,EAAE,0BAA0B;CACzB,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAA6B;IACnE,CAAC,mBAAmB,CAAC,aAAa,CAAC,EAAE;QACnC,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,SAAS;QACf,WAAW,EACT,2EAA2E;QAC7E,QAAQ,EAAE,MAAM;KACjB;IACD,CAAC,mBAAmB,CAAC,YAAY,CAAC,EAAE;QAClC,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,SAAS;QACf,WAAW,EAAE,mDAAmD;QAChE,QAAQ,EAAE,MAAM;KACjB;IACD,CAAC,mBAAmB,CAAC,SAAS,CAAC,EAAE;QAC/B,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,SAAS;QACf,WAAW,EACT,6EAA6E;QAC/E,QAAQ,EAAE,MAAM;KACjB;IACD,CAAC,mBAAmB,CAAC,GAAG,CAAC,EAAE;QACzB,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,OAAO;QACb,IAAI,EAAE,SAAS;QACf,WAAW,EACT,2EAA2E;QAC7E,QAAQ,EAAE,MAAM;KACjB;IACD,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE;QAChC,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,OAAO;QACb,WAAW,EACT,0FAA0F;QAC5F,QAAQ,EAAE,MAAM;KACjB;IACD,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE;QAC3B,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,OAAO;QACb,WAAW,EACT,sEAAsE;QACxE,QAAQ,EAAE,MAAM;KACjB;IACD,CAAC,mBAAmB,CAAC,eAAe,CAAC,EAAE;QACrC,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,IAAI;QACV,WAAW,EACT,kFAAkF;QACpF,QAAQ,EAAE,MAAM;KACjB;IACD,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE;QAC3B,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,SAAS;QACf,WAAW,EAAE,kDAAkD;QAC/D,QAAQ,EAAE,MAAM;KACjB;CACF,CAAC;AAmCF,kFAAkF;AAClF,SAAS,eAAe,CACtB,KAA6B,EAC7B,KAAuC;IAEvC,MAAM,GAAG,GAAoC,EAAE,CAAC;IAChD,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,GAAG,KAAK,EAAE,GAAG,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5D,IAAI,CAAC,KAAK,SAAS;YAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAO,kBAAkB;IACZ,KAAK,CAAyB;IAC9B,MAAM,CAAiB;IAExC,YAAY,IAA4B;QACtC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC;QAC7B,8DAA8D;QAC9D,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC;IAC5C,CAAC;IAEO,IAAI,CACV,IAAgB,EAChB,IAAY,EACZ,KAAa,EACb,KAAuC;QAEvC,IAAI,CAAC,MAAM,CAAC;YACV,IAAI;YACJ,IAAI;YACJ,KAAK;YACL,UAAU,EAAE,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC;SAC/C,CAAC,CAAC;IACL,CAAC;IAED,mDAAmD;IACnD,aAAa,CAAC,KAAuC;QACnD,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,aAAa,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;IACpE,CAAC;IAED,wDAAwD;IACxD,YAAY,CAAC,KAAuC;QAClD,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,YAAY,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;IACnE,CAAC;IAED,2EAA2E;IAC3E,SAAS,CAAC,KAAuC;QAC/C,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,SAAS,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;IAChE,CAAC;IAED,oDAAoD;IACpD,SAAS,CAAC,OAAe,EAAE,KAAuC;QAChE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED;;;;;OAKG;IACH,SAAS,CAAC,QAAgB,EAAE,KAAuC;QACjE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,UAAU,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IACtE,CAAC;IAED,qEAAqE;IACrE,WAAW,CAAC,KAAa,EAAE,KAAuC;QAChE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED,6DAA6D;IAC7D,mBAAmB,CACjB,EAAU,EACV,KAAuC;QAEvC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,mBAAmB,CAAC,eAAe,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;IACzE,CAAC;IAED,uDAAuD;IACvD,WAAW,CAAC,KAAa,EAAE,KAAuC;QAChE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAChE,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,QAAuB;IACnD,IAAI,CAAC,QAAQ;QAAE,OAAO,CAAC,CAAC;IACxB,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC3D,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IACvB,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpC,CAAC"}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import type { CheckpointStore } from "./checkpoint-store.js";
|
|
2
|
+
import type { DeadLetterSink } from "./dead-letter.js";
|
|
3
|
+
import { ObservabilityHooks } from "./observability-hooks.js";
|
|
4
|
+
import { type SingleActiveLeaseOpts } from "./single-active-lease.js";
|
|
5
|
+
import type { SqlExecutor } from "./tx.js";
|
|
6
|
+
import type { WorkerHandle, WorkerHwm, WorkerStopOptions, WorkerStopResult } from "./worker-handle.js";
|
|
7
|
+
/** A pending outbox row claimed for dispatch. The generic shape the drain needs. */
|
|
8
|
+
export interface OutboxRow {
|
|
9
|
+
/** Primary key (used to mark dispatched). */
|
|
10
|
+
id: string | number;
|
|
11
|
+
/** The dedupe id the consumer dedupes on. */
|
|
12
|
+
event_id: string;
|
|
13
|
+
/** The outbox topic. */
|
|
14
|
+
topic: string;
|
|
15
|
+
/** Source emission time (ms epoch or ISO) for lag + FIFO order. */
|
|
16
|
+
occurred_at: string | number;
|
|
17
|
+
/** The rest of the row (the publisher serializes whatever it needs). */
|
|
18
|
+
[key: string]: unknown;
|
|
19
|
+
}
|
|
20
|
+
/** The injected publisher — XADDs (or otherwise emits) a row + returns the id. */
|
|
21
|
+
export type OutboxPublisher = (row: OutboxRow) => Promise<string | null>;
|
|
22
|
+
export interface OutboxDispatcherOpts {
|
|
23
|
+
/** The injected publisher (e.g. an XADD to the topic's stream). */
|
|
24
|
+
publish: OutboxPublisher;
|
|
25
|
+
/**
|
|
26
|
+
* Opens a claim transaction. The drain claims rows + marks dispatched +
|
|
27
|
+
* advances the cursor on THIS tx so the mark + cursor commit together-or-none.
|
|
28
|
+
*/
|
|
29
|
+
begin: <T>(fn: (tx: SqlExecutor) => Promise<T>) => Promise<T>;
|
|
30
|
+
/** Claims a batch of pending rows ON the claim tx (FOR UPDATE SKIP LOCKED). */
|
|
31
|
+
claimBatch: (tx: SqlExecutor, batchSize: number) => Promise<OutboxRow[]>;
|
|
32
|
+
/** Marks a row dispatched ON the claim tx (sets dispatched_at + stream_id). */
|
|
33
|
+
markDispatched: (tx: SqlExecutor, id: string | number, streamId: string | null) => Promise<void>;
|
|
34
|
+
/** Records a publish failure ON the claim tx (D401: bump attempts; keep row). */
|
|
35
|
+
recordFailure?: (tx: SqlExecutor, id: string | number, errorMessage: string) => Promise<void>;
|
|
36
|
+
checkpoint: CheckpointStore;
|
|
37
|
+
/** Lease config (service + scope=outbox + ttlMs). */
|
|
38
|
+
lease: Omit<SingleActiveLeaseOpts, "onLeaseLost">;
|
|
39
|
+
/** Checkpoint subscription key. Default `outbox:<service>`. */
|
|
40
|
+
subscription?: string;
|
|
41
|
+
/**
|
|
42
|
+
* OPTIONAL and currently NOT invoked on any drain path. Per D401 the producer
|
|
43
|
+
* drain has NO source-side dead-letter: a terminal / unpublishable row is
|
|
44
|
+
* surfaced via `recordFailure` (keep-row, publish_attempts bumped) + the
|
|
45
|
+
* 24h-unpublished sweep, NOT a sink. The sink is accepted for API symmetry with
|
|
46
|
+
* the consumer workers (StreamsWorker/ReplicationWorker, which DO dead-letter
|
|
47
|
+
* poison events) and a future opt-in; if provided it is PII-redacted, but no
|
|
48
|
+
* path calls it today.
|
|
49
|
+
*/
|
|
50
|
+
deadLetter?: DeadLetterSink;
|
|
51
|
+
/** Rows claimed per drain pass. Default 100. */
|
|
52
|
+
batchSize?: number;
|
|
53
|
+
/** Poll interval (ms) when the loop runs autonomously. Default 1_000. */
|
|
54
|
+
pollMs?: number;
|
|
55
|
+
hooks?: ObservabilityHooks;
|
|
56
|
+
workerLabel?: string;
|
|
57
|
+
/** Launch the background drain loop on `start()`. Default true; tests pass
|
|
58
|
+
* false to drive `drainOnce()`/`runOnce()` deterministically. */
|
|
59
|
+
autoStart?: boolean;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* OutboxDispatcher — the Pattern-3 producer drain. Composes lease + checkpoint +
|
|
63
|
+
* hooks + drain. Single-active; the mark-dispatched + cursor advance are atomic
|
|
64
|
+
* in the claim tx; publish is at-least-once (the consumer dedupes).
|
|
65
|
+
*/
|
|
66
|
+
export declare class OutboxDispatcher implements WorkerHandle {
|
|
67
|
+
private readonly publish;
|
|
68
|
+
private readonly begin;
|
|
69
|
+
private readonly claimBatch;
|
|
70
|
+
private readonly markDispatched;
|
|
71
|
+
private readonly recordFailure?;
|
|
72
|
+
private readonly checkpoint;
|
|
73
|
+
private readonly lease;
|
|
74
|
+
private readonly subscription;
|
|
75
|
+
private readonly deadLetter?;
|
|
76
|
+
private readonly batchSize;
|
|
77
|
+
private readonly pollMs;
|
|
78
|
+
private readonly hooks;
|
|
79
|
+
private readonly autoStart;
|
|
80
|
+
private _running;
|
|
81
|
+
private _stopped;
|
|
82
|
+
private _inFlight;
|
|
83
|
+
private _hwm;
|
|
84
|
+
private _lastAppliedAtMs;
|
|
85
|
+
private loop;
|
|
86
|
+
constructor(opts: OutboxDispatcherOpts);
|
|
87
|
+
get running(): boolean;
|
|
88
|
+
get leaseAcquired(): boolean;
|
|
89
|
+
get lag(): number | null;
|
|
90
|
+
get currentHwm(): WorkerHwm;
|
|
91
|
+
start(): Promise<void>;
|
|
92
|
+
private hydrateCursor;
|
|
93
|
+
private runLoop;
|
|
94
|
+
/**
|
|
95
|
+
* One drain pass: claim a batch in ONE tx, publish each, mark dispatched +
|
|
96
|
+
* advance the HWM cursor (fence-guarded) in the SAME tx. Returns the count
|
|
97
|
+
* dispatched. The publish runs INSIDE the claim tx so the row-lock is held
|
|
98
|
+
* across it (a second dispatcher can't double-claim); at-least-once safe (a
|
|
99
|
+
* crash after publish but before commit re-publishes; the consumer dedupes).
|
|
100
|
+
*/
|
|
101
|
+
drainOnce(): Promise<number>;
|
|
102
|
+
/** Run exactly one drain pass (test seam / cron tick). */
|
|
103
|
+
runOnce(): Promise<number>;
|
|
104
|
+
stop(opts?: WorkerStopOptions): Promise<WorkerStopResult>;
|
|
105
|
+
}
|
|
106
|
+
//# sourceMappingURL=outbox-dispatcher.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"outbox-dispatcher.d.ts","sourceRoot":"","sources":["../../src/worker/outbox-dispatcher.ts"],"names":[],"mappings":"AA2CA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAE7D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAiB,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC7E,OAAO,EAIL,KAAK,qBAAqB,EAC3B,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,KAAK,EACV,YAAY,EACZ,SAAS,EACT,iBAAiB,EACjB,gBAAgB,EACjB,MAAM,oBAAoB,CAAC;AAE5B,oFAAoF;AACpF,MAAM,WAAW,SAAS;IACxB,6CAA6C;IAC7C,EAAE,EAAE,MAAM,GAAG,MAAM,CAAC;IACpB,6CAA6C;IAC7C,QAAQ,EAAE,MAAM,CAAC;IACjB,wBAAwB;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,mEAAmE;IACnE,WAAW,EAAE,MAAM,GAAG,MAAM,CAAC;IAC7B,wEAAwE;IACxE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,kFAAkF;AAClF,MAAM,MAAM,eAAe,GAAG,CAAC,GAAG,EAAE,SAAS,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;AAEzE,MAAM,WAAW,oBAAoB;IACnC,mEAAmE;IACnE,OAAO,EAAE,eAAe,CAAC;IACzB;;;OAGG;IACH,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;IAC9D,+EAA+E;IAC/E,UAAU,EAAE,CAAC,EAAE,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IACzE,+EAA+E;IAC/E,cAAc,EAAE,CACd,EAAE,EAAE,WAAW,EACf,EAAE,EAAE,MAAM,GAAG,MAAM,EACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,KACpB,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,iFAAiF;IACjF,aAAa,CAAC,EAAE,CACd,EAAE,EAAE,WAAW,EACf,EAAE,EAAE,MAAM,GAAG,MAAM,EACnB,YAAY,EAAE,MAAM,KACjB,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,UAAU,EAAE,eAAe,CAAC;IAC5B,qDAAqD;IACrD,KAAK,EAAE,IAAI,CAAC,qBAAqB,EAAE,aAAa,CAAC,CAAC;IAClD,+DAA+D;IAC/D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B,gDAAgD;IAChD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yEAAyE;IACzE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,kBAAkB,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;sEACkE;IAClE,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAMD;;;;GAIG;AACH,qBAAa,gBAAiB,YAAW,YAAY;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;IAC1C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAEN;IAChB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAGD;IAC1B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAIZ;IACnB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAIZ;IACnB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAkB;IAC7C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAoB;IAC1C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAGtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAiB;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAqB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAU;IAEpC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,IAAI,CAAqC;IACjD,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,IAAI,CAAoC;gBAEpC,IAAI,EAAE,oBAAoB;IA6BtC,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,GAAG,IAAI,MAAM,GAAG,IAAI,CAGvB;IAED,IAAI,UAAU,IAAI,SAAS,CAE1B;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAWd,aAAa;YAWb,OAAO;IA2BrB;;;;;;OAMG;IACG,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC;IA8ElC,0DAA0D;IACpD,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC;IAI1B,IAAI,CAAC,IAAI,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAsBhE"}
|