@end-close/relay 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/COMPATIBILITY.md +39 -0
- package/LICENSE +21 -0
- package/README.md +205 -0
- package/dist/config/schema.d.ts +993 -0
- package/dist/config/schema.d.ts.map +1 -0
- package/dist/config/schema.js +156 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/crypto/at-rest.d.ts +6 -0
- package/dist/crypto/at-rest.d.ts.map +1 -0
- package/dist/crypto/at-rest.js +19 -0
- package/dist/crypto/at-rest.js.map +1 -0
- package/dist/crypto/keys.d.ts +2 -0
- package/dist/crypto/keys.d.ts.map +1 -0
- package/dist/crypto/keys.js +12 -0
- package/dist/crypto/keys.js.map +1 -0
- package/dist/engine/codec.d.ts +12 -0
- package/dist/engine/codec.d.ts.map +1 -0
- package/dist/engine/codec.js +21 -0
- package/dist/engine/codec.js.map +1 -0
- package/dist/engine/hooks.d.ts +84 -0
- package/dist/engine/hooks.d.ts.map +1 -0
- package/dist/engine/hooks.js +34 -0
- package/dist/engine/hooks.js.map +1 -0
- package/dist/engine/ingest.d.ts +39 -0
- package/dist/engine/ingest.d.ts.map +1 -0
- package/dist/engine/ingest.js +131 -0
- package/dist/engine/ingest.js.map +1 -0
- package/dist/engine/memory-store.d.ts +35 -0
- package/dist/engine/memory-store.d.ts.map +1 -0
- package/dist/engine/memory-store.js +205 -0
- package/dist/engine/memory-store.js.map +1 -0
- package/dist/engine/relay.d.ts +116 -0
- package/dist/engine/relay.d.ts.map +1 -0
- package/dist/engine/relay.js +180 -0
- package/dist/engine/relay.js.map +1 -0
- package/dist/engine/secrets.d.ts +18 -0
- package/dist/engine/secrets.d.ts.map +1 -0
- package/dist/engine/secrets.js +42 -0
- package/dist/engine/secrets.js.map +1 -0
- package/dist/engine/settings.d.ts +29 -0
- package/dist/engine/settings.d.ts.map +1 -0
- package/dist/engine/settings.js +13 -0
- package/dist/engine/settings.js.map +1 -0
- package/dist/engine/store.d.ts +141 -0
- package/dist/engine/store.d.ts.map +1 -0
- package/dist/engine/store.js +52 -0
- package/dist/engine/store.js.map +1 -0
- package/dist/forward/backoff.d.ts +3 -0
- package/dist/forward/backoff.d.ts.map +1 -0
- package/dist/forward/backoff.js +10 -0
- package/dist/forward/backoff.js.map +1 -0
- package/dist/forward/dispatcher.d.ts +98 -0
- package/dist/forward/dispatcher.d.ts.map +1 -0
- package/dist/forward/dispatcher.js +444 -0
- package/dist/forward/dispatcher.js.map +1 -0
- package/dist/forward/endclose-client.d.ts +42 -0
- package/dist/forward/endclose-client.d.ts.map +1 -0
- package/dist/forward/endclose-client.js +76 -0
- package/dist/forward/endclose-client.js.map +1 -0
- package/dist/forward/enrich.d.ts +39 -0
- package/dist/forward/enrich.d.ts.map +1 -0
- package/dist/forward/enrich.js +77 -0
- package/dist/forward/enrich.js.map +1 -0
- package/dist/forward/mapper.d.ts +50 -0
- package/dist/forward/mapper.d.ts.map +1 -0
- package/dist/forward/mapper.js +141 -0
- package/dist/forward/mapper.js.map +1 -0
- package/dist/index.d.ts +36 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +24 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest/adapters/generic-hmac.d.ts +3 -0
- package/dist/ingest/adapters/generic-hmac.d.ts.map +1 -0
- package/dist/ingest/adapters/generic-hmac.js +60 -0
- package/dist/ingest/adapters/generic-hmac.js.map +1 -0
- package/dist/ingest/adapters/payabli.d.ts +3 -0
- package/dist/ingest/adapters/payabli.d.ts.map +1 -0
- package/dist/ingest/adapters/payabli.js +57 -0
- package/dist/ingest/adapters/payabli.js.map +1 -0
- package/dist/ingest/adapters/registry.d.ts +4 -0
- package/dist/ingest/adapters/registry.d.ts.map +1 -0
- package/dist/ingest/adapters/registry.js +16 -0
- package/dist/ingest/adapters/registry.js.map +1 -0
- package/dist/ingest/adapters/types.d.ts +28 -0
- package/dist/ingest/adapters/types.d.ts.map +1 -0
- package/dist/ingest/adapters/types.js +5 -0
- package/dist/ingest/adapters/types.js.map +1 -0
- package/dist/logger.d.ts +12 -0
- package/dist/logger.d.ts.map +1 -0
- package/dist/logger.js +14 -0
- package/dist/logger.js.map +1 -0
- package/dist/mask/defaults.d.ts +7 -0
- package/dist/mask/defaults.d.ts.map +1 -0
- package/dist/mask/defaults.js +51 -0
- package/dist/mask/defaults.js.map +1 -0
- package/dist/mask/paths.d.ts +7 -0
- package/dist/mask/paths.d.ts.map +1 -0
- package/dist/mask/paths.js +36 -0
- package/dist/mask/paths.js.map +1 -0
- package/dist/util/payload-shape.d.ts +6 -0
- package/dist/util/payload-shape.d.ts.map +1 -0
- package/dist/util/payload-shape.js +19 -0
- package/dist/util/payload-shape.js.map +1 -0
- package/dist/util/strings.d.ts +3 -0
- package/dist/util/strings.d.ts.map +1 -0
- package/dist/util/strings.js +7 -0
- package/dist/util/strings.js.map +1 -0
- package/package.json +57 -0
- package/src/config/schema.ts +177 -0
- package/src/crypto/at-rest.ts +21 -0
- package/src/crypto/keys.ts +13 -0
- package/src/engine/codec.ts +30 -0
- package/src/engine/hooks.ts +76 -0
- package/src/engine/ingest.ts +194 -0
- package/src/engine/memory-store.ts +232 -0
- package/src/engine/relay.ts +294 -0
- package/src/engine/secrets.ts +51 -0
- package/src/engine/settings.ts +42 -0
- package/src/engine/store.ts +188 -0
- package/src/forward/backoff.ts +10 -0
- package/src/forward/dispatcher.ts +504 -0
- package/src/forward/endclose-client.ts +102 -0
- package/src/forward/enrich.ts +106 -0
- package/src/forward/mapper.ts +191 -0
- package/src/index.ts +47 -0
- package/src/ingest/adapters/generic-hmac.ts +70 -0
- package/src/ingest/adapters/payabli.ts +67 -0
- package/src/ingest/adapters/registry.ts +21 -0
- package/src/ingest/adapters/types.ts +33 -0
- package/src/logger.ts +26 -0
- package/src/mask/defaults.ts +56 -0
- package/src/mask/paths.ts +36 -0
- package/src/util/payload-shape.ts +19 -0
- package/src/util/strings.ts +7 -0
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// Tuning knobs shared by createRelay() and the Dispatcher. One shape, camelCase.
|
|
2
|
+
export const DEFAULT_DISPATCH = {
|
|
3
|
+
batchMax: 100,
|
|
4
|
+
pollIntervalMs: 250,
|
|
5
|
+
backoffBaseMs: 1000,
|
|
6
|
+
backoffCapMs: 600_000,
|
|
7
|
+
parkAfterMs: 7 * 24 * 3600 * 1000,
|
|
8
|
+
leaseMs: 600_000,
|
|
9
|
+
recoverIntervalMs: 60_000,
|
|
10
|
+
enrichTimeoutMs: 5_000,
|
|
11
|
+
};
|
|
12
|
+
export const DEFAULT_RETENTION = { deliveredDays: 7, ledgerDays: 30 };
|
|
13
|
+
//# sourceMappingURL=settings.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"settings.js","sourceRoot":"","sources":["../../src/engine/settings.ts"],"names":[],"mappings":"AAAA,iFAAiF;AA8BjF,MAAM,CAAC,MAAM,gBAAgB,GAAqB;IAChD,QAAQ,EAAE,GAAG;IACb,cAAc,EAAE,GAAG;IACnB,aAAa,EAAE,IAAI;IACnB,YAAY,EAAE,OAAO;IACrB,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI;IACjC,OAAO,EAAE,OAAO;IAChB,iBAAiB,EAAE,MAAM;IACzB,eAAe,EAAE,KAAK;CACvB,CAAA;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAsB,EAAE,aAAa,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,CAAA"}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import type { RouteConfig } from '../config/schema.js';
|
|
2
|
+
export type EventStatus = 'pending' | 'retry' | 'delivering' | 'delivered' | 'parked' | 'dropped_by_filter';
|
|
3
|
+
export interface EventRecord {
|
|
4
|
+
/** Opaque store-assigned id. */
|
|
5
|
+
id: string;
|
|
6
|
+
route_id: string;
|
|
7
|
+
source: string;
|
|
8
|
+
event_id: string;
|
|
9
|
+
event_type: string | null;
|
|
10
|
+
/** Whatever the PayloadCodec produced; opaque to the store. */
|
|
11
|
+
payload: Buffer;
|
|
12
|
+
payload_iv: Buffer | null;
|
|
13
|
+
headers_json: string;
|
|
14
|
+
received_at: string;
|
|
15
|
+
status: EventStatus;
|
|
16
|
+
attempts: number;
|
|
17
|
+
next_attempt_at: string | null;
|
|
18
|
+
delivered_at: string | null;
|
|
19
|
+
bulk_request_id: string | null;
|
|
20
|
+
last_error: string | null;
|
|
21
|
+
idempotency_key: string;
|
|
22
|
+
claimed_by: string | null;
|
|
23
|
+
lease_until: string | null;
|
|
24
|
+
}
|
|
25
|
+
export interface NewEvent {
|
|
26
|
+
route_id: string;
|
|
27
|
+
source: string;
|
|
28
|
+
event_id: string;
|
|
29
|
+
event_type: string | null;
|
|
30
|
+
payload: Buffer;
|
|
31
|
+
payload_iv: Buffer | null;
|
|
32
|
+
headers_json: string;
|
|
33
|
+
received_at: string;
|
|
34
|
+
status: EventStatus;
|
|
35
|
+
idempotency_key: string;
|
|
36
|
+
}
|
|
37
|
+
export interface RouteStats {
|
|
38
|
+
route_id: string;
|
|
39
|
+
counts: Partial<Record<EventStatus, number>>;
|
|
40
|
+
last_delivered_at: string | null;
|
|
41
|
+
oldest_pending_at: string | null;
|
|
42
|
+
}
|
|
43
|
+
export type EventSummary = Omit<EventRecord, 'payload' | 'payload_iv' | 'headers_json' | 'idempotency_key' | 'claimed_by' | 'lease_until'>;
|
|
44
|
+
export type InsertResult = {
|
|
45
|
+
duplicate: false;
|
|
46
|
+
id: string;
|
|
47
|
+
} | {
|
|
48
|
+
duplicate: true;
|
|
49
|
+
};
|
|
50
|
+
/** A store operation failed. `op` names the operation for logs and hooks. */
|
|
51
|
+
export declare class StoreError extends Error {
|
|
52
|
+
readonly op: string;
|
|
53
|
+
constructor(message: string, op: string, options?: {
|
|
54
|
+
cause?: unknown;
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* The store is temporarily unable to serve a request (lock contention, connection
|
|
59
|
+
* loss). Ingest answers 503 so the processor retries; any other StoreError is 500.
|
|
60
|
+
*/
|
|
61
|
+
export declare class StoreUnavailableError extends StoreError {
|
|
62
|
+
constructor(message: string, op: string, options?: {
|
|
63
|
+
cause?: unknown;
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
export declare function storeOp(err: unknown): string | undefined;
|
|
67
|
+
/** What the engine needs from event storage. */
|
|
68
|
+
export interface EventStore {
|
|
69
|
+
/** Persist a new event; `duplicate: true` when the idempotency key already exists. */
|
|
70
|
+
insert(e: NewEvent): Promise<InsertResult>;
|
|
71
|
+
/**
|
|
72
|
+
* Routes with `pending`/`retry` events whose next attempt is at or before `now`.
|
|
73
|
+
* Called with a far-future `now` it answers "which routes hold any backlog at all".
|
|
74
|
+
*/
|
|
75
|
+
routesWithDueEvents(now: string): Promise<string[]>;
|
|
76
|
+
/**
|
|
77
|
+
* Atomically select up to `limit` due events for a route (oldest first), mark them
|
|
78
|
+
* `delivering` and record the lease. Concurrent callers must never receive overlapping
|
|
79
|
+
* rows (a SQL store would use SELECT ... FOR UPDATE SKIP LOCKED).
|
|
80
|
+
*/
|
|
81
|
+
claimDue(routeId: string, now: string, limit: number, lease: Lease): Promise<EventRecord[]>;
|
|
82
|
+
markDelivered(ids: string[], deliveredAt: string, bulkRequestId: string | null): Promise<void>;
|
|
83
|
+
markFailed(ids: string[], nextAttemptAt: string, error: string): Promise<void>;
|
|
84
|
+
markParked(ids: string[], error: string): Promise<void>;
|
|
85
|
+
/** Return the given rows to `retry` if (and only if) they are still `delivering`. */
|
|
86
|
+
releaseDelivering(ids: string[], nextAttemptAt: string, error: string): Promise<number>;
|
|
87
|
+
/**
|
|
88
|
+
* Return `delivering` rows whose lease has expired — or that were claimed by `owner`,
|
|
89
|
+
* so a restarted instance reclaims its own work immediately — to `retry`.
|
|
90
|
+
*/
|
|
91
|
+
recoverDelivering(now: string, owner?: string): Promise<number>;
|
|
92
|
+
/** Park events that have been retrying longer than `maxAgeMs`. */
|
|
93
|
+
parkExpired(now: string, maxAgeMs: number): Promise<number>;
|
|
94
|
+
/** One bounded retention step; callers loop until it returns zeros. */
|
|
95
|
+
pruneBatch(now: string, deliveredDays: number, ledgerDays: number, limit: number): Promise<{
|
|
96
|
+
wiped: number;
|
|
97
|
+
deleted: number;
|
|
98
|
+
}>;
|
|
99
|
+
close?(): Promise<void>;
|
|
100
|
+
}
|
|
101
|
+
export interface Lease {
|
|
102
|
+
owner: string;
|
|
103
|
+
until: string;
|
|
104
|
+
}
|
|
105
|
+
/** Optional inspection/replay capability used by an operator surface. */
|
|
106
|
+
export interface EventStoreAdmin {
|
|
107
|
+
getById(id: string): Promise<EventRecord | undefined>;
|
|
108
|
+
list(filter: {
|
|
109
|
+
status?: EventStatus;
|
|
110
|
+
route?: string;
|
|
111
|
+
limit?: number;
|
|
112
|
+
}): Promise<EventSummary[]>;
|
|
113
|
+
countByStatus(): Promise<Record<string, number>>;
|
|
114
|
+
perRouteStats(): Promise<RouteStats[]>;
|
|
115
|
+
replay(id: string): Promise<boolean>;
|
|
116
|
+
replayAllParked(): Promise<number>;
|
|
117
|
+
}
|
|
118
|
+
export declare function hasAdmin(store: EventStore): store is EventStore & EventStoreAdmin;
|
|
119
|
+
export type Killswitch = 'none' | 'pause' | 'panic';
|
|
120
|
+
/** Runtime-mutable operator state: the global killswitch and per-route pauses. */
|
|
121
|
+
export interface ControlStore {
|
|
122
|
+
getKillswitch(): Promise<Killswitch>;
|
|
123
|
+
setKillswitch(state: Killswitch): Promise<void>;
|
|
124
|
+
isRoutePaused(routeId: string): Promise<boolean>;
|
|
125
|
+
setRoutePaused(routeId: string, paused: boolean): Promise<void>;
|
|
126
|
+
}
|
|
127
|
+
/** Where the engine reads route definitions. Static for an embedding app; live for the application. */
|
|
128
|
+
export interface RouteProvider {
|
|
129
|
+
get(id: string): Promise<RouteConfig | undefined>;
|
|
130
|
+
all(): Promise<RouteConfig[]>;
|
|
131
|
+
}
|
|
132
|
+
export declare function staticRoutes(routes: RouteConfig[]): RouteProvider;
|
|
133
|
+
export declare class MemoryControlStore implements ControlStore {
|
|
134
|
+
private killswitch;
|
|
135
|
+
private paused;
|
|
136
|
+
getKillswitch(): Promise<Killswitch>;
|
|
137
|
+
setKillswitch(state: Killswitch): Promise<void>;
|
|
138
|
+
isRoutePaused(routeId: string): Promise<boolean>;
|
|
139
|
+
setRoutePaused(routeId: string, paused: boolean): Promise<void>;
|
|
140
|
+
}
|
|
141
|
+
//# sourceMappingURL=store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/engine/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AAMtD,MAAM,MAAM,WAAW,GACnB,SAAS,GACT,OAAO,GACP,YAAY,GACZ,WAAW,GACX,QAAQ,GACR,mBAAmB,CAAA;AAEvB,MAAM,WAAW,WAAW;IAC1B,gCAAgC;IAChC,EAAE,EAAE,MAAM,CAAA;IACV,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;IACzB,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAA;IACf,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;IACzB,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,MAAM,EAAE,WAAW,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;IACzB,eAAe,EAAE,MAAM,CAAA;IACvB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;CAC3B;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;IACzB,OAAO,EAAE,MAAM,CAAA;IACf,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;IACzB,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,MAAM,EAAE,WAAW,CAAA;IACnB,eAAe,EAAE,MAAM,CAAA;CACxB;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAA;IAC5C,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAA;IAChC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAA;CACjC;AAED,MAAM,MAAM,YAAY,GAAG,IAAI,CAC7B,WAAW,EACX,SAAS,GAAG,YAAY,GAAG,cAAc,GAAG,iBAAiB,GAAG,YAAY,GAAG,aAAa,CAC7F,CAAA;AAED,MAAM,MAAM,YAAY,GAAG;IAAE,SAAS,EAAE,KAAK,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,SAAS,EAAE,IAAI,CAAA;CAAE,CAAA;AAEjF,6EAA6E;AAC7E,qBAAa,UAAW,SAAQ,KAAK;IAGjC,QAAQ,CAAC,EAAE,EAAE,MAAM;gBADnB,OAAO,EAAE,MAAM,EACN,EAAE,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAKhC;AAED;;;GAGG;AACH,qBAAa,qBAAsB,SAAQ,UAAU;gBACvC,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAIvE;AAED,wBAAgB,OAAO,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAExD;AAED,gDAAgD;AAChD,MAAM,WAAW,UAAU;IACzB,sFAAsF;IACtF,MAAM,CAAC,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IAC1C;;;OAGG;IACH,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IACnD;;;;OAIG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAA;IAC3F,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC9F,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC9E,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACvD,qFAAqF;IACrF,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IACvF;;;OAGG;IACH,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IAC/D,kEAAkE;IAClE,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IAC3D,uEAAuE;IACvE,UAAU,CACR,GAAG,EAAE,MAAM,EACX,aAAa,EAAE,MAAM,EACrB,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC9C,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACxB;AAED,MAAM,WAAW,KAAK;IACpB,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;CACd;AAED,yEAAyE;AACzE,MAAM,WAAW,eAAe;IAC9B,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAAA;IACrD,IAAI,CAAC,MAAM,EAAE;QAAE,MAAM,CAAC,EAAE,WAAW,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;IAC/F,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IAChD,aAAa,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;IACtC,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;IACpC,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC,CAAA;CACnC;AAED,wBAAgB,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,KAAK,IAAI,UAAU,GAAG,eAAe,CAEjF;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,OAAO,GAAG,OAAO,CAAA;AAEnD,kFAAkF;AAClF,MAAM,WAAW,YAAY;IAC3B,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC,CAAA;IACpC,aAAa,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/C,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;IAChD,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAChE;AAED,uGAAuG;AACvG,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAAA;IACjD,GAAG,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAAA;CAC9B;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,WAAW,EAAE,GAAG,aAAa,CAMjE;AAED,qBAAa,kBAAmB,YAAW,YAAY;IACrD,OAAO,CAAC,UAAU,CAAqB;IACvC,OAAO,CAAC,MAAM,CAAoB;IAC5B,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC;IAGpC,aAAa,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAG/C,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAGhD,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;CAItE"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/** A store operation failed. `op` names the operation for logs and hooks. */
|
|
2
|
+
export class StoreError extends Error {
|
|
3
|
+
op;
|
|
4
|
+
constructor(message, op, options) {
|
|
5
|
+
super(message, options);
|
|
6
|
+
this.op = op;
|
|
7
|
+
this.name = 'StoreError';
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* The store is temporarily unable to serve a request (lock contention, connection
|
|
12
|
+
* loss). Ingest answers 503 so the processor retries; any other StoreError is 500.
|
|
13
|
+
*/
|
|
14
|
+
export class StoreUnavailableError extends StoreError {
|
|
15
|
+
constructor(message, op, options) {
|
|
16
|
+
super(message, op, options);
|
|
17
|
+
this.name = 'StoreUnavailableError';
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
export function storeOp(err) {
|
|
21
|
+
return err instanceof StoreError ? err.op : undefined;
|
|
22
|
+
}
|
|
23
|
+
export function hasAdmin(store) {
|
|
24
|
+
return typeof store.list === 'function';
|
|
25
|
+
}
|
|
26
|
+
export function staticRoutes(routes) {
|
|
27
|
+
const byId = new Map(routes.map((r) => [r.id, r]));
|
|
28
|
+
return {
|
|
29
|
+
get: async (id) => byId.get(id),
|
|
30
|
+
all: async () => [...byId.values()],
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
export class MemoryControlStore {
|
|
34
|
+
killswitch = 'none';
|
|
35
|
+
paused = new Set();
|
|
36
|
+
async getKillswitch() {
|
|
37
|
+
return this.killswitch;
|
|
38
|
+
}
|
|
39
|
+
async setKillswitch(state) {
|
|
40
|
+
this.killswitch = state;
|
|
41
|
+
}
|
|
42
|
+
async isRoutePaused(routeId) {
|
|
43
|
+
return this.paused.has(routeId);
|
|
44
|
+
}
|
|
45
|
+
async setRoutePaused(routeId, paused) {
|
|
46
|
+
if (paused)
|
|
47
|
+
this.paused.add(routeId);
|
|
48
|
+
else
|
|
49
|
+
this.paused.delete(routeId);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/engine/store.ts"],"names":[],"mappings":"AAgEA,6EAA6E;AAC7E,MAAM,OAAO,UAAW,SAAQ,KAAK;IAGxB;IAFX,YACE,OAAe,EACN,EAAU,EACnB,OAA6B;QAE7B,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QAHd,OAAE,GAAF,EAAE,CAAQ;QAInB,IAAI,CAAC,IAAI,GAAG,YAAY,CAAA;IAC1B,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,OAAO,qBAAsB,SAAQ,UAAU;IACnD,YAAY,OAAe,EAAE,EAAU,EAAE,OAA6B;QACpE,KAAK,CAAC,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,CAAA;QAC3B,IAAI,CAAC,IAAI,GAAG,uBAAuB,CAAA;IACrC,CAAC;CACF;AAED,MAAM,UAAU,OAAO,CAAC,GAAY;IAClC,OAAO,GAAG,YAAY,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAA;AACvD,CAAC;AAsDD,MAAM,UAAU,QAAQ,CAAC,KAAiB;IACxC,OAAO,OAAQ,KAAkC,CAAC,IAAI,KAAK,UAAU,CAAA;AACvE,CAAC;AAkBD,MAAM,UAAU,YAAY,CAAC,MAAqB;IAChD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IAClD,OAAO;QACL,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/B,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;KACpC,CAAA;AACH,CAAC;AAED,MAAM,OAAO,kBAAkB;IACrB,UAAU,GAAe,MAAM,CAAA;IAC/B,MAAM,GAAG,IAAI,GAAG,EAAU,CAAA;IAClC,KAAK,CAAC,aAAa;QACjB,OAAO,IAAI,CAAC,UAAU,CAAA;IACxB,CAAC;IACD,KAAK,CAAC,aAAa,CAAC,KAAiB;QACnC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAA;IACzB,CAAC;IACD,KAAK,CAAC,aAAa,CAAC,OAAe;QACjC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IACjC,CAAC;IACD,KAAK,CAAC,cAAc,CAAC,OAAe,EAAE,MAAe;QACnD,IAAI,MAAM;YAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;;YAC/B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAClC,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backoff.d.ts","sourceRoot":"","sources":["../../src/forward/backoff.ts"],"names":[],"mappings":"AACA,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,SAAO,EAAE,KAAK,SAAU,GAAG,MAAM,CAIlF;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAEvF"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// Exponential backoff with ±20% jitter, Sequin-style: base * 2^attempts, capped.
|
|
2
|
+
export function backoffMs(attempts, baseMs = 1000, capMs = 600_000) {
|
|
3
|
+
const raw = Math.min(baseMs * 2 ** Math.min(attempts, 30), capMs);
|
|
4
|
+
const jitter = 0.8 + Math.random() * 0.4;
|
|
5
|
+
return Math.round(raw * jitter);
|
|
6
|
+
}
|
|
7
|
+
export function nextAttemptAt(attempts, baseMs, capMs) {
|
|
8
|
+
return new Date(Date.now() + backoffMs(attempts, baseMs, capMs)).toISOString();
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=backoff.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backoff.js","sourceRoot":"","sources":["../../src/forward/backoff.ts"],"names":[],"mappings":"AAAA,iFAAiF;AACjF,MAAM,UAAU,SAAS,CAAC,QAAgB,EAAE,MAAM,GAAG,IAAI,EAAE,KAAK,GAAG,OAAO;IACxE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,KAAK,CAAC,CAAA;IACjE,MAAM,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAA;IACxC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,CAAA;AACjC,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAgB,EAAE,MAAe,EAAE,KAAc;IAC7E,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE,CAAA;AAChF,CAAC"}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import type { EventEmitter } from 'node:events';
|
|
2
|
+
import { type ControlStore, type EventStore, type RouteProvider } from '../engine/store.js';
|
|
3
|
+
import type { PayloadCodec } from '../engine/codec.js';
|
|
4
|
+
import type { DispatchSettings, RetentionSettings } from '../engine/settings.js';
|
|
5
|
+
import { RelayHooks } from '../engine/hooks.js';
|
|
6
|
+
import { type Logger } from '../logger.js';
|
|
7
|
+
import { type Enrichment } from './enrich.js';
|
|
8
|
+
import { EndCloseClient } from './endclose-client.js';
|
|
9
|
+
export interface DispatcherDeps {
|
|
10
|
+
store: EventStore;
|
|
11
|
+
control: ControlStore;
|
|
12
|
+
routes: RouteProvider;
|
|
13
|
+
dispatch: DispatchSettings;
|
|
14
|
+
/** null disables retention pruning. */
|
|
15
|
+
retention: RetentionSettings | null;
|
|
16
|
+
client: EndCloseClient;
|
|
17
|
+
codec: PayloadCodec;
|
|
18
|
+
maskingKey: Buffer;
|
|
19
|
+
/** Lease owner recorded on claimed rows; a stable id lets a restarted instance reclaim its own. */
|
|
20
|
+
instanceId: string;
|
|
21
|
+
signal: EventEmitter;
|
|
22
|
+
hooks?: RelayHooks;
|
|
23
|
+
logger?: Logger;
|
|
24
|
+
/** Host functions routes may name with `enrich:`; run per event after mapping. */
|
|
25
|
+
enrichments?: Record<string, Enrichment>;
|
|
26
|
+
}
|
|
27
|
+
export interface DispatchCounts {
|
|
28
|
+
delivered: number;
|
|
29
|
+
retried: number;
|
|
30
|
+
parked: number;
|
|
31
|
+
}
|
|
32
|
+
/** What one dispatch cycle did, and what it deliberately left alone. */
|
|
33
|
+
export interface CycleSummary extends DispatchCounts {
|
|
34
|
+
/** Routes that had due events when the cycle started. */
|
|
35
|
+
due: string[];
|
|
36
|
+
/** Due routes the cycle did not touch, and why. */
|
|
37
|
+
skipped: {
|
|
38
|
+
routeId: string;
|
|
39
|
+
reason: 'paused' | 'unknown_route';
|
|
40
|
+
}[];
|
|
41
|
+
/** True when the global killswitch stopped the cycle before any forwarding. */
|
|
42
|
+
halted: boolean;
|
|
43
|
+
}
|
|
44
|
+
export declare class Dispatcher {
|
|
45
|
+
private deps;
|
|
46
|
+
private log;
|
|
47
|
+
private hooks;
|
|
48
|
+
private running;
|
|
49
|
+
private wakeRequested;
|
|
50
|
+
private reclaimOwner;
|
|
51
|
+
private lastSweepAt;
|
|
52
|
+
private inFlight;
|
|
53
|
+
private pruneWork;
|
|
54
|
+
private timer;
|
|
55
|
+
private pruneTimer;
|
|
56
|
+
private stopped;
|
|
57
|
+
constructor(deps: DispatcherDeps);
|
|
58
|
+
start(): void;
|
|
59
|
+
private onSignal;
|
|
60
|
+
/**
|
|
61
|
+
* Run exactly one dispatch cycle, serialised with the background loop if it is running.
|
|
62
|
+
* Errors propagate to the caller (they are also emitted as hooks).
|
|
63
|
+
*/
|
|
64
|
+
runOnce(): Promise<CycleSummary>;
|
|
65
|
+
/** Run retention pruning to completion, serialised with any scheduled prune. */
|
|
66
|
+
pruneNow(): Promise<{
|
|
67
|
+
wiped: number;
|
|
68
|
+
deleted: number;
|
|
69
|
+
}>;
|
|
70
|
+
/** Stop accepting new work and wait for the in-flight cycle and prune to drain. */
|
|
71
|
+
stop(): Promise<void>;
|
|
72
|
+
private wake;
|
|
73
|
+
private reportCycleError;
|
|
74
|
+
private cycle;
|
|
75
|
+
private deliverBatch;
|
|
76
|
+
/**
|
|
77
|
+
* Fill the fields mapEvent left to the host, one enrichment at a time. Throws
|
|
78
|
+
* EnrichmentError for a bad reference or return value (park); rethrows anything else
|
|
79
|
+
* the host threw, including a timeout (retry).
|
|
80
|
+
*/
|
|
81
|
+
private resolveEnrichments;
|
|
82
|
+
/** Schedule one event for redelivery with backoff, leaving the rest of its batch alone. */
|
|
83
|
+
private retryOne;
|
|
84
|
+
private postWithRetries;
|
|
85
|
+
/**
|
|
86
|
+
* Bulk processing is async server-side: poll briefly for per-row results. Rows the API
|
|
87
|
+
* rejected park; everything else is delivered. If results don't settle in time, mark
|
|
88
|
+
* delivered with the bulk_request_id recorded for later inspection.
|
|
89
|
+
*/
|
|
90
|
+
private settleResults;
|
|
91
|
+
private recordDelivered;
|
|
92
|
+
private recordParked;
|
|
93
|
+
private schedulePrune;
|
|
94
|
+
/** Bounded batches with a yield between them so ingest and dispatch keep running. */
|
|
95
|
+
private runPrune;
|
|
96
|
+
private releaseLeftover;
|
|
97
|
+
}
|
|
98
|
+
//# sourceMappingURL=dispatcher.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dispatcher.d.ts","sourceRoot":"","sources":["../../src/forward/dispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC/C,OAAO,EAEL,KAAK,YAAY,EAEjB,KAAK,UAAU,EACf,KAAK,aAAa,EACnB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AAGhF,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAA;AAC/C,OAAO,EAAc,KAAK,MAAM,EAAE,MAAM,cAAc,CAAA;AAKtD,OAAO,EAKL,KAAK,UAAU,EAChB,MAAM,aAAa,CAAA;AACpB,OAAO,EACL,cAAc,EAIf,MAAM,sBAAsB,CAAA;AAS7B,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,UAAU,CAAA;IACjB,OAAO,EAAE,YAAY,CAAA;IACrB,MAAM,EAAE,aAAa,CAAA;IACrB,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,uCAAuC;IACvC,SAAS,EAAE,iBAAiB,GAAG,IAAI,CAAA;IACnC,MAAM,EAAE,cAAc,CAAA;IACtB,KAAK,EAAE,YAAY,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;IAClB,mGAAmG;IACnG,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,YAAY,CAAA;IACpB,KAAK,CAAC,EAAE,UAAU,CAAA;IAClB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,kFAAkF;IAClF,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;CACzC;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;CACf;AAED,wEAAwE;AACxE,MAAM,WAAW,YAAa,SAAQ,cAAc;IAClD,yDAAyD;IACzD,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,mDAAmD;IACnD,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,QAAQ,GAAG,eAAe,CAAA;KAAE,EAAE,CAAA;IAClE,+EAA+E;IAC/E,MAAM,EAAE,OAAO,CAAA;CAChB;AAID,qBAAa,UAAU;IAgBT,OAAO,CAAC,IAAI;IAfxB,OAAO,CAAC,GAAG,CAAQ;IACnB,OAAO,CAAC,KAAK,CAAY;IACzB,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,aAAa,CAAQ;IAI7B,OAAO,CAAC,YAAY,CAAoB;IACxC,OAAO,CAAC,WAAW,CAAI;IACvB,OAAO,CAAC,QAAQ,CAAmC;IACnD,OAAO,CAAC,SAAS,CAAsC;IACvD,OAAO,CAAC,KAAK,CAA4B;IACzC,OAAO,CAAC,UAAU,CAA4B;IAC9C,OAAO,CAAC,OAAO,CAA6B;gBAExB,IAAI,EAAE,cAAc;IAMxC,KAAK,IAAI,IAAI;IAYb,OAAO,CAAC,QAAQ,CAAoB;IAEpC;;;OAGG;IACH,OAAO,IAAI,OAAO,CAAC,YAAY,CAAC;IAShC,gFAAgF;IAChF,QAAQ,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAMvD,mFAAmF;IAC7E,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAS3B,OAAO,CAAC,IAAI;IAeZ,OAAO,CAAC,gBAAgB;YAMV,KAAK;YA4DL,YAAY;IAgF1B;;;;OAIG;YACW,kBAAkB;IA+ChC,2FAA2F;YAC7E,QAAQ;YAcR,eAAe;IAc7B;;;;OAIG;YACW,aAAa;IAoC3B,OAAO,CAAC,eAAe;IAUvB,OAAO,CAAC,YAAY;IASpB,OAAO,CAAC,aAAa;IAKrB,qFAAqF;YACvE,QAAQ;YA6BR,eAAe;CAgB9B"}
|