@things-factory/webhook 10.1.115 → 10.1.121
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-server/service/index.d.ts +5 -2
- package/dist-server/service/index.js +4 -1
- package/dist-server/service/index.js.map +1 -1
- package/dist-server/service/outbox/landed-probe.d.ts +26 -0
- package/dist-server/service/outbox/landed-probe.js +63 -0
- package/dist-server/service/outbox/landed-probe.js.map +1 -0
- package/dist-server/service/outbox/pump-loop.d.ts +9 -5
- package/dist-server/service/outbox/pump-loop.js +18 -34
- package/dist-server/service/outbox/pump-loop.js.map +1 -1
- package/dist-server/service/outbox/pump.d.ts +10 -0
- package/dist-server/service/outbox/pump.js +12 -0
- package/dist-server/service/outbox/pump.js.map +1 -1
- package/dist-server/service/webhook-destination/webhook-destination-resolver.d.ts +20 -0
- package/dist-server/service/webhook-destination/webhook-destination-resolver.js +30 -23
- package/dist-server/service/webhook-destination/webhook-destination-resolver.js.map +1 -1
- package/dist-server/service/webhook-outbox-event/webhook-outbox-resolver.d.ts +67 -0
- package/dist-server/service/webhook-outbox-event/webhook-outbox-resolver.js +338 -0
- package/dist-server/service/webhook-outbox-event/webhook-outbox-resolver.js.map +1 -0
- package/dist-server/tsconfig.tsbuildinfo +1 -1
- package/package.json +5 -5
|
@@ -1,20 +1,23 @@
|
|
|
1
1
|
import { WebhookDestination } from './webhook-destination/webhook-destination';
|
|
2
2
|
import { WebhookDestinationResolver } from './webhook-destination/webhook-destination-resolver';
|
|
3
3
|
import { WebhookOutboxEvent } from './webhook-outbox-event/webhook-outbox-event';
|
|
4
|
+
import { WebhookOutboxResolver } from './webhook-outbox-event/webhook-outbox-resolver';
|
|
4
5
|
import { WebhookSequence } from './webhook-sequence/webhook-sequence';
|
|
5
6
|
export * from './webhook-destination/webhook-destination';
|
|
6
7
|
export * from './webhook-destination/webhook-destination-resolver';
|
|
7
8
|
export * from './webhook-outbox-event/webhook-outbox-event';
|
|
9
|
+
export * from './webhook-outbox-event/webhook-outbox-resolver';
|
|
8
10
|
export * from './webhook-sequence/webhook-sequence';
|
|
9
|
-
export declare const entities: (typeof
|
|
11
|
+
export declare const entities: (typeof WebhookOutboxEvent | typeof WebhookSequence | typeof WebhookDestination)[];
|
|
10
12
|
export declare const schema: {
|
|
11
|
-
resolverClasses: (typeof WebhookDestinationResolver)[];
|
|
13
|
+
resolverClasses: (typeof WebhookOutboxResolver | typeof WebhookDestinationResolver)[];
|
|
12
14
|
};
|
|
13
15
|
export * from './outbox/delivery';
|
|
14
16
|
export * from './outbox/settlement';
|
|
15
17
|
export * from './outbox/row-effect';
|
|
16
18
|
export * from './outbox/seq-report';
|
|
17
19
|
export * from './outbox/reconciliation';
|
|
20
|
+
export * from './outbox/landed-probe';
|
|
18
21
|
export * from './outbox/standing';
|
|
19
22
|
export * from './outbox/requeue-note';
|
|
20
23
|
export * from './outbox/page';
|
|
@@ -14,10 +14,12 @@ const tslib_1 = require("tslib");
|
|
|
14
14
|
const webhook_destination_1 = require("./webhook-destination/webhook-destination");
|
|
15
15
|
const webhook_destination_resolver_1 = require("./webhook-destination/webhook-destination-resolver");
|
|
16
16
|
const webhook_outbox_event_1 = require("./webhook-outbox-event/webhook-outbox-event");
|
|
17
|
+
const webhook_outbox_resolver_1 = require("./webhook-outbox-event/webhook-outbox-resolver");
|
|
17
18
|
const webhook_sequence_1 = require("./webhook-sequence/webhook-sequence");
|
|
18
19
|
tslib_1.__exportStar(require("./webhook-destination/webhook-destination"), exports);
|
|
19
20
|
tslib_1.__exportStar(require("./webhook-destination/webhook-destination-resolver"), exports);
|
|
20
21
|
tslib_1.__exportStar(require("./webhook-outbox-event/webhook-outbox-event"), exports);
|
|
22
|
+
tslib_1.__exportStar(require("./webhook-outbox-event/webhook-outbox-resolver"), exports);
|
|
21
23
|
tslib_1.__exportStar(require("./webhook-sequence/webhook-sequence"), exports);
|
|
22
24
|
/*
|
|
23
25
|
* ⚠ **The shell builds the database and the schema from these two names.** Without them the module's
|
|
@@ -26,7 +28,7 @@ tslib_1.__exportStar(require("./webhook-sequence/webhook-sequence"), exports);
|
|
|
26
28
|
* append: its database had no webhook table at all). `test/append-store-db.test.ts` holds this line.
|
|
27
29
|
*/
|
|
28
30
|
exports.entities = [webhook_destination_1.WebhookDestination, webhook_outbox_event_1.WebhookOutboxEvent, webhook_sequence_1.WebhookSequence];
|
|
29
|
-
exports.schema = { resolverClasses: [webhook_destination_resolver_1.WebhookDestinationResolver] };
|
|
31
|
+
exports.schema = { resolverClasses: [webhook_destination_resolver_1.WebhookDestinationResolver, webhook_outbox_resolver_1.WebhookOutboxResolver] };
|
|
30
32
|
/*
|
|
31
33
|
* The outbox machine, moved from operato-plant (ADR-0078 decision 2). Step ①: the pure judgement —
|
|
32
34
|
* envelope, signature, reading the receiver's answer, settlement, what an answer does to our rows,
|
|
@@ -37,6 +39,7 @@ tslib_1.__exportStar(require("./outbox/settlement"), exports);
|
|
|
37
39
|
tslib_1.__exportStar(require("./outbox/row-effect"), exports);
|
|
38
40
|
tslib_1.__exportStar(require("./outbox/seq-report"), exports);
|
|
39
41
|
tslib_1.__exportStar(require("./outbox/reconciliation"), exports);
|
|
42
|
+
tslib_1.__exportStar(require("./outbox/landed-probe"), exports);
|
|
40
43
|
tslib_1.__exportStar(require("./outbox/standing"), exports);
|
|
41
44
|
tslib_1.__exportStar(require("./outbox/requeue-note"), exports);
|
|
42
45
|
tslib_1.__exportStar(require("./outbox/page"), exports);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../server/service/index.ts"],"names":[],"mappings":";;;;AAAA;;;;;;;;GAQG;AACH,mFAA8E;AAC9E,qGAA+F;AAC/F,sFAAgF;AAChF,0EAAqE;AAErE,oFAAyD;AACzD,6FAAkE;AAClE,sFAA2D;AAC3D,8EAAmD;AAEnD;;;;;GAKG;AACU,QAAA,QAAQ,GAAG,CAAC,wCAAkB,EAAE,yCAAkB,EAAE,kCAAe,CAAC,CAAA;AAEpE,QAAA,MAAM,GAAG,EAAE,eAAe,EAAE,CAAC,yDAA0B,CAAC,EAAE,CAAA;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../server/service/index.ts"],"names":[],"mappings":";;;;AAAA;;;;;;;;GAQG;AACH,mFAA8E;AAC9E,qGAA+F;AAC/F,sFAAgF;AAChF,4FAAsF;AACtF,0EAAqE;AAErE,oFAAyD;AACzD,6FAAkE;AAClE,sFAA2D;AAC3D,yFAA8D;AAC9D,8EAAmD;AAEnD;;;;;GAKG;AACU,QAAA,QAAQ,GAAG,CAAC,wCAAkB,EAAE,yCAAkB,EAAE,kCAAe,CAAC,CAAA;AAEpE,QAAA,MAAM,GAAG,EAAE,eAAe,EAAE,CAAC,yDAA0B,EAAE,+CAAqB,CAAC,EAAE,CAAA;AAE9F;;;;GAIG;AACH,4DAAiC;AACjC,8DAAmC;AACnC,8DAAmC;AACnC,8DAAmC;AACnC,kEAAuC;AACvC,gEAAqC;AACrC,4DAAiC;AACjC,gEAAqC;AACrC,wDAA6B;AAC7B,uEAA4C;AAE5C;;;GAGG;AACH,0DAA+B;AAE/B;+EAC+E;AAC/E,uEAA4C;AAC5C,wEAA6C;AAE7C,6GAA6G;AAC7G,yDAA8B;AAC9B,8DAAmC;AACnC,iEAAsC;AACtC,wEAA6C;AAE7C;;;;GAIG;AACH,8DAAmC;AACnC,wDAA6B;AAC7B,6DAAkC","sourcesContent":["/*\n * Each source file is named rather than re-exporting folder barrels.\n *\n * A name added to a folder barrel but not listed here stops at the package boundary, and the\n * consumer's error is `TS2305: has no exported member` — which reads like the name does not\n * exist rather than like it is unreachable. That happened on 2026-09-07 with\n * `work-shift/days-of-week`: the parser was in the folder barrel, not here, and the consuming\n * lane wrote its own until the miss was found.\n */\nimport { WebhookDestination } from './webhook-destination/webhook-destination'\nimport { WebhookDestinationResolver } from './webhook-destination/webhook-destination-resolver'\nimport { WebhookOutboxEvent } from './webhook-outbox-event/webhook-outbox-event'\nimport { WebhookOutboxResolver } from './webhook-outbox-event/webhook-outbox-resolver'\nimport { WebhookSequence } from './webhook-sequence/webhook-sequence'\n\nexport * from './webhook-destination/webhook-destination'\nexport * from './webhook-destination/webhook-destination-resolver'\nexport * from './webhook-outbox-event/webhook-outbox-event'\nexport * from './webhook-outbox-event/webhook-outbox-resolver'\nexport * from './webhook-sequence/webhook-sequence'\n\n/*\n * ⚠ **The shell builds the database and the schema from these two names.** Without them the module's\n * tables are never created in any application and the destination door is in no schema - the files stand\n * and nothing reaches them. That was the state through 10.1.37 (found 2026-09-24 by the first application to\n * append: its database had no webhook table at all). `test/append-store-db.test.ts` holds this line.\n */\nexport const entities = [WebhookDestination, WebhookOutboxEvent, WebhookSequence]\n\nexport const schema = { resolverClasses: [WebhookDestinationResolver, WebhookOutboxResolver] }\n\n/*\n * The outbox machine, moved from operato-plant (ADR-0078 decision 2). Step ①: the pure judgement —\n * envelope, signature, reading the receiver's answer, settlement, what an answer does to our rows,\n * the sequence report, reconciliation, standing. None of these touch a store.\n */\nexport * from './outbox/delivery'\nexport * from './outbox/settlement'\nexport * from './outbox/row-effect'\nexport * from './outbox/seq-report'\nexport * from './outbox/reconciliation'\nexport * from './outbox/landed-probe'\nexport * from './outbox/standing'\nexport * from './outbox/requeue-note'\nexport * from './outbox/page'\nexport * from './outbox/writing-transaction'\n\n/*\n * Step ②: the back row. Writing a fact down takes the caller's open transaction - in the type, not only\n * in a check - because the number and the row have to commit together or the number burns.\n */\nexport * from './outbox/append'\n\n/* What a delivery needs to say \"when\" and \"which request\" — moved from operato-plant with the dispatcher's\n * needs in mind (ADR-0078 decision 1). The inbound side reads the same two. */\nexport * from './observability/clock-offset'\nexport * from './observability/trace-context'\n\n/* Step ②: the back row. One dispatcher over a store interface; `webhookStore` is this module's own store. */\nexport * from './outbox/store'\nexport * from './outbox/dispatcher'\nexport * from './outbox/webhook-store'\nexport * from './outbox/webhook-append-store'\n\n/*\n * The pump: one round's judgement (`pump-round`), one round (`pump`), and the loop an application starts from its\n * boot hook (`pump-loop`). Keys come from the contract's ladder - the product-named `pump-gate` is gone\n * (ADR-0078 decision 5, re-ruled 2026-09-24).\n */\nexport * from './outbox/pump-round'\nexport * from './outbox/pump'\nexport * from './outbox/pump-loop'\n\n"]}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { type Reconciliation } from './reconciliation';
|
|
2
|
+
export interface LandedReport {
|
|
3
|
+
landed: number;
|
|
4
|
+
rejected?: number;
|
|
5
|
+
duplicates?: number;
|
|
6
|
+
vocabulary?: number;
|
|
7
|
+
}
|
|
8
|
+
export interface LandedDestination {
|
|
9
|
+
params?: {
|
|
10
|
+
landedEndpoint?: string;
|
|
11
|
+
source?: string;
|
|
12
|
+
} | null;
|
|
13
|
+
}
|
|
14
|
+
/** The receiver's own count for one stream, or `undefined` when it cannot be asked. */
|
|
15
|
+
export declare function askLanded(destination: LandedDestination | undefined, scope: string, fetcher?: typeof fetch): Promise<LandedReport | undefined>;
|
|
16
|
+
/**
|
|
17
|
+
* Sent against landed for one stream. `sent` and `rejected` are this side's counts; the receiver's own
|
|
18
|
+
* rejected count wins when it gives one. An error asking is reported to `onAskFailed` and reads as unknown.
|
|
19
|
+
*/
|
|
20
|
+
export declare function reconcileStream(destination: LandedDestination | undefined, scope: string, counts: {
|
|
21
|
+
sent: number;
|
|
22
|
+
rejected: number;
|
|
23
|
+
}, options?: {
|
|
24
|
+
fetcher?: typeof fetch;
|
|
25
|
+
onAskFailed?: (error: unknown) => void;
|
|
26
|
+
}): Promise<Reconciliation>;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.askLanded = askLanded;
|
|
4
|
+
exports.reconcileStream = reconcileStream;
|
|
5
|
+
const reconciliation_1 = require("./reconciliation");
|
|
6
|
+
/*
|
|
7
|
+
* Asking the receiver how many facts of a stream actually landed (ADR-0078 decision 1, moved from operato-plant).
|
|
8
|
+
*
|
|
9
|
+
* The receiver's count is the only evidence for "sent but never landed". Without it, sent and landed read as
|
|
10
|
+
* the same number and a silent loss looks like success.
|
|
11
|
+
*
|
|
12
|
+
* Where to ask is **declared, not derived**: the destination's `params.landedEndpoint`, with `params.source`
|
|
13
|
+
* passed along. The plant used to cut its endpoint at `/hook` and add `/landed` — a guess about one receiver's
|
|
14
|
+
* URL shape. A product that knows its receiver writes the address into the destination.
|
|
15
|
+
*
|
|
16
|
+
* It never says "matched" when it could not ask: no endpoint, an error or a timeout give `unknown`.
|
|
17
|
+
*/
|
|
18
|
+
const ASK_TIMEOUT_MS = 3_000;
|
|
19
|
+
function number(value) {
|
|
20
|
+
return Number.isFinite(value) ? Number(value) : undefined;
|
|
21
|
+
}
|
|
22
|
+
/** The receiver's own count for one stream, or `undefined` when it cannot be asked. */
|
|
23
|
+
async function askLanded(destination, scope, fetcher = fetch) {
|
|
24
|
+
const base = String(destination?.params?.landedEndpoint ?? '').trim();
|
|
25
|
+
if (!base)
|
|
26
|
+
return undefined;
|
|
27
|
+
const query = new URLSearchParams({ channel: scope });
|
|
28
|
+
const source = String(destination?.params?.source ?? '').trim();
|
|
29
|
+
if (source)
|
|
30
|
+
query.set('source', source);
|
|
31
|
+
const response = await fetcher(`${base}?${query}`, {
|
|
32
|
+
method: 'GET',
|
|
33
|
+
headers: { Accept: 'application/json' },
|
|
34
|
+
signal: AbortSignal.timeout(ASK_TIMEOUT_MS)
|
|
35
|
+
});
|
|
36
|
+
if (!response.ok)
|
|
37
|
+
return undefined;
|
|
38
|
+
const body = await response.json();
|
|
39
|
+
return Number.isFinite(body?.landed)
|
|
40
|
+
? { landed: Number(body.landed), rejected: number(body?.rejected), duplicates: number(body?.duplicates), vocabulary: number(body?.vocabulary) }
|
|
41
|
+
: undefined;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Sent against landed for one stream. `sent` and `rejected` are this side's counts; the receiver's own
|
|
45
|
+
* rejected count wins when it gives one. An error asking is reported to `onAskFailed` and reads as unknown.
|
|
46
|
+
*/
|
|
47
|
+
async function reconcileStream(destination, scope, counts, options = {}) {
|
|
48
|
+
let report;
|
|
49
|
+
try {
|
|
50
|
+
report = await askLanded(destination, scope, options.fetcher ?? fetch);
|
|
51
|
+
}
|
|
52
|
+
catch (error) {
|
|
53
|
+
options.onAskFailed?.(error);
|
|
54
|
+
}
|
|
55
|
+
return (0, reconciliation_1.reconcile)({
|
|
56
|
+
sent: counts.sent,
|
|
57
|
+
rejected: report?.rejected ?? counts.rejected,
|
|
58
|
+
duplicates: report?.duplicates,
|
|
59
|
+
vocabulary: report?.vocabulary,
|
|
60
|
+
landed: report?.landed
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
//# sourceMappingURL=landed-probe.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"landed-probe.js","sourceRoot":"","sources":["../../../server/service/outbox/landed-probe.ts"],"names":[],"mappings":";;AAiCA,8BAuBC;AAMD,0CAoBC;AAlFD,qDAAiE;AAEjE;;;;;;;;;;;GAWG;AAEH,MAAM,cAAc,GAAG,KAAK,CAAA;AAa5B,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAe,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;AACrE,CAAC;AAED,uFAAuF;AAChF,KAAK,UAAU,SAAS,CAC7B,WAA0C,EAC1C,KAAa,EACb,UAAwB,KAAK;IAE7B,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,cAAc,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IACrE,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAA;IAE3B,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAA;IACrD,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAC/D,IAAI,MAAM;QAAE,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IAEvC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,IAAI,IAAI,KAAK,EAAE,EAAE;QACjD,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE;QACvC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC;KAC5C,CAAC,CAAA;IACF,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,OAAO,SAAS,CAAA;IAElC,MAAM,IAAI,GAAQ,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;IACvC,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAClC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE;QAC/I,CAAC,CAAC,SAAS,CAAA;AACf,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,eAAe,CACnC,WAA0C,EAC1C,KAAa,EACb,MAA0C,EAC1C,UAA8E,EAAE;IAEhF,IAAI,MAAgC,CAAA;IACpC,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,SAAS,CAAC,WAAW,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC,CAAA;IACxE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAA;IAC9B,CAAC;IAED,OAAO,IAAA,0BAAS,EAAC;QACf,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,QAAQ,EAAE,MAAM,EAAE,QAAQ,IAAI,MAAM,CAAC,QAAQ;QAC7C,UAAU,EAAE,MAAM,EAAE,UAAU;QAC9B,UAAU,EAAE,MAAM,EAAE,UAAU;QAC9B,MAAM,EAAE,MAAM,EAAE,MAAM;KACvB,CAAC,CAAA;AACJ,CAAC","sourcesContent":["import { reconcile, type Reconciliation } from './reconciliation'\n\n/*\n * Asking the receiver how many facts of a stream actually landed (ADR-0078 decision 1, moved from operato-plant).\n *\n * The receiver's count is the only evidence for \"sent but never landed\". Without it, sent and landed read as\n * the same number and a silent loss looks like success.\n *\n * Where to ask is **declared, not derived**: the destination's `params.landedEndpoint`, with `params.source`\n * passed along. The plant used to cut its endpoint at `/hook` and add `/landed` — a guess about one receiver's\n * URL shape. A product that knows its receiver writes the address into the destination.\n *\n * It never says \"matched\" when it could not ask: no endpoint, an error or a timeout give `unknown`.\n */\n\nconst ASK_TIMEOUT_MS = 3_000\n\nexport interface LandedReport {\n landed: number\n rejected?: number\n duplicates?: number\n vocabulary?: number\n}\n\nexport interface LandedDestination {\n params?: { landedEndpoint?: string; source?: string } | null\n}\n\nfunction number(value: unknown): number | undefined {\n return Number.isFinite(value as number) ? Number(value) : undefined\n}\n\n/** The receiver's own count for one stream, or `undefined` when it cannot be asked. */\nexport async function askLanded(\n destination: LandedDestination | undefined,\n scope: string,\n fetcher: typeof fetch = fetch\n): Promise<LandedReport | undefined> {\n const base = String(destination?.params?.landedEndpoint ?? '').trim()\n if (!base) return undefined\n\n const query = new URLSearchParams({ channel: scope })\n const source = String(destination?.params?.source ?? '').trim()\n if (source) query.set('source', source)\n\n const response = await fetcher(`${base}?${query}`, {\n method: 'GET',\n headers: { Accept: 'application/json' },\n signal: AbortSignal.timeout(ASK_TIMEOUT_MS)\n })\n if (!response.ok) return undefined\n\n const body: any = await response.json()\n return Number.isFinite(body?.landed)\n ? { landed: Number(body.landed), rejected: number(body?.rejected), duplicates: number(body?.duplicates), vocabulary: number(body?.vocabulary) }\n : undefined\n}\n\n/**\n * Sent against landed for one stream. `sent` and `rejected` are this side's counts; the receiver's own\n * rejected count wins when it gives one. An error asking is reported to `onAskFailed` and reads as unknown.\n */\nexport async function reconcileStream(\n destination: LandedDestination | undefined,\n scope: string,\n counts: { sent: number; rejected: number },\n options: { fetcher?: typeof fetch; onAskFailed?: (error: unknown) => void } = {}\n): Promise<Reconciliation> {\n let report: LandedReport | undefined\n try {\n report = await askLanded(destination, scope, options.fetcher ?? fetch)\n } catch (error) {\n options.onAskFailed?.(error)\n }\n\n return reconcile({\n sent: counts.sent,\n rejected: report?.rejected ?? counts.rejected,\n duplicates: report?.duplicates,\n vocabulary: report?.vocabulary,\n landed: report?.landed\n })\n}\n"]}
|
|
@@ -11,12 +11,16 @@ import { type PumpSource } from './pump.js';
|
|
|
11
11
|
*
|
|
12
12
|
* ## One instance at a time, and one round at a time
|
|
13
13
|
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
* first reads at the receiver as a gap
|
|
14
|
+
* The loop is `startPeriodicTask` from `@things-factory/task-lease` (ADR-0110 adoption 2, step 3): a round never
|
|
15
|
+
* starts while the last is still sending, it runs under a lease so that two instances behind a load balancer
|
|
16
|
+
* do not both send (the later one arriving first reads at the receiver as a gap), the lease is renewed through
|
|
17
|
+
* a long round, and a round that throws backs off.
|
|
17
18
|
*
|
|
18
|
-
* **A paused pump keeps the lease
|
|
19
|
-
* nothing of it - takes over at once and knocks on the receiver that asked us to wait.
|
|
19
|
+
* **A paused pump keeps the lease** - returned as `holdMs`. The pause lives in this process. Let go, another
|
|
20
|
+
* instance - which knows nothing of it - takes over at once and knocks on the receiver that asked us to wait.
|
|
21
|
+
* It is `holdMs` and not `restMs` because a pause is about one stream: the streams that are not paused still
|
|
22
|
+
* go out from this instance at the usual interval. The pauses, their causes and their lengths stay in
|
|
23
|
+
* `pump.ts`; only the longest length reaches the loop.
|
|
20
24
|
*
|
|
21
25
|
* ## It stands without a key
|
|
22
26
|
*
|
|
@@ -11,11 +11,7 @@ const webhook_outbox_event_js_1 = require("../webhook-outbox-event/webhook-outbo
|
|
|
11
11
|
const pump_js_1 = require("./pump.js");
|
|
12
12
|
const webhook_store_js_1 = require("./webhook-store.js");
|
|
13
13
|
const DEFAULTS = { intervalSeconds: 5, batchSize: 200 };
|
|
14
|
-
const
|
|
15
|
-
/** Longer than a round, so it does not lapse mid-batch; short enough that a dead instance is replaced soon. */
|
|
16
|
-
function leaseTtlMs(intervalSeconds) {
|
|
17
|
-
return Math.max(60_000, intervalSeconds * 1000 * 6);
|
|
18
|
-
}
|
|
14
|
+
const TASK = 'webhook-outbox-pump';
|
|
19
15
|
/** The source over this module's tables. */
|
|
20
16
|
function webhookPumpSource(manager) {
|
|
21
17
|
return {
|
|
@@ -47,47 +43,35 @@ function webhookPumpSource(manager) {
|
|
|
47
43
|
}
|
|
48
44
|
};
|
|
49
45
|
}
|
|
50
|
-
let
|
|
51
|
-
let running = false;
|
|
46
|
+
let task;
|
|
52
47
|
let state = (0, pump_js_1.newPumpState)();
|
|
53
48
|
/** Start the loop. Starting twice does nothing. */
|
|
54
49
|
function startWebhookPump(options = {}) {
|
|
55
|
-
if (
|
|
50
|
+
if (task) {
|
|
56
51
|
return;
|
|
57
52
|
}
|
|
58
53
|
const intervalSeconds = options.intervalSeconds ?? DEFAULTS.intervalSeconds;
|
|
59
54
|
const batchSize = options.batchSize ?? DEFAULTS.batchSize;
|
|
60
55
|
const say = { info: (what) => env_1.logger.info(what), warn: (what) => env_1.logger.warn(what), error: (what) => env_1.logger.error(what) };
|
|
61
56
|
env_1.logger.info(`[webhook] sending queued facts every ${intervalSeconds}s`);
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
}
|
|
76
|
-
catch (error) {
|
|
77
|
-
/* A failed round must not stop the next one - stopped, facts pile up for good. */
|
|
78
|
-
env_1.logger.error(`[webhook] sending queued facts failed this round - ${String(error)}`);
|
|
79
|
-
}
|
|
80
|
-
finally {
|
|
81
|
-
running = false;
|
|
82
|
-
}
|
|
83
|
-
}, intervalSeconds * 1000);
|
|
57
|
+
task = (0, task_lease_1.startPeriodicTask)({
|
|
58
|
+
name: TASK,
|
|
59
|
+
intervalMs: intervalSeconds * 1000,
|
|
60
|
+
work: async () => {
|
|
61
|
+
await (0, pump_js_1.pumpOnce)({ source: webhookPumpSource((0, shell_1.getRepository)(webhook_outbox_event_js_1.WebhookOutboxEvent).manager), env: process.env, fetch, say }, state, { batchSize, ...(options.legacyBases?.length ? { legacyBases: options.legacyBases } : {}) });
|
|
62
|
+
return (0, pump_js_1.afterRound)(state, Date.now());
|
|
63
|
+
},
|
|
64
|
+
/* A failed round must not stop the next one - stopped, facts pile up for good. It backs off instead. */
|
|
65
|
+
onFailure: f => env_1.logger.error(`[webhook] sending queued facts failed (${f.consecutive} in a row since ${f.firstFailedAt}) - next try in ${Math.round(f.nextIntervalMs / 1000)}s - ${f.error}`),
|
|
66
|
+
onRecovered: r => env_1.logger.info(`[webhook] sending queued facts recovered after ${r.afterFailures} failed round(s)`),
|
|
67
|
+
onIntervalRaised: r => env_1.logger.warn(`[webhook] intervalSeconds raised from ${r.asked / 1000}s to ${r.used / 1000}s`),
|
|
68
|
+
onLeaseError: e => env_1.logger.error(`[webhook] pump lease "${e.name}" - ${e.error}`)
|
|
69
|
+
});
|
|
84
70
|
}
|
|
85
71
|
/** Stop the loop. For tests and shutdown - left running, the process does not end. */
|
|
86
72
|
function stopWebhookPump() {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
timer = undefined;
|
|
90
|
-
}
|
|
73
|
+
task?.stop();
|
|
74
|
+
task = undefined;
|
|
91
75
|
state = (0, pump_js_1.newPumpState)();
|
|
92
76
|
}
|
|
93
77
|
//# sourceMappingURL=pump-loop.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pump-loop.js","sourceRoot":"","sources":["../../../server/service/outbox/pump-loop.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"pump-loop.js","sourceRoot":"","sources":["../../../server/service/outbox/pump-loop.ts"],"names":[],"mappings":";;AAoDA,8CAgCC;AAMD,4CAgCC;AAGD,0CAIC;AA/HD,6CAA4C;AAC5C,iDAAqD;AACrD,2DAAiF;AAEjF,0FAAkF;AAClF,6FAAyG;AACzG,uCAA+F;AAE/F,yDAAiD;AAsCjD,MAAM,QAAQ,GAAG,EAAE,eAAe,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAA;AACvD,MAAM,IAAI,GAAG,qBAAqB,CAAA;AAElC,4CAA4C;AAC5C,SAAgB,iBAAiB,CAAC,OAAsB;IACtD,OAAO;QACL,KAAK,CAAC,cAAc;YAClB,MAAM,IAAI,GAAG,MAAM,OAAO;iBACvB,kBAAkB,CAAC,4CAAkB,EAAE,OAAO,CAAC;iBAC/C,SAAS,CAAC,cAAc,EAAE,QAAQ,CAAC;iBACnC,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC;iBAC/B,SAAS,CAAC,kBAAkB,EAAE,WAAW,CAAC;iBAC1C,SAAS,CAAC,YAAY,EAAE,MAAM,CAAC;iBAC/B,SAAS,CAAC,aAAa,EAAE,OAAO,CAAC;iBACjC,KAAK,CAAC,yBAAyB,EAAE,EAAE,OAAO,EAAE,6CAAmB,CAAC,OAAO,EAAE,CAAC;iBAC1E,OAAO,CAAC,WAAW,CAAC;iBACpB,UAAU,CAAC,kBAAkB,CAAC;iBAC9B,UAAU,CAAC,YAAY,CAAC;iBACxB,UAAU,CAAC,aAAa,CAAC;iBACzB,OAAO,CAAC,kBAAkB,CAAC;iBAC3B,UAAU,CAAC,YAAY,CAAC;iBACxB,UAAU,CAAC,aAAa,CAAC;iBACzB,UAAU,EAAE,CAAA;YAEf,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAClH,CAAC;QAED,KAAK,CAAC,aAAa,CAAC,QAAgB,EAAE,IAAY;YAChD,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,2CAAkB,EAAE,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;YACpG,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS,CAAA;QAC1G,CAAC;QAED,QAAQ,CAAC,MAAqB;YAC5B,OAAO,IAAA,+BAAY,EAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAA;QACrG,CAAC;KACF,CAAA;AACH,CAAC;AAED,IAAI,IAA8B,CAAA;AAClC,IAAI,KAAK,GAAc,IAAA,sBAAY,GAAE,CAAA;AAErC,mDAAmD;AACnD,SAAgB,gBAAgB,CAAC,UAA8B,EAAE;IAC/D,IAAI,IAAI,EAAE,CAAC;QACT,OAAM;IACR,CAAC;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,QAAQ,CAAC,eAAe,CAAA;IAC3E,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS,CAAA;IACzD,MAAM,GAAG,GAAG,EAAE,IAAI,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,YAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,YAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,YAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAA;IAEjJ,YAAM,CAAC,IAAI,CAAC,wCAAwC,eAAe,GAAG,CAAC,CAAA;IAEvE,IAAI,GAAG,IAAA,8BAAiB,EAAC;QACvB,IAAI,EAAE,IAAI;QACV,UAAU,EAAE,eAAe,GAAG,IAAI;QAClC,IAAI,EAAE,KAAK,IAAI,EAAE;YACf,MAAM,IAAA,kBAAQ,EACZ,EAAE,MAAM,EAAE,iBAAiB,CAAC,IAAA,qBAAa,EAAC,4CAAkB,CAAC,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EACtG,KAAK,EACL,EAAE,SAAS,EAAE,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAC5F,CAAA;YAED,OAAO,IAAA,oBAAU,EAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;QACtC,CAAC;QACD,wGAAwG;QACxG,SAAS,EAAE,CAAC,CAAC,EAAE,CACb,YAAM,CAAC,KAAK,CACV,0CAA0C,CAAC,CAAC,WAAW,mBAAmB,CAAC,CAAC,aAAa,mBAAmB,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAChK;QACH,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC,YAAM,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC,aAAa,kBAAkB,CAAC;QAClH,gBAAgB,EAAE,CAAC,CAAC,EAAE,CAAC,YAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC,KAAK,GAAG,IAAI,QAAQ,CAAC,CAAC,IAAI,GAAG,IAAI,GAAG,CAAC;QACnH,YAAY,EAAE,CAAC,CAAC,EAAE,CAAC,YAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;KACjF,CAAC,CAAA;AACJ,CAAC;AAED,sFAAsF;AACtF,SAAgB,eAAe;IAC7B,IAAI,EAAE,IAAI,EAAE,CAAA;IACZ,IAAI,GAAG,SAAS,CAAA;IAChB,KAAK,GAAG,IAAA,sBAAY,GAAE,CAAA;AACxB,CAAC","sourcesContent":["import { EntityManager } from 'typeorm'\n\nimport { logger } from '@things-factory/env'\nimport { getRepository } from '@things-factory/shell'\nimport { startPeriodicTask, type PeriodicTask } from '@things-factory/task-lease'\n\nimport { WebhookDestination } from '../webhook-destination/webhook-destination.js'\nimport { WebhookOutboxEvent, WebhookOutboxStatus } from '../webhook-outbox-event/webhook-outbox-event.js'\nimport { afterRound, newPumpState, pumpOnce, type PumpSource, type PumpState } from './pump.js'\nimport type { PendingStream } from './pump-round.js'\nimport { webhookStore } from './webhook-store.js'\n\n/**\n * **The pump's loop** — every few seconds, one round, on one instance of the cluster (ADR-0078 decision 6).\n *\n * ═════════════════════════════════════════════════════════════════════════\n * ## The application starts it\n *\n * The module does not start anything on its own. An application that appends facts calls `startWebhookPump`\n * from its boot hook, and passes the names its older installations used for keys, if it has any.\n *\n * ## One instance at a time, and one round at a time\n *\n * The loop is `startPeriodicTask` from `@things-factory/task-lease` (ADR-0110 adoption 2, step 3): a round never\n * starts while the last is still sending, it runs under a lease so that two instances behind a load balancer\n * do not both send (the later one arriving first reads at the receiver as a gap), the lease is renewed through\n * a long round, and a round that throws backs off.\n *\n * **A paused pump keeps the lease** - returned as `holdMs`. The pause lives in this process. Let go, another\n * instance - which knows nothing of it - takes over at once and knocks on the receiver that asked us to wait.\n * It is `holdMs` and not `restMs` because a pause is about one stream: the streams that are not paused still\n * go out from this instance at the usual interval. The pauses, their causes and their lengths stay in\n * `pump.ts`; only the longest length reaches the loop.\n *\n * ## It stands without a key\n *\n * Keys are looked up per stream, per round, on the contract's ladder. A missing key is said once, naming the\n * stream, and the facts wait. Refusing to start instead would say nothing at all about which domain lacks one.\n * ═════════════════════════════════════════════════════════════════════════\n */\n\nexport interface WebhookPumpOptions {\n intervalSeconds?: number\n batchSize?: number\n /** Names this product's installations used for keys before the contract's ladder. Most products have none. */\n legacyBases?: readonly string[]\n}\n\nconst DEFAULTS = { intervalSeconds: 5, batchSize: 200 }\nconst TASK = 'webhook-outbox-pump'\n\n/** The source over this module's tables. */\nexport function webhookPumpSource(manager: EntityManager): PumpSource {\n return {\n async pendingStreams(): Promise<PendingStream[]> {\n const rows = await manager\n .createQueryBuilder(WebhookOutboxEvent, 'event')\n .innerJoin('event.domain', 'domain')\n .select('domain.id', 'domainId')\n .addSelect('domain.subdomain', 'subdomain')\n .addSelect('event.peer', 'peer')\n .addSelect('event.scope', 'scope')\n .where('event.status = :pending', { pending: WebhookOutboxStatus.Pending })\n .groupBy('domain.id')\n .addGroupBy('domain.subdomain')\n .addGroupBy('event.peer')\n .addGroupBy('event.scope')\n .orderBy('domain.subdomain')\n .addOrderBy('event.peer')\n .addOrderBy('event.scope')\n .getRawMany()\n\n return rows.map(row => ({ domainId: row.domainId, subdomain: row.subdomain, peer: row.peer, scope: row.scope }))\n },\n\n async destinationOf(domainId: string, peer: string) {\n const row = await manager.findOne(WebhookDestination, { where: { domain: { id: domainId }, peer } })\n return row ? { endpoint: row.endpoint, active: row.active, params: row.params ?? undefined } : undefined\n },\n\n storeFor(stream: PendingStream) {\n return webhookStore(manager, { domainId: stream.domainId, peer: stream.peer, scope: stream.scope })\n }\n }\n}\n\nlet task: PeriodicTask | undefined\nlet state: PumpState = newPumpState()\n\n/** Start the loop. Starting twice does nothing. */\nexport function startWebhookPump(options: WebhookPumpOptions = {}): void {\n if (task) {\n return\n }\n\n const intervalSeconds = options.intervalSeconds ?? DEFAULTS.intervalSeconds\n const batchSize = options.batchSize ?? DEFAULTS.batchSize\n const say = { info: (what: string) => logger.info(what), warn: (what: string) => logger.warn(what), error: (what: string) => logger.error(what) }\n\n logger.info(`[webhook] sending queued facts every ${intervalSeconds}s`)\n\n task = startPeriodicTask({\n name: TASK,\n intervalMs: intervalSeconds * 1000,\n work: async () => {\n await pumpOnce(\n { source: webhookPumpSource(getRepository(WebhookOutboxEvent).manager), env: process.env, fetch, say },\n state,\n { batchSize, ...(options.legacyBases?.length ? { legacyBases: options.legacyBases } : {}) }\n )\n\n return afterRound(state, Date.now())\n },\n /* A failed round must not stop the next one - stopped, facts pile up for good. It backs off instead. */\n onFailure: f =>\n logger.error(\n `[webhook] sending queued facts failed (${f.consecutive} in a row since ${f.firstFailedAt}) - next try in ${Math.round(f.nextIntervalMs / 1000)}s - ${f.error}`\n ),\n onRecovered: r => logger.info(`[webhook] sending queued facts recovered after ${r.afterFailures} failed round(s)`),\n onIntervalRaised: r => logger.warn(`[webhook] intervalSeconds raised from ${r.asked / 1000}s to ${r.used / 1000}s`),\n onLeaseError: e => logger.error(`[webhook] pump lease \"${e.name}\" - ${e.error}`)\n })\n}\n\n/** Stop the loop. For tests and shutdown - left running, the process does not end. */\nexport function stopWebhookPump(): void {\n task?.stop()\n task = undefined\n state = newPumpState()\n}\n"]}
|
|
@@ -66,6 +66,16 @@ export declare function newPumpState(): PumpState;
|
|
|
66
66
|
* and knocks on it. Streams that are not paused still go out - they go out from this instance.
|
|
67
67
|
*/
|
|
68
68
|
export declare function restingFor(state: PumpState, nowMs: number): number;
|
|
69
|
+
/**
|
|
70
|
+
* What a round tells the periodic loop (task-lease `startPeriodicTask`).
|
|
71
|
+
*
|
|
72
|
+
* A pause is about one stream, so it is `holdMs` - other instances stay off while it stands, and this one
|
|
73
|
+
* goes on sending the streams that are not paused at the usual interval. Never `restMs`: that would stop
|
|
74
|
+
* every stream behind one tenant's wrong key.
|
|
75
|
+
*/
|
|
76
|
+
export declare function afterRound(state: PumpState, nowMs: number): {
|
|
77
|
+
holdMs: number;
|
|
78
|
+
} | undefined;
|
|
69
79
|
/** How long to rest after an answer that resending will not change. */
|
|
70
80
|
export declare const PAUSE_AFTER_UNRECOVERABLE_MS = 60000;
|
|
71
81
|
export interface PumpOptions extends RoundOptions {
|
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.PAUSE_AFTER_UNRECOVERABLE_MS = void 0;
|
|
4
4
|
exports.newPumpState = newPumpState;
|
|
5
5
|
exports.restingFor = restingFor;
|
|
6
|
+
exports.afterRound = afterRound;
|
|
6
7
|
exports.pumpOnce = pumpOnce;
|
|
7
8
|
const dispatcher_js_1 = require("./dispatcher.js");
|
|
8
9
|
const pump_round_js_1 = require("./pump-round.js");
|
|
@@ -22,6 +23,17 @@ function restingFor(state, nowMs) {
|
|
|
22
23
|
}
|
|
23
24
|
return latest - nowMs;
|
|
24
25
|
}
|
|
26
|
+
/**
|
|
27
|
+
* What a round tells the periodic loop (task-lease `startPeriodicTask`).
|
|
28
|
+
*
|
|
29
|
+
* A pause is about one stream, so it is `holdMs` - other instances stay off while it stands, and this one
|
|
30
|
+
* goes on sending the streams that are not paused at the usual interval. Never `restMs`: that would stop
|
|
31
|
+
* every stream behind one tenant's wrong key.
|
|
32
|
+
*/
|
|
33
|
+
function afterRound(state, nowMs) {
|
|
34
|
+
const resting = restingFor(state, nowMs);
|
|
35
|
+
return resting > 0 ? { holdMs: resting } : undefined;
|
|
36
|
+
}
|
|
25
37
|
/** How long to rest after an answer that resending will not change. */
|
|
26
38
|
exports.PAUSE_AFTER_UNRECOVERABLE_MS = 60_000;
|
|
27
39
|
async function pumpOnce(deps, state, options) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pump.js","sourceRoot":"","sources":["../../../server/service/outbox/pump.ts"],"names":[],"mappings":";;;AAkEA,oCAEC;AAQD,gCAMC;AASD,4BAwEC;AAnKD,mDAA2D;AAC3D,mDAA2G;AAiE3G,SAAgB,YAAY;IAC1B,OAAO,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,CAAA;AAC/C,CAAC;AAED;;;;;GAKG;AACH,SAAgB,UAAU,CAAC,KAAgB,EAAE,KAAa;IACxD,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QAC1C,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;IACxC,CAAC;IACD,OAAO,MAAM,GAAG,KAAK,CAAA;AACvB,CAAC;AAED,uEAAuE;AAC1D,QAAA,4BAA4B,GAAG,MAAM,CAAA;AAM3C,KAAK,UAAU,QAAQ,CAAC,IAAc,EAAE,KAAgB,EAAE,OAAoB;IACnF,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;IAE5C,uGAAuG;IACvG,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACxC,IAAI,KAAK,CAAC,KAAK,IAAI,GAAG,EAAE;YAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,GAAG,EAAE,CAAA;IAE5E,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,KAAa,EAAE,GAAW,EAAE,EAAE;QACxD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACtC,IAAI,QAAQ,EAAE,GAAG,KAAK,GAAG,EAAE,CAAC;YAC1B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,GAAG,cAAc,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAA;QAC1E,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;IACtF,CAAC,CAAA;IAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;IAElD,MAAM,YAAY,GAAG,IAAI,GAAG,EAA0C,CAAA;IACtE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,QAAQ,SAAS,MAAM,CAAC,IAAI,EAAE,CAAA;QACpD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;QACtF,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,IAAA,yBAAS,EAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,QAAQ,SAAS,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACrH,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAA;IAEvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAChC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC/B,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzB,SAAQ;QACV,CAAC;QAED,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,MAAM,CAAA;QAC7C,MAAM,QAAQ,GAAG,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAA;QAC5C,MAAM,MAAM,GAAG,UAAU,QAAQ,IAAI,IAAI,EAAE,CAAA;QAC3C,MAAM,MAAM,GAAG,UAAU,QAAQ,IAAI,IAAI,IAAI,KAAK,EAAE,CAAA;QAEpD,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5D,SAAQ;QACV,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAA,+BAAe,EAClC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EACjC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,EAC9H,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CACjE,CAAA;QAED,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;YACxB,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,QAAQ,wBAAwB,CAAC,CAAA;QAChF,CAAC;aAAM,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC1C,KAAK,CAAC,MAAM,EAAE,oCAA4B,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QACpE,CAAC;aAAM,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAClD,KAAK,CAAC,MAAM,EAAE,oCAA4B,EAAE,GAAG,IAAI,CAAC,QAAQ,qCAAqC,IAAI,CAAC,MAAM,CAAC,SAAS,wDAAwD,CAAC,CAAA;QACjL,CAAC;IACH,CAAC;IAED,4GAA4G;IAC5G,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAClC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IACrD,CAAC;AACH,CAAC","sourcesContent":["import { dispatchPending, type Say } from './dispatcher.js'\nimport { planRound, type DestinationReading, type PendingStream, type RoundOptions } from './pump-round.js'\nimport type { OutboxStore } from './store.js'\n\n/**\n * **One round of the pump** — every stream with facts waiting, one delivery each (ADR-0078 decision 6).\n *\n * ═════════════════════════════════════════════════════════════════════════\n * Built against operato-plant's `mes-outbox-pump.ts`, which learned these by running:\n *\n * - **Asked to wait, nothing more goes to that receiver until the wait is over.** Ignoring a 429 does not crash\n * anything - the receiver drops facts instead.\n * - **An answer only a person can fix pauses that stream.** A wrong key gets the same 401 every five seconds,\n * knocks on the receiver and buries the log. It pauses only what it is about - see `PumpState.pauses`.\n * - **Each cause of facts piling up is said once.** Said every round, the line covers everything else.\n *\n * Nothing here touches TypeORM, the process or a timer - the source, the environment, `fetch` and the clock are\n * handed in, so every branch runs in a test. The loop, the lease and the tables are in `pump-loop.ts`.\n * ═════════════════════════════════════════════════════════════════════════\n */\n\n/** Where a round reads from. `pump-loop.ts` supplies one over this module's tables. */\nexport interface PumpSource {\n pendingStreams(): Promise<PendingStream[]>\n destinationOf(domainId: string, peer: string): Promise<DestinationReading | undefined>\n storeFor(stream: PendingStream): OutboxStore\n}\n\nexport interface PumpDeps {\n source: PumpSource\n env: Record<string, string | undefined>\n fetch: typeof fetch\n say: Say\n nowMs?: () => number\n}\n\n/**\n * One pause: until when, and why. The same reason is not said twice in a row.\n */\ninterface Pause {\n until: number\n why: string\n}\n\n/** What a pump remembers between rounds. One per running pump. */\nexport interface PumpState {\n /**\n * Pauses **by what they are about** (Plant, 2026-09-24). The first cut held one pause for the whole pump, as\n * operato-plant's pump does: a tenant with a *wrong* key paused everything, and since the round goes in a fixed\n * order the tenants behind it never sent again - measured, five minutes, zero sent. So each cause pauses its\n * own unit:\n *\n * ```\n * receiver:<endpoint> asked to wait it is that receiver's load, whoever the stream belongs to\n * stream:<domain>/<peer> signature refused a key is per domain and peer\n * stream:<domain>/<peer>/<scope> stop the answer was about what this stream sent\n * ```\n *\n * Waiting is keyed by the endpoint rather than the peer's name: two domains may both call their twin `twin`\n * and mean two different servers.\n */\n pauses: Map<string, Pause>\n /** Notes said, by key. A key whose cause is gone this round is forgotten, so a cause that returns is said again. */\n said: Set<string>\n}\n\nexport function newPumpState(): PumpState {\n return { pauses: new Map(), said: new Set() }\n}\n\n/**\n * How much longer the longest pause stands - what the lease should be held past. Zero or less when nothing rests.\n *\n * Holding past the longest keeps the rule the single pause had: while anything rests, no other instance takes over\n * and knocks on it. Streams that are not paused still go out - they go out from this instance.\n */\nexport function restingFor(state: PumpState, nowMs: number): number {\n let latest = 0\n for (const pause of state.pauses.values()) {\n latest = Math.max(latest, pause.until)\n }\n return latest - nowMs\n}\n\n/** How long to rest after an answer that resending will not change. */\nexport const PAUSE_AFTER_UNRECOVERABLE_MS = 60_000\n\nexport interface PumpOptions extends RoundOptions {\n batchSize: number\n}\n\nexport async function pumpOnce(deps: PumpDeps, state: PumpState, options: PumpOptions): Promise<void> {\n const now = deps.nowMs ?? (() => Date.now())\n\n /* A pause that is over is forgotten - the next time the same answer comes, it has to be said again. */\n for (const [key, pause] of state.pauses) {\n if (pause.until <= now()) state.pauses.delete(key)\n }\n\n const resting = (key: string) => (state.pauses.get(key)?.until ?? 0) > now()\n\n const pause = (key: string, forMs: number, why: string) => {\n const standing = state.pauses.get(key)\n if (standing?.why !== why) {\n deps.say.warn(`[webhook] ${why} - waiting ${Math.round(forMs / 1000)}s`)\n }\n state.pauses.set(key, { until: Math.max(standing?.until ?? 0, now() + forMs), why })\n }\n\n const streams = await deps.source.pendingStreams()\n\n const destinations = new Map<string, DestinationReading | undefined>()\n for (const stream of streams) {\n const key = `${stream.domainId}\\u0000${stream.peer}`\n if (!destinations.has(key)) {\n destinations.set(key, await deps.source.destinationOf(stream.domainId, stream.peer))\n }\n }\n\n const steps = planRound(streams, (domainId, peer) => destinations.get(`${domainId}\\u0000${peer}`), deps.env, options)\n const saidThisRound = new Set<string>()\n\n for (const step of steps) {\n if (step.note) {\n saidThisRound.add(step.note.key)\n if (!state.said.has(step.note.key)) {\n state.said.add(step.note.key)\n deps.say.warn(step.note.line)\n }\n }\n\n if (step.kind === 'wait') {\n continue\n }\n\n const { domainId, peer, scope } = step.stream\n const receiver = `receiver:${step.endpoint}`\n const signer = `stream:${domainId}/${peer}`\n const stream = `stream:${domainId}/${peer}/${scope}`\n\n if (resting(receiver) || resting(signer) || resting(stream)) {\n continue\n }\n\n const report = await dispatchPending(\n deps.source.storeFor(step.stream),\n { endpoint: step.endpoint, secret: step.secret, headers: step.headers, clockKey: step.clockKey, batchSize: options.batchSize },\n { fetch: deps.fetch, say: deps.say, now: () => new Date(now()) }\n )\n\n if (report.retryAfterMs) {\n pause(receiver, report.retryAfterMs, `${step.endpoint} asked us to slow down`)\n } else if (report.outcome.kind === 'stop') {\n pause(stream, PAUSE_AFTER_UNRECOVERABLE_MS, report.outcome.reason)\n } else if (report.outcome.kind === 'unauthorized') {\n pause(signer, PAUSE_AFTER_UNRECOVERABLE_MS, `${step.endpoint} did not accept our signature for ${step.stream.subdomain} - check the key on both sides, then it will try again`)\n }\n }\n\n /* A note whose cause did not come up this round is forgotten. If the cause comes back, it is said again. */\n for (const key of [...state.said]) {\n if (!saidThisRound.has(key)) state.said.delete(key)\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"pump.js","sourceRoot":"","sources":["../../../server/service/outbox/pump.ts"],"names":[],"mappings":";;;AAkEA,oCAEC;AAQD,gCAMC;AASD,gCAGC;AASD,4BAwEC;AA/KD,mDAA2D;AAC3D,mDAA2G;AAiE3G,SAAgB,YAAY;IAC1B,OAAO,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,CAAA;AAC/C,CAAC;AAED;;;;;GAKG;AACH,SAAgB,UAAU,CAAC,KAAgB,EAAE,KAAa;IACxD,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QAC1C,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;IACxC,CAAC;IACD,OAAO,MAAM,GAAG,KAAK,CAAA;AACvB,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,UAAU,CAAC,KAAgB,EAAE,KAAa;IACxD,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IACxC,OAAO,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,SAAS,CAAA;AACtD,CAAC;AAED,uEAAuE;AAC1D,QAAA,4BAA4B,GAAG,MAAM,CAAA;AAM3C,KAAK,UAAU,QAAQ,CAAC,IAAc,EAAE,KAAgB,EAAE,OAAoB;IACnF,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;IAE5C,uGAAuG;IACvG,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACxC,IAAI,KAAK,CAAC,KAAK,IAAI,GAAG,EAAE;YAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,GAAG,EAAE,CAAA;IAE5E,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,KAAa,EAAE,GAAW,EAAE,EAAE;QACxD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACtC,IAAI,QAAQ,EAAE,GAAG,KAAK,GAAG,EAAE,CAAC;YAC1B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,GAAG,cAAc,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAA;QAC1E,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;IACtF,CAAC,CAAA;IAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;IAElD,MAAM,YAAY,GAAG,IAAI,GAAG,EAA0C,CAAA;IACtE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,QAAQ,SAAS,MAAM,CAAC,IAAI,EAAE,CAAA;QACpD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;QACtF,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,IAAA,yBAAS,EAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,QAAQ,SAAS,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACrH,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAA;IAEvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAChC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC/B,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzB,SAAQ;QACV,CAAC;QAED,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,MAAM,CAAA;QAC7C,MAAM,QAAQ,GAAG,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAA;QAC5C,MAAM,MAAM,GAAG,UAAU,QAAQ,IAAI,IAAI,EAAE,CAAA;QAC3C,MAAM,MAAM,GAAG,UAAU,QAAQ,IAAI,IAAI,IAAI,KAAK,EAAE,CAAA;QAEpD,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5D,SAAQ;QACV,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAA,+BAAe,EAClC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EACjC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,EAC9H,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CACjE,CAAA;QAED,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;YACxB,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,QAAQ,wBAAwB,CAAC,CAAA;QAChF,CAAC;aAAM,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC1C,KAAK,CAAC,MAAM,EAAE,oCAA4B,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QACpE,CAAC;aAAM,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAClD,KAAK,CAAC,MAAM,EAAE,oCAA4B,EAAE,GAAG,IAAI,CAAC,QAAQ,qCAAqC,IAAI,CAAC,MAAM,CAAC,SAAS,wDAAwD,CAAC,CAAA;QACjL,CAAC;IACH,CAAC;IAED,4GAA4G;IAC5G,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAClC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IACrD,CAAC;AACH,CAAC","sourcesContent":["import { dispatchPending, type Say } from './dispatcher.js'\nimport { planRound, type DestinationReading, type PendingStream, type RoundOptions } from './pump-round.js'\nimport type { OutboxStore } from './store.js'\n\n/**\n * **One round of the pump** — every stream with facts waiting, one delivery each (ADR-0078 decision 6).\n *\n * ═════════════════════════════════════════════════════════════════════════\n * Built against operato-plant's `mes-outbox-pump.ts`, which learned these by running:\n *\n * - **Asked to wait, nothing more goes to that receiver until the wait is over.** Ignoring a 429 does not crash\n * anything - the receiver drops facts instead.\n * - **An answer only a person can fix pauses that stream.** A wrong key gets the same 401 every five seconds,\n * knocks on the receiver and buries the log. It pauses only what it is about - see `PumpState.pauses`.\n * - **Each cause of facts piling up is said once.** Said every round, the line covers everything else.\n *\n * Nothing here touches TypeORM, the process or a timer - the source, the environment, `fetch` and the clock are\n * handed in, so every branch runs in a test. The loop, the lease and the tables are in `pump-loop.ts`.\n * ═════════════════════════════════════════════════════════════════════════\n */\n\n/** Where a round reads from. `pump-loop.ts` supplies one over this module's tables. */\nexport interface PumpSource {\n pendingStreams(): Promise<PendingStream[]>\n destinationOf(domainId: string, peer: string): Promise<DestinationReading | undefined>\n storeFor(stream: PendingStream): OutboxStore\n}\n\nexport interface PumpDeps {\n source: PumpSource\n env: Record<string, string | undefined>\n fetch: typeof fetch\n say: Say\n nowMs?: () => number\n}\n\n/**\n * One pause: until when, and why. The same reason is not said twice in a row.\n */\ninterface Pause {\n until: number\n why: string\n}\n\n/** What a pump remembers between rounds. One per running pump. */\nexport interface PumpState {\n /**\n * Pauses **by what they are about** (Plant, 2026-09-24). The first cut held one pause for the whole pump, as\n * operato-plant's pump does: a tenant with a *wrong* key paused everything, and since the round goes in a fixed\n * order the tenants behind it never sent again - measured, five minutes, zero sent. So each cause pauses its\n * own unit:\n *\n * ```\n * receiver:<endpoint> asked to wait it is that receiver's load, whoever the stream belongs to\n * stream:<domain>/<peer> signature refused a key is per domain and peer\n * stream:<domain>/<peer>/<scope> stop the answer was about what this stream sent\n * ```\n *\n * Waiting is keyed by the endpoint rather than the peer's name: two domains may both call their twin `twin`\n * and mean two different servers.\n */\n pauses: Map<string, Pause>\n /** Notes said, by key. A key whose cause is gone this round is forgotten, so a cause that returns is said again. */\n said: Set<string>\n}\n\nexport function newPumpState(): PumpState {\n return { pauses: new Map(), said: new Set() }\n}\n\n/**\n * How much longer the longest pause stands - what the lease should be held past. Zero or less when nothing rests.\n *\n * Holding past the longest keeps the rule the single pause had: while anything rests, no other instance takes over\n * and knocks on it. Streams that are not paused still go out - they go out from this instance.\n */\nexport function restingFor(state: PumpState, nowMs: number): number {\n let latest = 0\n for (const pause of state.pauses.values()) {\n latest = Math.max(latest, pause.until)\n }\n return latest - nowMs\n}\n\n/**\n * What a round tells the periodic loop (task-lease `startPeriodicTask`).\n *\n * A pause is about one stream, so it is `holdMs` - other instances stay off while it stands, and this one\n * goes on sending the streams that are not paused at the usual interval. Never `restMs`: that would stop\n * every stream behind one tenant's wrong key.\n */\nexport function afterRound(state: PumpState, nowMs: number): { holdMs: number } | undefined {\n const resting = restingFor(state, nowMs)\n return resting > 0 ? { holdMs: resting } : undefined\n}\n\n/** How long to rest after an answer that resending will not change. */\nexport const PAUSE_AFTER_UNRECOVERABLE_MS = 60_000\n\nexport interface PumpOptions extends RoundOptions {\n batchSize: number\n}\n\nexport async function pumpOnce(deps: PumpDeps, state: PumpState, options: PumpOptions): Promise<void> {\n const now = deps.nowMs ?? (() => Date.now())\n\n /* A pause that is over is forgotten - the next time the same answer comes, it has to be said again. */\n for (const [key, pause] of state.pauses) {\n if (pause.until <= now()) state.pauses.delete(key)\n }\n\n const resting = (key: string) => (state.pauses.get(key)?.until ?? 0) > now()\n\n const pause = (key: string, forMs: number, why: string) => {\n const standing = state.pauses.get(key)\n if (standing?.why !== why) {\n deps.say.warn(`[webhook] ${why} - waiting ${Math.round(forMs / 1000)}s`)\n }\n state.pauses.set(key, { until: Math.max(standing?.until ?? 0, now() + forMs), why })\n }\n\n const streams = await deps.source.pendingStreams()\n\n const destinations = new Map<string, DestinationReading | undefined>()\n for (const stream of streams) {\n const key = `${stream.domainId}\\u0000${stream.peer}`\n if (!destinations.has(key)) {\n destinations.set(key, await deps.source.destinationOf(stream.domainId, stream.peer))\n }\n }\n\n const steps = planRound(streams, (domainId, peer) => destinations.get(`${domainId}\\u0000${peer}`), deps.env, options)\n const saidThisRound = new Set<string>()\n\n for (const step of steps) {\n if (step.note) {\n saidThisRound.add(step.note.key)\n if (!state.said.has(step.note.key)) {\n state.said.add(step.note.key)\n deps.say.warn(step.note.line)\n }\n }\n\n if (step.kind === 'wait') {\n continue\n }\n\n const { domainId, peer, scope } = step.stream\n const receiver = `receiver:${step.endpoint}`\n const signer = `stream:${domainId}/${peer}`\n const stream = `stream:${domainId}/${peer}/${scope}`\n\n if (resting(receiver) || resting(signer) || resting(stream)) {\n continue\n }\n\n const report = await dispatchPending(\n deps.source.storeFor(step.stream),\n { endpoint: step.endpoint, secret: step.secret, headers: step.headers, clockKey: step.clockKey, batchSize: options.batchSize },\n { fetch: deps.fetch, say: deps.say, now: () => new Date(now()) }\n )\n\n if (report.retryAfterMs) {\n pause(receiver, report.retryAfterMs, `${step.endpoint} asked us to slow down`)\n } else if (report.outcome.kind === 'stop') {\n pause(stream, PAUSE_AFTER_UNRECOVERABLE_MS, report.outcome.reason)\n } else if (report.outcome.kind === 'unauthorized') {\n pause(signer, PAUSE_AFTER_UNRECOVERABLE_MS, `${step.endpoint} did not accept our signature for ${step.stream.subdomain} - check the key on both sides, then it will try again`)\n }\n }\n\n /* A note whose cause did not come up this round is forgotten. If the cause comes back, it is said again. */\n for (const key of [...state.said]) {\n if (!saidThisRound.has(key)) state.said.delete(key)\n }\n}\n"]}
|
|
@@ -1,4 +1,24 @@
|
|
|
1
|
+
import { EntityManager } from 'typeorm';
|
|
1
2
|
import { WebhookDestination } from './webhook-destination';
|
|
3
|
+
/**
|
|
4
|
+
* One change a person made to a destination — before and after, in the writing transaction.
|
|
5
|
+
*
|
|
6
|
+
* ADR-0078 decision 1: who changed where facts go, when, and from what to what, is kept. The module does not
|
|
7
|
+
* own a product's change history, so it hands the change to whoever listens, inside the same transaction; the
|
|
8
|
+
* product writes it into its own history (operato-plant: `fact-change`). A listener that throws rolls the
|
|
9
|
+
* change back — a change that cannot be recorded is not made.
|
|
10
|
+
*/
|
|
11
|
+
export interface WebhookDestinationChange {
|
|
12
|
+
domainId: string;
|
|
13
|
+
peer: string;
|
|
14
|
+
user?: unknown;
|
|
15
|
+
/** Absent when the destination is new. */
|
|
16
|
+
before?: Pick<WebhookDestination, 'endpoint' | 'description' | 'active' | 'params'>;
|
|
17
|
+
after: Pick<WebhookDestination, 'endpoint' | 'description' | 'active' | 'params'>;
|
|
18
|
+
}
|
|
19
|
+
export type WebhookDestinationListener = (tx: EntityManager, change: WebhookDestinationChange) => Promise<void> | void;
|
|
20
|
+
/** Listen to destination changes. Returns a function that stops listening. */
|
|
21
|
+
export declare function onWebhookDestinationChange(listener: WebhookDestinationListener): () => void;
|
|
2
22
|
export declare class WebhookDestinationResolver {
|
|
3
23
|
webhookDestinations(context: ResolverContext): Promise<WebhookDestination[]>;
|
|
4
24
|
webhookDestination(peer: string, context: ResolverContext): Promise<WebhookDestination | null>;
|
|
@@ -1,29 +1,22 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.WebhookDestinationResolver = void 0;
|
|
4
|
+
exports.onWebhookDestinationChange = onWebhookDestinationChange;
|
|
4
5
|
const tslib_1 = require("tslib");
|
|
5
6
|
const type_graphql_1 = require("type-graphql");
|
|
6
7
|
const shell_1 = require("@things-factory/shell");
|
|
7
8
|
const webhook_destination_1 = require("./webhook-destination");
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
* Signing and verifying keys come from the environment, per peer. A mutation that accepted
|
|
20
|
-
* them would put them in a request body, a resolver log and an audit row; a query that
|
|
21
|
-
* returned them would hand them to anyone who can open the settings screen.
|
|
22
|
-
*
|
|
23
|
-
* What the screen may say about a key is whether one is *present* — that is a fact about
|
|
24
|
-
* configuration, not the value.
|
|
25
|
-
* ═══════════════════════════════════════════════════════════════════════════
|
|
26
|
-
*/
|
|
9
|
+
const destinationListeners = new Set();
|
|
10
|
+
/** Listen to destination changes. Returns a function that stops listening. */
|
|
11
|
+
function onWebhookDestinationChange(listener) {
|
|
12
|
+
destinationListeners.add(listener);
|
|
13
|
+
return () => destinationListeners.delete(listener);
|
|
14
|
+
}
|
|
15
|
+
const shapeOf = (row) => row ? { endpoint: row.endpoint, description: row.description, active: row.active, params: row.params } : undefined;
|
|
16
|
+
async function tellListeners(tx, change) {
|
|
17
|
+
for (const listener of destinationListeners)
|
|
18
|
+
await listener(tx, change);
|
|
19
|
+
}
|
|
27
20
|
let WebhookDestinationResolver = class WebhookDestinationResolver {
|
|
28
21
|
async webhookDestinations(context) {
|
|
29
22
|
return await (0, shell_1.getRepository)(webhook_destination_1.WebhookDestination).find({
|
|
@@ -45,7 +38,8 @@ let WebhookDestinationResolver = class WebhookDestinationResolver {
|
|
|
45
38
|
*/
|
|
46
39
|
async saveWebhookDestination(peer, patch, context) {
|
|
47
40
|
const domainId = context.state.domain.id;
|
|
48
|
-
const
|
|
41
|
+
const tx = context.state.tx;
|
|
42
|
+
const repo = tx.getRepository(webhook_destination_1.WebhookDestination);
|
|
49
43
|
const existing = await repo.findOne({ where: { domain: { id: domainId }, peer } });
|
|
50
44
|
/*
|
|
51
45
|
* `params` merges by key for the same reason the patch does — a peer's options are added
|
|
@@ -55,7 +49,7 @@ let WebhookDestinationResolver = class WebhookDestinationResolver {
|
|
|
55
49
|
const params = patch.params
|
|
56
50
|
? { ...(existing?.params ?? {}), ...patch.params }
|
|
57
51
|
: existing?.params;
|
|
58
|
-
|
|
52
|
+
const saved = await repo.save(repo.create({
|
|
59
53
|
...(existing ?? {}),
|
|
60
54
|
domain: { id: domainId },
|
|
61
55
|
peer,
|
|
@@ -70,6 +64,8 @@ let WebhookDestinationResolver = class WebhookDestinationResolver {
|
|
|
70
64
|
creator: existing?.creator ?? context.state.user,
|
|
71
65
|
updater: context.state.user
|
|
72
66
|
}));
|
|
67
|
+
await tellListeners(tx, { domainId, peer, user: context.state.user, before: shapeOf(existing), after: shapeOf(saved) });
|
|
68
|
+
return saved;
|
|
73
69
|
}
|
|
74
70
|
/**
|
|
75
71
|
* Stop pushing to a peer without forgetting it.
|
|
@@ -80,8 +76,17 @@ let WebhookDestinationResolver = class WebhookDestinationResolver {
|
|
|
80
76
|
* numbers it has already seen.
|
|
81
77
|
*/
|
|
82
78
|
async setWebhookDestinationActive(peer, active, context) {
|
|
83
|
-
const
|
|
84
|
-
|
|
79
|
+
const domainId = context.state.domain.id;
|
|
80
|
+
const tx = context.state.tx;
|
|
81
|
+
const repo = tx.getRepository(webhook_destination_1.WebhookDestination);
|
|
82
|
+
const existing = await repo.findOne({ where: { domain: { id: domainId }, peer } });
|
|
83
|
+
if (!existing)
|
|
84
|
+
return false;
|
|
85
|
+
await repo.update({ domain: { id: domainId }, peer }, { active, updater: context.state.user });
|
|
86
|
+
if (existing.active !== active) {
|
|
87
|
+
await tellListeners(tx, { domainId, peer, user: context.state.user, before: shapeOf(existing), after: { ...shapeOf(existing), active } });
|
|
88
|
+
}
|
|
89
|
+
return true;
|
|
85
90
|
}
|
|
86
91
|
};
|
|
87
92
|
exports.WebhookDestinationResolver = WebhookDestinationResolver;
|
|
@@ -106,6 +111,7 @@ tslib_1.__decorate([
|
|
|
106
111
|
], WebhookDestinationResolver.prototype, "webhookDestination", null);
|
|
107
112
|
tslib_1.__decorate([
|
|
108
113
|
(0, type_graphql_1.Directive)('@privilege(category: "webhook", privilege: "mutation", domainOwnerGranted: true, superUserGranted: true)'),
|
|
114
|
+
(0, type_graphql_1.Directive)('@transaction'),
|
|
109
115
|
(0, type_graphql_1.Mutation)(returns => webhook_destination_1.WebhookDestination, { description: 'Create or update the destination for one peer. Fields absent from the patch keep their stored value.' }),
|
|
110
116
|
tslib_1.__param(0, (0, type_graphql_1.Arg)('peer')),
|
|
111
117
|
tslib_1.__param(1, (0, type_graphql_1.Arg)('patch', type => shell_1.ScalarObject)),
|
|
@@ -116,6 +122,7 @@ tslib_1.__decorate([
|
|
|
116
122
|
], WebhookDestinationResolver.prototype, "saveWebhookDestination", null);
|
|
117
123
|
tslib_1.__decorate([
|
|
118
124
|
(0, type_graphql_1.Directive)('@privilege(category: "webhook", privilege: "mutation", domainOwnerGranted: true, superUserGranted: true)'),
|
|
125
|
+
(0, type_graphql_1.Directive)('@transaction'),
|
|
119
126
|
(0, type_graphql_1.Mutation)(returns => Boolean, { description: 'Turn pushing to one peer off or on. The queue and the numbering are kept either way.' }),
|
|
120
127
|
tslib_1.__param(0, (0, type_graphql_1.Arg)('peer')),
|
|
121
128
|
tslib_1.__param(1, (0, type_graphql_1.Arg)('active')),
|