@things-factory/webhook 10.1.38 → 10.1.40

package/dist-server/service/index.d.ts

@@ -20,10 +20,12 @@ export * from './outbox/requeue-note';
 export * from './outbox/page';
 export * from './outbox/writing-transaction';
 export * from './outbox/append';
-export * from './outbox/pump-gate';
 export * from './observability/clock-offset';
 export * from './observability/trace-context';
 export * from './outbox/store';
 export * from './outbox/dispatcher';
 export * from './outbox/webhook-store';
 export * from './outbox/webhook-append-store';
+export * from './outbox/pump-round';
+export * from './outbox/pump';
+export * from './outbox/pump-loop';

package/dist-server/service/index.js

@@ -46,7 +46,6 @@ tslib_1.__exportStar(require("./outbox/writing-transaction"), exports);
  * in a check - because the number and the row have to commit together or the number burns.
  */
 tslib_1.__exportStar(require("./outbox/append"), exports);
-tslib_1.__exportStar(require("./outbox/pump-gate"), exports);
 /* What a delivery needs to say "when" and "which request" — moved from operato-plant with the dispatcher's
  * needs in mind (ADR-0078 decision 1). The inbound side reads the same two. */
 tslib_1.__exportStar(require("./observability/clock-offset"), exports);
@@ -56,4 +55,12 @@ tslib_1.__exportStar(require("./outbox/store"), exports);
 tslib_1.__exportStar(require("./outbox/dispatcher"), exports);
 tslib_1.__exportStar(require("./outbox/webhook-store"), exports);
 tslib_1.__exportStar(require("./outbox/webhook-append-store"), exports);
+/*
+ * The pump: one round's judgement (`pump-round`), one round (`pump`), and the loop an application starts from its
+ * boot hook (`pump-loop`). Keys come from the contract's ladder - the product-named `pump-gate` is gone
+ * (ADR-0078 decision 5, re-ruled 2026-09-24).
+ */
+tslib_1.__exportStar(require("./outbox/pump-round"), exports);
+tslib_1.__exportStar(require("./outbox/pump"), exports);
+tslib_1.__exportStar(require("./outbox/pump-loop"), exports);
 //# sourceMappingURL=index.js.map

package/dist-server/service/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../server/service/index.ts"],"names":[],"mappings":";;;;AAAA;;;;;;;;GAQG;AACH,mFAA8E;AAC9E,qGAA+F;AAC/F,sFAAgF;AAChF,0EAAqE;AAErE,oFAAyD;AACzD,6FAAkE;AAClE,sFAA2D;AAC3D,8EAAmD;AAEnD;;;;;GAKG;AACU,QAAA,QAAQ,GAAG,CAAC,wCAAkB,EAAE,yCAAkB,EAAE,kCAAe,CAAC,CAAA;AAEpE,QAAA,MAAM,GAAG,EAAE,eAAe,EAAE,CAAC,yDAA0B,CAAC,EAAE,CAAA;AAEvE;;;;GAIG;AACH,4DAAiC;AACjC,8DAAmC;AACnC,8DAAmC;AACnC,8DAAmC;AACnC,kEAAuC;AACvC,4DAAiC;AACjC,gEAAqC;AACrC,wDAA6B;AAC7B,uEAA4C;AAE5C;;;GAGG;AACH,0DAA+B;AAC/B,6DAAkC;AAElC;+EAC+E;AAC/E,uEAA4C;AAC5C,wEAA6C;AAE7C,6GAA6G;AAC7G,yDAA8B;AAC9B,8DAAmC;AACnC,iEAAsC;AACtC,wEAA6C","sourcesContent":["/*\n * Each source file is named rather than re-exporting folder barrels.\n *\n * A name added to a folder barrel but not listed here stops at the package boundary, and the\n * consumer's error is `TS2305: has no exported member` — which reads like the name does not\n * exist rather than like it is unreachable. That happened on 2026-09-07 with\n * `work-shift/days-of-week`: the parser was in the folder barrel, not here, and the consuming\n * lane wrote its own until the miss was found.\n */\nimport { WebhookDestination } from './webhook-destination/webhook-destination'\nimport { WebhookDestinationResolver } from './webhook-destination/webhook-destination-resolver'\nimport { WebhookOutboxEvent } from './webhook-outbox-event/webhook-outbox-event'\nimport { WebhookSequence } from './webhook-sequence/webhook-sequence'\n\nexport * from './webhook-destination/webhook-destination'\nexport * from './webhook-destination/webhook-destination-resolver'\nexport * from './webhook-outbox-event/webhook-outbox-event'\nexport * from './webhook-sequence/webhook-sequence'\n\n/*\n * ⚠ **The shell builds the database and the schema from these two names.** Without them the module's\n * tables are never created in any application and the destination door is in no schema - the files stand\n * and nothing reaches them. That was the state through 10.1.37 (found 2026-09-24 by the first application to\n * append: its database had no webhook table at all). `test/append-store-db.test.ts` holds this line.\n */\nexport const entities = [WebhookDestination, WebhookOutboxEvent, WebhookSequence]\n\nexport const schema = { resolverClasses: [WebhookDestinationResolver] }\n\n/*\n * The outbox machine, moved from operato-plant (ADR-0078 decision 2). Step ①: the pure judgement —\n * envelope, signature, reading the receiver's answer, settlement, what an answer does to our rows,\n * the sequence report, reconciliation, standing. None of these touch a store.\n */\nexport * from './outbox/delivery'\nexport * from './outbox/settlement'\nexport * from './outbox/row-effect'\nexport * from './outbox/seq-report'\nexport * from './outbox/reconciliation'\nexport * from './outbox/standing'\nexport * from './outbox/requeue-note'\nexport * from './outbox/page'\nexport * from './outbox/writing-transaction'\n\n/*\n * Step ②: the back row. Writing a fact down takes the caller's open transaction - in the type, not only\n * in a check - because the number and the row have to commit together or the number burns.\n */\nexport * from './outbox/append'\nexport * from './outbox/pump-gate'\n\n/* What a delivery needs to say \"when\" and \"which request\" — moved from operato-plant with the dispatcher's\n * needs in mind (ADR-0078 decision 1). The inbound side reads the same two. */\nexport * from './observability/clock-offset'\nexport * from './observability/trace-context'\n\n/* Step ②: the back row. One dispatcher over a store interface; `webhookStore` is this module's own store. */\nexport * from './outbox/store'\nexport * from './outbox/dispatcher'\nexport * from './outbox/webhook-store'\nexport * from './outbox/webhook-append-store'\n\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../server/service/index.ts"],"names":[],"mappings":";;;;AAAA;;;;;;;;GAQG;AACH,mFAA8E;AAC9E,qGAA+F;AAC/F,sFAAgF;AAChF,0EAAqE;AAErE,oFAAyD;AACzD,6FAAkE;AAClE,sFAA2D;AAC3D,8EAAmD;AAEnD;;;;;GAKG;AACU,QAAA,QAAQ,GAAG,CAAC,wCAAkB,EAAE,yCAAkB,EAAE,kCAAe,CAAC,CAAA;AAEpE,QAAA,MAAM,GAAG,EAAE,eAAe,EAAE,CAAC,yDAA0B,CAAC,EAAE,CAAA;AAEvE;;;;GAIG;AACH,4DAAiC;AACjC,8DAAmC;AACnC,8DAAmC;AACnC,8DAAmC;AACnC,kEAAuC;AACvC,4DAAiC;AACjC,gEAAqC;AACrC,wDAA6B;AAC7B,uEAA4C;AAE5C;;;GAGG;AACH,0DAA+B;AAE/B;+EAC+E;AAC/E,uEAA4C;AAC5C,wEAA6C;AAE7C,6GAA6G;AAC7G,yDAA8B;AAC9B,8DAAmC;AACnC,iEAAsC;AACtC,wEAA6C;AAE7C;;;;GAIG;AACH,8DAAmC;AACnC,wDAA6B;AAC7B,6DAAkC","sourcesContent":["/*\n * Each source file is named rather than re-exporting folder barrels.\n *\n * A name added to a folder barrel but not listed here stops at the package boundary, and the\n * consumer's error is `TS2305: has no exported member` — which reads like the name does not\n * exist rather than like it is unreachable. That happened on 2026-09-07 with\n * `work-shift/days-of-week`: the parser was in the folder barrel, not here, and the consuming\n * lane wrote its own until the miss was found.\n */\nimport { WebhookDestination } from './webhook-destination/webhook-destination'\nimport { WebhookDestinationResolver } from './webhook-destination/webhook-destination-resolver'\nimport { WebhookOutboxEvent } from './webhook-outbox-event/webhook-outbox-event'\nimport { WebhookSequence } from './webhook-sequence/webhook-sequence'\n\nexport * from './webhook-destination/webhook-destination'\nexport * from './webhook-destination/webhook-destination-resolver'\nexport * from './webhook-outbox-event/webhook-outbox-event'\nexport * from './webhook-sequence/webhook-sequence'\n\n/*\n * ⚠ **The shell builds the database and the schema from these two names.** Without them the module's\n * tables are never created in any application and the destination door is in no schema - the files stand\n * and nothing reaches them. That was the state through 10.1.37 (found 2026-09-24 by the first application to\n * append: its database had no webhook table at all). `test/append-store-db.test.ts` holds this line.\n */\nexport const entities = [WebhookDestination, WebhookOutboxEvent, WebhookSequence]\n\nexport const schema = { resolverClasses: [WebhookDestinationResolver] }\n\n/*\n * The outbox machine, moved from operato-plant (ADR-0078 decision 2). Step ①: the pure judgement —\n * envelope, signature, reading the receiver's answer, settlement, what an answer does to our rows,\n * the sequence report, reconciliation, standing. None of these touch a store.\n */\nexport * from './outbox/delivery'\nexport * from './outbox/settlement'\nexport * from './outbox/row-effect'\nexport * from './outbox/seq-report'\nexport * from './outbox/reconciliation'\nexport * from './outbox/standing'\nexport * from './outbox/requeue-note'\nexport * from './outbox/page'\nexport * from './outbox/writing-transaction'\n\n/*\n * Step ②: the back row. Writing a fact down takes the caller's open transaction - in the type, not only\n * in a check - because the number and the row have to commit together or the number burns.\n */\nexport * from './outbox/append'\n\n/* What a delivery needs to say \"when\" and \"which request\" — moved from operato-plant with the dispatcher's\n * needs in mind (ADR-0078 decision 1). The inbound side reads the same two. */\nexport * from './observability/clock-offset'\nexport * from './observability/trace-context'\n\n/* Step ②: the back row. One dispatcher over a store interface; `webhookStore` is this module's own store. */\nexport * from './outbox/store'\nexport * from './outbox/dispatcher'\nexport * from './outbox/webhook-store'\nexport * from './outbox/webhook-append-store'\n\n/*\n * The pump: one round's judgement (`pump-round`), one round (`pump`), and the loop an application starts from its\n * boot hook (`pump-loop`). Keys come from the contract's ladder - the product-named `pump-gate` is gone\n * (ADR-0078 decision 5, re-ruled 2026-09-24).\n */\nexport * from './outbox/pump-round'\nexport * from './outbox/pump'\nexport * from './outbox/pump-loop'\n\n"]}

package/dist-server/service/outbox/pump-loop.d.ts

@@ -0,0 +1,38 @@
+import { EntityManager } from 'typeorm';
+import { type PumpSource } from './pump.js';
+/**
+ * **The pump's loop** — every few seconds, one round, on one instance of the cluster (ADR-0078 decision 6).
+ *
+ * ═════════════════════════════════════════════════════════════════════════
+ * ## The application starts it
+ *
+ * The module does not start anything on its own. An application that appends facts calls `startWebhookPump`
+ * from its boot hook, and passes the names its older installations used for keys, if it has any.
+ *
+ * ## One instance at a time, and one round at a time
+ *
+ * `running` stops a round starting while the last is still sending - it would send the same facts twice. It
+ * only guards this process; behind a load balancer two instances would both send, and the later one arriving
+ * first reads at the receiver as a gap. So each round runs under a lease.
+ *
+ * **A paused pump keeps the lease.** The pause lives in this process. Let go, another instance - which knows
+ * nothing of it - takes over at once and knocks on the receiver that asked us to wait.
+ *
+ * ## It stands without a key
+ *
+ * Keys are looked up per stream, per round, on the contract's ladder. A missing key is said once, naming the
+ * stream, and the facts wait. Refusing to start instead would say nothing at all about which domain lacks one.
+ * ═════════════════════════════════════════════════════════════════════════
+ */
+export interface WebhookPumpOptions {
+    intervalSeconds?: number;
+    batchSize?: number;
+    /** Names this product's installations used for keys before the contract's ladder. Most products have none. */
+    legacyBases?: readonly string[];
+}
+/** The source over this module's tables. */
+export declare function webhookPumpSource(manager: EntityManager): PumpSource;
+/** Start the loop. Starting twice does nothing. */
+export declare function startWebhookPump(options?: WebhookPumpOptions): void;
+/** Stop the loop. For tests and shutdown - left running, the process does not end. */
+export declare function stopWebhookPump(): void;

package/dist-server/service/outbox/pump-loop.js

@@ -0,0 +1,93 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.webhookPumpSource = webhookPumpSource;
+exports.startWebhookPump = startWebhookPump;
+exports.stopWebhookPump = stopWebhookPump;
+const env_1 = require("@things-factory/env");
+const shell_1 = require("@things-factory/shell");
+const task_lease_1 = require("@things-factory/task-lease");
+const webhook_destination_js_1 = require("../webhook-destination/webhook-destination.js");
+const webhook_outbox_event_js_1 = require("../webhook-outbox-event/webhook-outbox-event.js");
+const pump_js_1 = require("./pump.js");
+const webhook_store_js_1 = require("./webhook-store.js");
+const DEFAULTS = { intervalSeconds: 5, batchSize: 200 };
+const LEASE = 'webhook-outbox-pump';
+/** Longer than a round, so it does not lapse mid-batch; short enough that a dead instance is replaced soon. */
+function leaseTtlMs(intervalSeconds) {
+    return Math.max(60_000, intervalSeconds * 1000 * 6);
+}
+/** The source over this module's tables. */
+function webhookPumpSource(manager) {
+    return {
+        async pendingStreams() {
+            const rows = await manager
+                .createQueryBuilder(webhook_outbox_event_js_1.WebhookOutboxEvent, 'event')
+                .innerJoin('event.domain', 'domain')
+                .select('domain.id', 'domainId')
+                .addSelect('domain.subdomain', 'subdomain')
+                .addSelect('event.peer', 'peer')
+                .addSelect('event.scope', 'scope')
+                .where('event.status = :pending', { pending: webhook_outbox_event_js_1.WebhookOutboxStatus.Pending })
+                .groupBy('domain.id')
+                .addGroupBy('domain.subdomain')
+                .addGroupBy('event.peer')
+                .addGroupBy('event.scope')
+                .orderBy('domain.subdomain')
+                .addOrderBy('event.peer')
+                .addOrderBy('event.scope')
+                .getRawMany();
+            return rows.map(row => ({ domainId: row.domainId, subdomain: row.subdomain, peer: row.peer, scope: row.scope }));
+        },
+        async destinationOf(domainId, peer) {
+            const row = await manager.findOne(webhook_destination_js_1.WebhookDestination, { where: { domain: { id: domainId }, peer } });
+            return row ? { endpoint: row.endpoint, active: row.active, params: row.params ?? undefined } : undefined;
+        },
+        storeFor(stream) {
+            return (0, webhook_store_js_1.webhookStore)(manager, { domainId: stream.domainId, peer: stream.peer, scope: stream.scope });
+        }
+    };
+}
+let timer;
+let running = false;
+let state = (0, pump_js_1.newPumpState)();
+/** Start the loop. Starting twice does nothing. */
+function startWebhookPump(options = {}) {
+    if (timer) {
+        return;
+    }
+    const intervalSeconds = options.intervalSeconds ?? DEFAULTS.intervalSeconds;
+    const batchSize = options.batchSize ?? DEFAULTS.batchSize;
+    const say = { info: (what) => env_1.logger.info(what), warn: (what) => env_1.logger.warn(what), error: (what) => env_1.logger.error(what) };
+    env_1.logger.info(`[webhook] sending queued facts every ${intervalSeconds}s`);
+    timer = setInterval(async () => {
+        if (running) {
+            return;
+        }
+        running = true;
+        try {
+            await (0, task_lease_1.withLease)(LEASE, leaseTtlMs(intervalSeconds), async () => {
+                await (0, pump_js_1.pumpOnce)({ source: webhookPumpSource((0, shell_1.getRepository)(webhook_outbox_event_js_1.WebhookOutboxEvent).manager), env: process.env, fetch, say }, state, { batchSize, ...(options.legacyBases?.length ? { legacyBases: options.legacyBases } : {}) });
+                /* How much longer to rest after this round. Zero or less when not resting. */
+                return (0, pump_js_1.restingFor)(state, Date.now());
+            }, 
+            /* A paused pump keeps the lease a little past the pause (see the head of this file). */
+            { hold: restingFor => (restingFor > 0 ? restingFor + leaseTtlMs(intervalSeconds) : undefined) });
+        }
+        catch (error) {
+            /* A failed round must not stop the next one - stopped, facts pile up for good. */
+            env_1.logger.error(`[webhook] sending queued facts failed this round - ${String(error)}`);
+        }
+        finally {
+            running = false;
+        }
+    }, intervalSeconds * 1000);
+}
+/** Stop the loop. For tests and shutdown - left running, the process does not end. */
+function stopWebhookPump() {
+    if (timer) {
+        clearInterval(timer);
+        timer = undefined;
+    }
+    state = (0, pump_js_1.newPumpState)();
+}
+//# sourceMappingURL=pump-loop.js.map

package/dist-server/service/outbox/pump-loop.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pump-loop.js","sourceRoot":"","sources":["../../../server/service/outbox/pump-loop.ts"],"names":[],"mappings":";;AAqDA,8CAgCC;AAOD,4CAyCC;AAGD,0CAMC;AA5ID,6CAA4C;AAC5C,iDAAqD;AACrD,2DAAsD;AAEtD,0FAAkF;AAClF,6FAAyG;AACzG,uCAA+F;AAE/F,yDAAiD;AAkCjD,MAAM,QAAQ,GAAG,EAAE,eAAe,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAA;AACvD,MAAM,KAAK,GAAG,qBAAqB,CAAA;AAEnC,+GAA+G;AAC/G,SAAS,UAAU,CAAC,eAAuB;IACzC,OAAO,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,CAAA;AACrD,CAAC;AAED,4CAA4C;AAC5C,SAAgB,iBAAiB,CAAC,OAAsB;IACtD,OAAO;QACL,KAAK,CAAC,cAAc;YAClB,MAAM,IAAI,GAAG,MAAM,OAAO;iBACvB,kBAAkB,CAAC,4CAAkB,EAAE,OAAO,CAAC;iBAC/C,SAAS,CAAC,cAAc,EAAE,QAAQ,CAAC;iBACnC,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC;iBAC/B,SAAS,CAAC,kBAAkB,EAAE,WAAW,CAAC;iBAC1C,SAAS,CAAC,YAAY,EAAE,MAAM,CAAC;iBAC/B,SAAS,CAAC,aAAa,EAAE,OAAO,CAAC;iBACjC,KAAK,CAAC,yBAAyB,EAAE,EAAE,OAAO,EAAE,6CAAmB,CAAC,OAAO,EAAE,CAAC;iBAC1E,OAAO,CAAC,WAAW,CAAC;iBACpB,UAAU,CAAC,kBAAkB,CAAC;iBAC9B,UAAU,CAAC,YAAY,CAAC;iBACxB,UAAU,CAAC,aAAa,CAAC;iBACzB,OAAO,CAAC,kBAAkB,CAAC;iBAC3B,UAAU,CAAC,YAAY,CAAC;iBACxB,UAAU,CAAC,aAAa,CAAC;iBACzB,UAAU,EAAE,CAAA;YAEf,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAClH,CAAC;QAED,KAAK,CAAC,aAAa,CAAC,QAAgB,EAAE,IAAY;YAChD,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,2CAAkB,EAAE,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;YACpG,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS,CAAA;QAC1G,CAAC;QAED,QAAQ,CAAC,MAAqB;YAC5B,OAAO,IAAA,+BAAY,EAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAA;QACrG,CAAC;KACF,CAAA;AACH,CAAC;AAED,IAAI,KAAiD,CAAA;AACrD,IAAI,OAAO,GAAG,KAAK,CAAA;AACnB,IAAI,KAAK,GAAc,IAAA,sBAAY,GAAE,CAAA;AAErC,mDAAmD;AACnD,SAAgB,gBAAgB,CAAC,UAA8B,EAAE;IAC/D,IAAI,KAAK,EAAE,CAAC;QACV,OAAM;IACR,CAAC;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,QAAQ,CAAC,eAAe,CAAA;IAC3E,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS,CAAA;IACzD,MAAM,GAAG,GAAG,EAAE,IAAI,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,YAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,YAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,YAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAA;IAEjJ,YAAM,CAAC,IAAI,CAAC,wCAAwC,eAAe,GAAG,CAAC,CAAA;IAEvE,KAAK,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;QAC7B,IAAI,OAAO,EAAE,CAAC;YACZ,OAAM;QACR,CAAC;QAED,OAAO,GAAG,IAAI,CAAA;QACd,IAAI,CAAC;YACH,MAAM,IAAA,sBAAS,EACb,KAAK,EACL,UAAU,CAAC,eAAe,CAAC,EAC3B,KAAK,IAAI,EAAE;gBACT,MAAM,IAAA,kBAAQ,EACZ,EAAE,MAAM,EAAE,iBAAiB,CAAC,IAAA,qBAAa,EAAC,4CAAkB,CAAC,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EACtG,KAAK,EACL,EAAE,SAAS,EAAE,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAC5F,CAAA;gBAED,8EAA8E;gBAC9E,OAAO,IAAA,oBAAU,EAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;YACtC,CAAC;YACD,wFAAwF;YACxF,EAAE,IAAI,EAAE,UAAU,CAAC,EAAE,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,CAChG,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,kFAAkF;YAClF,YAAM,CAAC,KAAK,CAAC,sDAAsD,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACrF,CAAC;gBAAS,CAAC;YACT,OAAO,GAAG,KAAK,CAAA;QACjB,CAAC;IACH,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC,CAAA;AAC5B,CAAC;AAED,sFAAsF;AACtF,SAAgB,eAAe;IAC7B,IAAI,KAAK,EAAE,CAAC;QACV,aAAa,CAAC,KAAK,CAAC,CAAA;QACpB,KAAK,GAAG,SAAS,CAAA;IACnB,CAAC;IACD,KAAK,GAAG,IAAA,sBAAY,GAAE,CAAA;AACxB,CAAC","sourcesContent":["import { EntityManager } from 'typeorm'\n\nimport { logger } from '@things-factory/env'\nimport { getRepository } from '@things-factory/shell'\nimport { withLease } from '@things-factory/task-lease'\n\nimport { WebhookDestination } from '../webhook-destination/webhook-destination.js'\nimport { WebhookOutboxEvent, WebhookOutboxStatus } from '../webhook-outbox-event/webhook-outbox-event.js'\nimport { newPumpState, pumpOnce, restingFor, type PumpSource, type PumpState } from './pump.js'\nimport type { PendingStream } from './pump-round.js'\nimport { webhookStore } from './webhook-store.js'\n\n/**\n * **The pump's loop** — every few seconds, one round, on one instance of the cluster (ADR-0078 decision 6).\n *\n * ═════════════════════════════════════════════════════════════════════════\n * ## The application starts it\n *\n * The module does not start anything on its own. An application that appends facts calls `startWebhookPump`\n * from its boot hook, and passes the names its older installations used for keys, if it has any.\n *\n * ## One instance at a time, and one round at a time\n *\n * `running` stops a round starting while the last is still sending - it would send the same facts twice. It\n * only guards this process; behind a load balancer two instances would both send, and the later one arriving\n * first reads at the receiver as a gap. So each round runs under a lease.\n *\n * **A paused pump keeps the lease.** The pause lives in this process. Let go, another instance - which knows\n * nothing of it - takes over at once and knocks on the receiver that asked us to wait.\n *\n * ## It stands without a key\n *\n * Keys are looked up per stream, per round, on the contract's ladder. A missing key is said once, naming the\n * stream, and the facts wait. Refusing to start instead would say nothing at all about which domain lacks one.\n * ═════════════════════════════════════════════════════════════════════════\n */\n\nexport interface WebhookPumpOptions {\n  intervalSeconds?: number\n  batchSize?: number\n  /** Names this product's installations used for keys before the contract's ladder. Most products have none. */\n  legacyBases?: readonly string[]\n}\n\nconst DEFAULTS = { intervalSeconds: 5, batchSize: 200 }\nconst LEASE = 'webhook-outbox-pump'\n\n/** Longer than a round, so it does not lapse mid-batch; short enough that a dead instance is replaced soon. */\nfunction leaseTtlMs(intervalSeconds: number): number {\n  return Math.max(60_000, intervalSeconds * 1000 * 6)\n}\n\n/** The source over this module's tables. */\nexport function webhookPumpSource(manager: EntityManager): PumpSource {\n  return {\n    async pendingStreams(): Promise<PendingStream[]> {\n      const rows = await manager\n        .createQueryBuilder(WebhookOutboxEvent, 'event')\n        .innerJoin('event.domain', 'domain')\n        .select('domain.id', 'domainId')\n        .addSelect('domain.subdomain', 'subdomain')\n        .addSelect('event.peer', 'peer')\n        .addSelect('event.scope', 'scope')\n        .where('event.status = :pending', { pending: WebhookOutboxStatus.Pending })\n        .groupBy('domain.id')\n        .addGroupBy('domain.subdomain')\n        .addGroupBy('event.peer')\n        .addGroupBy('event.scope')\n        .orderBy('domain.subdomain')\n        .addOrderBy('event.peer')\n        .addOrderBy('event.scope')\n        .getRawMany()\n\n      return rows.map(row => ({ domainId: row.domainId, subdomain: row.subdomain, peer: row.peer, scope: row.scope }))\n    },\n\n    async destinationOf(domainId: string, peer: string) {\n      const row = await manager.findOne(WebhookDestination, { where: { domain: { id: domainId }, peer } })\n      return row ? { endpoint: row.endpoint, active: row.active, params: row.params ?? undefined } : undefined\n    },\n\n    storeFor(stream: PendingStream) {\n      return webhookStore(manager, { domainId: stream.domainId, peer: stream.peer, scope: stream.scope })\n    }\n  }\n}\n\nlet timer: ReturnType<typeof setInterval> | undefined\nlet running = false\nlet state: PumpState = newPumpState()\n\n/** Start the loop. Starting twice does nothing. */\nexport function startWebhookPump(options: WebhookPumpOptions = {}): void {\n  if (timer) {\n    return\n  }\n\n  const intervalSeconds = options.intervalSeconds ?? DEFAULTS.intervalSeconds\n  const batchSize = options.batchSize ?? DEFAULTS.batchSize\n  const say = { info: (what: string) => logger.info(what), warn: (what: string) => logger.warn(what), error: (what: string) => logger.error(what) }\n\n  logger.info(`[webhook] sending queued facts every ${intervalSeconds}s`)\n\n  timer = setInterval(async () => {\n    if (running) {\n      return\n    }\n\n    running = true\n    try {\n      await withLease(\n        LEASE,\n        leaseTtlMs(intervalSeconds),\n        async () => {\n          await pumpOnce(\n            { source: webhookPumpSource(getRepository(WebhookOutboxEvent).manager), env: process.env, fetch, say },\n            state,\n            { batchSize, ...(options.legacyBases?.length ? { legacyBases: options.legacyBases } : {}) }\n          )\n\n          /* How much longer to rest after this round. Zero or less when not resting. */\n          return restingFor(state, Date.now())\n        },\n        /* A paused pump keeps the lease a little past the pause (see the head of this file). */\n        { hold: restingFor => (restingFor > 0 ? restingFor + leaseTtlMs(intervalSeconds) : undefined) }\n      )\n    } catch (error) {\n      /* A failed round must not stop the next one - stopped, facts pile up for good. */\n      logger.error(`[webhook] sending queued facts failed this round - ${String(error)}`)\n    } finally {\n      running = false\n    }\n  }, intervalSeconds * 1000)\n}\n\n/** Stop the loop. For tests and shutdown - left running, the process does not end. */\nexport function stopWebhookPump(): void {\n  if (timer) {\n    clearInterval(timer)\n    timer = undefined\n  }\n  state = newPumpState()\n}\n"]}

package/dist-server/service/outbox/pump-round.js

@@ -14,7 +14,7 @@ function planStream(stream, destination, env, options) {
             stream,
             note: {
                 key: `no-destination:${subdomain}:${peer}`,
-                line: `[webhook] facts are piling up for ${where} but no destination is declared for that peer - set it on the webhook destinations screen. Nothing is lost.`
+                line: `[webhook] facts are piling up for ${where} but no destination is declared for that peer - add one with peer "${peer}" on the webhook destinations screen; a row under any other name does not carry them. Nothing is lost.`
             }
         };
     }

package/dist-server/service/outbox/pump-round.js.map

@@ -1 +1 @@
-{"version":3,"file":"pump-round.js","sourceRoot":"","sources":["../../../server/service/outbox/pump-round.ts"],"names":[],"mappings":";;AAkEA,8BAOC;AAzED,wDAA4G;AAkE5G,SAAgB,SAAS,CACvB,OAAiC,EACjC,aAAiF,EACjF,GAAuC,EACvC,UAAwB,EAAE;IAE1B,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,CAAA;AAC7G,CAAC;AAED,SAAS,UAAU,CAAC,MAAqB,EAAE,WAA2C,EAAE,GAAuC,EAAE,OAAqB;IACpJ,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,MAAM,CAAA;IAClC,MAAM,KAAK,GAAG,IAAI,SAAS,OAAO,IAAI,EAAE,CAAA;IAExC,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,CAAC;QAC3B,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,MAAM;YACN,IAAI,EAAE;gBACJ,GAAG,EAAE,kBAAkB,SAAS,IAAI,IAAI,EAAE;gBAC1C,IAAI,EAAE,qCAAqC,KAAK,6GAA6G;aAC9J;SACF,CAAA;IACH,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;QACxB,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,MAAM;YACN,IAAI,EAAE;gBACJ,GAAG,EAAE,cAAc,SAAS,IAAI,IAAI,EAAE;gBACtC,IAAI,EAAE,qCAAqC,KAAK,gGAAgG;aACjJ;SACF,CAAA;IACH,CAAC;IAED,MAAM,MAAM,GAAG,EAAE,OAAO,EAAE,SAAkB,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAA;IACrJ,MAAM,KAAK,GAAG,IAAA,gCAAiB,EAAC,GAAG,EAAE,MAAM,CAAC,CAAA;IAE5C;;;OAGG;IACH,IAAI,CAAC,CAAC,QAAQ,IAAI,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,MAAM;YACN,IAAI,EAAE;gBACJ,GAAG,EAAE,aAAa,SAAS,IAAI,IAAI,EAAE;gBACrC,IAAI,EAAE,qCAAqC,KAAK,gCAAgC,IAAA,uCAAwB,EAAC,KAAK,CAAC,KAAK,CAAC,sDAAsD;aAC5K;SACF,CAAA;IACH,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAA;IAChC,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,EAAE,eAAe,CAAA;IAE3D,OAAO;QACL,IAAI,EAAE,SAAS;QACf,MAAM;QACN,QAAQ,EAAE,WAAW,CAAC,QAAQ;QAC9B,MAAM;QACN,2DAA2D;QAC3D,OAAO,EAAE,OAAO,eAAe,KAAK,QAAQ,IAAI,eAAe,CAAC,CAAC,CAAC,EAAE,yBAAyB,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE;QACrH,QAAQ,EAAE,GAAG,SAAS,IAAI,IAAI,EAAE;QAChC,GAAG,CAAC,MAAM,CAAC,aAAa;YACtB,CAAC,CAAC;gBACE,IAAI,EAAE;oBACJ,GAAG,EAAE,iBAAiB,SAAS,IAAI,IAAI,EAAE;oBACzC,IAAI,EAAE,aAAa,KAAK,mBAAmB,MAAM,CAAC,IAAI,qEAAqE,IAAA,sCAAuB,EAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG;iBAC/M;aACF;YACH,CAAC,CAAC,EAAE,CAAC;KACR,CAAA;AACH,CAAC","sourcesContent":["import { webhookSecretCandidates, webhookSecretFrom, webhookSecretMissingNote } from '@operato/ops-contract'\n\n/**\n * **What one round of the pump does with each stream** — a pure judgement: no database, no network, and\n * the environment arrives as an argument (ADR-0078 decision 5 as re-ruled on 2026-09-24).\n *\n * ═════════════════════════════════════════════════════════════════════════\n * ## Silence is the failure this guards\n *\n * Facts can pile up for three reasons, and none of them fails anything: no destination is declared for\n * that peer, a person turned the destination off, or nothing can sign. Each sends the administrator\n * somewhere different - declare it, turn it back on, set a key - so each is its own sentence, and each\n * comes with a key the pump says it once by. Said every round, the line buries everything else.\n *\n * ## The key's name belongs to the contract\n *\n * `webhookSecretFrom` walks the contract's ladder - per peer, then per domain - and a product passes no\n * names of its own. Pinned to the peer, two products on one installation never share a key. A product that\n * ran before this ladder passes `legacyBases`; a key found there is used and said once, because a quiet\n * success means nobody ever moves to the new name.\n *\n * The missing-key sentence names the variables. That is for the process log only - the contract says so\n * (`webhookSecretMissingNote`) - and this sentence never goes into a response.\n * ═════════════════════════════════════════════════════════════════════════\n */\n\n/** One stream with facts waiting. The pump reads these off the pending rows. */\nexport interface PendingStream {\n  domainId: string\n  /** The tenant's name - what the key's name is built from and what a person reads in the log. */\n  subdomain: string\n  peer: string\n  scope: string\n}\n\n/** What the destination row says, read by the caller. */\nexport interface DestinationReading {\n  endpoint: string\n  active: boolean\n  params?: { receivingDomain?: string } & Record<string, unknown>\n}\n\nexport interface RoundNote {\n  /** The same cause on the same stream carries the same key, so it is said once. */\n  key: string\n  line: string\n}\n\nexport type RoundStep =\n  | {\n      kind: 'deliver'\n      stream: PendingStream\n      endpoint: string\n      secret: string\n      headers: Record<string, string>\n      /** Clock readings are kept per peer - two peers' clocks are not one clock. */\n      clockKey: string\n      note?: RoundNote\n    }\n  | { kind: 'wait'; stream: PendingStream; note: RoundNote }\n\nexport interface RoundOptions {\n  /** Names this product's installations used before the ladder. Absent for a product that never had one. */\n  legacyBases?: readonly string[]\n}\n\nexport function planRound(\n  streams: readonly PendingStream[],\n  destinationOf: (domainId: string, peer: string) => DestinationReading | undefined,\n  env: Record<string, string | undefined>,\n  options: RoundOptions = {}\n): RoundStep[] {\n  return streams.map(stream => planStream(stream, destinationOf(stream.domainId, stream.peer), env, options))\n}\n\nfunction planStream(stream: PendingStream, destination: DestinationReading | undefined, env: Record<string, string | undefined>, options: RoundOptions): RoundStep {\n  const { subdomain, peer } = stream\n  const where = `\"${subdomain}\" → ${peer}`\n\n  if (!destination?.endpoint) {\n    return {\n      kind: 'wait',\n      stream,\n      note: {\n        key: `no-destination:${subdomain}:${peer}`,\n        line: `[webhook] facts are piling up for ${where} but no destination is declared for that peer - set it on the webhook destinations screen. Nothing is lost.`\n      }\n    }\n  }\n\n  if (!destination.active) {\n    return {\n      kind: 'wait',\n      stream,\n      note: {\n        key: `turned-off:${subdomain}:${peer}`,\n        line: `[webhook] facts are piling up for ${where} - its destination is turned off. Nothing is lost; turn it back on when the receiver is ready.`\n      }\n    }\n  }\n\n  const lookup = { purpose: 'signing' as const, domain: subdomain, peer, ...(options.legacyBases?.length ? { legacyBases: options.legacyBases } : {}) }\n  const found = webhookSecretFrom(env, lookup)\n\n  /*\n   * The answer is a union. This package compiles without `strictNullChecks`, and there a check on `secret`\n   * does not narrow it - so ask which branch it is by the field only that branch has.\n   */\n  if (!('opened' in found)) {\n    return {\n      kind: 'wait',\n      stream,\n      note: {\n        key: `no-secret:${subdomain}:${peer}`,\n        line: `[webhook] facts are piling up for ${where} but nothing can sign them - ${webhookSecretMissingNote(found.tried)} Nothing is lost; they go as soon as a key is there.`\n      }\n    }\n  }\n\n  const { secret, opened } = found\n  const receivingDomain = destination.params?.receivingDomain\n\n  return {\n    kind: 'deliver',\n    stream,\n    endpoint: destination.endpoint,\n    secret,\n    /* The receiver's own tenant - never assumed to be ours. */\n    headers: typeof receivingDomain === 'string' && receivingDomain ? { 'x-things-factory-domain': receivingDomain } : {},\n    clockKey: `${subdomain}:${peer}`,\n    ...(opened.compatibility\n      ? {\n          note: {\n            key: `legacy-secret:${subdomain}:${peer}`,\n            line: `[webhook] ${where} is signed with ${opened.name}, a name kept for installations that ran before - move the key to ${webhookSecretCandidates({ purpose: 'signing', domain: subdomain, peer })[0].name}.`\n          }\n        }\n      : {})\n  }\n}\n"]}
+{"version":3,"file":"pump-round.js","sourceRoot":"","sources":["../../../server/service/outbox/pump-round.ts"],"names":[],"mappings":";;AAkEA,8BAOC;AAzED,wDAA4G;AAkE5G,SAAgB,SAAS,CACvB,OAAiC,EACjC,aAAiF,EACjF,GAAuC,EACvC,UAAwB,EAAE;IAE1B,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,CAAA;AAC7G,CAAC;AAED,SAAS,UAAU,CAAC,MAAqB,EAAE,WAA2C,EAAE,GAAuC,EAAE,OAAqB;IACpJ,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,MAAM,CAAA;IAClC,MAAM,KAAK,GAAG,IAAI,SAAS,OAAO,IAAI,EAAE,CAAA;IAExC,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,CAAC;QAC3B,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,MAAM;YACN,IAAI,EAAE;gBACJ,GAAG,EAAE,kBAAkB,SAAS,IAAI,IAAI,EAAE;gBAC1C,IAAI,EAAE,qCAAqC,KAAK,sEAAsE,IAAI,wGAAwG;aACnO;SACF,CAAA;IACH,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;QACxB,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,MAAM;YACN,IAAI,EAAE;gBACJ,GAAG,EAAE,cAAc,SAAS,IAAI,IAAI,EAAE;gBACtC,IAAI,EAAE,qCAAqC,KAAK,gGAAgG;aACjJ;SACF,CAAA;IACH,CAAC;IAED,MAAM,MAAM,GAAG,EAAE,OAAO,EAAE,SAAkB,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAA;IACrJ,MAAM,KAAK,GAAG,IAAA,gCAAiB,EAAC,GAAG,EAAE,MAAM,CAAC,CAAA;IAE5C;;;OAGG;IACH,IAAI,CAAC,CAAC,QAAQ,IAAI,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,MAAM;YACN,IAAI,EAAE;gBACJ,GAAG,EAAE,aAAa,SAAS,IAAI,IAAI,EAAE;gBACrC,IAAI,EAAE,qCAAqC,KAAK,gCAAgC,IAAA,uCAAwB,EAAC,KAAK,CAAC,KAAK,CAAC,sDAAsD;aAC5K;SACF,CAAA;IACH,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAA;IAChC,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,EAAE,eAAe,CAAA;IAE3D,OAAO;QACL,IAAI,EAAE,SAAS;QACf,MAAM;QACN,QAAQ,EAAE,WAAW,CAAC,QAAQ;QAC9B,MAAM;QACN,2DAA2D;QAC3D,OAAO,EAAE,OAAO,eAAe,KAAK,QAAQ,IAAI,eAAe,CAAC,CAAC,CAAC,EAAE,yBAAyB,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE;QACrH,QAAQ,EAAE,GAAG,SAAS,IAAI,IAAI,EAAE;QAChC,GAAG,CAAC,MAAM,CAAC,aAAa;YACtB,CAAC,CAAC;gBACE,IAAI,EAAE;oBACJ,GAAG,EAAE,iBAAiB,SAAS,IAAI,IAAI,EAAE;oBACzC,IAAI,EAAE,aAAa,KAAK,mBAAmB,MAAM,CAAC,IAAI,qEAAqE,IAAA,sCAAuB,EAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG;iBAC/M;aACF;YACH,CAAC,CAAC,EAAE,CAAC;KACR,CAAA;AACH,CAAC","sourcesContent":["import { webhookSecretCandidates, webhookSecretFrom, webhookSecretMissingNote } from '@operato/ops-contract'\n\n/**\n * **What one round of the pump does with each stream** — a pure judgement: no database, no network, and\n * the environment arrives as an argument (ADR-0078 decision 5 as re-ruled on 2026-09-24).\n *\n * ═════════════════════════════════════════════════════════════════════════\n * ## Silence is the failure this guards\n *\n * Facts can pile up for three reasons, and none of them fails anything: no destination is declared for\n * that peer, a person turned the destination off, or nothing can sign. Each sends the administrator\n * somewhere different - declare it, turn it back on, set a key - so each is its own sentence, and each\n * comes with a key the pump says it once by. Said every round, the line buries everything else.\n *\n * ## The key's name belongs to the contract\n *\n * `webhookSecretFrom` walks the contract's ladder - per peer, then per domain - and a product passes no\n * names of its own. Pinned to the peer, two products on one installation never share a key. A product that\n * ran before this ladder passes `legacyBases`; a key found there is used and said once, because a quiet\n * success means nobody ever moves to the new name.\n *\n * The missing-key sentence names the variables. That is for the process log only - the contract says so\n * (`webhookSecretMissingNote`) - and this sentence never goes into a response.\n * ═════════════════════════════════════════════════════════════════════════\n */\n\n/** One stream with facts waiting. The pump reads these off the pending rows. */\nexport interface PendingStream {\n  domainId: string\n  /** The tenant's name - what the key's name is built from and what a person reads in the log. */\n  subdomain: string\n  peer: string\n  scope: string\n}\n\n/** What the destination row says, read by the caller. */\nexport interface DestinationReading {\n  endpoint: string\n  active: boolean\n  params?: { receivingDomain?: string } & Record<string, unknown>\n}\n\nexport interface RoundNote {\n  /** The same cause on the same stream carries the same key, so it is said once. */\n  key: string\n  line: string\n}\n\nexport type RoundStep =\n  | {\n      kind: 'deliver'\n      stream: PendingStream\n      endpoint: string\n      secret: string\n      headers: Record<string, string>\n      /** Clock readings are kept per peer - two peers' clocks are not one clock. */\n      clockKey: string\n      note?: RoundNote\n    }\n  | { kind: 'wait'; stream: PendingStream; note: RoundNote }\n\nexport interface RoundOptions {\n  /** Names this product's installations used before the ladder. Absent for a product that never had one. */\n  legacyBases?: readonly string[]\n}\n\nexport function planRound(\n  streams: readonly PendingStream[],\n  destinationOf: (domainId: string, peer: string) => DestinationReading | undefined,\n  env: Record<string, string | undefined>,\n  options: RoundOptions = {}\n): RoundStep[] {\n  return streams.map(stream => planStream(stream, destinationOf(stream.domainId, stream.peer), env, options))\n}\n\nfunction planStream(stream: PendingStream, destination: DestinationReading | undefined, env: Record<string, string | undefined>, options: RoundOptions): RoundStep {\n  const { subdomain, peer } = stream\n  const where = `\"${subdomain}\" → ${peer}`\n\n  if (!destination?.endpoint) {\n    return {\n      kind: 'wait',\n      stream,\n      note: {\n        key: `no-destination:${subdomain}:${peer}`,\n        line: `[webhook] facts are piling up for ${where} but no destination is declared for that peer - add one with peer \"${peer}\" on the webhook destinations screen; a row under any other name does not carry them. Nothing is lost.`\n      }\n    }\n  }\n\n  if (!destination.active) {\n    return {\n      kind: 'wait',\n      stream,\n      note: {\n        key: `turned-off:${subdomain}:${peer}`,\n        line: `[webhook] facts are piling up for ${where} - its destination is turned off. Nothing is lost; turn it back on when the receiver is ready.`\n      }\n    }\n  }\n\n  const lookup = { purpose: 'signing' as const, domain: subdomain, peer, ...(options.legacyBases?.length ? { legacyBases: options.legacyBases } : {}) }\n  const found = webhookSecretFrom(env, lookup)\n\n  /*\n   * The answer is a union. This package compiles without `strictNullChecks`, and there a check on `secret`\n   * does not narrow it - so ask which branch it is by the field only that branch has.\n   */\n  if (!('opened' in found)) {\n    return {\n      kind: 'wait',\n      stream,\n      note: {\n        key: `no-secret:${subdomain}:${peer}`,\n        line: `[webhook] facts are piling up for ${where} but nothing can sign them - ${webhookSecretMissingNote(found.tried)} Nothing is lost; they go as soon as a key is there.`\n      }\n    }\n  }\n\n  const { secret, opened } = found\n  const receivingDomain = destination.params?.receivingDomain\n\n  return {\n    kind: 'deliver',\n    stream,\n    endpoint: destination.endpoint,\n    secret,\n    /* The receiver's own tenant - never assumed to be ours. */\n    headers: typeof receivingDomain === 'string' && receivingDomain ? { 'x-things-factory-domain': receivingDomain } : {},\n    clockKey: `${subdomain}:${peer}`,\n    ...(opened.compatibility\n      ? {\n          note: {\n            key: `legacy-secret:${subdomain}:${peer}`,\n            line: `[webhook] ${where} is signed with ${opened.name}, a name kept for installations that ran before - move the key to ${webhookSecretCandidates({ purpose: 'signing', domain: subdomain, peer })[0].name}.`\n          }\n        }\n      : {})\n  }\n}\n"]}

package/dist-server/service/outbox/pump.d.ts

@@ -0,0 +1,75 @@
+import { type Say } from './dispatcher.js';
+import { type DestinationReading, type PendingStream, type RoundOptions } from './pump-round.js';
+import type { OutboxStore } from './store.js';
+/**
+ * **One round of the pump** — every stream with facts waiting, one delivery each (ADR-0078 decision 6).
+ *
+ * ═════════════════════════════════════════════════════════════════════════
+ * Built against operato-plant's `mes-outbox-pump.ts`, which learned these by running:
+ *
+ * - **Asked to wait, nothing more goes to that receiver until the wait is over.** Ignoring a 429 does not crash
+ *   anything - the receiver drops facts instead.
+ * - **An answer only a person can fix pauses that stream.** A wrong key gets the same 401 every five seconds,
+ *   knocks on the receiver and buries the log. It pauses only what it is about - see `PumpState.pauses`.
+ * - **Each cause of facts piling up is said once.** Said every round, the line covers everything else.
+ *
+ * Nothing here touches TypeORM, the process or a timer - the source, the environment, `fetch` and the clock are
+ * handed in, so every branch runs in a test. The loop, the lease and the tables are in `pump-loop.ts`.
+ * ═════════════════════════════════════════════════════════════════════════
+ */
+/** Where a round reads from. `pump-loop.ts` supplies one over this module's tables. */
+export interface PumpSource {
+    pendingStreams(): Promise<PendingStream[]>;
+    destinationOf(domainId: string, peer: string): Promise<DestinationReading | undefined>;
+    storeFor(stream: PendingStream): OutboxStore;
+}
+export interface PumpDeps {
+    source: PumpSource;
+    env: Record<string, string | undefined>;
+    fetch: typeof fetch;
+    say: Say;
+    nowMs?: () => number;
+}
+/**
+ * One pause: until when, and why. The same reason is not said twice in a row.
+ */
+interface Pause {
+    until: number;
+    why: string;
+}
+/** What a pump remembers between rounds. One per running pump. */
+export interface PumpState {
+    /**
+     * Pauses **by what they are about** (Plant, 2026-09-24). The first cut held one pause for the whole pump, as
+     * operato-plant's pump does: a tenant with a *wrong* key paused everything, and since the round goes in a fixed
+     * order the tenants behind it never sent again - measured, five minutes, zero sent. So each cause pauses its
+     * own unit:
+     *
+     * ```
+     * receiver:<endpoint>          asked to wait     it is that receiver's load, whoever the stream belongs to
+     * stream:<domain>/<peer>        signature refused a key is per domain and peer
+     * stream:<domain>/<peer>/<scope> stop            the answer was about what this stream sent
+     * ```
+     *
+     * Waiting is keyed by the endpoint rather than the peer's name: two domains may both call their twin `twin`
+     * and mean two different servers.
+     */
+    pauses: Map<string, Pause>;
+    /** Notes said, by key. A key whose cause is gone this round is forgotten, so a cause that returns is said again. */
+    said: Set<string>;
+}
+export declare function newPumpState(): PumpState;
+/**
+ * How much longer the longest pause stands - what the lease should be held past. Zero or less when nothing rests.
+ *
+ * Holding past the longest keeps the rule the single pause had: while anything rests, no other instance takes over
+ * and knocks on it. Streams that are not paused still go out - they go out from this instance.
+ */
+export declare function restingFor(state: PumpState, nowMs: number): number;
+/** How long to rest after an answer that resending will not change. */
+export declare const PAUSE_AFTER_UNRECOVERABLE_MS = 60000;
+export interface PumpOptions extends RoundOptions {
+    batchSize: number;
+}
+export declare function pumpOnce(deps: PumpDeps, state: PumpState, options: PumpOptions): Promise<void>;
+export {};

package/dist-server/service/outbox/pump.js

@@ -0,0 +1,87 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PAUSE_AFTER_UNRECOVERABLE_MS = void 0;
+exports.newPumpState = newPumpState;
+exports.restingFor = restingFor;
+exports.pumpOnce = pumpOnce;
+const dispatcher_js_1 = require("./dispatcher.js");
+const pump_round_js_1 = require("./pump-round.js");
+function newPumpState() {
+    return { pauses: new Map(), said: new Set() };
+}
+/**
+ * How much longer the longest pause stands - what the lease should be held past. Zero or less when nothing rests.
+ *
+ * Holding past the longest keeps the rule the single pause had: while anything rests, no other instance takes over
+ * and knocks on it. Streams that are not paused still go out - they go out from this instance.
+ */
+function restingFor(state, nowMs) {
+    let latest = 0;
+    for (const pause of state.pauses.values()) {
+        latest = Math.max(latest, pause.until);
+    }
+    return latest - nowMs;
+}
+/** How long to rest after an answer that resending will not change. */
+exports.PAUSE_AFTER_UNRECOVERABLE_MS = 60_000;
+async function pumpOnce(deps, state, options) {
+    const now = deps.nowMs ?? (() => Date.now());
+    /* A pause that is over is forgotten - the next time the same answer comes, it has to be said again. */
+    for (const [key, pause] of state.pauses) {
+        if (pause.until <= now())
+            state.pauses.delete(key);
+    }
+    const resting = (key) => (state.pauses.get(key)?.until ?? 0) > now();
+    const pause = (key, forMs, why) => {
+        const standing = state.pauses.get(key);
+        if (standing?.why !== why) {
+            deps.say.warn(`[webhook] ${why} - waiting ${Math.round(forMs / 1000)}s`);
+        }
+        state.pauses.set(key, { until: Math.max(standing?.until ?? 0, now() + forMs), why });
+    };
+    const streams = await deps.source.pendingStreams();
+    const destinations = new Map();
+    for (const stream of streams) {
+        const key = `${stream.domainId}\u0000${stream.peer}`;
+        if (!destinations.has(key)) {
+            destinations.set(key, await deps.source.destinationOf(stream.domainId, stream.peer));
+        }
+    }
+    const steps = (0, pump_round_js_1.planRound)(streams, (domainId, peer) => destinations.get(`${domainId}\u0000${peer}`), deps.env, options);
+    const saidThisRound = new Set();
+    for (const step of steps) {
+        if (step.note) {
+            saidThisRound.add(step.note.key);
+            if (!state.said.has(step.note.key)) {
+                state.said.add(step.note.key);
+                deps.say.warn(step.note.line);
+            }
+        }
+        if (step.kind === 'wait') {
+            continue;
+        }
+        const { domainId, peer, scope } = step.stream;
+        const receiver = `receiver:${step.endpoint}`;
+        const signer = `stream:${domainId}/${peer}`;
+        const stream = `stream:${domainId}/${peer}/${scope}`;
+        if (resting(receiver) || resting(signer) || resting(stream)) {
+            continue;
+        }
+        const report = await (0, dispatcher_js_1.dispatchPending)(deps.source.storeFor(step.stream), { endpoint: step.endpoint, secret: step.secret, headers: step.headers, clockKey: step.clockKey, batchSize: options.batchSize }, { fetch: deps.fetch, say: deps.say, now: () => new Date(now()) });
+        if (report.retryAfterMs) {
+            pause(receiver, report.retryAfterMs, `${step.endpoint} asked us to slow down`);
+        }
+        else if (report.outcome.kind === 'stop') {
+            pause(stream, exports.PAUSE_AFTER_UNRECOVERABLE_MS, report.outcome.reason);
+        }
+        else if (report.outcome.kind === 'unauthorized') {
+            pause(signer, exports.PAUSE_AFTER_UNRECOVERABLE_MS, `${step.endpoint} did not accept our signature for ${step.stream.subdomain} - check the key on both sides, then it will try again`);
+        }
+    }
+    /* A note whose cause did not come up this round is forgotten. If the cause comes back, it is said again. */
+    for (const key of [...state.said]) {
+        if (!saidThisRound.has(key))
+            state.said.delete(key);
+    }
+}
+//# sourceMappingURL=pump.js.map

package/dist-server/service/outbox/pump.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pump.js","sourceRoot":"","sources":["../../../server/service/outbox/pump.ts"],"names":[],"mappings":";;;AAkEA,oCAEC;AAQD,gCAMC;AASD,4BAwEC;AAnKD,mDAA2D;AAC3D,mDAA2G;AAiE3G,SAAgB,YAAY;IAC1B,OAAO,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,CAAA;AAC/C,CAAC;AAED;;;;;GAKG;AACH,SAAgB,UAAU,CAAC,KAAgB,EAAE,KAAa;IACxD,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QAC1C,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;IACxC,CAAC;IACD,OAAO,MAAM,GAAG,KAAK,CAAA;AACvB,CAAC;AAED,uEAAuE;AAC1D,QAAA,4BAA4B,GAAG,MAAM,CAAA;AAM3C,KAAK,UAAU,QAAQ,CAAC,IAAc,EAAE,KAAgB,EAAE,OAAoB;IACnF,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;IAE5C,uGAAuG;IACvG,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACxC,IAAI,KAAK,CAAC,KAAK,IAAI,GAAG,EAAE;YAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,GAAG,EAAE,CAAA;IAE5E,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,KAAa,EAAE,GAAW,EAAE,EAAE;QACxD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACtC,IAAI,QAAQ,EAAE,GAAG,KAAK,GAAG,EAAE,CAAC;YAC1B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,GAAG,cAAc,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAA;QAC1E,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;IACtF,CAAC,CAAA;IAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;IAElD,MAAM,YAAY,GAAG,IAAI,GAAG,EAA0C,CAAA;IACtE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,QAAQ,SAAS,MAAM,CAAC,IAAI,EAAE,CAAA;QACpD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;QACtF,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,IAAA,yBAAS,EAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,QAAQ,SAAS,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACrH,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAA;IAEvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAChC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC/B,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzB,SAAQ;QACV,CAAC;QAED,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,MAAM,CAAA;QAC7C,MAAM,QAAQ,GAAG,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAA;QAC5C,MAAM,MAAM,GAAG,UAAU,QAAQ,IAAI,IAAI,EAAE,CAAA;QAC3C,MAAM,MAAM,GAAG,UAAU,QAAQ,IAAI,IAAI,IAAI,KAAK,EAAE,CAAA;QAEpD,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5D,SAAQ;QACV,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAA,+BAAe,EAClC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EACjC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,EAC9H,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CACjE,CAAA;QAED,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;YACxB,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,QAAQ,wBAAwB,CAAC,CAAA;QAChF,CAAC;aAAM,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC1C,KAAK,CAAC,MAAM,EAAE,oCAA4B,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QACpE,CAAC;aAAM,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAClD,KAAK,CAAC,MAAM,EAAE,oCAA4B,EAAE,GAAG,IAAI,CAAC,QAAQ,qCAAqC,IAAI,CAAC,MAAM,CAAC,SAAS,wDAAwD,CAAC,CAAA;QACjL,CAAC;IACH,CAAC;IAED,4GAA4G;IAC5G,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAClC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IACrD,CAAC;AACH,CAAC","sourcesContent":["import { dispatchPending, type Say } from './dispatcher.js'\nimport { planRound, type DestinationReading, type PendingStream, type RoundOptions } from './pump-round.js'\nimport type { OutboxStore } from './store.js'\n\n/**\n * **One round of the pump** — every stream with facts waiting, one delivery each (ADR-0078 decision 6).\n *\n * ═════════════════════════════════════════════════════════════════════════\n * Built against operato-plant's `mes-outbox-pump.ts`, which learned these by running:\n *\n * - **Asked to wait, nothing more goes to that receiver until the wait is over.** Ignoring a 429 does not crash\n *   anything - the receiver drops facts instead.\n * - **An answer only a person can fix pauses that stream.** A wrong key gets the same 401 every five seconds,\n *   knocks on the receiver and buries the log. It pauses only what it is about - see `PumpState.pauses`.\n * - **Each cause of facts piling up is said once.** Said every round, the line covers everything else.\n *\n * Nothing here touches TypeORM, the process or a timer - the source, the environment, `fetch` and the clock are\n * handed in, so every branch runs in a test. The loop, the lease and the tables are in `pump-loop.ts`.\n * ═════════════════════════════════════════════════════════════════════════\n */\n\n/** Where a round reads from. `pump-loop.ts` supplies one over this module's tables. */\nexport interface PumpSource {\n  pendingStreams(): Promise<PendingStream[]>\n  destinationOf(domainId: string, peer: string): Promise<DestinationReading | undefined>\n  storeFor(stream: PendingStream): OutboxStore\n}\n\nexport interface PumpDeps {\n  source: PumpSource\n  env: Record<string, string | undefined>\n  fetch: typeof fetch\n  say: Say\n  nowMs?: () => number\n}\n\n/**\n * One pause: until when, and why. The same reason is not said twice in a row.\n */\ninterface Pause {\n  until: number\n  why: string\n}\n\n/** What a pump remembers between rounds. One per running pump. */\nexport interface PumpState {\n  /**\n   * Pauses **by what they are about** (Plant, 2026-09-24). The first cut held one pause for the whole pump, as\n   * operato-plant's pump does: a tenant with a *wrong* key paused everything, and since the round goes in a fixed\n   * order the tenants behind it never sent again - measured, five minutes, zero sent. So each cause pauses its\n   * own unit:\n   *\n   * ```\n   * receiver:<endpoint>          asked to wait     it is that receiver's load, whoever the stream belongs to\n   * stream:<domain>/<peer>        signature refused a key is per domain and peer\n   * stream:<domain>/<peer>/<scope> stop            the answer was about what this stream sent\n   * ```\n   *\n   * Waiting is keyed by the endpoint rather than the peer's name: two domains may both call their twin `twin`\n   * and mean two different servers.\n   */\n  pauses: Map<string, Pause>\n  /** Notes said, by key. A key whose cause is gone this round is forgotten, so a cause that returns is said again. */\n  said: Set<string>\n}\n\nexport function newPumpState(): PumpState {\n  return { pauses: new Map(), said: new Set() }\n}\n\n/**\n * How much longer the longest pause stands - what the lease should be held past. Zero or less when nothing rests.\n *\n * Holding past the longest keeps the rule the single pause had: while anything rests, no other instance takes over\n * and knocks on it. Streams that are not paused still go out - they go out from this instance.\n */\nexport function restingFor(state: PumpState, nowMs: number): number {\n  let latest = 0\n  for (const pause of state.pauses.values()) {\n    latest = Math.max(latest, pause.until)\n  }\n  return latest - nowMs\n}\n\n/** How long to rest after an answer that resending will not change. */\nexport const PAUSE_AFTER_UNRECOVERABLE_MS = 60_000\n\nexport interface PumpOptions extends RoundOptions {\n  batchSize: number\n}\n\nexport async function pumpOnce(deps: PumpDeps, state: PumpState, options: PumpOptions): Promise<void> {\n  const now = deps.nowMs ?? (() => Date.now())\n\n  /* A pause that is over is forgotten - the next time the same answer comes, it has to be said again. */\n  for (const [key, pause] of state.pauses) {\n    if (pause.until <= now()) state.pauses.delete(key)\n  }\n\n  const resting = (key: string) => (state.pauses.get(key)?.until ?? 0) > now()\n\n  const pause = (key: string, forMs: number, why: string) => {\n    const standing = state.pauses.get(key)\n    if (standing?.why !== why) {\n      deps.say.warn(`[webhook] ${why} - waiting ${Math.round(forMs / 1000)}s`)\n    }\n    state.pauses.set(key, { until: Math.max(standing?.until ?? 0, now() + forMs), why })\n  }\n\n  const streams = await deps.source.pendingStreams()\n\n  const destinations = new Map<string, DestinationReading | undefined>()\n  for (const stream of streams) {\n    const key = `${stream.domainId}\\u0000${stream.peer}`\n    if (!destinations.has(key)) {\n      destinations.set(key, await deps.source.destinationOf(stream.domainId, stream.peer))\n    }\n  }\n\n  const steps = planRound(streams, (domainId, peer) => destinations.get(`${domainId}\\u0000${peer}`), deps.env, options)\n  const saidThisRound = new Set<string>()\n\n  for (const step of steps) {\n    if (step.note) {\n      saidThisRound.add(step.note.key)\n      if (!state.said.has(step.note.key)) {\n        state.said.add(step.note.key)\n        deps.say.warn(step.note.line)\n      }\n    }\n\n    if (step.kind === 'wait') {\n      continue\n    }\n\n    const { domainId, peer, scope } = step.stream\n    const receiver = `receiver:${step.endpoint}`\n    const signer = `stream:${domainId}/${peer}`\n    const stream = `stream:${domainId}/${peer}/${scope}`\n\n    if (resting(receiver) || resting(signer) || resting(stream)) {\n      continue\n    }\n\n    const report = await dispatchPending(\n      deps.source.storeFor(step.stream),\n      { endpoint: step.endpoint, secret: step.secret, headers: step.headers, clockKey: step.clockKey, batchSize: options.batchSize },\n      { fetch: deps.fetch, say: deps.say, now: () => new Date(now()) }\n    )\n\n    if (report.retryAfterMs) {\n      pause(receiver, report.retryAfterMs, `${step.endpoint} asked us to slow down`)\n    } else if (report.outcome.kind === 'stop') {\n      pause(stream, PAUSE_AFTER_UNRECOVERABLE_MS, report.outcome.reason)\n    } else if (report.outcome.kind === 'unauthorized') {\n      pause(signer, PAUSE_AFTER_UNRECOVERABLE_MS, `${step.endpoint} did not accept our signature for ${step.stream.subdomain} - check the key on both sides, then it will try again`)\n    }\n  }\n\n  /* A note whose cause did not come up this round is forgotten. If the cause comes back, it is said again. */\n  for (const key of [...state.said]) {\n    if (!saidThisRound.has(key)) state.said.delete(key)\n  }\n}\n"]}