@requence/service 0.8.2-edge.1151 → 0.8.2-edge.1154

package/build/types/helpers/src/utils/amqpPublishRetry.d.ts

@@ -11,35 +11,130 @@ export declare const DEFAULT_AMQP_PUBLISH_RETRY_ATTEMPTS = 3;
  * out and retries) and far below the incident's multi-second stalls.
  */
 export declare const PUBLISH_STARVATION_THRESHOLD_MS = 1000;
-export interface LoopStarvationProbe {
+/**
+ * How long to wait, after a confirm deadline elapsed on a starved loop, before
+ * reading the link's incarnation for the second time.
+ *
+ * The stall that delayed the confirm delayed the socket's own close callback
+ * too, and Node runs the timers phase BEFORE the poll phase — so the deadline's
+ * timer can fire before the disconnect it was caused by has even been read off
+ * the socket. Deciding on the spot would call that lost message delivered. One
+ * settle turn puts the read after the poll phase; the margin above that costs
+ * nothing, because this only ever delays a publish that already spent its full
+ * timeout.
+ */
+export declare const PUBLISH_LINK_SETTLE_MS = 250;
+/**
+ * Reads the publish link's incarnation: a number that changes on every
+ * (re)connect, and `null` for as long as the link is down.
+ *
+ * Encoding "down" as `null` is what lets a single equality carry the whole
+ * liveness question — `start !== null && start === end` holds only if the link
+ * was up at both ends AND never cycled in between.
+ */
+export type PublishLinkEpoch = () => number | null;
+export interface PublishLinkTracker {
+    /** Pass this to `createPublishWindowProbe`. */
+    epoch: PublishLinkEpoch;
+    /** Call when the publish channel is (re)connected and usable. */
+    markConnected: () => void;
+    /** Call when the connection or the publish channel is lost. */
+    markDisconnected: () => void;
+}
+/**
+ * Tracks whether a publisher's link to the broker is up, and how many times it
+ * has been re-made. Every AMQP client wires this to its own connection and
+ * channel events; the counter is deliberately not derived from the library's
+ * `isConnected()`, because a drop-and-recover inside one publish window is
+ * invisible to a boolean and must not read as "the link held".
+ */
+export declare function createPublishLinkTracker(): PublishLinkTracker;
+export interface PublishTimeoutVerdict {
+    /** Longest stretch (ms) in which the loop could not run a callback. */
+    starvedMs: number;
+    /** The loop was starved past the threshold during the window. */
+    starved: boolean;
+    /** The link was down at some point while the confirm was outstanding. */
+    linkLost: boolean;
+    /**
+     * A reading taken while the message was still the library's responsibility
+     * found the publisher's send queue empty, which proves this message was handed
+     * to the socket. `false` means unproven, not "provably still queued".
+     */
+    leftSendQueue: boolean;
+    /**
+     * Only true when the loop was starved AND the link held throughout AND the
+     * message provably left the send queue: it is in the broker's hands and must
+     * NOT be re-sent.
+     */
+    delivered: boolean;
+}
+export interface PublishWindowProbe {
+    /**
+     * Take a send-queue reading now. Call it synchronously right after handing the
+     * message to the publisher: `ChannelWrapper.publish()` enqueues AND runs its
+     * send worker synchronously, so on a healthy channel the queue is already
+     * empty by the time it returns — which is the sharpest reading available.
+     */
+    observeSendQueue: () => void;
+    /** Stop sampling. Idempotent; call it on the success path. */
+    stop: () => void;
     /**
-     * Stop sampling and return the longest stretch (ms) during which the loop
-     * could not run a callback since the probe started. Idempotent — subsequent
-     * calls return the same value without re-reading the clock.
+     * Decide what an elapsed confirm deadline means. Stops sampling, then — only
+     * where the answer could still flip — waits out `settleMs` before re-reading
+     * the link. Idempotent: subsequent calls return the first verdict.
      */
-    stop: () => number;
+    readTimeoutVerdict: () => Promise<PublishTimeoutVerdict>;
+}
+interface PublishWindowProbeOptions {
+    epoch: PublishLinkEpoch;
+    /**
+     * Reads how many messages the publisher still holds unsent — the wrapper's
+     * `queueLength()`. Required, not optional: a publisher that skipped it would
+     * silently inherit the pre-#139 verdict and drop messages again.
+     */
+    sendQueueLength: () => number;
+    now?: () => number;
+    settleMs?: number;
 }
 /**
- * Measure how long the event loop was starved while a single publish is in
- * flight. It runs a self-rescheduling timer and, crucially, also reads the
- * pending gap synchronously at `stop()`: a stall that blocked the loop right up
- * to `stop()` is caught by `now() - expected` even though the timer never got a
- * chance to fire. That makes the reading immune to the very stall it measures
- * and independent of callback ordering after the loop resumes.
+ * Watch a single publish's in-flight window: how long the event loop was
+ * starved, whether the link to the broker survived, and whether the message
+ * provably left the publisher's send queue.
+ *
+ * The starvation term runs a self-rescheduling timer and, crucially, also reads
+ * the pending gap synchronously when sampling stops: a stall that blocked the
+ * loop right up to that moment is caught by `now() - expected` even though the
+ * timer never got a chance to fire. That makes the reading immune to the very
+ * stall it measures and independent of callback ordering after the loop resumes.
+ *
+ * The send-queue term samples on the same tick, and the caller adds the sharpest
+ * reading of all through `observeSendQueue()` — the synchronous one taken right
+ * after the publisher accepted the message, which no stall can delay.
  *
  * The clock is monotonic (`performance.now`), never wall time — the operator's
  * lag monitor uses the same, and a wall-clock step (NTP) must not read as
- * starvation and suppress a legitimate retry. `now` is injectable for tests.
+ * starvation and suppress a legitimate retry. `now` and `settleMs` are
+ * injectable for tests.
  */
-export declare function createLoopStarvationProbe({ now, }?: {
-    now?: () => number;
-}): LoopStarvationProbe;
+export declare function createPublishWindowProbe({ epoch, sendQueueLength, now, settleMs, }: PublishWindowProbeOptions): PublishWindowProbe;
 /**
- * Whether a confirm deadline that elapsed should be read as loop starvation
- * rather than a lost publish. When it returns true the publish must NOT be
- * re-sent: the message is in the broker's hands and only the confirm was late.
+ * The starvation term of the verdict — whether the loop was unresponsive long
+ * enough for the confirm deadline to be an artefact of the stall. On its own it
+ * is NOT a reason to suppress a re-send; see `createPublishWindowProbe`.
  */
 export declare function isStarvedPublishTimeout(starvedMs: number): boolean;
+/**
+ * The error a non-delivered confirm deadline becomes. Shared so the operator and
+ * the task SDK report the same thing, and worded to say WHICH of the two failure
+ * shapes happened — a starved window whose link died reads as a healthy broker
+ * timing out unless the message says otherwise.
+ *
+ * The `RabbitMQ publish timed out` prefix is load-bearing:
+ * `isRetryablePublishError` matches on it.
+ */
+export declare function formatPublishTimeoutError({ starved, starvedMs, linkLost, leftSendQueue, }: PublishTimeoutVerdict): string;
 export declare function isRetryablePublishError(error: unknown): boolean;
 export declare function computeAmqpPublishRetryDelay(attempt: number): number;
+export {};
 //# sourceMappingURL=amqpPublishRetry.d.ts.map

package/build/types/helpers/src/utils/amqpPublishRetry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"amqpPublishRetry.d.ts","sourceRoot":"","sources":["../../../../../../helpers/src/utils/amqpPublishRetry.ts"],"names":[],"mappings":"AA8BA,eAAO,MAAM,+BAA+B,QAAS,CAAA;AACrD,eAAO,MAAM,mCAAmC,IAAI,CAAA;AAqBpD;;;;;;;;;GASG;AACH,eAAO,MAAM,+BAA+B,OAAQ,CAAA;AAEpD,MAAM,WAAW,mBAAmB;IAClC;;;;OAIG;IACH,IAAI,EAAE,MAAM,MAAM,CAAA;CACnB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,yBAAyB,CAAC,EACxC,GAA6B,GAC9B,GAAE;IAAE,GAAG,CAAC,EAAE,MAAM,MAAM,CAAA;CAAO,GAAG,mBAAmB,CAyBnD;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,WAExD;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,WAQrD;AAID,wBAAgB,4BAA4B,CAAC,OAAO,EAAE,MAAM,UAM3D"}
+{"version":3,"file":"amqpPublishRetry.d.ts","sourceRoot":"","sources":["../../../../../../helpers/src/utils/amqpPublishRetry.ts"],"names":[],"mappings":"AA8BA,eAAO,MAAM,+BAA+B,QAAS,CAAA;AACrD,eAAO,MAAM,mCAAmC,IAAI,CAAA;AAoEpD;;;;;;;;;GASG;AACH,eAAO,MAAM,+BAA+B,OAAQ,CAAA;AAEpD;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,sBAAsB,MAAM,CAAA;AAEzC;;;;;;;GAOG;AACH,MAAM,MAAM,gBAAgB,GAAG,MAAM,MAAM,GAAG,IAAI,CAAA;AAElD,MAAM,WAAW,kBAAkB;IACjC,+CAA+C;IAC/C,KAAK,EAAE,gBAAgB,CAAA;IACvB,iEAAiE;IACjE,aAAa,EAAE,MAAM,IAAI,CAAA;IACzB,+DAA+D;IAC/D,gBAAgB,EAAE,MAAM,IAAI,CAAA;CAC7B;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,IAAI,kBAAkB,CAc7D;AAED,MAAM,WAAW,qBAAqB;IACpC,uEAAuE;IACvE,SAAS,EAAE,MAAM,CAAA;IACjB,iEAAiE;IACjE,OAAO,EAAE,OAAO,CAAA;IAChB,yEAAyE;IACzE,QAAQ,EAAE,OAAO,CAAA;IACjB;;;;OAIG;IACH,aAAa,EAAE,OAAO,CAAA;IACtB;;;;OAIG;IACH,SAAS,EAAE,OAAO,CAAA;CACnB;AAED,MAAM,WAAW,kBAAkB;IACjC;;;;;OAKG;IACH,gBAAgB,EAAE,MAAM,IAAI,CAAA;IAC5B,8DAA8D;IAC9D,IAAI,EAAE,MAAM,IAAI,CAAA;IAChB;;;;OAIG;IACH,kBAAkB,EAAE,MAAM,OAAO,CAAC,qBAAqB,CAAC,CAAA;CACzD;AAED,UAAU,yBAAyB;IACjC,KAAK,EAAE,gBAAgB,CAAA;IACvB;;;;OAIG;IACH,eAAe,EAAE,MAAM,MAAM,CAAA;IAC7B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,wBAAwB,CAAC,EACvC,KAAK,EACL,eAAe,EACf,GAA6B,EAC7B,QAAiC,GAClC,EAAE,yBAAyB,GAAG,kBAAkB,CAuEhD;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,WAExD;AAED;;;;;;;;GAQG;AACH,wBAAgB,yBAAyB,CAAC,EACxC,OAAO,EACP,SAAS,EACT,QAAQ,EACR,aAAa,GACd,EAAE,qBAAqB,UAevB;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,WAQrD;AAID,wBAAgB,4BAA4B,CAAC,OAAO,EAAE,MAAM,UAM3D"}

package/build/types/helpers/src/utils/sliceClock.d.ts

@@ -0,0 +1,61 @@
+/**
+ * THE SHARED YIELD RULE FOR LONG SYNCHRONOUS WORK (#134).
+ *
+ * A loop that does real CPU per item blocks every callback for its whole
+ * length — the AMQP heartbeat included. On 2026-08-07 that took the operator's
+ * connection down and lost every publish issued inside the dead window (#130).
+ * The fix in every such loop is the same shape: after each item, ask whether the
+ * current slice has run long enough, and if it has, hand the loop back.
+ *
+ * This is that rule, once. Two loops use it — `protocol/sliceTreeList.ts` (tree
+ * validation, the expensive one) and the operator's `machinePacker.unpack`
+ * (snapshot restore) — and any third should use it rather than re-time itself.
+ *
+ * It does NOT make work interruptible. A slice always overruns by the cost of
+ * the item that tripped it, because the item itself — an AJV compile, a msgpack
+ * decode — cannot be split part-way. The bound is `budgetMs` plus the single
+ * most expensive item, which is why the budget is set two orders of magnitude
+ * below the threshold anyone cares about.
+ */
+/**
+ * How long one uninterrupted slice may run before the loop yields.
+ *
+ * TIME, not a fixed item count, and that is deliberate: #134 asked for "every N
+ * items" picked by measurement, but N cannot hold the invariant that actually
+ * matters. Item cost spans four orders of magnitude across the callers — an edge
+ * validates in 0.004 ms, a service node with three JSON schemas in ~12 ms, a
+ * 20 kB snapshot payload in 0.01 ms — so any N tuned for one caller is wrong for
+ * the next. A time budget is the same rule expressed on the axis the loop cares
+ * about, and it makes N adaptive for free.
+ *
+ * 20 ms is two orders of magnitude below `LAG_WARN_THRESHOLD` (1000 ms in the
+ * operator's `utils/eventLoopLag.ts`), which leaves room for the overrun above.
+ */
+export declare const SLICE_BUDGET_MS = 20;
+export type SliceOptions = {
+    /** @see SLICE_BUDGET_MS */
+    budgetMs?: number;
+    /**
+     * How the loop hands control back. Injectable so a spec can count the yields
+     * instead of waiting for them.
+     */
+    yieldToEventLoop?: () => Promise<void>;
+};
+export type SliceStats = {
+    /** How many uninterrupted slices the work took. */
+    slices: number;
+    /** The longest of them, in ms — the stall the work actually caused. */
+    longestSliceMs: number;
+};
+/**
+ * The clock a sliced loop yields on. It measures the slice it is IN, so the
+ * reported worst slice includes the item that overran the budget — the honest
+ * number.
+ */
+export declare function createSliceClock(options?: SliceOptions): {
+    /** Called after every item. Yields only when the budget is spent. */
+    tick(): Promise<void>;
+    stats(): SliceStats;
+};
+export type SliceClock = ReturnType<typeof createSliceClock>;
+//# sourceMappingURL=sliceClock.d.ts.map

package/build/types/helpers/src/utils/sliceClock.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sliceClock.d.ts","sourceRoot":"","sources":["../../../../../../helpers/src/utils/sliceClock.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,eAAe,KAAK,CAAA;AAEjC,MAAM,MAAM,YAAY,GAAG;IACzB,2BAA2B;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CACvC,CAAA;AAED,MAAM,MAAM,UAAU,GAAG;IACvB,mDAAmD;IACnD,MAAM,EAAE,MAAM,CAAA;IACd,uEAAuE;IACvE,cAAc,EAAE,MAAM,CAAA;CACvB,CAAA;AAyBD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,GAAE,YAAiB;IASvD,qEAAqE;IAC/D,IAAI;IAWV,KAAK,IAAI,UAAU;EAOtB;AAED,MAAM,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAA"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@requence/service",
   "type": "module",
-  "version": "0.8.2-edge.1151",
+  "version": "0.8.2-edge.1154",
   "bin": {
     "requence-service": "build/cli.js"
   },