@cotal-ai/delivery 0.48.2 → 0.50.0

package/dist/watchdog.js

@@ -0,0 +1,374 @@
+/**
+ * The delivery daemon's two self-termination decisions, as PURE functions over evidence.
+ *
+ * Both decisions used to be taken inline from a wall clock, and a wall clock cannot tell "the
+ * broker is gone" from "this process did not get scheduled". On 2026-09-05 that cost Plane-3 nine
+ * minutes across two exits while `nats-server` had been up continuously for 7.3 days and was
+ * listening throughout; the real condition was load 311 on 12 cores (#1318). The detector failed in
+ * the direction of its own failure, and it did so exactly when delivery was most needed.
+ *
+ * Pulled out here for two reasons. The obvious one is that a decision expressed as a function of
+ * named evidence is readable. The load-bearing one is that it becomes GRADEABLE without a broker,
+ * a host under load, or a spawned process: every branch below is reachable from a literal, so each
+ * ACCEPTING branch can be paired with a REFUSING case that differs only in the evidence, and a
+ * mutation to any branch reddens a named cell rather than a timing-dependent end-to-end assertion.
+ */
+/** How often the broker watch is scheduled. The measured gap between two firings minus this is
+ *  LOCAL SCHEDULER LAG: a fact about this process, not about the server. */
+export const PROBE_INTERVAL_MS = 2000;
+/** The deadline `isReachable` gives a non-websocket probe. Named here because the probe's own
+ *  budget is the yardstick that makes a late answer legible as lateness. */
+export const PROBE_BUDGET_MS = 1000;
+/** How far past its own budget an answer may arrive and still be believed as a statement about the
+ *  server. A probe that answers at its deadline is a normal timeout; one that answers at twice its
+ *  deadline did not have that deadline enforced against the SERVER at all. It had it enforced
+ *  against a process that was not running when it expired. Two rather than something larger because
+ *  the measured case is 2554ms on a 1000ms budget (#1318's triage), and rather than something
+ *  smaller because it must never reclassify an honest timeout on a merely busy host as starvation,
+ *  which would blunt the true-positive path this repair is required to preserve. */
+export const PROBE_LATE_FACTOR = 2;
+/** How often the deschedule sampler wakes during a probe. Short enough to resolve the stalls that
+ *  matter (tens of ms), long enough that the measurement is not itself a meaningful load. */
+export const PROBE_SAMPLE_MS = 25;
+/** How much of its promised budget the server must have been DENIED before a refusal stops counting
+ *  as evidence about the server. Ordinary Node timer jitter measures tens of milliseconds on a
+ *  perfectly healthy host, so any non-zero threshold would disqualify every honest budget-ended
+ *  refusal and leave the daemon exiting only on its backstop. A server handed more than half the
+ *  time it was promised and still refusing is refusing on its own account. */
+export const STARVED_SHARE = 0.5;
+/**
+ * Classify one probe result.
+ *
+ * `ok === undefined` is a probe that REJECTED rather than resolving: an unanswered question. It was
+ * previously swallowed by `.catch(() => {})`, so it neither refreshed the window nor evaluated
+ * anything, and silently aged the daemon toward an exit it had gathered no evidence for.
+ */
+export function classifyProbe(ok, elapsedMs, budgetMs = PROBE_BUDGET_MS, lateFactor = PROBE_LATE_FACTOR, descheduledDuringMs = 0) {
+    if (ok === undefined)
+        return { counts: "incomplete" };
+    // A POSITIVE IS BELIEVED HOWEVER LATE IT IS. A slow yes still required a server to say it, so
+    // lateness cannot turn it into anything weaker; and refusing late positives would make a starved
+    // daemon unable to ever clear its own window, which is the defect again with the sign flipped.
+    if (ok)
+        return { counts: "positive" };
+    const ceiling = budgetMs * lateFactor;
+    // (1) THE ANSWER ARRIVED FAR PAST ITS OWN DEADLINE. The deadline was enforced against this
+    // process, not against the server: a 1000ms budget that reports at 2554ms did not measure a
+    // server for 2554ms, it measured a process that could not get back on the CPU to stop waiting.
+    // This is the form the #1318 triage captured directly, and it needs no other instrumentation.
+    if (elapsedMs > ceiling)
+        return { counts: "starved", lateBy: elapsedMs - budgetMs };
+    // (2) THE DEADLINE EXPIRED WITHOUT THE SERVER EVER GETTING THE BUDGET IT WAS PROMISED. A refusal
+    // is only evidence if the server had the time to answer in. Subtracting the stretch this process
+    // spent OFF the runqueue leaves what the server actually had, and the two cases separate cleanly:
+    //
+    //   - a genuinely dead port answers ECONNREFUSED in about a millisecond, so `elapsed` never
+    //     reaches the budget at all and this clause does not apply. That is a prompt, honest negative
+    //     and it must stay one, or the repair would buy availability by making a dead broker
+    //     survivable, which is the failure mode worse than the defect.
+    //   - a process getting short slices of CPU issues a connect, is descheduled, and its deadline
+    //     timer fires the instant it is scheduled again. Wall-clock elapsed looks like a normal,
+    //     prompt timeout; almost none of it was time the server was given. Judged by the clock alone
+    //     this is indistinguishable from a dead server, which is exactly the confusion #1318 is about.
+    //
+    // THE MATERIAL-SHARE TEST IS WHAT KEEPS THIS FROM SWALLOWING EVERY REFUSAL, and it was a review
+    // finding: requiring merely `attributable < budget` meant ANY non-zero measurement disqualified an
+    // honest budget-ended refusal, and ordinary Node timer jitter on a healthy host measures 10-30ms.
+    // Every blackholed probe everywhere then read as starvation, the completed-negative count could
+    // never rise, and a genuinely dead broker would be exited only by the backstop, four times slower
+    // than the shipped window, with the wrong reason in the log. So the server must have been denied a
+    // MATERIAL share of its budget, not merely an instant of it. Half is the line: a server given more
+    // than half the time it was promised and still refusing is refusing on its own account.
+    //
+    // Clamped to the probe's own span so a bad measurement cannot manufacture credit.
+    const attributableMs = elapsedMs - Math.max(0, Math.min(descheduledDuringMs, elapsedMs));
+    if (elapsedMs >= budgetMs && attributableMs < budgetMs * STARVED_SHARE) {
+        return { counts: "starved", lateBy: elapsedMs - attributableMs };
+    }
+    return { counts: "negative" };
+}
+/**
+ * Measure how long this process spends OFF the runqueue across a span, by watching a short timer's
+ * own lateness.
+ *
+ * A timer asked to fire every `sampleMs` that instead fires `sampleMs + d` later reports `d` of
+ * delay this process could not avoid: the event loop was ready and the process was not running.
+ * Summed across a probe. That is the part of the probe's wall-clock that the server was never
+ * actually given, which is the difference between "the server did not answer in a second" and "a
+ * second passed, and the server had 40ms of it".
+ *
+ * This measures the same underlying condition as {@link LoopLagMeter} at a finer grain and over a
+ * bounded span, which is what lets a per-probe verdict use it. It deliberately reads only the
+ * clock: `/proc` sampling, `getrusage`, or cgroup pressure would all be sharper, and all of them are
+ * platform-specific in a way that would make this daemon behave differently on the hosts that most
+ * need it. A late timer is available everywhere the daemon runs.
+ */
+export class DescheduleSampler {
+    sampleMs;
+    accumulated = 0;
+    last = 0;
+    timer;
+    constructor(sampleMs = PROBE_SAMPLE_MS) {
+        this.sampleMs = sampleMs;
+    }
+    start(now = Date.now()) {
+        this.accumulated = 0;
+        this.last = now;
+        this.timer = setInterval(() => {
+            const t = Date.now();
+            this.accumulated += Math.max(0, t - this.last - this.sampleMs);
+            this.last = t;
+        }, this.sampleMs);
+        // Never hold the process open for a measurement: the daemon's lifetime is decided elsewhere.
+        this.timer.unref?.();
+    }
+    /** Stop sampling and return the total off-runqueue time observed. Charging the final partial gap
+     *  matters: under heavy starvation the single longest stall is often the one still in progress
+     *  when the probe resolves, and dropping it would undercount exactly the worst case. */
+    stop(now = Date.now()) {
+        if (this.timer !== undefined) {
+            clearInterval(this.timer);
+            this.timer = undefined;
+        }
+        this.accumulated += Math.max(0, now - this.last - this.sampleMs);
+        return this.accumulated;
+    }
+}
+/**
+ * Decide whether the broker is GONE, from evidence alone.
+ *
+ * EXIT REQUIRES BOTH CONJUNCTS, and each exists because the other cannot cover its case:
+ *
+ *   1. `completedNegatives >= requiredNegatives`, probes that actually RAN and actually said no.
+ *      Without this, a window that expired while the process was descheduled is read as a server
+ *      failure, which is the reported defect in its purest form.
+ *   2. `msSinceLastReachable - starvedMs > windowMs`, the UNSTARVED part of the window. Without
+ *      this, a host that schedules the daemon just often enough to fire two probes into a
+ *      momentarily-saturated loopback exits on two negatives that a healthy host would never have
+ *      produced.
+ *
+ * THIS IS NOT A WIDER TIMEOUT. `windowMs` is unchanged from the shipped default; what changed is
+ * that the quantity compared against it is now evidence rather than the passage of time. A daemon
+ * whose broker is genuinely dead produces completed negatives as fast as it is scheduled, a dead
+ * port answers ECONNREFUSED immediately and a blackholed one answers inside the probe deadline ,
+ * AND loses its standing connection, so the true-positive path is not delayed by this at all, which
+ * is the property that separates a repair from a band-aid. A process so starved that it produces NO
+ * completed probe also cannot deliver anything, and ending it would not make Plane-3 more
+ * available; it would make the outage permanent for a stock install with no watchdog.
+ */
+export function brokerGoneVerdict(e) {
+    // Positive evidence inside the window outranks everything: nothing below can be true of a broker
+    // that answered us this recently.
+    if (e.msSinceLastReachable <= e.windowMs)
+        return { exit: false, reason: "reachable" };
+    // THE BACKSTOP IS ABSOLUTE, AND IT IS DECIDED FIRST because every clause below it is a reason to
+    // keep waiting. This ordering was a review finding, and it is the difference between a bound and
+    // a wish: two reviewers independently observed that an open transport ranked ABOVE the backstop,
+    // which made the backstop conditional on the very signal most likely to be stale.
+    //
+    // `transportConnected` is not an observation of the broker. It is this client's cached socket
+    // state, and it is only refreshed when nats.js decides the peer is gone. Under a SILENT death ,
+    // an OOM kill, a hypervisor pause, a firewall that starts dropping rather than refusing, anything
+    // that produces no FIN and no RST, the kernel keeps the connection ESTABLISHED and nats.js does
+    // not notice until its own ping cycle expires. With the shipped defaults that is roughly six
+    // minutes. Ranked above the backstop, a stale `true` suspended the exit for that entire window no
+    // matter how much elapsed time and how many completed refusals had piled up behind it, which is
+    // worse than the pre-fix behaviour in exactly the shape an operator would least expect.
+    //
+    // So the daemon is bounded unconditionally: past the backstop it exits, whatever it believes
+    // about its socket. That is the guarantee the coupling exists for, and a guarantee that any
+    // single stale flag can defer is not one.
+    if (e.msSinceLastReachable > e.backstopMs)
+        return { exit: true, reason: "backstop" };
+    // AN OPEN TRANSPORT IS ONGOING POSITIVE EVIDENCE, NOT AN EXCUSE FOR ITS ABSENCE, so inside the
+    // backstop it still outranks the starvation and evidence clauses below. This daemon's own
+    // connection to that same broker is up; it is serving on it; its lease renews across it. A fresh
+    // side-probe that cannot complete a handshake to an address we are CURRENTLY CONNECTED TO is a
+    // statement about this process's ability to open a new socket, not about the server.
+    //
+    // Inside the backstop this is still the strongest thing the daemon knows: when the broker really
+    // dies the flag goes false, the client detects the loss, the endpoint's status watcher turns it
+    // into `transport: connected=false`, and every clause below is live again from that instant. What
+    // the flag CANNOT do any more is defer the exit indefinitely while it is stale, because the bound
+    // above it has already been decided.
+    if (e.transportConnected)
+        return { exit: false, reason: "transport-live" };
+    // "I could not ask." Credit the lag this process MEASURED on itself before reading the clock as a
+    // statement about the server.
+    const unstarvedMs = e.msSinceLastReachable - e.starvedMs;
+    if (unstarvedMs <= e.windowMs)
+        return { exit: false, reason: "starved" };
+    // "I asked and was told no", but not yet often enough to be a verdict.
+    if (e.completedNegatives < e.requiredNegatives)
+        return { exit: false, reason: "insufficient-evidence" };
+    return { exit: true, reason: "broker-gone" };
+}
+/**
+ * Turn a lease reading into an action.
+ *
+ * A FAILED RENEW IS A QUESTION, NOT A VERDICT, the same split #1301 made for `down` and the
+ * manager's liveness lease already makes. The shipped code exited on ANY renew error, so the
+ * measured incident's `wrong last sequence: 0` (the key had EXPIRED during the stall, with nobody
+ * else holding it) read identically to a genuine takeover and ended a daemon that was still the
+ * only holder there was.
+ *
+ * The single-holder guarantee is preserved exactly, and by construction rather than by timing:
+ * `gone` recovers through an ATOMIC create, so if a replacement did take the slot first, the create
+ * fails and that is the genuine loss which exits. `taken` exits immediately. Only "the key is still
+ * ours" and "the broker could not be asked" keep serving, and neither of those is a second holder.
+ */
+export function leaseAction(reading) {
+    switch (reading.kind) {
+        // Our own key, at whatever revision the broker says: adopt it and carry on. Covers the renew
+        // whose write landed with only its acknowledgement lost.
+        case "held": return "keep-serving";
+        // Expired or released while we were still here. Put it back; the create arbitrates.
+        case "gone": return "reacquire";
+        // A different process holds this shard. Exit so the holder stays single.
+        case "taken": return "exit";
+        // Could not ask. An unanswerable question is not a negative answer; the broker watch above owns
+        // the "the server is gone" decision, and it decides on evidence.
+        case "unknown": return "keep-serving";
+    }
+}
+/**
+ * May this process hold Plane-3 bindings on the strength of this reading alone?
+ *
+ * SEPARATE FROM {@link leaseAction} BECAUSE THEY ANSWER DIFFERENT QUESTIONS, and conflating them was
+ * a review finding. `leaseAction` decides whether the PROCESS lives; this decides whether it may
+ * SERVE. They agree on `taken` and disagree everywhere else that matters:
+ *
+ *   • `gone` is `reacquire`, the process lives, but it must NOT serve on it. The create has not
+ *     been attempted yet, and a replacement may already hold the shard; serving through the
+ *     arbitration is how two daemons end up on one durable.
+ *   • `unknown` is `keep-serving` for the PROCESS, and a refusal here. Surviving an unanswerable
+ *     broker is the entire point of #1318, but not being able to ask who owns the shard is not
+ *     permission to keep acting on it. The daemon stays alive and stays quiet.
+ *
+ * So this is deliberately the STRICTER of the two: it says yes to exactly one reading, the one that
+ * carries positive proof of ownership from the broker. Winning the atomic create is the other way
+ * to earn it, and that is not a reading, which is why the daemon re-arms there separately.
+ */
+export function mayServeOn(reading) {
+    // The only reading that is itself proof: the broker was asked, it answered, and the holder it
+    // named is this process.
+    return reading.kind === "held";
+}
+/**
+ * Measures how late this process's own timers are running.
+ *
+ * A `setInterval(f, 2000)` that fires 30 seconds after its predecessor did not observe a slow
+ * server; it observed a runqueue it was not on. Node hands us no such signal, but the gap between
+ * consecutive firings of a timer we own is a direct measurement of it, and it costs one
+ * subtraction. Excess over the nominal period is clamped at zero: a timer can fire late, never
+ * early, so a negative reading is clock adjustment rather than scheduling and must not CREDIT time
+ * the process actually had.
+ */
+export class LoopLagMeter {
+    intervalMs;
+    last;
+    accumulated = 0;
+    /** Disjoint, sorted [start, end] intervals of measured non-running time in the current window.
+     *  Bounded: overlapping charges merge, and `reset` clears them on every positive. */
+    spans = [];
+    constructor(intervalMs = PROBE_INTERVAL_MS) {
+        this.intervalMs = intervalMs;
+    }
+    /** Record a firing at `now`; returns the lag this particular gap contributed.
+     *
+     *  WHAT THE GAP IS CHARGED AS, since this is the half that is easy to misread: the excess of the
+     *  observed gap over the nominal interval is charged as time THIS PROCESS WAS NOT RUNNING. It is a
+     *  measurement of the local scheduler, never of the broker - a timer we own fired late, which says
+     *  the runqueue did not reach us and says nothing at all about the server.
+     *
+     *  It is also NOT the only such charge. {@link credit} adds lateness observed on a probe answer,
+     *  and the two can describe the same stall or two adjacent ones; nothing in either number reveals
+     *  which. Neither method deduplicates, deliberately, because suppressing a charge on the guess
+     *  that it overlaps discards real stalls. The sum is bounded instead, at the point of comparison,
+     *  by {@link starvedMsWithin}. */
+    tick(now) {
+        const previous = this.last;
+        this.last = now;
+        if (previous === undefined)
+            return 0; // first firing has no gap to measure
+        const lag = Math.max(0, now - previous - this.intervalMs);
+        // Charged as a DATED INTERVAL, not a scalar. The stall ended when this firing ran, so it
+        // occupied [now - lag, now]. Recording WHEN lets an overlapping probe charge be unioned with it
+        // instead of added to it, which is the whole of the double-count repair.
+        this.chargeSpan(now - lag, now);
+        return lag;
+    }
+    /** Add lag measured somewhere OTHER than the interval gap, in practice, a probe answer that
+     *  arrived past its own deadline. Interval gaps alone miss the case where the process IS being
+     *  scheduled often enough to fire the timer but not often enough to finish a handshake, which is
+     *  the third of the issue's three mechanisms and the one that produces a completed `false` from a
+     *  perfectly live server. Clamped at zero for the same reason `tick` is: a measurement that ran
+     *  backwards must never credit time the process actually had. */
+    credit(lagMs, endedAt = Date.now()) {
+        // BOTH MEASUREMENTS ARE KEPT. An earlier repair here netted this against the most recent interval
+        // gap, on the theory that the two always observe ONE stall. They do not, and a reviewer produced
+        // the separating case: over a 4000ms span a timer can fire 2000ms late (a stall happening NOW)
+        // while a probe issued at the PREVIOUS tick answers 2500ms late (the stall before it). Netting
+        // credited 2500ms there and threw away a 2000ms stall the process had actually measured - which
+        // is the very mechanism this method exists for, a process scheduled often enough to fire a timer
+        // but not to finish a handshake. I measured my own netting against that case and it produced the
+        // lossy answer, so it is gone.
+        this.chargeSpan(endedAt - Math.max(0, lagMs), endedAt);
+    }
+    /** Charge [from, to] as time this process was not running, UNIONED with what is already charged.
+     *
+     *  This is the double-count repair, and it is why both call sites can stay. A stall observed by
+     *  the interval timer and by a probe answering across it is ONE interval of wall-clock time seen
+     *  by two instruments. Summing scalars charges it twice (measured: 17s for a 10s stall). Summing
+     *  INTERVALS cannot, because the union of overlapping intervals is their extent.
+     *
+     *  It also keeps what netting threw away. Two ADJACENT stalls (a timer late by one, a probe from
+     *  the previous tick late by another) do not overlap, so the union is their sum and both are
+     *  charged in full. Overlap is decided by the timestamps rather than guessed from the magnitudes,
+     *  which is the thing neither scalar arithmetic nor a span clamp can do. */
+    chargeSpan(from, to) {
+        if (!(to > from))
+            return; // zero or backwards: never credits time we had
+        const merged = [];
+        let lo = from, hi = to;
+        for (const [s, e] of this.spans) {
+            if (e < lo || s > hi) {
+                merged.push([s, e]);
+                continue;
+            } // disjoint: keep as-is
+            lo = Math.min(lo, s);
+            hi = Math.max(hi, e); // touching: absorb
+        }
+        merged.push([lo, hi]);
+        merged.sort((a, b) => a[0] - b[0]);
+        this.spans = merged;
+        this.accumulated = merged.reduce((n, [s, e]) => n + (e - s), 0);
+    }
+    /** Accumulated lag CLAMPED to the span it is about to be compared against.
+     *
+     *  This is the honest way to stop double counting, and the reason the meter cannot do it by
+     *  arithmetic alone: whether a tick charge and a probe charge describe the same stall or two
+     *  adjacent ones is not knowable from the two numbers. What IS knowable is that time the process
+     *  did not have can never exceed time that passed. Overlap is removed by that bound and nothing
+     *  else is discarded, so the sum stays as informative as its parts allow.
+     *
+     *  Load-bearing, not hygiene: `starvedMs` is SUBTRACTED from elapsed in {@link brokerGoneVerdict}.
+     *  Unclamped, a stall charged twice drives unstarved time negative, and NO amount of real outage
+     *  can then clear the evidence clause - a genuinely dead broker stops being detectable by evidence
+     *  and survives to the backstop instead. Measured: a 30s stall moved an exit from 44s to 62s. */
+    starvedMsWithin(spanMs) {
+        return Math.min(this.accumulated, Math.max(0, spanMs));
+    }
+    /** Lag accumulated since the last {@link reset}. */
+    get starvedMs() {
+        return this.accumulated;
+    }
+    /** Called when positive evidence arrives: the window restarts, so its lag budget does too. The
+     *  firing baseline is deliberately KEPT, so a stall spanning a reset is still measured. */
+    reset() {
+        this.accumulated = 0;
+        this.spans = []; // the intervals ARE the accumulator now; clearing one without the other
+        // would let a pre-reset stall be re-charged by a later overlapping probe.
+    }
+}
+//# sourceMappingURL=watchdog.js.map

package/dist/watchdog.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"watchdog.js","sourceRoot":"","sources":["../src/watchdog.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH;4EAC4E;AAC5E,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAEtC;4EAC4E;AAC5E,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,CAAC;AAEpC;;;;;;oFAMoF;AACpF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAEnC;6FAC6F;AAC7F,MAAM,CAAC,MAAM,eAAe,GAAG,EAAE,CAAC;AAElC;;;;8EAI8E;AAC9E,MAAM,CAAC,MAAM,aAAa,GAAG,GAAG,CAAC;AAyBjC;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAC3B,EAAuB,EACvB,SAAiB,EACjB,WAAmB,eAAe,EAClC,aAAqB,iBAAiB,EACtC,sBAA8B,CAAC;IAE/B,IAAI,EAAE,KAAK,SAAS;QAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IACtD,8FAA8F;IAC9F,iGAAiG;IACjG,+FAA+F;IAC/F,IAAI,EAAE;QAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IACtC,MAAM,OAAO,GAAG,QAAQ,GAAG,UAAU,CAAC;IACtC,2FAA2F;IAC3F,4FAA4F;IAC5F,+FAA+F;IAC/F,8FAA8F;IAC9F,IAAI,SAAS,GAAG,OAAO;QAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,GAAG,QAAQ,EAAE,CAAC;IACpF,iGAAiG;IACjG,iGAAiG;IACjG,kGAAkG;IAClG,EAAE;IACF,4FAA4F;IAC5F,kGAAkG;IAClG,yFAAyF;IACzF,mEAAmE;IACnE,+FAA+F;IAC/F,6FAA6F;IAC7F,iGAAiG;IACjG,mGAAmG;IACnG,EAAE;IACF,gGAAgG;IAChG,mGAAmG;IACnG,kGAAkG;IAClG,gGAAgG;IAChG,kGAAkG;IAClG,mGAAmG;IACnG,mGAAmG;IACnG,wFAAwF;IACxF,EAAE;IACF,kFAAkF;IAClF,MAAM,cAAc,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,mBAAmB,EAAE,SAAS,CAAC,CAAC,CAAC;IACzF,IAAI,SAAS,IAAI,QAAQ,IAAI,cAAc,GAAG,QAAQ,GAAG,aAAa,EAAE,CAAC;QACvE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,GAAG,cAAc,EAAE,CAAC;IACnE,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;AAChC,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,OAAO,iBAAiB;IAKC;IAJrB,WAAW,GAAG,CAAC,CAAC;IAChB,IAAI,GAAG,CAAC,CAAC;IACT,KAAK,CAA6C;IAE1D,YAA6B,WAAmB,eAAe;QAAlC,aAAQ,GAAR,QAAQ,CAA0B;IAAG,CAAC;IAEnE,KAAK,CAAC,MAAc,IAAI,CAAC,GAAG,EAAE;QAC5B,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;QAChB,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC5B,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACrB,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC/D,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;QAChB,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClB,6FAA6F;QAC7F,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IACvB,CAAC;IAED;;4FAEwF;IACxF,IAAI,CAAC,MAAc,IAAI,CAAC,GAAG,EAAE;QAC3B,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAAC,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QAAC,CAAC;QACpF,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjE,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;CACF;AAgED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,iBAAiB,CAAC,CAAsB;IACtD,iGAAiG;IACjG,kCAAkC;IAClC,IAAI,CAAC,CAAC,oBAAoB,IAAI,CAAC,CAAC,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IACtF,iGAAiG;IACjG,iGAAiG;IACjG,iGAAiG;IACjG,kFAAkF;IAClF,EAAE;IACF,8FAA8F;IAC9F,gGAAgG;IAChG,kGAAkG;IAClG,gGAAgG;IAChG,6FAA6F;IAC7F,kGAAkG;IAClG,gGAAgG;IAChG,wFAAwF;IACxF,EAAE;IACF,6FAA6F;IAC7F,4FAA4F;IAC5F,0CAA0C;IAC1C,IAAI,CAAC,CAAC,oBAAoB,GAAG,CAAC,CAAC,UAAU;QAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IACrF,+FAA+F;IAC/F,0FAA0F;IAC1F,iGAAiG;IACjG,+FAA+F;IAC/F,qFAAqF;IACrF,EAAE;IACF,iGAAiG;IACjG,gGAAgG;IAChG,kGAAkG;IAClG,kGAAkG;IAClG,qCAAqC;IACrC,IAAI,CAAC,CAAC,kBAAkB;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC3E,kGAAkG;IAClG,8BAA8B;IAC9B,MAAM,WAAW,GAAG,CAAC,CAAC,oBAAoB,GAAG,CAAC,CAAC,SAAS,CAAC;IACzD,IAAI,WAAW,IAAI,CAAC,CAAC,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IACzE,uEAAuE;IACvE,IAAI,CAAC,CAAC,kBAAkB,GAAG,CAAC,CAAC,iBAAiB;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,uBAAuB,EAAE,CAAC;IACxG,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;AAC/C,CAAC;AAeD;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,WAAW,CAAC,OAAqB;IAC/C,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;QACrB,6FAA6F;QAC7F,yDAAyD;QACzD,KAAK,MAAM,CAAC,CAAC,OAAO,cAAc,CAAC;QACnC,oFAAoF;QACpF,KAAK,MAAM,CAAC,CAAC,OAAO,WAAW,CAAC;QAChC,yEAAyE;QACzE,KAAK,OAAO,CAAC,CAAC,OAAO,MAAM,CAAC;QAC5B,gGAAgG;QAChG,iEAAiE;QACjE,KAAK,SAAS,CAAC,CAAC,OAAO,cAAc,CAAC;IACxC,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,UAAU,CAAC,OAAqB;IAC9C,8FAA8F;IAC9F,yBAAyB;IACzB,OAAO,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC;AACjC,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,OAAO,YAAY;IAMM;IALrB,IAAI,CAAqB;IACzB,WAAW,GAAG,CAAC,CAAC;IACxB;yFACqF;IAC7E,KAAK,GAA4B,EAAE,CAAC;IAC5C,YAA6B,aAAqB,iBAAiB;QAAtC,eAAU,GAAV,UAAU,CAA4B;IAAG,CAAC;IAEvE;;;;;;;;;;;sCAWkC;IAClC,IAAI,CAAC,GAAW;QACd,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;QAChB,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,CAAC,CAAC,CAAC,qCAAqC;QAC3E,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;QAC1D,yFAAyF;QACzF,gGAAgG;QAChG,yEAAyE;QACzE,IAAI,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;QAChC,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;;qEAKiE;IACjE,MAAM,CAAC,KAAa,EAAE,UAAkB,IAAI,CAAC,GAAG,EAAE;QAChD,kGAAkG;QAClG,iGAAiG;QACjG,+FAA+F;QAC/F,+FAA+F;QAC/F,gGAAgG;QAChG,iGAAiG;QACjG,iGAAiG;QACjG,+BAA+B;QAC/B,IAAI,CAAC,UAAU,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,OAAO,CAAC,CAAC;IACzD,CAAC;IAED;;;;;;;;;;gFAU4E;IACpE,UAAU,CAAC,IAAY,EAAE,EAAU;QACzC,IAAI,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC;YAAE,OAAO,CAAuB,+CAA+C;QAC/F,MAAM,MAAM,GAA4B,EAAE,CAAC;QAC3C,IAAI,EAAE,GAAG,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;gBAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAAC,SAAS;YAAC,CAAC,CAAG,uBAAuB;YAClF,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;YAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAgB,mBAAmB;QAChF,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;QACtB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAClE,CAAC;IAED;;;;;;;;;;;qGAWiG;IACjG,eAAe,CAAC,MAAc;QAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IACzD,CAAC;IAGD,oDAAoD;IACpD,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;+FAC2F;IAC3F,KAAK;QACH,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAG,wEAAwE;QACxE,0EAA0E;IAC/F,CAAC;CACF"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@cotal-ai/delivery",
   "description": "Cotal delivery daemon: the server-side Plane-3 durable backstop (fan-out writer + trusted reader + membership/ACL authority), a scoped least-privilege NATS client co-located with the broker.",
-  "version": "0.48.2",
+  "version": "0.50.0",
   "license": "Apache-2.0",
   "repository": {
     "type": "git",
@@ -18,8 +18,8 @@
     }
   },
   "dependencies": {
-    "@cotal-ai/core": "0.48.2",
-    "@cotal-ai/workspace": "0.48.2"
+    "@cotal-ai/core": "0.50.0",
+    "@cotal-ai/workspace": "0.50.0"
   },
   "files": [
     "dist"