@cotal-ai/core 0.66.1 → 0.67.0

package/dist/endpoint.js

@@ -8,7 +8,7 @@ import { credsClaims, credsFingerprint, credsRenewalDelayMs, idFromCreds } from
 import { requireBrokerFloor } from "./broker-floor.js";
 import { inspectCredHealth } from "./provision.js";
 import { parseSecretStoreIdentity, } from "./secret-store.js";
-import { resolveService, invokeCommand, submitAndFollowGoal } from "./endpoint-invoke.js";
+import { BIND_SPLIT_REISSUES, resolveService, invokeCommand, submitAndFollowGoal } from "./endpoint-invoke.js";
 import { EpEnvelopeError, respondedButUnbound, replyRefusedBeforeEffect, EP_BIND_REFUSED } from "./endpoint-envelope.js";
 import { isRepeatSafeCommand } from "./endpoint-grants.js";
 import { assertIdToken, assertGeneration } from "./endpoint-subjects.js";
@@ -98,15 +98,19 @@ const READ_HISTORY_DEFAULT_LIMIT = 100;
  *  request pull a channel's whole retained set through it. Clamped, not refused — a UI asking for
  *  more than a page should get a page, and `complete: false` already tells it more remains. */
 const READ_HISTORY_MAX_LIMIT = 200;
-/** The NEWEST prefix-from-the-end of `items` whose serialized size fits `budget` bytes, order
- *  preserved. Returns `[]` when not even the newest single message fits — the caller must refuse
- *  loudly there rather than serve an empty page, which would read as "no history".
+/** The NEWEST prefix-from-the-end of `items` whose serialized `readHistory` reply fits `budget`
+ *  bytes, order preserved. Returns `[]` when not even the newest single message fits — the caller
+ *  must refuse loudly there rather than serve an empty page, which would read as "no history".
+ *
+ *  The `ControlReply` around the items rides the same NATS message, so it is charged too: items
+ *  that fit on their own can still make a reply the broker refuses.
  *
  *  Measured in ENCODED bytes, not `string.length`: a page of multi-byte text would otherwise be
  *  undercounted and still overflow the broker. Same discipline as `assertFactFits`. */
 export function fitHistoryPage(items, budget) {
     const enc = new TextEncoder();
-    let used = 2; // the enclosing `[]`
+    // The reply with no items, at its longer `complete` value.
+    let used = enc.encode(JSON.stringify({ ok: true, data: { items: [], complete: false } })).length;
     let first = items.length; // index of the oldest kept item
     for (let i = items.length - 1; i >= 0; i--) {
         const size = enc.encode(JSON.stringify(items[i])).length + 1; // + the `,` separator
@@ -2017,7 +2021,8 @@ export class CotalEndpoint extends EventEmitter {
      *  and only knowing someone answered:
      *   - the RESPONDER fenced it on the request's `bind` (§13.2, an `ok:false` reply marked
      *     {@link replyRefusedBeforeEffect}): the command did not run, so the bind is dropped and the
-     *     call re-issued ONCE for any command. If that re-issue cannot be resolved, the refusal
+     *     call re-issued for any command, up to {@link BIND_SPLIT_REISSUES} times while each re-issue
+     *     is refused the same way. If a re-issue cannot be resolved, the refusal
      *     surfaces — still saying the command did not run — naming the resolve failure as why the
      *     repair could not be attempted.
      *   - this CLIENT caught it on the reply ({@link respondedButUnbound}: a different instance,
@@ -2066,7 +2071,7 @@ export class CotalEndpoint extends EventEmitter {
         const doInvoke = async (signal = opts.signal) => {
             signal?.throwIfAborted();
             try {
-                const r = await invokeResolved(await resolve(signal), signal);
+                let r = await invokeResolved(await resolve(signal), signal);
                 // THE RESPONDER FENCED IT (§13.2 `ai.cotal.ep.bind-refused`): a class member saw the call
                 // was bound to a different incarnation and refused BEFORE running the command.
                 //
@@ -2075,9 +2080,10 @@ export class CotalEndpoint extends EventEmitter {
                 // otherwise keep its stale bind and meet the same refusal forever.
                 //
                 // The re-issue is NOT gated on {@link isRepeatSafeCommand}: the responder states the command
-                // did not run, so this is a FIRST attempt, not a second. Exactly once; a second refusal
-                // surfaces.
-                if (r.reply.ok === false && replyRefusedBeforeEffect(r.reply.error)) {
+                // did not run, so each re-issue is a FIRST attempt, not a second. It repeats up to the bound
+                // the CLI uses, because the re-issue rides the same class queue and splits again at the same
+                // rate; repairing once left a quarter of all calls in a two-manager space failing (#443).
+                for (let reissues = 0; reissues < BIND_SPLIT_REISSUES && r.reply.ok === false && replyRefusedBeforeEffect(r.reply.error); reissues++) {
                     // Counted before it is repaired: a recovery that leaves no trace takes the split rate with it.
                     this.splitsRecovered++;
                     // `boundTo` is the other half of `servedBy`: who the handle THOUGHT it was talking to,
@@ -2127,7 +2133,7 @@ export class CotalEndpoint extends EventEmitter {
                         // re-issue never went out. Prose is for the reader; this is for the machine.
                         "not-executed");
                     }
-                    return await invokeResolved(reissueTarget, signal);
+                    r = await invokeResolved(reissueTarget, signal);
                 }
                 return r;
             }
@@ -4346,23 +4352,16 @@ export class CotalEndpoint extends EventEmitter {
         // So bound by BYTES too, keeping the NEWEST that fit — which needs no new vocabulary, because
         // `complete: false` already means "older history remains behind this page". Trimming here is the
         // documented truncation signal doing its job, not a silent degradation.
-        const fitted = fitHistoryPage(wanted, this.payloadBudget());
+        const fitted = fitHistoryPage(wanted, this.maxPayload);
         // `wanted.length > 0` is load-bearing, not defensive: an EMPTY channel also fits nothing, and
         // without this guard a channel nobody has posted to was refused with "the newest message exceeds
         // the payload budget" — a confident, entirely wrong explanation for a legitimately empty result.
         // Genuine emptiness is `{ items: [], complete: true }`; only a message too large to ever send is
         // the error.
         if (wanted.length > 0 && fitted.length === 0)
-            return { ok: false, error: `readHistory: the newest message on "${channel}" alone exceeds the broker payload budget (${this.payloadBudget()} bytes) - refused loudly rather than served as an empty page` };
+            return { ok: false, error: `readHistory: the newest message on "${channel}" alone exceeds the broker payload budget (${this.maxPayload} bytes) - refused loudly rather than served as an empty page` };
         return { ok: true, data: { items: fitted, complete: reachedStart && fitted.length === wanted.length } };
     }
-    /** Bytes a control reply may occupy: the broker's `max_payload` less headroom for the `ControlReply`
-     *  envelope wrapped around the items. Read from the live server info rather than assumed, since an
-     *  operator can raise or lower it. */
-    payloadBudget() {
-        const max = this.nc?.info?.max_payload ?? 1_048_576;
-        return Math.max(1, Math.floor(max * 0.9));
-    }
     /** Stop serving Plane-3 WITHOUT tearing down the connection, so a daemon that has just learned its
      *  lease may no longer be its own can stop acting on the shard while it finds out for certain.
      *