@cotal-ai/core 0.69.0 → 0.70.1

package/dist/endpoint.js

@@ -1,7 +1,7 @@
 import { EventEmitter } from "node:events";
 import { randomUUID } from "node:crypto";
 import { createConnection } from "node:net";
-import { connect, credsAuthenticator, headers, tokenAuthenticator, nanos, AuthorizationError, PermissionViolationError, UserAuthenticationExpiredError, NoRespondersError, RequestError, } from "@nats-io/transport-node";
+import { connect, credsAuthenticator, headers, tokenAuthenticator, nanos, AuthorizationError, PermissionViolationError, UserAuthenticationExpiredError, NoRespondersError, RequestError, TimeoutError, ClosedConnectionError, } from "@nats-io/transport-node";
 import { wsconnect } from "@nats-io/nats-core";
 import { parseLivenessAnswer, responderFromLease, responderFromProbe, isLivenessPlane, LIVENESS_PLANES, } from "./liveness.js";
 import { credsClaims, credsFingerprint, credsRenewalDelayMs, idFromCreds } from "./identity.js";
@@ -9,7 +9,7 @@ import { requireBrokerFloor } from "./broker-floor.js";
 import { inspectCredHealth } from "./provision.js";
 import { parseSecretStoreIdentity, } from "./secret-store.js";
 import { BIND_SPLIT_REISSUES, resolveService, invokeCommand, submitAndFollowGoal } from "./endpoint-invoke.js";
-import { EpEnvelopeError, respondedButUnbound, replyRefusedBeforeEffect, EP_BIND_REFUSED } from "./endpoint-envelope.js";
+import { EpEnvelopeError, respondedButUnbound, replyRefusedBeforeEffect, replyTargetUnmapped, EP_BIND_REFUSED } from "./endpoint-envelope.js";
 import { isRepeatSafeCommand } from "./endpoint-grants.js";
 import { assertIdToken, assertGeneration } from "./endpoint-subjects.js";
 import { readAcceptedRow } from "./issued-authority.js";
@@ -363,8 +363,9 @@ export class CotalEndpoint extends EventEmitter {
     /** Active goal followers awaiting outcome, tracked so endpoint stop can cancel them immediately
      *  and connection replacement / reconnect can trigger reconciliation. */
     activeGoalFollowers = new Set();
-    /** How many calls {@link invokeService} has silently recovered from a bind refusal (§13.2) — the
-     *  class-queue splits this endpoint hit and survived.
+    /** How many calls {@link invokeService} has silently recovered from a bind refusal (§13.2) or a
+     *  member holding no mapping for the target (§13.3) — the class-queue splits this endpoint hit
+     *  and survived.
      *
      *  Counted because it is recovered: handling the split is what makes it invisible, so this is the
      *  only evidence the split rate exists. Always on, never behind a flag — a counter you have to
@@ -1907,8 +1908,9 @@ export class CotalEndpoint extends EventEmitter {
                     this.emit("error", new Error(`rejected ${service} request on ${m.subject}: reply target "${m.reply ?? "(none)"}" is not under the sender's own reply subtree`));
                     continue;
                 }
-                let reply;
+                let body;
                 try {
+                    let reply;
                     const req = m.json();
                     // Authenticity guard (fail closed): control is the most privileged surface
                     // (start/stop). The sender is encoded in the subject (ctl.<svc>.<sender>), which
@@ -1923,15 +1925,25 @@ export class CotalEndpoint extends EventEmitter {
                     else {
                         reply = await handler(req);
                     }
+                    body = JSON.stringify(reply);
                 }
                 catch (e) {
-                    reply = { ok: false, error: e.message };
+                    body = JSON.stringify({ ok: false, error: e.message });
                 }
                 try {
-                    m.respond(JSON.stringify(reply));
+                    m.respond(body);
                 }
-                catch {
-                    /* no reply inbox */
+                catch (e) {
+                    // `respond` signals a missing reply inbox by returning false; it throws when the reply exceeds the
+                    // broker's `max_payload`. Silence would leave the caller a bare timeout it cannot tell apart from
+                    // a dead service.
+                    const refused = { ok: false, error: `the ${service} reply is ${Buffer.byteLength(body)} bytes and was not published: ${e.message}` };
+                    try {
+                        m.respond(JSON.stringify(refused));
+                    }
+                    catch (err) {
+                        this.emit("error", new Error(`could not answer ${service} request on ${m.subject}: ${refused.error}, and the refusal failed too: ${err.message}`));
+                    }
                 }
             }
         })().catch((e) => this.emit("error", e));
@@ -2032,7 +2044,9 @@ export class CotalEndpoint extends EventEmitter {
      *     call re-issued for any command, up to {@link BIND_SPLIT_REISSUES} times while each re-issue
      *     is refused the same way. If a re-issue cannot be resolved, the refusal
      *     surfaces — still saying the command did not run — naming the resolve failure as why the
-     *     repair could not be attempted.
+     *     repair could not be attempted. An unpinned targeted call that reached a member holding no
+     *     mapping for its target ({@link replyTargetUnmapped}) is repaired the same way, and when the
+     *     re-issues run out, the latest such refusal is the one surfaced.
      *   - this CLIENT caught it on the reply ({@link respondedButUnbound}: a different instance,
      *     `failed-precondition`; the same instance at any other epoch, `expired`), which is what a
      *     responder too old to know the field produces. A live instance received and answered it, so
@@ -2103,7 +2117,14 @@ export class CotalEndpoint extends EventEmitter {
                 // did not run, so each re-issue is a FIRST attempt, not a second. It repeats up to the bound
                 // the CLI uses, because the re-issue rides the same class queue and splits again at the same
                 // rate; repairing once left a quarter of all calls in a two-manager space failing (#443).
-                for (let reissues = 0; reissues < BIND_SPLIT_REISSUES && r.reply.ok === false && replyRefusedBeforeEffect(r.reply.error); reissues++) {
+                // A member that holds no mapping for the target also ran nothing, and on the class rail a
+                // sibling may host it. A pinned call named its instance, so that refusal is its answer.
+                // That member looked the target up, so when the re-issues run out its refusal is surfaced
+                // in place of a trailing bind refusal, which says nothing about the target.
+                const unmappedHere = (x) => opts.instanceId === undefined && replyTargetUnmapped(x.reply.error);
+                let unmapped = unmappedHere(r) ? r : undefined;
+                for (let reissues = 0; reissues < BIND_SPLIT_REISSUES && r.reply.ok === false
+                    && (replyRefusedBeforeEffect(r.reply.error) || unmappedHere(r)); reissues++) {
                     // Counted before it is repaired: a recovery that leaves no trace takes the split rate with it.
                     this.splitsRecovered++;
                     // `boundTo` is the other half of `servedBy`: who the handle THOUGHT it was talking to,
@@ -2154,8 +2175,10 @@ export class CotalEndpoint extends EventEmitter {
                         "not-executed");
                     }
                     r = await invokeResolved(reissueTarget, signal);
+                    if (unmappedHere(r))
+                        unmapped = r;
                 }
-                return r;
+                return replyRefusedBeforeEffect(r.reply.error) ? unmapped ?? r : r;
             }
             catch (e) {
                 // DO NOT auto-retry a command a responder already ANSWERED. This path covers the responders
@@ -2762,8 +2785,8 @@ export class CotalEndpoint extends EventEmitter {
                     watch.consumerName = undefined;
                 }
                 else {
-                    const closedEpoch = err.name === "ClosedConnectionError" || /^closed connection$/i.test(err.message);
-                    const timeout = isTimeoutError(err);
+                    const closedEpoch = err instanceof ClosedConnectionError;
+                    const timeout = err instanceof TimeoutError;
                     const dyingEpochTimeout = timeout && (this.reconnecting || !this.nc || this.nc.isClosed());
                     // Cleanup of an ordered consumer: a delete timeout means the broker did not answer in time,
                     // not that the endpoint is unusable. The broker reaps an idle/ephemeral consumer anyway.
@@ -4361,9 +4384,8 @@ export class CotalEndpoint extends EventEmitter {
         const reachedStart = page.length <= limit;
         const wanted = reachedStart ? page : page.slice(-limit);
         // A page that cannot be SENT is not a page. The reply rides one NATS message, so `limit` alone is
-        // the wrong bound: 200 large messages serialize past `max_payload`, `m.respond` throws inside
-        // `serveControl`'s swallow, and the caller sees a bare request timeout it cannot tell apart from a
-        // dead daemon. Measured, not predicted: 200 x ~6 KB timed out at 5s with the message "timeout".
+        // the wrong bound: 200 large messages serialize past `max_payload`, and `serveControl` can then only
+        // answer with a refusal in place of the page.
         //
         // So bound by BYTES too, keeping the NEWEST that fit — which needs no new vocabulary, because
         // `complete: false` already means "older history remains behind this page". Trimming here is the
@@ -4705,9 +4727,7 @@ export class CotalEndpoint extends EventEmitter {
     probeFailureOutcome(e) {
         if (e instanceof AuthorizationError || e instanceof PermissionViolationError)
             return "refused";
-        const name = e?.name;
-        const msg = e?.message ?? "";
-        if (name === "TimeoutError" || /timeout/i.test(msg))
+        if (e instanceof TimeoutError)
             return "timeout";
         // Anything else is a transport or client failure: it says something about our link, not about
         // the plane, so it is graded like a refusal rather than guessed at.
@@ -5450,7 +5470,9 @@ export class CotalEndpoint extends EventEmitter {
             await this.pump(taskStream(this.space), taskDurable(this.card.role));
         }
     }
-    /** Drive one consumer: decode, drop our own echo, and hand each message to listeners with ack control. */
+    /** Drive one consumer: decode and hand each message to listeners with ack control. Our own sends are
+     *  delivered too: on the DM inbox and the role queue they were addressed to us, so none is an echo,
+     *  and acking one unseen on the work queue would delete the only copy of the request. */
     async pump(stream, durable) {
         if (!this.js)
             throw new Error("endpoint not started");
@@ -5484,10 +5506,6 @@ export class CotalEndpoint extends EventEmitter {
                         `does not match subject sender ${parsed?.sender ?? "(unparseable)"}`));
                     continue;
                 }
-                if (msg.from.id === this.card.id) {
-                    m.ack(); // our own echo — advance past it
-                    continue;
-                }
                 // No-replay + dedup (chat only): drop a message at/below this channel's join watermark
                 // — pre-join history the New tail still carries for a *lagging* joiner (cursor behind the
                 // frontier), and the overlap a replay backfill already delivered. Must ack, or JetStream
@@ -7043,14 +7061,6 @@ export async function isReachable(servers = DEFAULT_SERVER, opts = {}) {
         return e instanceof AuthorizationError || e instanceof UserAuthenticationExpiredError;
     }
 }
-/** True when `err` is a dial/consumer-op timeout rather than a real refusal — the one shared test
- *  for "the operation ran out of its own budget", used both by {@link classifyProbeFailure} (#851:
- *  a probe timeout must never collapse into `unreachable`, which a TLS-required target then
- *  misreads as a trust failure) and by {@link Endpoint#disarmMembershipWatch}'s consumer-delete
- *  cleanup, which predates it. */
-function isTimeoutError(err) {
-    return err instanceof Error && (err.name === "TimeoutError" || /timeout/i.test(err.message));
-}
 /** Like {@link isReachable}, but distinguishes "up but won't take these creds" from "nothing there".
  *  `spawn` needs the difference: auth-required → name the trust dir + next step; unreachable → the
  *  mesh is down (prune the stale entry, tell the user to `cotal up`). Pass `creds` to confirm a
@@ -7116,10 +7126,10 @@ function classifyProbeFailure(e, opts) {
     // The broker answered but rejected these creds (so it IS up) — auth-required, not stale-auth.
     if (e instanceof AuthorizationError)
         return { ok: false, reason: "auth-required" };
-    // A dial that ran out of its own budget is neither a refusal nor a dead broker — it is latency.
-    // `e` is undefined when the tcpDialable gate refused before any connect() attempt; that path has
-    // no timeout to inspect and must stay `unreachable` (nothing answered at all).
-    if (e !== undefined && isTimeoutError(e))
+    // A dial that ran out of its own budget is neither a refusal nor a dead broker — it is latency,
+    // and grading it `unreachable` made a TLS-required target read as a trust failure (#851). Only
+    // nats-core's TimeoutError means that: a peer's `-ERR` that mentions a timeout answered at once.
+    if (e instanceof TimeoutError)
         return { ok: false, reason: "timeout" };
     return { ok: false, reason: "unreachable" };
 }