@cotal-ai/core 0.68.0 → 0.70.0

package/dist/endpoint.js

@@ -9,7 +9,7 @@ import { requireBrokerFloor } from "./broker-floor.js";
 import { inspectCredHealth } from "./provision.js";
 import { parseSecretStoreIdentity, } from "./secret-store.js";
 import { BIND_SPLIT_REISSUES, resolveService, invokeCommand, submitAndFollowGoal } from "./endpoint-invoke.js";
-import { EpEnvelopeError, respondedButUnbound, replyRefusedBeforeEffect, EP_BIND_REFUSED } from "./endpoint-envelope.js";
+import { EpEnvelopeError, respondedButUnbound, replyRefusedBeforeEffect, replyTargetUnmapped, EP_BIND_REFUSED } from "./endpoint-envelope.js";
 import { isRepeatSafeCommand } from "./endpoint-grants.js";
 import { assertIdToken, assertGeneration } from "./endpoint-subjects.js";
 import { readAcceptedRow } from "./issued-authority.js";
@@ -249,9 +249,6 @@ export class CotalEndpoint extends EventEmitter {
      *  EXPECTED async permission violation that joinChannel turns into a clean throw, so watchStatus
      *  suppresses it rather than surfacing a spurious connection error. */
     confirmingChatSubs = new Set();
-    /** Delete subjects of this endpoint's own consumers whose refusal watchStatus has not consumed yet
-     *  (see {@link deleteOwnConsumer}). */
-    ownConsumerDeletes = new Set();
     /** `$JS.API.CONSUMER.DELETE.<stream>.<prefix>_` for each of this connection's own KV watches,
      *  running membership scans and running history reads (see {@link recordOwnWatchConsumer}). */
     ownWatchDeletePrefixes = new Set();
@@ -366,8 +363,9 @@ export class CotalEndpoint extends EventEmitter {
     /** Active goal followers awaiting outcome, tracked so endpoint stop can cancel them immediately
      *  and connection replacement / reconnect can trigger reconciliation. */
     activeGoalFollowers = new Set();
-    /** How many calls {@link invokeService} has silently recovered from a bind refusal (§13.2) — the
-     *  class-queue splits this endpoint hit and survived.
+    /** How many calls {@link invokeService} has silently recovered from a bind refusal (§13.2) or a
+     *  member holding no mapping for the target (§13.3) — the class-queue splits this endpoint hit
+     *  and survived.
      *
      *  Counted because it is recovered: handling the split is what makes it invisible, so this is the
      *  only evidence the split rate exists. Always on, never behind a flag — a counter you have to
@@ -1222,7 +1220,6 @@ export class CotalEndpoint extends EventEmitter {
         this.chatSubs.clear();
         this.chatSubDenied.clear();
         this.confirmingChatSubs.clear();
-        this.ownConsumerDeletes.clear();
         this.ownWatchDeletePrefixes.clear();
         this.roster.clear();
         // #1356: the presence-refusal record is connection-scoped like everything else torn down here.
@@ -1911,8 +1908,9 @@ export class CotalEndpoint extends EventEmitter {
                     this.emit("error", new Error(`rejected ${service} request on ${m.subject}: reply target "${m.reply ?? "(none)"}" is not under the sender's own reply subtree`));
                     continue;
                 }
-                let reply;
+                let body;
                 try {
+                    let reply;
                     const req = m.json();
                     // Authenticity guard (fail closed): control is the most privileged surface
                     // (start/stop). The sender is encoded in the subject (ctl.<svc>.<sender>), which
@@ -1927,15 +1925,25 @@ export class CotalEndpoint extends EventEmitter {
                     else {
                         reply = await handler(req);
                     }
+                    body = JSON.stringify(reply);
                 }
                 catch (e) {
-                    reply = { ok: false, error: e.message };
+                    body = JSON.stringify({ ok: false, error: e.message });
                 }
                 try {
-                    m.respond(JSON.stringify(reply));
+                    m.respond(body);
                 }
-                catch {
-                    /* no reply inbox */
+                catch (e) {
+                    // `respond` signals a missing reply inbox by returning false; it throws when the reply exceeds the
+                    // broker's `max_payload`. Silence would leave the caller a bare timeout it cannot tell apart from
+                    // a dead service.
+                    const refused = { ok: false, error: `the ${service} reply is ${Buffer.byteLength(body)} bytes and was not published: ${e.message}` };
+                    try {
+                        m.respond(JSON.stringify(refused));
+                    }
+                    catch (err) {
+                        this.emit("error", new Error(`could not answer ${service} request on ${m.subject}: ${refused.error}, and the refusal failed too: ${err.message}`));
+                    }
                 }
             }
         })().catch((e) => this.emit("error", e));
@@ -2017,6 +2025,7 @@ export class CotalEndpoint extends EventEmitter {
                     return () => { reconnectHandler = undefined; };
                 },
                 signal: abortController.signal,
+                prepare: opts.prepare,
             });
         }
         finally {
@@ -2035,7 +2044,9 @@ export class CotalEndpoint extends EventEmitter {
      *     call re-issued for any command, up to {@link BIND_SPLIT_REISSUES} times while each re-issue
      *     is refused the same way. If a re-issue cannot be resolved, the refusal
      *     surfaces — still saying the command did not run — naming the resolve failure as why the
-     *     repair could not be attempted.
+     *     repair could not be attempted. An unpinned targeted call that reached a member holding no
+     *     mapping for its target ({@link replyTargetUnmapped}) is repaired the same way, and when the
+     *     re-issues run out, the latest such refusal is the one surfaced.
      *   - this CLIENT caught it on the reply ({@link respondedButUnbound}: a different instance,
      *     `failed-precondition`; the same instance at any other epoch, `expired`), which is what a
      *     responder too old to know the field produces. A live instance received and answered it, so
@@ -2079,10 +2090,22 @@ export class CotalEndpoint extends EventEmitter {
             }
             return invokeCommand(nc, this.space, service, command, args, { ...invokeOpts, signal });
         };
-        const doInvoke = async (signal = opts.signal) => {
+        // A `failed-precondition` from the resolve is its own refusal, raised before any command was
+        // published, so resolving once more is a repair.
+        const resolveFirst = async (signal = opts.signal) => {
+            try {
+                return await resolve(signal);
+            }
+            catch (e) {
+                if (!(e instanceof EpEnvelopeError) || e.code !== "failed-precondition")
+                    throw e;
+                return await resolve(signal);
+            }
+        };
+        const doInvoke = async (service, signal = opts.signal) => {
             signal?.throwIfAborted();
             try {
-                let r = await invokeResolved(await resolve(signal), signal);
+                let r = await invokeResolved(service, signal);
                 // THE RESPONDER FENCED IT (§13.2 `ai.cotal.ep.bind-refused`): a class member saw the call
                 // was bound to a different incarnation and refused BEFORE running the command.
                 //
@@ -2094,7 +2117,14 @@ export class CotalEndpoint extends EventEmitter {
                 // did not run, so each re-issue is a FIRST attempt, not a second. It repeats up to the bound
                 // the CLI uses, because the re-issue rides the same class queue and splits again at the same
                 // rate; repairing once left a quarter of all calls in a two-manager space failing (#443).
-                for (let reissues = 0; reissues < BIND_SPLIT_REISSUES && r.reply.ok === false && replyRefusedBeforeEffect(r.reply.error); reissues++) {
+                // A member that holds no mapping for the target also ran nothing, and on the class rail a
+                // sibling may host it. A pinned call named its instance, so that refusal is its answer.
+                // That member looked the target up, so when the re-issues run out its refusal is surfaced
+                // in place of a trailing bind refusal, which says nothing about the target.
+                const unmappedHere = (x) => opts.instanceId === undefined && replyTargetUnmapped(x.reply.error);
+                let unmapped = unmappedHere(r) ? r : undefined;
+                for (let reissues = 0; reissues < BIND_SPLIT_REISSUES && r.reply.ok === false
+                    && (replyRefusedBeforeEffect(r.reply.error) || unmappedHere(r)); reissues++) {
                     // Counted before it is repaired: a recovery that leaves no trace takes the split rate with it.
                     this.splitsRecovered++;
                     // `boundTo` is the other half of `servedBy`: who the handle THOUGHT it was talking to,
@@ -2145,12 +2175,12 @@ export class CotalEndpoint extends EventEmitter {
                         "not-executed");
                     }
                     r = await invokeResolved(reissueTarget, signal);
+                    if (unmappedHere(r))
+                        unmapped = r;
                 }
-                return r;
+                return replyRefusedBeforeEffect(r.reply.error) ? unmapped ?? r : r;
             }
             catch (e) {
-                if (!(e instanceof EpEnvelopeError))
-                    throw e;
                 // DO NOT auto-retry a command a responder already ANSWERED. This path covers the responders
                 // WITHOUT the fence above: they ignore `bind`, run the command, and the error is raised
                 // afterwards, so core cannot tell a repair from a duplicate and the allowlist is the only
@@ -2183,23 +2213,20 @@ export class CotalEndpoint extends EventEmitter {
                         throw e;
                     return await invokeResolved(await resolve(signal), signal);
                 }
-                // An UNMARKED `failed-precondition` is the resolve's own refusal, raised before any command
-                // was published, so re-resolving once is a repair. The `replyRefusedBeforeEffect` half keeps
-                // out the refusal THIS method raises after a failed re-issue: it carries that same code, and
-                // would otherwise fall into the re-resolve below as a THIRD attempt at a command whose
-                // second could not even be resolved. A marker means the disposition is already decided,
-                // whichever code carries it.
-                if (e.code !== "failed-precondition" || replyRefusedBeforeEffect(e.toEpError()))
-                    throw e;
-                this.resolvedServices.delete(endpoint);
-                return await invokeResolved(await resolve(signal), signal);
+                throw e;
             }
         };
         // P2 item 2 (2b): a goal-bearing command (spawn/launch) follows its acceptance to the terminal so
         // the caller still returns on the real outcome (UX unchanged); every other command replies directly.
         if (!opts.follow)
-            return doInvoke();
-        return this.followServiceGoal(endpoint, doInvoke, opts.deadlineMs ?? 10_000, { signal: opts.signal });
+            return doInvoke(await resolveFirst());
+        // Resolved before the submission starts: the follow reports an unmarked failure inside the
+        // submission as a command that may have run, and the resolve publishes none of this call.
+        let service;
+        return this.followServiceGoal(endpoint, (signal) => doInvoke(service, signal), opts.deadlineMs ?? 10_000, {
+            signal: opts.signal,
+            prepare: async (signal) => { service = await resolveFirst(signal); },
+        });
     }
     /** Send a durable-membership request to the SERVER-SIDE delivery daemon (`ctl.delivery`) and await its
      *  reply. Unlike {@link requestControl}, the reply rides a subject UNDER `ctl.delivery.<id>.>` (not the
@@ -2562,7 +2589,6 @@ export class CotalEndpoint extends EventEmitter {
         let scanPrefix;
         const entries = await liveKvEntries(kv, {
             onConsumer: (info) => { scanPrefix = this.recordOwnWatchConsumer(info); },
-            deleteOwnConsumer: (stream, name, del) => this.deleteOwnConsumer(stream, name, del),
         }).finally(() => {
             // The scan's own last delete goes out after nats.js closed its consumer, so every predecessor
             // delete the library sent is answered ahead of it on this connection.
@@ -2644,15 +2670,11 @@ export class CotalEndpoint extends EventEmitter {
         watch.consumer = consumer;
         watch.consumerStream = info.stream_name;
         watch.consumerName = info.name;
-        let pending = info.num_pending;
         let iter;
         try {
-            iter = await consumer.consume({ callback: (msg) => {
-                    const isUpdate = pending === 0 || --pending === 0;
-                    const entry = kv.jmToWatchEntry(msg, isUpdate);
+            iter = await consumer.consume({ callback: () => {
                     if (!watch.stopped && watch.consumer === consumer)
                         watch.onChange();
-                    void entry;
                 } });
         }
         catch (err) {
@@ -2694,31 +2716,12 @@ export class CotalEndpoint extends EventEmitter {
         watch.resolveStop = undefined;
         watch.rejectStop = undefined;
     }
-    /** Run one delete of a consumer this endpoint created. A profile without the delete row is refused
-     *  by design (#691), and nats.js reports a refused request twice: as the request's rejection, which
-     *  the caller handles, and on the connection status, which watchStatus would emit as an `error`.
-     *  A refused subject stays recorded until watchStatus drops that echo, since the two settle in
-     *  either order. */
-    async deleteOwnConsumer(stream, name, del) {
-        const subject = `$JS.API.CONSUMER.DELETE.${stream}.${name}`;
-        this.ownConsumerDeletes.add(subject);
-        let refused = false;
-        try {
-            return await del();
-        }
-        catch (err) {
-            refused = isPublishPermissionDenied(err);
-            throw err;
-        }
-        finally {
-            if (!refused)
-                this.ownConsumerDeletes.delete(subject);
-        }
-    }
     /** Record the consumer behind one of this connection's own KV watches, membership scans or history
      *  reads, and return the delete-subject prefix it recorded. nats.js names it `<prefix>_<serial>` and,
      *  rebuilding it after a stall or a sequence gap, deletes the predecessor itself with no hook before
-     *  the send, so a profile without the delete row (#691) has that delete refused. A watch's prefix
+     *  the send, so a profile without the delete row (#691) has that delete refused. A stalled link can
+     *  time that delete out before its refusal arrives, and nats.js then hands the refusal to no request,
+     *  so {@link trackRequestDenials} cannot own it and the consumer's name has to. A watch's prefix
      *  stays recorded for the connection because a retired watch's last rebuild can be refused after
      *  the watch has stopped; {@link readMembership} and {@link drainWindow} retire theirs after their own
      *  last delete, which goes out once nats.js has stopped the consumer and so is answered after every
@@ -2737,9 +2740,8 @@ export class CotalEndpoint extends EventEmitter {
      *  the elevated profile, which holds no stream-wide CONSUMER.DELETE (#691): the broker reaps the
      *  consumer at its inactive threshold. */
     async deleteReaderConsumer(consumer) {
-        const { stream_name, name } = await consumer.info(true);
         try {
-            await this.deleteOwnConsumer(stream_name, name, () => consumer.delete());
+            await consumer.delete();
         }
         catch (e) {
             if (!isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound) && !isPublishPermissionDenied(e))
@@ -2749,7 +2751,7 @@ export class CotalEndpoint extends EventEmitter {
     /** Delete one membership-watch consumer, swallowing ONLY already-gone and a refused delete. */
     async deleteMembershipConsumer(jsm, stream, name) {
         try {
-            return await this.deleteOwnConsumer(stream, name, () => jsm.consumers.delete(stream, name));
+            return await jsm.consumers.delete(stream, name);
         }
         catch (err) {
             if (membershipConsumerReleased(err))
@@ -2771,8 +2773,7 @@ export class CotalEndpoint extends EventEmitter {
         catch { /* already closed */ }
         if (consumer) {
             try {
-                const { stream_name, name } = await consumer.info(true);
-                const deleted = await this.deleteOwnConsumer(stream_name, name, () => consumer.delete());
+                const deleted = await consumer.delete();
                 if (deleted) {
                     watch.consumerStream = undefined;
                     watch.consumerName = undefined;
@@ -3271,15 +3272,18 @@ export class CotalEndpoint extends EventEmitter {
     // ---- internals -----------------------------------------------------------
     /**
      * Surface the connection's async status errors on our `error` event. NATS reports
-     * publish permission violations *only* here (subscription/request ones too), never on
-     * the failing call — so without this an over-tight ACL silently drops the agent's
-     * traffic and it just looks "absent". We annotate permission denials explicitly so a
-     * denial is never mistaken for absence (which already has a benign cause: MCP reconnect).
+     * publish permission violations *only* here (subscription ones too), never on the
+     * failing call — so without this an over-tight ACL silently drops the agent's
+     * traffic and it just looks "absent". A denied request is the exception: nats.js also
+     * rejects that request with the denial, so its caller owns it (see {@link trackRequestDenials}).
+     * We annotate permission denials explicitly so a denial is never mistaken for absence
+     * (which already has a benign cause: MCP reconnect).
      */
     watchStatus() {
         const nc = this.nc;
         if (!nc)
             return;
+        const handedToRequest = this.trackRequestDenials(nc);
         void (async () => {
             for await (const s of nc.status()) {
                 // A rebuild can replace `this.nc` before the old iterator finishes. Late disconnect/close
@@ -3309,11 +3313,12 @@ export class CotalEndpoint extends EventEmitter {
                 // and turns into a clean throw — it is not a connection error to surface.
                 if (s.error instanceof PermissionViolationError && this.confirmingChatSubs.has(s.error.subject))
                     continue;
-                // The echo of a refused delete of this endpoint's own consumer: one its caller already handled,
-                // or the predecessor nats.js deleted while rebuilding one of this connection's watches, scans or
-                // history reads.
+                if (s.error instanceof PermissionViolationError && handedToRequest.has(s.error))
+                    continue;
+                // The predecessor nats.js deleted while rebuilding one of this connection's watches, scans or
+                // history reads, whose refusal can arrive after that delete timed out.
                 if (s.error instanceof PermissionViolationError && s.error.operation === "publish"
-                    && (this.ownConsumerDeletes.delete(s.error.subject) || this.isOwnWatchDelete(s.error.subject)))
+                    && this.isOwnWatchDelete(s.error.subject))
                     continue;
                 this.emit("error", describeStatusError(s.error));
             }
@@ -3337,6 +3342,29 @@ export class CotalEndpoint extends EventEmitter {
             return;
         this.emit("transport", { connected: true, server: nc.getServer() });
     }
+    /** Wrap `nc.request` to record each publish denial nats.js rejects one of this connection's requests
+     *  with. nats.js rejects the pending request on the denied subject with the denial and then dispatches
+     *  that same error instance on the connection status, so the caller that received it (an empty history
+     *  read, a refused consumer delete, nats.js's own delete of a rebuilt watch's predecessor) owns it and
+     *  the status loop skips it. nats.js settles the request before it dispatches the status and this
+     *  reaction sits directly on the promise it returns, so it runs before the status loop body, which is
+     *  further microtasks away behind the status iterator. Any other denial is a different instance and is
+     *  not recorded, including one on the same subject and one that arrives after its request timed out.
+     *  A refused request inbox is rejected into every pending request but breaks every later one, so it
+     *  is not recorded either. */
+    trackRequestDenials(nc) {
+        const handed = new WeakSet();
+        const request = nc.request.bind(nc);
+        nc.request = (subject, payload, opts) => {
+            const reply = request(subject, payload, opts);
+            reply.catch(({ cause }) => {
+                if (cause instanceof PermissionViolationError && cause.operation === "publish")
+                    handed.add(cause);
+            });
+            return reply;
+        };
+        return handed;
+    }
     /** The error message for a guard that finds the endpoint unbound: "reconnecting" during a
      *  rebuild's null window OR an inter-retry backoff (so a concurrent op reports the real
      *  reason, not "not started" — `reestablishing` spans the whole retry loop incl. backoff),
@@ -4356,9 +4384,8 @@ export class CotalEndpoint extends EventEmitter {
         const reachedStart = page.length <= limit;
         const wanted = reachedStart ? page : page.slice(-limit);
         // A page that cannot be SENT is not a page. The reply rides one NATS message, so `limit` alone is
-        // the wrong bound: 200 large messages serialize past `max_payload`, `m.respond` throws inside
-        // `serveControl`'s swallow, and the caller sees a bare request timeout it cannot tell apart from a
-        // dead daemon. Measured, not predicted: 200 x ~6 KB timed out at 5s with the message "timeout".
+        // the wrong bound: 200 large messages serialize past `max_payload`, and `serveControl` can then only
+        // answer with a refusal in place of the page.
         //
         // So bound by BYTES too, keeping the NEWEST that fit — which needs no new vocabulary, because
         // `complete: false` already means "older history remains behind this page". Trimming here is the
@@ -6041,28 +6068,29 @@ export class CotalEndpoint extends EventEmitter {
             stuck: this.presenceWriteEscalated,
         };
     }
-    /** Bind a presence watch on the current connection. Resolves true when the watch was
-     *  installed, false when the endpoint stopped or rebuilt while the bind was in flight: that
-     *  bind's iterator is released here and nothing is installed, because the epoch that asked
-     *  for it is gone and the epoch that replaced it binds its own watch through
+    /** Bind a presence watch on the current connection. Resolves the bound consumer's info when
+     *  the watch was installed, undefined when the endpoint stopped or rebuilt while the bind was
+     *  in flight: that bind's iterator is released here and nothing is installed, because the
+     *  epoch that asked for it is gone and the epoch that replaced it binds its own watch through
      *  {@link connectAndBind}. Without this fence a bind that completes after {@link stop} would
      *  resurrect a watch on a stopped endpoint, and one that completes after a rebuild would
      *  overwrite the fresh epoch's watch with a dead-connection iterator. */
     async startPresenceWatch() {
         if (!this.kv)
-            return false;
+            return undefined;
         const epoch = this.presenceEpoch;
         let hydrated;
         this.presenceSnapshot = new Promise((resolve) => { hydrated = resolve; });
         const iter = await this.kv.watch();
-        this.recordOwnWatchConsumer(await kvWatchConsumer(iter).info(true));
+        const info = await kvWatchConsumer(iter).info(true);
+        this.recordOwnWatchConsumer(info);
         if (epoch !== this.presenceEpoch) {
             try {
                 iter.stop();
             }
             catch { /* its connection may already be gone */ }
             hydrated();
-            return false;
+            return undefined;
         }
         this.presenceWatchIter = iter;
         void (async () => {
@@ -6083,7 +6111,7 @@ export class CotalEndpoint extends EventEmitter {
             }
             hydrated();
         })().catch((e) => this.emit("error", e));
-        return true;
+        return info;
     }
     /**
      * Replace a presence watch that has gone silent past TTL while the connection is up. The new
@@ -6111,11 +6139,11 @@ export class CotalEndpoint extends EventEmitter {
                 // one a held link never answers must leave the old watch in place: on a plain stall that
                 // watch is the one that recovers by itself, and its replay is still guarded against
                 // expired PUTs. Only a successfully bound watch retires its predecessor.
-                const installed = await this.startPresenceWatch();
+                const info = await this.startPresenceWatch();
                 // Retired mid-bind (stop or rebuild moved the epoch): the late iterator is already
                 // released and the old watch was torn down by whoever moved the epoch. Nothing to
                 // retire, nothing to report.
-                if (!installed)
+                if (!info)
                     return;
                 if (old && old !== this.presenceWatchIter) {
                     try {
@@ -6124,11 +6152,10 @@ export class CotalEndpoint extends EventEmitter {
                     catch { /* already closed with its consumer */ }
                 }
                 // A bucket with no keys replays nothing, so the new watch cannot refresh
-                // `lastPresenceWatchAt` by delivering. It IS current knowledge: nobody is present. Read
-                // the consumer's initial pending count for that one fact; nats.js's KV watch computed it
-                // from the same `info(true)` it used to place the isUpdate marker.
-                const pending = this.presenceWatchIter?._data?._info?.num_pending;
-                if (pending === 0)
+                // `lastPresenceWatchAt` by delivering. It IS current knowledge: nobody is present. The
+                // bind's consumer info carries the initial pending count for that one fact; nats.js's KV
+                // watch placed the isUpdate marker from the same `info(true)`.
+                if (info.num_pending === 0)
                     await this.onPresenceBucketEmpty();
                 this.emit("warning", new Error(`presence watch silent for ${silentMs}ms with the connection up; rebound it from the bucket's current state`));
             }