@cotal-ai/core 0.3.2 → 0.4.0

package/dist/endpoint.js

@@ -2,16 +2,21 @@ import { EventEmitter } from "node:events";
 import { randomUUID } from "node:crypto";
 import { connect, credsAuthenticator, nanos, AuthorizationError, PermissionViolationError, UserAuthenticationExpiredError, } from "@nats-io/transport-node";
 import { idFromCreds } from "./identity.js";
-import { createSpaceStreams, dmDurableConfig, taskDurableConfig, MAX_MSGS_PER_SUBJECT } from "./streams.js";
+import { assertValidName } from "./resolve.js";
+import { createSpaceStreams, chatDurableConfig, dmDurableConfig, taskDurableConfig, MAX_MSGS_PER_SUBJECT } from "./streams.js";
 import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, } from "@nats-io/jetstream";
 import { Kvm } from "@nats-io/kv";
 import { openChannelRegistry, effectiveReplay, effectiveReplayWindowMs, readChannelConfig, readChannelDefaults, } from "./channels.js";
-import { anycastSubject, CHANNEL_DEFAULTS_KEY, chatStream, chatDurable, chatSubject, collapseFilterSubjects, controlServiceSubject, dmStream, dmDurable, isConcreteChannel, normalizeMentions, parseSubject, presenceBucket, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, token, unicastSubject, } from "./subjects.js";
+import { anycastSubject, CHANNEL_DEFAULTS_KEY, chatStream, chatDurable, chatHistDurable, chatSubject, collapseFilterSubjects, controlServiceSubject, CONTROL_SELF_SERVICE, dmStream, dmDurable, isConcreteChannel, normalizeMentions, parseSubject, presenceBucket, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, token, unicastSubject, } from "./subjects.js";
 export const DEFAULT_SERVER = "nats://127.0.0.1:4222";
 /** Space joined when none is given on the CLI (the `cotal-<space>` cmux tab, etc.). */
 export const DEFAULT_SPACE = "main";
 /**
- * Events: "message" (CotalMessage), "presence" (PresenceEvent), "roster" (Presence[]), "error" (Error).
+ * Events: "message" (CotalMessage), "presence" (PresenceEvent), "roster" (Presence[]), "error" (Error),
+ * "connection" ({ connected: boolean }) — true on every successful (re)bind (initial start, manual
+ * reconnect, AND background self-heal), false the moment the connection drops (rebuild null window /
+ * terminal close). Lets an in-process agent track connectedness off the endpoint's own (re)binds
+ * instead of an imperative flag the self-heal path can't reach.
  *
  * Callers MUST attach an "error" listener before `start()`: async faults (incl. NATS
  * permission denials, surfaced via `watchStatus`) are emitted as "error", and Node throws
@@ -48,6 +53,9 @@ export class CotalEndpoint extends EventEmitter {
      *  a lagging joiner + dedups the backfill overlap). Keyed by the subscription pattern (may be
      *  wildcard), so the drop matches every concrete channel the pattern subsumes. */
     joinSeq = new Map();
+    /** Serializes history reads ({@link collectHistory}): they share the fixed per-instance
+     *  `chathist_<id>` consumer, so overlapping reads would delete/recreate it under one another. */
+    histLock = Promise.resolve();
     subs = [];
     streamMsgs = [];
     heartbeatTimer;
@@ -56,9 +64,27 @@ export class CotalEndpoint extends EventEmitter {
     status = "idle";
     activity;
     stopped = false;
+    /** In-flight rebuild (drain+rebind) — serializes manual reconnect, the supervisor's
+     *  closed(), and reestablishLoop so only ONE rebuild runs at a time (a second trigger
+     *  coalesces onto the shared promise, never starts a parallel connectAndBind). */
+    rebuildPromise;
+    /** True only during the null window of a rebuild (this.nc unset) — user-facing ops then
+     *  throw a "reconnecting" message instead of the misleading "endpoint not started". */
+    reconnecting = false;
+    /** One reestablishLoop at a time; concurrent triggers coalesce via rebuild(). */
+    reestablishing = false;
+    /** Interruptible backoff for reestablishLoop — reconnect()/stop() resolves this to retry
+     *  now instead of awaiting the full retryMs. */
+    backoffResolve;
+    backoffTimer;
+    retryMs = 3000;
     constructor(opts) {
         super();
         this.space = opts.space;
+        // A display name is the client-side handle a peer is addressed by; reject the reserved `/`
+        // (the future owner/name separator) and surrounding whitespace at the one identity choke
+        // point every join/spawn path flows through.
+        assertValidName(opts.card.name);
         // Identity precedence: an explicit card.id, else the creds' identity, else a random
         // uuid. When both an id and creds are given they MUST name the same nkey — otherwise
         // the subject sender token wouldn't match the authenticated user and every publish
@@ -87,6 +113,19 @@ export class CotalEndpoint extends EventEmitter {
         return { id: this.card.id, name: this.card.name, role: this.card.role };
     }
     async start() {
+        await this.connectAndBind();
+        // nats.js auto-reconnects transient drops; when it exhausts its attempts and the
+        // connection closes for good, rebuild from scratch so an in-process agent (e.g. the
+        // OpenCode plugin) recovers without a host respawn. Armed only after a successful first
+        // connect — a first-connect failure throws to the caller's connect-retry loop instead.
+        this.superviseConnection();
+    }
+    /** Open the connection and bind everything that hangs off it: status watch, presence
+     *  watch + heartbeat, channel registry, and the durable consumers. Re-runnable — a
+     *  reconnect calls it again after {@link clearConnectionScoped}; every binding is
+     *  idempotent (durables bind by name, JetStream dedups by msgID, KV opens are idempotent). */
+    async connectAndBind() {
+        this.clearConnectionScoped();
         this.nc = await connect({
             servers: this.servers,
             name: `cotal:${this.card.name}`,
@@ -134,11 +173,184 @@ export class CotalEndpoint extends EventEmitter {
                 await this.ensureStreams();
             await this.startConsumers();
         }
+        // Bound and live — covers initial start, manual reconnect, AND background self-heal (every
+        // path lands here). The single signal an in-process agent's connected flag tracks.
+        this.emit("connection", { connected: true });
+    }
+    /** Tear down everything {@link connectAndBind} (re)creates, so a rebind can't leak a
+     *  second heartbeat, double-pump a consumer, or keep stale roster ghosts. Caller-owned
+     *  subs (tap/serve) are left alone — they aren't rebuilt here. */
+    clearConnectionScoped() {
+        if (this.heartbeatTimer) {
+            clearInterval(this.heartbeatTimer);
+            this.heartbeatTimer = undefined;
+        }
+        if (this.sweepTimer) {
+            clearInterval(this.sweepTimer);
+            this.sweepTimer = undefined;
+        }
+        for (const msgs of this.streamMsgs) {
+            try {
+                msgs.stop();
+            }
+            catch {
+                /* already closed with the connection */
+            }
+        }
+        this.streamMsgs.length = 0;
+        this.roster.clear();
+        this.joinSeq.clear();
+        this.channelConfigs.clear();
+        this.channelDefaults = {};
+    }
+    /** If stop() ran during a rebuild's `await connectAndBind`, the just-bound connection +
+     *  heartbeat + supervisor would be left live on a stopped endpoint. Tear that fresh
+     *  connection back down and report it. Reads `this.nc` in its own scope (a bare `this.nc`
+     *  in doRebuild narrows to `never` via TS inlining connectAndBind's assignment). Returns
+     *  true iff it tore something down (caller bails out of the rebuild). */
+    async tearDownIfStopped() {
+        if (!this.stopped)
+            return false;
+        const nc = this.nc;
+        this.clearConnectionScoped();
+        try {
+            await nc?.drain();
+        }
+        catch {
+            /* already closing */
+        }
+        this.nc = undefined;
+        return true;
+    }
+    /** Watch for a terminal close (nats.js has exhausted its own reconnect) and rebuild.
+     *  Our own stop()/drain also resolves closed(), so the `stopped` guard keeps a clean
+     *  shutdown from re-establishing. The identity guard (`this.nc !== nc`) no-ops a STALE
+     *  supervisor — one whose connection reconnect()/rebuild already replaced — so only a
+     *  close of the CURRENT connection triggers a rebuild. The rebuild itself is serialized
+     *  with the manual path via {@link rebuild}. */
+    superviseConnection() {
+        const nc = this.nc;
+        if (!nc)
+            return;
+        void nc.closed().then((err) => {
+            if (this.stopped)
+                return;
+            if (this.nc !== nc)
+                return; // epoch-stale — a rebuild already swapped this connection
+            this.emit("connection", { connected: false }); // dropped — report it before the rebuild kicks in
+            this.emit("error", new Error(`mesh connection closed${err ? `: ${err.message}` : ""} — re-establishing`));
+            void this.reestablishLoop();
+        });
+    }
+    /** Single serialized rebuild: drain the old connection and rebind via {@link connectAndBind},
+     *  guarded so concurrent triggers (manual {@link reconnect}, the supervisor's closed(), the
+     *  retry loop) coalesce onto ONE in-flight rebuild instead of racing two connectAndBinds and
+     *  leaking a connection. Returns the shared promise; a second caller gets the in-flight one. */
+    rebuild() {
+        if (this.rebuildPromise)
+            return this.rebuildPromise;
+        const p = this.doRebuild().finally(() => {
+            if (this.rebuildPromise === p)
+                this.rebuildPromise = undefined;
+        });
+        this.rebuildPromise = p;
+        return p;
+    }
+    /** The transition: stop the connection-scoped timers FIRST (so nothing live touches
+     *  this.nc during the null window), drop the connection refs, drain the old nc, then
+     *  rebind + re-arm the supervisor on the fresh connection. clearConnectionScoped is
+     *  idempotent, so connectAndBind's own call here is a noop. */
+    async doRebuild() {
+        const oldNc = this.nc;
+        this.reconnecting = true;
+        try {
+            this.clearConnectionScoped();
+            this.nc = undefined;
+            this.js = undefined;
+            this.jsm = undefined;
+            this.kv = undefined;
+            this.channelKv = undefined;
+            this.emit("connection", { connected: false }); // null window opened — not live until the rebind below
+            try {
+                await oldNc?.drain();
+            }
+            catch {
+                /* already closing */
+            }
+            await this.connectAndBind();
+            // stop() may have run during the await — don't leave a live connection + heartbeat +
+            // supervisor on a stopped endpoint. (Reads this.nc in its own scope — a bare `this.nc`
+            // here in doRebuild narrows to `never` via TS inlining connectAndBind's assignment.)
+            if (await this.tearDownIfStopped())
+                return;
+            this.superviseConnection(); // re-arm on the fresh nc
+        }
+        finally {
+            this.reconnecting = false;
+        }
+    }
+    /** Rebuild with backoff until it sticks or we're stopped. Interruptible: a manual
+     *  {@link reconnect} kicks the backoff so the next attempt runs immediately instead of
+     *  awaiting the full retryMs. One loop at a time ({@link reestablishing}); concurrent
+     *  triggers coalesce via {@link rebuild}. */
+    async reestablishLoop() {
+        if (this.reestablishing)
+            return;
+        this.reestablishing = true;
+        try {
+            while (!this.stopped) {
+                try {
+                    await this.rebuild();
+                    return; // success — re-armed; the supervisor re-triggers on the next terminal close
+                }
+                catch (e) {
+                    if (!this.stopped)
+                        this.emit("error", e);
+                    await new Promise((resolve) => {
+                        this.backoffResolve = resolve;
+                        this.backoffTimer = setTimeout(resolve, this.retryMs);
+                    });
+                }
+            }
+        }
+        finally {
+            this.reestablishing = false;
+        }
+    }
+    /** Cut an in-flight reestablish backoff short so the next attempt runs immediately, and
+     *  clear its timer so it can't fire later on a stopped/restarted loop. */
+    kickBackoff() {
+        this.backoffResolve?.();
+        if (this.backoffTimer) {
+            clearTimeout(this.backoffTimer);
+            this.backoffTimer = undefined;
+        }
+    }
+    /** Manual reconnect: tear down the current connection and rebuild, WITHOUT the permanent
+     *  stop (stopped/stopping stay false). Serialized with the self-heal supervisor via
+     *  {@link rebuild}, and interruptible — if a backoff is in flight, kick it so the attempt
+     *  is now, not in retryMs. Throws if stopped. On failure, leaves {@link reestablishLoop}
+     *  running in the background so the endpoint never stays dead, and rethrows so the caller
+     *  can report it. */
+    async reconnect() {
+        if (this.stopped)
+            throw new Error("endpoint stopped — cannot reconnect");
+        this.kickBackoff();
+        try {
+            await this.rebuild();
+        }
+        catch (e) {
+            void this.reestablishLoop(); // background retry until success or stop
+            throw e;
+        }
     }
     async stop() {
         if (this.stopped)
             return;
         this.stopped = true;
+        // Wake a reestablishLoop sitting in backoff so it sees `stopped` and exits instead of
+        // sleeping out retryMs; also clears the timer so it can't fire later.
+        this.kickBackoff();
         if (this.heartbeatTimer)
             clearInterval(this.heartbeatTimer);
         if (this.sweepTimer)
@@ -286,7 +498,7 @@ export class CotalEndpoint extends EventEmitter {
     /** Send a control request to a service and await its reply (client side). */
     async requestControl(service, req, timeoutMs = 5000) {
         if (!this.nc)
-            throw new Error("endpoint not started");
+            throw new Error(this.notLiveMsg());
         const body = { ...req, from: req.from ?? this.ref() };
         const m = await this.nc.request(controlServiceSubject(this.space, service, this.card.id), JSON.stringify(body), { timeout: timeoutMs });
         return m.json();
@@ -327,17 +539,20 @@ export class CotalEndpoint extends EventEmitter {
      */
     async joinChannel(channel) {
         if (!this.jsm)
-            throw new Error("endpoint not started");
+            throw new Error(this.notLiveMsg());
         if (this.channels.includes(channel))
             return { joined: false, backfilled: 0 };
-        const next = collapseFilterSubjects([...this.channels, channel].map((ch) => chatSubject(this.space, "*", ch)));
         // Arm the watermark BEFORE the filter flip (single-delivery: a tail message on the new
         // channel is then either ≤ frontier → backfill-only or > frontier → tail-only, never both),
         // and filter BEFORE backfill (gap-safe: backfill-first leaves a window in neither stream).
         const armed = await this.armJoin([channel]);
-        await this.jsm.consumers.update(chatStream(this.space), chatDurable(this.card.id), {
-            filter_subjects: next,
-        });
+        try {
+            await this.setChatFilter([...this.channels, channel]);
+        }
+        catch (e) {
+            this.joinSeq.delete(channel); // the flip was rejected (e.g. outside allowSubscribe) — undo the arm
+            throw e;
+        }
         this.channels.push(channel);
         const backfilled = await this.backfillArmed(armed);
         return { joined: true, backfilled };
@@ -347,26 +562,53 @@ export class CotalEndpoint extends EventEmitter {
      *  leaving). Returns whether anything changed. */
     async leaveChannel(channel) {
         if (!this.jsm)
-            throw new Error("endpoint not started");
+            throw new Error(this.notLiveMsg());
         const i = this.channels.indexOf(channel);
         if (i < 0)
             return { left: false };
         if (this.channels.length === 1)
             throw new Error(`cannot leave "${channel}" — it is your only channel (an empty filter would subscribe to all)`);
         const remaining = this.channels.filter((c) => c !== channel);
-        await this.jsm.consumers.update(chatStream(this.space), chatDurable(this.card.id), {
-            filter_subjects: collapseFilterSubjects(remaining.map((ch) => chatSubject(this.space, "*", ch))),
-        });
+        await this.setChatFilter(remaining);
         this.channels.splice(i, 1);
         this.joinSeq.delete(channel);
         return { left: true };
     }
+    /** Move the chat live-tail durable to a new channel set. OPEN mode self-serves the
+     *  `consumers.update` (the agent owns its durable). AUTH mode is bind-only — the agent has no
+     *  UPDATE grant — so it sends a mediated control request to the manager, which validates the set
+     *  ⊆ its `allowSubscribe` before moving the filter. Throws clearly when no privileged responder is
+     *  present: a manager-less standalone auth session is fixed to its boot subscribe set — a
+     *  documented limitation, not a silent degrade. */
+    async setChatFilter(channels) {
+        if (!this.jsm)
+            throw new Error(this.notLiveMsg());
+        if (!this.creds) {
+            await this.jsm.consumers.update(chatStream(this.space), chatDurable(this.card.id), {
+                filter_subjects: collapseFilterSubjects(channels.map((ch) => chatSubject(this.space, "*", ch))),
+            });
+            return;
+        }
+        let reply;
+        try {
+            reply = await this.requestControl(CONTROL_SELF_SERVICE, { op: "setChannels", args: { channels } });
+        }
+        catch (e) {
+            const msg = e.message;
+            if (/no responders/i.test(msg))
+                throw new Error("cannot change channels at runtime: no privileged provisioner (manager) is serving the mesh — " +
+                    "this session is fixed to its boot subscribe set");
+            throw e;
+        }
+        if (!reply.ok)
+            throw new Error(reply.error ?? "channel change rejected");
+    }
     /** One coherent channel model for dashboards: every channel that has messages OR a registry
      *  entry (configured-but-empty), each tagged with its {@link ChannelConfig}. Works even on
      *  observer endpoints (no consumers needed). */
     async listChannels() {
         if (!this.nc)
-            throw new Error("endpoint not started");
+            throw new Error(this.notLiveMsg());
         const mgr = await jetstreamManager(this.nc);
         // Subjects carry the sender (chat.<sender>.<channel>), so collapse across senders: sum
         // each channel's counts regardless of who published.
@@ -512,9 +754,18 @@ export class CotalEndpoint extends EventEmitter {
                 this.emit("error", e);
         });
     }
+    /** The error message for a guard that finds the endpoint unbound: "reconnecting" during a
+     *  rebuild's null window OR an inter-retry backoff (so a concurrent op reports the real
+     *  reason, not "not started" — `reestablishing` spans the whole retry loop incl. backoff),
+     *  else "endpoint not started" (genuine pre-start). */
+    notLiveMsg() {
+        return this.reconnecting || this.reestablishing
+            ? "reconnecting — try again shortly"
+            : "endpoint not started";
+    }
     async publishMsg(subject, msg) {
         if (!this.js)
-            throw new Error("endpoint not started");
+            throw new Error(this.notLiveMsg());
         // msgID = message id → free server-side dedup across JetStream redelivery.
         await this.js.publish(subject, JSON.stringify(msg), { msgID: msg.id });
     }
@@ -525,6 +776,29 @@ export class CotalEndpoint extends EventEmitter {
             throw new Error("endpoint not started");
         await createSpaceStreams(this.jsm, this.space);
     }
+    /**
+     * Privileged: pre-create an agent's bind-only chat live-tail durable (auth mode), filtered to its
+     * `subscribe` set, so the agent can BIND it without holding CONSUMER.CREATE/UPDATE on CHAT — its
+     * live read can't be self-widened past `allowSubscribe`. The creator sets the filter; the agent
+     * never does (mirrors {@link provisionDmInbox}). Idempotent. The caller must be permissive on CHAT.
+     */
+    async provisionChatDurable(targetId, subscribe) {
+        const jsm = await this.manager();
+        await jsm.consumers.add(chatStream(this.space), chatDurableConfig(this.space, targetId, subscribe));
+    }
+    /**
+     * Privileged: move an agent's bind-only chat durable to a new channel set — the write half of the
+     * mediated join/leave. The manager calls this AFTER validating the set ⊆ the agent's
+     * `allowSubscribe`; the agent itself has no UPDATE grant, so this trusted path is the only way its
+     * live filter moves. The filter is rebuilt from channel names here (not from agent-supplied
+     * subjects) so a caller can't smuggle a hand-built filter.
+     */
+    async setChatFilterFor(targetId, channels) {
+        const jsm = await this.manager();
+        await jsm.consumers.update(chatStream(this.space), chatDurable(targetId), {
+            filter_subjects: collapseFilterSubjects(channels.map((ch) => chatSubject(this.space, "*", ch))),
+        });
+    }
     /**
      * Privileged: pre-create an agent's DM inbox durable (auth mode), so the agent can BIND
      * it without holding CONSUMER.CREATE on DM_<space>. The creator sets the filter to
@@ -559,8 +833,6 @@ export class CotalEndpoint extends EventEmitter {
         if (!this.jsm)
             throw new Error("endpoint not started");
         const id = this.card.id;
-        const ack_wait = nanos(this.ackWaitMs);
-        const inactive_threshold = nanos(this.inactiveThresholdMs);
         // Unicast: this instance's private DM inbox. Open mode self-creates; auth mode BINDS a
         // durable the provisioner pre-created (agents are denied CONSUMER.CREATE on DM_<space>,
         // since the create-time filter_subject is the attack surface — see provisionDmInbox).
@@ -577,37 +849,53 @@ export class CotalEndpoint extends EventEmitter {
         if (this.channels.length) {
             const durable = chatDurable(id);
             const want = collapseFilterSubjects(this.channels.map((ch) => chatSubject(this.space, "*", ch)));
+            // Auth mode: the chat live-tail durable is pre-created BIND-ONLY by the provisioner (the agent
+            // is denied CONSUMER.CREATE/UPDATE on CHAT — its filter is the read boundary). Open mode: the
+            // agent owns it and self-creates. Either way it is a DeliverPolicy.New tail; per-channel
+            // history is the explicit backfill below (the only shape that honors per-channel replay
+            // policy given deliver_policy is consumer-wide).
             const info = await this.consumerInfo(chatStream(this.space), durable);
             if (!info) {
-                // Fresh durable: a New tail (history is the explicit backfill below — the only shape
-                // that honors per-channel policy given deliver_policy is consumer-wide).
-                await this.jsm.consumers.add(chatStream(this.space), {
-                    durable_name: durable,
-                    filter_subjects: want,
-                    ack_policy: AckPolicy.Explicit,
-                    ack_wait,
-                    deliver_policy: DeliverPolicy.New,
-                    inactive_threshold,
-                });
+                if (this.creds)
+                    throw new Error(`chat durable ${durable} not pre-created — a launcher must call provisionChatDurable ` +
+                        `(auth mode binds the durable, it never self-creates)`);
+                await this.jsm.consumers.add(chatStream(this.space), chatDurableConfig(this.space, id, this.channels, {
+                    ackWaitMs: this.ackWaitMs,
+                    inactiveThresholdMs: this.inactiveThresholdMs,
+                }));
+            }
+            // First bind to this durable (open self-create, or an auth pre-create never consumed) ⇒
+            // backfill the full subscribe set. A later reconnect (the consumed cursor has advanced)
+            // backfills only channels the config GAINED — un-acked live messages auto-redeliver, so a full
+            // re-backfill would double up. With pre-create, `info` always exists under auth, so the
+            // consumed cursor — not the durable's existence — is what tells first-bind from reconnect.
+            //
+            // Caveat (best-effort, by design): `consumer_seq > 0` proves the durable has delivered at
+            // least once, NOT that the initial backfill completed. A crash between the first delivery and
+            // backfillArmed() makes the next bind take the reconnect path and skip the full pre-bind
+            // backfill. This is unchanged from the prior self-create path (which keyed on durable
+            // existence and had the same gap — and was actually weaker: a crash before any delivery left
+            // the durable existing, so it never re-backfilled; consumer_seq still 0 here re-backfills).
+            // Reliable FORWARD delivery is the durable's job (un-acked redelivery); pre-bind history is
+            // opportunistic. A backfill-completion marker would make it reliable — a deferred follow-up.
+            const consumed = (info?.delivered?.consumer_seq ?? 0) > 0;
+            if (!consumed) {
                 // Arm the tail-drop watermarks BEFORE pump starts, so the tail can never deliver a
-                // just-created channel's message un-watermarked (which would double-emit: live + backfill).
+                // just-bound channel's message un-watermarked (which would double-emit: live + backfill).
                 const armed = await this.armJoin(this.channels);
                 await this.pump(chatStream(this.space), durable);
                 await this.backfillArmed(armed);
             }
             else {
-                // Rebind: reconcile the durable's filter to the CURRENT config (a config that changed
-                // between restarts is honored). Channels the config GAINED are backfilled like a fresh
-                // join; channels it LOST are dropped from the filter. An unchanged config = pure resume,
-                // empty diff, no re-replay.
+                // Reconnect: resume the tail, then backfill any channels the config GAINED since.
                 await this.pump(chatStream(this.space), durable);
                 const haveFilters = info.config.filter_subjects ?? (info.config.filter_subject ? [info.config.filter_subject] : []);
-                // Channels the config gained = those not already covered by the durable's filters (a
-                // wildcard already covers its sub-channels). Backfill only those.
                 const gained = this.channels.filter((c) => !haveFilters.some((f) => subjectMatches(f, chatSubject(this.space, "*", c))));
-                // Arm watermarks for the gained channels BEFORE the filter reconcile flips them on.
                 const armed = gained.length ? await this.armJoin(gained) : undefined;
-                if (!sameSet(haveFilters, want))
+                // Reconcile the durable's filter to the CURRENT config — OPEN MODE ONLY. Auth mode is
+                // bind-only (no UPDATE grant): the durable's filter is authoritative, moved solely by the
+                // mediated join/leave control op, so the agent never self-reconciles it.
+                if (!this.creds && !sameSet(haveFilters, want))
                     await this.jsm.consumers.update(chatStream(this.space), durable, { filter_subjects: want });
                 if (armed)
                     await this.backfillArmed(armed);
@@ -738,73 +1026,122 @@ export class CotalEndpoint extends EventEmitter {
     async joinPolicyFresh(channel) {
         if (!this.channelKv)
             return { replay: effectiveReplay(undefined, undefined) };
+        // A wildcard subscription (`review.>`) has no single registry entry — and `>`/`*` are illegal
+        // KV keys, so a per-channel get would throw. Read only the space defaults for it; concrete
+        // channels still get their per-channel override.
         const [cfg, defaults] = await Promise.all([
-            readChannelConfig(this.channelKv, channel),
+            isConcreteChannel(channel) ? readChannelConfig(this.channelKv, channel) : Promise.resolve(undefined),
             readChannelDefaults(this.channelKv),
         ]);
         return { replay: effectiveReplay(cfg, defaults), windowMs: effectiveReplayWindowMs(cfg, defaults) };
     }
-    /** Read a channel's retained history up to `upToSeq` via JetStream **Direct Get** (a read
-     *  verb — no consumer create, so it rides a read-only grant) and emit each message as a
-     *  `historical` "message" event. `sinceMs` bounds how far back via a native Direct-Get
-     *  `start_time` (now − window); unset ⇒ the full retained window. New messages (`seq > upToSeq`)
-     *  are skipped — the live tail owns them. Pages the batch API; the ack handle is a no-op. */
-    async backfillChannel(channel, upToSeq, sinceMs) {
-        if (!this.jsm)
+    /**
+     * Read retained chat history on ONE channel subject through a name-scoped, single-filter
+     * EPHEMERAL pull consumer — the broker-contained replacement for the removed Direct Get. The
+     * create rides `$JS.API.CONSUMER.CREATE.<CHAT>.<chathist_id>.<subject>`, whose trailing filter
+     * token nats-server pins to the request body (JSConsumerCreateFilterSubjectMismatchErr, code
+     * 10131) — so an agent can only ever replay a channel its `allowSubscribe` grants. Single filter
+     * only (plural isn't ACL-constrainable); `AckPolicy.None` + `mem_storage` so it leaves no durable
+     * state, and it is deleted right after. Returns raw messages in stream order from `start`,
+     * stopping once past `untilSeq` (exclusive of it) or after `limit`. The per-instance name means
+     * calls must be serial — every reader here awaits to completion, so they are.
+     */
+    async collectHistory(subject, start, opts = {}) {
+        // Serialize on the per-instance lock: the fixed `chathist_<id>` name means two concurrent reads
+        // (recall + join-backfill + drop-marker can race in-process) would delete/recreate the consumer
+        // under each other and cross-feed results. The chain makes the "serial callers" assumption true.
+        const run = this.histLock.then(() => this.collectHistoryInner(subject, start, opts));
+        this.histLock = run.catch(() => { }); // keep the chain alive on error
+        return run;
+    }
+    async collectHistoryInner(subject, start, opts = {}) {
+        if (!this.jsm || !this.js)
             throw new Error("endpoint not started");
-        const subject = chatSubject(this.space, "*", channel);
-        const collected = [];
-        // First page starts by time when a window is set (native), else from seq 1; after that we
-        // always page by sequence.
-        const startTime = sinceMs === undefined ? undefined : new Date(Date.now() - sinceMs);
-        let startSeq = 1;
-        let first = true;
-        pages: for (;;) {
-            let last = 0;
-            let got = 0;
-            try {
-                const query = first && startTime !== undefined
-                    ? { start_time: startTime, next_by_subj: subject, batch: 256 }
-                    : { seq: startSeq, next_by_subj: subject, batch: 256 };
-                first = false;
-                const iter = await this.jsm.direct.getBatch(chatStream(this.space), query);
-                for await (const sm of iter) {
+        const stream = chatStream(this.space);
+        const name = chatHistDurable(this.card.id);
+        const out = [];
+        // Clear any consumer leaked by a crashed prior read before re-creating it with THIS read's
+        // single filter (the read ACL is enforced at create — see the doc above).
+        try {
+            await this.jsm.consumers.delete(stream, name);
+        }
+        catch { /* none — fine */ }
+        await this.jsm.consumers.add(stream, {
+            name,
+            filter_subject: subject,
+            ack_policy: AckPolicy.None,
+            mem_storage: true,
+            inactive_threshold: nanos(30_000),
+            ...("time" in start
+                ? { deliver_policy: DeliverPolicy.StartTime, opt_start_time: start.time.toISOString() }
+                : { deliver_policy: DeliverPolicy.StartSequence, opt_start_seq: start.seq }),
+        });
+        try {
+            const consumer = await this.js.consumers.get(stream, name);
+            let pending = (await consumer.info()).num_pending;
+            while (pending > 0) {
+                const want = Math.min(pending, 256);
+                const iter = await consumer.fetch({ max_messages: want, expires: 5_000 });
+                let got = 0;
+                for await (const m of iter) {
                     got++;
-                    if (sm.seq > upToSeq)
-                        break pages; // crossed the frontier — the tail owns the rest
-                    last = sm.seq;
-                    let msg;
-                    try {
-                        msg = sm.json();
-                    }
-                    catch {
-                        continue; // skip undecodable
-                    }
-                    // Same authenticity guard as the tail; skip our own echoes in history.
-                    const parsed = parseSubject(sm.subject);
-                    if (!parsed || msg.from?.id !== parsed.sender || msg.from.id === this.card.id)
+                    if (opts.untilSeq !== undefined && m.seq > opts.untilSeq)
+                        return out; // crossed the frontier
+                    // Belt-and-suspenders over the lock: only keep messages on the requested channel subject
+                    // (the consumer's filter already bounds this; guards against any stale-consumer edge).
+                    if (!subjectMatches(subject, m.subject))
                         continue;
-                    collected.push({ msg, seq: sm.seq });
+                    out.push(m);
+                    if (opts.limit !== undefined && out.length >= opts.limit)
+                        return out;
                 }
+                if (got < want)
+                    break; // drained early
+                pending -= got;
             }
-            catch (e) {
-                // Batch Direct Get raises a 404 ("message not found") when no message matches from
-                // `start` — the normal "no more history" signal (empty channel or last page), not a
-                // fault. Anything else is real.
-                if (e.code === 404)
-                    break;
-                this.emit("error", e);
-                break;
+        }
+        finally {
+            try {
+                await this.jsm.consumers.delete(stream, name);
             }
-            if (got === 0 || last === 0)
-                break; // drained
-            startSeq = last + 1;
+            catch { /* already gone */ }
+        }
+        return out;
+    }
+    /** Read a channel's retained history up to `upToSeq` (the join frontier) and emit each message
+     *  as a `historical` "message" event. `sinceMs` bounds how far back via a native consumer
+     *  `start_time` (now − window); unset ⇒ the full retained window. New messages (`seq > upToSeq`)
+     *  are skipped — the live tail owns them. Reads through the contained {@link collectHistory}. */
+    async backfillChannel(channel, upToSeq, sinceMs) {
+        const subject = chatSubject(this.space, "*", channel);
+        const start = sinceMs === undefined ? { seq: 1 } : { time: new Date(Date.now() - sinceMs) };
+        let msgs;
+        try {
+            msgs = await this.collectHistory(subject, start, { untilSeq: upToSeq });
+        }
+        catch (e) {
+            this.emit("error", e);
+            return 0;
         }
         const noop = { ack: () => { }, nak: () => { } };
-        for (const { msg } of collected)
-            // Backfill only ever pages the chat stream, so the authenticated class is always "channel".
+        let n = 0;
+        for (const sm of msgs) {
+            let msg;
+            try {
+                msg = sm.json();
+            }
+            catch {
+                continue; // skip undecodable
+            }
+            // Same authenticity guard as the tail; skip our own echoes in history.
+            const parsed = parseSubject(sm.subject);
+            if (!parsed || msg.from?.id !== parsed.sender || msg.from.id === this.card.id)
+                continue;
+            // Backfill only ever reads the chat stream, so the authenticated class is always "channel".
             this.emit("message", msg, noop, { historical: true, kind: "channel" });
-        return collected.length;
+            n++;
+        }
+        return n;
     }
     /**
      * Replay-gated pull of a channel's retained ambient from `sinceSeq` (exclusive) forward — the
@@ -815,55 +1152,40 @@ export class CotalEndpoint extends EventEmitter {
      *
      * Honors the **same** per-channel replay gate as join-backfill ({@link joinPolicyFresh}): a
      * `replay=off` channel returns nothing, so `focus` can't become a history bypass for a channel
-     * that denies replay to everyone else (chat is `allow_direct` with no broker-level ACL, so this
-     * app gate is the entire boundary).
+     * that denies replay to everyone else (the read ACL bounds *which* channels recall can touch; this
+     * app gate bounds *whether* a permitted channel replays).
      */
     async recallChannel(channel, sinceSeq) {
         if (!this.jsm)
-            throw new Error("endpoint not started");
+            throw new Error(this.notLiveMsg());
         if (!isConcreteChannel(channel))
             return { messages: [], dropped: false };
         const policy = await this.joinPolicyFresh(channel);
         if (!policy.replay)
             return { messages: [], dropped: false };
         const subject = chatSubject(this.space, "*", channel);
+        let raw;
+        try {
+            raw = await this.collectHistory(subject, { seq: sinceSeq + 1 });
+        }
+        catch (e) {
+            this.emit("error", e);
+            raw = [];
+        }
         const collected = [];
-        let startSeq = sinceSeq + 1;
-        pages: for (;;) {
-            let last = 0;
-            let got = 0;
+        for (const sm of raw) {
+            let msg;
             try {
-                const iter = await this.jsm.direct.getBatch(chatStream(this.space), {
-                    seq: startSeq,
-                    next_by_subj: subject,
-                    batch: 256,
-                });
-                for await (const sm of iter) {
-                    got++;
-                    last = sm.seq;
-                    let msg;
-                    try {
-                        msg = sm.json();
-                    }
-                    catch {
-                        continue; // skip undecodable
-                    }
-                    // Same authenticity guard as the tail/backfill; skip our own echoes.
-                    const parsed = parseSubject(sm.subject);
-                    if (!parsed || msg.from?.id !== parsed.sender || msg.from.id === this.card.id)
-                        continue;
-                    collected.push(msg);
-                }
+                msg = sm.json();
             }
-            catch (e) {
-                if (e.code === 404)
-                    break; // no more history (empty or last page)
-                this.emit("error", e);
-                break;
+            catch {
+                continue; // skip undecodable
             }
-            if (got === 0 || last === 0)
-                break;
-            startSeq = last + 1;
+            // Same authenticity guard as the tail/backfill; skip our own echoes.
+            const parsed = parseSubject(sm.subject);
+            if (!parsed || msg.from?.id !== parsed.sender || msg.from.id === this.card.id)
+                continue;
+            collected.push(msg);
         }
         const dropped = await this.channelDropped(subject, sinceSeq);
         return { messages: collected, dropped };
@@ -895,24 +1217,18 @@ export class CotalEndpoint extends EventEmitter {
         return oldest !== undefined && oldest > sinceSeq + 1;
     }
     /** Sequence of the earliest message still retained on a channel subject (any sender), or
-     *  undefined if nothing is retained. One 1-message Direct Get — used for the recall drop marker. */
+     *  undefined if nothing is retained. One message through the contained {@link collectHistory} —
+     *  used for the recall drop marker. */
     async channelOldestSeq(subject) {
         if (!this.jsm)
             return undefined;
         try {
-            const iter = await this.jsm.direct.getBatch(chatStream(this.space), {
-                seq: 1,
-                next_by_subj: subject,
-                batch: 1,
-            });
-            for await (const sm of iter)
-                return sm.seq;
-            return undefined;
+            const [first] = await this.collectHistory(subject, { seq: 1 }, { limit: 1 });
+            return first?.seq;
         }
         catch (e) {
-            if (e.code !== 404)
-                this.emit("error", e);
-            return undefined; // 404 = nothing retained on this subject (normal)
+            this.emit("error", e);
+            return undefined;
         }
     }
     async publishPresence() {
@@ -1088,6 +1404,19 @@ function describeStatusError(err) {
     }
     return err;
 }
+/** True when a failure is a NATS *permission denial* — the subject is forbidden to this
+ *  endpoint's creds — rather than a missing responder or a timeout. The two need opposite
+ *  fixes (grant the capability vs. start/await the service), so callers (e.g. a control
+ *  request that can't reach the manager) must tell them apart instead of defaulting to
+ *  "service down". Unwraps a wrapped `cause` and falls back to the server's error text, since
+ *  a denied publish can surface either as the typed error or inside a request rejection. */
+export function isPermissionDenied(e) {
+    if (e instanceof PermissionViolationError)
+        return true;
+    if (e?.cause instanceof PermissionViolationError)
+        return true;
+    return /permissions?\s+violation/i.test(String(e?.message ?? ""));
+}
 /** Whether a NATS server is *running* at `servers`. True on a successful connect AND on an
  *  auth rejection — an auth error means a server is there, just refusing these creds (so the
  *  caller should surface the real auth failure, not a misleading "server down", and `up`