@cotal-ai/core 0.115.4 → 0.116.1

package/dist/endpoint.js

@@ -162,6 +162,31 @@ export function fitHistoryPage(items, budget) {
     }
     return first === items.length ? [] : items.slice(first);
 }
+/** One feed entry, decoded the way {@link CotalEndpoint.readMembership} decodes it: the reserved
+ *  heartbeat key carries `asOf`, every other key is one agent's record, and an undecodable record
+ *  is a member change with no record (never a crash in the consumer callback). */
+function membershipChange(e, update) {
+    const operation = e.operation;
+    if (e.key === MEMBERSHIP_FEED_KEY) {
+        let asOf;
+        if (operation === "PUT") {
+            try {
+                asOf = e.json().observedAt;
+            }
+            catch { /* heartbeat garbled; leave undefined */ }
+        }
+        return { type: "heartbeat", revision: e.revision, update, asOf };
+    }
+    let member;
+    if (operation === "PUT") {
+        try {
+            const rec = e.json();
+            member = { id: e.key, live: rec.live ?? [], durable: rec.durable ?? [], observedAt: rec.observedAt };
+        }
+        catch { /* undecodable: delivered as a change with no record */ }
+    }
+    return { type: "member", id: e.key, operation, revision: e.revision, update, member };
+}
 export class CotalEndpoint extends EventEmitter {
     card;
     space;
@@ -226,6 +251,15 @@ export class CotalEndpoint extends EventEmitter {
      *  `DrainingConnectionError` out of `reset()` on a timer nothing awaits. */
     presenceWatchIter;
     channelWatchIter;
+    /** Consumer deletes of retired presence and channel-registry watches still in flight. A stop
+     *  alone leaves each watch's ordered consumer on the broker for its five-minute inactive
+     *  threshold (#3914), so {@link stop} and a live-epoch rebuild await these before they close the
+     *  connection that carries them, as they do for the lease watch. */
+    kvWatchDeletes = new Set();
+    /** Presence and channel-registry watch binds still awaiting the broker. One that lands after
+     *  {@link stop} or a rebuild moved the epoch retires its watch and starts that delete itself, so
+     *  the same two paths wait for these first. */
+    kvWatchBinds = new Set();
     /** The daemon's lease-loss watch (#1596): the INTENT that survives a connection rebuild, plus the
      *  live iterator. See {@link watchDeliveryLease}. */
     leaseWatchIntent;
@@ -1413,19 +1447,11 @@ export class CotalEndpoint extends EventEmitter {
         this.streamMsgs.length = 0;
         this.presenceEpoch++;
         this.presenceRebind = undefined;
-        try {
-            this.presenceWatchIter?.stop();
-        }
-        catch {
-            /* already closed with the connection */
-        }
+        if (this.presenceWatchIter)
+            this.retireKvWatch(this.presenceWatchIter);
         this.presenceWatchIter = undefined;
-        try {
-            this.channelWatchIter?.stop();
-        }
-        catch {
-            /* already closed with the connection */
-        }
+        if (this.channelWatchIter)
+            this.retireKvWatch(this.channelWatchIter);
         this.channelWatchIter = undefined;
         this.stopDeliveryLeaseWatch();
         for (const sub of this.chatSubs.values()) {
@@ -1635,9 +1661,10 @@ export class CotalEndpoint extends EventEmitter {
         try {
             this.clearConnectionScoped();
             // Manual reconnect still has a live old epoch: complete broker-consumer cleanup before close.
-            // Terminal self-heal has an already-closed epoch: disarm retains stream/name for fresh cleanup.
+            // Terminal self-heal has an already-closed epoch: disarm retains stream/name for fresh cleanup,
+            // and the KV watch consumers that closed with it are left to the broker's inactive threshold.
             if (oldNc && !oldNc.isClosed())
-                await Promise.all([...[...this.membershipFeedWatches].map((watch) => watch.arm), this.settleDeliveryLeaseWatches()]);
+                await Promise.all([...[...this.membershipFeedWatches].map((watch) => watch.arm), this.settleDeliveryLeaseWatches(), this.settleKvWatches()]);
             this.nc = undefined;
             this.js = undefined;
             this.jsm = undefined;
@@ -1818,19 +1845,11 @@ export class CotalEndpoint extends EventEmitter {
                 /* already closed */
             }
         }
-        try {
-            this.presenceWatchIter?.stop();
-        }
-        catch {
-            /* already closed */
-        }
+        if (this.presenceWatchIter)
+            this.retireKvWatch(this.presenceWatchIter);
         this.presenceWatchIter = undefined;
-        try {
-            this.channelWatchIter?.stop();
-        }
-        catch {
-            /* already closed */
-        }
+        if (this.channelWatchIter)
+            this.retireKvWatch(this.channelWatchIter);
         this.channelWatchIter = undefined;
         this.stopDeliveryLeaseWatch();
         try {
@@ -1842,7 +1861,7 @@ export class CotalEndpoint extends EventEmitter {
         catch {
             /* best-effort graceful leave */
         }
-        await this.settleDeliveryLeaseWatches();
+        await Promise.all([this.settleDeliveryLeaseWatches(), this.settleKvWatches()]);
         // DM-only: the pull above is stopped, so no new copy reaches the handler. Let a running handler
         // settle while the connection is still open, so its ack can reach the broker; close either way.
         let dmOnlyUnsettled = false;
@@ -2320,18 +2339,20 @@ export class CotalEndpoint extends EventEmitter {
         }
     }
     /** The resolved, digest-verified command surface of `endpoint` as this connection's caller sees
-     *  it: the same resolve and the same per-name cache {@link invokeService} uses, so a describe
-     *  followed by an invoke binds both to one incarnation. `refresh` drops the cached entry first. */
+     *  it. Unpinned calls share {@link invokeService}'s per-name cache, and `refresh` drops that entry.
+     *  `instanceId` selects one instance without reading or populating the class cache, as a pinned
+     *  invoke does. The caller's existing credential must permit that route. */
     async describeService(endpoint, opts = {}) {
         if (!this.nc)
             throw new Error(this.notLiveMsg());
         if (opts.refresh)
             this.resolvedServices.delete(endpoint);
-        const cached = this.resolvedServices.get(endpoint);
+        const cached = opts.instanceId === undefined ? this.resolvedServices.get(endpoint) : undefined;
         if (cached)
             return cached;
-        const svc = await resolveService(this.nc, this.space, endpoint, this.serviceCaller(), { deadlineMs: opts.deadlineMs ?? 10_000, signal: opts.signal });
-        this.resolvedServices.set(endpoint, svc);
+        const svc = await resolveService(this.nc, this.space, endpoint, this.serviceCaller(), { deadlineMs: opts.deadlineMs ?? 10_000, signal: opts.signal, ...(opts.instanceId !== undefined ? { instanceId: opts.instanceId } : {}) });
+        if (opts.instanceId === undefined)
+            this.resolvedServices.set(endpoint, svc);
         return svc;
     }
     /** GENERIC v0.4 service invoke over this endpoint's own connection: resolve
@@ -3016,19 +3037,20 @@ export class CotalEndpoint extends EventEmitter {
         const kv = await this.membershipFeedRegistry();
         const members = [];
         let asOf;
-        // ONE pass. This was `kv.keys()` followed by a sequential `kv.get()` per key — O(N) round trips,
-        // measured at 30-34s for 89 entries against a mesh at 534ms RTT. `liveKvEntries` is ~3 round
-        // trips regardless of N, and (unlike the old loop) refuses to return a truncated view rather
-        // than reporting a partial roster as the whole one.
-        let scanPrefix;
-        const entries = await liveKvEntries(kv, {
-            onConsumer: (info) => { scanPrefix = this.recordOwnWatchConsumer(info); },
-        }).finally(() => {
-            // The scan's own last delete goes out after nats.js closed its consumer, so every predecessor
-            // delete the library sent is answered ahead of it on this connection.
-            if (scanPrefix)
-                this.ownWatchDeletePrefixes.delete(scanPrefix);
-        });
+        // Paged batched Direct Gets, as the on-demand presence read: a few round trips regardless of N,
+        // a short page refused rather than reported as the whole feed, and no consumer bound. The scan
+        // this replaced bound an ordered consumer per read whose delete the observer and admin profiles
+        // may not send (#691), so every dashboard poll left a consumer to the broker's inactive threshold
+        // and a refused `CONSUMER.DELETE` line in its log (#4123).
+        let entries;
+        try {
+            entries = await lastKvEntries(kv);
+        }
+        catch (e) {
+            if (!isPublishPermissionDenied(e))
+                throw e;
+            throw new Error(`membership read refused for ${this.card.id}: this credential may not publish $JS.API.DIRECT.GET.KV_${membershipBucket(this.space)}, the membership feed read (SPEC Appendix B). A credential minted by an older issuer can lack this grant even when newly minted; upgrade the manager or auth service that issues credentials, then mint a new credential. Reusing credentials or reconnecting does not add the grant.`, { cause: e });
+        }
         for (const e of entries) {
             if (e.key === MEMBERSHIP_FEED_KEY) {
                 try {
@@ -3045,10 +3067,12 @@ export class CotalEndpoint extends EventEmitter {
         }
         return { asOf, members };
     }
-    /** Watch the membership feed for changes (admin/observer): `onChange` fires on every KV entry,
-     *  including the initial replay — the caller debounces + re-reads {@link readMembership}. The async
-     *  stop handle resolves only after its ordered broker consumer is deleted. Best-effort: a feed the
-     *  cred can't read (or absent) surfaces as an `error` event and the dashboard keeps its last snapshot. */
+    /** Watch the membership feed for changes (admin/observer): `onChange` receives every KV entry as a
+     *  {@link MembershipChange} — the decoded member record (or its deletion), or the feed heartbeat's
+     *  `asOf` — so a caller keeps its snapshot current without re-reading the bucket; the entries replayed
+     *  when the watch (re)arms carry `update: false` and end with a `replayed` change. The async stop
+     *  handle resolves only after its ordered broker consumer is deleted. Best-effort: a feed the cred
+     *  can't read (or absent) surfaces as an `error` event and the dashboard keeps its last snapshot. */
     async watchMembership(onChange, onClosed) {
         if (this.stopped)
             throw new Error("endpoint stopped - cannot watch membership");
@@ -3102,11 +3126,20 @@ export class CotalEndpoint extends EventEmitter {
         watch.consumer = consumer;
         watch.consumerStream = info.stream_name;
         watch.consumerName = info.name;
+        // The replay is the bucket's last value per key, bound at `info`; it ends with the entry that
+        // leaves nothing pending (the same rule as nats.js `watch()`'s `initializedFn`).
+        let replaying = info.num_pending > 0;
         let iter;
         try {
-            iter = await consumer.consume({ callback: () => {
-                    if (!watch.stopped && watch.consumer === consumer)
-                        watch.onChange();
+            iter = await consumer.consume({ callback: (jm) => {
+                    if (watch.stopped || watch.consumer !== consumer)
+                        return;
+                    const entry = kv.jmToWatchEntry(jm, !replaying);
+                    watch.onChange(membershipChange(entry, !replaying));
+                    if (replaying && jm.info.pending === 0) {
+                        replaying = false;
+                        watch.onChange({ type: "replayed", revision: entry.revision });
+                    }
                 } });
         }
         catch (err) {
@@ -3118,6 +3151,8 @@ export class CotalEndpoint extends EventEmitter {
             await this.disarmMembershipWatch(watch);
             return;
         }
+        if (!replaying)
+            watch.onChange({ type: "replayed", revision: 0 }); // an empty feed replays nothing
         iter.closed().then(() => {
             if (watch.stopped || watch.consumer !== consumer)
                 return;
@@ -4363,6 +4398,52 @@ export class CotalEndpoint extends EventEmitter {
         await Promise.allSettled(this.leaseWatchBinds);
         await Promise.all(this.leaseWatchDeletes);
     }
+    /** Stop a presence or channel-registry watch and delete its ordered consumer, which the stop
+     *  alone leaves on the broker until its five-minute inactive threshold (#3914). A connection
+     *  that closed on its own cannot carry the delete, so that consumer is left to the threshold. A
+     *  failed delete is a warning: the broker still reaps the consumer, and a rejected {@link stop}
+     *  would read as unproved consumer custody. */
+    retireKvWatch(iter) {
+        try {
+            iter.stop();
+        }
+        catch { /* already closed with its connection */ }
+        if (!this.nc || this.nc.isClosed())
+            return;
+        const deleting = this.deleteOwnConsumer(kvWatchConsumer(iter)).catch((e) => this.emitRecoverable(new Error(`a retired KV watch consumer was not deleted (${e.message}); the broker reaps it at its inactive threshold`)));
+        this.kvWatchDeletes.add(deleting);
+        void deleting.then(() => this.kvWatchDeletes.delete(deleting));
+    }
+    /** Wait for the presence and channel-registry watch binds still in flight, then for every retired
+     *  watch's consumer delete. {@link stop} and a live-epoch rebuild run this before they close the
+     *  connection those deletes need (#3914). */
+    async settleKvWatches() {
+        await Promise.allSettled(this.kvWatchBinds);
+        await Promise.all(this.kvWatchDeletes);
+    }
+    /** Bind a watch on `kv` for the current epoch and record its consumer as this connection's own.
+     *  A bind that {@link stop} or a rebuild overtook retires its late watch, consumer delete
+     *  included, and resolves undefined: the epoch that asked for it is gone, and the one that
+     *  replaced it binds its own through {@link connectAndBind}. A stopped endpoint binds nothing,
+     *  because its teardown may already have settled the flights it waits for. */
+    bindKvWatch(kv) {
+        if (this.stopped)
+            return Promise.resolve(undefined);
+        const epoch = this.presenceEpoch;
+        const binding = (async () => {
+            const iter = await kv.watch();
+            const info = await kvWatchConsumer(iter).info(true);
+            this.recordOwnWatchConsumer(info);
+            if (epoch === this.presenceEpoch && !this.stopped)
+                return { iter, info };
+            this.retireKvWatch(iter);
+            return undefined;
+        })();
+        const settled = () => { this.kvWatchBinds.delete(binding); };
+        this.kvWatchBinds.add(binding);
+        void binding.then(settled, settled);
+        return binding;
+    }
     /** Bind the lease watch on the current connection. A bind that lands after a newer watch replaced
      *  its intent, or after a stop or connection teardown moved the epoch, retires its watch instead of
      *  installing it: the epoch it was bound for is gone, and a rebuilt one binds its own. */
@@ -5021,6 +5102,18 @@ export class CotalEndpoint extends EventEmitter {
             return { ok: false, error: e.message };
         }
     }
+    /** Whether `lifecycleUid`'s exact ACL row still allows reads, so an inventory of it came from a
+     *  caller that did not fence. A row narrowed to no reads, or retired or absent, is fenced. A read
+     *  that fails answers true: the caller keeps its in-memory mark, which only ever refuses. */
+    async lifecycleAclUnfenced(principal, lifecycleUid) {
+        try {
+            const row = await readAcl(await this.aclRegistry(), principal, lifecycleUid);
+            return row !== undefined && row.record.allowSubscribe.length > 0;
+        }
+        catch {
+            return true;
+        }
+    }
     /** Run one durable join or leave that writes `principal`'s rows of `lifecycleUid`, refused once that
      *  lifecycle's inventory has been asked for ({@link retiringLifecycles}). The check and the
      *  registration have no await between them, so an inventory either finds this write running and
@@ -5643,7 +5736,13 @@ export class CotalEndpoint extends EventEmitter {
             for (const earlier of [...marked])
                 if (earlier !== uid && await aclRetired(await this.aclRegistry(), principal, earlier).catch(() => false))
                     marked.delete(earlier);
-            marked.add(uid);
+            // A mark is kept only for an inventory whose caller did not fence (#4996): a lifecycle whose ACL
+            // row is narrowed to no reads, or already retired, needs none, since every join re-reads that
+            // row around its write. A read that fails proves nothing, so the mark is kept, which only refuses.
+            if (await this.lifecycleAclUnfenced(principal, uid))
+                marked.add(uid);
+            else if (marked.size === 0)
+                this.retiringLifecycles.delete(principal);
             await Promise.allSettled(this.membershipWrites.get(`${principal}/${uid}`) ?? []);
             try {
                 const channels = await listLifecycleMemberChannels(await this.membersRegistry(), principal, uid);
@@ -5859,6 +5958,20 @@ export class CotalEndpoint extends EventEmitter {
         })().catch((e) => { if (!this.stopped)
             this.emit("error", e); });
     }
+    /** Remove a retired lifecycle's member row for `channel`, when one exists. Best effort: the row
+     *  authorizes nothing (its ACL row is retired), and the next entry fan-out writes against it
+     *  brings the reader back here. */
+    async removeRetiredMemberRow(channel, owner, lifecycleUid) {
+        try {
+            const kv = await this.membersRegistry();
+            if (await readMember(kv, channel, owner, lifecycleUid) === undefined)
+                return;
+            await deleteMember(kv, channel, owner, lifecycleUid);
+        }
+        catch (e) {
+            this.emit("warning", new Error(`plane-3 reader: could not remove retired lifecycle ${owner}.${lifecycleUid}'s "${channel}" member row: ${e.message}`));
+        }
+    }
     /** Re-authorize ONE mixed-inbox entry and transfer it to the owner's DELIVER store. Deny (drop) on a
      *  revoked/narrowed ACL or out-of-interval seq; on transfer success, ack the mixed entry (durability
      *  has moved to DLV — an §8 equivalent per-member at-least-once mechanism). The agent acks DLV. */
@@ -5924,6 +6037,15 @@ export class CotalEndpoint extends EventEmitter {
                     m.nak(2000);
                     return;
                 }
+                // The member row that addressed this entry is one a daemon left behind when it died between
+                // its create and its post-write ACL read (#4994). It authorizes nothing, but fan-out copies
+                // every later message on the channel into this inbox until it goes, and this reader then
+                // removes each copy. Remove the row where that cost is paid, by its exact key, only when it
+                // exists: a purge of an absent key would write a marker per dropped entry.
+                if (entry.reason === "durable-channel")
+                    await this.removeRetiredMemberRow(entry.channel, owner, pr.lifecycleUid);
+                if (!this.plane3MayAct())
+                    return;
                 m.ack();
                 return;
             }
@@ -6991,28 +7113,22 @@ export class CotalEndpoint extends EventEmitter {
     }
     /** Bind a presence watch on the current connection. Resolves the bound consumer's info when
      *  the watch was installed, undefined when the endpoint stopped or rebuilt while the bind was
-     *  in flight: that bind's iterator is released here and nothing is installed, because the
-     *  epoch that asked for it is gone and the epoch that replaced it binds its own watch through
-     *  {@link connectAndBind}. Without this fence a bind that completes after {@link stop} would
-     *  resurrect a watch on a stopped endpoint, and one that completes after a rebuild would
-     *  overwrite the fresh epoch's watch with a dead-connection iterator. */
+     *  in flight: that bind's watch is retired by {@link bindKvWatch} and nothing is installed,
+     *  because the epoch that asked for it is gone and the epoch that replaced it binds its own
+     *  watch through {@link connectAndBind}. Without this fence a bind that completes after
+     *  {@link stop} would resurrect a watch on a stopped endpoint, and one that completes after a
+     *  rebuild would overwrite the fresh epoch's watch with a dead-connection iterator. */
     async startPresenceWatch() {
         if (!this.kv)
             return undefined;
-        const epoch = this.presenceEpoch;
         let hydrated;
         this.presenceSnapshot = new Promise((resolve) => { hydrated = resolve; });
-        const iter = await this.kv.watch();
-        const info = await kvWatchConsumer(iter).info(true);
-        this.recordOwnWatchConsumer(info);
-        if (epoch !== this.presenceEpoch) {
-            try {
-                iter.stop();
-            }
-            catch { /* its connection may already be gone */ }
+        const bound = await this.bindKvWatch(this.kv);
+        if (!bound) {
             hydrated();
             return undefined;
         }
+        const { iter, info } = bound;
         this.presenceWatchIter = iter;
         // A bucket with no keys replays nothing, so no entry carries the end-of-replay marker. For an
         // observer that does not register, that empty replay is the whole snapshot: nobody is present.
@@ -7064,7 +7180,7 @@ export class CotalEndpoint extends EventEmitter {
      * next sweep exactly as if the watch had never stalled. Rate-limited to one attempt per TTL
      * per observer, never overlapping, never on a stopped or rebuilding endpoint (those own their
      * watch through {@link connectAndBind}). A stop or rebuild that lands while the bind is in
-     * flight retires it: {@link startPresenceWatch} releases the late iterator and reports
+     * flight retires it: {@link startPresenceWatch} retires the late watch and reports
      * nothing, since the epoch that was silent no longer exists. A failed bind is reported and
      * the view stays stale.
      */
@@ -7084,17 +7200,15 @@ export class CotalEndpoint extends EventEmitter {
                 // watch is the one that recovers by itself, and its replay is still guarded against
                 // expired PUTs. Only a successfully bound watch retires its predecessor.
                 const info = await this.startPresenceWatch();
-                // Retired mid-bind (stop or rebuild moved the epoch): the late iterator is already
-                // released and the old watch was torn down by whoever moved the epoch. Nothing to
+                // Retired mid-bind (stop or rebuild moved the epoch): the late watch is already
+                // retired and the old watch was torn down by whoever moved the epoch. Nothing to
                 // retire, nothing to report.
                 if (!info)
                     return;
-                if (old && old !== this.presenceWatchIter) {
-                    try {
-                        old.stop();
-                    }
-                    catch { /* already closed with its consumer */ }
-                }
+                // The predecessor's consumer goes with it: left alone, every rebind of a silent bucket
+                // would add one idle consumer to the presence stream per TTL (#3914).
+                if (old && old !== this.presenceWatchIter)
+                    this.retireKvWatch(old);
                 // A bucket with no keys replays nothing, so the new watch cannot refresh
                 // `lastPresenceWatchAt` by delivering. It IS current knowledge: nobody is present. The
                 // bind's consumer info carries the initial pending count for that one fact; nats.js's KV
@@ -7123,8 +7237,10 @@ export class CotalEndpoint extends EventEmitter {
     async startChannelWatch() {
         if (!this.channelKv)
             return;
-        const iter = await this.channelKv.watch();
-        this.recordOwnWatchConsumer(await kvWatchConsumer(iter).info(true));
+        const bound = await this.bindKvWatch(this.channelKv);
+        if (!bound)
+            return;
+        const { iter } = bound;
         this.channelWatchIter = iter;
         void (async () => {
             for await (const e of iter)
@@ -7408,6 +7524,8 @@ function authenticatedMessage(msg, parsed) {
         return authenticatedChannelMessage(msg, parsed.rest);
     if (parsed.kind === "inst")
         return authenticatedDmMessage(msg, parsed.rest);
+    if (parsed.kind === "svc")
+        return authenticatedAnycastMessage(msg, parsed.rest);
     return msg;
 }
 function authenticatedChannelMessage(msg, channel) {
@@ -7422,6 +7540,12 @@ function authenticatedDmMessage(msg, to) {
     const { channel: _channel, toService: _toService, ...base } = msg;
     return { ...base, to };
 }
+function authenticatedAnycastMessage(msg, toService) {
+    if (msg.toService === toService && msg.to === undefined && msg.channel === undefined)
+        return msg;
+    const { to: _to, channel: _channel, ...base } = msg;
+    return { ...base, toService };
+}
 /** The ack wait the broker reports for a bound consumer: how long a copy it delivers stays with
  *  its reader without an ack. */
 async function ackWaitOf(consumer) {