@cotal-ai/core 0.61.0 → 0.63.0

package/dist/endpoint.js

@@ -245,6 +245,12 @@ export class CotalEndpoint extends EventEmitter {
      *  EXPECTED async permission violation that joinChannel turns into a clean throw, so watchStatus
      *  suppresses it rather than surfacing a spurious connection error. */
     confirmingChatSubs = new Set();
+    /** Delete subjects of this endpoint's own consumers whose refusal watchStatus has not consumed yet
+     *  (see {@link deleteOwnConsumer}). */
+    ownConsumerDeletes = new Set();
+    /** `$JS.API.CONSUMER.DELETE.<stream>.<prefix>_` for each of this connection's own KV watches,
+     *  running membership scans and running history reads (see {@link recordOwnWatchConsumer}). */
+    ownWatchDeletePrefixes = new Set();
     /** True until the first successful connect completes its boot backfill — distinguishes first-connect
      *  (backfill the boot channels' history) from a reconnect (reopen the core-subs, no re-backfill).
      *  Persists across reconnect (NOT connection-scoped). Replaces the legacy chat-durable consumed-cursor
@@ -296,6 +302,10 @@ export class CotalEndpoint extends EventEmitter {
     presenceRebindAt = 0;
     status = "idle";
     activity;
+    /** When {@link status} or {@link activity} last changed. Heartbeats republish it unchanged. */
+    statusSince = Date.now();
+    /** When {@link activity} last changed. A status change leaves it alone. */
+    activitySince;
     /** Last harness-reported work progress. Carried by the next heartbeat, never published per event. */
     activeAt;
     condition;
@@ -1207,6 +1217,8 @@ export class CotalEndpoint extends EventEmitter {
         this.chatSubs.clear();
         this.chatSubDenied.clear();
         this.confirmingChatSubs.clear();
+        this.ownConsumerDeletes.clear();
+        this.ownWatchDeletePrefixes.clear();
         this.roster.clear();
         // #1356: the presence-refusal record is connection-scoped like everything else torn down here.
         // It says "the broker on THIS connection refuses writes to this bucket", so it cannot outlive the
@@ -2246,10 +2258,14 @@ export class CotalEndpoint extends EventEmitter {
         }
     }
     async setActivity(activity) {
+        if (activity !== this.activity)
+            this.statusSince = this.activitySince = Date.now();
         this.activity = activity;
         await this.publishPresence();
     }
     async setStatus(status) {
+        if (status !== this.status)
+            this.statusSince = Date.now();
         this.status = status;
         await this.publishPresence();
     }
@@ -2525,7 +2541,17 @@ export class CotalEndpoint extends EventEmitter {
         // measured at 30-34s for 89 entries against a mesh at 534ms RTT. `liveKvEntries` is ~3 round
         // trips regardless of N, and (unlike the old loop) refuses to return a truncated view rather
         // than reporting a partial roster as the whole one.
-        for (const e of await liveKvEntries(kv)) {
+        let scanPrefix;
+        const entries = await liveKvEntries(kv, {
+            onConsumer: (info) => { scanPrefix = this.recordOwnWatchConsumer(info); },
+            deleteOwnConsumer: (stream, name, del) => this.deleteOwnConsumer(stream, name, del),
+        }).finally(() => {
+            // The scan's own last delete goes out after nats.js closed its consumer, so every predecessor
+            // delete the library sent is answered ahead of it on this connection.
+            if (scanPrefix)
+                this.ownWatchDeletePrefixes.delete(scanPrefix);
+        });
+        for (const e of entries) {
             if (e.key === MEMBERSHIP_FEED_KEY) {
                 try {
                     asOf = e.json().observedAt;
@@ -2594,6 +2620,7 @@ export class CotalEndpoint extends EventEmitter {
         const cc = kv._buildCC(">", KvWatchInclude.LastValue, { headers_only: false });
         const consumer = await kv.js.consumers.getPushConsumer(kv.stream, cc);
         const info = await consumer.info(true);
+        this.recordOwnWatchConsumer(info);
         // The broker resource exists now. Record its identity BEFORE consume() so a concurrent stop or
         // connection close always leaves enough state for strict cleanup or fresh-epoch retry.
         watch.consumer = consumer;
@@ -2649,19 +2676,72 @@ export class CotalEndpoint extends EventEmitter {
         watch.resolveStop = undefined;
         watch.rejectStop = undefined;
     }
-    /** Delete one membership-watch consumer, swallowing ONLY already-gone. */
+    /** Run one delete of a consumer this endpoint created. A profile without the delete row is refused
+     *  by design (#691), and nats.js reports a refused request twice: as the request's rejection, which
+     *  the caller handles, and on the connection status, which watchStatus would emit as an `error`.
+     *  A refused subject stays recorded until watchStatus drops that echo, since the two settle in
+     *  either order. */
+    async deleteOwnConsumer(stream, name, del) {
+        const subject = `$JS.API.CONSUMER.DELETE.${stream}.${name}`;
+        this.ownConsumerDeletes.add(subject);
+        let refused = false;
+        try {
+            return await del();
+        }
+        catch (err) {
+            refused = isPublishPermissionDenied(err);
+            throw err;
+        }
+        finally {
+            if (!refused)
+                this.ownConsumerDeletes.delete(subject);
+        }
+    }
+    /** Record the consumer behind one of this connection's own KV watches, membership scans or history
+     *  reads, and return the delete-subject prefix it recorded. nats.js names it `<prefix>_<serial>` and,
+     *  rebuilding it after a stall or a sequence gap, deletes the predecessor itself with no hook before
+     *  the send, so a profile without the delete row (#691) has that delete refused. A watch's prefix
+     *  stays recorded for the connection because a retired watch's last rebuild can be refused after
+     *  the watch has stopped; {@link readMembership} and {@link drainWindow} retire theirs after their own
+     *  last delete, which goes out once nats.js has stopped the consumer and so is answered after every
+     *  predecessor delete. */
+    recordOwnWatchConsumer({ stream_name, name }) {
+        const prefix = `$JS.API.CONSUMER.DELETE.${stream_name}.${name.slice(0, name.lastIndexOf("_") + 1)}`;
+        this.ownWatchDeletePrefixes.add(prefix);
+        return prefix;
+    }
+    /** Whether `subject` deletes `<prefix>_<serial>` of a consumer {@link recordOwnWatchConsumer} recorded. */
+    isOwnWatchDelete(subject) {
+        const cut = subject.lastIndexOf("_") + 1;
+        return /^\d+$/.test(subject.slice(cut)) && this.ownWatchDeletePrefixes.has(subject.slice(0, cut));
+    }
+    /** Delete a history reader's own ephemeral consumer. A refused delete is the designed outcome for
+     *  the elevated profile, which holds no stream-wide CONSUMER.DELETE (#691): the broker reaps the
+     *  consumer at its inactive threshold. */
+    async deleteReaderConsumer(consumer) {
+        const { stream_name, name } = await consumer.info(true);
+        try {
+            await this.deleteOwnConsumer(stream_name, name, () => consumer.delete());
+        }
+        catch (e) {
+            if (!isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound) && !isPublishPermissionDenied(e))
+                throw e;
+        }
+    }
+    /** Delete one membership-watch consumer, swallowing ONLY already-gone and a refused delete. */
     async deleteMembershipConsumer(jsm, stream, name) {
         try {
-            return await jsm.consumers.delete(stream, name);
+            return await this.deleteOwnConsumer(stream, name, () => jsm.consumers.delete(stream, name));
         }
         catch (err) {
-            if (err.code === 404 || /(consumer|stream) not found/i.test(err.message))
+            if (membershipConsumerReleased(err))
                 return true;
             throw err;
         }
     }
-    /** Stop the local iterator AND delete its ordered consumer. The admin/observer grant already holds
-     *  the bucket-scoped consumer-delete row, so a reconnect leaves no five-minute predecessor. */
+    /** Stop the local iterator AND delete its ordered consumer. The admin/observer profile holds no
+     *  consumer delete on this bucket (#691), so its refused delete leaves the consumer to the broker's
+     *  five-minute inactive threshold. */
     async disarmMembershipWatch(watch) {
         const iter = watch.iter;
         const consumer = watch.consumer;
@@ -2673,14 +2753,15 @@ export class CotalEndpoint extends EventEmitter {
         catch { /* already closed */ }
         if (consumer) {
             try {
-                const deleted = await consumer.delete();
+                const { stream_name, name } = await consumer.info(true);
+                const deleted = await this.deleteOwnConsumer(stream_name, name, () => consumer.delete());
                 if (deleted) {
                     watch.consumerStream = undefined;
                     watch.consumerName = undefined;
                 }
             }
             catch (err) {
-                if (err.code === 404 || /(consumer|stream) not found/i.test(err.message)) {
+                if (membershipConsumerReleased(err)) {
                     watch.consumerStream = undefined;
                     watch.consumerName = undefined;
                 }
@@ -3049,13 +3130,7 @@ export class CotalEndpoint extends EventEmitter {
             throw new Error(`history: the broker reported messages on ${subjectLabel(subjects)} but delivered none - the read was cut short, not empty`);
         }
         finally {
-            try {
-                await consumer.delete();
-            }
-            catch (e) {
-                if (!isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
-                    throw e;
-            }
+            await this.deleteReaderConsumer(consumer);
         }
     }
     /** The newest stream sequence matching `subject`, or 0 when the subject has no messages.
@@ -3095,13 +3170,7 @@ export class CotalEndpoint extends EventEmitter {
             throw new Error(`history: the broker reported messages on ${subjectLabel(subjects)} but delivered none - the read was cut short, not empty`);
         }
         finally {
-            try {
-                await consumer.delete();
-            }
-            catch (e) {
-                if (!isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
-                    throw e;
-            }
+            await this.deleteReaderConsumer(consumer);
         }
     }
     /** Drain every message matching `subject` with sequence in `[start, ceiling]`, oldest-first.
@@ -3112,6 +3181,8 @@ export class CotalEndpoint extends EventEmitter {
         signal?.throwIfAborted();
         const out = [];
         const consumer = await js.consumers.get(stream, { filter_subjects: subjects, opt_start_seq: start });
+        // A delivery-sequence gap makes nats.js rebuild this consumer and delete the predecessor itself.
+        const readerPrefix = this.recordOwnWatchConsumer(await consumer.info(true));
         try {
             // A freshly created consumer already carries its ConsumerInfo, so read the CACHED copy: the
             // explicit uncached `info()` this used to call was a round trip for data we already had.
@@ -3174,13 +3245,9 @@ export class CotalEndpoint extends EventEmitter {
             // DMs, and the single-channel routes still make one each.
             // Left alone that accumulates consumers on the broker until the thresholds expire, and the
             // resulting resource exhaustion would land in streamHistory's catch and read as empty history.
-            try {
-                await consumer.delete();
-            }
-            catch (e) {
-                if (!isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
-                    throw e;
-            }
+            // The observer and admin profiles may not delete (#691), so the dashboard's readers do live out
+            // that threshold; only a profile holding the delete row reclaims them here.
+            await this.deleteReaderConsumer(consumer).finally(() => { this.ownWatchDeletePrefixes.delete(readerPrefix); });
         }
     }
     // ---- internals -----------------------------------------------------------
@@ -3224,6 +3291,12 @@ export class CotalEndpoint extends EventEmitter {
                 // and turns into a clean throw — it is not a connection error to surface.
                 if (s.error instanceof PermissionViolationError && this.confirmingChatSubs.has(s.error.subject))
                     continue;
+                // The echo of a refused delete of this endpoint's own consumer: one its caller already handled,
+                // or the predecessor nats.js deleted while rebuilding one of this connection's watches, scans or
+                // history reads.
+                if (s.error instanceof PermissionViolationError && s.error.operation === "publish"
+                    && (this.ownConsumerDeletes.delete(s.error.subject) || this.isOwnWatchDelete(s.error.subject)))
+                    continue;
                 this.emit("error", describeStatusError(s.error));
             }
         })().catch((e) => {
@@ -5838,14 +5911,17 @@ export class CotalEndpoint extends EventEmitter {
             condition: this.condition,
             environment: this.environment,
             activity: this.activity,
+            statusSince: this.statusSince,
+            activitySince: this.activitySince,
             activeAt: this.activeAt,
             attention: this.attentionMode,
             channelModes: this.channelModes,
             ts: Date.now(),
         };
-        // Wire contract (SPEC §6): an OFFLINE record must not carry the advisory attention fields. Scrub at
-        // the publisher — this covers stop(), setStatus("offline"), and any future offline publish site, so
-        // the raw KV record is compliant, not only the observer-side roster materialization.
+        // Wire contract (SPEC §6): an OFFLINE record must not carry the advisory attention fields or
+        // `statusSince`. Scrub at the publisher — this covers stop(), setStatus("offline"), and any future
+        // offline publish site, so the raw KV record is compliant, not only the observer-side roster
+        // materialization.
         const record = this.status === "offline" ? this.toOffline(p) : p;
         // #1356: this put can still be in flight when a teardown runs, because a rebind reaches
         // publishPresence through onPresenceBucketEmpty and no teardown awaits that flight. Take the same
@@ -5968,6 +6044,7 @@ export class CotalEndpoint extends EventEmitter {
         let hydrated;
         this.presenceSnapshot = new Promise((resolve) => { hydrated = resolve; });
         const iter = await this.kv.watch();
+        this.recordOwnWatchConsumer(await kvWatchConsumer(iter).info(true));
         if (epoch !== this.presenceEpoch) {
             try {
                 iter.stop();
@@ -6065,6 +6142,7 @@ export class CotalEndpoint extends EventEmitter {
         if (!this.channelKv)
             return;
         const iter = await this.channelKv.watch();
+        this.recordOwnWatchConsumer(await kvWatchConsumer(iter).info(true));
         this.channelWatchIter = iter;
         void (async () => {
             for await (const e of iter)
@@ -6236,9 +6314,11 @@ export class CotalEndpoint extends EventEmitter {
     }
     /** Materialize an OFFLINE presence record: drop the advisory attention fields. An offline peer must
      *  not show a stale `[focus]` or "locally muted #x" hint — SPEC: attention removed on offline sweep,
-     *  channel modes reset on restart. card/activity/ts are kept. */
+     *  channel modes reset on restart. `statusSince` goes too: it dated the live status, and an observer
+     *  that derives offline from a stale heartbeat does not know when the peer left. card/activity/ts
+     *  are kept. */
     toOffline(p) {
-        return { ...p, status: "offline", attention: undefined, channelModes: undefined };
+        return { ...p, status: "offline", statusSince: undefined, attention: undefined, channelModes: undefined };
     }
     /** Mark a known peer offline (on KV delete/purge), keeping it in the roster. */
     markOffline(id) {
@@ -6676,6 +6756,19 @@ export function isPermissionDenied(e) {
 function isJetStreamMissing(e, ...codes) {
     return e instanceof JetStreamApiError && codes.includes(e.code);
 }
+/** The ordered consumer behind a KV watch, which nats.js keeps on the iterator and the KV types hide. */
+function kvWatchConsumer(iter) {
+    const consumer = iter._data;
+    if (!consumer)
+        throw new Error("KV watch carries no consumer - the pinned nats.js changed shape");
+    return consumer;
+}
+/** A membership-watch delete that leaves nothing to retry: the consumer is already gone, or the broker
+ *  refused the delete because the reading profile holds no stream-wide CONSUMER.DELETE (#691) and
+ *  reaps the consumer at its inactive threshold. */
+function membershipConsumerReleased(err) {
+    return err.code === 404 || /(consumer|stream) not found/i.test(err.message) || isPublishPermissionDenied(err);
+}
 /** The signal's exact abort reason, or the platform-standard AbortError when none was supplied. */
 function abortReason(signal) {
     return signal?.reason instanceof Error