@cotal-ai/core 0.100.2 → 0.101.0

package/dist/endpoint.js

@@ -1,7 +1,7 @@
 import { EventEmitter } from "node:events";
 import { randomUUID } from "node:crypto";
 import { setTimeout as delay } from "node:timers/promises";
-import { createConnection } from "node:net";
+import { createConnection, isIP } from "node:net";
 import { connect, credsAuthenticator, headers, tokenAuthenticator, nanos, millis, AuthorizationError, PermissionViolationError, UserAuthenticationExpiredError, NoRespondersError, RequestError, TimeoutError, ClosedConnectionError, } from "@nats-io/transport-node";
 import { wsconnect } from "@nats-io/nats-core";
 import { parseLivenessAnswer, responderFromLease, responderFromProbe, isLivenessPlane, LIVENESS_PLANES, } from "./liveness.js";
@@ -63,6 +63,8 @@ export const MULTI_FILTER_READ_CONCURRENCY = 4;
  *  minutes is what the nats.js ordered consumers it replaces carried, and it keeps a reader alive
  *  across the gap between pulls on a slow link. */
 const HISTORY_READER_INACTIVE_MS = 5 * 60_000;
+/** The longest delay `setTimeout` honors, 2^31-1 ms. */
+const MAX_TIMER_MS = 2_147_483_647;
 /** A DM-only endpoint's own `dm_` durable does not exist. The endpoint never creates it. */
 export class DmDurableMissingError extends Error {
     stream;
@@ -898,10 +900,14 @@ export class CotalEndpoint extends EventEmitter {
                 : undefined;
         if (expiryMs === undefined || this.stopped)
             return;
-        this.authExpiryReconnectTimer = setTimeout(() => {
-            if (this.nc === nc)
-                this.disableLibraryReconnect(nc);
-        }, Math.max(0, expiryMs - Date.now() - 500));
+        // Node fires a delay above 2^31-1 ms (about 24.8 days) after 1 ms, which would disable reconnect
+        // at once. A later expiry waits in steps of that bound, and the fence is armed again from there.
+        const delay = Math.max(0, expiryMs - Date.now() - 500);
+        this.authExpiryReconnectTimer = delay > MAX_TIMER_MS
+            ? setTimeout(() => { if (this.nc === nc)
+                this.armAuthExpiryReconnectFence(nc); }, MAX_TIMER_MS)
+            : setTimeout(() => { if (this.nc === nc)
+                this.disableLibraryReconnect(nc); }, delay);
         this.authExpiryReconnectTimer.unref?.();
     }
     /** The connectAndBind PRE-CONNECT fetch: pull the freshest source cred and pin it into
@@ -2131,7 +2137,7 @@ export class CotalEndpoint extends EventEmitter {
     }
     /** This endpoint's v0.4 caller triple (§13.2) — the identity its minted ep-rail rows pin. The
      *  owner/actor principal is mode-correct by construction (static: DEV_OWNER + the connection
-     *  identity; user mode: the bearer's callout-derived pair — 1c.2c), and the lifecycle UID is the
+     *  identity; user mode: the bearer's callout-derived pair), and the lifecycle UID is the
      *  launcher-supplied incarnation the rows are keyed on (ledger-consistent: the §13.1 presence
      *  lifecycle-proof refuses a divergent uid before any publish). */
     serviceCaller() {
@@ -2480,7 +2486,13 @@ export class CotalEndpoint extends EventEmitter {
         this.activity = activity;
         await this.publishPresence();
     }
-    async setStatus(status) {
+    /** Set the status, and with `activity` the activity line too, in one presence write, so no peer sees
+     *  the new activity beside the old status. */
+    async setStatus(status, activity) {
+        if (activity !== undefined && activity !== this.activity)
+            this.statusSince = this.activitySince = Date.now();
+        if (activity !== undefined)
+            this.activity = activity;
         if (status !== this.status)
             this.statusSince = Date.now();
         this.status = status;
@@ -2562,7 +2574,8 @@ export class CotalEndpoint extends EventEmitter {
     }
     /** A channel's effective replay policy read straight from the registry bucket, for a caller that
      *  reads it right after start, before any watch cache has caught up, or that watches nothing. A
-     *  wildcard reads the space defaults only. A space with no registry bucket has the built-in policy. */
+     *  wildcard reads the space defaults only. A space with no registry bucket has the built-in policy.
+     *  `registered` says whether the channel has its own registry entry. */
     async channelReplayPolicy(channel) {
         if (!this.nc)
             throw new Error(this.notLiveMsg());
@@ -2580,7 +2593,7 @@ export class CotalEndpoint extends EventEmitter {
             if (!isJetStreamMissing(e, JetStreamApiCodes.StreamNotFound))
                 throw e;
         }
-        return { replay: effectiveReplay(cfg, defaults), window: cfg?.replayWindow ?? defaults?.replayWindow ?? null };
+        return { replay: effectiveReplay(cfg, defaults), window: cfg?.replayWindow ?? defaults?.replayWindow ?? null, registered: cfg !== undefined };
     }
     /** Effective delivery class for a channel (per-channel override ?? space default ?? "durable"),
      *  from the live watch cache — drives the non-gating delivery-health surface (only durable-class
@@ -2699,7 +2712,8 @@ export class CotalEndpoint extends EventEmitter {
      * message on the channel is discarded. `maxAgeMs` is the stream's age limit. Each is `null` when
      * the stream sets none. `senders` counts each sender's retained messages per channel, so a sender
      * at the cap marks a channel whose retained history has lost that sender's older messages while
-     * other senders' messages from the same time remain.
+     * other senders' messages from the same time remain. `stream` is false when the chat stream does
+     * not exist, which holds nothing. `lastSeq` is the stream's newest sequence, 0 when it has none.
      */
     async chatRetention() {
         if (!this.nc)
@@ -2713,7 +2727,7 @@ export class CotalEndpoint extends EventEmitter {
             // As in listChannels: an absent CHAT stream holds nothing yet; any other failure stays loud.
             if (!isJetStreamMissing(e, JetStreamApiCodes.StreamNotFound))
                 throw e;
-            return { perSender: null, maxAgeMs: null, senders: [] };
+            return { perSender: null, maxAgeMs: null, senders: [], stream: false, lastSeq: 0 };
         }
         const senders = [];
         for (const [subject, messages] of Object.entries(info.state.subjects ?? {})) {
@@ -2727,6 +2741,8 @@ export class CotalEndpoint extends EventEmitter {
             perSender: typeof cap === "number" && cap > 0 ? cap : null,
             maxAgeMs: typeof age === "number" && age > 0 ? Math.round(age / 1_000_000) : null,
             senders: senders.sort((a, b) => a.channel.localeCompare(b.channel) || a.sender.localeCompare(b.sender)),
+            stream: true,
+            lastSeq: info.state.last_seq,
         };
     }
     /** One coherent channel model for dashboards: every channel that has messages OR a registry
@@ -3163,22 +3179,13 @@ export class CotalEndpoint extends EventEmitter {
      * The newest `limit` chat messages across MANY channels at once, oldest-first within the page,
      * each tagged with the channel the BROKER delivered it on.
      *
-     * **One read, not one per channel.** The CHAT stream already interleaves every channel into one
-     * sequence space, so "the newest N across these channels" is the tail of ONE stream, and a
-     * consumer takes a SET of filter subjects. The dashboard's activity feed used to answer this by
-     * calling {@link channelHistory} once per channel and merging: each of those is a widening probe
-     * loop, so the cost carried two multipliers (a probe loop per channel, and a fan-out across every
-     * channel). Counted on the wire over a seeded corpus of 69 chat channels and 24 event channels at
-     * limit 200: 2524 broker requests and about 8.0 MB transferred to return a 143,401-byte page,
-     * against 143 requests and about 0.91 MB here. With no link cost the counts and the page size
-     * repeat exactly across runs; the byte totals move by tens of bytes. `pnpm smoke:web-activity-read-cost` reproduces the
-     * second column and a frozen copy of the fan-out shape; the first is that same suite run against
-     * `544a974b7` (Cotal #1210).
+     * The CHAT stream interleaves every channel into one sequence space. Consumers take sets of
+     * filter subjects for the requested channels. Large filter sets are read in batches, and the
+     * pages are merged by stream sequence to select the newest `limit` messages overall.
      *
      * **The broker does the filtering, so the wire carries only what is asked for.** A channel left
      * out of `channels` costs nothing: its messages are never delivered, so a space whose volume is
-     * dominated by channels the caller does not want stays cheap. That is the same "filter before the
-     * fetch" property the per-channel fan-out had, kept rather than traded away.
+     * dominated by channels the caller does not want stays cheap.
      *
      * **The channel comes from the SUBJECT, never from the payload.** A message claims a `channel`
      * field, and this method ignores it: the tag is derived from the subject the broker routed the
@@ -3956,7 +3963,7 @@ export class CotalEndpoint extends EventEmitter {
     /**
      * Privileged: pre-create a role's shared TASK work-queue durable (auth mode), so agents
      * of that role can BIND it without holding CONSUMER.CREATE on TASK_<space>. The creator
-     * sets the filter to svc.<role>.* — agents never choose it, which stops cross-role drain.
+     * sets the filter to svc.<role>.> — agents never choose it, which stops cross-role drain.
      * Idempotent per role. The caller must be permissive on TASK_<space>.
      */
     async provisionTaskQueue(role) {
@@ -4090,8 +4097,8 @@ export class CotalEndpoint extends EventEmitter {
     async renewDeliveryLease(shardIndex, revision) {
         return (await this.deliveryRegistry()).update(leaseKey(shardIndex), this.encodeLease(shardIndex, true), revision);
     }
-    /** Release the held lease on clean shutdown so a replacement daemon re-acquires immediately (best
-     *  effort, a crash just lets the bucket TTL expire it).
+    /** Release the held lease on clean shutdown so a replacement daemon re-acquires immediately (a
+     *  crash just lets the bucket TTL expire it).
      *
      *  THE REVISION IS WHAT MAKES THIS A RELEASE RATHER THAN A DELETE. An unconditional delete removes
      *  whatever row is there, and by shutdown time the row is not necessarily still ours: the exit
@@ -4101,6 +4108,10 @@ export class CotalEndpoint extends EventEmitter {
      *  swap (`previousSeq` becomes JetStream's `ExpectedLastSubjectSequence`), so a row that has moved
      *  on is left alone.
      *
+     *  A RELEASE THAT DID NOT COMMIT REJECTS. The broker's CAS refusal and any broker failure reach the
+     *  caller, because only the caller knows whether that matters: a shutdown that must finish anyway
+     *  catches it, and a close that reports whether a successor can start learns the outcome here.
+     *
      *  THE ARGUMENT IS REQUIRED, AND EXPLICITLY NULLABLE RATHER THAN OPTIONAL. `undefined` means "this
      *  process no longer holds a revision it can argue for", which is the takeover paths' honest
      *  answer and correctly releases nothing, the bucket TTL is the crash-safe authority and expires
@@ -4112,14 +4123,7 @@ export class CotalEndpoint extends EventEmitter {
     async releaseDeliveryLease(shardIndex, revision) {
         if (revision === undefined)
             return;
-        try {
-            await (await this.deliveryRegistry()).delete(leaseKey(shardIndex), { previousSeq: revision });
-        }
-        catch {
-            // Intentionally best-effort for EVERY failure: the lease TTL is the crash-safe release authority,
-            // and clean shutdown must continue even when the broker is already gone or draining. A refused
-            // CAS lands here too, which is the correct outcome, someone else owns the row.
-        }
+        await (await this.deliveryRegistry()).delete(leaseKey(shardIndex), { previousSeq: revision });
     }
     /** Read a shard's delivery lease (the daemon-availability signal), or `undefined` if none is live.
      *  READ-ONLY surface — drives Component 6's `cotal_channels` delivery-health field (an agent reads it
@@ -4339,8 +4343,8 @@ export class CotalEndpoint extends EventEmitter {
             await (await this.managerLeaseRegistry()).delete(MANAGER_RENEWAL_LEASE_KEY, { previousSeq: held });
         }
         catch {
-            // Best-effort, like releaseManagerLease: a moved revision means it is not ours, and a broker
-            // failure is recovered by the bucket TTL. Shutdown must not claim deletion.
+            // Best-effort: a moved revision means it is not ours, and a broker failure is recovered by the
+            // bucket TTL. Shutdown must not claim deletion.
         }
     }
     encodeDaemonRenewalLease(instanceId) {
@@ -4389,20 +4393,15 @@ export class CotalEndpoint extends EventEmitter {
         return { info: JSON.parse(new TextDecoder().decode(e.value)), revision: e.revision, created: e.created.getTime() };
     }
     /** Release THIS instance's key on clean shutdown so a same-id restart re-acquires immediately. CAS-guarded
-     *  by `revision`: if we already LOST it (renew gap) the stored revision has moved, the conditional delete
-     *  no-ops. Keyed per instance, so a release NEVER touches a sibling manager's key (security pin 6). */
+     *  by `revision`: if we already LOST it (renew gap) the stored revision has moved, and the conditional
+     *  delete is refused and rejects, like any broker failure; a shutdown that must finish anyway catches
+     *  it. Keyed per instance, so a release NEVER touches a sibling manager's key (security pin 6). */
     async releaseManagerLease(instanceId, revision) {
-        try {
-            const kv = await this.managerLeaseRegistry();
-            if (revision === undefined)
-                await kv.delete(managerLeaseKey(instanceId));
-            else
-                await kv.delete(managerLeaseKey(instanceId), { previousSeq: revision });
-        }
-        catch {
-            // Intentionally best-effort for EVERY failure: a revision mismatch means the lease is no longer
-            // ours, while a broker failure is recovered by the bucket TTL. Shutdown must not claim deletion.
-        }
+        const kv = await this.managerLeaseRegistry();
+        if (revision === undefined)
+            await kv.delete(managerLeaseKey(instanceId));
+        else
+            await kv.delete(managerLeaseKey(instanceId), { previousSeq: revision });
     }
     /** Read a live manager liveness lease, or undefined if NONE (no manager instance holds the space). A
      *  presence/existence check for the CLI's `spawn -f` reuse and `waitLeaseGone`, which only need "is any
@@ -5309,7 +5308,7 @@ export class CotalEndpoint extends EventEmitter {
                 : { ok: true, data: { delivery, membership } };
         }
         if (req.op === "evictPrincipal") {
-            // The LIVE-EVICTION executor (D5 slice 6): force-drop a denied principal's connections.
+            // The LIVE-EVICTION executor: force-drop a denied principal's connections.
             // Composition-root hook because the $SYS observer/evictor creds live outside this endpoint's
             // trust boundary; absent hook = a daemon build without the executor, refused loudly.
             if (!this.plane3?.evictPrincipal)
@@ -5964,7 +5963,7 @@ export class CotalEndpoint extends EventEmitter {
                 inactiveThresholdMs: this.inactiveThresholdMs,
             });
         }
-        await this.pump(dmStream(this.space), dmDurable(this.owner, this.actor, ownUid));
+        await this.pump(dmStream(this.space), dmDurable(this.owner, this.actor, ownUid), { inbox: true });
         // Plane-3 (SPEC §8): bind + pump our per-member DELIVER durable (`dlv_<id>`) — the re-authorized
         // durable-backstop channel copies the trusted reader transfers to us. No-op when it isn't present
         // (open mode / un-provisioned). Auth-only feature; the pump self-guards on the durable's existence.
@@ -6008,13 +6007,15 @@ export class CotalEndpoint extends EventEmitter {
     /** Drive one consumer: decode and hand each message to listeners with ack control. Our own sends are
      *  delivered too: on the DM inbox and the role queue they were addressed to us, so none is an echo,
      *  and acking one unseen on the work queue would delete the only copy of the request. */
-    async pump(stream, durable) {
+    async pump(stream, durable, opts = {}) {
         if (!this.js)
             throw new Error("endpoint not started");
         const consumer = await this.js.consumers.get(stream, durable);
         const ackWaitMs = await ackWaitOf(consumer);
         const msgs = await consumer.consume();
         this.streamMsgs.push(msgs);
+        if (opts.inbox)
+            this.watchInbox(msgs, stream, durable);
         void (async () => {
             for await (const m of msgs) {
                 const authed = this.authenticateStreamMsg(m);
@@ -6048,6 +6049,42 @@ export class CotalEndpoint extends EventEmitter {
                 this.emit("error", e);
         });
     }
+    /** Warn once when the broker reports this endpoint's own DM inbox missing while the connection is
+     *  up, as after a store reset that the client's reconnect rode through. nats.js keeps checking and
+     *  resumes pulling if the inbox comes back, so this only reports. One report can be a passing state,
+     *  so the warning waits for a second one; the several 409s a delete answers pending pulls with count
+     *  once. A pull means the consumer was found again, which re-arms the warning. The loop ends when
+     *  the consumer is stopped, by `stop()` or by a rebuild replacing this connection. */
+    watchInbox(msgs, stream, durable) {
+        const boundNc = this.nc;
+        void (async () => {
+            let absences = 0;
+            let deleted = false;
+            for await (const s of msgs.status()) {
+                if (this.stopped || this.nc !== boundNc)
+                    break;
+                if (s.type === "next") {
+                    absences = 0;
+                    deleted = false;
+                    continue;
+                }
+                if (s.type === "consumer_deleted") {
+                    if (deleted)
+                        continue;
+                    deleted = true;
+                }
+                else if (s.type !== "consumer_not_found" && s.type !== "stream_not_found")
+                    continue;
+                if (++absences === 2)
+                    this.emitRecoverable(new Error(`DM inbox missing: the broker reports consumer "${durable}" on stream "${stream}" missing, so direct ` +
+                        "messages to this endpoint are not delivered. The endpoint keeps checking; if the inbox does not " +
+                        "come back, end this session and start a new one"));
+            }
+        })().catch((e) => {
+            if (!this.stopped)
+                this.emit("error", e);
+        });
+    }
     /** Decode one stream copy and apply the authenticity guard (fail closed): the sender is encoded in
      *  the subject, which the server policed who could publish. The payload `from` is advisory — it
      *  must match, and a missing `from` or an unparseable subject on a delivery is itself an anomaly.
@@ -6280,6 +6317,12 @@ export class CotalEndpoint extends EventEmitter {
         const jsm = await this.manager();
         return (await jsm.streams.info(chatStream(this.space))).state.last_seq;
     }
+    /** The bytes this connection's account holds in JetStream file storage, as `$JS.API.INFO`
+     *  reports them. A broker's `max_file_store` cap counts every account it serves, so this is a
+     *  lower bound on what that cap measures. */
+    async storedBytes() {
+        return (await (await this.manager()).getAccountInfo()).storage;
+    }
     /** Phase 1 of a join — arm each channel's tail-drop watermark at the current frontier. MUST run
      *  BEFORE opening the core subscription so the live tail can never carry a just-joined message
      *  un-watermarked — which would double-emit it (live + backfill).
@@ -6919,7 +6962,7 @@ export class CotalEndpoint extends EventEmitter {
         this.applyPresence(e.key, p);
     }
     applyPresence(id, raw) {
-        // Defense-in-depth (per-user-auth closure (i), residual 3): the KV key IS the publisher's identity —
+        // Defense-in-depth: the KV key IS the publisher's identity —
         // publishPresence() writes its record under card.id — so a record whose embedded card.id disagrees
         // with its bucket key is forged or corrupt. Drop it rather than surface a spoofed roster identity.
         // The write-side scoping ($KV.<presenceBucket>.<own-id>) is the primary guard; this rejects a
@@ -7537,15 +7580,38 @@ export function defaultProbeTimeoutMs(servers) {
  *  `wss://host/path`), everything else through the TCP transport. The websocket dial OWNS its
  *  transport options: the URL scheme already decides TLS there, and the w3c transport refuses a
  *  `tls` block outright ("'tls' is not configurable"), so it is stripped here — at the one point
- *  that knows which transport is dialing — rather than at every caller composing auth options. */
+ *  that knows which transport is dialing — rather than at every caller composing auth options.
+ *  A TCP dial that requires TLS to one IP address names that address as the host its certificate
+ *  must cover (see {@link withDialedTlsHost}). */
 export function dialerFor(servers) {
     if (!wsServers(servers))
-        return connect;
+        return ((opts) => connect(withDialedTlsHost(servers, opts)));
     return ((opts) => {
         const { tls: _tls, ...rest } = (opts ?? {});
         return wsconnect(rest);
     });
 }
+/** `opts` for a dial to `servers`, with the IP address it names as the host a TLS certificate must
+ *  cover when the dial requires TLS: a `tls` block, or a `tls://` address. The TCP transport upgrades
+ *  to TLS with no server name for an IP, and Node then checks the certificate against "localhost":
+ *  a certificate for the dialed address alone is refused, and one naming localhost passes for any
+ *  address. A dial that does not require TLS keeps its options, because a `tls` block refuses a
+ *  plaintext broker, and so does a list of several servers, whose one host the options cannot name. */
+function withDialedTlsHost(servers, opts) {
+    const urls = servers.split(",").map((url) => url.trim()).filter(Boolean);
+    const tls = opts?.tls;
+    if (urls.length !== 1 || tls === null || tls === undefined && !/^tls:\/\//i.test(urls[0]))
+        return opts;
+    let host;
+    try {
+        ({ host } = hostPort(urls[0]));
+    }
+    catch {
+        return opts;
+    }
+    // Node checks an upgraded socket's certificate against `host` when no server name is set.
+    return isIP(host) ? { ...opts, tls: { host, ...tls } } : opts;
+}
 /** Parse a NATS server URL (`nats://host:port`, `host:port`, a bare host, or a comma list — the
  *  first entry wins) into a host+port for {@link tcpInfoProbe}. Defaults the port to 4222. */
 function hostPort(server) {