@cotal-ai/core 0.54.0 → 0.56.0

package/dist/endpoint.js

@@ -48,6 +48,27 @@ export const MULTI_FILTER_BATCH = 1_000;
  *  issuing one read per channel: a space large enough to need batches must not get the fan-out back
  *  under another name. */
 export const MULTI_FILTER_READ_CONCURRENCY = 4;
+/** A presence bucket that has refused consecutive writes for at least one full presence TTL.
+ * The condition is non-transient: ordinary heartbeat retry has already failed for the whole
+ * liveness window, so callers should report the roster as last-known until a write succeeds. */
+export class PresenceWriteStuckError extends Error {
+    bucket;
+    since;
+    consecutiveFailures;
+    ttlMs;
+    lastError;
+    code = "presence-write-stuck";
+    transient = false;
+    constructor(bucket, since, consecutiveFailures, ttlMs, lastError) {
+        super(`presence writes to bucket ${JSON.stringify(bucket)} have failed ${consecutiveFailures} consecutive times for at least one TTL (${ttlMs}ms); the presence view is not live until a write succeeds or the broker store is repaired${lastError ? ` (last refusal: ${lastError})` : ""}`);
+        this.bucket = bucket;
+        this.since = since;
+        this.consecutiveFailures = consecutiveFailures;
+        this.ttlMs = ttlMs;
+        this.lastError = lastError;
+        this.name = "PresenceWriteStuckError";
+    }
+}
 /**
  * Events: "message" (CotalMessage), "presence" (PresenceEvent), "roster" (Presence[]), "error" (Error),
  * "connection" ({ connected: boolean }) — true on every successful (re)bind (initial start, manual
@@ -225,6 +246,10 @@ export class CotalEndpoint extends EventEmitter {
     presenceWriteFailingSince;
     /** #1356: the broker's last refusal message, kept alongside the start time for diagnosis. */
     lastPresenceWriteError;
+    /** Consecutive latest-evidence refusals on the current connection epoch. A success resets it. */
+    presenceWriteFailures = 0;
+    /** One named non-transient warning per failed run. A success re-arms it. */
+    presenceWriteEscalated = false;
     /** #1461: monotonic per-put generation on the presence path, so a settle can tell whether it is
      *  the latest evidence on its epoch. {@link publishPresence} snapshots it at put start and every
      * settle records its generation here: a settle is the latest evidence only while no put that
@@ -1949,11 +1974,18 @@ export class CotalEndpoint extends EventEmitter {
         const caller = this.serviceCaller();
         const resolve = async (signal = opts.signal) => {
             signal?.throwIfAborted();
-            const cached = this.resolvedServices.get(endpoint);
+            const cached = opts.instanceId === undefined ? this.resolvedServices.get(endpoint) : undefined;
             if (cached)
                 return cached;
-            const svc = await resolveService(nc, this.space, endpoint, caller, { deadlineMs: opts.deadlineMs ?? 10_000, signal });
-            this.resolvedServices.set(endpoint, svc);
+            const svc = await resolveService(nc, this.space, endpoint, caller, {
+                deadlineMs: opts.deadlineMs ?? 10_000,
+                signal,
+                ...(opts.instanceId !== undefined ? { instanceId: opts.instanceId } : {}),
+            });
+            // A pinned resolve never enters the endpoint-only class cache. Otherwise a later unpinned call
+            // could silently inherit that instance, or a different pin could reuse the wrong manager.
+            if (opts.instanceId === undefined)
+                this.resolvedServices.set(endpoint, svc);
             return svc;
         };
         const invokeOpts = { ...(opts.target ? { target: opts.target } : {}), ...(opts.deadlineMs !== undefined ? { deadlineMs: opts.deadlineMs } : {}) };
@@ -2453,10 +2485,10 @@ export class CotalEndpoint extends EventEmitter {
      *  including the initial replay — the caller debounces + re-reads {@link readMembership}. The async
      *  stop handle resolves only after its ordered broker consumer is deleted. Best-effort: a feed the
      *  cred can't read (or absent) surfaces as an `error` event and the dashboard keeps its last snapshot. */
-    async watchMembership(onChange) {
+    async watchMembership(onChange, onClosed) {
         if (this.stopped)
             throw new Error("endpoint stopped - cannot watch membership");
-        const watch = { onChange, stopped: false, arm: Promise.resolve() };
+        const watch = { onChange, onClosed, stopped: false, arm: Promise.resolve() };
         this.membershipFeedWatches.add(watch);
         watch.arm = watch.arm.catch(() => { }).then(() => this.armMembershipWatch(watch));
         try {
@@ -2528,8 +2560,11 @@ export class CotalEndpoint extends EventEmitter {
             return;
         }
         iter.closed().then(() => {
-            if (!watch.stopped && watch.consumer === consumer)
-                this.emit("error", new Error("membership watch closed"));
+            if (watch.stopped || watch.consumer !== consumer)
+                return;
+            const err = new Error("membership watch closed");
+            watch.onClosed?.(err);
+            this.emit("error", err);
         }).catch(() => { });
     }
     /** Delete identity retained across a failed/closed-epoch consumer object using the CURRENT connection. */
@@ -2591,7 +2626,7 @@ export class CotalEndpoint extends EventEmitter {
                 }
                 else {
                     const closedEpoch = err.name === "ClosedConnectionError" || /^closed connection$/i.test(err.message);
-                    const timeout = err.name === "TimeoutError" || /timeout/i.test(err.message);
+                    const timeout = isTimeoutError(err);
                     const dyingEpochTimeout = timeout && (this.reconnecting || !this.nc || this.nc.isClosed());
                     // Cleanup of an ordered consumer: a delete timeout means the broker did not answer in time,
                     // not that the endpoint is unusable. The broker reaps an idle/ephemeral consumer anyway.
@@ -3605,6 +3640,16 @@ export class CotalEndpoint extends EventEmitter {
     encodeDaemonRenewalLease(instanceId) {
         return new TextEncoder().encode(JSON.stringify({ instanceId, since: Date.now() }));
     }
+    /** Read the per-space daemon-credential renewal-lease row (the one {@link holdDaemonRenewalLease}
+     *  CAS-writes), or `undefined` when nothing holds it. A `doctor auth --fix` that lost `hold` calls
+     *  this through the SAME bucket the hold attempt opened, to name the live holder in its refusal —
+     *  `managerLeaseKv` is private, so this is the exported read beside {@link readManagerLease}. */
+    async readDaemonRenewalLease() {
+        const e = await (await this.managerLeaseRegistry()).get(MANAGER_RENEWAL_LEASE_KEY);
+        if (!e || e.operation !== "PUT")
+            return undefined;
+        return JSON.parse(new TextDecoder().decode(e.value));
+    }
     encodeManagerLease(info) {
         return new TextEncoder().encode(JSON.stringify(info));
     }
@@ -3761,6 +3806,13 @@ export class CotalEndpoint extends EventEmitter {
         const matches = [...this.roster.values()].filter((p) => p.card.name.toLowerCase() === name.toLowerCase());
         return matches.length === 1 ? matches[0].card.id : undefined;
     }
+    /** Plane-3 publish-dedupe key for a fan-out/transfer entry (#673, SPEC §4: two `id: ""` posts
+     *  MUST NOT collapse to one delivery). An empty `msg.id` contributes NO msgID — a plain publish,
+     *  never an empty or salted one — so an id-less message is at-least-once on the durable plane (a
+     *  redelivery may reach the member twice; the receiver already treats each as its own delivery). */
+    plane3MsgId(entry, keyPrefix) {
+        return entry.msg.id === "" ? {} : { msgID: `${entry.msg.id}:${keyPrefix}` };
+    }
     /** Publish one fan-out entry into a member LIFECYCLE's mixed inbox (`dinbox.<o>.<a>.<uid>`, SPEC
      *  §13.1: fan-out addresses the member row's RECORDED lifecycle, never the alias's current
      *  occupant), idempotent via `Nats-Msg-Id` (`<msgId>:<principal>:<generation>`) so a catch-up copy
@@ -3775,7 +3827,7 @@ export class CotalEndpoint extends EventEmitter {
             // JetStream dedupe is STREAM-WIDE, so the id must carry the LIFECYCLE too: with an alias-keyed
             // id, lifecycle A's copy would suppress a same-alias successor B's copy of the same message
             // (both start at generation 1) — cross-lifecycle suppression, not dedup.
-            msgID: `${entry.msg.id}:${principal}:${lifecycleUid}:${entry.generation}`,
+            ...this.plane3MsgId(entry, `${principal}:${lifecycleUid}:${entry.generation}`),
         });
     }
     /** The fan-out consumer's delivered stream-seq — the activation-fence upper bound (red-team
@@ -4639,7 +4691,7 @@ export class CotalEndpoint extends EventEmitter {
             // subject (streams.ts filter_subject), and a predecessor's transferred copy must never suppress
             // a successor's under the same alias (disjoint lifecycles, stream-wide dedupe).
             await this.js.publish(dlvSubject(this.space, pr.owner, pr.actor, pr.lifecycleUid), JSON.stringify(frame), {
-                msgID: `${entry.msg.id}:${owner}:${pr.lifecycleUid}:${entry.generation}`,
+                ...this.plane3MsgId(entry, `${owner}:${pr.lifecycleUid}:${entry.generation}`),
                 headers: frameHeaders,
             });
         }
@@ -4673,15 +4725,20 @@ export class CotalEndpoint extends EventEmitter {
             return;
         if (!this.ownLifecycleUid)
             return; // no lifecycle uid — never provisioned for Plane-3 (its durable is lifecycle-keyed)
+        const durable = dlvDurable(this.owner, this.actor, this.ownLifecycleUid);
         let consumer;
         try {
-            consumer = await this.js.consumers.get(dlvStream(this.space), dlvDurable(this.owner, this.actor, this.ownLifecycleUid));
+            consumer = await this.js.consumers.get(dlvStream(this.space), durable);
         }
         catch (e) {
             if (isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
                 return;
             throw e; // a denied bind is not proof Plane-3 is absent
         }
+        const lease = await this.readDeliveryLease(0);
+        const liveDeliveryPlane = lease?.ready === true;
+        if (!liveDeliveryPlane)
+            this.emit("warning", new Error(`delivery durable "${durable}" for space "${this.space}" bound while the plane this connection reaches has no ready delivery lease, so the durable delivers nothing until a delivery daemon serves this plane. If a daemon serves this space on another plane, reconnect against it, which re-binds the durable there.`));
         const msgs = await consumer.consume();
         this.streamMsgs.push(msgs);
         void (async () => {
@@ -5433,6 +5490,8 @@ export class CotalEndpoint extends EventEmitter {
             if (epoch === this.presenceEpoch && !this.stopped && !superseded) {
                 this.presenceWriteFailingSince ??= Date.now();
                 this.lastPresenceWriteError = e?.message ?? String(e);
+                this.presenceWriteFailures++;
+                this.escalatePresenceWriteFailureIfStuck();
             }
             throw e;
         }
@@ -5471,6 +5530,21 @@ export class CotalEndpoint extends EventEmitter {
     clearPresenceWriteFailure() {
         this.presenceWriteFailingSince = undefined;
         this.lastPresenceWriteError = undefined;
+        this.presenceWriteFailures = 0;
+        this.presenceWriteEscalated = false;
+    }
+    /** A heartbeat retry remains recoverable inside one TTL. Once two or more consecutive writes have
+     * failed across the full window, ordinary retry has exhausted the roster's liveness budget: raise
+     * one named, non-transient warning. Later failures keep the duration/count current without flooding
+     * the operator log; the next successful write clears and re-arms the condition. */
+    escalatePresenceWriteFailureIfStuck() {
+        if (this.presenceWriteEscalated ||
+            this.presenceWriteFailingSince === undefined ||
+            this.presenceWriteFailures < 2 ||
+            Date.now() - this.presenceWriteFailingSince < this.ttlMs)
+            return;
+        this.presenceWriteEscalated = true;
+        this.emitRecoverable(new PresenceWriteStuckError(presenceBucket(this.space), this.presenceWriteFailingSince, this.presenceWriteFailures, this.ttlMs, this.lastPresenceWriteError));
     }
     /** #1356: the presence bucket has been refusing writes since this time, or `undefined` when the
      *  last publish succeeded. Cleared by a successful write that is the latest evidence on its
@@ -5488,6 +5562,8 @@ export class CotalEndpoint extends EventEmitter {
             forMs: Date.now() - this.presenceWriteFailingSince,
             error: this.lastPresenceWriteError,
             bucket: presenceBucket(this.space),
+            consecutiveFailures: this.presenceWriteFailures,
+            stuck: this.presenceWriteEscalated,
         };
     }
     /** Bind a presence watch on the current connection. Resolves true when the watch was
@@ -6364,7 +6440,8 @@ function tcpInfoProbe(server, timeoutMs) {
  *  after the probe already returned its answer. That is issue #389, and it is upstream: nothing a
  *  caller passes (`reconnect: false`, `timeout`) reaches the orphan. Our socket, our `destroy()`,
  *  on every exit path — never an `unref`/force-exit, which would hide the symptom and a future
- *  real hang with it. */
+ *  real hang with it. A second orphan case (#2156) needs a greeting, not just a handshake, so it is
+ *  gated by {@link tcpInfoProbe} instead wherever the dial will be plaintext. */
 function tcpDialable(server, timeoutMs) {
     return new Promise((resolve) => {
         let socket;
@@ -6424,13 +6501,17 @@ export async function isReachable(servers = DEFAULT_SERVER, opts = {}) {
         return tcpInfoProbe(servers, timeoutMs);
     }
     // The credless branch above already owns its socket. This one reaches `connect()`, so it carries
-    // the same orphaned-socket defect probeConnect did (#389) and takes the same gate: reach the
-    // address on a socket we own first, and give `connect()` the remainder of the budget its own
+    // the same orphaned-socket defects probeConnect did (#389, and the slow-greeting case #2156) and
+    // takes the same gate choice: reach the address on a socket we own first, requiring the NATS
+    // greeting for a plaintext dial (the upstream transport orphans a socket whose handshake
+    // completed but whose greeting came late) and only the handshake for a TLS-required or websocket
+    // dial (no plaintext greeting to read), then give `connect()` the remainder of the budget its own
     // timeout always covered. A gate failure is a genuine connection failure, which is exactly the
     // `false` the catch below already returns for one — an auth rejection cannot reach us from an
     // address that never completed a handshake.
     const started = Date.now();
-    if (!(await tcpDialable(servers, timeoutMs)))
+    const gate = opts.tls || wsServers(servers) ? tcpDialable : tcpInfoProbe;
+    if (!(await gate(servers, timeoutMs)))
         return false;
     try {
         const nc = await dialerFor(servers)({
@@ -6447,6 +6528,14 @@ export async function isReachable(servers = DEFAULT_SERVER, opts = {}) {
         return e instanceof AuthorizationError || e instanceof UserAuthenticationExpiredError;
     }
 }
+/** True when `err` is a dial/consumer-op timeout rather than a real refusal — the one shared test
+ *  for "the operation ran out of its own budget", used both by {@link classifyProbeFailure} (#851:
+ *  a probe timeout must never collapse into `unreachable`, which a TLS-required target then
+ *  misreads as a trust failure) and by {@link Endpoint#disarmMembershipWatch}'s consumer-delete
+ *  cleanup, which predates it. */
+function isTimeoutError(err) {
+    return err instanceof Error && (err.name === "TimeoutError" || /timeout/i.test(err.message));
+}
 /** Like {@link isReachable}, but distinguishes "up but won't take these creds" from "nothing there".
  *  `spawn` needs the difference: auth-required → name the trust dir + next step; unreachable → the
  *  mesh is down (prune the stale entry, tell the user to `cotal up`). Pass `creds` to confirm a
@@ -6456,14 +6545,22 @@ export async function probeConnect(server = DEFAULT_SERVER, opts = {}) {
     const timeoutMs = opts.timeoutMs ?? defaultProbeTimeoutMs(server);
     const started = Date.now();
     // Reach the address on a socket we own BEFORE handing it to `connect()`, which orphans the
-    // connection it never established (see {@link tcpDialable} for the upstream mechanism, #389).
-    // This cannot change any verdict: every address that gets past here had to complete a TCP
-    // handshake for `connect()` to have gotten anywhere either, and a gate failure is routed through
-    // the SAME classification the catch uses — so a locally-dead cred is still `stale-auth` and not
-    // silently downgraded to `unreachable` by the address being dark. The cost is one extra
-    // handshake on the reachable path; the deadline below is the REMAINDER of the budget, because
-    // `connect()`'s own `timeout` always covered its handshake too.
-    if (!(await tcpDialable(server, timeoutMs)))
+    // connection it never established (see {@link tcpDialable} for the upstream mechanism, #389,
+    // and {@link tcpInfoProbe} for the second orphan case below, #2156). For a plaintext dial the
+    // gate must see the server's NATS greeting, not just the handshake: `connect()`'s own timeout
+    // can fire while it is still waiting on that greeting, and the upstream transport only destroys
+    // a socket whose handshake never completed, not one whose greeting came late (`_closed`'s guard
+    // on `connected`, which is set only after the greeting is read) — so `tcpInfoProbe` (which owns
+    // its socket and requires the greeting) is the gate there, while a TLS-required or websocket
+    // dial has no plaintext greeting to read and keeps the handshake-only `tcpDialable` gate. This
+    // cannot change any verdict: every address that gets past here had to complete a TCP handshake
+    // for `connect()` to have gotten anywhere either, and a gate failure is routed through the SAME
+    // classification the catch uses — so a locally-dead cred is still `stale-auth` and not silently
+    // downgraded to `unreachable` by the address being dark. The cost is one extra handshake (and,
+    // off TLS/ws, one extra greeting read) on the reachable path; the deadline below is the
+    // REMAINDER of the budget, because `connect()`'s own `timeout` always covered its handshake too.
+    const gate = opts.tls || wsServers(server) ? tcpDialable : tcpInfoProbe;
+    if (!(await gate(server, timeoutMs)))
         return classifyProbeFailure(undefined, opts);
     try {
         const nc = await dialerFor(server)({
@@ -6504,6 +6601,11 @@ function classifyProbeFailure(e, opts) {
     // The broker answered but rejected these creds (so it IS up) — auth-required, not stale-auth.
     if (e instanceof AuthorizationError)
         return { ok: false, reason: "auth-required" };
+    // A dial that ran out of its own budget is neither a refusal nor a dead broker — it is latency.
+    // `e` is undefined when the tcpDialable gate refused before any connect() attempt; that path has
+    // no timeout to inspect and must stay `unreachable` (nothing answered at all).
+    if (e !== undefined && isTimeoutError(e))
+        return { ok: false, reason: "timeout" };
     return { ok: false, reason: "unreachable" };
 }
 //# sourceMappingURL=endpoint.js.map