@cotal-ai/core 0.54.0 → 0.55.0

package/dist/endpoint.js

@@ -48,6 +48,27 @@ export const MULTI_FILTER_BATCH = 1_000;
  *  issuing one read per channel: a space large enough to need batches must not get the fan-out back
  *  under another name. */
 export const MULTI_FILTER_READ_CONCURRENCY = 4;
+/** A presence bucket that has refused consecutive writes for at least one full presence TTL.
+ * The condition is non-transient: ordinary heartbeat retry has already failed for the whole
+ * liveness window, so callers should report the roster as last-known until a write succeeds. */
+export class PresenceWriteStuckError extends Error {
+    bucket;
+    since;
+    consecutiveFailures;
+    ttlMs;
+    lastError;
+    code = "presence-write-stuck";
+    transient = false;
+    constructor(bucket, since, consecutiveFailures, ttlMs, lastError) {
+        super(`presence writes to bucket ${JSON.stringify(bucket)} have failed ${consecutiveFailures} consecutive times for at least one TTL (${ttlMs}ms); the presence view is not live until a write succeeds or the broker store is repaired${lastError ? ` (last refusal: ${lastError})` : ""}`);
+        this.bucket = bucket;
+        this.since = since;
+        this.consecutiveFailures = consecutiveFailures;
+        this.ttlMs = ttlMs;
+        this.lastError = lastError;
+        this.name = "PresenceWriteStuckError";
+    }
+}
 /**
  * Events: "message" (CotalMessage), "presence" (PresenceEvent), "roster" (Presence[]), "error" (Error),
  * "connection" ({ connected: boolean }) — true on every successful (re)bind (initial start, manual
@@ -225,6 +246,10 @@ export class CotalEndpoint extends EventEmitter {
     presenceWriteFailingSince;
     /** #1356: the broker's last refusal message, kept alongside the start time for diagnosis. */
     lastPresenceWriteError;
+    /** Consecutive latest-evidence refusals on the current connection epoch. A success resets it. */
+    presenceWriteFailures = 0;
+    /** One named non-transient warning per failed run. A success re-arms it. */
+    presenceWriteEscalated = false;
     /** #1461: monotonic per-put generation on the presence path, so a settle can tell whether it is
      *  the latest evidence on its epoch. {@link publishPresence} snapshots it at put start and every
      * settle records its generation here: a settle is the latest evidence only while no put that
@@ -1949,11 +1974,18 @@ export class CotalEndpoint extends EventEmitter {
         const caller = this.serviceCaller();
         const resolve = async (signal = opts.signal) => {
             signal?.throwIfAborted();
-            const cached = this.resolvedServices.get(endpoint);
+            const cached = opts.instanceId === undefined ? this.resolvedServices.get(endpoint) : undefined;
             if (cached)
                 return cached;
-            const svc = await resolveService(nc, this.space, endpoint, caller, { deadlineMs: opts.deadlineMs ?? 10_000, signal });
-            this.resolvedServices.set(endpoint, svc);
+            const svc = await resolveService(nc, this.space, endpoint, caller, {
+                deadlineMs: opts.deadlineMs ?? 10_000,
+                signal,
+                ...(opts.instanceId !== undefined ? { instanceId: opts.instanceId } : {}),
+            });
+            // A pinned resolve never enters the endpoint-only class cache. Otherwise a later unpinned call
+            // could silently inherit that instance, or a different pin could reuse the wrong manager.
+            if (opts.instanceId === undefined)
+                this.resolvedServices.set(endpoint, svc);
             return svc;
         };
         const invokeOpts = { ...(opts.target ? { target: opts.target } : {}), ...(opts.deadlineMs !== undefined ? { deadlineMs: opts.deadlineMs } : {}) };
@@ -2591,7 +2623,7 @@ export class CotalEndpoint extends EventEmitter {
                 }
                 else {
                     const closedEpoch = err.name === "ClosedConnectionError" || /^closed connection$/i.test(err.message);
-                    const timeout = err.name === "TimeoutError" || /timeout/i.test(err.message);
+                    const timeout = isTimeoutError(err);
                     const dyingEpochTimeout = timeout && (this.reconnecting || !this.nc || this.nc.isClosed());
                     // Cleanup of an ordered consumer: a delete timeout means the broker did not answer in time,
                     // not that the endpoint is unusable. The broker reaps an idle/ephemeral consumer anyway.
@@ -4673,15 +4705,20 @@ export class CotalEndpoint extends EventEmitter {
             return;
         if (!this.ownLifecycleUid)
             return; // no lifecycle uid — never provisioned for Plane-3 (its durable is lifecycle-keyed)
+        const durable = dlvDurable(this.owner, this.actor, this.ownLifecycleUid);
         let consumer;
         try {
-            consumer = await this.js.consumers.get(dlvStream(this.space), dlvDurable(this.owner, this.actor, this.ownLifecycleUid));
+            consumer = await this.js.consumers.get(dlvStream(this.space), durable);
         }
         catch (e) {
             if (isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
                 return;
             throw e; // a denied bind is not proof Plane-3 is absent
         }
+        const lease = await this.readDeliveryLease(0);
+        const liveDeliveryPlane = lease?.ready === true;
+        if (!liveDeliveryPlane)
+            this.emit("warning", new Error(`delivery durable "${durable}" for space "${this.space}" bound while the plane this connection reaches has no ready delivery lease, so the durable delivers nothing until a delivery daemon serves this plane. If a daemon serves this space on another plane, reconnect against it, which re-binds the durable there.`));
         const msgs = await consumer.consume();
         this.streamMsgs.push(msgs);
         void (async () => {
@@ -5433,6 +5470,8 @@ export class CotalEndpoint extends EventEmitter {
             if (epoch === this.presenceEpoch && !this.stopped && !superseded) {
                 this.presenceWriteFailingSince ??= Date.now();
                 this.lastPresenceWriteError = e?.message ?? String(e);
+                this.presenceWriteFailures++;
+                this.escalatePresenceWriteFailureIfStuck();
             }
             throw e;
         }
@@ -5471,6 +5510,21 @@ export class CotalEndpoint extends EventEmitter {
     clearPresenceWriteFailure() {
         this.presenceWriteFailingSince = undefined;
         this.lastPresenceWriteError = undefined;
+        this.presenceWriteFailures = 0;
+        this.presenceWriteEscalated = false;
+    }
+    /** A heartbeat retry remains recoverable inside one TTL. Once two or more consecutive writes have
+     * failed across the full window, ordinary retry has exhausted the roster's liveness budget: raise
+     * one named, non-transient warning. Later failures keep the duration/count current without flooding
+     * the operator log; the next successful write clears and re-arms the condition. */
+    escalatePresenceWriteFailureIfStuck() {
+        if (this.presenceWriteEscalated ||
+            this.presenceWriteFailingSince === undefined ||
+            this.presenceWriteFailures < 2 ||
+            Date.now() - this.presenceWriteFailingSince < this.ttlMs)
+            return;
+        this.presenceWriteEscalated = true;
+        this.emitRecoverable(new PresenceWriteStuckError(presenceBucket(this.space), this.presenceWriteFailingSince, this.presenceWriteFailures, this.ttlMs, this.lastPresenceWriteError));
     }
     /** #1356: the presence bucket has been refusing writes since this time, or `undefined` when the
      *  last publish succeeded. Cleared by a successful write that is the latest evidence on its
@@ -5488,6 +5542,8 @@ export class CotalEndpoint extends EventEmitter {
             forMs: Date.now() - this.presenceWriteFailingSince,
             error: this.lastPresenceWriteError,
             bucket: presenceBucket(this.space),
+            consecutiveFailures: this.presenceWriteFailures,
+            stuck: this.presenceWriteEscalated,
         };
     }
     /** Bind a presence watch on the current connection. Resolves true when the watch was
@@ -6447,6 +6503,14 @@ export async function isReachable(servers = DEFAULT_SERVER, opts = {}) {
         return e instanceof AuthorizationError || e instanceof UserAuthenticationExpiredError;
     }
 }
+/** True when `err` is a dial/consumer-op timeout rather than a real refusal — the one shared test
+ *  for "the operation ran out of its own budget", used both by {@link classifyProbeFailure} (#851:
+ *  a probe timeout must never collapse into `unreachable`, which a TLS-required target then
+ *  misreads as a trust failure) and by {@link Endpoint#disarmMembershipWatch}'s consumer-delete
+ *  cleanup, which predates it. */
+function isTimeoutError(err) {
+    return err instanceof Error && (err.name === "TimeoutError" || /timeout/i.test(err.message));
+}
 /** Like {@link isReachable}, but distinguishes "up but won't take these creds" from "nothing there".
  *  `spawn` needs the difference: auth-required → name the trust dir + next step; unreachable → the
  *  mesh is down (prune the stale entry, tell the user to `cotal up`). Pass `creds` to confirm a
@@ -6504,6 +6568,11 @@ function classifyProbeFailure(e, opts) {
     // The broker answered but rejected these creds (so it IS up) — auth-required, not stale-auth.
     if (e instanceof AuthorizationError)
         return { ok: false, reason: "auth-required" };
+    // A dial that ran out of its own budget is neither a refusal nor a dead broker — it is latency.
+    // `e` is undefined when the tcpDialable gate refused before any connect() attempt; that path has
+    // no timeout to inspect and must stay `unreachable` (nothing answered at all).
+    if (e !== undefined && isTimeoutError(e))
+        return { ok: false, reason: "timeout" };
     return { ok: false, reason: "unreachable" };
 }
 //# sourceMappingURL=endpoint.js.map