@cotal-ai/core 0.53.0 → 0.55.0

package/dist/endpoint.js

@@ -48,6 +48,27 @@ export const MULTI_FILTER_BATCH = 1_000;
  *  issuing one read per channel: a space large enough to need batches must not get the fan-out back
  *  under another name. */
 export const MULTI_FILTER_READ_CONCURRENCY = 4;
+/** A presence bucket that has refused consecutive writes for at least one full presence TTL.
+ * The condition is non-transient: ordinary heartbeat retry has already failed for the whole
+ * liveness window, so callers should report the roster as last-known until a write succeeds. */
+export class PresenceWriteStuckError extends Error {
+    bucket;
+    since;
+    consecutiveFailures;
+    ttlMs;
+    lastError;
+    code = "presence-write-stuck";
+    transient = false;
+    constructor(bucket, since, consecutiveFailures, ttlMs, lastError) {
+        super(`presence writes to bucket ${JSON.stringify(bucket)} have failed ${consecutiveFailures} consecutive times for at least one TTL (${ttlMs}ms); the presence view is not live until a write succeeds or the broker store is repaired${lastError ? ` (last refusal: ${lastError})` : ""}`);
+        this.bucket = bucket;
+        this.since = since;
+        this.consecutiveFailures = consecutiveFailures;
+        this.ttlMs = ttlMs;
+        this.lastError = lastError;
+        this.name = "PresenceWriteStuckError";
+    }
+}
 /**
  * Events: "message" (CotalMessage), "presence" (PresenceEvent), "roster" (Presence[]), "error" (Error),
  * "connection" ({ connected: boolean }) — true on every successful (re)bind (initial start, manual
@@ -225,6 +246,10 @@ export class CotalEndpoint extends EventEmitter {
     presenceWriteFailingSince;
     /** #1356: the broker's last refusal message, kept alongside the start time for diagnosis. */
     lastPresenceWriteError;
+    /** Consecutive latest-evidence refusals on the current connection epoch. A success resets it. */
+    presenceWriteFailures = 0;
+    /** One named non-transient warning per failed run. A success re-arms it. */
+    presenceWriteEscalated = false;
     /** #1461: monotonic per-put generation on the presence path, so a settle can tell whether it is
      *  the latest evidence on its epoch. {@link publishPresence} snapshots it at put start and every
      * settle records its generation here: a settle is the latest evidence only while no put that
@@ -310,6 +335,9 @@ export class CotalEndpoint extends EventEmitter {
     /** Per-endpoint-name {@link resolveService} cache for {@link invokeService} — dropped on a
      *  `failed-precondition` currency refusal (the described incarnation was superseded). */
     resolvedServices = new Map();
+    /** Active goal followers awaiting outcome, tracked so endpoint stop can cancel them immediately
+     *  and connection replacement / reconnect can trigger reconciliation. */
+    activeGoalFollowers = new Set();
     /** How many calls {@link invokeService} has silently recovered from a bind refusal (§13.2) — the
      *  class-queue splits this endpoint hit and survived.
      *
@@ -1083,6 +1111,11 @@ export class CotalEndpoint extends EventEmitter {
         if (this.stopped)
             return;
         this.emit("connection", { connected: true });
+        if (this.nc) {
+            for (const follower of this.activeGoalFollowers) {
+                follower.reconnected(this.nc);
+            }
+        }
     }
     /** Tear down everything {@link connectAndBind} (re)creates, so a rebind can't leak a
      *  second heartbeat, double-pump a consumer, or keep stale roster ghosts. Caller-owned
@@ -1446,6 +1479,10 @@ export class CotalEndpoint extends EventEmitter {
         if (this.nc)
             this.disableLibraryReconnect(this.nc);
         this.stopped = true;
+        for (const follower of this.activeGoalFollowers) {
+            follower.cancel();
+        }
+        this.activeGoalFollowers.clear();
         this.presenceEpoch++;
         this.presenceRebind = undefined;
         // Wake a reestablishLoop sitting in backoff so it sees `stopped` and exits instead of
@@ -1862,6 +1899,55 @@ export class CotalEndpoint extends EventEmitter {
             throw new Error("invokeService: the issued generation is not yet discovered on this connection");
         return { ...triple, generation: this.issuedGeneration };
     }
+    /** Follow this caller's goal on its standing connection, subscribing before submission.
+     *  A submitter may use a separate short-lived control credential for the same caller triple;
+     *  the accepted action's readiness budget can outlive that credential. This endpoint retains
+     *  its existing credential renewal and caller-scoped progress grants. */
+    async followServiceGoal(endpoint, submit, deadlineMs = 10_000, opts = {}) {
+        if (!this.nc)
+            throw new Error(this.notLiveMsg());
+        if (this.stopped)
+            throw new Error("endpoint stopped - cannot follow goal");
+        const abortController = new AbortController();
+        const onAbort = () => abortController.abort();
+        if (opts.signal?.aborted)
+            onAbort();
+        else
+            opts.signal?.addEventListener("abort", onAbort, { once: true });
+        let reconnectHandler;
+        const entry = {
+            cancel: () => abortController.abort(),
+            reconnected: (nc) => reconnectHandler?.(nc),
+        };
+        this.activeGoalFollowers.add(entry);
+        try {
+            const reconcile = opts.reconcile ?? (async (goalId, _attributed, context) => {
+                const current = this.serviceCaller();
+                if (current.owner !== context.caller.owner || current.actor !== context.caller.actor || current.uid !== context.caller.uid)
+                    throw new Error("the renewed goal reader no longer has the accepting caller's full identity");
+                // Static callers retain their authorized class route; knowing an instance id grants no rights.
+                const q = await this.invokeService(endpoint, "goal-result", { goalId }, {
+                    deadlineMs: Math.min(deadlineMs, context.deadlineMs), signal: context.signal,
+                });
+                if (q.reply.ok !== true)
+                    throw new Error(q.reply.error?.message ?? "goal-result query refused");
+                return q.reply.data;
+            });
+            return await submitAndFollowGoal(this.nc, this.space, endpoint, this.serviceCaller(), deadlineMs, submit, {
+                reconcile,
+                currentNc: () => this.nc,
+                onReconnect: (cb) => {
+                    reconnectHandler = cb;
+                    return () => { reconnectHandler = undefined; };
+                },
+                signal: abortController.signal,
+            });
+        }
+        finally {
+            opts.signal?.removeEventListener("abort", onAbort);
+            this.activeGoalFollowers.delete(entry);
+        }
+    }
     /** GENERIC v0.4 service invoke over this endpoint's own connection (P2 item 1, 1c.2b): resolve
      *  the named endpoint's registered surface — describe, §13.7 store fetch, digest-verified
      *  recompile ({@link resolveService}; cached per endpoint name) — and invoke one command. The
@@ -1886,18 +1972,40 @@ export class CotalEndpoint extends EventEmitter {
             throw new Error(this.notLiveMsg());
         const nc = this.nc;
         const caller = this.serviceCaller();
-        const resolve = async () => {
-            const cached = this.resolvedServices.get(endpoint);
+        const resolve = async (signal = opts.signal) => {
+            signal?.throwIfAborted();
+            const cached = opts.instanceId === undefined ? this.resolvedServices.get(endpoint) : undefined;
             if (cached)
                 return cached;
-            const svc = await resolveService(nc, this.space, endpoint, caller, { deadlineMs: opts.deadlineMs ?? 10_000 });
-            this.resolvedServices.set(endpoint, svc);
+            const svc = await resolveService(nc, this.space, endpoint, caller, {
+                deadlineMs: opts.deadlineMs ?? 10_000,
+                signal,
+                ...(opts.instanceId !== undefined ? { instanceId: opts.instanceId } : {}),
+            });
+            // A pinned resolve never enters the endpoint-only class cache. Otherwise a later unpinned call
+            // could silently inherit that instance, or a different pin could reuse the wrong manager.
+            if (opts.instanceId === undefined)
+                this.resolvedServices.set(endpoint, svc);
             return svc;
         };
         const invokeOpts = { ...(opts.target ? { target: opts.target } : {}), ...(opts.deadlineMs !== undefined ? { deadlineMs: opts.deadlineMs } : {}) };
-        const doInvoke = async () => {
+        const invokeResolved = (service, signal) => {
+            signal?.throwIfAborted();
+            if (opts.follow && !service.commands.has("goal-result")) {
+                return Promise.resolve({
+                    reply: { v: 1, id: randomUUID(), ok: false, error: {
+                            code: "failed-precondition", outcome: "not-executed",
+                            message: `endpoint "${endpoint}" does not support "goal-result"; upgrade manager to enable durable goal following (SPEC 13.6)`,
+                        } },
+                    responder: { endpoint, instanceId: service.responder.instanceId, epoch: service.responder.epoch },
+                });
+            }
+            return invokeCommand(nc, this.space, service, command, args, { ...invokeOpts, signal });
+        };
+        const doInvoke = async (signal = opts.signal) => {
+            signal?.throwIfAborted();
             try {
-                const r = await invokeCommand(nc, this.space, await resolve(), command, args, invokeOpts);
+                const r = await invokeResolved(await resolve(signal), signal);
                 // THE RESPONDER FENCED IT (§13.2 `ai.cotal.ep.bind-refused`): a class member saw the call
                 // was bound to a different incarnation and refused BEFORE running the command.
                 //
@@ -1945,7 +2053,10 @@ export class CotalEndpoint extends EventEmitter {
                     // the fence's marker does: a responder answered and the effect may have landed.
                     let reissueTarget;
                     try {
-                        reissueTarget = await resolve();
+                        reissueTarget = await resolve(signal);
+                        if (opts.follow && !reissueTarget.commands.has("goal-result")) {
+                            throw new Error(`endpoint "${endpoint}" does not support "goal-result"; upgrade manager to enable durable goal following (SPEC 13.6)`);
+                        }
                     }
                     catch (reissue) {
                         throw new EpEnvelopeError(refusalCode, `${endpoint}.${command} WAS NOT RUN - the incarnation that received it refused it before any effect, and the re-issue could not be resolved: ${reissue instanceof Error ? reissue.message : String(reissue)}. Re-resolve and re-issue when the endpoint is reachable (SPEC 13.2)`, r.reply.error?.details, 
@@ -1955,7 +2066,7 @@ export class CotalEndpoint extends EventEmitter {
                         // re-issue never went out. Prose is for the reader; this is for the machine.
                         "not-executed");
                     }
-                    return await invokeCommand(nc, this.space, reissueTarget, command, args, invokeOpts);
+                    return await invokeResolved(reissueTarget, signal);
                 }
                 return r;
             }
@@ -1992,7 +2103,7 @@ export class CotalEndpoint extends EventEmitter {
                     this.resolvedServices.delete(endpoint);
                     if (!isRepeatSafeCommand(endpoint, command))
                         throw e;
-                    return await invokeCommand(nc, this.space, await resolve(), command, args, invokeOpts);
+                    return await invokeResolved(await resolve(signal), signal);
                 }
                 // An UNMARKED `failed-precondition` is the resolve's own refusal, raised before any command
                 // was published, so re-resolving once is a repair. The `replyRefusedBeforeEffect` half keeps
@@ -2003,14 +2114,14 @@ export class CotalEndpoint extends EventEmitter {
                 if (e.code !== "failed-precondition" || replyRefusedBeforeEffect(e.toEpError()))
                     throw e;
                 this.resolvedServices.delete(endpoint);
-                return await invokeCommand(nc, this.space, await resolve(), command, args, invokeOpts);
+                return await invokeResolved(await resolve(signal), signal);
             }
         };
         // P2 item 2 (2b): a goal-bearing command (spawn/launch) follows its acceptance to the terminal so
         // the caller still returns on the real outcome (UX unchanged); every other command replies directly.
         if (!opts.follow)
             return doInvoke();
-        return submitAndFollowGoal(nc, this.space, endpoint, caller, opts.deadlineMs ?? 10_000, doInvoke);
+        return this.followServiceGoal(endpoint, doInvoke, opts.deadlineMs ?? 10_000, { signal: opts.signal });
     }
     /** Send a durable-membership request to the SERVER-SIDE delivery daemon (`ctl.delivery`) and await its
      *  reply. Unlike {@link requestControl}, the reply rides a subject UNDER `ctl.delivery.<id>.>` (not the
@@ -2512,7 +2623,7 @@ export class CotalEndpoint extends EventEmitter {
                 }
                 else {
                     const closedEpoch = err.name === "ClosedConnectionError" || /^closed connection$/i.test(err.message);
-                    const timeout = err.name === "TimeoutError" || /timeout/i.test(err.message);
+                    const timeout = isTimeoutError(err);
                     const dyingEpochTimeout = timeout && (this.reconnecting || !this.nc || this.nc.isClosed());
                     // Cleanup of an ordered consumer: a delete timeout means the broker did not answer in time,
                     // not that the endpoint is unusable. The broker reaps an idle/ephemeral consumer anyway.
@@ -3034,6 +3145,9 @@ export class CotalEndpoint extends EventEmitter {
                 if (s.type === "reconnect") {
                     this.armAuthExpiryReconnectFence(nc);
                     this.emit("transport", { connected: true, server: s.server });
+                    for (const follower of this.activeGoalFollowers) {
+                        follower.reconnected(nc);
+                    }
                     continue;
                 }
                 if (s.type === "close") {
@@ -3395,12 +3509,41 @@ export class CotalEndpoint extends EventEmitter {
             }
         })().catch((e) => this.emit("error", e));
     }
+    /** The principal recorded as holding a shard's delivery lease, read by THIS process from the row,
+     *  or `undefined` when no row is live or the read failed.
+     *
+     *  THE POINT IS WHO ESTABLISHES THE FACT. Everything a requester learns from a request/reply rail
+     *  is something a responder chose to send, and the delivery-admin rail is queue-grouped, so any
+     *  process permitted to serve it decides what arrives. The lease row is the other kind of fact: a
+     *  KV key whose only writer is the `delivery` credential's own `lease.*` grant, read here under
+     *  THIS endpoint's credential, which no reply on any rail participates in. A caller comparing an
+     *  answerer against the holder needs that, rather than the answerer's word for which one it is.
+     *
+     *  `undefined` is a genuine unknown and is NEVER a statement that nobody holds the shard: a read
+     *  that throws (a denied or unreadable bucket, a broker that will not answer) must not collapse
+     *  into "nothing is there". Callers must treat it as a failure to determine (#1694). */
+    async deliveryLeaseHolder(shardIndex) {
+        try {
+            return (await this.readDeliveryLeaseEntry(shardIndex))?.info.holder;
+        }
+        catch {
+            return undefined;
+        }
+    }
     /** The lease row AND the KV revision it is at. The revision is the CAS token every renew and the
      *  CAS release are argued against, so a caller re-establishing ownership after a failed renew
      *  needs the BROKER's sequence, not the one it last cached: a renew can fail with its write
      *  already applied (a lost reply, a reconnect mid-request), which leaves the cached revision one
      *  behind forever and every subsequent CAS refused over a sequence this process itself moved ,
-     *  read as somebody else's takeover, which is the #1318 misreading in a second costume. */
+     *  read as somebody else's takeover, which is the #1318 misreading in a second costume.
+     *
+     *  ABSENT and UNREADABLE are different facts and this method keeps them apart (#1694): `undefined`
+     *  means NO row is live (no key, or DEL/PURGE), while a key that EXISTS but does not parse as a
+     *  {@link DeliveryLeaseInfo} THROWS. Folding the parse failure into `undefined` made a live-but-
+     *  corrupt row read as "nothing there", and the store-identity challenge's absence settlement
+     *  then certified a remint over a row it could not read. A caller that wants the old swallow
+     *  (an owner testing "is this provably MINE") still gets its safe answer: an unparseable row is
+     *  not provably anyone's, and catching here reads as `false` exactly as before. */
     async readDeliveryLeaseEntry(shardIndex) {
         const e = await (await this.deliveryRegistry()).get(leaseKey(shardIndex));
         if (!e || e.operation === "DEL" || e.operation === "PURGE")
@@ -3408,8 +3551,8 @@ export class CotalEndpoint extends EventEmitter {
         try {
             return { info: e.json(), revision: e.revision };
         }
-        catch {
-            return undefined;
+        catch (err) {
+            throw new Error(`delivery lease row "${leaseKey(shardIndex)}" exists but does not parse as a lease record (${err.message}) - it is unreadable, not absent`);
         }
     }
     /** Ensure + bind the manager singleton-lease bucket. Mirrors the presence-bucket pattern (connectAndBind):
@@ -4281,8 +4424,28 @@ export class CotalEndpoint extends EventEmitter {
                 return { ok: false, error: "reloadStoreIdentity: this daemon did not name the SecretStore it reloads from" };
             try {
                 const identity = this.plane3.reloadStoreIdentity();
+                // WHO IS ANSWERING is part of the answer. This rail is queue-grouped, so the broker hands
+                // the request to any bound responder, and every process with a `delivery` credential for
+                // the space can bind it, while only the DELIVERY LEASE HOLDER actually reloads the standing
+                // credentials. A bare store identity therefore lets a second responder's store stand in for
+                // the reloading process's, and the manager's remint decision is then made about a process
+                // that reloads nothing. So the reply carries this responder's wire identity and whether it
+                // holds the lease, and the caller requires the binding (#1694).
+                //
+                // The lease read is the LIVE row, not a cached claim: a holder that lost the shard must not
+                // keep asserting it. A read that fails is reported as `false` rather than thrown, because
+                // "I cannot prove I hold the shard" is what a non-holder's answer says, and it sends the
+                // caller down the same fail-closed path rather than inventing an authority.
+                let holdsDeliveryLease = false;
+                try {
+                    const own = await this.readDeliveryLeaseEntry(0);
+                    holdsDeliveryLease = own !== undefined && this.ownsDeliveryLease(own.info);
+                }
+                catch {
+                    holdsDeliveryLease = false;
+                }
                 // Round-trip through the closed parser so a hook cannot smuggle extra fields onto the rail.
-                return { ok: true, data: parseSecretStoreIdentity(identity) };
+                return { ok: true, data: { identity: parseSecretStoreIdentity(identity), responder: this.card.id, holdsDeliveryLease } };
             }
             catch (e) {
                 return { ok: false, error: e.message };
@@ -4542,15 +4705,20 @@ export class CotalEndpoint extends EventEmitter {
             return;
         if (!this.ownLifecycleUid)
             return; // no lifecycle uid — never provisioned for Plane-3 (its durable is lifecycle-keyed)
+        const durable = dlvDurable(this.owner, this.actor, this.ownLifecycleUid);
         let consumer;
         try {
-            consumer = await this.js.consumers.get(dlvStream(this.space), dlvDurable(this.owner, this.actor, this.ownLifecycleUid));
+            consumer = await this.js.consumers.get(dlvStream(this.space), durable);
         }
         catch (e) {
             if (isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
                 return;
             throw e; // a denied bind is not proof Plane-3 is absent
         }
+        const lease = await this.readDeliveryLease(0);
+        const liveDeliveryPlane = lease?.ready === true;
+        if (!liveDeliveryPlane)
+            this.emit("warning", new Error(`delivery durable "${durable}" for space "${this.space}" bound while the plane this connection reaches has no ready delivery lease, so the durable delivers nothing until a delivery daemon serves this plane. If a daemon serves this space on another plane, reconnect against it, which re-binds the durable there.`));
         const msgs = await consumer.consume();
         this.streamMsgs.push(msgs);
         void (async () => {
@@ -5302,6 +5470,8 @@ export class CotalEndpoint extends EventEmitter {
             if (epoch === this.presenceEpoch && !this.stopped && !superseded) {
                 this.presenceWriteFailingSince ??= Date.now();
                 this.lastPresenceWriteError = e?.message ?? String(e);
+                this.presenceWriteFailures++;
+                this.escalatePresenceWriteFailureIfStuck();
             }
             throw e;
         }
@@ -5340,6 +5510,21 @@ export class CotalEndpoint extends EventEmitter {
     clearPresenceWriteFailure() {
         this.presenceWriteFailingSince = undefined;
         this.lastPresenceWriteError = undefined;
+        this.presenceWriteFailures = 0;
+        this.presenceWriteEscalated = false;
+    }
+    /** A heartbeat retry remains recoverable inside one TTL. Once two or more consecutive writes have
+     * failed across the full window, ordinary retry has exhausted the roster's liveness budget: raise
+     * one named, non-transient warning. Later failures keep the duration/count current without flooding
+     * the operator log; the next successful write clears and re-arms the condition. */
+    escalatePresenceWriteFailureIfStuck() {
+        if (this.presenceWriteEscalated ||
+            this.presenceWriteFailingSince === undefined ||
+            this.presenceWriteFailures < 2 ||
+            Date.now() - this.presenceWriteFailingSince < this.ttlMs)
+            return;
+        this.presenceWriteEscalated = true;
+        this.emitRecoverable(new PresenceWriteStuckError(presenceBucket(this.space), this.presenceWriteFailingSince, this.presenceWriteFailures, this.ttlMs, this.lastPresenceWriteError));
     }
     /** #1356: the presence bucket has been refusing writes since this time, or `undefined` when the
      *  last publish succeeded. Cleared by a successful write that is the latest evidence on its
@@ -5357,6 +5542,8 @@ export class CotalEndpoint extends EventEmitter {
             forMs: Date.now() - this.presenceWriteFailingSince,
             error: this.lastPresenceWriteError,
             bucket: presenceBucket(this.space),
+            consecutiveFailures: this.presenceWriteFailures,
+            stuck: this.presenceWriteEscalated,
         };
     }
     /** Bind a presence watch on the current connection. Resolves true when the watch was
@@ -6316,6 +6503,14 @@ export async function isReachable(servers = DEFAULT_SERVER, opts = {}) {
         return e instanceof AuthorizationError || e instanceof UserAuthenticationExpiredError;
     }
 }
+/** True when `err` is a dial/consumer-op timeout rather than a real refusal — the one shared test
+ *  for "the operation ran out of its own budget", used both by {@link classifyProbeFailure} (#851:
+ *  a probe timeout must never collapse into `unreachable`, which a TLS-required target then
+ *  misreads as a trust failure) and by {@link Endpoint#disarmMembershipWatch}'s consumer-delete
+ *  cleanup, which predates it. */
+function isTimeoutError(err) {
+    return err instanceof Error && (err.name === "TimeoutError" || /timeout/i.test(err.message));
+}
 /** Like {@link isReachable}, but distinguishes "up but won't take these creds" from "nothing there".
  *  `spawn` needs the difference: auth-required → name the trust dir + next step; unreachable → the
  *  mesh is down (prune the stale entry, tell the user to `cotal up`). Pass `creds` to confirm a
@@ -6373,6 +6568,11 @@ function classifyProbeFailure(e, opts) {
     // The broker answered but rejected these creds (so it IS up) — auth-required, not stale-auth.
     if (e instanceof AuthorizationError)
         return { ok: false, reason: "auth-required" };
+    // A dial that ran out of its own budget is neither a refusal nor a dead broker — it is latency.
+    // `e` is undefined when the tcpDialable gate refused before any connect() attempt; that path has
+    // no timeout to inspect and must stay `unreachable` (nothing answered at all).
+    if (e !== undefined && isTimeoutError(e))
+        return { ok: false, reason: "timeout" };
     return { ok: false, reason: "unreachable" };
 }
 //# sourceMappingURL=endpoint.js.map