@cotal-ai/core 0.17.0 → 0.19.0

package/dist/endpoint.js

@@ -5,11 +5,13 @@ import { connect, credsAuthenticator, headers, tokenAuthenticator, nanos, Author
 import { credsClaims, credsFingerprint, credsRenewalDelayMs, idFromCreds } from "./identity.js";
 import { inspectCredHealth } from "./provision.js";
 import { resolveService, invokeCommand, submitAndFollowGoal } from "./endpoint-invoke.js";
-import { EpEnvelopeError } from "./endpoint-envelope.js";
+import { EpEnvelopeError, respondedButUnbound, replyRefusedBeforeEffect, EP_BIND_REFUSED } from "./endpoint-envelope.js";
+import { isRepeatSafeCommand } from "./endpoint-grants.js";
+import { assertIdToken } from "./endpoint-subjects.js";
 import { liveKvEntries } from "./kv-scan.js";
 import { ARTIFACT_PART_KIND, isArtifactPart } from "./artifact.js";
 import { assertValidName } from "./resolve.js";
-import { createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig, MAX_MSGS_PER_SUBJECT, MANAGER_LEASE_TTL_MS } from "./streams.js";
+import { createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig, MAX_MSGS_PER_SUBJECT, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS } from "./streams.js";
 import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, } from "@nats-io/jetstream";
 import { Kvm } from "@nats-io/kv";
 import { openMembersRegistry, commitMember, tombstoneMember, activateMember, readMember, listMembers, durableEligible, StaleMembershipWrite, } from "./members.js";
@@ -199,16 +201,34 @@ export class CotalEndpoint extends EventEmitter {
     owner;
     /** This endpoint's actor token (principal half 2) — the connection id in the dev default. */
     actor;
+    /** True when {@link actor} was SELF-MINTED at construction — a fresh random token, because the
+     *  card declared no actor and no id and no creds named one. Such a principal differs on every
+     *  restart, so nothing can be granted to it in advance and nothing durable may be keyed on it.
+     *  Exposed via {@link actorIsEphemeral} so a caller deriving a per-agent resource name can refuse
+     *  the mode instead of silently keying on a value that will not survive the process. */
+    actorIsEphemeral;
     /** This incarnation's lifecycle UID (opts.lifecycleUid) — see {@link EndpointOptions.lifecycleUid}. */
     ownLifecycleUid;
     /** Per-endpoint-name {@link resolveService} cache for {@link invokeService} — dropped on a
      *  `failed-precondition` currency refusal (the described incarnation was superseded). */
     resolvedServices = new Map();
+    /** How many calls {@link invokeService} has silently recovered from a bind refusal (§13.2) — the
+     *  class-queue splits this endpoint hit and survived.
+     *
+     *  Counted because it is recovered: handling the split is what makes it invisible, so this is the
+     *  only evidence the split rate exists. Always on, never behind a flag — a counter you have to
+     *  enable is not there when the thing you needed it for happened. */
+    splitsRecovered = 0;
     /** This endpoint's wire principal (owner + actor tokens, §13.2) — what its minted grant rows
      *  pin. Public so a caller can build owner-mode target blocks for {@link invokeService}. */
     get principal() {
         return { owner: this.owner, actor: this.actor };
     }
+    /** Class-queue splits this endpoint has hit and silently survived ({@link splitsRecovered}).
+     *  Pull it, or listen for `split-recovered` — the event can be missed, the count cannot. */
+    get splitRecoveryCount() {
+        return this.splitsRecovered;
+    }
     /** The endpoint's own lifecycle UID, REQUIRED for every lifecycle-keyed messaging resource; absent
      *  ⇒ loud refusal naming the operation (the hard cut of SPEC §13.1 — no alias-keyed fallback). */
     requireLifecycleUid(what) {
@@ -218,6 +238,9 @@ export class CotalEndpoint extends EventEmitter {
     }
     constructor(opts) {
         super();
+        /** Did the dev/static branch fall through to a random connId? Set on every path that assigns
+         *  `connId` so the ephemeral verdict below can never read an unassigned value. */
+        let selfMintedConnId = false;
         this.space = opts.space;
         // A display name is the client-side handle a peer is addressed by; reject the reserved `/`
         // (the future owner/name separator) and surrounding whitespace at the one identity choke
@@ -258,6 +281,10 @@ export class CotalEndpoint extends EventEmitter {
             }
             this.connId = assertInboxConnId(`ibx${randomUUID().replace(/-/g, "")}`);
             this.sentinelCreds = opts.sentinelCreds;
+            // User mode's actor is SERVER-AUTHORED (bearer claims or a declared card checked against
+            // them), never self-minted — the ephemeral value here is the inbox nonce, which is the
+            // connection id and not the principal.
+            this.actorIsEphemeral = false;
         }
         else {
             // DEV / STATIC. Connection identity precedence: an explicit card.id, else the creds' identity, else
@@ -272,16 +299,22 @@ export class CotalEndpoint extends EventEmitter {
                     throw new Error("a creds source requires an explicit card.id (no cred to derive the identity from at construction)");
                 this.credsSource = opts.creds;
                 this.connId = opts.card.id;
+                selfMintedConnId = false;
             }
             else {
                 const credId = opts.creds ? idFromCreds(opts.creds) : undefined;
                 if (opts.card.id && credId && opts.card.id !== credId)
                     throw new Error(`card.id ${opts.card.id} != creds identity ${credId} - they must be the same nkey`);
                 this.currentCreds = opts.creds;
+                selfMintedConnId = opts.card.id === undefined && credId === undefined;
                 this.connId = opts.card.id ?? credId ?? randomUUID().replace(/-/g, "");
             }
             this.owner = opts.card.owner ?? DEV_OWNER;
             this.actor = opts.card.actor ?? this.connId;
+            // The actor is EPHEMERAL only when it inherited a self-minted connId — a declared `card.actor`
+            // is stable even on an otherwise identity-less connection. Recorded here, at the one site the
+            // fallback fires, so no caller has to re-derive the precedence rule from the outside.
+            this.actorIsEphemeral = opts.card.actor === undefined && selfMintedConnId;
         }
         // The incarnation's lifecycle UID (SPEC §13.1). AUTH mode (JWT creds/bearer) REQUIRES the
         // launcher to supply it — the dm/dlv/chathist durable names must match the exact names the
@@ -924,6 +957,164 @@ export class CotalEndpoint extends EventEmitter {
         await this.publishMsg(chatSubject(this.space, this.owner, this.actor, channel), msg);
         return msg;
     }
+    /** The broker's live `max_payload` — the CEILING a frame is measured against, not a budget for a
+     *  caller's own payload.
+     *
+     *  Exposed because the connection is private and callers outside core (a connector assembling a
+     *  batched payload) otherwise have no way to learn the ceiling except by failing a publish.
+     *  Throws rather than guessing a default: a wrong ceiling is worse than none, because it splits
+     *  either too eagerly or too late and both look like working code.
+     *
+     *  THIS ALONE CANNOT SIZE A MESSAGE. The envelope this endpoint adds after the publish call, and
+     *  the client's own headers, are charged against the same ceiling and the caller never sees them.
+     *  Use {@link encodedSize}, which measures what will actually be sent. */
+    get maxPayload() {
+        const max = this.nc?.info?.max_payload;
+        if (typeof max !== "number" || !Number.isFinite(max) || max <= 0)
+            throw new Error(`${this.notLiveMsg()} - max_payload is only known while connected`);
+        return max;
+    }
+    /**
+     * Verify the PRECONDITION {@link multicastExpecting} depends on: that the chat stream evaluates
+     * the subject expectation BEFORE the `Nats-Msg-Id` dedup cache. **Throws if it cannot be
+     * guaranteed.** Call before the first serialized append on a given endpoint.
+     *
+     * **The ordering follows the stream's REPLICATION FACTOR, not the deployment.** A standalone R1
+     * stream and an R1 stream inside a real 3-node cluster both refuse a stale expectation with a CAS
+     * error; only an R3 stream evaluates dedup first and answers a retry with `duplicate: true`. A
+     * check written against cluster size would pass on exactly the deployment that breaks.
+     *
+     * Every stream Cotal creates is `num_replicas: 1`, from the same canonical config the restore path
+     * uses, so the property holds by construction today. This exists because "by construction" is an
+     * observation until something checks it: nothing in the wire contract reserves the replica factor.
+     * A caller appending under a stale assumption does not fail loudly; it accepts a retry as success
+     * and drops a message.
+     *
+     * Evidence is `smoke:cas-preflight-cluster`, which records the server version it measured against
+     * rather than naming one here — the suites resolve `nats-server` from `PATH`, so a hardcoded
+     * provenance ages into a claim about a machine that no longer exists.
+     *
+     * @throws if the stream is unreadable (no `STREAM.INFO` grant, or absent) or reports more than
+     *   one replica. Never degrades to a warning: the failure it prevents is silent.
+     */
+    async assertExpectationSemantics() {
+        if (!this.jsm)
+            throw new Error(this.notLiveMsg());
+        const stream = chatStream(this.space);
+        let replicas;
+        try {
+            replicas = (await this.jsm.streams.info(stream)).config.num_replicas;
+        }
+        catch (e) {
+            throw new Error(`cannot verify expectation semantics: stream "${stream}" info unavailable (${e.message}). ` +
+                `Serialized appends are refused rather than run on an unverified stream.`);
+        }
+        // `undefined` is NOT treated as 1. A server that does not report the field is a server whose
+        // ordering we have not established, which is the case this check exists for.
+        if (replicas !== 1)
+            throw new Error(`stream "${stream}" reports num_replicas=${String(replicas)}; serialized appends on THIS ` +
+                `stream require 1 — a property of this one stream, not of the broker, so a clustered ` +
+                `deployment is fine so long as this stream is R1, which a cluster can host. ` +
+                `On a replicated stream the dedup cache is consulted before the subject expectation, so a ` +
+                `retry returns a duplicate ack instead of a conflict and a lost message reads as success.`);
+    }
+    /** The envelope {@link multicastExpecting} publishes, built in ONE place so that a frame and any
+     *  measurement of that frame cannot describe different messages. The fields this adds — `ts`,
+     *  `space`, `from`, `channel` and the normalized `mentions` — are exactly the ones a caller
+     *  holding only its parts cannot account for. */
+    casEnvelope(opts) {
+        return {
+            id: opts.id,
+            ts: Date.now(),
+            space: this.space,
+            from: this.ref(),
+            channel: opts.channel,
+            mentions: normalizeMentions(opts.mentions),
+            parts: opts.parts,
+            replyTo: opts.replyTo,
+            contextId: opts.contextId,
+        };
+    }
+    /**
+     * The bytes this frame will ACTUALLY put on the wire, to compare against {@link maxPayload}.
+     *
+     * Caller-side arithmetic is wrong in the dangerous direction: a split sized against the caller's
+     * own payload produces a frame the broker REJECTS, and a rejected truncation makes the loss silent
+     * again — the failure splitting exists to prevent.
+     *
+     * It lives on the surface that BUILDS the envelope so measurement and construction cannot drift
+     * apart unnoticed: it shares {@link casEnvelope} with the publish path, sets the same two headers,
+     * and lets the client's own encoder encode them rather than re-implementing the wire format.
+     * `frame-size.smoke.ts` binary-searches a real broker's ceiling and requires this number to land
+     * on it exactly.
+     *
+     * `expectedLastSubjectSeq` is a parameter because it is a header VALUE: sizing at 0 and publishing
+     * at 123456 differ by five bytes.
+     *
+     * Residual: `ts` is re-stamped at publish, so the two differ in value — not in length until
+     * epoch-millis needs a 14th digit.
+     */
+    encodedSize(opts) {
+        // The same argument validation the publish path applies, so a caller cannot size a frame that
+        // would have been refused before it ever reached the wire.
+        if (!isConcreteChannel(opts.channel))
+            throw new Error(`cannot publish to wildcard channel "${opts.channel}" - pick a concrete sub-channel`);
+        assertIdToken(opts.id, "publish id");
+        if (!Number.isSafeInteger(opts.expectedLastSubjectSeq) || opts.expectedLastSubjectSeq < 0)
+            throw new Error(`expectedLastSubjectSeq must be a non-negative safe integer, got ${JSON.stringify(opts.expectedLastSubjectSeq)}`);
+        if (!Array.isArray(opts.parts) || opts.parts.length === 0)
+            throw new Error("encodedSize requires at least one part");
+        const mh = headers();
+        mh.set("Nats-Msg-Id", opts.id);
+        mh.set("Nats-Expected-Last-Subject-Sequence", `${opts.expectedLastSubjectSeq}`);
+        // `encode()` is on the client's header implementation but not on the published `MsgHdrs` type,
+        // so it is reached through a cast rather than copied. If a client version drops it, this throws
+        // immediately and loudly — the calibration cell would also fail — instead of returning a number
+        // that is quietly wrong near the ceiling.
+        const headerBytes = mh.encode().length;
+        return headerBytes + Buffer.byteLength(JSON.stringify(this.casEnvelope(opts)), "utf8");
+    }
+    /**
+     * Multicast with an OPTIMISTIC-CONCURRENCY expectation and a caller-chosen dedup id, returning
+     * the `PubAck` fields instead of discarding them. The serialized-append primitive: two writers
+     * racing one subject cannot interleave, because the loser's expectation no longer holds.
+     *
+     * **Why a separate method rather than options on {@link multicast}.** `multicast` mints a fresh
+     * `id` per call and drops the ack; both are right for ordinary chat and both are fatal to a
+     * caller that must retry an append idempotently. Keeping them apart means no existing caller
+     * changes behaviour, and the stricter validation below applies only where a caller opted in.
+     *
+     * - `id` becomes the JetStream `Nats-Msg-Id`, so the SAME id may be republished on retry and the
+     *   server dedups it within the stream's duplicate window. It is validated rather than trusted:
+     *   it lands in a wire header, and the dedup cache is **stream-wide**, so a caller-supplied id is
+     *   both an injection surface and a way to suppress another publisher's message.
+     * - `expectedLastSubjectSeq` is the sequence this publisher believes is the subject's tip; `0`
+     *   means "the subject must be empty". A mismatch throws, and the throw stays classifiable by the
+     *   already-public {@link isCasLoss} — the error is deliberately **not wrapped**, since wrapping
+     *   would hide the `err_code` that classification reads.
+     *
+     * @throws if the endpoint is not live, the channel is not concrete, `id` is malformed, `parts` is
+     *   empty, or `expectedLastSubjectSeq` is not a non-negative safe integer.
+     */
+    async multicastExpecting(opts) {
+        if (!this.js)
+            throw new Error(this.notLiveMsg());
+        if (!isConcreteChannel(opts.channel))
+            throw new Error(`cannot publish to wildcard channel "${opts.channel}" - pick a concrete sub-channel`);
+        // Reuse the existing id grammar rather than mint a second one: [A-Za-z0-9_-]{1,64} admits a
+        // UUID and rejects every character that could break a wire header (CR, LF, space, colon).
+        assertIdToken(opts.id, "publish id");
+        const expected = opts.expectedLastSubjectSeq;
+        if (!Number.isSafeInteger(expected) || expected < 0)
+            throw new Error(`expectedLastSubjectSeq must be a non-negative safe integer, got ${JSON.stringify(expected)}`);
+        if (!Array.isArray(opts.parts) || opts.parts.length === 0)
+            throw new Error("multicastExpecting requires at least one part");
+        const message = this.casEnvelope(opts);
+        // Publish DIRECTLY rather than through publishMsg: this path must set the expectation and read
+        // the ack, and publishMsg deliberately does neither.
+        const ack = await this.js.publish(chatSubject(this.space, this.owner, this.actor, opts.channel), JSON.stringify(message), { msgID: opts.id, expect: { lastSubjectSequence: expected } });
+        return { message, ack: { seq: ack.seq, duplicate: ack.duplicate === true } };
+    }
     /** Unicast: direct message to one specific instance. */
     async unicast(instanceId, text, opts) {
         const msg = {
@@ -1071,11 +1262,22 @@ export class CotalEndpoint extends EventEmitter {
     /** GENERIC v0.4 service invoke over this endpoint's own connection (P2 item 1, 1c.2b): resolve
      *  the named endpoint's registered surface — describe, §13.7 store fetch, digest-verified
      *  recompile ({@link resolveService}; cached per endpoint name) — and invoke one command. The
-     *  resolve is describe-bound currency: when a DIFFERENT instance answers a later invoke
-     *  (`failed-precondition`, a restart/supersede), the cache is dropped and resolved ONCE more
-     *  against the current incarnation. Errors from the responder come back structurally on the
-     *  attributed reply (`reply.ok === false`); transport/validation refusals throw
-     *  {@link EpEnvelopeError}. */
+     *  resolve is describe-bound currency, and a call that reaches the wrong incarnation is recovered
+     *  two ways depending on WHO caught it — the difference between knowing the command did not run
+     *  and only knowing someone answered:
+     *   - the RESPONDER fenced it on the request's `bind` (§13.2, an `ok:false` reply marked
+     *     {@link replyRefusedBeforeEffect}): the command did not run, so the bind is dropped and the
+     *     call re-issued ONCE for any command. If that re-issue cannot be resolved, the refusal
+     *     surfaces — still saying the command did not run — naming the resolve failure as why the
+     *     repair could not be attempted.
+     *   - this CLIENT caught it on the reply ({@link respondedButUnbound}: a different instance,
+     *     `failed-precondition`; the same instance at any other epoch, `expired`), which is what a
+     *     responder too old to know the field produces. A live instance received and answered it, so
+     *     the bind is dropped but the call is re-issued only for a command on the
+     *     {@link isRepeatSafeCommand} allowlist; anything else surfaces, since a second attempt could
+     *     duplicate its effect.
+     *  Errors from the responder come back structurally on the attributed reply
+     *  (`reply.ok === false`); transport/validation refusals throw {@link EpEnvelopeError}. */
     async invokeService(endpoint, command, args, opts = {}) {
         if (!this.nc)
             throw new Error(this.notLiveMsg());
@@ -1092,13 +1294,111 @@ export class CotalEndpoint extends EventEmitter {
         const invokeOpts = { ...(opts.target ? { target: opts.target } : {}), ...(opts.deadlineMs !== undefined ? { deadlineMs: opts.deadlineMs } : {}) };
         const doInvoke = async () => {
             try {
-                return await invokeCommand(nc, this.space, await resolve(), command, args, invokeOpts);
+                const r = await invokeCommand(nc, this.space, await resolve(), command, args, invokeOpts);
+                // THE RESPONDER FENCED IT (§13.2 `ai.cotal.ep.bind-refused`): a class member saw the call
+                // was bound to a different incarnation and refused BEFORE running the command.
+                //
+                // Handled here rather than below because it arrives as an ordinary `ok:false` REPLY, not a
+                // throw, so the `respondedButUnbound` recovery never sees it — a long-lived client would
+                // otherwise keep its stale bind and meet the same refusal forever.
+                //
+                // The re-issue is NOT gated on {@link isRepeatSafeCommand}: the responder states the command
+                // did not run, so this is a FIRST attempt, not a second. Exactly once; a second refusal
+                // surfaces.
+                if (r.reply.ok === false && replyRefusedBeforeEffect(r.reply.error)) {
+                    // Counted before it is repaired: a recovery that leaves no trace takes the split rate with it.
+                    this.splitsRecovered++;
+                    // `boundTo` is the other half of `servedBy`: who the handle THOUGHT it was talking to,
+                    // against who actually answered. Without it a listener sees that a split was recovered
+                    // but not which bind went stale, so it cannot tell one handle's repeated staleness from
+                    // splits spread across many — and that is the difference between a handle to drop and a
+                    // class that is churning.
+                    this.emit("split-recovered", {
+                        endpoint, command, servedBy: r.responder, splitsRecovered: this.splitsRecovered,
+                        boundTo: (r.reply.error?.details ?? []).find((d) => d.kind === EP_BIND_REFUSED)?.boundTo,
+                    });
+                    this.resolvedServices.delete(endpoint);
+                    // A FAILED RE-ISSUE RETHROWS THE ORIGINAL REFUSAL, not the resolve error: if the endpoint
+                    // has since retired, the resolve times out and the caller would otherwise get `no describe
+                    // reply within 10000ms` — about a describe it never asked for — losing the one fact that
+                    // says its handle is stale AND that nothing ran. The resolve failure is named as the
+                    // reason the repair could not be attempted.
+                    //
+                    // The code must be the refusal's own (`failed-precondition` for a different instance,
+                    // `expired` for the same instance at ANY other epoch — the fence does not compare
+                    // direction, so a superseded incarnation still answering produces it too); the marker on
+                    // any other code is incoherent, so it is rejected rather than trusted (§13.3).
+                    const raw = r.reply.error?.code;
+                    const refusalCode = raw === "expired" ? "expired"
+                        : raw === "failed-precondition" ? "failed-precondition" : undefined;
+                    if (refusalCode === undefined)
+                        throw new EpEnvelopeError("internal", `${endpoint}.${command} came back marked as refused before it ran, but with code ${String(raw)}; the fence produces only failed-precondition or expired (SPEC 13.2)`, r.reply.error?.details);
+                    // ONLY THE RESOLVE IS WRAPPED — do not widen this `try` to cover the re-issue. A re-issue
+                    // that PUBLISHED AND RAN can still throw (an unfenced responder answers and the post-reply
+                    // currency check raises `respondedButUnbound`); wrapping that as the first hop's refusal
+                    // would hand the caller `WAS NOT RUN` for a command that executed, and its next attempt
+                    // would be a second one believing it was the first. A re-issue that fails on its own terms
+                    // must propagate its OWN error, because `respondedButUnbound` asserts the opposite of what
+                    // the fence's marker does: a responder answered and the effect may have landed.
+                    let reissueTarget;
+                    try {
+                        reissueTarget = await resolve();
+                    }
+                    catch (reissue) {
+                        throw new EpEnvelopeError(refusalCode, `${endpoint}.${command} WAS NOT RUN - the incarnation that received it refused it before any effect, and the re-issue could not be resolved: ${reissue instanceof Error ? reissue.message : String(reissue)}. Re-resolve and re-issue when the endpoint is reachable (SPEC 13.2)`, r.reply.error?.details, 
+                        // §13.3: the message asserts the command did not run, so the FIELD a caller keys on
+                        // must assert it too. Omitted, it MUST be read as `unknown`, which is the opposite of
+                        // what this path knows: the responder fenced the call before any effect and the
+                        // re-issue never went out. Prose is for the reader; this is for the machine.
+                        "not-executed");
+                    }
+                    return await invokeCommand(nc, this.space, reissueTarget, command, args, invokeOpts);
+                }
+                return r;
             }
             catch (e) {
-                if (!(e instanceof EpEnvelopeError) || e.code !== "failed-precondition")
+                if (!(e instanceof EpEnvelopeError))
+                    throw e;
+                // DO NOT auto-retry a command a responder already ANSWERED. This path covers the responders
+                // WITHOUT the fence above: they ignore `bind`, run the command, and the error is raised
+                // afterwards, so core cannot tell a repair from a duplicate and the allowlist is the only
+                // guard left. (`failed-precondition` here is not only supersession — it also fires when a
+                // DIFFERENT live instance wins the class queue and replies, an ordinary split.)
+                //
+                // Dropping the retry outright is not an option: in a two-manager space roughly half of all
+                // class-queue calls split, so every other `ps` would surface an error this absorbs, and
+                // re-running a read costs nothing.
+                //
+                // So it is gated on an ALLOWLIST (`REPEAT_SAFE_COMMANDS`), keyed by ENDPOINT because this
+                // method is endpoint-agnostic. Polarity matters more than membership: an allowlist fails
+                // CLOSED. Not a `GOAL_BEARING_COMMANDS`-only gate — `purge` is neither goal-bearing nor
+                // convergent, and its second run deletes messages published after the first completed, so
+                // the boundary is "changes anything", which core cannot see: commands carry no idempotency
+                // declaration and every manager command shares `class: "ephemeral"`.
+                //
+                // Keyed on the MARKER, not the error code, because the same fact has two producers: a
+                // DIFFERENT instance answering (`failed-precondition`) and the SAME instance at ANY OTHER
+                // EPOCH (`expired`) — epoch inequality, not "a successor", so a superseded incarnation
+                // still answering produces it too. Same rule for both, or a long-lived client stays bound
+                // to a dead epoch while every call reaches whoever is actually there.
+                if (respondedButUnbound(e)) {
+                    // Drop the stale bind FIRST, whichever way this goes: it names an incarnation that is not
+                    // the one answering, so every later call would reuse it and meet the same refusal. This is
+                    // not a retry — nothing is re-issued by it, and for a command that is not repeat-safe the
+                    // next call is the caller's, made after it has verified.
+                    this.resolvedServices.delete(endpoint);
+                    if (!isRepeatSafeCommand(endpoint, command))
+                        throw e;
+                    return await invokeCommand(nc, this.space, await resolve(), command, args, invokeOpts);
+                }
+                // An UNMARKED `failed-precondition` is the resolve's own refusal, raised before any command
+                // was published, so re-resolving once is a repair. The `replyRefusedBeforeEffect` half keeps
+                // out the refusal THIS method raises after a failed re-issue: it carries that same code, and
+                // would otherwise fall into the re-resolve below as a THIRD attempt at a command whose
+                // second could not even be resolved. A marker means the disposition is already decided,
+                // whichever code carries it.
+                if (e.code !== "failed-precondition" || replyRefusedBeforeEffect(e.toEpError()))
                     throw e;
-                // The describe-bound incarnation is gone (restart/supersede) — re-resolve once, then invoke
-                // against the CURRENT one; a second failure is the real state and throws.
                 this.resolvedServices.delete(endpoint);
                 return await invokeCommand(nc, this.space, await resolve(), command, args, invokeOpts);
             }
@@ -1735,7 +2035,7 @@ export class CotalEndpoint extends EventEmitter {
         await this.ensureDmDurable(owner, actor, lifecycleUid, {});
     }
     /** Idempotent-PER-LIFECYCLE create of a `dm_<o>-<a>-<uid>` durable with its ACTIVATION FRONTIER
-     *  (SPEC :467). Info-first: an existing durable (a manager-restart re-provision of the SAME uid, or
+     *  (SPEC §8). Info-first: an existing durable (a manager-restart re-provision of the SAME uid, or
      *  the same lifecycle's own restart) is kept as-is, preserving the ORIGINAL frontier — the
      *  activation moment never moves. A fresh lifecycle captures the DM stream's current `last_seq` and
      *  starts delivery at frontier+1, so a same-alias successor inherits none of the predecessor's
@@ -1921,7 +2221,11 @@ export class CotalEndpoint extends EventEmitter {
             throw new Error("endpoint not started");
         if (this.managerLeaseKv)
             return this.managerLeaseKv;
-        const kvm = new Kvm(this.nc);
+        // A JetStream client of its own, so every operation on this bucket carries the lease budget's
+        // attempt deadline rather than the library default. The default is TTL/2, which would let one
+        // attempt spend the whole renew window (see MANAGER_LEASE_ATTEMPT_MS). Scoped to this bucket:
+        // every op on it is a single small keyed request, and nothing else shares this client.
+        const kvm = new Kvm(jetstream(this.nc, { timeout: MANAGER_LEASE_ATTEMPT_MS }));
         if (this.authed) {
             this.managerLeaseKv = await kvm.open(managerBucket(this.space));
         }
@@ -1952,6 +2256,18 @@ export class CotalEndpoint extends EventEmitter {
     async renewManagerLease(info, revision) {
         return (await this.managerLeaseRegistry()).update(managerLeaseKey(info.instanceId), this.encodeManagerLease({ ...info, since: Date.now() }), revision);
     }
+    /** Read THIS instance's OWN lease key, keyed (not the `lease.*` sweep {@link readManagerLease} does).
+     *
+     *  `undefined` means the key IS NOT THERE — a definite absence, established by a completed read.
+     *  A read that could not be completed THROWS instead, so a caller can tell "it is gone" from "I could
+     *  not find out". That distinction is the whole point of the method: a renew that got no answer has
+     *  proved nothing, and only a definite answer here may be acted on. */
+    async readOwnManagerLease(instanceId) {
+        const e = await (await this.managerLeaseRegistry()).get(managerLeaseKey(instanceId));
+        if (!e || e.operation !== "PUT")
+            return undefined;
+        return { info: JSON.parse(new TextDecoder().decode(e.value)), revision: e.revision };
+    }
     /** Release THIS instance's key on clean shutdown so a same-id restart re-acquires immediately. CAS-guarded
      *  by `revision`: if we already LOST it (renew gap) the stored revision has moved, the conditional delete
      *  no-ops. Keyed per instance, so a release NEVER touches a sibling manager's key (security pin 6). */
@@ -2240,7 +2556,7 @@ export class CotalEndpoint extends EventEmitter {
     async startPlane3(aclFor, opts = {}) {
         if (!this.js)
             throw new Error("endpoint not started");
-        this.plane3 = { aclFor, reloadMembershipCreds: opts.reloadMembershipCreds, evictPrincipal: opts.evictPrincipal, planeConnLiveness: opts.planeConnLiveness };
+        this.plane3 = { aclFor, reloadMembershipCreds: opts.reloadMembershipCreds, evictPrincipal: opts.evictPrincipal, planeConnLiveness: opts.planeConnLiveness, principalLiveness: opts.principalLiveness };
         await this.armPlane3();
     }
     /** Serve one runtime durable-membership control request (the server-side delivery daemon). The caller
@@ -2562,6 +2878,24 @@ export class CotalEndpoint extends EventEmitter {
                 return { ok: false, error: e.message };
             }
         }
+        if (req.op === "principalLiveness") {
+            // The freeze-holder liveness probe (#391): the READ-ONLY half of `evictPrincipal`. A repair
+            // that must REFUSE while the holder is alive cannot use eviction as its own precheck — that
+            // kills the holder before anything can refuse on its behalf. Same executor-hook shape as the
+            // verbs above (the $SYS observer cred lives outside this trust boundary); absent hook =
+            // refused loudly, and the caller maps a refusal to UNKNOWN and never repairs over it.
+            if (!this.plane3?.principalLiveness)
+                return { ok: false, error: "principalLiveness: no liveness oracle wired on this daemon" };
+            const principal = typeof req.args?.principal === "string" ? req.args.principal.trim() : "";
+            if (!principal)
+                return { ok: false, error: "principalLiveness: a principal (owner.actor dot-form) is required" };
+            try {
+                return { ok: true, data: await this.plane3.principalLiveness(principal) };
+            }
+            catch (e) {
+                return { ok: false, error: e.message };
+            }
+        }
         return { ok: false, error: `op "${req.op}" not supported on the delivery admin service` };
     }
     /** Fan-out loop: bind the privileged `fanout` durable on CHAT and route each message (routing only —
@@ -3465,7 +3799,7 @@ export class CotalEndpoint extends EventEmitter {
             return; // observers watch but never publish their own record
         const p = {
             card: this.card,
-            // SPEC §6/:315: presence carries the incarnation's lifecycle UID (MUST in auth mode from v0.4);
+            // SPEC §6: presence carries the incarnation's lifecycle UID (MUST in auth mode from v0.4);
             // omitted only where the endpoint has none (a pure operator/daemon connection never registers).
             ...(this.ownLifecycleUid !== undefined ? { lifecycleUid: this.ownLifecycleUid } : {}),
             status: this.status,
@@ -3806,6 +4140,33 @@ export function isPermissionDenied(e) {
         return true;
     return /permissions?\s+violation/i.test(String(e?.message ?? ""));
 }
+/** True ONLY for a denial on a **publish** — the single case that proves the message was never
+ *  ACCEPTED or stored. (Not "never reached the server": the server necessarily received enough of
+ *  it to reject it. The distinction matters precisely here, because this helper exists to separate
+ *  provably-not-stored from possibly-stored, and the looser phrasing overstates the very thing
+ *  being measured.) {@link isPermissionDenied} deliberately does not look at the operation: it exists to
+ *  separate "denied" from "service down", and that answer is the same either way. The operation
+ *  matters enormously to a caller that reports *delivery*, because a JetStream publish is
+ *  request/PubAck and the subscription half is the reply inbox — a denial THERE rejects
+ *  `js.publish()` while the stream may already hold the message. Verified against a live broker: a
+ *  user allowed to publish but denied its `_INBOX` subscription got
+ *  `Permissions Violation for Subscription to "_INBOX.….*"` back from `js.publish()`, and an
+ *  unrestricted observer then read `messages: 1` off the stream.
+ *
+ *  Note what is deliberately NOT accepted: the untyped text fallback above. A permission-shaped
+ *  message string carries no operation, so it cannot prove non-delivery, and guessing "publish"
+ *  from wording would reintroduce exactly the false certainty this exists to prevent. Anything not
+ *  provably a publish denial is unknown, and a caller reporting delivery must fail toward
+ *  "I could not confirm" rather than toward "it did not happen" — the costly mistake is telling
+ *  someone to re-send a message that was in fact stored. */
+export function isPublishPermissionDenied(e) {
+    const typed = e instanceof PermissionViolationError
+        ? e
+        : e?.cause instanceof PermissionViolationError
+            ? (e.cause)
+            : undefined;
+    return typed?.operation === "publish";
+}
 /** Parse a NATS server URL (`nats://host:port`, `host:port`, a bare host, or a comma list — the
  *  first entry wins) into a host+port for {@link tcpInfoProbe}. Defaults the port to 4222. */
 function hostPort(server) {
@@ -3873,6 +4234,51 @@ function tcpInfoProbe(server, timeoutMs) {
         socket.on("close", () => finish(false)); // closed before a full INFO line arrived
     });
 }
+/** Bounded TCP reachability on a socket we OWN: does a handshake to `server` complete within
+ *  `timeoutMs`? Deliberately narrower than {@link tcpInfoProbe} — it asks only whether the
+ *  transport can be reached, never whether NATS is speaking there, so the TLS-first listener that
+ *  `tcpInfoProbe` reports false for still passes this gate and goes on to a real connect.
+ *
+ *  It exists because `connect()` cannot be trusted to release a connection it never established.
+ *  `@nats-io/transport-node`'s `NodeTransport.dial()` keeps its socket in a local until the
+ *  handshake resolves (`this.socket = await this.dial(hp)`), so `this.socket` is STILL UNDEFINED
+ *  when the client's own connect timeout wins the race in `protocol.ts`'s `dial` and the catch
+ *  calls `transport.close()` — whose teardown is `this.socket?.destroy()`, i.e. a destroy of
+ *  nothing. Against an address that BLACKHOLES (SYN unanswered) rather than REFUSES (RST), the
+ *  socket is orphaned in libuv until the OS SYN timeout, and the process cannot exit for minutes
+ *  after the probe already returned its answer. That is issue #389, and it is upstream: nothing a
+ *  caller passes (`reconnect: false`, `timeout`) reaches the orphan. Our socket, our `destroy()`,
+ *  on every exit path — never an `unref`/force-exit, which would hide the symptom and a future
+ *  real hang with it. */
+function tcpDialable(server, timeoutMs) {
+    return new Promise((resolve) => {
+        let socket;
+        let done = false;
+        const finish = (dialable) => {
+            if (done)
+                return;
+            done = true;
+            try {
+                socket?.destroy();
+            }
+            catch { /* already gone */ }
+            resolve(dialable);
+        };
+        let host, port;
+        try {
+            ({ host, port } = hostPort(server));
+        }
+        catch {
+            return resolve(false);
+        }
+        socket = createConnection({ host, port });
+        socket.setTimeout(timeoutMs);
+        socket.on("connect", () => finish(true));
+        socket.on("timeout", () => finish(false)); // blackhole: SYN unanswered inside our deadline
+        socket.on("error", () => finish(false)); // refused / reset / DNS failure
+        socket.on("close", () => finish(false));
+    });
+}
 /** Whether a NATS server is *running* at `servers`. With NO creds this is a SILENT plaintext
  *  liveness check ({@link tcpInfoProbe}): it reads the server's pre-auth `INFO` greeting and closes
  *  WITHOUT authenticating, so a live broker (open OR auth — INFO precedes auth) returns true while
@@ -3882,12 +4288,22 @@ function tcpInfoProbe(server, timeoutMs) {
  *  refuses these creds is still up — so the caller surfaces the real auth failure, and `up` won't
  *  start a duplicate on the bound port). Only a genuine connection failure (refused/timeout) is false. */
 export async function isReachable(servers = DEFAULT_SERVER, opts = {}) {
+    const timeoutMs = opts.timeoutMs ?? 1000;
     if (!opts.creds && !opts.token && !opts.user && !opts.pass && !opts.tls)
-        return tcpInfoProbe(servers, opts.timeoutMs ?? 1000);
+        return tcpInfoProbe(servers, timeoutMs);
+    // The credless branch above already owns its socket. This one reaches `connect()`, so it carries
+    // the same orphaned-socket defect probeConnect did (#389) and takes the same gate: reach the
+    // address on a socket we own first, and give `connect()` the remainder of the budget its own
+    // timeout always covered. A gate failure is a genuine connection failure, which is exactly the
+    // `false` the catch below already returns for one — an auth rejection cannot reach us from an
+    // address that never completed a handshake.
+    const started = Date.now();
+    if (!(await tcpDialable(servers, timeoutMs)))
+        return false;
     try {
         const nc = await connect({
             servers,
-            timeout: opts.timeoutMs ?? 1000,
+            timeout: Math.max(1, timeoutMs - (Date.now() - started)),
             reconnect: false,
             maxReconnectAttempts: 0,
             ...authOpts(opts),
@@ -3905,10 +4321,22 @@ export async function isReachable(servers = DEFAULT_SERVER, opts = {}) {
  *  specific identity is accepted (`ok`); omit them to probe mere liveness (an auth broker answers
  *  `auth-required`, which still proves it's up). */
 export async function probeConnect(server = DEFAULT_SERVER, opts = {}) {
+    const timeoutMs = opts.timeoutMs ?? 1000;
+    const started = Date.now();
+    // Reach the address on a socket we own BEFORE handing it to `connect()`, which orphans the
+    // connection it never established (see {@link tcpDialable} for the upstream mechanism, #389).
+    // This cannot change any verdict: every address that gets past here had to complete a TCP
+    // handshake for `connect()` to have gotten anywhere either, and a gate failure is routed through
+    // the SAME classification the catch uses — so a locally-dead cred is still `stale-auth` and not
+    // silently downgraded to `unreachable` by the address being dark. The cost is one extra
+    // handshake on the reachable path; the deadline below is the REMAINDER of the budget, because
+    // `connect()`'s own `timeout` always covered its handshake too.
+    if (!(await tcpDialable(server, timeoutMs)))
+        return classifyProbeFailure(undefined, opts);
     try {
         const nc = await connect({
             servers: server,
-            timeout: opts.timeoutMs ?? 1000,
+            timeout: Math.max(1, timeoutMs - (Date.now() - started)),
             reconnect: false,
             maxReconnectAttempts: 0,
             ...authOpts(opts),
@@ -3917,26 +4345,33 @@ export async function probeConnect(server = DEFAULT_SERVER, opts = {}) {
         return { ok: true };
     }
     catch (e) {
-        // A presented cred that is PROVABLY expired by its own JWT is stale-auth (credential death) —
-        // and that is knowable LOCALLY, without the network, so it is decided FIRST, before the error
-        // type. On the wire the broker's rejection and the socket close race: a slow CI/Windows handshake
-        // can surface a bare transport failure (→ "unreachable") instead of a clean AuthorizationError,
-        // which used to misclassify a dead cred. Reading the cred removes that timing dependency entirely,
-        // so the classification is deterministic. Unreadable content falls through to the wire truth (no
-        // false stale diagnosis from garbage).
-        if (typeof opts.creds === "string") {
-            try {
-                if (inspectCredHealth(opts.creds).state === "expired")
-                    return { ok: false, reason: "stale-auth" };
-            }
-            catch { /* not introspectable — keep the wire truth */ }
+        return classifyProbeFailure(e, opts);
+    }
+}
+/** Why a {@link probeConnect} attempt did not end in `ok`. Shared by the pre-connect reachability
+ *  gate and the connect catch so both classify identically — the gate must never turn a diagnosable
+ *  credential death into a bare `unreachable` just because the address went dark.
+ *
+ *  A presented cred that is PROVABLY expired by its own JWT is stale-auth (credential death) — and
+ *  that is knowable LOCALLY, without the network, so it is decided FIRST, before the error type. On
+ *  the wire the broker's rejection and the socket close race: a slow CI/Windows handshake can
+ *  surface a bare transport failure (→ "unreachable") instead of a clean AuthorizationError, which
+ *  used to misclassify a dead cred. Reading the cred removes that timing dependency entirely, so the
+ *  classification is deterministic. Unreadable content falls through to the wire truth (no false
+ *  stale diagnosis from garbage). `e` is undefined when the gate refused before any connect. */
+function classifyProbeFailure(e, opts) {
+    if (typeof opts.creds === "string") {
+        try {
+            if (inspectCredHealth(opts.creds).state === "expired")
+                return { ok: false, reason: "stale-auth" };
         }
-        if (e instanceof UserAuthenticationExpiredError)
-            return { ok: false, reason: "stale-auth" };
-        // The broker answered but rejected these creds (so it IS up) — auth-required, not stale-auth.
-        if (e instanceof AuthorizationError)
-            return { ok: false, reason: "auth-required" };
-        return { ok: false, reason: "unreachable" };
+        catch { /* not introspectable — keep the wire truth */ }
     }
+    if (e instanceof UserAuthenticationExpiredError)
+        return { ok: false, reason: "stale-auth" };
+    // The broker answered but rejected these creds (so it IS up) — auth-required, not stale-auth.
+    if (e instanceof AuthorizationError)
+        return { ok: false, reason: "auth-required" };
+    return { ok: false, reason: "unreachable" };
 }
 //# sourceMappingURL=endpoint.js.map