@cotal-ai/core 0.58.0 → 0.60.0

package/dist/endpoint.js

@@ -3,6 +3,7 @@ import { randomUUID } from "node:crypto";
 import { createConnection } from "node:net";
 import { connect, credsAuthenticator, headers, tokenAuthenticator, nanos, AuthorizationError, PermissionViolationError, UserAuthenticationExpiredError, NoRespondersError, RequestError, } from "@nats-io/transport-node";
 import { wsconnect } from "@nats-io/nats-core";
+import { parseLivenessAnswer, responderFromLease, responderFromProbe, isLivenessPlane, LIVENESS_PLANES, } from "./liveness.js";
 import { credsClaims, credsFingerprint, credsRenewalDelayMs, idFromCreds } from "./identity.js";
 import { requireBrokerFloor } from "./broker-floor.js";
 import { inspectCredHealth } from "./provision.js";
@@ -15,16 +16,17 @@ import { readAcceptedRow } from "./issued-authority.js";
 import { liveKvEntries } from "./kv-scan.js";
 import { ARTIFACT_PART_KIND, isArtifactPart } from "./artifact.js";
 import { assertValidName } from "./resolve.js";
-import { createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig, MAX_MSGS_PER_SUBJECT, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS, TTL_RECONCILE_CANARY_KEY } from "./streams.js";
+import { EVICT_PRINCIPALS_MAX } from "./evict.js";
+import { createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig, MAX_MSGS_PER_SUBJECT, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS, TTL_RECONCILE_CANARY_KEY, PRESENCE_STORAGE } from "./streams.js";
 import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes, JetStreamApiError, } from "@nats-io/jetstream";
 import {} from "@nats-io/jetstream";
 import { Kvm } from "@nats-io/kv";
 import { Bucket, KvWatchInclude } from "@nats-io/kv/internal";
 import { openMembersRegistry, commitMember, tombstoneMember, activateMember, readMember, listMembers, listLifecycleMemberChannels, durableEligible, StaleMembershipWrite, } from "./members.js";
-import { openAclRegistry, readAcl, readAclForAlias, AmbiguousAclAlias, commitAcl as writeAclRecord, reissueAcl as writeAclReissue } from "./acls.js";
+import { openAclRegistry, readAcl, readAclForAlias, aclRetired, AmbiguousAclAlias, commitAcl as writeAclRecord, reissueAcl as writeAclReissue } from "./acls.js";
 import { openDeliveryRegistry } from "./lease.js";
 import { openChannelRegistry, effectiveReplay, effectiveReplayWindowMs, effectiveDeliveryClass, readChannelConfig, readChannelDefaults, } from "./channels.js";
-import { anycastSubject, CHANNEL_DEFAULTS_KEY, chatStream, chatHistDurable, chatSubject, controlServiceSubject, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dlvDurable, dlvSubject, dinboxSubject, inboxStream, parseDinboxPrincipal, FANOUT_DURABLE, INBOX_READER_DURABLE, leaseKey, managerBucket, MANAGER_LEASE_KEY, MANAGER_RENEWAL_LEASE_KEY, managerLeaseKey, chatWildcard, assertValidChannel, channelInAllow, isConcreteChannel, normalizeMentions, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, membershipBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
+import { anycastSubject, CHANNEL_DEFAULTS_KEY, chatStream, chatHistDurable, chatSubject, controlServiceSubject, livenessSubject, livenessServeFilter, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dlvDurable, dlvSubject, dinboxSubject, inboxStream, parseDinboxPrincipal, FANOUT_DURABLE, INBOX_READER_DURABLE, leaseKey, managerBucket, MANAGER_LEASE_KEY, MANAGER_RENEWAL_LEASE_KEY, managerLeaseKey, chatWildcard, assertValidChannel, channelInAllow, isConcreteChannel, normalizeMentions, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, membershipBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
 export const DEFAULT_SERVER = "nats://127.0.0.1:4222";
 const PLANE3_FRAME_HEADER = "Cotal-Delivery-Frame";
 /** Space joined when none is given on the CLI (the `cotal-<space>` cmux tab, etc.). */
@@ -187,12 +189,12 @@ export class CotalEndpoint extends EventEmitter {
      *  {@link armDeliveryControl}; tracked so the stale one is dropped on reconnect. */
     deliveryServeSub;
     deliveryAdminServeSub;
-    /** When set, this endpoint hosts the Plane-3 fan-out writer + trusted reader (the server-side delivery
-     *  daemon). `aclFor` maps an owner id to its current read ACL (`allowSubscribe`) for the reader's
-     *  re-authorization — read FRESH per entry from the durable ACL registry KV, hence async. */
     /** True once {@link quiescePlane3} has stopped serving this shard pending an ownership answer.
      *  Guards {@link armPlane3} so a RECONNECT cannot silently resume serving mid-question. */
     plane3Quiesced = false;
+    /** When set, this endpoint hosts the Plane-3 fan-out writer + trusted reader (the server-side delivery
+     *  daemon). `aclFor` maps an owner id to its current read ACL (`allowSubscribe`) for the reader's
+     *  re-authorization — read FRESH per entry from the durable ACL registry KV, hence async. */
     plane3;
     /** Live local cache of the channel registry (key = channel token), kept by a KV watch. */
     channelConfigs = new Map();
@@ -206,6 +208,11 @@ export class CotalEndpoint extends EventEmitter {
      *  `chathist_<id>` consumer, so overlapping reads would delete/recreate it under one another. */
     histLock = Promise.resolve();
     subs = [];
+    /** The liveness responders {@link serveLiveness} was asked for, by plane. This is INTENT, re-bound
+     *  by {@link bindLiveness} on every (re)connect: a rebuild closes the old connection and every
+     *  subscription on it, so a responder bound once would leave its plane to the broker's
+     *  no-responders answer, which a peer grades `unbound`, while this endpoint is connected and serving. */
+    livenessResponders = new Map();
     streamMsgs = [];
     /** Per-channel native core subscriptions (SPEC v0.3) — the manager-free live read path for boot +
      *  runtime channels (there is no per-instance chat durable). Keyed by channel so leave unsubscribes
@@ -289,6 +296,8 @@ export class CotalEndpoint extends EventEmitter {
     presenceRebindAt = 0;
     status = "idle";
     activity;
+    /** Last harness-reported work progress. Carried by the next heartbeat, never published per event. */
+    activeAt;
     condition;
     /** Advances on every condition change so an older in-flight put cannot be the final KV state. */
     conditionRevision = 0;
@@ -1030,7 +1039,7 @@ export class CotalEndpoint extends EventEmitter {
             // OPENs it (it's pre-created at `cotal up`; KV stream-create is denied to agents).
             this.kv = this.authed
                 ? await kvm.open(presenceBucket(this.space))
-                : await kvm.create(presenceBucket(this.space), { ttl: this.ttlMs });
+                : await kvm.create(presenceBucket(this.space), { ttl: this.ttlMs, storage: PRESENCE_STORAGE });
         }
         if (this.doWatch) {
             await this.startPresenceWatch();
@@ -1118,6 +1127,10 @@ export class CotalEndpoint extends EventEmitter {
         // endpoint hosts it. The first arm comes from startPlane3 (after start()); this re-binds the loops
         // a reconnect's clearConnectionScoped() tore down, so a broker blip doesn't silently kill the backstop.
         await this.armPlane3();
+        // Re-bind the liveness responders on this connection, for armPlane3's reason: the rebuild closed
+        // the old connection and the responders with it.
+        for (const plane of this.livenessResponders.keys())
+            this.bindLiveness(plane);
         // Bound and live — covers initial start, manual reconnect, AND background self-heal (every
         // path lands here). The single signal an in-process agent's connected flag tracks.
         //
@@ -2240,6 +2253,12 @@ export class CotalEndpoint extends EventEmitter {
         this.status = status;
         await this.publishPresence();
     }
+    /** Record harness-reported work progress as `activeAt`. It writes nothing itself: the next heartbeat
+     *  carries it, so a stream of events costs no presence writes. */
+    noteActivity(at = Date.now()) {
+        if (Number.isFinite(at) && at > (this.activeAt ?? 0))
+            this.activeAt = at;
+    }
     /** Publish a harness-reported condition, or clear it. Core stores the relay without interpretation. */
     async setCondition(condition) {
         this.condition = condition ?? undefined;
@@ -3407,8 +3426,11 @@ export class CotalEndpoint extends EventEmitter {
      *  daemon's cred is a file on disk that every restart re-reads, so `card.id` is stable across
      *  processes by design. See {@link DeliveryLeaseInfo.incarnation}. */
     leaseIncarnation = randomUUID();
-    encodeLease(ready) {
-        return new TextEncoder().encode(JSON.stringify({ holder: this.card.id, incarnation: this.leaseIncarnation, since: Date.now(), ready }));
+    /** When this endpoint last WON each shard's lease create. Every later write of that row carries it
+     *  unchanged, while `since` is re-stamped. See {@link DeliveryLeaseInfo.acquiredAt}. */
+    leaseAcquiredAt = new Map();
+    encodeLease(shardIndex, ready, acquiredAt = this.leaseAcquiredAt.get(shardIndex)) {
+        return new TextEncoder().encode(JSON.stringify({ holder: this.card.id, incarnation: this.leaseIncarnation, acquiredAt, since: Date.now(), ready }));
     }
     /** Is this shard's lease row one THIS ENDPOINT INSTANCE wrote? The question a daemon whose renew
      *  just failed has to answer before it decides whether it still owns the shard.
@@ -3428,13 +3450,17 @@ export class CotalEndpoint extends EventEmitter {
      *  freeing a re-acquire. Acquired BEFORE binding (single-flight gate); {@link markDeliveryLeaseReady}
      *  flips it ready AFTER the loops + `ctl.delivery` are bound. Returns the lease revision. */
     async acquireDeliveryLease(shardIndex) {
-        return (await this.deliveryRegistry()).create(leaseKey(shardIndex), this.encodeLease(false));
+        // Recorded only once the create wins: a refused create leaves the held row, and so its time, as it was.
+        const acquiredAt = Date.now();
+        const revision = await (await this.deliveryRegistry()).create(leaseKey(shardIndex), this.encodeLease(shardIndex, false, acquiredAt));
+        this.leaseAcquiredAt.set(shardIndex, acquiredAt);
+        return revision;
     }
     /** Flip the held lease to READY (CAS `kv.update`) AFTER `startPlane3` has bound the loops + the
      *  `ctl.delivery` responder — so "lease ready" proves the responder is up, not just that the slot was
      *  claimed. Returns the new revision. */
     async markDeliveryLeaseReady(shardIndex, revision) {
-        return (await this.deliveryRegistry()).update(leaseKey(shardIndex), this.encodeLease(true), revision);
+        return (await this.deliveryRegistry()).update(leaseKey(shardIndex), this.encodeLease(shardIndex, true), revision);
     }
     /** Flip the held lease back to NOT-ready, the counterpart to {@link markDeliveryLeaseReady}, for a
      *  holder that has UNBOUND its loops and control responder but has not given up the shard.
@@ -3446,13 +3472,13 @@ export class CotalEndpoint extends EventEmitter {
      *  the row (rather than deleting it) is deliberate: the shard is still claimed, so no third daemon
      *  should be invited in; what is being withdrawn is only the claim to be answering. */
     async markDeliveryLeaseNotReady(shardIndex, revision) {
-        return (await this.deliveryRegistry()).update(leaseKey(shardIndex), this.encodeLease(false), revision);
+        return (await this.deliveryRegistry()).update(leaseKey(shardIndex), this.encodeLease(shardIndex, false), revision);
     }
     /** Renew the held lease (CAS `kv.update` against `revision`, keeping `ready:true`) to refresh it before
      *  the bucket TTL expires it. Returns the new revision. Throws if the revision moved (lost the lease —
      *  the daemon should exit). */
     async renewDeliveryLease(shardIndex, revision) {
-        return (await this.deliveryRegistry()).update(leaseKey(shardIndex), this.encodeLease(true), revision);
+        return (await this.deliveryRegistry()).update(leaseKey(shardIndex), this.encodeLease(shardIndex, true), revision);
     }
     /** Release the held lease on clean shutdown so a replacement daemon re-acquires immediately (best
      *  effort, a crash just lets the bucket TTL expire it).
@@ -3997,7 +4023,7 @@ export class CotalEndpoint extends EventEmitter {
                         continue;
                     }
                     const parsed = parseSubject(m.subject);
-                    if (!parsed || msg.from?.id !== parsed.sender || !isPrincipalOwnerToken(parsed.owner) || msg.from.id === owner)
+                    if (!isRecord(msg) || !parsed || msg.from?.id !== parsed.sender || !isPrincipalOwnerToken(parsed.owner) || msg.from.id === owner)
                         continue;
                     await this.publishDinbox(owner, lifecycleUid, { msg, channel, seq: m.seq, reason: "durable-channel", generation });
                     copied++;
@@ -4022,13 +4048,9 @@ export class CotalEndpoint extends EventEmitter {
     async startPlane3(aclFor, opts = {}) {
         if (!this.js)
             throw new Error("endpoint not started");
-        this.plane3 = { aclFor, reloadMembershipCreds: opts.reloadMembershipCreds, evictPrincipal: opts.evictPrincipal, planeConnLiveness: opts.planeConnLiveness, principalLiveness: opts.principalLiveness, reloadStoreIdentity: opts.reloadStoreIdentity, onDeliveryCredsAdopted: opts.onDeliveryCredsAdopted };
+        this.plane3 = { aclFor, reloadMembershipCreds: opts.reloadMembershipCreds, evictPrincipal: opts.evictPrincipal, evictPrincipals: opts.evictPrincipals, planeConnLiveness: opts.planeConnLiveness, principalLiveness: opts.principalLiveness, reloadStoreIdentity: opts.reloadStoreIdentity, onDeliveryCredsAdopted: opts.onDeliveryCredsAdopted, startupComplete: opts.startupComplete };
         await this.armPlane3();
     }
-    /** Serve one runtime durable-membership control request (the server-side delivery daemon). The caller
-     *  id is the authenticated subject sender ({@link serveControl} fail-closes on a mismatch). Validation
-     *  is against the durable ACL registry — the SAME KV the reader re-auths against (single source of
-     *  truth, no in-memory ledger to drift). */
     /** Whether an ALREADY-DISPATCHED unit of Plane-3 work may still take effect.
      *
      *  Unsubscribing stops NEW work; it cannot recall work already in flight. A handler that entered
@@ -4046,6 +4068,10 @@ export class CotalEndpoint extends EventEmitter {
     plane3MayAct() {
         return !this.plane3Quiesced;
     }
+    /** Serve one runtime durable-membership control request (the server-side delivery daemon). The caller
+     *  id is the authenticated subject sender ({@link serveControl} fail-closes on a mismatch). Validation
+     *  is against the durable ACL registry — the SAME KV the reader re-auths against (single source of
+     *  truth, no in-memory ledger to drift). */
     async handleDeliveryControl(req) {
         // FENCE: entered before a quiesce, resuming after it. Answering now would put a second server on
         // this shard's control rail while the winner is already READY.
@@ -4388,6 +4414,216 @@ export class CotalEndpoint extends EventEmitter {
         }
         this.deliveryAdminServeSub = this.serveControl(CONTROL_DELIVERY_ADMIN, (req) => this.handleDeliveryAdmin(req), { boundReply: true });
     }
+    // ---- the peer-readable liveness surface (#1577) --------------------------
+    /** Bind this endpoint as the liveness RESPONDER for one plane, answering presence and nothing
+     *  else to any credentialed peer that asks (`live.<plane>.<owner>.<actor>`).
+     *
+     *  THE HANDLER IS THE PRIVACY BOUNDARY, and that is the entire reason this is a request/reply
+     *  probe rather than a KV read grant. `readState` returns the responder's OWN verdict, derived
+     *  from the lease it can already read; the LEASE ROW NEVER LEAVES THIS PROCESS. A peer therefore
+     *  learns one enum about one plane and cannot learn the holder, the workspace root, the pid, the
+     *  instance id, the runtime, or that any of those exist. Handing a peer the manager bucket instead
+     *  would have handed it the operator's filesystem path and a pid to signal.
+     *
+     *  `queue`-grouped by plane, so several manager instances in one space answer a probe ONCE rather
+     *  than N times. A queue group is correct here and would be wrong on a roster-style surface: the
+     *  question is "is ANY responder bound", which any one member can answer.
+     *
+     *  THE ANSWER NAMES WHICH RESPONDER GAVE IT (`instance`), and that field exists because the queue
+     *  group alone leaves a real ambiguity. Manager instances coexist per instance id by design, each
+     *  member answers only about ITSELF, and the group delivers one probe to an arbitrary member. So
+     *  when two instances hold opposite verdicts, identical probes alternate between them, and without
+     *  a discriminator the two answers are indistinguishable from one instance changing state. With
+     *  the field, a caller that probes more than once can see that two DIFFERENT responders answered
+     *  and that the verdicts disagree. The field is the responder's own endpoint-scoped instance
+     *  token, never the lease row's `instanceId`, its holder, its pid or its root: see
+     *  {@link LivenessAnswer} for why none of those may cross the wire.
+     *
+     *  ONE PROBE STILL SAMPLES ONE MEMBER. This does not aggregate N instances, and a single probe
+     *  cannot report a split; it makes the split OBSERVABLE to a caller that asks again, where before
+     *  it was not observable at all.
+     *
+     *  BOUNDED REPLY, for the same confused-deputy reason `serveControl` documents: this responder
+     *  holds a wildcard publish grant over `live.<plane>.*.*.reply.>`, so without the bound check an
+     *  authenticated caller could name a PEER's reply lane as its reply target and have us publish
+     *  into it. The broker does not permission-check a requester's embedded reply subject; we do.
+     *
+     *  A HANDLER THAT THROWS ANSWERS `unknown`, never silence and never health. Silence would reach
+     *  the prober as a timeout, which it correctly grades `unknown` anyway — but only after burning
+     *  the full deadline, and an operator staring at a hung probe learns less than one told plainly
+     *  that the axis could not be checked.
+     *
+     *  THE BINDING IS KEPT AS INTENT and re-bound on every (re)connect, so a reconnect does not leave
+     *  the plane answered by the broker's no-responders status while this endpoint is serving. A second
+     *  call for the same plane replaces the first. */
+    serveLiveness(plane, readState) {
+        if (!this.nc)
+            throw new Error("endpoint not started");
+        const held = this.livenessResponders.get(plane);
+        if (held)
+            held.readState = readState;
+        else
+            this.livenessResponders.set(plane, { readState });
+        this.bindLiveness(plane);
+    }
+    /** (Re)bind one plane's liveness responder on the CURRENT connection, dropping the stale sub first
+     *  as {@link armDeliveryControl} does. */
+    bindLiveness(plane) {
+        const held = this.livenessResponders.get(plane);
+        if (!held || !this.nc)
+            return;
+        if (held.sub) {
+            try {
+                held.sub.unsubscribe();
+            }
+            catch { /* dead with the old connection */ }
+            const i = this.subs.indexOf(held.sub);
+            if (i >= 0)
+                this.subs.splice(i, 1);
+        }
+        const { readState } = held;
+        const sub = this.nc.subscribe(livenessServeFilter(this.space, plane), { queue: `live.${plane}` });
+        held.sub = sub;
+        this.subs.push(sub);
+        // The responder token this bind answers under: AN OPAQUE VALUE MINTED HERE, and deliberately
+        // not any name this process already has. It is what makes "which responder answered" askable
+        // without making "who is this responder" answerable — the two properties the surface has to
+        // hold at once. The endpoint's own `card.id` would be its principal, the lease row's
+        // `instanceId` is a field of a bucket an agent holds no grant on, and either would turn a
+        // presence probe into an identity read for every credentialed peer in the space. A fresh
+        // random token correlates with nothing but this plane's other answers, which is the whole job:
+        // two answers carrying two different tokens came from two different responders.
+        //
+        // Per BIND rather than per process, so a responder that goes away and comes back answers under
+        // a new token. That is the honest reading: a caller comparing answers across a restart is
+        // comparing two different incarnations, and a token that survived the restart would say
+        // otherwise. A re-bind after a reconnect is such a return: the plane had no responder here for
+        // the gap.
+        const instance = randomUUID();
+        void (async () => {
+            for await (const m of sub) {
+                // Sender-bound reply guard. Drop silently rather than publishing somewhere else: a reply
+                // aimed outside the sender's own subtree is not a request we can answer safely.
+                //
+                // REPORTED ON `warning`, NOT ON `error`, AND THAT IS A SAFETY PROPERTY RATHER THAN A CHOICE
+                // OF CHANNEL. A rejected probe is a condition this responder is already surviving: it drops
+                // the frame and keeps serving. `error` cannot carry such a notice on this class, because
+                // Node's `EventEmitter` RETHROWS an `error` emit that has no listener attached, so the
+                // notice would end the process of any embedder that had not attached one — and the plane
+                // would then genuinely be unbound, which a peer's next probe reports as `unbound`. The
+                // condition would have manufactured the state it described. `warning` is observable and
+                // never fatal without a listener (see {@link emitRecoverable} and #891, where retry notices
+                // on `error` killed hosts the endpoint intended to keep running).
+                //
+                // Since ANY credentialed peer may publish a probe, and the reply target inside it is chosen
+                // by that peer and not permission-checked by the broker, reaching this branch is a peer's
+                // decision, not an operator's. Safety here must therefore hold with no listener attached,
+                // which is what CELL F1 runs.
+                if (!m.reply || !m.reply.startsWith(`${m.subject}.reply.`)) {
+                    this.emitRecoverable(new Error(`rejected liveness probe on ${m.subject}: reply target "${m.reply ?? "(none)"}" is not under the sender's own reply subtree`));
+                    continue;
+                }
+                let responder;
+                try {
+                    responder = await readState();
+                }
+                catch {
+                    responder = "unknown";
+                }
+                const answer = { plane, responder, instance };
+                try {
+                    m.respond(JSON.stringify(answer));
+                }
+                catch { /* the requester is gone */ }
+            }
+            // The loop itself ends only on a SUBSCRIPTION-level fault (the connection went away), which
+            // is not something a peer's probe can cause: the guard above `continue`s, `readState` is
+            // caught, and the respond is caught. So this arm stays on `error` — it reports that this
+            // responder has stopped answering at all, which is a fault an embedder should not be able to
+            // miss, and no credentialed peer can reach it.
+        })().catch((e) => this.emit("error", e));
+    }
+    /** Bind the liveness responder for the DELIVERY plane, grading itself from its own shard-0 lease
+     *  through the shared classifier. The daemon is the one process that can answer this honestly: it
+     *  knows its own incarnation, so a predecessor's `ready:true` corpse in the bucket classifies
+     *  `stale` rather than as its own health.
+     *
+     *  It reads the lease at PROBE time rather than caching a flag set at bind time. A cached flag
+     *  would answer `bound` for as long as this process lived, including after it had lost the shard
+     *  and stood down — a responder that reports its intent instead of its state, which is the whole
+     *  family of bug this issue is about. */
+    serveDeliveryLiveness(shardIndex = 0) {
+        this.serveLiveness("delivery", async () => responderFromLease(await this.readDeliveryLease(shardIndex), this.card.id));
+    }
+    /** Ask whether a responder is bound for `plane`, as a credentialed peer that need not own it.
+     *
+     *  EVERY WAY THIS CAN END IS CLASSIFIED, and the classification lives in ONE place
+     *  ({@link responderFromProbe}) so no call site can invent its own mapping. Only the broker's own
+     *  no-responders 503 becomes `unbound`. A timeout, a permission refusal, a transport failure and
+     *  an unreadable reply ALL become `unknown`, because each is a failure to find out, and this
+     *  issue exists because failures to find out were being reported as findings.
+     *
+     *  `noMux` with a named reply subject is required rather than stylistic: the muxed inbox would
+     *  swallow the 503 into an ordinary timeout, and the 503 is the only outcome that carries a
+     *  positive verdict. Collapsing it would leave a probe that can say `bound` or `unknown` and
+     *  never `unbound` — an instrument that cannot report the failure it was built to report.
+     *
+     *  The reply rides `<request>.reply.<uuid>`, the sender's own subtree, so the responder's bound
+     *  reply guard accepts it and the responder needs no broad inbox-publish grant to answer. */
+    async probeLiveness(plane, timeoutMs = 2_000) {
+        if (!isLivenessPlane(plane))
+            // A closed set, refused loudly. Answering an unknown plane at all would make this a probe
+            // that returns something for every input, which is indistinguishable from one that is not
+            // measuring anything.
+            throw new Error(`liveness: "${plane}" is not a plane this surface answers for (${[...LIVENESS_PLANES].join(", ")})`);
+        if (!this.nc)
+            throw new Error(this.notLiveMsg());
+        const reqSubject = livenessSubject(this.space, plane, this.owner, this.actor);
+        const reply = `${reqSubject}.reply.${randomUUID()}`;
+        let outcome;
+        let answered;
+        let instance;
+        try {
+            const m = await this.nc.request(reqSubject, "", { timeout: timeoutMs, noMux: true, reply });
+            let body;
+            try {
+                body = m.json();
+            }
+            catch {
+                body = undefined;
+            }
+            const parsed = parseLivenessAnswer(body, plane);
+            // A reply we cannot read is `malformed`, which grades `unknown`. It is NOT promoted to
+            // `bound` on the strength of having replied at all: that promotion is the `pgrep` error,
+            // where evidence a process exists was read as evidence it works.
+            outcome = parsed ? "replied" : "malformed";
+            answered = parsed?.responder;
+            instance = parsed?.instance;
+        }
+        catch (e) {
+            outcome = this.isNoResponders(e) ? "noResponders" : this.probeFailureOutcome(e);
+        }
+        // `instance` rides ONLY a reply that was read. Every other outcome means no responder answered,
+        // so there is no responder to name, and a token attached to an `unbound` or an `unknown` would
+        // claim one had. It stays absent on those arms by construction: nothing assigns it.
+        const verdict = { plane, responder: responderFromProbe(outcome, answered) };
+        return instance === undefined ? verdict : { ...verdict, instance };
+    }
+    /** Grade a failed probe that was NOT a no-responders answer. Both arms return an outcome that
+     *  maps to `unknown`; they are told apart so the distinction stays legible at the call site and so
+     *  a future caller can render the two differently (a refusal is about YOUR credential, a timeout
+     *  is about something being wedged). Neither may ever mean health. */
+    probeFailureOutcome(e) {
+        if (e instanceof AuthorizationError || e instanceof PermissionViolationError)
+            return "refused";
+        const name = e?.name;
+        const msg = e?.message ?? "";
+        if (name === "TimeoutError" || /timeout/i.test(msg))
+            return "timeout";
+        // Anything else is a transport or client failure: it says something about our link, not about
+        // the plane, so it is graded like a refusal rather than guessed at.
+        return "refused";
+    }
     /** Serve one PRIVILEGED delivery-admin request (the D5 rail-split). The cred layer is the caller
      *  boundary — only the supervisor profile can publish here — and `serveControl`'s sender check +
      *  bounded reply still apply on top. `reloadCreds` is the class-2 renewal ADOPTION step: re-read
@@ -4399,6 +4635,12 @@ export class CotalEndpoint extends EventEmitter {
         if (!this.plane3MayAct())
             return { ok: false, error: "delivery: this daemon is not serving this shard (it is re-checking ownership); retry" };
         if (req.op === "reloadCreds") {
+            // NOT BEFORE START-UP IS DONE (#2304). Both halves below would run: the delivery half commits
+            // and schedules `nc.reconnect()`, which lands underneath start-up awaits still in flight on
+            // this connection (the lease watch's consumer create times out and the daemon exits). Refuse
+            // before either half acts, so nothing is adopted and the renewal owner records why.
+            if (this.plane3?.startupComplete?.() === false)
+                return { ok: false, error: "delivery: this daemon has not finished starting; nothing adopted - the next renewal pass or the daemon's 75% re-read adopts the re-signed creds" };
             // The renewal owner's EXPECTED-generation tokens (SHA-256 of each JWT it re-signed), per
             // component. A missing entry means "no expectation" (the passive backstop still adopts).
             const expected = (req.args?.expected ?? {});
@@ -4446,6 +4688,24 @@ export class CotalEndpoint extends EventEmitter {
                 return { ok: false, error: e.message };
             }
         }
+        if (req.op === "evictPrincipals") {
+            // The same executor over a SET: one shared sweep instead of one request and one scan per
+            // principal. The set is bounded so one request stays inside one sweep's work.
+            if (!this.plane3?.evictPrincipals)
+                return { ok: false, error: "evictPrincipals: no batch eviction executor wired on this daemon" };
+            const raw = req.args?.principals;
+            const principals = Array.isArray(raw) ? raw.map((p) => (typeof p === "string" ? p.trim() : "")) : [];
+            if (principals.length === 0 || principals.some((p) => !p))
+                return { ok: false, error: "evictPrincipals: principals must be a non-empty array of owner.actor dot-form principals" };
+            if (principals.length > EVICT_PRINCIPALS_MAX || new Set(principals).size !== principals.length)
+                return { ok: false, error: `evictPrincipals: principals must be distinct and at most ${EVICT_PRINCIPALS_MAX}` };
+            try {
+                return { ok: true, data: await this.plane3.evictPrincipals(principals) };
+            }
+            catch (e) {
+                return { ok: false, error: e.message };
+            }
+        }
         if (req.op === "planeConnLiveness") {
             // The plane-claim liveness oracle (#29 HIGH 3): a CLOSED read-only verb — two claimed
             // scanner tuples in, two bound verdicts + sweep completeness out. The executor hook owns
@@ -4592,7 +4852,7 @@ export class CotalEndpoint extends EventEmitter {
             m.ack();
             return;
         }
-        if (!msg.from || msg.from.id !== parsed.sender || !isPrincipalOwnerToken(parsed.owner)) {
+        if (!isRecord(msg) || !msg.from || msg.from.id !== parsed.sender || !isPrincipalOwnerToken(parsed.owner)) {
             m.ack();
             return;
         } // authenticity (owner must be a real principal, not an old-shape alias)
@@ -4702,12 +4962,52 @@ export class CotalEndpoint extends EventEmitter {
             m.ack();
             return;
         } // undecodable — drop
+        if (!isRecord(entry)) {
+            m.ack();
+            return;
+        } // non-object envelope — permanently invalid
         const redeliveries = m.info?.deliveryCount ?? 1; // JsMsg delivery attempts (1 on first delivery)
         // Lifecycle-exact ACL re-auth (SPEC 13.1): the entry was addressed to pr.lifecycleUid's inbox, so
         // the row read is that lifecycle's exact key — a retired lifecycle's purged row reads as unknown
-        // and its residual entries terminate at the redelivery ceiling, never against the successor's row.
+        // and its residual entries are removed below, never authorized against the successor's row.
         const acl = await this.plane3?.aclFor(owner, pr.lifecycleUid);
         if (acl === undefined) {
+            // RETIRED lifecycle: its exact ACL key carries retirement's tombstone, so no attempt can ever
+            // deliver this entry. Drop it AND remove it from the store. Deferring it to the ceiling and
+            // keeping it would make every reader that starts from a fresh cursor pay the same sweep again.
+            // An ACL row that is merely absent proves nothing and still defers below.
+            if (await aclRetired(await this.aclRegistry(), owner, pr.lifecycleUid)) {
+                if (!this.plane3MayAct())
+                    return;
+                // Remove BEFORE the ack. The ack ends the reader's work on this entry, so acking first and
+                // then failing the delete would keep the entry with nothing left to remove it. A failed delete
+                // stays pending and retries, bounded by the same ceiling; an entry the stream no longer holds
+                // is already removed (the broker answers a delete of a missing sequence with 10043).
+                let failure;
+                try {
+                    await this.jsm.streams.deleteMessage(inboxStream(this.space), m.seq, false);
+                }
+                catch (e) {
+                    if (!isJetStreamMissing(e, 10043))
+                        failure = e instanceof Error ? e : new Error(String(e));
+                }
+                // FENCE AFTER THE DELETE, which is broker I/O. A daemon that stopped serving while it was in
+                // flight leaves the entry to the holder: a term here would move the shared durable past an entry
+                // the failed delete left stored, so the holder would never retry it.
+                if (!this.plane3MayAct())
+                    return;
+                if (failure) {
+                    if (redeliveries >= READER_MAX_REDELIVERIES) {
+                        m.term();
+                        this.emit("error", new Error(`plane-3 reader: gave up removing entry ${m.seq} for retired lifecycle ${owner}.${pr.lifecycleUid} after ${redeliveries} redeliveries: ${failure.message}`));
+                        return;
+                    }
+                    m.nak(2000);
+                    return;
+                }
+                m.ack();
+                return;
+            }
             // UNKNOWN owner — the manager has not (re)hydrated this owner's ACL yet (e.g. right after a
             // manager PROCESS restart). This is NOT a revocation: DEFER (redeliver), never drop — an ack here
             // would lose at-least-once on restart (impl-review BLOCKER-2). A delayed nak + a redelivery
@@ -4889,7 +5189,7 @@ export class CotalEndpoint extends EventEmitter {
             }
             catch (e) {
                 if (attempt === 0)
-                    this.emit("error", new Error(`channel "${channel}": Plane-3 durable membership (generation ${generation}) not yet tombstoned after a refused live sub - retrying; §7 boundary may be open until it succeeds (${e.message})`));
+                    this.emitRecoverable(new Error(`channel "${channel}": Plane-3 durable membership (generation ${generation}) not yet tombstoned after a refused live sub - retrying; §7 boundary may be open until it succeeds (${e.message})`));
                 await new Promise((r) => setTimeout(r, Math.min(30_000, 1000 * 2 ** attempt)));
             }
         }
@@ -5083,7 +5383,7 @@ export class CotalEndpoint extends EventEmitter {
                 // server policed who could publish. The payload `from` is advisory — it must match,
                 // and a missing `from` or an unparseable subject on a delivery is itself an anomaly.
                 // Reject (term — a spoof is permanently invalid, never redeliver) BEFORE any handler.
-                if (!isUsableMessageId(msg.id)) {
+                if (!isRecord(msg) || !isUsableMessageId(msg.id)) {
                     m.term(); // malformed envelope (SPEC sec 5): absent/non-string id — permanently invalid
                     this.emit("error", new Error(`dropped message on ${m.subject}: absent or non-string id`));
                     continue;
@@ -5178,7 +5478,7 @@ export class CotalEndpoint extends EventEmitter {
                     this.emit("error", e);
                     return;
                 }
-                if (!isUsableMessageId(msg.id))
+                if (!isRecord(msg) || !isUsableMessageId(msg.id))
                     return; // malformed envelope (SPEC sec 5) — live is at-most-once: drop
                 if (!msg.from || msg.from.id !== parsed.sender || !isPrincipalOwnerToken(parsed.owner))
                     return; // spoof/malformed/old-shape-alias — drop (at-most-once)
@@ -5414,7 +5714,7 @@ export class CotalEndpoint extends EventEmitter {
                 continue; // skip undecodable
             }
             // Same authenticity guard as the tail; skip our own echoes in history.
-            if (!isUsableMessageId(msg.id))
+            if (!isRecord(msg) || !isUsableMessageId(msg.id))
                 continue; // malformed envelope (SPEC sec 5) — history skips
             const parsed = parseSubject(sm.subject);
             if (!parsed || msg.from?.id !== parsed.sender || !isPrincipalOwnerToken(parsed.owner) || msg.from.id === this.card.id)
@@ -5431,7 +5731,10 @@ export class CotalEndpoint extends EventEmitter {
      * not a push into context) plus `dropped: true` when the window is not complete: either the
      * channel's earliest *retained* message is already newer than the watermark (some ambient aged
      * out of the per-subject window), or replay is off for the channel below. Either way the caller
-     * must say so rather than silently reporting an empty, complete window.
+     * must say so rather than silently reporting an empty, complete window. `seqs[i]` is the chat
+     * stream sequence of `messages[i]`, the one identity two identical messages do not share.
+     * `unanswered` is true when no history read answered (replay is off, or the read failed), so
+     * `messages` says nothing about what the channel retains.
      *
      * Honors the **same** per-channel replay gate as join-backfill ({@link joinPolicyFresh}): a
      * `replay=off` channel returns no messages, so `focus` can't become a history bypass for a
@@ -5445,10 +5748,10 @@ export class CotalEndpoint extends EventEmitter {
         if (!this.jsm)
             throw new Error(this.notLiveMsg());
         if (!isConcreteChannel(channel))
-            return { messages: [], dropped: false };
+            return { messages: [], seqs: [], dropped: false, unanswered: true };
         const policy = await this.joinPolicyFresh(channel);
         if (!policy.replay)
-            return { messages: [], dropped: true };
+            return { messages: [], seqs: [], dropped: true, unanswered: true };
         const subject = chatSubject(this.space, "*", "*", channel);
         let raw;
         try {
@@ -5458,9 +5761,10 @@ export class CotalEndpoint extends EventEmitter {
             this.emit("error", e);
             if (isPermissionDenied(e))
                 throw e;
-            raw = [];
+            return { messages: [], seqs: [], dropped: true, unanswered: true };
         }
         const collected = [];
+        const seqs = [];
         for (const sm of raw) {
             let msg;
             try {
@@ -5470,15 +5774,16 @@ export class CotalEndpoint extends EventEmitter {
                 continue; // skip undecodable
             }
             // Same authenticity guard as the tail/backfill; skip our own echoes.
-            if (!isUsableMessageId(msg.id))
+            if (!isRecord(msg) || !isUsableMessageId(msg.id))
                 continue; // malformed envelope (SPEC sec 5) — recall skips
             const parsed = parseSubject(sm.subject);
             if (!parsed || msg.from?.id !== parsed.sender || !isPrincipalOwnerToken(parsed.owner) || msg.from.id === this.card.id)
                 continue;
             collected.push(authenticatedMessage(msg, parsed));
+            seqs.push(sm.seq);
         }
         const dropped = await this.channelDropped(subject, sinceSeq);
-        return { messages: collected, dropped };
+        return { messages: collected, seqs, dropped, unanswered: false };
     }
     /** Did focus recall on `subject` miss ambient that aged out past the watermark? Ambient is only
      *  ever discarded once a sender-subject reaches {@link MAX_MSGS_PER_SUBJECT} (`DiscardPolicy.Old`);
@@ -5533,6 +5838,7 @@ export class CotalEndpoint extends EventEmitter {
             condition: this.condition,
             environment: this.environment,
             activity: this.activity,
+            activeAt: this.activeAt,
             attention: this.attentionMode,
             channelModes: this.channelModes,
             ts: Date.now(),
@@ -5992,22 +6298,22 @@ export class CotalEndpoint extends EventEmitter {
             this.emit("roster", this.getRoster());
     }
 }
-/** Map an authenticated parsed-subject kind to the message class surfaced to "message" listeners.
- *  Throws on `ctl` (control-plane is request/reply, never a "message") — per repo convention, no
- *  silent default: an unexpected delivering kind is a bug, not something to swallow. */
-/** A usable delivery-message id (#624): a string, possibly empty (the never-a-key case), but
- *  never absent and never a non-string. An absent or non-string id is a malformed envelope under
- *  SPEC sec 5; each delivery pump handles it per its own class (durable term, live drop, history
- *  skip) so it never reaches the receiver's id-keyed machinery as `undefined`. */
 /** What a history read failure NAMES when it could not finish. One filter subject is the useful
  *  thing to print; a set of sixty-nine of them is a wall of text in a message a human has to read,
  *  so a set says its size and the stream it was read from instead. */
 function subjectLabel(subjects) {
     return subjects.length === 1 ? subjects[0] : `${subjects.length} filtered subjects`;
 }
+/** A usable delivery-message id (#624): a string, possibly empty (the never-a-key case), but
+ *  never absent and never a non-string. An absent or non-string id is a malformed envelope under
+ *  SPEC sec 5; each delivery pump handles it per its own class (durable term, live drop, history
+ *  skip) so it never reaches the receiver's id-keyed machinery as `undefined`. */
 function isUsableMessageId(id) {
     return typeof id === "string";
 }
+/** Map an authenticated parsed-subject kind to the message class surfaced to "message" listeners.
+ *  Throws on `ctl` (control-plane is request/reply, never a "message") — per repo convention, no
+ *  silent default: an unexpected delivering kind is a bug, not something to swallow. */
 function kindFromParsed(kind) {
     switch (kind) {
         case "chat":
@@ -6304,11 +6610,6 @@ function authOpts(a) {
     }
     return { token: a.token, user: a.user, pass: a.pass, tls };
 }
-/** Decode the owner+actor PRINCIPAL from a user bearer WITHOUT verifying it — the client trusts its own
- *  bearer only to build its subjects; the broker's minted grant (from the callout, which DOES verify the
- *  bearer) is the real boundary, so a client that lied to itself would just be denied. Per the token
- *  claim semantics the OWNER is the JWT `sub` (`act.owner` merely restates it) and the ACTOR is
- *  `act.actor`. Throws on a structurally-unusable bearer (fail-loud). */
 /** The bearer's `exp` as epoch ms — what the refresh schedule keys on. A bearer without a numeric
  *  `exp` is structurally unusable for a refreshing endpoint (fail-loud, like the principal decode). */
 function bearerExpiryMs(bearer) {
@@ -6326,6 +6627,11 @@ function bearerExpiryMs(bearer) {
         throw new Error("user-mode bearer is missing a numeric exp claim");
     return claims.exp * 1000;
 }
+/** Decode the owner+actor PRINCIPAL from a user bearer WITHOUT verifying it — the client trusts its own
+ *  bearer only to build its subjects; the broker's minted grant (from the callout, which DOES verify the
+ *  bearer) is the real boundary, so a client that lied to itself would just be denied. Per the token
+ *  claim semantics the OWNER is the JWT `sub` (`act.owner` merely restates it) and the ACTOR is
+ *  `act.actor`. Throws on a structurally-unusable bearer (fail-loud). */
 function decodeBearerPrincipal(bearer) {
     const payload = bearer.split(".")[1];
     if (!payload)