@cotal-ai/core 0.112.0 → 0.113.0

package/dist/endpoint.js

@@ -32,7 +32,7 @@ import { openMembersRegistry, commitMember, tombstoneMember, activateMember, rea
 import { openAclRegistry, readAcl, readAclForAlias, aclRetired, AmbiguousAclAlias, commitAcl as writeAclRecord, reissueAcl as writeAclReissue } from "./acls.js";
 import { openDeliveryRegistry } from "./lease.js";
 import { openChannelRegistry, effectiveReplay, effectiveReplayWindowMs, effectiveDeliveryClass, readChannelConfig, readChannelDefaults, decodeChannelEntry, } from "./channels.js";
-import { anycastSubject, assertValidRole, CHANNEL_DEFAULTS_KEY, chatStream, chatHistDurable, chatSubject, historyReaderConsumer, calloutReaderId, HISTORY_READERS, controlServiceSubject, livenessSubject, livenessServeFilter, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dlvDurable, dlvSubject, dinboxSubject, inboxStream, parseDinboxPrincipal, FANOUT_DURABLE, INBOX_READER_DURABLE, leaseKey, managerBucket, MANAGER_LEASE_KEY, MANAGER_RENEWAL_LEASE_KEY, managerLeaseKey, chatWildcard, assertValidChannel, channelInAllow, isConcreteChannel, normalizeMentions, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, membershipBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
+import { anycastSubject, assertValidRole, CHANNEL_DEFAULTS_KEY, chatStream, chatHistDurable, chatSubject, historyReaderConsumer, calloutReaderId, HISTORY_READERS, controlServiceSubject, livenessSubject, livenessServeFilter, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dlvDurable, dlvSubject, dinboxSubject, inboxStream, parseDinboxPrincipal, FANOUT_DURABLE, INBOX_READER_DURABLE, leaseKey, managerBucket, MANAGER_LEASE_KEY, MANAGER_RENEWAL_LEASE_KEY, managerLeaseKey, chatWildcard, assertValidChannel, channelInAllow, isConcreteChannel, normalizeMentions, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, isPresenceRow, membershipBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
 import { isPermissionDenied } from "./permission-denied.js";
 export { isPermissionDenied };
 export const DEFAULT_SERVER = "nats://127.0.0.1:4222";
@@ -255,6 +255,15 @@ export class CotalEndpoint extends EventEmitter {
      *  {@link armDeliveryControl}; tracked so the stale one is dropped on reconnect. */
     deliveryServeSub;
     deliveryAdminServeSub;
+    /** The lifecycles whose membership inventory this daemon has been asked for, as principal to uids
+     *  (#4965). A durable join or leave bound to one of them is refused, so no row of a retiring lifecycle
+     *  is written after its inventory read starts. An entry goes when a later inventory for the same
+     *  principal finds its ACL row retired; an alias that never retires again keeps its one entry until
+     *  the daemon restarts. */
+    retiringLifecycles = new Map();
+    /** The durable joins and leaves running now, by `<principal>/<lifecycleUid>`: the writes an
+     *  inventory of that lifecycle waits for before it reads. */
+    membershipWrites = new Map();
     /** True once {@link quiescePlane3} has stopped serving this shard pending an ownership answer.
      *  Guards {@link armPlane3} so a RECONNECT cannot silently resume serving mid-question. */
     plane3Quiesced = false;
@@ -2139,7 +2148,12 @@ export class CotalEndpoint extends EventEmitter {
      *  PEER's reply lane (`ctl.delivery.<victim>.reply.<n>`) and turn the responder into a confused
      *  deputy — the broker does NOT permission-check the requester's embedded reply subject. With it, a
      *  reply is published only when `m.reply` is under the AUTHENTICATED request subject
-     *  (`${m.subject}.reply.…`), binding the reply to the broker-policed sender token. */
+     *  (`${m.subject}.reply.…`), binding the reply to the broker-policed sender token.
+     *
+     *  Requests are answered one at a time, in arrival order. A request `concurrent` accepts is answered
+     *  in its own time instead: it waits for none of the requests in front of it, including one still
+     *  running, and holds up none behind it. Only for a request that shares no state with the service's
+     *  other requests. */
     serveControl(service, handler, opts = {}) {
         if (!this.nc)
             throw new Error("endpoint not started");
@@ -2147,6 +2161,11 @@ export class CotalEndpoint extends EventEmitter {
             queue: service,
         });
         this.subs.push(sub);
+        const inOwnTime = opts.concurrent;
+        // With `concurrent`, the in-order requests are answered one after another on this chain, and the
+        // reading below never waits on it, so a request answered in its own time is read and answered
+        // while an in-order one is still running.
+        let inOrder = Promise.resolve();
         void (async () => {
             for await (const m of sub) {
                 // Sender-bound reply guard (confused-deputy fix): never respond to a reply target outside the
@@ -2155,47 +2174,64 @@ export class CotalEndpoint extends EventEmitter {
                     this.emit("error", new Error(`rejected ${service} request on ${m.subject}: reply target "${m.reply ?? "(none)"}" is not under the sender's own reply subtree`));
                     continue;
                 }
-                let body;
-                try {
-                    let reply;
-                    const req = m.json();
-                    // Authenticity guard (fail closed): control is the most privileged surface
-                    // (start/stop). The sender is encoded in the subject (ctl.<svc>.<sender>), which
-                    // the server policed who could publish; the payload `from` is advisory and must
-                    // match. Reject before the handler acts on a request claiming a forged sender.
-                    const parsed = parseSubject(m.subject);
-                    if (!parsed || req.from?.id !== parsed.sender || !isPrincipalOwnerToken(parsed.owner)) {
-                        this.emit("error", new Error(`rejected control request on ${m.subject}: from ${req.from?.id ?? "(none)"} ` +
-                            `does not match subject sender ${parsed?.sender ?? "(unparseable)"}`));
-                        reply = { ok: false, error: "sender mismatch - request rejected" };
-                    }
-                    else {
-                        reply = await handler(req);
-                    }
-                    body = JSON.stringify(reply);
-                }
-                catch (e) {
-                    body = JSON.stringify({ ok: false, error: e.message });
+                const answer = () => this.answerControl(service, handler, m);
+                if (inOwnTime === undefined) {
+                    await answer();
+                    continue;
                 }
+                let own = false;
                 try {
-                    m.respond(body);
-                }
-                catch (e) {
-                    // `respond` signals a missing reply inbox by returning false; it throws when the reply exceeds the
-                    // broker's `max_payload`. Silence would leave the caller a bare timeout it cannot tell apart from
-                    // a dead service.
-                    const refused = { ok: false, error: `the ${service} reply is ${Buffer.byteLength(body)} bytes and was not published: ${e.message}` };
-                    try {
-                        m.respond(JSON.stringify(refused));
-                    }
-                    catch (err) {
-                        this.emit("error", new Error(`could not answer ${service} request on ${m.subject}: ${refused.error}, and the refusal failed too: ${err.message}`));
-                    }
+                    own = inOwnTime(m.json());
                 }
+                catch { /* unreadable: answered in order, as a refusal */ }
+                if (own)
+                    void answer().catch((e) => this.emit("error", e));
+                else
+                    inOrder = inOrder.then(answer).catch((e) => { this.emit("error", e); });
             }
         })().catch((e) => this.emit("error", e));
         return sub;
     }
+    /** Answer one control request for {@link serveControl}: the sender guard, the handler, and the reply. */
+    async answerControl(service, handler, m) {
+        let body;
+        try {
+            let reply;
+            const req = m.json();
+            // Authenticity guard (fail closed): control is the most privileged surface
+            // (start/stop). The sender is encoded in the subject (ctl.<svc>.<sender>), which
+            // the server policed who could publish; the payload `from` is advisory and must
+            // match. Reject before the handler acts on a request claiming a forged sender.
+            const parsed = parseSubject(m.subject);
+            if (!parsed || req.from?.id !== parsed.sender || !isPrincipalOwnerToken(parsed.owner)) {
+                this.emit("error", new Error(`rejected control request on ${m.subject}: from ${req.from?.id ?? "(none)"} ` +
+                    `does not match subject sender ${parsed?.sender ?? "(unparseable)"}`));
+                reply = { ok: false, error: "sender mismatch - request rejected" };
+            }
+            else {
+                reply = await handler(req);
+            }
+            body = JSON.stringify(reply);
+        }
+        catch (e) {
+            body = JSON.stringify({ ok: false, error: e.message });
+        }
+        try {
+            m.respond(body);
+        }
+        catch (e) {
+            // `respond` signals a missing reply inbox by returning false; it throws when the reply exceeds the
+            // broker's `max_payload`. Silence would leave the caller a bare timeout it cannot tell apart from
+            // a dead service.
+            const refused = { ok: false, error: `the ${service} reply is ${Buffer.byteLength(body)} bytes and was not published: ${e.message}` };
+            try {
+                m.respond(JSON.stringify(refused));
+            }
+            catch (err) {
+                this.emit("error", new Error(`could not answer ${service} request on ${m.subject}: ${refused.error}, and the refusal failed too: ${err.message}`));
+            }
+        }
+    }
     /** Send a control request to a service and await its reply (client side). Like {@link requestDelivery},
      *  the reply rides a BOUNDED subject UNDER the request subject (`ctl.<service>.<id>.reply.<uuid>`), not
      *  the per-id `_INBOX`.
@@ -4922,13 +4958,36 @@ export class CotalEndpoint extends EventEmitter {
             return { ok: false, error: `durableJoin: no read ACL on record for ${caller} (not provisioned for durable delivery)` };
         if (!channelInAllow(acl.allowSubscribe, channel))
             return { ok: false, error: `channel "${channel}" is not within your read ACL [${acl.allowSubscribe.join(", ")}]` };
+        const uid = acl.lifecycleUid;
         try {
-            return { ok: true, data: await this.durableJoinFor(caller, channel, acl.lifecycleUid) };
+            return { ok: true, data: await this.membershipWrite(caller, uid, () => this.durableJoinFor(caller, channel, uid)) };
         }
         catch (e) {
             return { ok: false, error: e.message };
         }
     }
+    /** Run one durable join or leave that writes `principal`'s rows of `lifecycleUid`, refused once that
+     *  lifecycle's inventory has been asked for ({@link retiringLifecycles}). The check and the
+     *  registration have no await between them, so an inventory either finds this write running and
+     *  waits for it, or this write finds the inventory's mark and is refused. */
+    async membershipWrite(principal, lifecycleUid, write) {
+        if (this.retiringLifecycles.get(principal)?.has(lifecycleUid))
+            throw new Error(`lifecycle ${lifecycleUid} of ${principal} is retiring: its membership inventory was requested, so its durable memberships no longer change`);
+        const key = `${principal}/${lifecycleUid}`;
+        const running = write();
+        let writes = this.membershipWrites.get(key);
+        if (writes === undefined)
+            this.membershipWrites.set(key, (writes = new Set()));
+        writes.add(running);
+        try {
+            return await running;
+        }
+        finally {
+            writes.delete(running);
+            if (writes.size === 0)
+                this.membershipWrites.delete(key);
+        }
+    }
     /** LEAVE must NOT require current-ACL coverage. Leave fires precisely when the ACL was narrowed/revoked
      *  (a refused live sub → {@link closeRefusedMembership}); gating the tombstone on the current ACL would
      *  loop forever and leave the SPEC §7 boundary open (the membership could resume if the ACL is later
@@ -4959,8 +5018,9 @@ export class CotalEndpoint extends EventEmitter {
         const existing = await readMember(await this.membersRegistry(), channel, caller, uid);
         if (!existing)
             return { ok: true, data: { channel, alreadyLeft: true } }; // nothing to tombstone — idempotent
+        const generation = args.generation;
         try {
-            await this.durableLeaveFor(caller, channel, uid, args.generation);
+            await this.membershipWrite(caller, uid, () => this.durableLeaveFor(caller, channel, uid, generation));
         }
         catch (e) {
             return { ok: false, error: e.message };
@@ -5183,7 +5243,13 @@ export class CotalEndpoint extends EventEmitter {
             if (i >= 0)
                 this.subs.splice(i, 1);
         }
-        this.deliveryAdminServeSub = this.serveControl(CONTROL_DELIVERY_ADMIN, (req) => this.handleDeliveryAdmin(req), { boundReply: true });
+        // The membership inventory is read-only and shares nothing with the other admin commands, so it
+        // does not queue behind them. In a mass stop each retirement's inventory followed every other
+        // seat's `evictPrincipal` on this rail and missed its 5 s deadline (#4965).
+        this.deliveryAdminServeSub = this.serveControl(CONTROL_DELIVERY_ADMIN, (req) => this.handleDeliveryAdmin(req), {
+            boundReply: true,
+            concurrent: (req) => req.op === "lifecycleMemberships",
+        });
     }
     // ---- the peer-readable liveness surface (#1577) --------------------------
     /** Bind this endpoint as the liveness RESPONDER for one plane, answering presence and nothing
@@ -5509,6 +5575,19 @@ export class CotalEndpoint extends EventEmitter {
             catch (e) {
                 return { ok: false, error: `lifecycleMemberships: ${e.message}` };
             }
+            // The lifecycle is retiring, so its memberships stop changing here (#4965): a join or leave bound
+            // to it is refused from now on, and the read starts after the ones already running have finished.
+            // A row they write is then in the read; nothing it misses can be written afterwards.
+            let marked = this.retiringLifecycles.get(principal);
+            if (marked === undefined)
+                this.retiringLifecycles.set(principal, (marked = new Set()));
+            // An earlier lifecycle of this principal whose ACL row is retired can bind no join or leave any
+            // more, so its mark goes. A failed read keeps the mark, which only ever refuses.
+            for (const earlier of [...marked])
+                if (earlier !== uid && await aclRetired(await this.aclRegistry(), principal, earlier).catch(() => false))
+                    marked.delete(earlier);
+            marked.add(uid);
+            await Promise.allSettled(this.membershipWrites.get(`${principal}/${uid}`) ?? []);
             try {
                 const channels = await listLifecycleMemberChannels(await this.membersRegistry(), principal, uid);
                 return { ok: true, data: { complete: true, channels } };
@@ -7099,21 +7178,14 @@ export class CotalEndpoint extends EventEmitter {
         catch {
             return undefined;
         }
-        if (!isUsablePresence(p)) {
+        if (!isPresenceRow(p, e.key)) {
             this.presenceBindingDrops++;
-            this.emitRecoverable(new Error(`dropped presence entry for key ${JSON.stringify(e.key)}: not a record with a string card.name, a known status, a finite ts, string display fields and a condition with a string code`));
-            return undefined;
-        }
-        // Defense-in-depth: the KV key IS the publisher's identity —
-        // publishPresence() writes its record under card.id — so a record whose embedded card.id disagrees
-        // with its bucket key is forged or corrupt. Drop it rather than surface a spoofed roster identity.
-        // The write-side scoping ($KV.<presenceBucket>.<own-id>) is the primary guard; this rejects a
-        // mis-keyed record even if a broad writer slips one in under another agent's key.
-        if (p.card.id !== e.key) {
-            this.presenceBindingDrops++;
-            this.emitRecoverable(new Error(`dropped presence entry for key ${JSON.stringify(e.key)}: card.id ${JSON.stringify(p.card.id)} does not match its KV key`));
+            this.emitRecoverable(new Error(`dropped presence entry for key ${JSON.stringify(e.key)}: not a usable presence row (a record with card.id equal to its KV key, a string card.name, a known status, a finite ts, string display fields and a condition with a string code)`));
             return undefined;
         }
+        // isPresenceRow is the one gate: it refuses a row whose card.id disagrees with the KV key, so a
+        // forged or mis-keyed record never surfaces a spoofed roster identity. The write-side scoping
+        // ($KV.<presenceBucket>.<own-id>) is the primary guard.
         return p;
     }
     applyPresence(id, raw) {
@@ -7537,29 +7609,6 @@ function assertPartsSerializable(parts) {
         }
     }
 }
-/** A `Record` so that adding a status to {@link PresenceStatus} fails to compile until the reader accepts it. */
-const PRESENCE_STATUSES = { idle: true, waiting: true, working: true, offline: true };
-/** The presence fields this observer computes with, in their SPEC §6 types. A missing `ts` or text
- *  such as `"nope"` makes the liveness arithmetic NaN, and a JSON `1e400` parses as `Infinity`;
- *  neither age is ever greater than the TTL, so the row would stay live after its key expired
- *  (#2612). `card.name` is the roster sort key: a non-string throws inside the watch loop and
- *  ends the watch. `status` decides offline and is what roster consumers read. `card.role`,
- *  `activity`, `attention` and `condition.code` are printed as text, and the peer writes its own
- *  row: an object such as `{"toString":null,"valueOf":null}` throws on conversion and would fail
- *  every reader's roster. A present `condition` must be an object with a string `code`, which SPEC
- *  §6 makes required: `null`, a bare string or an object with no code is dropped too. `card.id` is
- *  left to the KV-key check, which rejects anything but the key string. */
-function isUsablePresence(value) {
-    return isRecord(value) && isRecord(value.card) && typeof value.card.name === "string" &&
-        isOptionalString(value.card.role) &&
-        typeof value.status === "string" && Object.hasOwn(PRESENCE_STATUSES, value.status) &&
-        typeof value.ts === "number" && Number.isFinite(value.ts) &&
-        isOptionalString(value.activity) && isOptionalString(value.attention) &&
-        (value.condition === undefined || (isRecord(value.condition) && typeof value.condition.code === "string"));
-}
-function isOptionalString(value) {
-    return value === undefined || typeof value === "string";
-}
 /** Shallow-equal two per-channel-mode maps (presence dedup): a change must re-emit, so an attention
  *  toggle isn't swallowed as a quiet heartbeat. Absent and empty compare equal. */
 function sameChannelModes(a, b) {