@cotal-ai/core 0.58.0 → 0.59.0

package/dist/provision.js

@@ -18,7 +18,8 @@ import { TTL_RECONCILE_CANARY_KEY, ttlBuckets } from "./streams.js";
 import { join } from "node:path";
 import { decode, encodeOperator, encodeAccount, encodeUser, fmtCreds, } from "@nats-io/jwt";
 import { createOperator, createAccount, fromPublic, fromSeed } from "@nats-io/nkeys";
-import { token, spacePrefix, chatSubject, assertValidChannel, channelInAllow, principalKey, parsePrincipalKey, deprovisionTargetPrincipal, principalTags, assertInboxConnId, DEV_OWNER, unicastSubject, anycastSubject, controlServiceSubject, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, artifactBucket, objectStoreStream, chatStream, dmStream, taskStream, dlvStream, inboxStream, chatHistDurable, dmDurable, taskDurable, dlvDurable, presenceBucket, channelBucket, membersBucket, aclBucket, aclKey, memberKey, assertLifecycleToken, membershipBucket, deliveryBucket, managerBucket, MANAGER_LEASE_KEY, MANAGER_RENEWAL_LEASE_KEY, connzRequestSubject, accountConnectSubject, accountDisconnectSubject, MEMBERSHIP_INBOX_PREFIX, FANOUT_DURABLE, INBOX_READER_DURABLE, } from "./subjects.js";
+import { token, spacePrefix, chatSubject, assertValidChannel, channelInAllow, principalKey, parsePrincipalKey, deprovisionTargetPrincipal, principalTags, assertInboxConnId, DEV_OWNER, unicastSubject, anycastSubject, controlServiceSubject, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, artifactBucket, objectStoreStream, chatStream, dmStream, taskStream, dlvStream, inboxStream, chatHistDurable, dmDurable, taskDurable, dlvDurable, presenceBucket, channelBucket, membersBucket, aclBucket, aclKey, memberKey, assertLifecycleToken, membershipBucket, deliveryBucket, managerBucket, MANAGER_LEASE_KEY, MANAGER_RENEWAL_LEASE_KEY, connzRequestSubject, accountConnectSubject, accountDisconnectSubject, MEMBERSHIP_INBOX_PREFIX, FANOUT_DURABLE, INBOX_READER_DURABLE, livenessSubject, livenessServeFilter, livenessReplyGrant, } from "./subjects.js";
+import { LIVENESS_PLANES } from "./liveness.js";
 import { epCallerGrantRows, epServeGrantRows, epBaselineGrantRows, baselineCallerCapabilities, spawnCallerCapabilities, runCallerCapabilities, epRequestGrantRows, operatorInstrumentCapabilities, instanceOnlyManagerCapabilities, epDescribeAllGrantRow, BASELINE_LIFECYCLE_ENDPOINT, } from "./endpoint-grants.js";
 import { assertServeGrantMintable, finalizeServeIssuance } from "./endpoint-service.js";
 import { effectsBindGrants, poolOwnerBindGrants, goalWriterGrants, sessionLedgerGrants, epAuthBucket, sessionsBucket, epcStreamName, endpointPlaneStreamNames, eptReqStreamName, eptStreamName, hardenedAuthorityStreamNames, timerWriterDurable, timerWriterGrants } from "./endpoint-binding.js";
@@ -532,6 +533,9 @@ export async function mintPublicUserJwt(auth, publicId, profile, opts) {
     }
     return { jwt, exp: validDates.exp };
 }
+/** The profiles whose credential carries caller rails, and can therefore be minted as an
+ *  issuance (SPEC 13.15). Everything else is infrastructure or a one-shot with no rail to bind. */
+const ISSUABLE_PROFILES = new Set(["agent", "control-caller-privileged", "control-caller-admin", "deployer"]);
 /** Build the NATS user permission object for a profile: a default-deny allow-list scoped to
  *  exactly what each profile does. Every profile is now enumerated least-privilege — the former
  *  allow-all `manager` is gone (its roles split across supervisor/provisioner/operator/purger and the
@@ -545,9 +549,6 @@ export async function mintPublicUserJwt(auth, publicId, profile, opts) {
  *  owner + ledger actor + the per-connection ephemeral nkey; the static/dev mint passes
  *  `{owner:"local", actor:<id>, connId:<id>}` via {@link principalOf}. EXPORTED so the callout's injected
  *  `permissionsFor` hook can feed a validated principal straight into the same builder. */
-/** The profiles whose credential carries caller rails, and can therefore be minted as an
- *  issuance (SPEC 13.15). Everything else is infrastructure or a one-shot with no rail to bind. */
-const ISSUABLE_PROFILES = new Set(["agent", "control-caller-privileged", "control-caller-admin", "deployer"]);
 export function permissionsFor(profile, space, pr, opts) {
     // Guard the connId BEFORE any profile builds `_INBOX_<connId>.>`: in user mode connId is a client-
     // chosen nonce (untrusted), so a metacharacter here would escalate the inbox grant to every inbox.
@@ -843,6 +844,20 @@ export function permissionsFor(profile, space, pr, opts) {
         // daemon (NOT the manager). The reply rides this same subtree (`ctl.delivery.<o>.<a>.reply.<n>`, in
         // sub.allow below) so the daemon can answer without broad inbox-publish — see CONTROL_DELIVERY.
         controlServiceSubject(space, CONTROL_DELIVERY, pr.owner, pr.actor),
+        // THE PEER-READABLE LIVENESS PROBE (#1577) — one publish row per plane, each pinned to THIS
+        // agent's own principal slots, so a peer asks as itself and can never forge a probe from
+        // another. This is the row that makes liveness askable by a NON-OWNER: without it, the only
+        // hypothesis a peer could form for any failure was "my credentials are wrong", because the
+        // subjects that answer "is the manager alive" were owner-only.
+        //
+        // WHAT THIS DOES NOT GRANT, which is the half that keeps it from being a back door. It is
+        // PUBLISH ON A REQUEST SUBJECT, not a read of anything. The agent gains NO grant on the manager
+        // bucket (still absent by deliberate omission, below) and NO new read of the delivery bucket.
+        // The answer it can obtain is one enum per plane, because the responder reduces the lease row
+        // before replying and the row never crosses the wire. It cannot subscribe the SERVE filter
+        // either: `live.<plane>.*.*` is not in `sub.allow`, so an agent cannot impersonate a responder
+        // and answer a peer's probe with a comforting lie.
+        ...LIVENESS_PLANES.map((plane) => livenessSubject(space, plane, pr.owner, pr.actor)),
         // JetStream control plane — scoped to this agent's own streams/durables.
         "$JS.API.INFO",
         // STREAM.INFO: CHAT (join watermark, recall drop-marker, channel-list counts — a documented
@@ -1017,9 +1032,15 @@ export function permissionsFor(profile, space, pr, opts) {
     // Replies to this agent's durable join/leave/list requests ride `ctl.delivery.<o>.<a>.>` (NOT the
     // per-id _INBOX), so the scoped delivery daemon can answer without broad inbox-publish.
     const deliveryReplies = `${controlServiceSubject(space, CONTROL_DELIVERY, pr.owner, pr.actor)}.>`;
+    // Liveness answers (#1577) ride the probe's OWN request subtree, exactly like the delivery
+    // control replies above and for the same reason: the responder answers without needing broad
+    // inbox-publish. One row per plane, pinned to this agent's principal, so it can hear answers to
+    // ITS OWN probes and nothing else — it cannot subscribe a peer's reply lane and harvest answers
+    // addressed elsewhere.
+    const livenessReplies = LIVENESS_PLANES.map((plane) => `${livenessSubject(space, plane, pr.owner, pr.actor)}.>`);
     // Manager control replies ride the v0.4 ep reply rail (in `epSub`, keyed on the caller triple) —
     // the `ctl.<tier>.<id>.reply.>` subtrees are gone with the ctl rail (1d).
-    return { pub: { allow: pubAllow, deny: pubDeny }, sub: { allow: [inbox, deliveryReplies, ...subChat, ...epSub] } };
+    return { pub: { allow: pubAllow, deny: pubDeny }, sub: { allow: [inbox, deliveryReplies, ...livenessReplies, ...subChat, ...epSub] } };
 }
 /** One seat's manager control view. It carries no agent messaging or delivery surface, and every
  * manager request, including describe, is pinned to one exact registered instance. */
@@ -1098,6 +1119,15 @@ function supervisorPermissions(space, pr) {
                 // files it requests `reloadCreds` here so adoption is an explicit, auditable event. Self-scoped
                 // request subject (its own owner+actor slots), bounded reply subtree in sub.allow below.
                 controlServiceSubject(space, CONTROL_DELIVERY_ADMIN, pr.owner, pr.actor),
+                // LIVENESS REPLIES for the MANAGER plane (#1577), bounded to the `.reply.` leaf under a
+                // caller's own request subject. The supervisor is the natural responder for this plane: it
+                // is the process that HOLDS the manager lease, so it is the one that can answer the
+                // question honestly without anyone else being granted a read of the lease row.
+                //
+                // REPLIES ONLY, exactly like the delivery daemon's `ctl.delivery.*.*.reply.>`. The row stops
+                // at the leaf, so the supervisor cannot publish to the liveness REQUEST subjects themselves
+                // and therefore cannot forge a probe from a peer.
+                livenessReplyGrant(space, "manager"),
                 // #1694: a POINT READ of the delivery lease row, so the store-identity challenge can verify
                 // that the process which answered the queue-grouped admin rail is the holder that actually
                 // reloads the standing credentials. Without it the challenge can only report what some
@@ -1116,7 +1146,16 @@ function supervisorPermissions(space, pr) {
             // Own reply inbox + the delivery-admin reply subtree for its OWN requests. NO chat/inst/dlv
             // native sub (the supervisor reads no feed), NO manager control-tier serve (1d: that moved to
             // the endpoint-serve credential), NO broad `$JS.>`/`$KV.>` (the residual-2 read/admin path is gone).
-            allow: [`_INBOX_${pr.connId}.>`, `${controlServiceSubject(space, CONTROL_DELIVERY_ADMIN, pr.owner, pr.actor)}.>`],
+            //
+            // PLUS the manager-plane liveness SERVE filter (#1577): `live.manager.*.*`, queue-grouped, so
+            // any credentialed peer's presence probe reaches the lease holder. This is a serve
+            // subscription on a presence-only rail, not a widening of what the supervisor may read: the
+            // request body is empty and the subject carries only the caller's own principal.
+            allow: [
+                `_INBOX_${pr.connId}.>`,
+                `${controlServiceSubject(space, CONTROL_DELIVERY_ADMIN, pr.owner, pr.actor)}.>`,
+                livenessServeFilter(space, "manager"),
+            ],
         },
     };
 }
@@ -1143,6 +1182,12 @@ function remoteManagerPermissions(space, pr, pin) {
     const gateKey = epgateKey("manager", iid);
     const credPrefix = epcredFamilyPrefix("manager", iid);
     const repairKey = eprepairKey("manager", iid);
+    // The SUPERVISOR actor of the pair, which is the one that runs `Manager.start()` and therefore
+    // the only one that binds the manager plane's liveness responder (`service.ts` hands
+    // `credentials.supervisor` to the manager; the executor gets the instance-scoped registration
+    // surface). The liveness rows below are pinned to it so the executor does not carry authority to
+    // answer a probe it never serves.
+    const isSupervisorActor = pin.actor === `manager_${iid}`;
     const recordKeys = [
         recordSpecKey(RECORD_KINDS.svc, ["manager", iid]),
         recordStatusKey(RECORD_KINDS.svc, ["manager", iid]),
@@ -1177,9 +1222,35 @@ function remoteManagerPermissions(space, pr, pin) {
                 `$JS.API.CONSUMER.DELETE.KV_${AUTH}.>`,
                 ...recordKeys.map((key) => `$JS.API.DIRECT.GET.KV_${REC}.$KV.${REC}.${key}`),
                 `$JS.API.STREAM.MSG.GET.KV_${REC}`,
+                // LIVENESS REPLIES for the MANAGER plane (#1577), bounded to the `.reply.` leaf under a
+                // caller's own request subject — the same row and the same bound as the local supervisor
+                // profile holds.
+                //
+                // WHY THIS PROFILE NEEDS IT AT ALL. `Manager.start()` binds the manager plane's liveness
+                // responder unconditionally, and a remote manager runs the same `start()` under this
+                // credential. Without the serve subscription below the bind is denied, and a peer probing
+                // the manager plane then gets the broker's own no-responders answer, which this surface
+                // grades `unbound` — a definite verdict about the plane, produced by a gap in a credential,
+                // while the remote manager is bound and serving. A surface that exists to stop a failure to
+                // find out being rendered as a finding must not render its own missing grant as one.
+                //
+                // REPLIES ONLY. The row stops at the leaf, so a remote manager cannot publish to the
+                // liveness REQUEST subjects and therefore cannot forge a probe that appears to come from a
+                // peer, exactly as the supervisor row cannot.
+                ...(isSupervisorActor ? [livenessReplyGrant(space, "manager")] : []),
+            ],
+        },
+        sub: {
+            allow: [
+                `_INBOX_${pr.connId}.>`,
+                // The manager-plane liveness SERVE filter (#1577): `live.manager.*.*`, queue-grouped, so a
+                // credentialed peer's presence probe reaches this instance. A serve subscription on a
+                // presence-only rail and not a widening of what a remote manager may read: the request body
+                // is empty and the subject carries only the caller's own principal. The MANAGER plane only —
+                // this credential cannot serve `delivery`, so each plane still answers for itself.
+                ...(isSupervisorActor ? [livenessServeFilter(space, "manager")] : []),
             ],
         },
-        sub: { allow: [`_INBOX_${pr.connId}.>`] },
     };
 }
 /** The human-CLI OPERATOR permission set (closure (ii), residual 2) — the ephemeral key the headless
@@ -1718,17 +1789,6 @@ function provisionerPermissions(space, pr) {
         sub: { allow: [`_INBOX_${pr.connId}.>`] },
     };
 }
-/** The ephemeral, LIFECYCLE-PINNED §13.1 state-write permission set for the STATIC manager's
- *  lifecycle executor (Unit B). One credential per lifecycle OPERATION (activation, terminal,
- *  renewal ledger append): every grant names exactly ONE incarnation's keys — the alias head,
- *  the uid reservation, the manager slot row, the issuance gate, and the `cred.<uid>.>` ledger
- *  family — so a leaked executor cred can move one incarnation's state machine and nothing else.
- *
- *  Reads: records reads ride the keyed Direct Get form (the key is ON the subject, so the read
- *  grant stays key-pinned); the auth store is leader-served (`allow_direct=false`), so its reads
- *  are body-selected `STREAM.MSG.GET` — stream-scoped, NOT key-scoped (the requested key rides
- *  the PAYLOAD, which a subject grant cannot see). NAMED RESIDUAL: for its one-shot lifetime the
- *  executor can READ (never write) other rows in the auth store. */
 /** The ISSUER permission set (SPEC 13.15): value-writes on the evidence store (evidence, attempt,
  *  source index: create-only and CAS rows, keyed by generation) and on the accepted-row store
  *  (create-only, keyed by the client's token); the leader-served point read on the evidence
@@ -1775,6 +1835,17 @@ function runAdmitterPermissions(space, pr, pin) {
         sub: { allow: [`_INBOX_${pr.connId}.>`] },
     };
 }
+/** The ephemeral, LIFECYCLE-PINNED §13.1 state-write permission set for the STATIC manager's
+ *  lifecycle executor (Unit B). One credential per lifecycle OPERATION (activation, terminal,
+ *  renewal ledger append): every grant names exactly ONE incarnation's keys — the alias head,
+ *  the uid reservation, the manager slot row, the issuance gate, and the `cred.<uid>.>` ledger
+ *  family — so a leaked executor cred can move one incarnation's state machine and nothing else.
+ *
+ *  Reads: records reads ride the keyed Direct Get form (the key is ON the subject, so the read
+ *  grant stays key-pinned); the auth store is leader-served (`allow_direct=false`), so its reads
+ *  are body-selected `STREAM.MSG.GET` — stream-scoped, NOT key-scoped (the requested key rides
+ *  the PAYLOAD, which a subject grant cannot see). NAMED RESIDUAL: for its one-shot lifetime the
+ *  executor can READ (never write) other rows in the auth store. */
 function lifecycleExecutorPermissions(space, pr, pin) {
     const REC = recordsBucket(space), AUTH = epAuthBucket(space);
     // ALL keys DERIVED here from the pin coordinates — the slot key is `staticSlotKey(owner, alias)`,
@@ -1810,19 +1881,6 @@ function lifecycleExecutorPermissions(space, pr, pin) {
         sub: { allow: [`_INBOX_${pr.connId}.>`] },
     };
 }
-/** The ephemeral, ENDPOINT-INSTANCE-PINNED endpoint-serve executor permission set (P2 item 1,
- *  1a-serve): the manager mints this per registration/serve-mint op and drives the endpoint
- *  registration barrier's `epgate` CAS + the mint fence's `epcred` stage/revoke THROUGH it — never
- *  its standing seed/supervisor connection (critic #1's manager-specific "no seed shortcut"). Every
- *  WRITE is key-pinned to exactly ONE (endpoint, instanceId): the gate `epgate.<ep>.<iid>`, its
- *  serving ledger family `epcred.<ep>.<iid>.>`, and the registration's two records keys (the
- *  instance's `svc` spec + the endpoint's governance head — `registerServiceInstance` drives the
- *  slot-take/promote over this same connection). A leaked/mis-constructed executor can move exactly
- *  one endpoint instance's serve state and nothing else. The auth store is `allow_direct=false`, so
- *  reads are leader-served `STREAM.MSG.GET` (stream-scoped, NOT key-scoped — the key rides the
- *  payload); enumeration of the epcred family rides an ordered `keys()` consumer. NAMED RESIDUAL:
- *  for its one-shot lifetime the executor can READ (never write) other auth rows — endpoint/
- *  credential metadata, no bearer bytes; every WRITE stays key-pinned. */
 /** The SELF-MEDIATED goal-writer profile (P2 item 2, spawn-as-action): exactly
  *  {@link goalWriterGrants} for ITS endpoint — the goal bind + terminal facts, the goal-record KV
  *  writes, and the leader-served fencing reads — plus the connection-scoped reply inbox. Disjoint
@@ -1880,6 +1938,19 @@ function sessionLedgerPermissions(space, pr) {
     const g = sessionLedgerGrants(space, pr.connId);
     return { pub: { allow: g.publish }, sub: { allow: g.subscribe } };
 }
+/** The ephemeral, ENDPOINT-INSTANCE-PINNED endpoint-serve executor permission set (P2 item 1,
+ *  1a-serve): the manager mints this per registration/serve-mint op and drives the endpoint
+ *  registration barrier's `epgate` CAS + the mint fence's `epcred` stage/revoke THROUGH it — never
+ *  its standing seed/supervisor connection (critic #1's manager-specific "no seed shortcut"). Every
+ *  WRITE is key-pinned to exactly ONE (endpoint, instanceId): the gate `epgate.<ep>.<iid>`, its
+ *  serving ledger family `epcred.<ep>.<iid>.>`, and the registration's two records keys (the
+ *  instance's `svc` spec + the endpoint's governance head — `registerServiceInstance` drives the
+ *  slot-take/promote over this same connection). A leaked/mis-constructed executor can move exactly
+ *  one endpoint instance's serve state and nothing else. The auth store is `allow_direct=false`, so
+ *  reads are leader-served `STREAM.MSG.GET` (stream-scoped, NOT key-scoped — the key rides the
+ *  payload); enumeration of the epcred family rides an ordered `keys()` consumer. NAMED RESIDUAL:
+ *  for its one-shot lifetime the executor can READ (never write) other auth rows — endpoint/
+ *  credential metadata, no bearer bytes; every WRITE stays key-pinned. */
 function endpointServeExecutorPermissions(space, pr, pin) {
     const AUTH = epAuthBucket(space), REC = recordsBucket(space);
     // DERIVED from (endpoint, instanceId) via the core builders — never a caller literal.
@@ -2053,6 +2124,10 @@ function deliveryPermissions(space, pr) {
         `$JS.API.CONSUMER.MSG.NEXT.${INBOX}.${INBOX_READER_DURABLE}`,
         `$JS.API.CONSUMER.DELETE.${INBOX}.${INBOX_READER_DURABLE}`,
         `$JS.ACK.${INBOX}.${INBOX_READER_DURABLE}.>`,
+        // The reader removes an entry addressed to a retired lifecycle (its ACL row is tombstoned), so the
+        // store does not keep what no reader can ever deliver. Nothing new in substance: the reader can
+        // already ack any INBOX entry away.
+        `$JS.API.STREAM.MSG.DELETE.${INBOX}`,
         "$JS.FC.>", // ordered-consumer flow control
         // Reads: presence (@mention resolve) + channel registry (delivery class) + members + ACL (re-auth).
         ...kvRead(PKV), ...kvRead(CHKV), ...kvRead(MKV), ...kvRead(AKV),
@@ -2098,11 +2173,21 @@ function deliveryPermissions(space, pr) {
         // The privileged delivery-admin rail (D5 slice 5/6): same replies-only shape. Requests reach the
         // daemon on the sub below; only the supervisor cred can PUBLISH them (nats-server is the boundary).
         `${p}.ctl.delivery-admin.*.*.reply.>`,
+        // LIVENESS REPLIES for the DELIVERY plane (#1577) — same replies-only shape as the two control
+        // rails above. The daemon is the only process that can grade its own responder honestly: it
+        // knows its own incarnation, so a predecessor's `ready:true` corpse in the lease classifies as
+        // stale rather than as its own health. Bounded to the `.reply.` leaf, so it cannot publish to
+        // the request subjects and forge a probe from a peer.
+        livenessReplyGrant(space, "delivery"),
     ];
     const sub = [
         `_INBOX_${pr.connId}.>`,
         `${p}.ctl.delivery.*.*`, // serve the delivery control service (queue-grouped; owner+actor caller slots)
         `${p}.ctl.delivery-admin.*.*`, // serve the privileged admin rail (reloadCreds; eviction executor next)
+        // Serve the delivery plane's liveness probe (#1577), queue-grouped. Presence-only: the handler
+        // reduces the lease row to one enum before replying, so this subscription lets the daemon ANSWER
+        // a peer without any peer gaining a read of the lease.
+        livenessServeFilter(space, "delivery"),
     ];
     return { pub: { allow: pub }, sub: { allow: sub } };
 }
@@ -2204,17 +2289,6 @@ export async function mintConnectionEvictorCreds(auth, identity, opts = {}) {
     const creds = fmtCreds(userJwt, fromSeed(new TextEncoder().encode(identity.seed)));
     return new TextDecoder().decode(creds);
 }
-/** Render the `nats-server` config that trusts ONE broker operator and serves N spaces' accounts via
- *  the in-config MEMORY resolver.
- *
- *  Broker trust (operator + system account) comes from `broker` and has exactly one owner; the
- *  per-space data accounts are listed in `spaces`. Every space account is asserted to be signed by
- *  THIS broker's operator before it is preloaded: rendering a foreign-signed account would either
- *  refuse broker boot or, worse, advertise a tenant the broker cannot actually authenticate.
- *
- *  NOTE (W4): the MEMORY resolver is one static whole-broker map, so every mutation rewrites all of
- *  it. Concurrent add/remove of spaces needs a broker-authoritative inventory with generation/CAS
- *  and atomic promotion above this function; this renderer is deliberately pure. */
 /**
  * Render the config for an OPEN (no-auth) broker.
  *
@@ -2271,6 +2345,17 @@ function renderTlsBlock(transport) {
 }
 `;
 }
+/** Render the `nats-server` config that trusts ONE broker operator and serves N spaces' accounts via
+ *  the in-config MEMORY resolver.
+ *
+ *  Broker trust (operator + system account) comes from `broker` and has exactly one owner; the
+ *  per-space data accounts are listed in `spaces`. Every space account is asserted to be signed by
+ *  THIS broker's operator before it is preloaded: rendering a foreign-signed account would either
+ *  refuse broker boot or, worse, advertise a tenant the broker cannot actually authenticate.
+ *
+ *  NOTE (W4): the MEMORY resolver is one static whole-broker map, so every mutation rewrites all of
+ *  it. Concurrent add/remove of spaces needs a broker-authoritative inventory with generation/CAS
+ *  and atomic promotion above this function; this renderer is deliberately pure. */
 export function serverConfig(broker, spaces, opts) {
     if (!spaces.length)
         throw new Error("serverConfig: at least one space account is required");