@cotal-ai/manager 0.52.1 → 0.54.0

package/dist/manager.js

@@ -4,8 +4,8 @@ import { hostname } from "node:os";
 import { credsAuthenticator } from "@nats-io/transport-node";
 import { existsSync, lstatSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
 import { isAbsolute, join, dirname, resolve } from "node:path";
-import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, divergentSecretStoreNotice, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, epRequestSubject, epCallerReplyFilter, parseEpSubject, controlServiceSubject, eventChannelPrincipal, } from "@cotal-ai/core";
-import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, createManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
+import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, epRequestSubject, epCallerReplyFilter, parseEpSubject, controlServiceSubject, eventChannelPrincipal, } from "@cotal-ai/core";
+import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, createManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
 import { createRuntime, isCustodialRuntime, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, } from "./runtime/index.js";
 import { AttachEndpoint } from "./attach-endpoint.js";
 import { makeManagerEndpointEvictionEvidence, makeManagerEndpointEvictor } from "./endpoint-evict.js";
@@ -536,6 +536,7 @@ export class Manager {
     resumeDurableCommitToken;
     resumedAgentNames = new Set();
     remoteAuthority;
+    remoteExecutorCreds;
     /** Every broker dial this manager makes, through ONE transport decision.
      *
      *  `this.servers` is whatever the mesh record held (`cotal supervise` reads it from there), so it
@@ -555,6 +556,7 @@ export class Manager {
         this.eventsRequired = opts.eventsRequired === true;
         this.maxSessions = opts.maxSessions;
         this.remoteAuthority = opts.remoteAuthority;
+        this.remoteExecutorCreds = opts.remoteAuthority?.executorCreds;
         if (opts.remoteAuthority)
             this.managerLifecycleUid = opts.remoteAuthority.lifecycleUid;
         this.secrets = opts.secretStore ?? workspaceSecretStore(this.workspaceRoot);
@@ -663,9 +665,24 @@ export class Manager {
         // through the SecretStore seam (`this.secrets` — the injected `ManagerOptions.secretStore`, or
         // the local `.cotal/auth/auth.json` FS default), so a HOSTED composition mints from its KMS/Vault
         // and no signing seed is ever read from the hosted disk. `this.space` cross-checks the bundle.
-        this.auth = await getSpaceAuth(this.secrets, this.space);
-        if (this.remoteAuthority && this.auth)
-            throw new Error("remote manager-service authority cannot be combined with local space signing trust - choose one authority path, never a fallback");
+        //
+        // REMOTE AUTHORITY (#1944): a manager holding host-issued remote authority mints nothing from a
+        // local signer, so it never composes or validates local trust. It asks ONE question of the
+        // store — does THIS space's own local signing trust exist (`localTrustOfSpace`) — because that
+        // alone is the conflict of authorities a supervisor refuses. The stock behaviour this replaces
+        // read the whole root's trust chain (`getSpaceAuth`): with no records for this space it fell
+        // back to the legacy monolith of an UNRELATED static tenant and reported that bundle as
+        // "corrupt or mislabeled". Other tenants' records — split (their account + the shared
+        // per-root broker record) or monolith — are not this space's trust and never refuse; an
+        // unreadable record on a key that is read refuses loud with the store's own parse message.
+        if (this.remoteAuthority) {
+            if ((await localTrustOfSpace(this.secrets, this.space)).present)
+                throw new Error("remote manager-service authority cannot be combined with local space signing trust - choose one authority path, never a fallback");
+            this.auth = undefined;
+        }
+        else {
+            this.auth = await getSpaceAuth(this.secrets, this.space);
+        }
         // USER-MODE detection is FAIL-CLOSED on the on-disk marker (the space-scoped state dir), never
         // on the mutable mesh registry alone — registry drift/tamper must not let a user-auth space
         // take the static self-mint branch. A marker/registry disagreement is a refused start with the
@@ -818,6 +835,11 @@ export class Manager {
             this.credRenewTimer = setInterval(() => { void this.renewDaemonCreds(); }, credRenewIntervalMs(STANDING_RENEWABLE_TTL_SEC));
             this.credRenewTimer.unref?.();
         }
+        else if (this.remoteAuthority) {
+            await this.renewRemoteExecutor();
+            this.credRenewTimer = setInterval(() => { void this.renewRemoteExecutor(); }, credRenewIntervalMs(5 * 60));
+            this.credRenewTimer.unref?.();
+        }
         // stop() fences before it waits for an accepted startup reconciliation terminal. Once that
         // terminal drains, start() and stop() resume from the same await boundary; start must observe
         // the fence before it can publish a fresh service registration after shutdown.
@@ -868,13 +890,27 @@ export class Manager {
         // manager records each agent's read ACL at spawn (`commitAcl`, in provisionAgent) so the daemon can
         // re-authorize it; that is the only Plane-3 state the manager touches, and it rides minting.
     }
+    /** The delivery shard whose lease names the process that reloads the standing credentials.
+     *  Delivery is single-shard today and the endpoint's own responder reads row 0; naming it here
+     *  keeps the manager's binding check and that reply arguing about the same row. */
+    static DELIVERY_SHARD = 0;
     /** Proof that this manager remints into the store the delivery daemon reloads from.
      *  Fingerprint-only `reloadCreds` is safe only after this. Two named, different stores
      *  refuse the pass. A daemon that is not bound yet is not a named store: start and
      *  remint still proceed (tests, delayed delivery), writing this manager's store. The
      *  challenge runs again before every remint, so a later daemon on a foreign store is
      *  refused then rather than certified by an earlier absence. A request timeout is not
-     *  absence: a hung rail would skip the proof, so it fails closed. */
+     *  absence: a hung rail would skip the proof, so it fails closed.
+     *
+     *  ABSENCE IS READ OFF THE LEASE ROW, never off the rail's no-responder shape alone (#1694).
+     *  The rail is queue-grouped, so "no responder answered" is itself something the rail reports,
+     *  and a responder that answers in a shape the client turns into that error makes the pass read
+     *  "absent" while a holder is live on the row — certifying every later remint with no proof at
+     *  all. So a no-responder outcome is decided by the row: a row that names a holder makes the
+     *  pass refuse (undetermined — a live daemon went unanswered, which must not read as "no
+     *  daemon"), and only a row that is absent or holderless settles "absent", the state start()
+     *  treats as not-yet-bound. An unreadable row (a failed read, or a key that exists but does not
+     *  parse) is a genuine unknown and refuses fail-closed, like a hung rail. */
     async daemonStoreRelation() {
         let reply;
         try {
@@ -883,24 +919,77 @@ export class Manager {
         catch (e) {
             const msg = e.message;
             if (isAbsentDeliveryAdmin(msg))
-                return "absent";
+                return this.absentByLeaseRow();
             throw new Error(`could not challenge the delivery daemon's SecretStore: ${msg}`);
         }
         if (!reply.ok)
             throw new Error(reply.error ?? "delivery daemon refused to name the SecretStore it reloads from");
-        let daemon;
+        let answer;
         try {
-            daemon = parseSecretStoreIdentity(reply.data);
+            answer = parseDaemonStoreAnswer(reply.data);
         }
         catch (e) {
-            throw new Error(`delivery daemon named an unreadable SecretStore: ${e.message}`);
-        }
+            throw new Error(`delivery daemon named an unreadable SecretStore: ${e.message}. A daemon older ` +
+                `than #1694 replies with a bare store identity and no answerer binding; this manager ` +
+                `cannot establish which process answered, so nothing reminted - upgrade the delivery daemon`);
+        }
+        // The answerer's own claim first, so an honest non-holder is refused by its own admission and
+        // the operator-facing reason names that. This is the CHEAP half of the test and NOT the binding
+        // one: a responder that does not hold the lease can assert `true` here just as easily.
+        if (!answer.holdsDeliveryLease)
+            throw new Error("the delivery-admin rail was answered by a responder that does not hold this space's " +
+                "delivery lease, so its SecretStore is not the store the daemon reloads from - nothing reminted");
+        // THE BINDING TEST, measured by this manager rather than asserted by the answerer. The lease
+        // row's holder is read here under this manager's own credential; the answerer must BE that
+        // principal. A reply asserting the claim while some other process holds the shard fails here,
+        // which is the case a bare store identity could not see. An unreadable row is an unknown and
+        // refuses too: this test may never be satisfied by failing to run.
+        const holder = await this.ep.deliveryLeaseHolder(Manager.DELIVERY_SHARD);
+        if (holder === undefined)
+            throw new Error("this manager could not read the delivery lease row, so it cannot verify that the answering " +
+                "responder is the process that reloads the standing credentials - nothing reminted");
+        if (holder !== answer.responder)
+            throw new Error(`the delivery-admin rail was answered by ${answer.responder}, which is not the holder of ` +
+                `this space's delivery lease (${holder}), so its SecretStore is not the store the daemon ` +
+                `reloads from - nothing reminted`);
+        const daemon = answer.identity;
         if (!sameSecretStoreIdentity(this.secretStoreIdentity, daemon)) {
             console.error(`! ${divergentSecretStoreNotice(this.secretStoreIdentity, daemon)}`);
             return "divergent";
         }
         return "shared";
     }
+    /** Settle a no-responder challenge outcome from the lease row, never from the rail's own
+     *  reporting shape (#1694). The three cases:
+     *
+     *  1. The row READS but names a holder: a live daemon holds the shard and the rail went
+     *     unanswered. That is undetermined, not "no daemon" — treating it as absent would let this
+     *     manager remint with no proof while a reloading daemon runs, which is exactly what the
+     *     challenge exists to prevent. Refuse.
+     *  2. The row is unreadable — a read that fails (a denied or erroring bucket) OR a key that
+     *     exists but does not parse as a lease record (`readDeliveryLeaseEntry` throws for both):
+     *     undetermined, and it refuses the same fail-closed way a hung rail does. Never a silent
+     *     "absent" over a row the manager could not read.
+     *  3. The row is absent or holderless (no key, a DEL/PURGE tombstone, or a row that names no
+     *     holder): today's "absent", the state start() treats as a daemon that is not bound yet.
+     *
+     *  `deliveryLeaseHolder` swallows its read failures into `undefined`, which would fold case 2
+     *  into case 3 — so this reads the entry directly and lets the read's own failure refuse. */
+    async absentByLeaseRow() {
+        let entry;
+        try {
+            entry = await this.ep.readDeliveryLeaseEntry(Manager.DELIVERY_SHARD);
+        }
+        catch (e) {
+            throw new Error(`could not read the delivery lease row to settle whether the delivery daemon is absent ` +
+                `(${e.message}) - absence is undetermined, nothing reminted`);
+        }
+        const holder = entry?.info.holder;
+        if (holder !== undefined && holder !== "")
+            throw new Error(`the delivery-admin rail reported no responder while the delivery lease names a holder ` +
+                `(${holder}) - absence cannot be read off the rail alone, and nothing reminted`);
+        return "absent";
+    }
     /** Keep this manager's renewal lease alive between credential passes (#1634). The manager bucket
      *  expires keys at MANAGER_LEASE_TTL_MS, far below the credential pass's cadence, so the lease
      *  needs its own heartbeat like the per-instance liveness lease. A non-owner calls this too: the
@@ -933,7 +1022,23 @@ export class Manager {
      *
      *  A manager failing either test serves the space normally and skips only the daemon remint. */
     async claimDaemonRenewalOwnership() {
-        if ((await this.daemonStoreRelation()) === "divergent") {
+        // EVERY refusal outcome drops a held lease, not only "divergent". The refusal paths THROW
+        // (#1694: a non-holder answer, an unbindable answerer, an unreadable or holder-naming row on a
+        // no-responder rail), and a thrown refusal caught by renewDaemonCreds would otherwise leave a
+        // manager that once owned the renewal still holding the lease behind its own heartbeat — the
+        // pass refuses, the log says so, and the space keeps a renewal owner that must not remint.
+        // Deciding here, inside the claim, is what keeps the release on the refusal path itself.
+        let relation;
+        try {
+            relation = await this.daemonStoreRelation();
+        }
+        catch (e) {
+            if (this.daemonRenewalOwner)
+                await this.ep.releaseDaemonRenewalLease().catch(() => { });
+            this.daemonRenewalOwner = false;
+            throw e;
+        }
+        if (relation === "divergent") {
             // Not our daemon's store: drop any lease we hold rather than sit on it, so the manager that
             // CAN remint is not blocked behind us.
             if (this.daemonRenewalOwner)
@@ -1106,6 +1211,23 @@ export class Manager {
             release();
         }
     }
+    /** Renew the remote registration executor through the typed host protocol. A healthy credential
+     * is retained; near-expiry and expired credentials are replaced for the same nkey. */
+    async renewRemoteExecutor(force = false) {
+        if (!this.remoteAuthority)
+            return;
+        const current = this.remoteExecutorCreds;
+        if (!force && current && inspectCredHealth(current).state === "healthy")
+            return;
+        try {
+            this.remoteExecutorCreds = await this.remoteAuthority.renewExecutor();
+        }
+        catch (e) {
+            console.error(`! remote manager executor renewal: ${e.message} - registration maintenance and clean deregistration remain unavailable until renewal succeeds`);
+            if (force)
+                throw e;
+        }
+    }
     /** Warn, on every renewal pass, when a $SYS credential is at or past its renewal point.
      *
      *  The manager is the renewal owner for every credential it CAN re-sign, and these two are the ones
@@ -2193,6 +2315,16 @@ export class Manager {
                 const data = unwrap(await this.opModels(args(ctx)));
                 return { catalogs: Array.isArray(data) ? data : [data] };
             }),
+            goalResult: (ctx) => this.serveGated(ctx, async () => {
+                const gw = this.goalWriter;
+                if (!gw || gw.nc.isClosed())
+                    throw new EpEnvelopeError("unavailable", "the manager goal-reader connection is not standing; the accepted goal is unaffected");
+                const goalId = args(ctx).goalId;
+                // Only the broker-authenticated caller's full incarnation can name this read. The
+                // trusted goal-writer keeps the raw EPF authority; serveEndpoint derives the reply rail.
+                const result = await readGoalResult(gw.ctx, goalRefOf(ctx.subject, goalId));
+                return { goalId, ...(result === undefined ? {} : { result }) };
+            }),
             resolveCwd: (ctx) => this.serveGated(ctx, () => this.resolveSpawnCwd(args(ctx).cwd)),
             // P2 item 2: `spawn` is an ACTION - accept a goal + reply the acceptance floor payload, drive
             // progress + terminal off-handler (no ~30s block). The blocking reply path is gone (pin 8).
@@ -3122,6 +3254,7 @@ export class Manager {
                     ? { ...restart.opts, resume: undefined, prompt: undefined, continueSession }
                     : { ...restart.opts, resume: undefined, prompt: undefined };
                 const spec = connector.buildLaunch(opts);
+                spec.env = { ...spec.env, COTAL_MANAGER_INSTANCE: this.managerInstanceId };
                 const wanted = this.managedPrincipal(a);
                 const joinedAfter = this.ep.getRoster()
                     .filter((p) => p.card.id === wanted && p.lifecycleUid === a.lifecycleUid)
@@ -4122,6 +4255,7 @@ export class Manager {
                 workspaceRoot: this.workspaceRoot,
             };
             const spec = connector.buildLaunch(launchOpts);
+            spec.env = { ...spec.env, COTAL_MANAGER_INSTANCE: this.managerInstanceId };
             const handle = await this.spawnCustodied(name, spec, cwd, custody);
             hooks?.onLaunched?.(); // P2 item 2: the "launched" progress edge (process spawned, pre-presence)
             const managed = {
@@ -4681,6 +4815,7 @@ export class Manager {
                     workspaceRoot: this.workspaceRoot,
                 };
                 const spec = connector.buildLaunch(launchOpts);
+                spec.env = { ...spec.env, COTAL_MANAGER_INSTANCE: this.managerInstanceId };
                 const value = { spec, launchOpts, ...authority };
                 prepared?.set(entry.name, value);
                 if (preflightOnly)
@@ -5108,7 +5243,9 @@ export class Manager {
      *  This is NEVER the manager's standing seed/supervisor connection. */
     async withEndpointServeExecutor(fn) {
         const identity = this.remoteAuthority?.identities.executor ?? newIdentity();
-        const creds = this.remoteAuthority?.executorCreds ?? (this.auth
+        if (this.remoteAuthority && (!this.remoteExecutorCreds || inspectCredHealth(this.remoteExecutorCreds).state !== "healthy"))
+            await this.renewRemoteExecutor(true);
+        const creds = this.remoteExecutorCreds ?? (this.auth
             ? await mintCreds(this.auth, identity, "endpoint-serve-executor", {
                 endpointServeExecutor: { endpoint: MANAGER_ENDPOINT, instanceId: this.managerInstanceId },
                 ...(this.endpointServeExecutorExpiresInSeconds !== undefined