@cotal-ai/manager 0.70.0 → 0.70.1

package/dist/manager.js

@@ -1,11 +1,10 @@
-import { execFile } from "node:child_process";
 import { createHash, randomUUID, randomBytes } from "node:crypto";
 import { hostname } from "node:os";
 import { credsAuthenticator } from "@nats-io/transport-node";
 import { existsSync, lstatSync, mkdirSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
 import { isAbsolute, join, dirname, resolve } from "node:path";
 import Headless from "@xterm/headless";
-import { CotalEndpoint, MANAGED_HANDOFF_KIND, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, modelPolicyRefusal, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, resolveReadAcl, bearerCommandFailure, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
+import { CotalEndpoint, MANAGED_HANDOFF_KIND, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, modelPolicyRefusal, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, resolveReadAcl, runAgentBearer, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
 import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, claimManagerSiblingIdentities, connectorInstallHint, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveAgentType, resolveOnPath, createManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
 import { createRuntime, isCustodialRuntime, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, } from "./runtime/index.js";
 import { AttachEndpoint } from "./attach-endpoint.js";
@@ -31,7 +30,7 @@ import { ManagerSessionPlane, openSessionLedgerKv } from "./session/index.js";
 // P2 item 1 (1a-serve): the manager as an ordinary v0.4 `service` endpoint — the §13.1
 // endpoint-serve credential subsystem (gate provisioning, registration barrier, mint fence) plus
 // the register/authorize/serve seams, all driven over a scoped one-shot executor connection.
-import { provisionEndpointGateOpen, endpointRegistrationBarrier, serveIssuanceGateKv, commitSiblingIssuance, markLedgerRowRevoked, epcredRowKey, epgateKey, parseEndpointGate, readEndpointGateGeneration, registerServingInstance, deregisterServiceInstance, serveEndpoint, bindGoal, createGoal, transitionGoal, commitGoalResult, settleGoalUncertain, readGoalResult, readGoalStatus, readGoalSpec, recordGoalIndex, readGoalIndex, clearGoalIndex, listGoalIndex, mintCheckpoint, resumeCheckpoint, readCheckpointSettle, readCheckpointSpec, expireCheckpoint, GOAL_TERMINAL_STATES, goalRefOf, goalProgressTopic, epeSubject, submissionFingerprint, EpEnvelopeError, lifecycleBlocked, lifecycleBlockedFrom, renderLifecycleBlocked, } from "@cotal-ai/core";
+import { provisionEndpointGateOpen, endpointRegistrationBarrier, serveIssuanceGateKv, commitSiblingIssuance, markLedgerRowRevoked, epcredRowKey, epgateKey, parseEndpointGate, readEndpointGateGeneration, registerServingInstance, deregisterServiceInstance, serveEndpoint, bindGoal, createGoal, transitionGoal, commitGoalResult, requestGoalCancel, goalAlreadyTerminal, settleGoalUncertain, readGoalResult, readGoalStatus, readGoalSpec, recordGoalIndex, readGoalIndex, clearGoalIndex, listGoalIndex, mintCheckpoint, resumeCheckpoint, readCheckpointSettle, readCheckpointSpec, expireCheckpoint, GOAL_TERMINAL_STATES, goalRefOf, goalProgressTopic, epeSubject, submissionFingerprint, EpEnvelopeError, lifecycleBlocked, lifecycleBlockedFrom, renderLifecycleBlocked, } from "@cotal-ai/core";
 import { MANAGER_ENDPOINT, managerClusterArtifacts, managerCommandDefs, managerContractArtifactValues } from "./manager-service-contract.js";
 import { staticLifecycleTransport, activateStaticLifecycle, runStaticTerminal, readStaticSlot, casStaticSlot, recordSlotCredential, appendStaticCredentialRow, planStaticSlotResume, ownedBySibling, observeStaticSlot, listStaticSlotObservations, renderStaticSlotObservation, StaticSlotReadError, STATIC_SLOT_READ_FAILED_DETAIL, } from "./static-lifecycle.js";
 /** Minimum slot lifetime for rate-flooring (P4c). A slot freed (by despawn OR natural exit/reap)
@@ -148,19 +147,6 @@ async function attemptCleanup(failed, what, step) {
         failed.push(`${what}: ${rejectionText(err)}`);
     }
 }
-/** Run the agent's bearer argv once, pre-launch — the end-to-end auth preflight (state dir, daemon,
- *  ledger row, secret). Its stderr is the provider command's operator-exact sentence; surface it
- *  verbatim as the spawn refusal. */
-function execBearerPreflight(argv) {
-    const timeout = 30_000;
-    return new Promise((res, rej) => {
-        execFile(argv[0], argv.slice(1), { timeout, maxBuffer: 64 * 1024 }, (err, _stdout, stderr) => {
-            if (err)
-                return rej(bearerCommandFailure(err, stderr, timeout));
-            res();
-        });
-    });
-}
 /** Reject `p` with `Error(msg)` if it hasn't settled within `ms`; clears the timer when `p` settles so it
  *  never keeps the loop alive. Used to bound the detached deprovision so its fail-loud log is guaranteed. */
 function withTimeout(p, ms, msg) {
@@ -252,6 +238,20 @@ function foreignEventChannels(channels, owner, actor) {
         return p !== null && !(p.owner === owner && p.actor === actor);
     });
 }
+/** One connector's declared harness binaries resolved on PATH now. The refusal sentence is built only
+ *  here, so the boot row and the check for a connector registered after boot can never word it apart. */
+function resolveHarness(name, requires) {
+    const binaries = {};
+    const missing = [];
+    for (const bin of requires) {
+        const path = resolveOnPath(bin);
+        if (path)
+            binaries[bin] = path;
+        else
+            missing.push(bin);
+    }
+    return missing.length ? { binaries, reason: `${name} harness needs ${missing.join(", ")} on PATH - not found` } : { binaries };
+}
 function injectedManagerStoreIdentity(store) {
     if (store.identity !== undefined)
         return parseSecretStoreIdentity(store.identity);
@@ -2875,6 +2875,9 @@ export class Manager {
             // turns addressed to ITS incarnation and yields them; no reach beyond itself exists here.
             turnPending: (ctx) => this.serveGated(ctx, () => this.turnPendingFor(ctx.subject.caller)),
             turnYield: (ctx) => this.serveGated(ctx, () => this.serveTurnYield(ctx.subject.caller, args(ctx))),
+            // The reserved goal cancel (SPEC 13.6 item 4), served for the relay: the goal ref is the
+            // authenticated caller's own, so a run withdraws only the turns it submitted.
+            cancel: (ctx) => this.serveGated(ctx, () => this.serveTurnCancel(ctx)),
             stopSelf: (ctx) => this.serveGated(ctx, () => unwrap(this.opStopSelf(callerOf(ctx), args(ctx)))),
             definePersona: (ctx) => this.serveGated(ctx, () => unwrap(this.opDefinePersona(args(ctx), callerOf(ctx), false))),
             listPersonas: (ctx) => this.serveGated(ctx, () => unwrap(this.opListPersonas(callerOf(ctx), false))),
@@ -3248,7 +3251,9 @@ export class Manager {
                 "--token-file", tokenPath,
                 "--health-file", healthPath,
             ];
-            await execBearerPreflight(bearerCmd);
+            // One bearer run before the launch is the end-to-end auth preflight (state dir, daemon, ledger
+            // row, secret); its failure sentence is the spawn refusal.
+            await runAgentBearer(bearerCmd);
             return { owner, lifecycleUid: opts.lifecycleUid, files, launch: { owner, actor: name, sentinelCredsPath: sentinelPath, bearerCmd } };
         }
         catch (e) {
@@ -3350,7 +3355,7 @@ export class Manager {
                 "--token-file", files.actorToken,
                 "--health-file", files.health,
             ];
-            await execBearerPreflight(bearerCmd);
+            await runAgentBearer(bearerCmd);
             return {
                 owner: material.owner,
                 lifecycleUid: material.lifecycleUid,
@@ -4496,19 +4501,9 @@ export class Manager {
         else {
             declared = registry.all("connector").map((connector) => ({ name: connector.name, requires: connector.requires ?? [] }));
         }
-        for (const connector of declared) {
-            const name = connector.name;
-            const binaries = {};
-            const missing = [];
-            for (const bin of connector.requires) {
-                const path = resolveOnPath(bin);
-                if (path)
-                    binaries[bin] = path;
-                else
-                    missing.push(bin);
-            }
-            if (missing.length) {
-                const reason = `${name} harness needs ${missing.join(", ")} on PATH - not found`;
+        for (const { name, requires } of declared) {
+            const { binaries, reason } = resolveHarness(name, requires);
+            if (reason) {
                 rows.push({ agent: name, state: "unavailable", binaries, reason });
                 console.error(`! manager boot: connector ${name} unavailable - ${reason}`);
             }
@@ -4540,18 +4535,17 @@ export class Manager {
         return `${reason}. This manager cannot launch that harness. The space may have other managers; pin one with --on <instance> (the whole id, as ps prints it)`;
     }
     /** The harness binaries a managed launch runs, as boot resolved them, or the refusal naming what is
-     *  missing. Spawn and resume both take it, so a seat launches from the same files whichever path
-     *  starts it. A connector registered after boot has no inventory row and is checked on PATH now. */
+     *  missing. Spawn, resume and the model catalog all take it, so a seat launches from the same files
+     *  whichever path starts it and the catalog never disagrees with a launch about whether the harness
+     *  is there. A connector registered after boot has no inventory row and is checked on PATH now. */
     launchHarness(connector) {
         const row = this.connectorStatuses.find((status) => status.agent === connector.name);
         if (row?.state === "unavailable")
             return { refusal: row.reason ?? `${connector.name} harness is unavailable` };
         if (row)
             return { binaries: row.binaries };
-        const missing = (connector.requires ?? []).filter((bin) => !resolveOnPath(bin));
-        if (missing.length)
-            return { refusal: `${connector.name} harness needs ${missing.join(", ")} on PATH - not found` };
-        return {};
+        const { reason } = resolveHarness(connector.name, connector.requires ?? []);
+        return reason ? { refusal: reason } : {};
     }
     /** The refusal for a launch choice the connector cannot honour. Spawn and resume both ask before
      *  any reserve or mint, so a rule changed here holds on both paths. */
@@ -4573,14 +4567,9 @@ export class Manager {
         const one = async (connector) => {
             if (!connector.listModels)
                 return { agent: connector.name, supported: false, models: [] };
-            const missing = (connector.requires ?? []).filter((bin) => !resolveOnPath(bin));
-            if (missing.length)
-                return {
-                    agent: connector.name,
-                    supported: true,
-                    models: [],
-                    error: `${connector.name} harness needs ${missing.join(", ")} on PATH - not found`,
-                };
+            const harness = this.launchHarness(connector);
+            if ("refusal" in harness)
+                return { agent: connector.name, supported: true, models: [], error: harness.refusal };
             try {
                 const catalog = await connector.listModels({ refresh });
                 return { agent: connector.name, supported: true, ...catalog };
@@ -8176,6 +8165,58 @@ export class Manager {
         this.maybeStopTurnSweep();
         return { goalId, state: fact.state };
     }
+    /** The caller withdraws a turn it submitted (the reserved `cancel`, SPEC 13.6 item 4), so the
+     *  seat is not shown it again and a late yield is answered from the cancelled terminal. Only a
+     *  relay this incarnation holds is moved to `cancelling`, because only its holder can commit the
+     *  terminal; an ended goal is refused with its cached outcome, which keeps a repeated cancel
+     *  idempotent. The pending entry is the latch the yield and the deadline sweep race on, taken
+     *  before the first await so a concurrent cancel finds nothing to take. A failed cancel puts it
+     *  back so the relay still ends at its deadline, unless something ended the relay while it was
+     *  held: a yield that read the entry first, or this commit with only its reply lost. An entry
+     *  whose outcome cannot be read back stays out and converges through the goal index at the
+     *  next boot, as a failed deadline commit does. */
+    async serveTurnCancel(ctx) {
+        if (!this.goalReconcileDone)
+            throw new EpEnvelopeError("unavailable", "the manager is still reconciling accepted goals at boot; the pending-turn index is not rebuilt yet, retry (SPEC 13.6)");
+        const gw = this.goalWriter;
+        if (!gw)
+            throw new EpEnvelopeError("unavailable", "the manager goal-writer connection is not standing (SPEC 13.6)");
+        const raw = (ctx.request.args ?? {});
+        const goalId = String(raw.goalId);
+        const ref = goalRefOf(ctx.subject, goalId);
+        const p = this.pendingTurns.get(goalId);
+        const c = p?.ref.caller;
+        if (p === undefined || c?.owner !== ref.caller.owner || c.actor !== ref.caller.actor || c.uid !== ref.caller.uid) {
+            const ended = await readGoalResult(gw.ctx, ref);
+            if (ended !== undefined)
+                throw goalAlreadyTerminal(goalId, ended);
+            throw new EpEnvelopeError("failed-precondition", `no turn "${goalId}" of this caller is pending on this manager; cancel withdraws a relayed turn (SPEC 13.6)`);
+        }
+        this.pendingTurns.delete(goalId);
+        const epoch = this.serviceServe?.grant.epoch ?? 0;
+        let fact;
+        try {
+            await requestGoalCancel(gw.ctx, { request: ctx.subject, goalId, mode: (raw.mode ?? "graceful") });
+            await this.assertGoalWriterEpochCurrent(epoch);
+            ({ fact } = await commitGoalResult(gw.ctx, { ref, now: Date.now(), cause: "cancel", data: { cancelledBy: "caller" }, committer: { instanceId: this.managerInstanceId, epoch } }));
+        }
+        catch (e) {
+            // The remembered answer is read after the durable one, with no await before the put back:
+            // a yield that has dropped the entry remembered its answer, and one still committing drops
+            // the entry it finds. An ended relay is stamped so its acceptance still ages off.
+            if ((await readGoalResult(gw.ctx, ref)) !== undefined)
+                this.markTurnLatchDropped(goalId);
+            else if (this.turnAcceptances.get(goalId)?.settled === undefined)
+                this.pendingTurns.set(goalId, p);
+            throw e;
+        }
+        this.emitGoalProgress(ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
+        this.markTurnLatchDropped(goalId);
+        this.rememberSettledTurn(goalId, fact.state, Date.now());
+        await clearGoalIndex(gw.ctx, ref);
+        this.maybeStopTurnSweep();
+        return { goalId, state: fact.state };
+    }
     /** Commit the deadline terminal for one pending turn whose hold settled EXPIRED: the deny's
      *  predicate is that recorded settle, verified by core against the spec's goal binding. The map
      *  delete is the idempotency latch (yield-loss and sweep race here); a commit failure after it
@@ -8537,11 +8578,17 @@ export class Manager {
                 console.error(`static retirement ${a.name}: retired ${retired} issuance(s) bound to uid ${a.lifecycleUid}`);
             const secrets = this.secrets;
             const files = a.secretPaths ?? agentLifecycleSecretFilePaths(this.workspaceRoot, this.space, a.name, a.lifecycleUid);
+            // The broker teardown mints its own credential, so a creds file that cannot be removed must not
+            // skip it. The failures are thrown together, so the terminal still stops at an incomplete cleanup.
+            const failed = [];
             if (files.creds) {
-                await secrets.delete(agentSecretKeyForFile(files.creds, this.space));
-                rmSync(files.creds, { force: true });
+                const creds = files.creds;
+                await attemptCleanup(failed, `secret ${creds}`, () => secrets.delete(agentSecretKeyForFile(creds, this.space)));
+                await attemptCleanup(failed, `file ${creds}`, () => rmSync(creds, { force: true }));
             }
-            await this.deprovisionBroker(a);
+            await attemptCleanup(failed, "broker teardown", () => this.deprovisionBroker(a));
+            if (failed.length)
+                throw new Error(`cleanup failed: ${failed.join("; ")}`);
         };
         // The process goes before its footprint: reap the orphan seat by reference, then tear down.
         const reapThenCleanup = async () => {