@cotal-ai/runtime 0.49.0 → 0.50.0

package/dist/mesh-handler.js

@@ -17,7 +17,9 @@
  * again.
  */
 import { createHash } from "node:crypto";
-import { mintCheckpoint, heartbeatCheckpoint, resumeCheckpoint, readCheckpointSettle, readCheckpointAnswer, readCheckpointStatus, readCheckpointSpec, reconcileCheckpointSchedule, handleCheckpointFire, eptStreamName, eptSubject, chatStream, waitConsumerName, waitConsumerConfig, isConcreteChannel, assertSafePattern, assertValidChannel, presenceBucket, principalKey, liveKvEntries, IncompleteKvScan, openMembersRegistry, openChannelRegistry, writeChannelConfig, readMember, commitMember, tombstoneMember, StaleMembershipWrite, runNoticeId, writeRunNotice, actionContext, invokeCommand, readGoalResult, readGoalStatus, resolveService, listRunNotices, listRunMigrations, markRunNoticeConsumed, listRunNoticesForRun, readRunRecord, writeRunStatus, EpEnvelopeError, assertAdmittedPublish, assertAdmittedSubscribe, assertNotRevoked, } from "@cotal-ai/core";
+import { isAbsolute } from "node:path";
+import { realpathSync } from "node:fs";
+import { mintCheckpoint, heartbeatCheckpoint, resumeCheckpoint, readCheckpointSettle, readCheckpointAnswer, readCheckpointStatus, readCheckpointSpec, reconcileCheckpointSchedule, handleCheckpointFire, eptStreamName, eptSubject, chatStream, waitConsumerName, waitConsumerConfig, isConcreteChannel, assertSafePattern, assertValidChannel, presenceBucket, principalKey, liveKvEntries, IncompleteKvScan, openMembersRegistry, openChannelRegistry, writeChannelConfig, readMember, commitMember, tombstoneMember, StaleMembershipWrite, runNoticeId, writeRunNotice, actionContext, invokeCommand, replyRefusedBeforeEffect, readGoalResult, readGoalStatus, resolveService, listRunNotices, listRunMigrations, markRunNoticeConsumed, listRunNoticesForRun, readRunRecord, writeRunStatus, EpEnvelopeError, assertAdmittedPublish, assertAdmittedSubscribe, assertNotRevoked, } from "@cotal-ai/core";
 import { renderRunContext } from "./run-context.js";
 import { migrationSeats } from "./migrate.js";
 import { Kvm } from "@nats-io/kv";
@@ -92,7 +94,17 @@ export class MeshHandler {
      * the next effect instead of poisoning every spawn for the handler's lifetime.
      */
     managerService;
-    manager() {
+    manager(instanceId) {
+        // #1616 ITEM 3 — PINNED DISPATCH. An explicit placement target resolves through the EXISTING
+        // instance-dispatch API: `resolveService`'s `instanceId` opt (endpoint-invoke.ts:274-280)
+        // becomes `EpRoute { mode: "inst", instanceId }` at :113, and the handle it returns carries
+        // `pinnedInstanceId` (:315) so `invokeCommand` addresses that instance and never the class
+        // queue. It is deliberately NOT served from `managerService`: that memo holds the class-anycast
+        // resolution, and handing a pinned caller the anycast handle would reinstate the exact fallback
+        // this item removes. A wrong, unavailable or replaced instance therefore fails to resolve —
+        // before a child exists — instead of quietly succeeding somewhere else.
+        if (instanceId !== undefined)
+            return resolveService(this.nc, this.binding.space, this.binding.endpoint, this.binding.caller, { instanceId });
         this.managerService ??= resolveService(this.nc, this.binding.space, this.binding.endpoint, this.binding.caller)
             .catch((e) => {
             this.managerService = undefined;
@@ -100,6 +112,99 @@ export class MeshHandler {
         });
         return this.managerService;
     }
+    /**
+     * One manager call, with a SPEC 13.2 bind refusal REPAIRED rather than raised.
+     *
+     * A run resolves the manager on the class rail and binds the incarnation that answered its
+     * describe. The invoke is a second, independent trip through the same anycast queue, so in a
+     * space with more than one manager it routinely reaches another member, and that member refuses
+     * before dispatching. The refusal is honest for one command and destructive for a run: it says
+     * the command did not run and its remedy is to re-issue, but raised as the effect's own failure
+     * it ends the run and consumes the run id and its journal (#1638).
+     *
+     * So a refusal the responder MARKS as pre-effect is re-issued instead of returned. It is a first
+     * attempt and not a second: the marker together with `not-executed` is the responder's own
+     * statement that no effect of the command exists, which is what {@link replyRefusedBeforeEffect}
+     * checks, and it is the same licence core's `Endpoint.invokeService` re-issues on. The stale
+     * class handle is dropped first, so the re-issue re-describes rather than rebinding the
+     * incarnation that was just refused.
+     *
+     * BOUNDED, because a re-issue draws the same queue again. The describe and the invoke stay two
+     * independent trips, so a space of m managers still splits (m-1)/m of the time and the repair
+     * converges geometrically rather than deterministically; after {@link BIND_SPLIT_REISSUES} of
+     * them the refusal surfaces unchanged, still stating that the command did not run. What removes
+     * the residual is addressing one instance, and the run's caller holds no instance-rail grant for
+     * a command its program did not place (SPEC 13.9, `run-driver-grants.ts`), so that is a wider
+     * change than this one.
+     *
+     * A PINNED handle is never repaired. It addresses one instance by name, so a refusal from it is
+     * that incarnation answering about itself, and re-resolving onto the class rail would reinstate
+     * the anycast fallback #1616 removed.
+     */
+    async invokeManager(service, command, args, opts) {
+        let handle = service;
+        for (let reissues = 0;; reissues += 1) {
+            const reply = await invokeCommand(this.nc, this.binding.space, handle, command, args, opts);
+            if (reply.reply.ok !== false || !replyRefusedBeforeEffect(reply.reply.error))
+                return reply;
+            if (handle.pinnedInstanceId !== undefined || reissues === BIND_SPLIT_REISSUES)
+                return reply;
+            this.managerService = undefined;
+            try {
+                handle = await this.manager();
+            }
+            catch {
+                // The repair could not be attempted. The REFUSAL is what surfaces, not the resolve failure:
+                // every caller of this method already reads a refused reply as "the manager declined", and
+                // this one states that nothing ran, which is the fact a describe timeout raised in its
+                // place would lose.
+                return reply;
+            }
+        }
+    }
+    /**
+     * #1616 item 5 — PHASE A, RESOLVE ON THE HOST THAT WILL LAUNCH. A directory is a fact about one
+     * filesystem, so the only process that can canonicalize it is the manager instance that will
+     * `chdir` into it. This asks the PINNED instance for the canonical form and returns it together
+     * with the identity that answered, so phase B dispatches the resolved path and the journal records
+     * which host resolved it. There is deliberately NO envelope `id`: a resolve binds no goal, takes
+     * no reservation and allocates nothing, so a retry or a crash between the phases costs nothing.
+     *
+     * EVERY failure direction is a REFUSAL. A path the target cannot resolve is not passed through as
+     * the raw string: the raw string would launch somewhere plausible on the manager's own root, which
+     * is the fail-open outcome this repair exists to remove.
+     */
+    async resolveCwd(req, service, cwd, instanceId) {
+        let reply;
+        try {
+            reply = await invokeCommand(this.nc, this.binding.space, service, "resolve-cwd", { cwd }, {
+                deadlineMs: SPAWN_ACCEPT_DEADLINE_MS,
+            });
+        }
+        catch (err) {
+            // An older manager does not list `resolve-cwd`, and `invokeCommand` refuses an unlisted
+            // command with `not-found` before it publishes anything (endpoint-invoke.ts:349-350). That is
+            // the fail-CLOSED direction and it stays closed: a host that cannot answer the question does
+            // not get handed the caller's guess.
+            throw cwdResolutionRefusal(req.persona, instanceId, cwd, err instanceof EpEnvelopeError && err.code === "not-found"
+                ? `this manager serves no resolve-cwd command, so it can state no canonical form (${err.message})`
+                : String(err?.message ?? err));
+        }
+        if (reply.reply.ok === false)
+            throw cwdResolutionRefusal(req.persona, instanceId, cwd, reply.reply.error?.message ?? "refused with no message");
+        const data = reply.reply.data;
+        if (typeof data?.cwd !== "string" || data.cwd.length === 0 || !isAbsolute(data.cwd))
+            throw cwdResolutionRefusal(req.persona, instanceId, cwd, `the reply names no absolute directory (${JSON.stringify(data?.cwd)})`);
+        // The AUTHORITATIVE identity is the one that answered, off the attributed reply's subject, not
+        // the one the caller asked for: the pinned resolve already rejects a reply from another
+        // instance, so recording the responder is recording what was checked.
+        return {
+            cwd: data.cwd,
+            endpoint: reply.responder.endpoint,
+            instanceId: reply.responder.instanceId,
+            ...(typeof data.host === "string" ? { host: data.host } : {}),
+        };
+    }
     /** The branded goal-fact context over this handler's own connection, memoized the same way. */
     actions;
     actionCtx() {
@@ -484,8 +589,7 @@ export class MeshHandler {
             console.error(`! discharge: the cancelled spawn goal "${goalId}" settled ${fact.state} with no readable agent identity; if its seat is up it must be despawned by hand (cotal ps)`);
             return;
         }
-        const service = await this.manager();
-        const reply = await invokeCommand(this.nc, this.binding.space, service, "despawn", { graceful: true }, {
+        const reply = await this.invokeManager(await this.manager(), "despawn", { graceful: true }, {
             target: { mode: "owner", ...target },
             deadlineMs: SPAWN_ACCEPT_DEADLINE_MS,
         });
@@ -1025,6 +1129,28 @@ export class MeshHandler {
         if (ctx.signal.cancelled)
             throw new Cancelled(ctx.signal.reason ?? "cancelled");
         const goalId = ctx.requestId;
+        // Placement is explicit and host-local. Refuse malformed values before manager discovery or
+        // submission, so an invalid request can never turn into an omitted `cwd` and inherit the
+        // manager's workspace root. Existence and launch authority stay with the serving manager.
+        if (req.cwd !== undefined && (typeof req.cwd !== "string" || req.cwd.length === 0 || !isAbsolute(req.cwd)))
+            throw new EffectError("L4000", "spawn", `spawn(${req.persona}) cwd must be a non-empty absolute directory on the serving manager's host; refusing ${JSON.stringify(req.cwd)} rather than falling back`);
+        // #1616 ITEM 2 — THE AFFINITY GATE. A directory is HOST-LOCAL, so a request that names one but
+        // names no manager instance rides the class `one` queue and lands wherever the anycast fell.
+        // The design record calls that combination not acceptable, so it REFUSES. There is deliberately
+        // NO anycast fallback: falling back IS the defect. And the refusal is raised HERE, ahead of
+        // `readPermits`, ahead of `this.manager()`'s describe round-trip, ahead of any submission,
+        // acceptance or launch — so nothing is reserved, claimed or started before it. Legacy
+        // cwd-omitted spawns never reach this line and keep their prior behaviour byte for byte.
+        if (req.cwd !== undefined && req.placement === undefined)
+            throw new EffectError("L4000", "spawn", `spawn(${req.persona}) names a cwd but no placement target: a host-local directory needs an explicit { endpoint, instanceId } manager instance, and this spawn refuses rather than falling back to class anycast`);
+        if (req.placement !== undefined
+            && (typeof req.placement.endpoint !== "string" || req.placement.endpoint.length === 0
+                || typeof req.placement.instanceId !== "string" || req.placement.instanceId.length === 0))
+            throw new EffectError("L4000", "spawn", `spawn(${req.persona}) placement must name both an endpoint and an instanceId; refusing ${JSON.stringify(req.placement)} rather than dispatching unpinned`);
+        // Naming a target the run is not bound to is a mismatched target, not a request to go find it:
+        // automatic manager discovery is outside this bounded repair, so it refuses here too.
+        if (req.placement !== undefined && req.placement.endpoint !== this.binding.endpoint)
+            throw new EffectError("L4000", "spawn", `spawn(${req.persona}) placement targets endpoint ${JSON.stringify(req.placement.endpoint)} but this run is bound to ${JSON.stringify(this.binding.endpoint)}; refusing rather than dispatching off-binding`);
         // A recorded goalId is a previous attempt's ACCEPTANCE: the submission landed and its
         // identity was bound before the crash. Go straight back to the terminal. An entry that says
         // `adoptedFrom` names the goal the ORPHANED spawn submitted, not this step's own request id:
@@ -1073,14 +1199,31 @@ export class MeshHandler {
         try {
             if (ext === undefined) {
                 let reply;
+                // #1616 item 5 — PHASE A, then phase B. A previous attempt's journalled resolution is
+                // reused rather than re-asked, so the path a resumed spawn launches in is the one the
+                // resolve recorded and cannot drift under a re-resolve. A spawn naming no cwd has nothing
+                // to resolve and skips both the invoke and the bind, so its behaviour is unchanged.
+                let resolution = readCwdResolution(recorded?.resolution);
                 try {
-                    const service = await this.manager();
-                    reply = await invokeCommand(this.nc, this.binding.space, service, "spawn", spawnArgs(req), {
+                    const service = await this.manager(req.placement?.instanceId);
+                    if (req.cwd !== undefined && resolution === undefined) {
+                        // `req.placement` is guaranteed here: a cwd without one refused above, at :1203.
+                        resolution = await this.resolveCwd(req, service, req.cwd, req.placement?.instanceId ?? "");
+                        // PERSIST BEFORE SUBMITTING. The resolution is what phase B dispatches and what a
+                        // resume re-reads; binding it after the submission would leave a crash in between with
+                        // a launched seat whose directory no record names.
+                        await ctx.bind({ resolution });
+                    }
+                    reply = await this.invokeManager(service, "spawn", spawnArgs(resolution === undefined ? req : { ...req, cwd: resolution.cwd }), {
                         id: goalId,
                         deadlineMs: SPAWN_ACCEPT_DEADLINE_MS,
                     });
                 }
                 catch (err) {
+                    // A refusal this handler RAISED is never a lost reply: it was decided here, before
+                    // anything was submitted, so it is re-thrown rather than weighed against a goal record.
+                    if (err instanceof EffectError)
+                        throw err;
                     // The invoke did not come back — which does not prove nothing happened: the request may
                     // have been accepted while the reply was lost. The goal record is the arbiter: a durable
                     // trace under this goalId means the submission landed, so proceed to its terminal; none
@@ -1107,6 +1250,9 @@ export class MeshHandler {
                 ext = {
                     goalId,
                     ...(floor !== undefined ? pickAcceptanceFloor(floor) : {}),
+                    // `bind` REPLACES the external record, so the phase-A resolution is re-stated here or the
+                    // acceptance bind would erase it and a resume would re-resolve against a moved target.
+                    ...(resolution !== undefined ? { resolution } : {}),
                     ...(req.worktree !== undefined ? { worktree: req.worktree } : {}),
                     ...(req.onFork !== undefined ? { onFork: req.onFork } : {}),
                     ...(req.permits !== undefined ? { permits: req.permits } : {}),
@@ -1234,7 +1380,7 @@ export class MeshHandler {
                     throw new EffectError("L4001", "permit-wall-clock", `turn(${name}#${uid}): a ${deadlineMs}ms deadline runs past the agent's remaining wall clock (${remaining}ms of its ${entry.permits.wallClockMs}ms permit)`);
             }
             const payload = JSON.stringify({ run: this.binding.runId, step, context, noticeIds: notices.map((n) => n.noticeId) });
-            const submit = async () => invokeCommand(this.nc, this.binding.space, await this.manager(), "turn", { payload, deadlineMs, ...(handoffFrom !== undefined ? { handoffFrom } : {}) }, {
+            const submit = async () => this.invokeManager(await this.manager(), "turn", { payload, deadlineMs, ...(handoffFrom !== undefined ? { handoffFrom } : {}) }, {
                 id: goalId,
                 deadlineMs: TURN_ACCEPT_DEADLINE_MS,
                 target: { mode: "owner", owner, actor, lifecycleUid: uid },
@@ -1593,7 +1739,7 @@ export class MeshHandler {
             return;
         let reply;
         try {
-            reply = await invokeCommand(this.nc, this.binding.space, await this.manager(), "turn", { payload, deadlineMs: Math.max(1_000, deadlineAt - this.now()) }, {
+            reply = await this.invokeManager(await this.manager(), "turn", { payload, deadlineMs: Math.max(1_000, deadlineAt - this.now()) }, {
                 id: goalId,
                 deadlineMs: TURN_ACCEPT_DEADLINE_MS,
                 target: { mode: "owner", owner: seat.owner, actor: seat.actor, lifecycleUid: uid },
@@ -2205,6 +2351,14 @@ const GOAL_POLL_MS = 2_000;
 const SPAWN_ACCEPT_DEADLINE_MS = 30_000;
 /** Bound on the manager's synchronous `turn` ACCEPT reply (the relay registration, not the yield). */
 const TURN_ACCEPT_DEADLINE_MS = 30_000;
+/** How many times {@link MeshHandler.invokeManager} re-issues one manager call after a
+ *  `not-executed` bind refusal. Every re-issue is a first attempt, so the bound is a loop guard and
+ *  not a duplication guard: it stops a class whose describe and invoke never agree from re-issuing
+ *  forever. Nine attempts leave a two-manager space a 1-in-512 residual where the unrepaired refusal
+ *  was 1-in-2 (#1638). The loop turns only on a refusal that has already been ANSWERED, so an
+ *  attempt costs a describe and an invoke round trip and never an elapsed deadline: a call nobody
+ *  answers raises its own `deadline-exceeded`, which is not a bind refusal and is not re-issued. */
+const BIND_SPLIT_REISSUES = 8;
 /** A step key's enclosing scope: the journal's own rendering (`entry.scope`), re-derived so the
  *  live path and the adoption rebuild key the handoff memos identically. */
 function scopeOf(key) {
@@ -2217,11 +2371,46 @@ const DISCHARGE_TERMINAL_BOUND_MS = 30_000;
 /** The manager `spawn` args a {@link SpawnRequest} submits: persona names the persona file
  *  (`name`), `join` becomes the seat's channel subscriptions. `permits` stay on the run (they
  *  bind at `turn`); `supervise` travels because the manager is who restarts the process. */
+/**
+ * #1616 proof item 5 — ALIAS NORMALIZATION POLICY. One clone reached through a symlink and through
+ * its realpath is ONE writable directory, and the single-writer rule keys on identity, so the two
+ * forms must collapse before any claim is taken. THE CANONICAL FORM IS THE REALPATH: it is the form
+ * the kernel and the child's own `process.cwd()` report, so a claim keyed on it matches what any
+ * concurrent run or imperative spawn path observes, whichever alias that caller typed. The symlink
+ * path is NOT canonical — normalizing toward it would require resolving every other alias to it,
+ * which has no unique answer. Resolution is done ONCE, on the serving host, before the claim and
+ * before launch; an unresolvable path is a refusal, never a pass-through of the raw string.
+ */
+export function canonicalCwd(cwd) {
+    return realpathSync(cwd);
+}
+/** The single named refusal every phase-A failure direction produces: the target does not serve
+ *  `resolve-cwd`, refuses the path, or answers with something that is not an absolute directory.
+ *  `L4000` is the host declining the request, and it is raised before any submission, so nothing is
+ *  bound, allocated or launched. There is no raw-path arm: a fall-back to the caller's string is
+ *  fail-OPEN, and launching somewhere plausible is the outcome being removed. */
+export function cwdResolutionRefusal(persona, instanceId, cwd, cause) {
+    return new EffectError("L4000", "spawn", `spawn(${persona}) cwd ${JSON.stringify(cwd)} was not resolved by manager instance ${instanceId}: ${cause}; refusing rather than dispatching a path this host never canonicalized`);
+}
+/** A journalled phase-A resolution, read back on resume. A garbled entry is treated as absent and
+ *  phase A runs again: re-resolving is free (no goal, no reservation), trusting a broken record is
+ *  not. */
+function readCwdResolution(value) {
+    const r = value;
+    if (typeof r?.cwd !== "string" || r.cwd.length === 0 || typeof r.endpoint !== "string" || typeof r.instanceId !== "string")
+        return undefined;
+    return { cwd: r.cwd, endpoint: r.endpoint, instanceId: r.instanceId, ...(typeof r.host === "string" ? { host: r.host } : {}) };
+}
 export function spawnArgs(req) {
     return {
         name: req.persona,
         ...(req.model !== undefined ? { model: req.model } : {}),
         ...(req.variant !== undefined ? { variant: req.variant } : {}),
+        // #1616 item 5: the cwd dispatched here is ALREADY the canonical form, resolved by the serving
+        // manager in phase A (see MeshEffectHandler.resolveCwd) and read back off the journalled
+        // resolution. This projection no longer canonicalizes anything: `realpathSync` in the driver
+        // answers about the DRIVER's filesystem, which is a different host's answer to the question.
+        ...(req.cwd !== undefined ? { cwd: req.cwd } : {}),
         ...(req.role !== undefined ? { role: req.role } : {}),
         ...(req.join !== undefined && req.join.length > 0 ? { subscribe: req.join.map((c) => c.channel) } : {}),
         ...(req.supervise !== undefined ? { supervise: readSupervise(req.supervise, req.persona) } : {}),