@cotal-ai/runtime 0.67.0 → 0.69.0

package/dist/mesh-handler.js

@@ -19,12 +19,13 @@
 import { createHash } from "node:crypto";
 import { isAbsolute } from "node:path";
 import { realpathSync } from "node:fs";
-import { mintCheckpoint, heartbeatCheckpoint, resumeCheckpoint, readCheckpointSettle, readCheckpointAnswer, readCheckpointStatus, readCheckpointSpec, reconcileCheckpointSchedule, handleCheckpointFire, eptStreamName, eptSubject, chatStream, waitConsumerName, waitConsumerConfig, isConcreteChannel, assertSafePattern, assertValidChannel, presenceBucket, principalKey, liveKvEntries, IncompleteKvScan, openMembersRegistry, openChannelRegistry, writeChannelConfig, readMember, commitMember, tombstoneMember, StaleMembershipWrite, runNoticeId, writeRunNotice, actionContext, invokeCommand, replyRefusedBeforeEffect, readGoalResult, readGoalStatus, resolveService, listRunNotices, listRunMigrations, markRunNoticeConsumed, listRunNoticesForRun, readRunRecord, writeRunStatus, EpEnvelopeError, assertAdmittedPublish, assertAdmittedSubscribe, assertNotRevoked, } from "@cotal-ai/core";
+import { mintCheckpoint, heartbeatCheckpoint, resumeCheckpoint, readCheckpointSettle, readCheckpointAnswer, readCheckpointStatus, readCheckpointSpec, reconcileCheckpointSchedule, handleCheckpointFire, eptStreamName, eptSubject, chatStream, waitConsumerName, waitConsumerConfig, isConcreteChannel, assertSafePattern, assertValidChannel, presenceBucket, principalKey, liveKvEntries, IncompleteKvScan, openMembersRegistry, openChannelRegistry, writeChannelConfig, readMember, commitMember, tombstoneMember, StaleMembershipWrite, runNoticeId, writeRunNotice, actionContext, invokeCommand, invokeRepairingSplit, replyRefusedBeforeEffect, readGoalResult, readGoalStatus, resolveService, listRunNotices, listRunMigrations, markRunNoticeConsumed, listRunNoticesForRun, readRunRecord, writeRunStatus, EpEnvelopeError, assertAdmittedPublish, assertAdmittedSubscribe, assertNotRevoked, } from "@cotal-ai/core";
 import { renderRunContext } from "./run-context.js";
 import { migrationSeats } from "./migrate.js";
 import { Kvm } from "@nats-io/kv";
-import { parseDuration, Cancelled, EffectError, Journal, EffectRefused, askSchemaShape, conformsToAskSchema, holdRequestId, journalEntryKeyString, stepKeyString, } from "@cotal-ai/lang";
+import { parseDuration, Cancelled, EffectError, Journal, EffectRefused, askSchemaShape, conformsToAskSchema, journalEntryKeyString, stepKeyString, } from "@cotal-ai/lang";
 import { loopLag, servedDespiteStarvation } from "./host-starvation.js";
+import { askAttemptToken, derivedToken, pauseTokens } from "./pause-tokens.js";
 /**
  * A checkpoint resumed with no answer to read.
  *
@@ -113,54 +114,23 @@ export class MeshHandler {
         return this.managerService;
     }
     /**
-     * One manager call, with a SPEC 13.2 bind refusal REPAIRED rather than raised.
+     * One manager call, with a SPEC 13.2 bind refusal REPAIRED rather than raised
+     * ({@link invokeRepairingSplit}).
      *
-     * A run resolves the manager on the class rail and binds the incarnation that answered its
-     * describe. The invoke is a second, independent trip through the same anycast queue, so in a
-     * space with more than one manager it routinely reaches another member, and that member refuses
-     * before dispatching. The refusal is honest for one command and destructive for a run: it says
-     * the command did not run and its remedy is to re-issue, but raised as the effect's own failure
-     * it ends the run and consumes the run id and its journal (#1638).
-     *
-     * So a refusal the responder MARKS as pre-effect is re-issued instead of returned. It is a first
-     * attempt and not a second: the marker together with `not-executed` is the responder's own
-     * statement that no effect of the command exists, which is what {@link replyRefusedBeforeEffect}
-     * checks, and it is the same licence core's `Endpoint.invokeService` re-issues on. The stale
-     * class handle is dropped first, so the re-issue re-describes rather than rebinding the
-     * incarnation that was just refused.
-     *
-     * BOUNDED, because a re-issue draws the same queue again. The describe and the invoke stay two
-     * independent trips, so a space of m managers still splits (m-1)/m of the time and the repair
-     * converges geometrically rather than deterministically; after {@link BIND_SPLIT_REISSUES} of
-     * them the refusal surfaces unchanged, still stating that the command did not run. What removes
-     * the residual is addressing one instance, and the run's caller holds no instance-rail grant for
-     * a command its program did not place (SPEC 13.9, `run-driver-grants.ts`), so that is a wider
-     * change than this one.
-     *
-     * A PINNED handle is never repaired. It addresses one instance by name, so a refusal from it is
-     * that incarnation answering about itself, and re-resolving onto the class rail would reinstate
-     * the anycast fallback #1616 removed.
+     * A run resolves the manager on the class rail, so in a space with more than one manager its
+     * invoke routinely reaches a member other than the one that answered the describe, and that
+     * member refuses before dispatching. Raised as the effect's own failure, that refusal ends the
+     * run and consumes the run id and its journal (#1638), although it says the command did not run.
+     * The class memo is dropped before each re-describe, so later calls do not start from the bind
+     * that was just refused. After the shared bound the refusal surfaces unchanged; what removes that
+     * residual is addressing one instance, and the run's caller holds no instance-rail grant for a
+     * command its program did not place (SPEC 13.9, `run-driver-grants.ts`).
      */
     async invokeManager(service, command, args, opts) {
-        let handle = service;
-        for (let reissues = 0;; reissues += 1) {
-            const reply = await invokeCommand(this.nc, this.binding.space, handle, command, args, opts);
-            if (reply.reply.ok !== false || !replyRefusedBeforeEffect(reply.reply.error))
-                return reply;
-            if (handle.pinnedInstanceId !== undefined || reissues === BIND_SPLIT_REISSUES)
-                return reply;
+        return invokeRepairingSplit(this.nc, this.binding.space, service, command, args, opts, () => {
             this.managerService = undefined;
-            try {
-                handle = await this.manager();
-            }
-            catch {
-                // The repair could not be attempted. The REFUSAL is what surfaces, not the resolve failure:
-                // every caller of this method already reads a refused reply as "the manager declined", and
-                // this one states that nothing ran, which is the fact a describe timeout raised in its
-                // place would lose.
-                return reply;
-            }
-        }
+            return this.manager();
+        });
     }
     /**
      * #1616 item 5 — PHASE A, RESOLVE ON THE HOST THAT WILL LAUNCH. A directory is a fact about one
@@ -411,7 +381,7 @@ export class MeshHandler {
                 await new Promise((r) => setTimeout(r, WAIT_POLL_MS).unref());
             }
         }
-        if (rows.some((p) => p.card?.name === holder.name && p.lifecycleUid === holder.uid))
+        if (rows.some((p) => p.card.name === holder.name && p.lifecycleUid === holder.uid))
             throw new EffectError("L4008", "worktree", `spawn(${persona}) would put a second agent into the worktree "${worktree}" while ${holder.name}#${holder.uid} is live in it; two agents MUST NOT share a worktree concurrently (L3022/L4008)`);
         const now = this.worktreeHolders.get(worktree);
         if (now !== undefined && now !== holder)
@@ -460,8 +430,7 @@ export class MeshHandler {
         for (const e of owed) {
             if (e.requestId === undefined)
                 continue;
-            if (e.hold !== undefined)
-                await this.cancelTimer({ endpoint: this.binding.endpoint, token: holdRequestId(e.requestId) });
+            await this.endPause(e);
             if (e.kind === "spawn") {
                 await this.dischargeSpawn(e);
                 continue;
@@ -480,52 +449,29 @@ export class MeshHandler {
                 if (typeof x?.name === "string" && typeof x?.uid === "string" && typeof x?.goalId === "string")
                     this.turnGoals.get(`${x.name}#${x.uid}`)?.delete(x.goalId);
             }
-            if (e.kind === "waitUntil") {
-                // A `waitUntil` arms ONE cadence pause per observation, each under a derived token, so the
-                // sweep releases the one that is actually open: the observation the entry is on. The
-                // earlier ones already settled (that is how the wait got here) and `cancelTimer` tolerates
-                // a claim that loses its own race, so releasing the current index is both necessary and
-                // sufficient. Attempt 0 never arms anything, which is why the index is the observation
-                // COUNT rather than the count minus one.
-                const attempt = (e.observations ?? []).length;
-                if (attempt > 0)
-                    await this.cancelTimer({
-                        endpoint: this.binding.endpoint,
-                        token: derivedToken(e.requestId, `observe-${attempt}`),
-                    });
-                continue;
-            }
-            await this.endPause(e);
         }
     }
     /**
-     * End the pause a step armed: claim the kind's armed pauses and, for a `wait`, close its
-     * consumer. A cancelled loser's discharge calls it, and so does a hold before its first bind
-     * (spec/cotal-lang.md §7.8), which claims an `ask`'s open attempt. An `ask`'s relay goal is left
-     * as the discharge leaves it.
+     * End the pauses a step armed: claim every token {@link pauseTokens} says it owns and, for a
+     * `wait`, close its consumer. A cancelled loser's discharge calls it, and so does a hold before
+     * its first bind (spec/cotal-lang.md §7.8), which claims the held step's open attempt. An `ask`'s
+     * relay goal is left as the discharge leaves it.
      */
     async endPause(e) {
         if (e.requestId === undefined)
             return;
-        if (e.kind !== "sleep" && e.kind !== "checkpoint" && e.kind !== "wait" && e.kind !== "ask" && e.kind !== "turn")
+        for (const token of pauseTokens(e))
+            await this.cancelTimer({ endpoint: this.binding.endpoint, token });
+        if (e.kind !== "wait")
+            return;
+        if (this.services) {
+            await this.services.waits.close(e.requestId);
             return;
-        // An ask's armed timer is its CURRENT attempt's, whose token is bound as `askToken`; a
-        // crash before the first bind leaves attempt 1, which is the request id itself.
-        const current = e.kind === "ask" && typeof e.external?.askToken === "string"
-            ? e.external.askToken
-            : e.requestId;
-        await this.cancelTimer({ endpoint: this.binding.endpoint, token: current });
-        if (e.kind === "wait") {
-            await this.cancelTimer({ endpoint: this.binding.endpoint, token: derivedToken(e.requestId, "wait-timeout") });
-            if (this.services) {
-                await this.services.waits.close(e.requestId);
-                return;
-            }
-            try {
-                await this.jsm.consumers.delete(chatStream(this.binding.space), waitConsumerName(e.requestId));
-            }
-            catch { /* never created, or already deleted — nothing is held either way */ }
         }
+        try {
+            await this.jsm.consumers.delete(chatStream(this.binding.space), waitConsumerName(e.requestId));
+        }
+        catch { /* never created, or already deleted — nothing is held either way */ }
     }
     /**
      * Withdraw a cancelled `notify`'s undelivered notices.
@@ -1028,8 +974,8 @@ export class MeshHandler {
                 }
                 throw e;
             }
-            if (!rows.some((p) => p.card?.name === name && p.lifecycleUid === uid)) {
-                const reason = rows.some((p) => p.card?.name === name) ? "superseded" : "lapsed";
+            if (!rows.some((p) => p.card.name === name && p.lifecycleUid === uid)) {
+                const reason = rows.some((p) => p.card.name === name) ? "superseded" : "lapsed";
                 lapsedSince = lapseWindow(lapsedSince, lastReadAt, this.now());
                 if (reason === "superseded" || readAt - lapsedSince >= LAPSE_CONFIRM_MS) {
                     if (primary !== undefined)
@@ -1527,8 +1473,8 @@ export class MeshHandler {
                     }
                     throw e;
                 }
-                if (!rows.some((pr) => pr.card?.name === name && pr.lifecycleUid === uid)) {
-                    const reason = rows.some((pr) => pr.card?.name === name) ? "superseded" : "lapsed";
+                if (!rows.some((pr) => pr.card.name === name && pr.lifecycleUid === uid)) {
+                    const reason = rows.some((pr) => pr.card.name === name) ? "superseded" : "lapsed";
                     lapsedSince = lapseWindow(lapsedSince, lastReadAt, this.now());
                     if (reason === "superseded" || readAt - lapsedSince >= LAPSE_CONFIRM_MS) {
                         await this.cancelTimer(primary);
@@ -2032,19 +1978,22 @@ export class MeshHandler {
             await (await this.channelRegistry()).delete(plan.channel);
         }
     }
-    /** Every live presence row that decodes. Foreign bytes in the bucket are skipped — a peer's
-     *  malformed self-publish must not break another agent's name resolution — and what an ABSENT
-     *  row means belongs to the caller: `resolveMemberPrincipal` refuses the join loudly, and
-     *  `waitDown` reads it as the death it is waiting for. */
+    /** Every live presence row that decodes to a card with a string id and name. Foreign bytes in
+     *  the bucket are skipped — a peer's malformed self-publish must not break another agent's name
+     *  resolution — and what an ABSENT row means belongs to the caller: `resolveMemberPrincipal`
+     *  refuses the join loudly, and `waitDown` reads it as the death it is waiting for. */
     async presenceRows() {
         const rows = [];
         for (const e of await liveKvEntries(await this.presenceRegistry())) {
+            let row;
             try {
-                rows.push(e.json());
+                row = e.json();
             }
             catch {
-                // not a presence row
+                continue;
             }
+            if (isPresenceRow(row))
+                rows.push(row);
         }
         return rows;
     }
@@ -2462,19 +2411,11 @@ const GOAL_POLL_MS = 2_000;
 const SPAWN_ACCEPT_DEADLINE_MS = 30_000;
 /** Bound on the manager's synchronous `turn` ACCEPT reply (the relay registration, not the yield). */
 const TURN_ACCEPT_DEADLINE_MS = 30_000;
-/** How many times {@link MeshHandler.invokeManager} re-issues one manager call after a
- *  `not-executed` bind refusal. Every re-issue is a first attempt, so the bound is a loop guard and
- *  not a duplication guard: it stops a class whose describe and invoke never agree from re-issuing
- *  forever. Nine attempts leave a two-manager space a 1-in-512 residual where the unrepaired refusal
- *  was 1-in-2 (#1638). The loop turns only on a refusal that has already been ANSWERED, so an
- *  attempt costs a describe and an invoke round trip and never an elapsed deadline: a call nobody
- *  answers raises its own `deadline-exceeded`, which is not a bind refusal and is not re-issued. */
-const BIND_SPLIT_REISSUES = 8;
 /** How many class-rail despawns a discharge sends before it stops waiting for the allocating
  *  manager to answer. Each is one {@link MeshHandler.invokeManager} call, whose describe and invoke
  *  both land on the allocator with probability 1/m^2 per trip in a space of m managers, so one call
- *  reaches it with probability (1 - ((m-1)/m)^9) / m: about 1/2 for m = 2 and 0.23 for m = 4. All
- *  65 then miss with probability about 2^-65 and 4e-8. */
+ *  reaches it with probability (1 - ((m-1)/m)^17) / m: about 1/2 for m = 2 and 0.25 for m = 4. All
+ *  65 then miss with probability about 2^-65 and 9e-9. */
 const DESPAWN_ROUTE_ATTEMPTS = 64;
 /** A step key's enclosing scope: the journal's own rendering (`entry.scope`), re-derived so the
  *  live path and the adoption rebuild key the handoff memos identically. */
@@ -2635,6 +2576,17 @@ function isConclavePlan(v) {
             && typeof r.generation === "number" && typeof r.joined === "boolean";
     });
 }
+/** Whether a decoded presence value carries the string `card.id` and `card.name` that every
+ *  presence reader here matches on or returns. */
+function isPresenceRow(v) {
+    if (typeof v !== "object" || v === null)
+        return false;
+    const card = v.card;
+    if (typeof card !== "object" || card === null)
+        return false;
+    const c = card;
+    return typeof c.id === "string" && typeof c.name === "string";
+}
 /** Split an agent handle `<name>#<lifecycleUid>` on its LAST `#` — the uid alphabet
  *  (`[a-z0-9]{26,32}`) cannot carry one, a name could. */
 function parseAgentHandle(agent) {
@@ -2647,7 +2599,7 @@ function parseAgentHandle(agent) {
  *  the effect's own catchable failure — the agent is down or gone (L4002) — and more than one is
  *  an ambiguity no membership row may be written under. */
 function resolveMemberPrincipal(rows, agent, name, uid) {
-    const matches = rows.filter((p) => p.card?.name === name && p.lifecycleUid === uid && typeof p.card?.id === "string");
+    const matches = rows.filter((p) => p.card.name === name && p.lifecycleUid === uid);
     if (matches.length === 1)
         return matches[0].card.id;
     if (matches.length === 0)
@@ -2665,15 +2617,6 @@ function assertConclaveChannel(channel) {
 /** How often a pause that nobody will answer looks for the broker's fire. Same argument as
  *  `WAIT_POLL_MS`: the deadline is durable and this is only how late its observation can be. */
 const FIRE_POLL_MS = 2_000;
-/** A second deadline for one step, derived so a resume re-derives it instead of remembering it.
- *  Same shape and alphabet as a request id, so it is a valid `<token>` by construction. */
-function derivedToken(requestId, purpose) {
-    return createHash("sha256").update(`${requestId}:${purpose}`, "utf8").digest("base64url").slice(0, 43);
-}
-/** An ask attempt's pause token: attempt 1 IS the step's request id; a re-ask derives its own. */
-function askAttemptToken(requestId, attempt) {
-    return attempt === 1 ? requestId : derivedToken(requestId, `ask-attempt-${attempt}`);
-}
 /** The recorded ask progress a resume re-enters at, or undefined for a fresh first attempt. The
  *  external is bytes from an earlier process, so the shape is checked rather than trusted. */
 function askResume(v) {
@@ -2782,46 +2725,15 @@ export async function rearmOutstandingPauses(deps, binding, entries) {
  *
  * An entry is open when its LAST record is `pending`, so the map is built in append order and the
  * later record wins — a step that settled has a settled entry after its pending one, and reading
- * only the first would re-arm timers for pauses that are already over.
- *
- * THE KINDS ARE THE FIVE THAT ARM A TIMER, and `wait`, `ask` and `turn` are three of them. None of
- * the three mints a pause that looks like its own, but a wait's idle window and timeout, an ask
- * attempt's deadline, and a turn's deadline authority are mediated deadlines exactly as `sleep`'s
- * is, and one adopted at a new epoch would otherwise wait on a deadline no live epoch fires.
- *
- * An idle wait with a timeout arms TWO, and the second is DERIVED rather than recorded, so it is
- * re-derived here for the same reason the live path derives it: a resume that had to remember it
- * would be carrying state the key already determines. Emitting it for a wait that never minted one
- * is harmless by construction — the reconciler reads the checkpoint's status first and re-emits
- * nothing when there is none — and the alternative, reading the request shape back out of the
- * entry to decide, would make the repair depend on a field a replay is not guaranteed to carry.
- * An ask's armed pause is its CURRENT attempt's, whose token is bound as `askToken`; before the
- * first bind it is attempt 1, which is the request id itself. A `turn`'s is under its goal id,
- * which is the request id: that pause is the client-side L4003 authority the run keeps for a
- * manager that dies, so leaving it armed at the predecessor's coordinates would go dark in exactly
- * the window recovery opens. A held step's hold is armed under its hold id once the hold binds; the
- * attempt token the hold claimed is still listed and re-arms nothing, because the reconciler
- * re-emits a schedule only for a waiting pause.
+ * only the first would re-arm timers for pauses that are already over. What each open step owns is
+ * {@link pauseTokens}: a held step's attempt token is still listed after its hold binds and
+ * re-arms nothing, because the reconciler re-emits a schedule only for a waiting pause.
  */
 export function outstandingPauseTokens(entries) {
     const last = new Map();
     for (const e of entries)
         last.set(journalEntryKeyString(e), e);
-    const tokens = [];
-    for (const e of last.values()) {
-        if (e.state !== "pending" || e.requestId === undefined)
-            continue;
-        // `turn` arms its client-side deadline authority under the step's request id (the goal id).
-        if (e.kind === "sleep" || e.kind === "checkpoint" || e.kind === "turn")
-            tokens.push(e.requestId);
-        else if (e.kind === "ask")
-            tokens.push(typeof e.external?.askToken === "string" ? e.external.askToken : e.requestId);
-        else if (e.kind === "wait")
-            tokens.push(e.requestId, derivedToken(e.requestId, "wait-timeout"));
-        if (e.hold !== undefined)
-            tokens.push(holdRequestId(e.requestId));
-    }
-    return tokens;
+    return [...last.values()].filter((e) => e.state === "pending").flatMap(pauseTokens);
 }
 /**
  * The settle watcher over EPF, which is where the one-use settle fact lives.