catherd-cli 1.3.0 → 1.5.0

package/src/services/routing-service.ts

@@ -1,3 +1,4 @@
+import { assertNativeHost } from "./backends.ts";
 import { BUDGET_CHEAP_AT } from "../domain/budget.ts";
 import { errorMessage } from "../domain/errors.ts";
 import {
@@ -12,11 +13,13 @@ import {
 import { type Difficulty, type Kind, parseLaneHeader } from "../domain/lane.ts";
 import type { Role } from "../domain/roles.ts";
 import type { RouteJev } from "../domain/route.ts";
+import type { Catalog } from "../domain/catalog.ts";
 import {
   candidates,
   defaultDifficulty,
   defaultLadder,
   type Pick,
+  quotaUsage,
   type RoutingProfile,
   select,
 } from "../domain/select.ts";
@@ -25,11 +28,17 @@ import { catalogQuery, freshenDiscovery, loadCatalog } from "./catalog-service.t
 import { type Asked, askJev, type JevOpts, jevQuestions, logJev } from "./jev-service.ts";
 import type { ProfileView, RouteAnswer, RouteRequest, RoutingPort, Verdict } from "./ports.ts";
 import { provenanceOf } from "./provenance.ts";
-import { runEvidence } from "./run-evidence.ts";
+import { type EvidenceTable, runEvidence } from "./run-evidence.ts";
 
-function routingProfile(v: ProfileView, role: Role, spentFraction: number): RoutingProfile {
+function routingProfile(
+  v: ProfileView,
+  role: Role,
+  spentFraction: number,
+  usage: Record<string, number> = {},
+): RoutingProfile {
   const rc = v.roles[role];
   return {
+    usage,
     // spec §4.6: from 80 % of the budget on, start at the cheapest rung that clears the bar
     objective: spentFraction >= BUDGET_CHEAP_AT ? "cost" : v.objective,
     billing: v.billing,
@@ -49,14 +58,60 @@ const answer = (
   asked: Asked | null,
   jev: RouteJev | null,
 ): RouteAnswer => ({
-  ...pick,
+  rung: pick.rung,
+  ladder: pick.ladder,
   source,
   kind,
   difficulty,
   questionSet: asked?.answers ? asked.meta.questionSet : null,
   jev,
+  jevSaid: null,
+  why: "",
+  ...(pick.noClear ? { noClear: pick.noClear } : {}),
+  ...(pick.tie ? { tie: pick.tie } : {}),
 });
 
+/** Spec 1.5 plan 24: what Jev said when it differs from the kind and difficulty the route used, else null. */
+function jevSaid(
+  judged: ReturnType<typeof judgeRoute> | null,
+  kind: Kind | null,
+  difficulty: Difficulty | null,
+): string | null {
+  if (!judged) return null;
+  const k = judged.kind;
+  const d = judged.track ? judged.difficulty : null;
+  if ((k === null || k === kind) && (d === null || d === difficulty)) return null;
+  return `Jev said ${k ?? "no sure kind"}/${d ?? "no sure difficulty"}`;
+}
+
+/** Spec 1.5 plan 24: the route's one-line why, for the coordinator; the provenance goes to routes.jsonl. */
+function whyOf(a: RouteAnswer, hasLane: boolean): string {
+  const what = `${a.kind}/${a.difficulty}`;
+  const head =
+    a.source === "lane"
+      ? `the lane's Kind/Difficulty, ${what}`
+      : a.source === "jev"
+        ? `Jev: ${what}`
+        : a.source === "jev-kind"
+          ? `Jev's kind with ${what}`
+          : hasLane
+            ? "the role's default rung: neither Jev nor the lane file gave a kind and difficulty"
+            : "the role's default rung";
+  const start =
+    a.noClear ?? (a.source === "default" ? null : "the first rung in objective order that clears it");
+  return [head, a.jevSaid, start, a.tie].filter((x): x is string => Boolean(x)).join("; ");
+}
+
+/** Spec 1.2 §8 evidence is display only: an unreadable run on this machine never fails a route. */
+function evidenceOrNone(c: Catalog): EvidenceTable | null {
+  try {
+    return runEvidence(c);
+  } catch (e) {
+    log("debug", "route", { evidence: errorMessage(e) });
+    return null;
+  }
+}
+
 /** How long `route` waits on the daily discovery refresh before routing on the cached listing. */
 const DISCOVERY_BUDGET_MS = 5_000;
 
@@ -78,53 +133,77 @@ async function freshenWithin(rungs: string[], repo: string, ms: number): Promise
   clearTimeout(timer);
 }
 
+/** What every route of one call shares: the role's routing profile and the catalog. */
+interface Prepared {
+  p: RoutingProfile;
+  c: Catalog;
+}
+
+async function prepare(req: RouteRequest, o: RoutingOpts): Promise<Prepared> {
+  const p = routingProfile(req.profile, req.role, req.spentFraction, req.usage);
+  await freshenWithin(p.role.rungs, req.repo, o.discoveryBudgetMs ?? DISCOVERY_BUDGET_MS);
+  return { p, c: loadCatalog({ repo: req.repo }) };
+}
+
+/** Jev's answer about a lane; null when the route reads no lane, empty when it asks nothing (Jev off, one usable rung). */
+interface Judged {
+  asked: Asked | null;
+  judged: ReturnType<typeof judgeRoute> | null;
+}
+
+async function judge(req: RouteRequest, { p, c }: Prepared, o: RoutingOpts): Promise<Judged | null> {
+  if (req.laneText === null) return null;
+  // one usable rung: nothing for Jev to choose, but the lane's header still names its kind and difficulty
+  if (req.profile.jev.use === "off" || candidates(c, p, req.role).length <= 1)
+    return { asked: null, judged: null };
+  const asked = await askJev(req.runDir, "route-v2", laneState(req.laneText), o);
+  const judged = asked.answers ? judgeRoute(jevQuestions().sets["route-v2"].rule, asked.answers) : null;
+  return { asked, judged };
+}
+
 /**
- * Spec §5.4: kind and difficulty from Jev (§5.5's rule), else the lane file's `Kind:`/`Difficulty:`,
- * else the role's default rung. A kind Jev is sure of survives a difficulty in the dead band: the
- * difficulty then comes from the lane, else the role's default difficulty (`jev-kind`). A role with one
- * usable rung never asks Jev.
+ * Spec §5.4: kind and difficulty from the lane file's `Kind:`/`Difficulty:` when it declares both (spec 1.5
+ * plan 24: a declared header wins, and `jevSaid` names where Jev disagreed), else from Jev (§5.5's rule), else
+ * the role's default rung. A kind Jev is sure of survives a difficulty in the dead band: the difficulty then
+ * comes from the lane, else the role's default difficulty (`jev-kind`). A role with one usable rung never asks
+ * Jev.
  */
-async function route(req: RouteRequest, o: RoutingOpts): Promise<RouteAnswer> {
-  const p = routingProfile(req.profile, req.role, req.spentFraction);
-  await freshenWithin(p.role.rungs, req.repo, o.discoveryBudgetMs ?? DISCOVERY_BUDGET_MS);
-  const c = loadCatalog({ repo: req.repo });
+function decide(req: RouteRequest, { p, c }: Prepared, j: Judged | null): RouteAnswer {
   const fallback = () => defaultLadder(c, p, req.role);
-  if (req.laneText === null || candidates(c, p, req.role).length <= 1) {
-    const out = answer(fallback(), "default", null, null, null, null);
-    return { ...out, provenance: provenanceOf(c, out.rung, null, null, p.billing, runEvidence(c)) };
-  }
+  const finish = (out: RouteAnswer): RouteAnswer => {
+    out.why = whyOf(out, req.laneText !== null);
+    out.provenance = provenanceOf(c, out.rung, out.kind, out.difficulty, p.billing, evidenceOrNone(c));
+    return out;
+  };
+  if (req.laneText === null || j === null)
+    return finish(answer(fallback(), "default", null, null, null, null));
   const lane = parseLaneHeader(req.laneText);
-  let asked: Asked | null = null;
-  let judged: ReturnType<typeof judgeRoute> | null = null;
-  if (req.profile.jev.use !== "off") {
-    asked = await askJev(req.runDir, "route-v2", laneState(req.laneText), o);
-    if (asked.answers) judged = judgeRoute(jevQuestions().sets["route-v2"].rule, asked.answers);
-  }
+  const { asked, judged } = j;
   const jev: RouteJev | null = judged
     ? { pKind: judged.pKind, pA: judged.pA, pB: judged.pB, nouls: judged.nouls }
     : null;
   let out: RouteAnswer;
-  if (judged?.track && judged.difficulty) {
-    const kind = judged.kind ?? lane.kind ?? "repo_code";
-    out = answer(select(c, p, req.role, kind, judged.difficulty), "jev", kind, judged.difficulty, asked, jev);
-  } else if (judged?.kind) {
-    const difficulty = lane.difficulty ?? defaultDifficulty(c, p, req.role, judged.kind);
+  if (lane.kind && lane.difficulty) {
     out = answer(
-      select(c, p, req.role, judged.kind, difficulty),
-      "jev-kind",
-      judged.kind,
-      difficulty,
+      select(c, p, req.role, lane.kind, lane.difficulty),
+      "lane",
+      lane.kind,
+      lane.difficulty,
       asked,
       jev,
     );
+  } else if (judged?.track && judged.difficulty) {
+    const kind = lane.kind ?? judged.kind ?? "repo_code";
+    out = answer(select(c, p, req.role, kind, judged.difficulty), "jev", kind, judged.difficulty, asked, jev);
+  } else if (judged?.kind || (judged && lane.kind)) {
+    const kind = (lane.kind ?? judged.kind) as Kind;
+    const difficulty = lane.difficulty ?? defaultDifficulty(c, p, req.role, kind);
+    out = answer(select(c, p, req.role, kind, difficulty), "jev-kind", kind, difficulty, asked, jev);
   } else {
-    const kind = lane.kind;
-    out =
-      kind && lane.difficulty
-        ? answer(select(c, p, req.role, kind, lane.difficulty), "lane", kind, lane.difficulty, asked, jev)
-        : answer(fallback(), "default", null, null, asked, jev);
+    out = answer(fallback(), "default", null, null, asked, jev);
   }
-  out.provenance = provenanceOf(c, out.rung, out.kind, out.difficulty, p.billing, runEvidence(c));
+  out.jevSaid = out.source === "default" ? null : jevSaid(judged, out.kind, out.difficulty);
+  finish(out);
   if (asked) {
     logJev(req.runDir, {
       ...asked.meta,
@@ -142,6 +221,31 @@ async function route(req: RouteRequest, o: RoutingOpts): Promise<RouteAnswer> {
   return out;
 }
 
+async function route(req: RouteRequest, o: RoutingOpts): Promise<RouteAnswer> {
+  const prep = await prepare(req, o);
+  return decide(req, prep, await judge(req, prep, o));
+}
+
+/**
+ * Spec 1.5 plan 24, batch `route`: the lanes of one role (one run, one profile) share one discovery refresh and
+ * one catalog, and Jev is asked about all of them at once (Ruling 6: one request per lane, all in flight
+ * together, so each keeps its own cache key and jev.jsonl row). Then each lane is decided in order, every
+ * routed start counting as a use of its quota, so lanes that tie spread over the quotas.
+ */
+async function routeMany(reqs: RouteRequest[], o: RoutingOpts): Promise<RouteAnswer[]> {
+  const first = reqs[0];
+  if (!first) return [];
+  const prep = await prepare(first, o);
+  const judged = await Promise.all(reqs.map((r) => judge(r, prep, o)));
+  const usage: Record<string, number> = { ...first.usage };
+  return reqs.map((req, i) => {
+    const out = decide(req, { ...prep, p: { ...prep.p, usage: { ...usage } } }, judged[i] ?? null);
+    const q = quotaUsage([out.rung]);
+    for (const [k, n] of Object.entries(q)) usage[k] = (usage[k] ?? 0) + n;
+    return out;
+  });
+}
+
 async function verdict<T extends string>(
   runDir: string,
   set: Extract<SetName, "finding" | "same-defect">,
@@ -171,7 +275,16 @@ async function verdict<T extends string>(
 /** The routing port over the 1.0 catalog and Jev (spec §5); `o` lets tests inject Jev's transport. */
 export function routingService(o: RoutingOpts = {}): RoutingPort {
   return {
-    route: (req) => route(req, o),
+    route: async (req) => {
+      const result = await route(req, o);
+      assertNativeHost(result.rung, req.host ?? "unknown");
+      return result;
+    },
+    routeMany: async (reqs) => {
+      const results = await routeMany(reqs, o);
+      results.forEach((r, i) => assertNativeHost(r.rung, reqs[i]?.host ?? "unknown"));
+      return results;
+    },
     finding: (runDir, laneText, finding, use) =>
       verdict(
         runDir,

package/src/services/run-debug.ts

@@ -1,15 +1,107 @@
-import { closeSync, fstatSync, openSync, readSync } from "node:fs";
+import { closeSync, existsSync, fstatSync, openSync, readSync } from "node:fs";
+import { sessionKey, type HostContext, type HostSessionRef } from "../domain/host.ts";
+import { deliveryState, readDelivery, type DeliveryState } from "../infra/delivery.ts";
 import type { ExitInfo, RunRecord } from "../domain/record.ts";
-import { dispatchPaths, readExit } from "../infra/dispatch-dir.ts";
+import { awaitsCollect, dispatchPaths, readExit } from "../infra/dispatch-dir.ts";
 import { redact } from "../infra/log.ts";
-import { listDispatches } from "./dispatches.ts";
+import { type Dispatch, listDispatches } from "./dispatches.ts";
 import { registerSavedSecrets } from "./jev-service.ts";
+import { runOwner } from "./sessions.ts";
 import { readRecords, type Run } from "./run-store.ts";
 
 /** How many trailing lines of stderr, events and the supervisor log `runs show --debug` prints. */
 export const TAIL_LINES = 20;
 
-export interface DispatchDebug {
+export function inspectionHost(host: HostContext): HostContext {
+  const target = host.session;
+  return {
+    host: host.host,
+    conflict: host.conflict,
+    session: target
+      ? {
+          host: target.host,
+          sessionId: target.sessionId,
+          hostSessionId: target.hostSessionId,
+          name: target.name,
+        }
+      : null,
+  };
+}
+
+export interface DeliveryInspection {
+  eventId: string;
+  delivery: DeliveryState;
+  receipt: { msgId: string; target: HostSessionRef; at: string } | null;
+}
+
+export function inspectDelivery(
+  run: Run,
+  d: Dispatch,
+  recorded: boolean,
+  eventId = JSON.stringify([
+    run.id,
+    d.admit.dispatchId,
+    !recorded && existsSync(dispatchPaths(d.dir).stall) ? "stalled" : "finished",
+  ]),
+): DeliveryInspection {
+  const owner = runOwner(run);
+  const target = owner
+    ? { host: owner.host, sessionId: owner.sessionId, hostSessionId: null, name: null }
+    : null;
+  const finished = eventId === JSON.stringify([run.id, d.admit.dispatchId, "finished"]);
+  const collected = finished && recorded && !awaitsCollect(d.dir);
+  try {
+    const receipt = target
+      ? readDelivery(d.dir).findLast(
+          (a) =>
+            a.status === "accepted" &&
+            a.eventIds.includes(eventId) &&
+            sessionKey(a.target) === sessionKey(target),
+        )
+      : undefined;
+    return {
+      eventId,
+      delivery: collected
+        ? "collected"
+        : (recorded || !finished) && target
+          ? deliveryState(d.dir, target, eventId)
+          : "pending",
+      receipt: receipt?.msgId
+        ? {
+            msgId: receipt.msgId,
+            at: receipt.at,
+            target: {
+              host: receipt.target.host,
+              sessionId: receipt.target.sessionId,
+              hostSessionId: receipt.target.hostSessionId,
+              name: receipt.target.name,
+            },
+          }
+        : null,
+    };
+  } catch {
+    return { eventId, delivery: collected ? "collected" : "ambiguous", receipt: null };
+  }
+}
+
+/**
+ * Stored stalled and finished events are separate; only finished records can be collected. A stall the role
+ * outlived that is still pending for the current owner can never go out (the notifier drops it and retry
+ * refuses it), so it is not shown; one with an outcome for this owner (accepted, ambiguous) still is.
+ */
+export function inspectDeliveries(run: Run, d: Dispatch, recorded: boolean): DeliveryInspection[] {
+  const exited = existsSync(dispatchPaths(d.dir).exit);
+  return [...(recorded ? ["finished"] : []), ...(existsSync(dispatchPaths(d.dir).stall) ? ["stalled"] : [])]
+    .map((kind) => ({
+      kind,
+      ...inspectDelivery(run, d, recorded, JSON.stringify([run.id, d.admit.dispatchId, kind])),
+    }))
+    .filter((e) => !(e.kind === "stalled" && exited && e.delivery === "pending"))
+    .map(({ kind: _kind, ...e }) => e);
+}
+
+export interface DispatchDebug extends DeliveryInspection {
+  deliveries: DeliveryInspection[];
   name: string;
   dispatchId: string;
   rung: string;
@@ -65,6 +157,8 @@ export function runDebug(run: Run, name?: string): DispatchDebug[] {
     .map((d) => {
       const p = dispatchPaths(d.dir);
       return redact({
+        ...inspectDelivery(run, d, records.has(d.admit.dispatchId)),
+        deliveries: inspectDeliveries(run, d, records.has(d.admit.dispatchId)),
         name: d.admit.name,
         dispatchId: d.admit.dispatchId,
         rung: d.admit.rung,

package/src/services/run-pin.ts

@@ -0,0 +1,125 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { z } from "zod";
+import { errorMessage } from "../domain/errors.ts";
+import { ACCESS } from "../domain/record.ts";
+import { ROLES } from "../domain/roles.ts";
+import { readVersioned, writeJsonAtomic } from "../infra/store.ts";
+import type { Deps, ProfileView } from "./ports.ts";
+import { findRun, type Run } from "./run-store.ts";
+import { refreshState } from "./state.ts";
+
+// Spec 1.5 "Pinned per run": run_start records the profile the run started on, each role's access and each
+// backend's isolation. A later change (another active profile, a toggled isolation, a version that changed a
+// default) is logged in state.md and status, and dispatch keeps the pinned values until the owner re-pins.
+
+const PinSchema = z.looseObject({
+  schema: z.literal(1),
+  at: z.string(),
+  profile: z.string(),
+  access: z.partialRecord(z.enum(ROLES), z.enum(ACCESS)),
+  isolated: z.record(z.string(), z.boolean()),
+});
+export type Pin = z.infer<typeof PinSchema>;
+
+export const pinFile = (run: Run): string => join(run.dir, "pin.json");
+
+/** The run's pin, or null: a run from before 1.5, or a pin that cannot be read (the profile then rules). */
+export function readPin(run: Run): Pin | null {
+  if (!existsSync(pinFile(run))) return null;
+  try {
+    return readVersioned(pinFile(run), PinSchema, 1);
+  } catch {
+    return null;
+  }
+}
+
+const pinOf = (view: ProfileView, at: string): Pin => ({
+  schema: 1,
+  at,
+  profile: view.name,
+  access: Object.fromEntries(
+    Object.entries(view.roles).flatMap(([role, rc]) => (rc ? [[role, rc.access]] : [])),
+  ),
+  isolated: Object.fromEntries(Object.entries(view.isolated).map(([k, v]) => [k, v === true])),
+});
+
+/** Records what the run runs on now: the profile its repo runs on, each role's access, each backend's isolation. */
+export function writePin(deps: Deps, run: Run): Pin {
+  const pin = pinOf(deps.profiles.forRepo(run.meta.repo), new Date(deps.now()).toISOString());
+  writeJsonAtomic(pinFile(run), pin);
+  return pin;
+}
+
+/** The pinned profile's view, or null with why when it can no longer be read (deleted, invalid). */
+function pinnedView(deps: Deps, run: Run, pin: Pin): { view: ProfileView | null; why: string | null } {
+  const current = deps.profiles.forRepo(run.meta.repo);
+  if (current.name === pin.profile) return { view: current, why: null };
+  try {
+    return { view: deps.profiles.get(pin.profile, run.meta.repo).profile, why: null };
+  } catch (e) {
+    return { view: null, why: errorMessage(e) };
+  }
+}
+
+/**
+ * The profile a run's dispatch, routing and failover read: its pinned profile, with its pinned access and
+ * isolation laid over it. A run with no pin, or whose pinned profile is gone, reads its repo's profile.
+ */
+export function runProfile(deps: Deps, run: Run): ProfileView {
+  const pin = readPin(run);
+  if (!pin) return deps.profiles.forRepo(run.meta.repo);
+  const view = pinnedView(deps, run, pin).view ?? deps.profiles.forRepo(run.meta.repo);
+  const roles: ProfileView["roles"] = {};
+  for (const [role, rc] of Object.entries(view.roles) as [
+    keyof ProfileView["roles"],
+    NonNullable<ProfileView["roles"][keyof ProfileView["roles"]]>,
+  ][])
+    roles[role] = { ...rc, access: pin.access[role] ?? rc.access };
+  // a backend the pin does not name ran with no isolation when the run started
+  const backends = new Set([...Object.keys(view.isolated), ...Object.keys(pin.isolated)]);
+  const isolated = Object.fromEntries([...backends].map((k) => [k, pin.isolated[k] === true]));
+  return { ...view, roles, isolated };
+}
+
+/** What changed since the pin, one line each, in words; none while the run's repo still runs as pinned. */
+export function pinChanges(deps: Deps, run: Run): string[] {
+  const pin = readPin(run);
+  if (!pin) return [];
+  let now: ProfileView;
+  try {
+    now = deps.profiles.forRepo(run.meta.repo);
+  } catch {
+    // a profile that cannot be read is doctor's and profile validate's to report, not a pin change
+    return [];
+  }
+  const out: string[] = [];
+  if (now.name !== pin.profile) {
+    const { why } = pinnedView(deps, run, pin);
+    out.push(
+      `profile: pinned ${pin.profile}, the repo now runs on ${now.name}${why ? ` (${pin.profile} cannot be read: ${why}; dispatch reads ${now.name})` : ""}`,
+    );
+  }
+  for (const [role, access] of Object.entries(pin.access)) {
+    const was = now.roles[role as keyof ProfileView["roles"]]?.access;
+    if (was !== undefined && was !== access) out.push(`${role} access: pinned ${access}, now ${was}`);
+  }
+  for (const backend of new Set([...Object.keys(pin.isolated), ...Object.keys(now.isolated)])) {
+    const isolated = pin.isolated[backend] === true;
+    const was = now.isolated[backend] === true;
+    if (was !== isolated) out.push(`${backend} isolated: pinned ${isolated}, now ${was}`);
+  }
+  return out;
+}
+
+/** `run_pin`: the owner re-pins the run to what its repo runs on now. Returns the new pin and what it changed. */
+export async function repin(
+  deps: Deps,
+  i: { run: string },
+): Promise<{ pin: Pin; changed: string[]; hints?: string[] }> {
+  const run = findRun(i.run);
+  const changed = pinChanges(deps, run);
+  const pin = writePin(deps, run);
+  const { hints } = await refreshState(run, { pinChanges: [] });
+  return { pin, changed, ...(hints.length ? { hints } : {}) };
+}