@agentswf/engine 0.1.0-beta.5 → 0.1.0

package/src/run-read.ts

@@ -1,27 +1,35 @@
 import { createHash } from "node:crypto";
 import { join } from "node:path";
-import type { Activity, RunLogLine } from "@agentswf/contract/records";
+import type { Activity, RunLine, RunLogLine } from "@agentswf/contract/records";
 import type { AgentKey, HarnessKind } from "@agentswf/contract/workflow";
 import { findHarness, harnessState, readActivity } from "@agentswf/harness";
+import { messageOf } from "./errors";
+import { readRunLines } from "./run-lines";
+import { unfenced } from "./run-policy/waits";
 import { readNumberedEvents } from "./runs";
 
+/** An agent's native session, as `run.jsonl` records it. */
+export type SessionLine = Extract<RunLine, { kind: "session" }>;
+
 /**
- * A native session the run's log names, numbered by the line that first named it, so a session
- * named later never renumbers the others. Its files are found under its agent's home and working
+ * A native session `run.jsonl` names, numbered by the line that first named it, so a session named
+ * later never renumbers the others. Its files are found under its agent's home and working
  * directory: a sandbox's, for an agent in one.
  */
-export type NativeSession = {
+type NativeSession = {
   n: number;
   harness: HarnessKind;
   id: string;
   agent?: AgentKey;
+  /** The home its line names, for an agent whose opening the log lost. */
+  home?: string;
 };
 
 /** A span of time, open while it has no end. */
 type Span = { from: number; to?: number };
 
 /** What the log says of an agent: where its files are, and the spans its activity is the run's. */
-export type RunAgent = {
+type RunAgent = {
   key: AgentKey;
   cwd?: string;
   home?: string;
@@ -36,8 +44,14 @@ export type RunAgent = {
 
 export type RunLog = {
   lines: { number: number; line: RunLogLine }[];
+  /** `run.jsonl`'s lines that first name a session, each with its line number there, from 1. */
+  named: { number: number; line: SessionLine }[];
+  /** How many complete lines `run.jsonl` has: where a read of it stopped. */
+  runLines: number;
   sessions: NativeSession[];
   agents: Map<AgentKey, RunAgent>;
+  /** Why `run.jsonl` could not be read, so no session is known. */
+  unreadSessions?: string;
 };
 
 /** `ended`: no attempt is running, so a span still open on the calling session ended with it. */
@@ -45,10 +59,26 @@ export async function readRunLog(
   runDir: string,
   options: { ended?: boolean } = {},
 ): Promise<RunLog> {
-  const lines = await readNumberedEvents(runDir);
+  const [lines, recorded] = await Promise.all([
+    readNumberedEvents(runDir),
+    readRunLines(runDir).then(
+      ({ lines }) => {
+        const kept = new Set(unfenced(lines));
+        const sessions = lines.flatMap((line, index) =>
+          line.kind === "session" && kept.has(line) ? [{ number: index + 1, line }] : [],
+        );
+        return { sessions, count: lines.length };
+      },
+      (error: unknown) => ({
+        sessions: [] as { number: number; line: SessionLine }[],
+        count: 0,
+        unread: messageOf(error),
+      }),
+    ),
+  ]);
+  const named: RunLog["named"] = [];
   const sessions: NativeSession[] = [];
   const agents = new Map<AgentKey, RunAgent>();
-  const agentOf = new Map<string, AgentKey>();
   const agent = (key: AgentKey) => {
     const found = agents.get(key) ?? { key, caller: false, operations: [], windows: [] };
     agents.set(key, found);
@@ -58,14 +88,21 @@ export async function readRunLog(
     if (held.windows.at(-1)?.to === undefined && held.windows.length > 0) return;
     held.windows.push({ from });
   };
-  const name = (harness: HarnessKind, id: string, key?: AgentKey) => {
-    const known = sessions.find((each) => each.harness === harness && each.id === id);
-    if (known) {
-      if (key && !known.agent) known.agent = key;
-      return;
+  // Each session's agent is on its line; which of its activity is the run's, its agent's operations
+  // and windows below say.
+  for (const { number, line } of recorded.sessions) {
+    if (sessions.some((each) => each.harness === line.harness && each.id === line.session)) {
+      continue;
     }
-    sessions.push({ n: sessions.length + 1, harness, id, ...(key ? { agent: key } : {}) });
-  };
+    named.push({ number, line });
+    sessions.push({
+      n: sessions.length + 1,
+      harness: line.harness,
+      id: line.session,
+      agent: line.agent,
+      ...(line.home === undefined ? {} : { home: line.home }),
+    });
+  }
   // An attempt that died left its spans open. An agent's end where the next attempt begins; the
   // calling session's at the dead attempt's last line, past which its file is the operator's.
   let attempt: number | undefined;
@@ -107,13 +144,6 @@ export async function readRunLog(
         const from = Date.parse(line.deliveredAt ?? line.at);
         if (!started.caller) open(started, from);
         started.operations.push({ id: line.operationId, from });
-        agentOf.set(line.operationId, line.agent);
-        for (const session of line.sessions) name(session.harness, session.id, line.agent);
-        break;
-      }
-      case "session-known": {
-        const key = line.agent ?? (line.operationId ? agentOf.get(line.operationId) : undefined);
-        name(line.session.harness, line.session.id, key);
         break;
       }
       case "turn":
@@ -129,7 +159,6 @@ export async function readRunLog(
           if (!settled.caller) open(settled, from);
           settled.operations.push({ id: line.operationId, from, to: to ?? at });
         }
-        for (const session of line.sessions) name(session.harness, session.id, line.agent);
         break;
       }
       case "operation-failed": {
@@ -145,7 +174,14 @@ export async function readRunLog(
     }
   }
   if (options.ended && lastAt !== undefined) closeAll(Number.POSITIVE_INFINITY, lastAt);
-  return { lines, sessions, agents };
+  return {
+    lines,
+    named,
+    runLines: recorded.count,
+    sessions,
+    agents,
+    ...("unread" in recorded ? { unreadSessions: recorded.unread } : {}),
+  };
 }
 
 const within = (span: Span, at: number) =>
@@ -177,7 +213,7 @@ const TAIL_BYTES = 4 * 1024 * 1024;
 const READ_BYTES = 4 * 1024 * 1024;
 
 /** One native file of a session, by a key its name gives, so a late file renumbers nothing. */
-export type SessionFile = { session: NativeSession; key: string; path: string };
+type SessionFile = { session: NativeSession; key: string; path: string };
 
 /** A session's files, the first its own and keyed `""`; none where they cannot be found. */
 async function sessionFiles(
@@ -188,7 +224,9 @@ async function sessionFiles(
   const spec = findHarness(session.harness);
   if (!spec?.sessionFiles || !spec.activityRow) return [];
   const home =
-    agent?.home ?? harnessState()[session.harness as keyof ReturnType<typeof harnessState>];
+    agent?.home ??
+    session.home ??
+    harnessState()[session.harness as keyof ReturnType<typeof harnessState>];
   if (!home) return [];
   const found = await spec
     .sessionFiles(home, session.id, agent?.cwd ?? runCwd)
@@ -206,27 +244,33 @@ async function sessionFiles(
 }
 
 /**
- * Where a read stopped: the event log's last line number, and each native file's byte offset.
- * Opaque to its holder, who copies it back; versioned, so a later awf can refuse an old one.
+ * Where a read stopped: the event log's last line number, `run.jsonl`'s, and each native file's
+ * byte offset. Opaque to its holder, who copies it back; versioned, so a later awf can refuse an
+ * old one.
  */
-export type Cursor = { log: number; files: Record<string, number> };
+export type Cursor = { log: number; run: number; files: Record<string, number> };
+
+/** A read from the start. */
+export const START: Cursor = Object.freeze({ log: 0, run: 0, files: Object.freeze({}) });
 
 export function encodeCursor(cursor: Cursor): string {
-  return `c1.${Buffer.from(JSON.stringify([cursor.log, cursor.files])).toString("base64url")}`;
+  const fields = [cursor.log, cursor.run, cursor.files];
+  return `c2.${Buffer.from(JSON.stringify(fields)).toString("base64url")}`;
 }
 
 export function decodeCursor(text: string): Cursor | undefined {
-  if (!text.startsWith("c1.")) return undefined;
+  if (!text.startsWith("c2.")) return undefined;
   try {
-    const [log, files] = JSON.parse(Buffer.from(text.slice(3), "base64url").toString("utf8"));
-    if (!Number.isSafeInteger(log) || log < 0 || typeof files !== "object" || files === null) {
+    const [log, run, files] = JSON.parse(Buffer.from(text.slice(3), "base64url").toString("utf8"));
+    const line = (n: unknown) => Number.isSafeInteger(n) && (n as number) >= 0;
+    if (!line(log) || !line(run) || typeof files !== "object" || files === null) {
       return undefined;
     }
     const offsets = Object.entries(files as Record<string, unknown>);
     if (!offsets.every(([, offset]) => Number.isSafeInteger(offset) && (offset as number) >= 0)) {
       return undefined;
     }
-    return { log, files: Object.fromEntries(offsets) as Record<string, number> };
+    return { log, run, files: Object.fromEntries(offsets) as Record<string, number> };
   } catch {
     return undefined;
   }
@@ -237,13 +281,15 @@ function fileName(file: Pick<SessionFile, "session" | "key">): string {
   return `a${file.session.n}${file.key ? `-${file.key}` : ""}`;
 }
 
-/** An event's handle: its line in the log, or its file and its place in that file. */
+/** An event's handle: its line in the log or in `run.jsonl`, or its file and its place in that file. */
 export type Handle =
   | { kind: "event"; line: number }
+  | { kind: "run"; line: number }
   | { kind: "activity"; file: string; offset: number; index: number };
 
 export function encodeHandle(handle: Handle): string {
   if (handle.kind === "event") return `e${handle.line}`;
+  if (handle.kind === "run") return `r${handle.line}`;
   const index = handle.index > 0 ? `.${handle.index}` : "";
   return `${handle.file}.${handle.offset.toString(16)}${index}`;
 }
@@ -252,6 +298,8 @@ export function decodeHandle(text: string): Handle | undefined {
   const plain = text.startsWith("@") ? text.slice(1) : text;
   const event = /^e([1-9]\d*)$/.exec(plain);
   if (event) return { kind: "event", line: Number(event[1]) };
+  const run = /^r([1-9]\d*)$/.exec(plain);
+  if (run) return { kind: "run", line: Number(run[1]) };
   const activity = /^(a[1-9]\d*(?:-[0-9a-f]{12})?)\.([0-9a-f]+)(?:\.([1-9]\d*))?$/.exec(plain);
   if (!activity) return undefined;
   return {
@@ -262,7 +310,10 @@ export function decodeHandle(text: string): Handle | undefined {
   };
 }
 
-/** One event of a read: a line of the log, or an agent's activity, with its handle. */
+/**
+ * One event of a read: a line of the log, a line of `run.jsonl` that first names a session, or an
+ * agent's activity, with its handle.
+ */
 export type ReadEvent = {
   handle: string;
   at: string;
@@ -270,6 +321,7 @@ export type ReadEvent = {
   time: number;
 } & (
   | { source: "log"; line: RunLogLine }
+  | { source: "run"; line: SessionLine }
   | {
       source: "activity";
       agent?: AgentKey;
@@ -280,13 +332,14 @@ export type ReadEvent = {
 );
 
 /**
- * Everything after `cursor`: the log's lines and each session's activity that is the run's, merged
- * by time. The cursor returned is where this read stopped, past nothing torn.
+ * Everything after `cursor`: the log's lines, the sessions `run.jsonl` names, and each session's
+ * activity that is the run's, merged by time. The cursor returned is where this read stopped, past
+ * nothing torn.
  */
 export async function readRun(
   runDir: string,
   runCwd: string,
-  cursor: Cursor = { log: 0, files: {} },
+  cursor: Cursor = START,
   options: {
     full?: boolean;
     ended?: boolean;
@@ -325,6 +378,20 @@ export async function readRun(
       line,
     });
   }
+  // As the log's, `run.jsonl`'s lines keep their order.
+  let since = Number.NEGATIVE_INFINITY;
+  for (const { number, line } of log.named) {
+    if (number <= cursor.run) continue;
+    since = Math.max(since, Date.parse(line.at) || since);
+    events.push({
+      handle: encodeHandle({ kind: "run", line: number }),
+      at: line.at,
+      time: since,
+      source: "run",
+      line,
+    });
+  }
+  const run = Math.max(cursor.run, log.runLines);
   const files = { ...cursor.files };
   // A pane's pi names its session twice, by id and by path: one file, read once.
   const seen = new Set<string>();
@@ -396,8 +463,8 @@ export async function readRun(
   events.sort((a, b) => a.time - b.time);
   const more = horizon < Number.POSITIVE_INFINITY;
   const beyond = events.findIndex((event) => event.time > horizon);
-  if (beyond < 0) return { events, cursor: { log: last, files }, log, cut, more };
-  const next = cutCursor(events, beyond, { log: last, files });
+  if (beyond < 0) return { events, cursor: { log: last, run, files }, log, cut, more };
+  const next = cutCursor(events, beyond, { log: last, run, files });
   return { events: events.slice(0, beyond), cursor: next, log, cut, more };
 }
 
@@ -406,10 +473,11 @@ export async function readRun(
  * time order and the log's lines in line order among them, so nothing shown is read again.
  */
 export function cutCursor(all: readonly ReadEvent[], cut: number, end: Cursor): Cursor {
-  const next: Cursor = { log: end.log, files: { ...end.files } };
+  const next: Cursor = { log: end.log, run: end.run, files: { ...end.files } };
   for (const event of all.slice(cut)) {
     const handle = decodeHandle(event.handle)!;
     if (handle.kind === "event") next.log = Math.min(next.log, handle.line - 1);
+    else if (handle.kind === "run") next.run = Math.min(next.run, handle.line - 1);
     else
       next.files[handle.file] = Math.min(next.files[handle.file] ?? handle.offset, handle.offset);
   }

package/src/run-state.ts

@@ -1,18 +1,23 @@
-import { join } from "node:path";
-import type { AttemptRecord, MailboxRecord, WaitingJson } from "@agentswf/contract/records";
+import { dirname, join } from "node:path";
+import type { AttemptRecord, ClaimedAttempt, WaitingJson } from "@agentswf/contract/records";
 import type { AttemptOutcome } from "@agentswf/contract/workflow";
 import { exists } from "./files";
-import { foldMailbox, nextTimer, readMailbox } from "./mailbox";
-import { listProjects, projectRunRoot } from "./projects";
+import { checkProjectEngine } from "./project-engine";
+import { listProjects, projectOf, projectRunRoot, rootProject } from "./projects";
+import { lockHeld, runLockFile } from "./run-lock";
+import { nextTimer, untakenAtReturn, type WaitState } from "./run-policy/waits";
 import {
+  claimedOf,
   entries,
   idProblem,
   openRun,
   probeProcess,
   type Run,
-  readAttempts,
+  RunUnreadable,
+  readAttemptFiles,
   sameStart,
 } from "./runs";
+import { readWaits } from "./waits";
 
 /** What `ps` says of a process: its start, `gone`, or `unknown`. */
 export type ProcessStatus = (pid: number) => string | "gone" | "unknown";
@@ -28,12 +33,20 @@ export type RunStatus = {
   run: Run;
   /** `{workflow}/{id}`. */
   ref: string;
-  attempts: AttemptRecord[];
+  /** The attempts that ran, in order: what its state is decided from. */
+  attempts: ClaimedAttempt[];
+  /** Every attempt file, in order, a background launch that never claimed its number included. */
+  files: AttemptRecord[];
   state: RunState;
   /** False for a `running` whose process could not be asked about. */
   checked: boolean;
-  /** A run of a workflow with `events` at its wait, live or parked: what its mailbox says. */
+  /** A run of a workflow with `events` at its wait, live or parked: what its last `wait` line says. */
   waiting?: WaitingJson;
+  /**
+   * A completed run of a workflow with `events`: how many events no wait took by when it returned,
+   * from its lines. Its last batch taken was handled, as the code returned after it.
+   */
+  unacknowledged?: number;
 };
 
 /** How long a run folder with no attempt yet may be an `awf run` still starting. */
@@ -44,18 +57,43 @@ export async function runStatus(
   probe: ProcessStatus = probeProcess,
   at = Date.now(),
 ): Promise<RunStatus> {
-  const status = statusOf(run, await readAttempts(run), probe, at);
-  if (!run.record.events) return status;
-  const records = await readMailbox(run.dir);
-  const waiting = waitingOf(status, records);
-  const stuck = status.state !== "running" && stuckOf(status.attempts, records);
-  return { ...status, ...(waiting ? { waiting } : {}), ...(stuck ? { state: "stuck" } : {}) };
+  let status = statusOf(run, await readAttemptFiles(run), probe, at);
+  if (status.attempts.length === 0 && (status.files.length > 0 || status.state === "interrupted")) {
+    // Its launch failed, or is past the minute an attempt has to claim it: unless a live sender
+    // holds its lock, launching it, nothing runs it.
+    const { workflow, id } = run.record;
+    const launching = await lockHeld(runLockFile(run.root, workflow, id));
+    status = { ...status, state: launching ? "running" : "interrupted", checked: true };
+  }
+  if (!run.record.events) {
+    // A plain run's failed attempt is final; only a program's start, whose launches failed, sticks.
+    const stuck = status.state !== "running" && launchesFailed(status.files) >= STUCK_AFTER;
+    return stuck ? { ...status, state: "stuck" } : status;
+  }
+  // A `run.jsonl` that can't be read shows no wait; `awf runs` says why it can't be read.
+  const waiting =
+    status.state === "running" || status.state === "parked"
+      ? await readWaits(run.dir, at).then(
+          (state) => waitingOf(status, state),
+          () => undefined,
+        )
+      : undefined;
+  const returned = status.state === "completed" ? status.attempts.at(-1)?.ended : undefined;
+  const unacknowledged =
+    returned === undefined
+      ? undefined
+      : await readWaits(run.dir, Date.parse(returned)).then(untakenAtReturn, () => undefined);
+  const stuck = status.state !== "running" && stuckOf(status.files);
+  return {
+    ...status,
+    ...(waiting ? { waiting } : {}),
+    ...(unacknowledged === undefined ? {} : { unacknowledged }),
+    ...(stuck ? { state: "stuck" } : {}),
+  };
 }
 
 /** An event run's wait, while it is running or parked there: its label and `until`. */
-function waitingOf(status: RunStatus, records: readonly MailboxRecord[]): WaitingJson | undefined {
-  if (status.state !== "running" && status.state !== "parked") return undefined;
-  const state = foldMailbox(records);
+function waitingOf(status: RunStatus, state: WaitState): WaitingJson | undefined {
   const wait = state.lastWait;
   // A wait an earlier attempt died in is not this one's.
   if (!wait?.waiting || wait.attempt !== status.attempts.at(-1)?.attempt) return undefined;
@@ -66,32 +104,46 @@ function waitingOf(status: RunStatus, records: readonly MailboxRecord[]): Waitin
   };
 }
 
-/** As `runStatus`, from its attempts as already read. */
+/**
+ * As `runStatus`, from its attempt files as already read: decided from the attempts that ran, so a
+ * launch that never claimed its number leaves a parked run parked.
+ */
 export function statusOf(
   run: Run,
-  attempts: AttemptRecord[],
+  files: readonly AttemptRecord[],
   probe: ProcessStatus = probeProcess,
   at = Date.now(),
 ): RunStatus {
+  const attempts = claimedOf(files);
+  const head = { run, ref: `${run.record.workflow}/${run.record.id}`, attempts, files: [...files] };
   const last = attempts.at(-1);
-  const ref = `${run.record.workflow}/${run.record.id}`;
   // Between its folder and its first attempt's claim: an `awf run` starting, or, past a minute,
   // one refused or killed before it claimed one.
   if (!last) {
     const starting = at - Date.parse(run.record.created) < STARTING_MS;
-    return { run, ref, attempts, state: starting ? "running" : "interrupted", checked: !starting };
+    return { ...head, state: starting ? "running" : "interrupted", checked: !starting };
   }
-  if (last.outcome !== undefined) return { run, ref, attempts, state: last.outcome, checked: true };
+  if (last.outcome !== undefined) return { ...head, state: last.outcome, checked: true };
   const now = probe(last.pid);
-  if (now === "unknown") return { run, ref, attempts, state: "running", checked: false };
+  if (now === "unknown") return { ...head, state: "running", checked: false };
   const same = now !== "gone" && sameStart(now, last.processStart);
-  return { run, ref, attempts, state: same ? "running" : "interrupted", checked: true };
+  return { ...head, state: same ? "running" : "interrupted", checked: true };
 }
 
 const ALSO_RUNNING_CHECKED = 20;
 
-/** A reference the commands take, or why there is none to read. */
-export class RunNotFound extends Error {}
+/**
+ * A reference the commands take, or why there is none to read: `no-run`, none by that ref; `id`, a
+ * ref no run could have; `ambiguous`, one naming runs of two workflows or projects.
+ */
+export class RunNotFound extends Error {
+  constructor(
+    message: string,
+    readonly code: "no-run" | "id" | "ambiguous" = "no-run",
+  ) {
+    super(message);
+  }
+}
 
 /**
  * Every run under `runRoot`, newest `created` first. A folder whose `run.json` cannot be read is
@@ -102,16 +154,33 @@ export async function listRuns(
   runRoot: string,
   known?: (dir: string) => Run | undefined,
 ): Promise<Run[]> {
-  const runs: Run[] = [];
+  return (await listRunEntries(runRoot, known)).filter(
+    (each): each is Run => !(each instanceof RunUnreadable),
+  );
+}
+
+/**
+ * The same, with each run of a `format` this awf does not read in its place by `created`, the
+ * last when it has none: `awf runs` lists them `unreadable`.
+ */
+export async function listRunEntries(
+  runRoot: string,
+  known?: (dir: string) => Run | undefined,
+): Promise<(Run | RunUnreadable)[]> {
+  const runs: (Run | RunUnreadable)[] = [];
   for (const workflow of await entries(runRoot).catch(() => [])) {
     for (const id of await entries(join(runRoot, workflow)).catch(() => [])) {
       const run =
         known?.(join(runRoot, workflow, id)) ??
-        (await openRun(runRoot, workflow, id).catch(() => undefined));
+        (await openRun(runRoot, workflow, id).catch((error: unknown) =>
+          error instanceof RunUnreadable ? error : undefined,
+        ));
       if (run) runs.push(run);
     }
   }
-  return runs.sort((a, b) => b.record.created.localeCompare(a.record.created));
+  const created = (each: Run | RunUnreadable) =>
+    each instanceof RunUnreadable ? (each.created ?? "") : each.record.created;
+  return runs.sort((a, b) => created(b).localeCompare(created(a)));
 }
 
 /**
@@ -139,7 +208,7 @@ export async function findRun(
   if (slash >= 0) {
     const [workflow, id] = [ref.slice(0, slash), ref.slice(slash + 1)];
     const problem = idProblem(workflow) ?? idProblem(id);
-    if (problem) throw new RunNotFound(`no run ${ref}: ${problem}`);
+    if (problem) throw new RunNotFound(`no run ${ref}: ${problem}`, "id");
     // A record that is there but cannot be read is not a missing run, and says why itself.
     if (!(await exists(join(runRoot, workflow, id, "run.json")))) {
       const missing = await openRun(runRoot, workflow, id).catch((error: unknown) => error);
@@ -148,11 +217,19 @@ export async function findRun(
     const run = await openRun(runRoot, workflow, id);
     return { status: await runStatus(run, probe), alsoRunning: [] };
   }
-  const found = (await listRuns(runRoot)).filter((run) => run.record.id === ref);
-  if (found.length === 0) throw new RunNotFound(`no run ${ref} under ${runRoot}`);
+  const listed = await listRunEntries(runRoot);
+  const found = listed.filter(
+    (run): run is Run => !(run instanceof RunUnreadable) && run.record.id === ref,
+  );
+  if (found.length === 0) {
+    // A run of an earlier format says so, as it does named by its workflow.
+    const old = listed.find((run) => run instanceof RunUnreadable && run.ref.endsWith(`/${ref}`));
+    if (old) throw old;
+    throw new RunNotFound(`no run ${ref} under ${runRoot}`);
+  }
   if (found.length > 1) {
     const refs = found.map((run) => `${run.record.workflow}/${ref}`).join(", ");
-    throw new RunNotFound(`${ref} names a run of more than one workflow: ${refs}`);
+    throw new RunNotFound(`${ref} names a run of more than one workflow: ${refs}`, "ambiguous");
   }
   return { status: await runStatus(found[0]!, probe), alsoRunning: [] };
 }
@@ -160,16 +237,22 @@ export async function findRun(
 /** Failed or crashed attempts in a row, counted together, after which a run is `stuck` (Q13). */
 export const STUCK_AFTER = 3;
 
+/** Launches that never claimed their number since the last attempt that did. */
+function launchesFailed(files: readonly AttemptRecord[]): number {
+  let n = 0;
+  for (const file of [...files].reverse()) {
+    if (file.kind !== "failed-launch") break;
+    n++;
+  }
+  return n;
+}
+
 /**
- * Failed, timed-out or crashed attempts at the end of the run, and launches since the last attempt
- * that never claimed one, counted together; any other ending resets them.
+ * Failed, timed-out or crashed attempts at the end of the run, a background launch that never
+ * claimed its number among them, as its launcher wrote it failed; any other ending resets them.
  */
-function failuresInARow(
-  attempts: readonly AttemptRecord[],
-  records: readonly MailboxRecord[],
-): number {
-  const since = attempts.at(-1)?.started ?? "";
-  let n = records.filter((record) => record.kind === "launch-failed" && record.at > since).length;
+function failuresInARow(attempts: readonly AttemptRecord[]): number {
+  let n = 0;
   for (const attempt of [...attempts].reverse()) {
     const failed =
       attempt.outcome === undefined ||
@@ -185,13 +268,10 @@ function failuresInARow(
  * Whether an event run is `stuck` (Q13): failed, crashed or failed to launch three times in a row,
  * whatever its last ending says, short of one the operator or the code chose.
  */
-export function stuckOf(
-  attempts: readonly AttemptRecord[],
-  records: readonly MailboxRecord[],
-): boolean {
+export function stuckOf(attempts: readonly AttemptRecord[]): boolean {
   const last = attempts.at(-1)?.outcome;
   if (last === "completed" || last === "stopped" || last === "cancelled") return false;
-  return failuresInARow(attempts, records) >= STUCK_AFTER;
+  return failuresInARow(attempts) >= STUCK_AFTER;
 }
 
 /**
@@ -204,12 +284,47 @@ export async function findRunByRef(
   place: { runRoot: string },
   home: string,
 ): Promise<RunStatus> {
-  // Here first: what `--cwd` names wins.
-  let here: RunNotFound;
+  const status = await findAnywhere(ref, place, home);
+  await checkRunEngine(status, home);
+  return status;
+}
+
+/**
+ * Refuses a run this engine doesn't run: as `awf run` checked it, the project of the file it last
+ * ran, not the folder a `--cwd` put it to work in; without a file, the project it is kept in.
+ */
+export async function checkRunEngine(
+  status: Pick<RunStatus, "run" | "attempts">,
+  home: string,
+): Promise<void> {
+  const file = status.attempts.at(-1)?.file ?? status.run.record.file;
+  if (file === undefined) return checkProjectEngine(runProject(status.run));
+  const project = await projectOf(file, home).catch(() => dirname(file));
+  checkProjectEngine(project, { file });
+}
+
+/**
+ * The project a run belongs to: the folder holding the `.awf/runs` it is kept in, else, under a
+ * `--run-root` of the operator's, the directory it works in.
+ */
+function runProject(run: Pick<Run, "root" | "record">): string {
+  return rootProject(run.root, run.record.cwd);
+}
+
+async function findAnywhere(
+  ref: string,
+  place: { runRoot: string },
+  home: string,
+): Promise<RunStatus> {
+  // Here first: what `--cwd` names wins. A run an earlier awf made is said only where no run
+  // that reads has the ref.
+  let here: RunNotFound | RunUnreadable;
+  let old: RunUnreadable | undefined;
   try {
     return (await findRun(place.runRoot, ref)).status;
   } catch (error) {
-    if (!(error instanceof RunNotFound)) throw error;
+    if (error instanceof RunUnreadable) old = error;
+    else if (!(error instanceof RunNotFound)) throw error;
     here = error;
   }
   const found: RunStatus[] = [];
@@ -217,17 +332,22 @@ export async function findRunByRef(
     try {
       found.push((await findRun(projectRunRoot(project), ref)).status);
     } catch (error) {
-      if (!(error instanceof RunNotFound)) throw error;
+      if (error instanceof RunUnreadable) old ??= error;
+      else if (!(error instanceof RunNotFound)) throw error;
     }
   }
   // Workflows of one name in two projects make the same ref: naming the project is the caller's.
   if (found.length > 1)
     throw new RunNotFound(
       `${ref} is a run in ${found.length} projects, ${found.map((each) => each.run.record.cwd).join(" and ")}; --cwd names one`,
+      "ambiguous",
     );
   if (found[0]) return found[0];
+  if (old) throw old;
   // Nowhere: a ref that can't name a run, malformed or naming two workflows, says why.
-  if (malformedRef(ref) || here.message.includes("more than one workflow")) throw here;
+  const problem = malformedRef(ref);
+  if (problem) throw new RunNotFound(`no run ${ref}: ${problem}`, "id");
+  if (here.message.includes("more than one workflow")) throw here;
   throw new RunNotFound(`no run ${ref} under ${place.runRoot}, or in a project awf knows`);
 }