@mmerterden/multi-agent-pipeline 20.23.2 → 20.24.0

package/pipeline/lib/task-graph.mjs

@@ -0,0 +1,446 @@
+/**
+ * @file task-graph.mjs  -  what a plan can work on next, and why the rest waits.
+ *
+ * A plan's todos form a graph through `deps`. A pending todo is:
+ *
+ *   ready     every dependency is completed or skipped
+ *   waiting   a dependency is still pending or in progress
+ *   blocked   it can never start as the plan stands: a dependency failed (or
+ *             ended in any status other than completed / skipped), a
+ *             dependency id is not in the plan, or it sits on a cycle
+ *
+ * A blocked todo names the root cause (`on`), not the nearest dependency, so a
+ * chain behind one failed task points at that task.
+ *
+ * A pending todo carrying an unconfirmed `needsReplan` is waiting with
+ * `reason: "needs-replan"` and `because: <the drifted todo>`, even when its
+ * dependencies are done: a todo upstream ended off-plan, so the plan this one
+ * was written against no longer holds, and someone confirms it before it
+ * starts (plan-todos.sh confirm-replan). markDrift sets the flag on every
+ * pending transitive dependent of the drifted todo, and only on those.
+ *
+ * The ready set is ranked by how much pending work each task unblocks
+ * (transitive dependents), then by the longest chain of dependents behind it
+ * (critical path), then by plan order. Doing the task that frees the most work
+ * first keeps parallel workers fed.
+ *
+ * Zero runtime dependencies (ADR-0004).
+ *
+ * @module pipeline/lib/task-graph
+ */
+
+import { readFileSync } from "node:fs";
+import { normalizeFile } from "../scripts/_plan-overlap.mjs";
+import { invokedDirectly } from "./invoked-directly.mjs";
+
+const DONE = new Set(["completed", "skipped"]);
+const OPEN = new Set(["pending", "in_progress"]);
+
+const depsOf = (todo) => [...new Set((Array.isArray(todo.deps) ? todo.deps : []).map(String))];
+
+const awaitsReplan = (todo) =>
+  Boolean(
+    todo.needsReplan && typeof todo.needsReplan === "object" && !todo.needsReplan.confirmedAt,
+  );
+
+/**
+ * The strongly connected components of the `deps` graph that hold a cycle:
+ * every component of two or more todos, and a todo that depends on itself.
+ * Tarjan's algorithm, so a todo that closes a second loop through a member
+ * already visited is still found. Each component is ordered by a walk along
+ * `deps` from its first todo in plan order; components come in plan order.
+ *
+ * @returns {string[][]}
+ */
+function cyclicComponents(order, byId) {
+  const index = new Map();
+  const low = new Map();
+  const stack = [];
+  const onStack = new Set();
+  const found = [];
+  let next = 0;
+  const strong = (id) => {
+    index.set(id, next);
+    low.set(id, next);
+    next++;
+    stack.push(id);
+    onStack.add(id);
+    for (const d of depsOf(byId.get(id))) {
+      if (!byId.has(d)) continue;
+      if (!index.has(d)) {
+        strong(d);
+        low.set(id, Math.min(low.get(id), low.get(d)));
+      } else if (onStack.has(d)) {
+        low.set(id, Math.min(low.get(id), index.get(d)));
+      }
+    }
+    if (low.get(id) !== index.get(id)) return;
+    const members = new Set();
+    let m;
+    do {
+      m = stack.pop();
+      onStack.delete(m);
+      members.add(m);
+    } while (m !== id);
+    if (members.size > 1 || depsOf(byId.get(id)).includes(id)) found.push(members);
+  };
+  for (const id of order) if (!index.has(id)) strong(id);
+
+  const position = new Map(order.map((id, i) => [id, i]));
+  return found
+    .map((members) => {
+      const first = [...members].sort((a, b) => position.get(a) - position.get(b))[0];
+      const walk = [];
+      const seen = new Set();
+      const go = (id) => {
+        if (seen.has(id)) return;
+        seen.add(id);
+        walk.push(id);
+        for (const d of depsOf(byId.get(id))) if (members.has(d)) go(d);
+      };
+      go(first);
+      return walk;
+    })
+    .sort((a, b) => position.get(a[0]) - position.get(b[0]));
+}
+
+function indexTodos(todos) {
+  const list = (Array.isArray(todos) ? todos : []).filter((t) => t && t.id != null);
+  return {
+    order: list.map((t) => String(t.id)),
+    byId: new Map(list.map((t) => [String(t.id), t])),
+  };
+}
+
+/**
+ * Which cycle each todo sits on: todo id -> the first todo of its strongly
+ * connected component. Todos on no cycle are absent, including one that only
+ * depends on a cycle. Two todos share a value exactly when each reaches the
+ * other through `deps`.
+ *
+ * @param {Array<{id: string, deps?: string[]}>} todos
+ * @returns {Map<string, string>}
+ */
+export function cyclicComponentOf(todos) {
+  const { order, byId } = indexTodos(todos);
+  const of = new Map();
+  for (const c of cyclicComponents(order, byId)) for (const id of c) of.set(id, c[0]);
+  return of;
+}
+
+/**
+ * @param {Array<{id: string, task?: string, deps?: string[], status?: string}>} todos
+ * @returns {{
+ *   ready: Array<{id: string, task: string|null, unblocks: number, criticalPath: number}>,
+ *   waiting: Array<{id: string, on: string[], reason?: "needs-replan", because?: string}>,
+ *   blocked: Array<{id: string, reason: "failed-dependency"|"missing-dependency"|"cycle", on: string}>,
+ *   cycles: string[][],
+ * }}  `cycles`: one entry per strongly connected component on a cycle, every member listed.
+ */
+export function analyzeGraph(todos) {
+  const { order, byId } = indexTodos(todos);
+  const cycles = cyclicComponents(order, byId);
+  const onCycle = new Map();
+  for (const c of cycles) for (const id of c) onCycle.set(id, c[0]);
+
+  // Resolved state per todo, memoised. A cycle member resolves to blocked
+  // before its dependencies are read, so the recursion always ends.
+  const state = new Map();
+  const resolve = (id) => {
+    if (state.has(id)) return state.get(id);
+    const todo = byId.get(id);
+    const status = String(todo.status || "pending");
+    let s;
+    if (DONE.has(status)) s = { kind: "done" };
+    else if (status === "in_progress") s = { kind: "active" };
+    else if (!OPEN.has(status)) s = { kind: "failed" };
+    else if (onCycle.has(id)) s = { kind: "blocked", reason: "cycle", on: onCycle.get(id) };
+    else {
+      state.set(id, { kind: "resolving" });
+      s = null;
+      const pendingOn = [];
+      for (const d of depsOf(todo)) {
+        if (!byId.has(d)) {
+          s = { kind: "blocked", reason: "missing-dependency", on: d };
+          break;
+        }
+        const ds = resolve(d);
+        if (ds.kind === "failed") {
+          s = { kind: "blocked", reason: "failed-dependency", on: d };
+          break;
+        }
+        if (ds.kind === "blocked") {
+          s = { kind: "blocked", reason: ds.reason, on: ds.on };
+          break;
+        }
+        if (ds.kind !== "done") pendingOn.push(d);
+      }
+      if (!s && awaitsReplan(todo))
+        s = {
+          kind: "waiting",
+          on: pendingOn,
+          reason: "needs-replan",
+          because: String(todo.needsReplan.because ?? ""),
+        };
+      if (!s) s = pendingOn.length ? { kind: "waiting", on: pendingOn } : { kind: "ready" };
+    }
+    state.set(id, s);
+    return s;
+  };
+  for (const id of order) resolve(id);
+
+  // Reverse edges among open todos, for the ranking.
+  const dependents = new Map(order.map((id) => [id, []]));
+  for (const id of order) {
+    if (!OPEN.has(String(byId.get(id).status || "pending"))) continue;
+    for (const d of depsOf(byId.get(id))) if (dependents.has(d)) dependents.get(d).push(id);
+  }
+  const reach = (id) => {
+    const seen = new Set();
+    const stack = [...dependents.get(id)];
+    while (stack.length) {
+      const n = stack.pop();
+      if (seen.has(n) || n === id) continue;
+      seen.add(n);
+      stack.push(...dependents.get(n));
+    }
+    return seen.size;
+  };
+  const depthMemo = new Map();
+  const depth = (id, trail = new Set()) => {
+    if (depthMemo.has(id)) return depthMemo.get(id);
+    if (trail.has(id)) return 0;
+    trail.add(id);
+    let best = 0;
+    for (const n of dependents.get(id)) best = Math.max(best, 1 + depth(n, trail));
+    trail.delete(id);
+    depthMemo.set(id, best);
+    return best;
+  };
+
+  const ready = [];
+  const waiting = [];
+  const blocked = [];
+  order.forEach((id, index) => {
+    if (String(byId.get(id).status || "pending") !== "pending") return;
+    const s = state.get(id);
+    if (s.kind === "ready")
+      ready.push({
+        id,
+        task: byId.get(id).task ?? null,
+        unblocks: reach(id),
+        criticalPath: depth(id),
+        index,
+      });
+    else if (s.kind === "waiting")
+      waiting.push(
+        s.reason ? { id, on: s.on, reason: s.reason, because: s.because } : { id, on: s.on },
+      );
+    else if (s.kind === "blocked") blocked.push({ id, reason: s.reason, on: s.on });
+  });
+  ready.sort(
+    (a, b) => b.unblocks - a.unblocks || b.criticalPath - a.criticalPath || a.index - b.index,
+  );
+  return { ready: ready.map(({ index: _index, ...r }) => r), waiting, blocked, cycles };
+}
+
+/**
+ * Every todo that depends on `id`, directly or through other todos, in plan
+ * order. `id` itself is never listed, also when it sits on a cycle.
+ *
+ * @param {Array<{id: string, deps?: string[]}>} todos
+ * @param {string} id
+ * @returns {string[]}
+ */
+export function dependentsOf(todos, id) {
+  const list = (Array.isArray(todos) ? todos : []).filter((t) => t && t.id != null);
+  const start = String(id);
+  const reverse = new Map(list.map((t) => [String(t.id), []]));
+  for (const t of list)
+    for (const d of depsOf(t)) if (reverse.has(d)) reverse.get(d).push(String(t.id));
+  const seen = new Set();
+  const stack = [...(reverse.get(start) || [])];
+  while (stack.length) {
+    const n = stack.pop();
+    if (n === start || seen.has(n)) continue;
+    seen.add(n);
+    stack.push(...reverse.get(n));
+  }
+  return list.map((t) => String(t.id)).filter((x) => seen.has(x));
+}
+
+const baseName = (path) => path.slice(path.lastIndexOf("/") + 1);
+const stemOf = (path) => {
+  const name = baseName(path);
+  const dot = name.indexOf(".", 1);
+  return dot > 0 ? name.slice(0, dot) : name;
+};
+
+/**
+ * The stem of the source file a test file covers, or "" when `path` is not a
+ * test file by name: `a.test.ts`, `a.spec.ts`, `util_test.go`, `FooTests.swift`,
+ * `BarTest.kt` and `test_calc.py` cover `a`, `a`, `util`, `Foo`, `Bar`, `calc`.
+ */
+function testedStem(path) {
+  const name = baseName(path);
+  const lastDot = name.lastIndexOf(".");
+  const bare = lastDot > 0 ? name.slice(0, lastDot) : name;
+  const m =
+    /^(.+)[._-](?:test|spec)$/.exec(bare) || /^(.+?)Tests?$/.exec(bare) || /^test_(.+)$/.exec(bare);
+  return m ? m[1] : "";
+}
+
+/**
+ * Record that todo `id` ended off-plan and flag what was planned on top of it.
+ *
+ * Drift is what the scope self-check reports for one todo
+ * (features/scope-check.md): files it changed that its plan entry does not
+ * list, or planned items it did not do. An explicit `reason` counts as drift
+ * on its own. Planned files are `files` plus `creates`, compared after
+ * normalizeFile on both sides; a changed test file whose name covers the stem
+ * of a planned file (testedStem) is planned too. A todo with neither list
+ * cannot be judged on files, so only `notDone` and `reason` apply to it.
+ *
+ * On drift the todo gets `drift: {outside, notDone, at}` and every pending
+ * transitive dependent gets `needsReplan: {because, at, reason}`, replacing an
+ * earlier flag and its confirmation. Started or finished dependents are left
+ * alone: their plan is already spent. Pure: returns a new todos array.
+ *
+ * @param {Array<object>} todos
+ * @param {string} id
+ * @param {{changedFiles?: string[], notDone?: number, reason?: string, at?: string}} signal
+ * @returns {{drifted: boolean, outside: string[], notDone: number, flagged: string[], todos: object[]}}
+ */
+export function markDrift(todos, id, signal = {}) {
+  const list = Array.isArray(todos) ? todos : [];
+  const self = list.find((t) => t && String(t.id) === String(id));
+  if (!self) throw new Error(`todo '${id}' is not in the plan`);
+  const at = signal.at || new Date().toISOString();
+  const listed = [self.files, self.creates].filter(Array.isArray);
+  const planned = listed.length
+    ? new Set(listed.flat().map(String).filter(Boolean).map(normalizeFile))
+    : null;
+  const plannedStems = new Set([...(planned || [])].map(stemOf));
+  const isPlanned = (f) => planned.has(f) || plannedStems.has(testedStem(f) || "\0");
+  const changed = [
+    ...new Set((signal.changedFiles || []).map(String).filter(Boolean).map(normalizeFile)),
+  ];
+  const outside = planned ? changed.filter((f) => !isPlanned(f)) : [];
+  const notDone = Number.isInteger(signal.notDone) && signal.notDone > 0 ? signal.notDone : 0;
+  const reason =
+    signal.reason ||
+    [
+      outside.length
+        ? `${outside.length} file${outside.length === 1 ? "" : "s"} outside the plan`
+        : "",
+      notDone ? `${notDone} planned item${notDone === 1 ? "" : "s"} not done` : "",
+    ]
+      .filter(Boolean)
+      .join(", ");
+  if (!reason) return { drifted: false, outside, notDone, flagged: [], todos: list };
+
+  const pending = new Set(
+    list.filter((t) => String(t.status || "pending") === "pending").map((t) => String(t.id)),
+  );
+  const flagged = dependentsOf(list, id).filter((d) => pending.has(d));
+  const flag = new Set(flagged);
+  const because = String(id);
+  const next = list.map((t) => {
+    if (String(t.id) === because) return { ...t, drift: { outside, notDone, at } };
+    if (flag.has(String(t.id))) return { ...t, needsReplan: { because, at, reason } };
+    return t;
+  });
+  return { drifted: true, outside, notDone, flagged, todos: next };
+}
+
+/** The todo to start next, or null when nothing is ready. */
+export function nextReady(todos) {
+  return analyzeGraph(todos).ready[0] ?? null;
+}
+
+/**
+ * CLI: `task-graph.mjs next <agent-state.json> [--task-id <id>]` prints
+ * `id<TAB>task` for the todo to start next, or nothing. When nothing is ready it
+ * says on stderr which todos are blocked and by what, and which wait for a
+ * re-plan confirmation, so an empty answer is never silent; with --task-id the
+ * confirm-replan hint names that task, ready to run.
+ * `task-graph.mjs analyze <agent-state.json>` prints the whole analysis as JSON.
+ * `task-graph.mjs drift <agent-state.json> <todo-id> [--files <path|->]
+ * [--not-done N] [--reason <text>]` prints markDrift's result as JSON, the
+ * updated todos included; it never writes the state file (plan-todos.sh does).
+ */
+function readTodos(file) {
+  return JSON.parse(readFileSync(file, "utf8"))?.plan?.todos ?? [];
+}
+
+function driftMain(file, id, rest) {
+  const opts = {};
+  for (let i = 0; i < rest.length; i++) {
+    const a = rest[i];
+    if (a === "--files") opts.files = rest[++i];
+    else if (a === "--not-done") opts.notDone = Number(rest[++i]);
+    else if (a === "--reason") opts.reason = rest[++i];
+    else throw Object.assign(new Error(`unknown drift option '${a}'`), { usage: true });
+  }
+  if (opts.notDone !== undefined && !(Number.isInteger(opts.notDone) && opts.notDone >= 0))
+    throw Object.assign(new Error("--not-done takes a whole number"), { usage: true });
+  const changedFiles = opts.files
+    ? readFileSync(opts.files === "-" ? 0 : opts.files, "utf8")
+        .split("\n")
+        .map((l) => l.trim())
+        .filter(Boolean)
+    : [];
+  return markDrift(readTodos(file), id, {
+    changedFiles,
+    notDone: opts.notDone,
+    reason: opts.reason,
+  });
+}
+
+function main(argv) {
+  const [cmd, file, ...rest] = argv;
+  if (!["next", "analyze", "drift"].includes(cmd) || !file || (cmd === "drift" && !rest[0])) {
+    process.stderr.write(
+      "usage: task-graph.mjs next <agent-state.json> [--task-id <id>] | analyze <agent-state.json> | drift <agent-state.json> <todo-id> [--files <path|->] [--not-done N] [--reason <text>]\n",
+    );
+    process.exitCode = 64;
+    return;
+  }
+  if (cmd === "drift") {
+    try {
+      process.stdout.write(`${JSON.stringify(driftMain(file, rest[0], rest.slice(1)))}\n`);
+    } catch (e) {
+      process.stderr.write(`task-graph: ${e.message}\n`);
+      process.exitCode = e.usage ? 64 : 1;
+    }
+    return;
+  }
+  let todos;
+  try {
+    todos = readTodos(file);
+  } catch (e) {
+    process.stderr.write(`task-graph: ${e.message}\n`);
+    process.exitCode = 3;
+    return;
+  }
+  const g = analyzeGraph(todos);
+  if (cmd === "analyze") {
+    process.stdout.write(`${JSON.stringify(g)}\n`);
+    return;
+  }
+  const taskId = rest[0] === "--task-id" && rest[1] ? rest[1] : "<task-id>";
+  const top = g.ready[0];
+  if (top) {
+    process.stdout.write(`${top.id}\t${top.task ?? ""}\n`);
+    return;
+  }
+  for (const b of g.blocked)
+    process.stderr.write(`plan-todos: ${b.id} is blocked (${b.reason}: ${b.on})\n`);
+  for (const w of g.waiting)
+    if (w.reason === "needs-replan")
+      process.stderr.write(
+        `plan-todos: ${w.id} waits for re-plan confirmation (${w.because} drifted): plan-todos.sh confirm-replan ${taskId} ${w.id}\n`,
+      );
+}
+
+if (invokedDirectly(import.meta.url)) main(process.argv.slice(2));