omp-conductor 0.2.2 → 0.3.1

package/src/store.ts

@@ -18,8 +18,12 @@ import type { RunRecord, RunState, Store } from "./types.ts";
 /**
  * States backed by a worker process. These are what worker capacity counts:
  * a slot is a process, and only a claimed or running attempt has one.
+ *
+ * Exported because it is also the answer to "is anything still writing to this
+ * run's transcript?" — `omp-conductor tail` needs that and must not re-derive
+ * it, or the two definitions drift the first time a state is added.
  */
-const LIVE_STATES: readonly RunState[] = ["claimed", "running"];
+export const LIVE_STATES: readonly RunState[] = ["claimed", "running"];
 
 /**
  * States that keep an *issue* occupied. `pushed-green` belongs here but not in
@@ -182,6 +186,16 @@ export function openStore(dbPath: string): Store {
   const countAttempts = db.query<{ n: number }, [string, number]>(
     `SELECT COUNT(*) AS n FROM runs WHERE project = ? AND issue = ?`,
   );
+  // Newest attempt for one issue. `startedAt` is millisecond-resolution and two
+  // attempts could in principle share one, so rowid breaks the tie by insertion
+  // order — a `tail` that attached to the older of two same-millisecond attempts
+  // would follow a transcript nobody is writing to any more.
+  const selectLatestRun = db.query<RunRow, [string, number]>(
+    `SELECT * FROM runs
+      WHERE project = ? AND issue = ?
+      ORDER BY startedAt DESC, rowid DESC
+      LIMIT 1`,
+  );
   const countStartedSince = db.query<{ n: number }, [string, number]>(
     `SELECT COUNT(*) AS n FROM runs WHERE project = ? AND startedAt >= ?`,
   );
@@ -255,6 +269,11 @@ export function openStore(dbPath: string): Store {
       return countAttempts.get(project, issue)?.n ?? 0;
     },
 
+    latestRun(project: string, issue: number): RunRecord | undefined {
+      const row = selectLatestRun.get(project, issue);
+      return row ? toRecord(row) : undefined;
+    },
+
     runsStartedSince(project: string, sinceEpochMs: number): number {
       return countStartedSince.get(project, sinceEpochMs)?.n ?? 0;
     },

package/src/tracker/github.ts

@@ -10,7 +10,7 @@
  * ~200-400ms) and failure classification by matching human-readable stderr
  * instead of reading a status code. Upgrade path when either bites: replace the
  * body of `gh()` with `fetch("https://api.github.com/...")` using a token from
- * `gh auth token`; the six Tracker methods above it stay untouched.
+ * `gh auth token`; the seven Tracker methods above it stay untouched.
  */
 
 import type { ProjectConfig, ReadyIssue, Tracker } from "../types.ts";
@@ -26,6 +26,47 @@ interface GhIssue {
   updatedAt: string;
 }
 
+/**
+ * Pull requests that would close an issue, with the one field that decides it.
+ *
+ * `gh issue view <n> --json closedByPullRequestsReferences` returns `id`,
+ * `number`, `repository` and `url` and no state, so a guard built on it holds an
+ * issue forever on a reference that is merged or closed-unmerged — which is
+ * exactly what a reopened issue looks like. GraphQL is the only spelling that
+ * yields `state`, so it is the spelling used.
+ *
+ * `includeClosedPrs:false` is deliberately not passed: it is not the filter it
+ * sounds like. Measured on gh 2.86.0 (2026-08-06), `veltro#260` returned a
+ * MERGED reference with the argument both true and false. The state is filtered
+ * in this consumer instead.
+ *
+ * ponytail: one page of ten. An issue closed by more than ten PRs is not a
+ * dispatch problem, and the first OPEN one already answers the question.
+ */
+const CLOSERS_QUERY = `query($owner:String!,$repo:String!,$n:Int!){
+  repository(owner:$owner,name:$repo){
+    issue(number:$n){
+      closedByPullRequestsReferences(first:10){
+        nodes{ number state isDraft url repository{ nameWithOwner } }
+      }
+    }
+  }
+}`;
+
+/** The `gh api graphql` envelope for {@link CLOSERS_QUERY}. Every level is
+ *  nullable: a deleted or wrong-numbered issue answers `null`, not an error. */
+interface ClosersResponse {
+  data?: {
+    repository?: {
+      issue?: {
+        closedByPullRequestsReferences?: {
+          nodes?: ({ state: string; isDraft: boolean; url: string } | null)[] | null;
+        } | null;
+      } | null;
+    } | null;
+  } | null;
+}
+
 /** Carries the captured stderr so callers can classify a failure without
  *  re-running the command or parsing the message text of a plain Error. */
 class GhError extends Error {
@@ -74,17 +115,45 @@ async function gh(argv: string[], stdin?: string): Promise<string> {
  * True when a label edit failed only because the requested end state already
  * holds. Adding a label is idempotent server-side, but removing one that is not
  * present is a 404, and a concurrent daemon restart can easily race into both.
+ *
+ * The quoted form is gh's own, and it is why the two operations are classified
+ * separately: `gh issue edit --remove-label x` on a label the *repository* does
+ * not define exits 1 with `'x' not found` (2.97.0), which for a removal is the
+ * end state already holding — a label nobody defined cannot be on an issue —
+ * while for an add it is a genuine failure. Removing a label the repo defines
+ * but the issue does not carry is already a silent success, so this path is
+ * reached only by the undefined-label case, most often a state label an
+ * operator declined to create at setup.
  */
 function isLabelNoop(err: unknown, op: "add" | "remove"): boolean {
   if (!(err instanceof GhError)) return false;
   const stderr = err.stderr;
   return op === "add"
     ? /already (?:has|had|exists|applied|added)|label .* already/i.test(stderr)
-    : /label does not exist|not labeled|does not have (?:that|the|this) label|label .* not found|not found on (?:this )?issue/i.test(
+    : /label does not exist|not labeled|does not have (?:that|the|this) label|label .* not found|not found on (?:this )?issue|'[^']+' not found/i.test(
         stderr,
       );
 }
 
+/**
+ * The URL of the first OPEN closer in a `gh api graphql` reply, if any.
+ *
+ * Split from the call so the state filter — the only real logic in this file —
+ * is pinned against recorded payloads instead of a live repo.
+ *
+ * A draft counts. `isDraft` is selected because the API offers it, not because
+ * it changes the answer: draft means "not ready to review", not "not pushed",
+ * and the branch behind a draft still holds the only copy of the work. Sending
+ * a second worker at it duplicates that work exactly as much as a ready PR
+ * would, so OPEN is the whole test.
+ */
+export function firstOpenCloser(raw: string): string | undefined {
+  const nodes =
+    (JSON.parse(raw) as ClosersResponse).data?.repository?.issue?.closedByPullRequestsReferences
+      ?.nodes ?? [];
+  return nodes.find((n) => n !== null && n.state === "OPEN")?.url;
+}
+
 export function makeTracker(p: ProjectConfig): Tracker {
   const repo = p.tracker.repo;
 
@@ -156,5 +225,28 @@ export function makeTracker(p: ProjectConfig): Tracker {
       // maintaining a second index of it.
       await gh(["issue", "edit", String(parent), "--repo", repo, "--add-sub-issue", String(child)]);
     },
+
+    async openCloserFor(issue: number): Promise<string | undefined> {
+      // GraphQL wants the halves of `owner/repo` separately. Config validates
+      // that spelling, so an empty half means a hand-edited config: `gh` then
+      // errors and the caller holds the candidate rather than guessing.
+      const [owner = "", name = ""] = repo.split("/");
+      const raw = await gh([
+        "api",
+        "graphql",
+        "-f",
+        `query=${CLOSERS_QUERY}`,
+        "-F",
+        `owner=${owner}`,
+        "-F",
+        `repo=${name}`,
+        // -F, not -f: the query declares $n as Int! and a string would be a
+        // type error rather than a coerced number.
+        "-F",
+        `n=${issue}`,
+      ]);
+
+      return firstOpenCloser(raw);
+    },
   };
 }

package/src/types.ts

@@ -13,7 +13,7 @@
  * counts turns, wall clock and dollars itself and kills anything over the line.
  */
 export interface Caps {
-  /** Parallel omp sessions. Two, because the homelab only has 3 CI runners and
+  /** Parallel omp sessions. Two by default: on a small self-hosted runner pool
    *  a third worker would starve its own PR checks. */
   maxConcurrentWorkers: number;
   /** Rolling-day spend ceiling; the loop stops claiming work once it is hit. */
@@ -65,6 +65,33 @@ export type ReportScope = (typeof REPORT_SCOPES)[number];
  */
 export const DEFAULT_REPORT_SCOPE: ReportScope = "material";
 
+/**
+ * Who holds an authority the daemon itself never exercises. Declared as data
+ * for the same reason as {@link REPORT_SCOPES}: the validator, the wizard and
+ * the brief renderer all enumerate the same two holders, so a third one cannot
+ * be added while any of them still knows only two.
+ */
+export const AUTHORITY_HOLDERS = ["human", "orchestrator"] as const;
+
+export type AuthorityHolder = (typeof AUTHORITY_HOLDERS)[number];
+
+/**
+ * What a project that never answered the question gets: humans keep both. A
+ * default that delegated merging would hand a fresh fleet write access to its
+ * own main branch on the strength of an unread config file.
+ */
+export const DEFAULT_AUTHORITY: ProjectConfig["authority"] = { merge: "human", release: "human" };
+
+/**
+ * Where the session that triages escalations lives. `embedded` is the daemon's
+ * own child session; `external` means an operator already runs one — a visible
+ * TUI session, say — and the daemon must not start a second brain that would
+ * re-triage the same issues from a different transcript.
+ */
+export const ORCHESTRATOR_MODES = ["embedded", "external"] as const;
+
+export type OrchestratorMode = (typeof ORCHESTRATOR_MODES)[number];
+
 /**
  * Everything the dispatcher needs to service one product: where work comes
  * from, where code goes, and what it may spend doing it. Config is per project
@@ -91,8 +118,18 @@ export interface ProjectConfig {
    * answered the question wants.
    */
   workerModel?: string;
-  /** How a stuck run reaches a human, and what to do when it cannot. */
-  escalation: { telegramChatId?: string; fallbackToIssueComment: boolean };
+  /**
+   * How a stuck run reaches a human, what to do when it cannot, and who runs
+   * the session that triages it. See {@link ORCHESTRATOR_MODES}.
+   */
+  escalation: { telegramChatId?: string; fallbackToIssueComment: boolean; orchestrator: OrchestratorMode };
+  /**
+   * Who lands green PRs and who cuts releases. The daemon never acts on this
+   * itself — it words the orchestrator's standing orders and the rendered brief
+   * scaffold with it, so config and prompt can never disagree about which of
+   * them is holding the merge button.
+   */
+  authority: { merge: AuthorityHolder; release: AuthorityHolder };
   /**
    * How loud the orchestrator is. Optional on disk — a config written before
    * this key existed loads as {@link DEFAULT_REPORT_SCOPE} — so read it through
@@ -156,6 +193,18 @@ export interface Tracker {
   /** Records that a worker split its issue, so follow-up work stays traceable
    *  to the request that spawned it. */
   linkParent(child: number, parent: number): Promise<void>;
+  /**
+   * The URL of an OPEN pull request that already closes `issue`, or undefined
+   * when none does.
+   *
+   * Admission has to ask the tracker because the store cannot answer. The busy
+   * set is built from run rows, so it only knows work *this* database recorded:
+   * a migration onto the daemon, a wiped or relocated state directory, a
+   * restore onto a new host, or simply a database younger than the PRs all
+   * present pushed-and-open work as an untouched queue item. The tracker is the
+   * only party that remembers across all of those.
+   */
+  openCloserFor(issue: number): Promise<string | undefined>;
 }
 
 /**
@@ -214,6 +263,10 @@ export interface Store {
   /** Runs backed by a worker process — what capacity counts. Subset of {@link Store.activeRuns}. */
   liveRuns(project: string): RunRecord[];
   attemptsFor(project: string, issue: number): number;
+  /** Newest attempt for one issue, whatever state it reached. `omp-conductor
+   *  tail` resolves an issue number to a transcript through this; the number is
+   *  what an operator has, the run id is not. */
+  latestRun(project: string, issue: number): RunRecord | undefined;
   runsStartedSince(project: string, sinceEpochMs: number): number;
   spendSince(project: string, sinceEpochMs: number): number;
   /** Idempotence guard so a retry loop cannot page a human repeatedly for the

package/src/unblock.ts

@@ -0,0 +1,113 @@
+/**
+ * Giving an answered block a way back into the queue.
+ *
+ * A blocked or failed run leaves its state label on the issue, and eligibility
+ * treats any state label as disqualifying (`routing.isEligible`). That is the
+ * right interlock — it is what stops a second worker landing on live work
+ * across a daemon restart — but until this verb existed it had no exit: the
+ * orchestrator's brief tells it never to hand-edit a state label, so an issue
+ * whose question it had just answered could only be left labelled, which means
+ * never re-claimed and the answer inert. Nothing surfaces that either: the
+ * issue does not fail, it simply stops existing as far as dispatch is
+ * concerned.
+ *
+ * So the clearing is a daemon-owned act, through the same tracker port the
+ * dispatcher writes labels with, and the brief's rule stays absolute. That
+ * absoluteness is worth more than the exception it replaces: orphan detection
+ * is only trustworthy while every state label on the tracker was written by
+ * this package.
+ *
+ * Nothing here writes to the store, and that is a decision rather than an
+ * omission. `RunState` describes what a worker process did; an answer is the
+ * one event that happens outside every run, so no member fits it — folding it
+ * into `merged` or `killed` would make `status` describe a run that never
+ * reached either. Eligibility is read off the tracker's labels and never off a
+ * run row, so the store has nothing to say here. Leaving the history alone is
+ * also what keeps `maxAttemptsPerIssue` honest: an answered block still spent a
+ * worker's whole budget, and the same question answered twice is a loop the cap
+ * exists to stop.
+ */
+
+import { LIVE_STATES } from "./store.ts";
+import type { Caps, ProjectConfig, RunRecord, Store, Tracker } from "./types.ts";
+
+/** What one `unblock` did, and the state it found around it. */
+export interface UnblockOutcome {
+  /** State labels the tracker was asked to drop. */
+  cleared: string[];
+  /** Attempts this issue has already spent. Unchanged by the unblock. */
+  attemptsUsed: number;
+  /** Newest attempt, when the store has one for this issue at all. */
+  latest?: RunRecord;
+}
+
+/**
+ * Drop both terminal state labels, whichever the issue is actually carrying.
+ *
+ * Both unconditionally, because the tracker is the only source of truth for
+ * which one is set and this process cannot read that back through the Tracker
+ * port — inferring it from the newest run row would be a guess that goes wrong
+ * exactly when a human has relabelled something by hand. Removing a label an
+ * issue does not carry is a no-op: `gh issue edit --remove-label` exits 0 on an
+ * absent label (verified against gh 2.97.0), and the adapter swallows the 404
+ * older paths return for one.
+ *
+ * `agent:in-progress` is deliberately not in the set. It means a worker process
+ * exists, which is not something an operator can answer away, and clearing it
+ * from under a live run is how two workers end up on one issue.
+ */
+export async function unblockIssue(
+  project: ProjectConfig,
+  tracker: Tracker,
+  store: Store,
+  issue: number,
+): Promise<UnblockOutcome> {
+  const cleared: string[] = [];
+  for (const label of new Set([project.stateLabels.blocked, project.stateLabels.failed])) {
+    await tracker.removeLabel(issue, label);
+    cleared.push(label);
+  }
+
+  const latest = store.latestRun(project.name, issue);
+  return {
+    cleared,
+    attemptsUsed: store.attemptsFor(project.name, issue),
+    ...(latest === undefined ? {} : { latest }),
+  };
+}
+
+/**
+ * What the operator reads back. It promises a re-claim only when one can
+ * actually happen: a live run still owns the issue through `agent:in-progress`,
+ * and a spent attempt budget makes the next tick escalate rather than dispatch.
+ * Either promised blindly would send someone away believing work had resumed.
+ */
+export function formatUnblock(issue: number, o: UnblockOutcome, project: ProjectConfig, caps: Caps): string {
+  const latest = o.latest;
+  const lines = [`#${issue}: cleared ${o.cleared.join(", ")}`];
+
+  if (latest === undefined) {
+    lines.push("  attempts   none recorded — the labels were cleared anyway; eligibility is read off the tracker");
+  } else {
+    lines.push(
+      `  attempts   ${o.attemptsUsed} of ${caps.maxAttemptsPerIssue} used, newest ${latest.state} — ` +
+        "unchanged, an answered block still spent a worker",
+    );
+  }
+
+  if (latest !== undefined && LIVE_STATES.includes(latest.state)) {
+    lines.push(
+      `  in flight  attempt ${latest.attempt} is ${latest.state}, so the issue keeps ` +
+        `"${project.stateLabels.inProgress}" until it ends — nothing is re-claimed before then`,
+    );
+  } else if (o.attemptsUsed >= caps.maxAttemptsPerIssue) {
+    lines.push(
+      `  next tick  not eligible: all ${caps.maxAttemptsPerIssue} attempts are spent, so the next tick escalates ` +
+        "instead of re-claiming. Raise maxAttemptsPerIssue with /conductor setup, or rewrite the issue.",
+    );
+  } else {
+    lines.push(`  next tick  eligible again, as long as the issue still carries "${project.queueLabel}"`);
+  }
+
+  return lines.join("\n");
+}

package/src/worker.ts

@@ -43,6 +43,15 @@ export interface WorkerOpts {
    */
   model?: string;
   onTurn?: (n: number) => void;
+  /**
+   * The transcript path, handed over the moment the session opens it rather
+   * than at the end with {@link WorkerResult.sessionFile}. Both report the same
+   * path; only this one arrives while there is still something to watch, which
+   * is what `omp-conductor tail` attaches to. Never called for a session that
+   * opened no transcript — there is no path to report, and an empty string
+   * would be a path that fails to open rather than an absence.
+   */
+  onSessionFile?: (path: string) => void;
 }
 
 /**
@@ -145,6 +154,11 @@ export async function runWorker(
     ...(o.model === undefined ? {} : { model: o.model }),
   });
 
+  // Before the first turn, not after the last: a caller that only learns the
+  // transcript path from the result learns it once the run it wanted to watch
+  // is already over.
+  if (session.sessionFile !== undefined) o.onSessionFile?.(session.sessionFile);
+
   // Every exit below reports the session's own facts the same way: the
   // transcript it actually opened, and any model downgrade it announced. Read at
   // return time so a session that materialises either late is still reported

package/src/worktree.ts

@@ -268,6 +268,125 @@ export async function addWorktree(
   return worktreePath;
 }
 
+/**
+ * What one salvage attempt did. A failure is a *value* rather than a throw
+ * because the only caller is a run that is already ending badly: it has to log
+ * the outcome, word it into the escalation and still close its run record, and
+ * none of that may be skipped by an exception from the last-ditch step.
+ */
+export type SalvageOutcome =
+  | { kind: "salvaged"; sha: string; branch: string; pushed: boolean; pushError?: string }
+  /** Nothing uncommitted was there to save — a clean tree, or no tree at all. */
+  | { kind: "nothing" }
+  /** There was work and git would not commit it. This is the loud one. */
+  | { kind: "failed"; error: string };
+
+/**
+ * The salvage commit is the daemon's own, made unattended in a worktree whose
+ * config it does not control. A machine with no global git identity, a global
+ * `commit.gpgsign=true` whose key needs a passphrase nobody can type, or a repo
+ * pre-commit hook that rejects half-finished code are all ordinary states — and
+ * every one of them would turn "save the work" into "lose the work". So the
+ * commit brings its own identity, signs nothing, and skips hooks: it is a
+ * snapshot for a human to sort out, never something anyone merges.
+ */
+const SALVAGE_COMMIT_CONFIG = [
+  "-c",
+  "user.name=conductor",
+  "-c",
+  "user.email=conductor@invalid",
+  "-c",
+  "commit.gpgsign=false",
+];
+
+/**
+ * Commits a dead run's uncommitted work to the run's own branch and pushes it,
+ * so that the tree the next attempt destroys is no longer the only copy.
+ *
+ * This closes a deliberate asymmetry. `addWorktree` preserves the run branch
+ * precisely because "that branch can hold the only copy of work attempt 1
+ * committed but never pushed", while `removeWorktree` runs `worktree remove
+ * --force` and `addWorktree` refuses to reuse a tree that "may hold a previous
+ * attempt's uncommitted work" — committed work is kept by design, uncommitted
+ * work is discarded by design. That trade is fair for a worker that *stops*:
+ * blocking is a decision it makes with turns left to commit first. It is not
+ * fair for one killed by the turns cap or the wall clock, or one that crashes:
+ * that end is external, unannounced, mid-sentence, and it lands hardest on the
+ * long refactors carrying the most unsaved work. So: non-graceful ends only.
+ *
+ * Never throws. Every outcome, including its own failure, comes back as a value
+ * for the caller to log and to put in front of a human.
+ *
+ * The push is best-effort and deliberately last, after the sha exists: a
+ * refused push (diverged branch, no credentials, no network) still leaves the
+ * commit in this host's mirror, which is strictly better than nothing. It is a
+ * plain fast-forward push — never a force — and if the run already had a PR
+ * open, that PR gains the WIP commit and re-runs its checks. That is the price
+ * of work outliving its host, and only a run that already failed ever pays it.
+ */
+export async function salvageWip(
+  worktree: string,
+  issue: number,
+  attempt: number,
+  reason: string,
+): Promise<SalvageOutcome> {
+  try {
+    // A tree that is not there cannot be holding work. Checked before spawning
+    // git, because a missing cwd fails at spawn time rather than as an exit
+    // code, and "no tree" is not a salvage failure worth alarming anyone with.
+    if (!existsSync(worktree)) return { kind: "nothing" };
+
+    if ((await git(["status", "--porcelain"], worktree)) === "") return { kind: "nothing" };
+
+    // The tree's own branch, not one the caller believes it should be on: this
+    // string ends up in an escalation as the place to go looking.
+    const branch = await git(["rev-parse", "--abbrev-ref", "HEAD"], worktree);
+
+    // `-A` on purpose: the losses this exists for were mostly *new* files.
+    await git(["add", "-A"], worktree);
+    await git(
+      [
+        ...SALVAGE_COMMIT_CONFIG,
+        "commit",
+        "--no-verify",
+        "-m",
+        `wip(#${issue}): attempt ${attempt} killed by ${reason} — auto-salvaged`,
+      ],
+      worktree,
+    );
+    const sha = await git(["rev-parse", "HEAD"], worktree);
+
+    if (branch === "HEAD") {
+      // Detached: the commit is real but reachable only by sha, and pushing
+      // `HEAD` from here would publish a branch literally named HEAD.
+      return {
+        kind: "salvaged",
+        sha,
+        branch,
+        pushed: false,
+        pushError: "detached HEAD — no branch to push",
+      };
+    }
+
+    const push = await runGit(["push", "origin", `HEAD:refs/heads/${branch}`], worktree);
+    if (push.code === 0) return { kind: "salvaged", sha, branch, pushed: true };
+
+    return {
+      kind: "salvaged",
+      sha,
+      branch,
+      pushed: false,
+      pushError: (
+        push.stderr.trim() ||
+        push.stdout.trim() ||
+        `git push exited ${push.code}`
+      ).replace(URL_USERINFO, "$1***@"),
+    };
+  } catch (err) {
+    return { kind: "failed", error: err instanceof Error ? err.message : String(err) };
+  }
+}
+
 /**
  * Removes a run's worktree and its registration in the mirror. Idempotent: a
  * path that is already gone resolves, so cleanup can be retried and can run on