omp-conductor 0.5.6 → 0.7.0

package/src/store.ts

@@ -61,7 +61,7 @@ export const LIVE_STATES: readonly RunState[] = ["claimed", "running"];
  * worktrees removed, so they must not consume slots, while their live PRs must
  * still block duplicate attempts.
  */
-const ACTIVE_STATES: readonly RunState[] = [...LIVE_STATES, "pushed-pending", "pushed-green"];
+export const ACTIVE_STATES: readonly RunState[] = [...LIVE_STATES, "pushed-pending", "pushed-green"];
 
 const LIVE_PLACEHOLDERS = LIVE_STATES.map(() => "?").join(", ");
 const ACTIVE_PLACEHOLDERS = ACTIVE_STATES.map(() => "?").join(", ");
@@ -773,12 +773,19 @@ export function openStore(dbPath: string): Store {
   const countFailures = db.query<{ n: number }, [string, number]>(
     `SELECT COUNT(*) AS n FROM runs
       WHERE project = ? AND issue = ? AND state = 'failed'
-        AND (failureClass IS NULL OR failureClass NOT IN ('ci-infra', 'settlement-stuck', 'env-start-failure'))`,
+        AND (failureClass IS NULL OR failureClass NOT IN ('ci-infra', 'settlement-stuck', 'env-start-failure', 'dispatch-infra'))`,
   );
   const countContinuations = db.query<{ n: number }, [string, number]>(
     `SELECT COUNT(*) AS n FROM runs
       WHERE project = ? AND issue = ? AND state IN ('killed', 'orphaned', 'blocked')
-        AND (failureClass IS NULL OR failureClass NOT IN ('admin-kill', 'settlement-stuck', 'env-start-failure'))`,
+        AND (failureClass IS NULL OR failureClass NOT IN ('admin-kill', 'settlement-stuck', 'env-start-failure', 'dispatch-infra'))`,
+  );
+  // How many times one issue reached a given class. Recovery uses it to bound a
+  // retry loop whose cause is persistent (e.g. a mirror that will not refresh):
+  // after `DISPATCH_INFRA_MAX_STRIKES` the daemon escalates instead of looping.
+  const countByClass = db.query<{ n: number }, [string, number, FailureClass]>(
+    `SELECT COUNT(*) AS n FROM runs
+      WHERE project = ? AND issue = ? AND failureClass = ?`,
   );
   // Newest first, and bounded: every row this returns costs `gh` calls to gather
   // facts for, so a fleet with a long unclassified history classifies over
@@ -1142,6 +1149,10 @@ export function openStore(dbPath: string): Store {
       return countContinuations.get(project, issue)?.n ?? 0;
     },
 
+    classCountFor(project: string, issue: number, cls: FailureClass): number {
+      return countByClass.get(project, issue, cls)?.n ?? 0;
+    },
+
     latestRun(project: string, issue: number): RunRecord | undefined {
       const row = selectLatestRun.get(project, issue);
       return row ? toRecord(row) : undefined;

package/src/tracker/github.ts

@@ -695,6 +695,22 @@ export function makeTracker(p: ProjectConfig, runGh: typeof gh = gh): Tracker {
       }
     },
 
+    async checkLog(detailsUrl: string): Promise<string | undefined> {
+      const args = runLogArgs(detailsUrl);
+      if (args === undefined) return undefined;
+      try {
+        const raw = await runGh(args);
+        const lines = raw.replaceAll(/\u001b\[[0-9;]*m/g, "").split("\n");
+        return lines.slice(-400).join("\n");
+      } catch {
+        // Best-effort, never throws: a revoked token, a deleted run or a flaky
+        // network must not error the classifier out of gathering facts. The
+        // caller treats `undefined` as "could not read the log" and classifies
+        // conservatively (#177).
+        return undefined;
+      }
+    },
+
     async mergeable(prUrl: string): Promise<"conflicting" | "clean" | "unknown"> {
       if (!PR_URL.test(prUrl)) return "unknown";
       try {

package/src/types.ts

@@ -766,6 +766,12 @@ export interface Tracker {
    * port: nothing downstream may read it as "no checks failed".
    */
   checkConclusions(prUrl: string): Promise<{ name: string; state: string; link?: string }[]>;
+  /** The tail of a failed workflow run's log, ANSI stripped, or `undefined`
+   *  when it cannot be fetched. Best-effort, never throws: the classifier uses
+   *  it to tell an infrastructure outage from a deterministic test failure
+   *  (#177), and "could not read the log" must fall through to the existing
+   *  conservative classification rather than error out of fact-gathering. */
+  checkLog(detailsUrl: string): Promise<string | undefined>;
   /** Whether the PR can merge into its base. `unknown` on any doubt, so a
    *  mergeability nobody could read never becomes a conflict recovery. */
   mergeable(prUrl: string): Promise<"conflicting" | "clean" | "unknown">;
@@ -803,6 +809,7 @@ export const FAILURE_CLASSES = [
   "admin-kill",
   "ci-infra",
   "ci-deterministic",
+  "dispatch-infra",
   "merge-conflict",
   "question",
   "orphan-clean",
@@ -1155,6 +1162,10 @@ export interface Store {
   failuresFor(project: string, issue: number): number;
   /** Operational stops that require a bounded continuation resume. */
   continuationsFor(project: string, issue: number): number;
+  /** How many times one issue has reached a given failure class. A recovery
+   *  that keeps landing on the same class (e.g. a persistently broken mirror)
+   *  uses this to escalate instead of retrying forever. */
+  classCountFor(project: string, issue: number, cls: FailureClass): number;
   /** Newest attempt for one issue, whatever state it reached. `omp-conductor
    *  tail` resolves an issue number to a transcript through this; the number is
    *  what an operator has, the run id is not. */
@@ -1321,6 +1332,7 @@ export const VERB_NAMES = [
   "conductor_push",
   "conductor_pr_create",
   "conductor_pr_update_branch",
+  "conductor_pr_update",
   "conductor_pr_merge",
   "conductor_label",
   "conductor_release",

package/src/unblock.ts

@@ -40,6 +40,10 @@ export interface UnblockOutcome {
   continuationsUsed: number;
   /** Newest attempt, when the store has one for this issue at all. */
   latest?: RunRecord;
+  /** Whether any run for the issue is still in an active state — the answer
+   *  to "will the dispatcher hold this issue as issue-active?" that #178
+   *  found this verb guessing at. */
+  active: boolean;
   /** Set when nothing was cleared because the newest attempt's work exists
    *  only in its worktree. Carries the salvage failure verbatim. */
   refused?: string;
@@ -92,10 +96,12 @@ export async function unblockIssue(
   // for clearing in-progress, so the row that carries it decides the set.
   const latest = store.latestRun(project.name, issue);
   const terminal = latest !== undefined && !LIVE_STATES.includes(latest.state);
+  const active = store.activeRuns(project.name).some((r) => r.issue === issue);
   const counts = {
     attemptsUsed: store.attemptsFor(project.name, issue),
     failuresUsed: store.failuresFor(project.name, issue),
     continuationsUsed: store.continuationsFor(project.name, issue),
+    active,
   };
 
   // The one case where this verb refuses. Clearing the labels here re-queues an
@@ -205,8 +211,31 @@ export function formatUnblock(
         `run row to prove the worker is gone, "${project.stateLabels.inProgress}" was left in place, and ` +
         "on its own it keeps the issue ineligible",
     );
+  } else if (latest.state === "pushed-green" || latest.state === "pushed-pending") {
+    // #178: an unblocked issue whose newest run pushed a green PR is not "free
+    // for a fresh attempt" — the dispatcher continues the pushed run's branch,
+    // and only once the PR resolves does the issue settle (#175). Say that
+    // instead of promising a re-claim the next tick withholds.
+    lines.push(
+      latest.prUrl === undefined
+        ? `  next tick  eligible again, as long as the issue still carries "${project.queueLabel}" ` +
+            `(newest run is ${latest.state} with no recorded PR; the dispatcher still applies its open-PR check at claim time)`
+        : `  next tick  eligible as a continuation of ${latest.prUrl} — the pushed run stays active until ` +
+            "that PR resolves; dispatch continues on its branch",
+    );
+  } else if (o.active) {
+    // A run other than the newest is still active (e.g. a live sibling under a
+    // terminal newest row), so the dispatcher will hold the issue as
+    // issue-active whatever the labels now say. #178's misleading case.
+    lines.push(
+      `  in flight  a run for this issue is still active, so the dispatcher holds the issue until it settles — ` +
+        `nothing is re-claimed before then ("${project.stateLabels.inProgress}" stays unless already released)`,
+    );
   } else {
-    lines.push(`  next tick  eligible again, as long as the issue still carries "${project.queueLabel}"`);
+    lines.push(
+      `  next tick  eligible again, as long as the issue still carries "${project.queueLabel}" ` +
+        "(the dispatcher still applies its open-PR check at claim time)",
+    );
   }
 
   return lines.join("\n");

package/src/verbs/actions.ts

@@ -97,6 +97,14 @@ export function githubVerbActions(project: ProjectConfig, run: CommandRunner = s
 
     updatePrBranch: (prUrl) => gh(["pr", "update-branch", prUrl]),
 
+    updatePr: async (prUrl, fields) => {
+      const argv = ["pr", "edit", prUrl];
+      if (fields.title !== undefined) argv.push("--title", fields.title);
+      if (fields.body !== undefined) argv.push("--body", fields.body);
+      const outcome = await gh(argv);
+      return outcome.ok ? { ok: true } : { ok: false, stderr: outcome.stderr };
+    },
+
     // `--match-head-commit` is the server-side half of the exact-head rule: the
     // daemon already re-read the head and refused a stale one, and this makes
     // GitHub refuse too if the branch moved in the milliseconds between. Belt

package/src/verbs/protocol.ts

@@ -145,6 +145,35 @@ export const VERB_SPECS: Readonly<Record<VerbName, VerbSpec>> = {
     },
     roleRefusalText: (role) => `conductor_pr_update_branch is not open to a ${role} session.`,
   },
+  conductor_pr_update: {
+    name: "conductor_pr_update",
+    mutating: true,
+    allowedRoles: ["worker", "orchestrator"],
+    description:
+      "Update this run's pull request title and/or body. Never touches base, head or state — " +
+      "a field you may not change is a field this verb refuses rather than renames. A worker " +
+      "may only name its own run's PR.",
+    args: {
+      prUrl: {
+        type: "string",
+        required: false,
+        description:
+          "Full pull request URL. Required on an orchestrator channel; a run channel infers its own PR.",
+      },
+      title: {
+        type: "string",
+        required: false,
+        description: "New pull request title. At least one of title/body must be a non-empty string.",
+      },
+      body: {
+        type: "string",
+        required: false,
+        description: "New pull request body. At least one of title/body must be a non-empty string.",
+      },
+      rationale: RATIONALE_ARG,
+    },
+    roleRefusalText: (role) => `conductor_pr_update is not open to a ${role} session.`,
+  },
   conductor_pr_merge: {
     name: "conductor_pr_merge",
     mutating: true,

package/src/verbs/server.ts

@@ -145,6 +145,10 @@ export interface VerbActions {
     opts: { title: string; body: string; base: string },
   ): Promise<{ ok: true; url: string } | { ok: false; stderr: string }>;
   updatePrBranch(prUrl: string): Promise<ActionOutcome>;
+  updatePr(
+    prUrl: string,
+    fields: { title?: string; body?: string },
+  ): Promise<{ ok: true } | { ok: false; stderr: string }>;
   mergePr(prUrl: string, headSha: string): Promise<ActionOutcome>;
   setLabel(issue: number, label: string, action: "add" | "remove"): Promise<ActionOutcome>;
   release(execution: ReleaseExecution): Promise<ActionOutcome>;
@@ -167,6 +171,14 @@ export interface VerbDeps {
    * sentinel alongside disarming ticks, so one read covers both.
    */
   fleetStop: () => string | undefined;
+  /**
+   * The epoch-ms timestamp the current pause began, or `undefined` when not
+   * paused (or when the sentinel does not parse). Lets the fleet-stop gate tell
+   * a mutating call from a run admitted *before* the pause — that run may
+   * finish — apart from one admitted after, which must be refused. Parallels
+   * {@link VerbDeps.fleetStop} by living behind the same per-call read.
+   */
+  pausedAt: () => number | undefined;
   log: (message: string) => void;
   now: () => number;
 }
@@ -387,17 +399,11 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
     );
   }
 
-  // 3. Stop always wins, and is re-read here rather than remembered. Reads are
-  //    exempt: a run finishing its report during a `hold` still has to be able
-  //    to say what it saw, and refusing that would only make it guess.
-  const stopped = spec.mutating ? deps.fleetStop() : undefined;
-  if (stopped !== undefined) {
-    return refuse("fleet-paused", `refused: ${stopped}. Mutating verbs are refused while the fleet is stopped.`);
-  }
-
-  // 4. The run's own row, re-read. A worker whose run was killed, orphaned or
+  // 3. The run's own row, re-read. A worker whose run was killed, orphaned or
   //    settled between its last turn and this call is no longer holding
-  //    anything, and a push from it would land work nothing is tracking.
+  //    anything, and a push from it would land work nothing is tracking. Read
+  //    *before* the stop gate below, because the stop gate needs this row to
+  //    tell an in-flight run (which may finish) from a post-pause call.
   let run: RunRecord | undefined;
   if (channel.kind === "run") {
     run = deps.store.getRun(channel.runId);
@@ -409,6 +415,22 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
     }
   }
 
+  // 4. Stop always wins, and is re-read here rather than remembered. Reads are
+  //    exempt: a run finishing its report during a `hold` still has to be able
+  //    to say what it saw, and refusing that would only make it guess. A run
+  //    admitted *before* the pause may finish its in-flight mutations — pause
+  //    stops new claims, not the work already running (#174) — so only a call
+  //    whose run started at or after the pause (or an orchestrator's call, with
+  //    no run row at all) is refused.
+  const stopped = spec.mutating ? deps.fleetStop() : undefined;
+  if (stopped !== undefined) {
+    const at = deps.pausedAt();
+    const admittedBeforePause = run !== undefined && at !== undefined && run.startedAt < at;
+    if (!admittedBeforePause) {
+      return refuse("fleet-paused", `refused: ${stopped}. The pause stops new claims and new mutations; a run admitted before the pause may finish.`);
+    }
+  }
+
   switch (verb) {
     case "conductor_push":
       return pushVerb(deps, channel, args, refuse, allow);
@@ -416,6 +438,8 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
       return prCreateVerb(deps, project, channel, args, refuse, allow);
     case "conductor_pr_update_branch":
       return prUpdateBranchVerb(deps, project, channel, args, refuse, allow);
+    case "conductor_pr_update":
+      return prUpdateVerb(deps, project, channel, args, refuse, allow);
     case "conductor_pr_merge":
       return prMergeVerb(deps, project, channel, args, refuse, allow);
     case "conductor_label":
@@ -530,6 +554,18 @@ async function prCreateVerb(
     );
   }
   if (closer !== undefined) {
+    // Adoption (#176): a continuation run is never bound to its issue's existing
+    // PR — the old row owns the prUrl, and this new row's prUrl stayed NULL — so
+    // when the open PR was opened from *this* run's branch, it is the same work.
+    // Bind it to this run instead of refusing: pr_status then answers from the
+    // live PR, and a merge can act on it. Branch mismatch (or an empty headRefName,
+    // which is never identity) keeps the recorded refusal.
+    if (closer.headRefName !== "" && closer.headRefName === run.branch) {
+      deps.store.updateRun(run.runId, { prUrl: closer.url });
+      return allow(
+        `adopted ${closer.url} — already open for ${run.branch} and closes #${run.issue}; title/body arguments were not applied`,
+      );
+    }
     return refuse("open-pr-exists", `refused: ${closer.url} is already open and already closes #${run.issue}.`);
   }
 
@@ -925,6 +961,73 @@ async function prStatusVerb(
   return allow(`${prUrl} at ${headSha}: ${verification.status} — ${verification.reason}`, undefined, issue);
 }
 
+/**
+ * Update one pull request's title and/or body — nothing else.
+ *
+ * Same resolution as {@link prStatusVerb}: a worker edits its own run's PR (and
+ * only that one), an orchestrator names one this project's runs opened.
+ * `conductor_pr_update` exists so the floor's own edits go through the same
+ * checks and the same ledger as every other mutation, instead of a raw
+ * `gh pr edit` from a session that already disclosed it would do exactly that
+ * (#176).
+ */
+async function prUpdateVerb(
+  deps: VerbDeps,
+  project: ProjectConfig,
+  channel: VerbChannel,
+  args: Record<string, unknown>,
+  refuse: Refuse,
+  allow: Allow,
+): Promise<Verdict> {
+  const asked = args["prUrl"];
+  let prUrl: string;
+  let issue: number | undefined;
+
+  if (channel.kind === "run") {
+    const own = deps.store.getRun(channel.runId);
+    if (own?.prUrl === undefined) {
+      return refuse("pr-missing", "refused: this run has no pull request yet. Open one with conductor_pr_create.");
+    }
+    if (typeof asked === "string" && asked !== own.prUrl) {
+      return refuse(
+        "pr-not-this-run",
+        `refused: this socket belongs to the run for #${channel.issue}, whose pull request is ${own.prUrl}. ` +
+          "A worker edits only its own run's PR.",
+      );
+    }
+    prUrl = own.prUrl;
+    issue = channel.issue;
+  } else {
+    if (typeof asked !== "string") {
+      return refuse("malformed-argument", "refused: conductor_pr_update needs prUrl when it has no run to infer one from.");
+    }
+    const target = runForPr(deps, project.name, asked);
+    if (target === undefined) {
+      return refuse("pr-not-this-run", `refused: ${asked} is not a pull request any run in ${project.name} opened.`);
+    }
+    prUrl = asked;
+    issue = target.issue;
+  }
+
+  const title = args["title"];
+  const body = args["body"];
+  if (!(typeof title === "string" && title !== "") && !(typeof body === "string" && body !== "")) {
+    return refuse(
+      "malformed-argument",
+      "refused: conductor_pr_update changes nothing unless at least one of title or body is a non-empty string.",
+    );
+  }
+
+  const outcome = await deps.actions.updatePr(prUrl, {
+    ...(typeof title === "string" && title !== "" ? { title } : {}),
+    ...(typeof body === "string" && body !== "" ? { body } : {}),
+  });
+  if (!outcome.ok) {
+    return refuse("action-failed", `refused: gh could not update the pull request:\n${outcome.stderr}`, issue);
+  }
+  return allow(`updated ${prUrl}.`, undefined, issue);
+}
+
 // ------------------------------------------------------------------ the listener
 
 export interface VerbListener {

package/src/worktree.ts

@@ -108,6 +108,29 @@ async function gitSucceeds(args: string[], cwd?: string): Promise<boolean> {
   return code === 0;
 }
 
+/** Fences the block below so it can be found, replaced, and never duplicated. */
+const REF_LOCK_SIGNATURES = /cannot lock ref|unable to update local ref/i;
+
+/**
+ * Refreshes a bare mirror's refs from its remote. Two dispatch loops can race
+ * on git's *packed-refs* / ref locks for the same repo and one lands a transient
+ * `cannot lock ref` / `unable to update local ref` — that is not corruption, but
+ * an {@link ensureMirror} that throws on it aborts the whole dispatch at turn 0
+ * and chars a failed attempt (#168). Retry once after a short backoff; a second
+ * failure (or any non-ref-lock failure) propagates unchanged, so a genuinely
+ * broken mirror still surfaces loudly.
+ */
+async function refreshMirror(mirrorPath: string): Promise<string> {
+  try {
+    return await git(["remote", "update", "--prune"], mirrorPath);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    if (!REF_LOCK_SIGNATURES.test(message)) throw err;
+    await new Promise<void>((r) => setTimeout(r, 500));
+    return git(["remote", "update", "--prune"], mirrorPath);
+  }
+}
+
 /** Fences the block below so it can be found, replaced, and never duplicated. */
 const EXCLUDE_BEGIN = "# >>> omp-conductor (managed; edit outside this block)";
 const EXCLUDE_END = "# <<< omp-conductor";
@@ -262,7 +285,7 @@ export async function ensureMirror(
 
   if (existsSync(mirrorPath)) {
     await configureMirror(mirrorPath);
-    await git(["remote", "update", "--prune"], mirrorPath);
+    await refreshMirror(mirrorPath);
     return mirrorPath;
   }