omp-conductor 0.4.3 → 0.4.4

package/README.md

@@ -1382,7 +1382,7 @@ persists the class on the row, and performs the one recovery that class names.
 | `orphan-dirty` | orphaned with a failed salvage and no operator ack | hold — recorded only; the tree is the only copy | none |
 | `orphan-clean` | orphaned with nothing uncommitted | requeue | continuation |
 | `turn-cap-progress` | at the turn ceiling **with** a PR, head or salvage commit | continue from the branch | continuation |
-| `turn-cap-spinning` | at the ceiling with no PR and no commits | escalate with the last tool calls the transcript recorded | none |
+| `turn-cap-spinning` | at the ceiling with no PR and no commits | escalate with the last tool calls the transcript recorded — and the completion path deliberately does **not** requeue it | none |
 | `admin-kill` | killed *below* its own ceiling — a restart or a drain | requeue | none |
 | `ci-infra` | PR open, every unresolved check cancelled / timed out / stale | re-run the failed jobs | none |
 | `ci-deterministic` | PR open, a check genuinely reports `FAILURE` | escalate with the failing check names and links | failed attempt |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omp-conductor",
-  "version": "0.4.3",
+  "version": "0.4.4",
   "type": "module",
   "license": "MIT",
   "description": "A 24/7 dispatcher that takes ready GitHub issues to green, mergeable PRs using omp coding sessions, with tiered escalation first to an orchestrator session and then to a human.",

package/src/daemon.ts

@@ -903,6 +903,38 @@ export async function collectSettlementFlags(
  * One attempt at one issue, from claim to terminal state. Everything is inside
  * a single try/catch so that a bad issue costs its own run and nothing else.
  */
+/**
+ * Whether a turns-cap kill is handed straight back to the queue.
+ *
+ * A turns cap consumes the independent continuation budget rather than an
+ * implementation-failure attempt, so a run that ran out of turns mid-work is
+ * requeued and the next tick reattaches its branch (#50).
+ *
+ * The artifact test is what keeps that from becoming a loop. A run that reached
+ * the ceiling with no PR, no observed head and no salvage commit produced
+ * nothing to continue *from*, and requeueing it spends another continuation on
+ * the same spin — the seven rows on this fleet that burned $109 producing no
+ * merged PR. That shape is `turn-cap-spinning`, which #132 says must escalate
+ * with evidence instead; requeueing it here would settle the question before the
+ * classifier ever saw the row, and the queue label would already be back on.
+ *
+ * Pure so the rule is testable without a worker session — the same split as
+ * {@link settlementFor}.
+ */
+export function shouldContinueAfterTurnsCap(f: {
+  killedBy?: string;
+  prUrl?: string;
+  headSha?: string;
+  salvageSha?: string;
+  continuation: number;
+  maxContinuations: number;
+}): boolean {
+  if (f.killedBy !== "turns") return false;
+  const hasArtifacts = f.prUrl !== undefined || f.headSha !== undefined || f.salvageSha !== undefined;
+  if (!hasArtifacts) return false;
+  return hasContinuationBudget(f.continuation, f.maxContinuations);
+}
+
 async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
   const { project, caps, tracker, store } = d;
   const issue = r.issue.number;
@@ -1299,13 +1331,17 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
         detail: [`${r.issue.title}`, r.issue.url, "", ...salvaged, "", result.report].join("\n"),
       });
     } else if (state === "failed" || state === "killed") {
-      // A turns cap consumes the independent continuation budget, not an
-      // implementation-failure attempt. The row is already `killed`, so this
-      // count includes the segment that just ended.
+      // The row is already `killed`, so this count includes the segment that
+      // just ended.
       const continuation = store.continuationsFor(project.name, issue);
-      const continueTurns =
-        result.killedBy === "turns" &&
-        hasContinuationBudget(continuation, caps.maxContinuationsPerIssue);
+      const continueTurns = shouldContinueAfterTurnsCap({
+        killedBy: result.killedBy,
+        prUrl: result.prUrl,
+        headSha: result.headSha,
+        salvageSha: settlement?.patch?.salvageSha,
+        continuation,
+        maxContinuations: caps.maxContinuationsPerIssue,
+      });
 
       if (continueTurns) {
         await tracker.removeLabel(issue, inProgress);
@@ -2891,8 +2927,13 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
     // base does move under it.
     if (run.state === "pushed-green" && cls === "unknown") continue;
 
+    const retry = run.failureClass !== undefined;
     store.updateRun(run.id, { failureClass: cls, recoveryAction: recovery });
-    log(`#${run.issue} classified ${cls} → ${recovery}: ${evidence}`);
+    log(
+      retry
+        ? `#${run.issue} retrying ${recovery} for ${cls}: ${evidence}`
+        : `#${run.issue} classified ${cls} → ${recovery}: ${evidence}`,
+    );
     await recoverRun(d, run, cls, recovery, evidence);
   }
 }

package/src/store.ts

@@ -735,10 +735,20 @@ export function openStore(dbPath: string): Store {
   // Newest first, and bounded: every row this returns costs `gh` calls to gather
   // facts for, so a fleet with a long unclassified history classifies over
   // several ticks rather than spending one tick's budget on all of it.
+  //
+  // Classified-but-unrecovered rows are included, not just unclassified ones: a
+  // recovery whose tracker write failed has to come back, or the log line saying
+  // "retrying next tick" is a lie and the row is stranded with a class and no
+  // action. `hold` is excluded because it is *recorded only* by design — its
+  // `recoveredAt` stays NULL forever, and re-offering it would spin the sweep.
   const selectUnclassified = db.query<RunRow, [string, number]>(
     `SELECT * FROM runs
-      WHERE project = ? AND failureClass IS NULL
+      WHERE project = ?
         AND state IN ('blocked', 'killed', 'failed', 'orphaned', 'pushed-green')
+        AND (
+          failureClass IS NULL
+          OR (recoveredAt IS NULL AND recoveryAction IN ('settle', 'continue', 'requeue', 'rerun-checks'))
+        )
       ORDER BY startedAt DESC, rowid DESC
       LIMIT ?`,
   );

package/src/tracker/github.ts

@@ -113,18 +113,29 @@ interface GhPrVerification {
 
 /** Carries the captured stderr so callers can classify a failure without
  *  re-running the command or parsing the message text of a plain Error. */
-class GhError extends Error {
+export class GhError extends Error {
   readonly argv: string[];
   readonly code: number;
   readonly stderr: string;
-
-  constructor(argv: string[], code: number, stderr: string) {
+  /**
+   * Whatever the command printed before it failed.
+   *
+   * Carried because a nonzero exit is not always a failure to *read*: `gh pr
+   * checks` exits 1 when a check failed and 8 when one is pending, having
+   * already written the complete JSON payload asked for. Discarding stdout here
+   * made both CI failure classes unreachable on exactly the pull requests they
+   * exist to classify — a red PR answered "no checks" (#132 follow-up).
+   */
+  readonly stdout: string;
+
+  constructor(argv: string[], code: number, stderr: string, stdout = "") {
     const detail = stderr.trim() || "(no stderr)";
     super(`\`gh ${argv.join(" ")}\` exited ${code}: ${detail}`);
     this.name = "GhError";
     this.argv = argv;
     this.code = code;
     this.stderr = stderr;
+    this.stdout = stdout;
   }
 }
 
@@ -155,7 +166,7 @@ async function gh(argv: string[], stdin?: string): Promise<string> {
 
   const signal = proc.signalCode;
   if (code !== 0 || signal) {
-    throw new GhError(argv, code, signal ? `${stderr}\nterminated by ${signal}` : stderr);
+    throw new GhError(argv, code, signal ? `${stderr}\nterminated by ${signal}` : stderr, stdout);
   }
   return stdout;
 }
@@ -671,11 +682,17 @@ export function makeTracker(p: ProjectConfig, runGh: typeof gh = gh): Tracker {
       if (!PR_URL.test(prUrl)) return [];
       try {
         return checkConclusionsFrom(await runGh(["pr", "checks", prUrl, "--json", "name,state,link"]));
-      } catch {
-        // `gh pr checks` exits non-zero when checks are failing *and* when it
-        // could not read them at all, so an exception here says nothing about the
-        // PR. Empty means "could not tell", and the classifier reads it that way.
-        return [];
+      } catch (err) {
+        // A nonzero exit is the *documented* answer for a PR with a failing or
+        // pending check — `gh pr checks` exits 1 and 8 respectively, having
+        // already printed the whole payload. Reading only the throw is what made
+        // `ci-infra` and `ci-deterministic` unreachable on live data: every red
+        // PR answered "no checks", so it classified `unknown` and escalated.
+        //
+        // So the payload wins when there is one, and only a genuinely unreadable
+        // answer falls through to "could not tell".
+        const stdout = err instanceof GhError ? err.stdout : "";
+        return checkConclusionsFrom(stdout);
       }
     },
 

package/src/types.ts

@@ -1299,10 +1299,14 @@ export interface Store {
   /** Everything an operator still has to care about: pending, sending, failed. */
   openReports(project: string): ReportRecord[];
   /**
-   * Terminal rows the classification sweep has not looked at yet, newest first
-   * and bounded (#132). Bounded because each row costs tracker calls to gather
-   * facts for: a fleet with a long unclassified history works through it over
-   * several ticks rather than spending one tick on all of it.
+   * Terminal rows the classification sweep still owes something to, newest first
+   * and bounded (#132): never classified, or classified with a retryable recovery
+   * that has not run. The second half is what makes "retrying next tick" true
+   * when a tracker write fails mid-recovery.
+   *
+   * Bounded because each row costs tracker calls to gather facts for: a fleet
+   * with a long unclassified history works through it over several ticks rather
+   * than spending one tick on all of it.
    */
   runsNeedingClassification(project: string, limit?: number): RunRecord[];
   /** Unrecovered rows per class, for `status`. Empty when nothing is carrying one. */