omp-conductor 0.19.7 → 0.20.1

package/src/verbs/server.ts

@@ -50,6 +50,11 @@ import { wakeDispatch } from "../wake.ts";
 import { cancelDrain } from "../fleet.ts";
 import { effectiveLane, laneEcho, writeLaneSectionHeading } from "../admission.ts";
 import { chainEntriesFromDiff, chainViolations } from "../chain-check.ts";
+// The runner-infrastructure waiver a red check's own log can earn (#1043 lane).
+// The worker keeps its two-red corrective budget in session, so this reply is
+// the only place that waiver can be stated — and it is stated from the same
+// classifier the daemon's own infra accounting uses, never a second copy.
+import { runnerInfraFailure } from "../failure-class.ts";
 import {
   repoSlugFor,
   type LaneFile,
@@ -60,6 +65,7 @@ import {
 import { releaseRefusal } from "../release-policy.ts";
 import { PR_LOOKUP_WINDOW_MS, REVISABLE_RUN_STATES, prReviewReadiness } from "../decisions.ts";
 import { LIVE_STATES } from "../store.ts";
+import { readyGate } from "../ready-gate.ts";
 import { parseToSpecEvidence } from "../to-spec.ts";
 import { DENIED_RELEASE_GRANTS } from "../types.ts";
 import type {
@@ -83,7 +89,7 @@ import type {
   VerbName,
   VerbRefusal,
 } from "../types.ts";
-import { prUrlParts, GhPrMissingError } from "../tracker/github.ts";
+import { failedRunIds, prUrlParts, GhPrMissingError } from "../tracker/github.ts";
 import { parseVerbRequest, roleRefusal, VERB_SPECS, type VerbReply } from "./protocol.ts";
 import {
   peerVerdict,
@@ -652,6 +658,8 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
       return prRecoverVerb(deps, project, channel, args, refuse, allow);
     case "conductor_pr_status":
       return prStatusVerb(deps, project, channel, args, refuse, allow);
+    case "conductor_ci_logs":
+      return ciLogsVerb(deps, project, channel, args, refuse, allow);
   }
 }
 
@@ -1344,6 +1352,17 @@ async function labelVerb(
     return refuse("action-failed", `refused: the tracker rejected the label change:\n${why}`, ref.issue);
   }
 
+  // Promotion provenance (#1041): the tick audits what reached the queue
+  // unattended, and "who queued this" is only answerable if every promotion
+  // path stamps it — a verdict promoted by hand that carried no actor would
+  // read in the audit exactly like one the daemon promoted. Stamped after the
+  // mutation committed, so a refused or failed label change leaves no
+  // provenance, and latched inside the store: a re-add of a label the issue
+  // already carries returns false and never restamps.
+  if (action === "add" && label === project.queueLabel) {
+    deps.store.markGroomingPromoted(project.name, ref.issue, Date.now(), "orchestrator");
+  }
+
   // A promotion that committed has made an issue claimable *now*, so the
   // resident daemon is poked rather than leaving the candidate for the next
   // scheduled pass (#878): three claimable issues once sat against `workers
@@ -1379,52 +1398,63 @@ async function laneForEcho(deps: VerbDeps, issue: number): Promise<{
   readable: boolean;
   lane: FileLane | undefined;
   malformed: string | undefined;
+  /** What the gate re-reads: the same body, thread and labels this parse used.
+   *  Absent exactly when `readable` is false. */
+  read: { body: string; comments: IssueComment[]; labels: readonly string[] } | undefined;
 }> {
   let body: string;
   let comments: IssueComment[];
+  let labels: readonly string[];
   try {
     const row = await deps.tracker.getIssue(issue);
-    if (row === undefined) return { readable: false, lane: undefined, malformed: undefined };
+    if (row === undefined) return { readable: false, lane: undefined, malformed: undefined, read: undefined };
     body = row.body;
+    labels = row.labels;
     comments = await deps.tracker.listComments(issue);
   } catch {
-    return { readable: false, lane: undefined, malformed: undefined };
+    return { readable: false, lane: undefined, malformed: undefined, read: undefined };
   }
+  const read = { body, comments, labels };
   const lane = effectiveLane(body, comments);
-  if (lane !== undefined) return { readable: true, lane, malformed: undefined };
+  if (lane !== undefined) return { readable: true, lane, malformed: undefined, read };
   // No declaration parsed anywhere. If a clearly delimited write-lane section
   // exists anyway, the section tried to declare and failed — the echo must
   // refuse, never claim fail-open.
   const heading =
     writeLaneSectionHeading(body) ??
     comments.map((c) => writeLaneSectionHeading(c.body)).find((h) => h !== undefined);
-  return { readable: true, lane: undefined, malformed: heading };
+  return { readable: true, lane: undefined, malformed: heading, read };
 }
 
 /**
- * The promotion-brief gate (#1036): a durable `promotable` grooming verdict is
- * an admission contract only while the dispatch brief still matches it, so a
+ * The promotion gate (#1036, #1041): a durable `promotable` grooming verdict is
+ * an admission contract only while the issue still matches it, so a
  * queue-label add must prove the match before any tracker mutation or daemon
- * wake. Compares the lane admission would enforce (the same parse the echo
- * reports) against the verdict's `fileLane` as deduplicated,
- * order-independent path sets, exact in both directions — missing and extra
- * paths refuse alike. Returns the refusal detail, or `undefined` when
- * promotion may proceed: no grooming row, a non-`promotable` row, or a
- * verified match.
+ * wake. Returns the refusal detail, or `undefined` when promotion may proceed:
+ * no grooming row, a non-`promotable` row, or a verdict that passes the gate.
+ *
+ * The judgement itself is {@link readyGate}, not this function. It used to be
+ * a lane comparison written here, which was correct and also the wrong place:
+ * since #1041 the daemon promotes passing verdicts unattended, and two copies
+ * of "is this ready" drift the moment one of them learns something. So this is
+ * the verb-shaped wrapper — recover the result, hand the gate the same body,
+ * thread and labels the echo parsed, and render whatever it refuses for. The
+ * gate names every missing element at once: a promotion that fails four checks
+ * costs one round trip, not four.
  *
  * Fail-closed by design: evidence behind a `promotable` row that does not
  * recover as a strict PROMOTABLE result (`parseToSpecEvidence`, the same
  * restart round-trip the selection side trusts), or an issue that cannot be
  * read to compare at all, refuses the promotion rather than dispatching on an
- * unverifiable brief. The gate never edits the issue — it names both path
- * sets and points at the verdict's `proposedBrief`; correcting the dispatch
+ * unverifiable brief. The gate never edits the issue — it names what is
+ * missing and points at the verdict's `proposedBrief`; correcting the dispatch
  * brief is an explicit orchestrator act.
  */
 function promotionBriefRefusal(
   deps: VerbDeps,
   project: ProjectConfig,
   issue: number,
-  read: { readable: boolean; lane: FileLane | undefined },
+  read: { readable: boolean; read: { body: string; comments: IssueComment[]; labels: readonly string[] } | undefined },
 ): string | undefined {
   const row = deps.store.grooming(project.name, issue);
   if (row?.verdict !== "promotable") return undefined;
@@ -1436,30 +1466,22 @@ function promotionBriefRefusal(
       "brief still matches what was groomed. Re-groom the issue so a fresh verdict replaces the row, then add the label again."
     );
   }
-  if (!read.readable) {
-    return (
-      `refused: ${project.queueLabel} was not added — #${issue} carries a durable PROMOTABLE grooming verdict, but ` +
-      "the issue could not be read, so the current brief cannot be compared with the verdict's file lane. " +
-      "Retry when the tracker responds."
-    );
-  }
-  // One closure so both sides of the comparison normalize in lockstep: a set
-  // comparison whose halves dedupe differently lies silently (#1036).
-  const normalizedPaths = (paths: readonly string[]): string[] =>
-    [...new Set(paths.map((path) => path.trim()).filter((path) => path.length > 0))].sort();
-  const brief = normalizedPaths(read.lane?.files ?? []);
-  const verdict = normalizedPaths(result.fileLane);
-  if (
-    brief.length === verdict.length &&
-    brief.every((path, index) => path === verdict[index])
-  ) {
-    return undefined;
-  }
+  // `"unread"` rather than an empty thread when the read failed: behind a
+  // promotable row "could not look" is a refusal, not an empty lane (#1036).
+  const verdict = readyGate({
+    result,
+    issueBody: read.read?.body ?? "",
+    comments: read.readable && read.read !== undefined ? read.read.comments : "unread",
+    labels: read.read?.labels ?? [],
+    project,
+  });
+  if (verdict.ok) return undefined;
   return (
-    `refused: ${project.queueLabel} was not added — #${issue}'s current write lane [${brief.join(", ")}] disagrees ` +
-    `with its durable PROMOTABLE verdict's file lane [${verdict.join(", ")}]. The durable verdict is the admission ` +
-    "contract: apply the verdict's proposedBrief to the issue (its ## Exact write lane included), then add the " +
-    "label again. The gate never rewrites the issue itself."
+    `refused: ${project.queueLabel} was not added — #${issue} carries a durable PROMOTABLE grooming verdict, but ` +
+    `the promotion is not ready:\n${verdict.missing.map((miss) => `- ${miss}`).join("\n")}\n` +
+    "The durable verdict is the admission contract: apply the verdict's proposedBrief to the issue (its " +
+    "## Exact write lane included) and fix every element above, then add the label again. The gate never " +
+    "rewrites the issue itself."
   );
 }
 
@@ -1901,6 +1923,275 @@ async function prStatusVerb(
   return allow(`${prUrl} at ${verification.headSha}: ${verification.status} — ${verification.reason}`, undefined, issue);
 }
 
+/**
+ * How much failing-job log text one `conductor_ci_logs` reply may carry.
+ *
+ * Characters rather than bytes: `gh run view --log-failed` output is
+ * effectively ASCII, so this is the ~32 KB posture the daemon's own log reads
+ * already work at, and a character bound cannot end mid-multibyte-sequence the
+ * way a byte slice can. It bounds the log text; the few framing lines are
+ * small and constant.
+ */
+export const CI_LOGS_MAX_CHARS = 32 * 1024;
+
+/** The smallest slice of one failed job worth showing. A PR whose matrix
+ *  exploded into forty failed jobs must not answer with forty useless
+ *  fragments — better to show the first few in full and say how many were left
+ *  out. */
+const CI_LOGS_MIN_JOB_CHARS = 2 * 1024;
+
+/**
+ * How many of the head's failing workflow runs are read, and how many attempts
+ * of each.
+ *
+ * Both exist because every attempt of every failing job is a separate guarded
+ * `gh` call, and one pathologically re-run pull request must not turn a worker's
+ * single diagnostic call into a hundred of them. The attempt bound is the
+ * tracker's own contract (`runFailedAttemptLogs` refuses rather than presenting
+ * attempts 1..N as the whole history), so exceeding it surfaces as the honest
+ * read failure below, never as a partial log the worker would read as complete.
+ */
+const CI_LOGS_MAX_RUNS = 5;
+const CI_LOGS_MAX_ATTEMPTS = 10;
+
+/**
+ * One bounded rendering of the failing jobs' logs.
+ *
+ * Pure and exported so the bound is tested directly rather than inferred from a
+ * verb reply. Empty chunks are dropped: `runFailedAttemptLogs` deliberately
+ * emits one per failed job even when its failed steps printed nothing, which
+ * the daemon's all-infra guard needs and a reader does not.
+ *
+ * Each shown job gets a fair share of the budget, so one enormous test job
+ * cannot starve the sibling that actually explains the failure. An over-share
+ * job keeps its head *and* its tail — the head carries the step banner and the
+ * runner's own setup sentences, the tail carries the failure — with the elision
+ * stated in the middle rather than left to be guessed at.
+ *
+ * No redaction is applied, deliberately and consistently with every other place
+ * these same chunks are read (escalations, the historical infra repair): the
+ * text is GitHub's own log, which Actions has already masked its registered
+ * secrets out of. A second, weaker masking here would only suggest a guarantee
+ * this does not make.
+ */
+export function boundCiLogs(chunks: readonly string[], maxChars = CI_LOGS_MAX_CHARS): string {
+  const failing = chunks.filter((chunk) => chunk.trim() !== "");
+  if (failing.length === 0) return "";
+  const share = Math.max(CI_LOGS_MIN_JOB_CHARS, Math.floor(maxChars / failing.length));
+  const room = Math.max(1, Math.floor(maxChars / share));
+  const shown = failing.slice(0, room);
+  const parts = shown.map((chunk, i) => {
+    // Head a quarter, tail the rest: the head carries the step banner and the
+    // runner's own setup sentences, the tail carries the failure.
+    const head = Math.floor(share / 4);
+    const body =
+      chunk.length <= share
+        ? chunk
+        : `${chunk.slice(0, head)}\n--- ${chunk.length - share} characters elided ---\n` +
+          chunk.slice(chunk.length - (share - head));
+    return `--- failing job ${i + 1} of ${failing.length} ---\n${body}`;
+  });
+  if (shown.length < failing.length) {
+    parts.push(
+      `--- ${failing.length - shown.length} further failing job(s) not shown: this reply is bounded to ` +
+        `${maxChars} characters of log. Re-run this call after fixing the ones above, or read the rest in the ` +
+        "Actions UI. ---",
+    );
+  }
+  return parts.join("\n");
+}
+
+/**
+ * The second read verb: what CI actually printed when it went red (#1043 lane).
+ *
+ * A worker that has to fix a red check otherwise has three bad options — guess
+ * from the check name, re-run the whole suite locally hoping to reproduce a
+ * runner-only failure, or report blocked. This hands it the failing jobs' own
+ * failed steps through the tracker read the daemon already uses for its infra
+ * classification, so there is exactly one transport for "what did CI say" and
+ * the worker reads the same evidence the classifier does.
+ *
+ * Same resolution as {@link prStatusVerb}: a worker reads its own run's PR and
+ * no other; an orchestrator names one. `headSha` is an assertion, never a
+ * selector — logs from a superseded commit are worse than no logs, because they
+ * describe a failure the current head may not even have.
+ *
+ * An unreadable log is a refusal, never an empty success. "No failing jobs" and
+ * "the log could not be read" invite opposite next moves, and a verb that
+ * spelled them the same way would teach a worker to declare a red check fixed
+ * because nothing came back.
+ */
+async function ciLogsVerb(
+  deps: VerbDeps,
+  project: ProjectConfig,
+  channel: VerbChannel,
+  args: Record<string, unknown>,
+  refuse: Refuse,
+  allow: Allow,
+): Promise<Verdict> {
+  const asked = args["prUrl"];
+  let prUrl: string;
+  let issue: number | undefined;
+
+  if (channel.kind === "run") {
+    const own = deps.store.getRun(channel.runId);
+    if (own?.prUrl === undefined) {
+      return refuse("pr-missing", "refused: this run has no pull request yet, so it has no CI to read.");
+    }
+    if (typeof asked === "string" && asked !== own.prUrl) {
+      return refuse(
+        "pr-not-this-run",
+        `refused: this socket belongs to the run for #${channel.issue}, whose pull request is ${own.prUrl}. ` +
+          "A worker reads only its own run's CI — the daemon resolves the run from the socket.",
+      );
+    }
+    prUrl = own.prUrl;
+    issue = channel.issue;
+  } else {
+    if (typeof asked !== "string") {
+      return refuse("malformed-argument", "refused: conductor_ci_logs needs prUrl when it has no run to infer one from.");
+    }
+    prUrl = asked;
+    issue = runForPr(deps, project.name, asked)?.issue;
+  }
+
+  const parts = prUrlParts(prUrl);
+  if (parts === undefined) {
+    return refuse("malformed-argument", `refused: ${prUrl} is not a pull request URL.`, issue);
+  }
+  const repo = `${parts.owner}/${parts.repo}`;
+
+  const askedHead = args["headSha"];
+  const expectedHead = typeof askedHead === "string" ? askedHead : undefined;
+  let verification: PrVerification | undefined;
+  try {
+    // Read mode, exactly as `conductor_pr_status`: a merged or closed PR is a
+    // fact worth reading logs against, not the merge gate's `expected OPEN`
+    // refusal.
+    verification = await deps.tracker.verifyPr(prUrl, expectedHead, { read: true });
+  } catch (err) {
+    verification = undefined;
+    deps.log(`verb ci_logs could not read ${prUrl}: ${err instanceof Error ? err.message : String(err)}`);
+  }
+  if (verification === undefined) {
+    return refuse(
+      "head-unresolvable",
+      `refused: ${prUrl} could not be read, so there is no head to pin these logs to. Wait and ask again, ` +
+        "and report blocked rather than guessing if it stays unreadable.",
+      issue,
+    );
+  }
+  if (isHeadMismatch(verification.reason)) {
+    return refuse(
+      "head-stale",
+      `refused: ${verification.reason}. These logs would describe a commit that is no longer the head — ` +
+        "read the current head with conductor_pr_status and ask again for that one.",
+      issue,
+    );
+  }
+
+  const runIds = failedRunIds(await deps.tracker.checkConclusions(prUrl));
+  if (runIds.length === 0) {
+    // Honest and distinguishable from an unreadable log: the check rollup was
+    // read and named no non-green run. `verification.status` says which kind of
+    // "nothing to diagnose" this is, because a pending head is worth asking
+    // again about and a green one is not.
+    return allow(
+      `${prUrl} at ${verification.headSha}: no failing workflow run to read logs from — ${verification.reason}. ` +
+        (verification.status === "pending"
+          ? "Checks are still running; poll conductor_pr_status and ask again if one goes red."
+          : "Nothing here needs diagnosing."),
+      undefined,
+      issue,
+    );
+  }
+
+  const chunks: string[] = [];
+  for (const runId of runIds.slice(0, CI_LOGS_MAX_RUNS)) {
+    let logs: string[] | undefined;
+    try {
+      logs = await deps.tracker.runFailedAttemptLogs(
+        repo,
+        `https://github.com/${repo}/actions/runs/${runId}`,
+        CI_LOGS_MAX_ATTEMPTS,
+      );
+    } catch (err) {
+      logs = undefined;
+      deps.log(`verb ci_logs could not read run ${runId} of ${repo}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    if (logs === undefined) {
+      // Fail loudly. The tracker answers `undefined` for a read it could not
+      // make *or* for an attempt history past the bound; either way the chunks
+      // gathered so far are not the evidence set, and handing them over as if
+      // they were is how a worker concludes the wrong thing about a red check.
+      return refuse(
+        "action-failed",
+        `refused: the failing-job logs of ${repo} run ${runId} could not be read (an unreadable job log, a ` +
+          `revoked token, or more than ${CI_LOGS_MAX_ATTEMPTS} re-run attempts to gather). That is not "no ` +
+          `failures": ${prUrl} is ${verification.status} at ${verification.headSha}. Read the run in the Actions ` +
+          "UI, or ask again — this call mutates nothing.",
+        issue,
+      );
+    }
+    chunks.push(...logs);
+  }
+
+  const rendered = boundCiLogs(chunks);
+  const omitted =
+    runIds.length > CI_LOGS_MAX_RUNS
+      ? ` (${CI_LOGS_MAX_RUNS} of ${runIds.length} non-green workflow runs read)`
+      : "";
+  if (rendered === "") {
+    // Every failed job's failed steps printed nothing readable. Determinate —
+    // the reads succeeded — and still not a diagnosis, so it says which run to
+    // look at rather than pretending the PR is fine.
+    return allow(
+      `${prUrl} at ${verification.headSha} is ${verification.status}, but its failing jobs printed no log ` +
+        `output${omitted}. Read the run itself: https://github.com/${repo}/actions/runs/${runIds[0]}`,
+      undefined,
+      issue,
+    );
+  }
+
+  // The two-red corrective budget is a rule the *worker* keeps, in session,
+  // from its brief — nothing in the daemon counts a run's reds, so nothing in
+  // the daemon can waive one on the worker's behalf. This reply is therefore
+  // the only surface where "that red was the runner, not your diff" can become
+  // true, and the brief's promise of an infra exemption is kept here or nowhere.
+  //
+  // Classified from the chunks already fetched: no second read, no store write.
+  // The verdict line goes *above* the log, because a worker that has just been
+  // handed 32k of log must not have to reach the end to learn the red was free.
+  const classified = chunks.filter((chunk) => chunk.trim() !== "").map((chunk) => runnerInfraFailure(chunk));
+  const wordings = new Set(classified.filter((match): match is string => match !== undefined));
+  let note = "";
+  if (wordings.size > 0 && classified.every((match) => match !== undefined)) {
+    note =
+      `RUNNER INFRASTRUCTURE: every failing job's own log names the runner, not your diff — ` +
+      `${[...wordings].map((w) => `"${w}"`).join(", ")}. This red does not spend one of your two corrective ` +
+      "pushes. Do not change the diff for it and do not push a fix: wait, poll conductor_pr_status again, and " +
+      "if it is still red stop and report with that wording quoted — the dispatcher re-runs infrastructure " +
+      "checks itself and does not charge the attempt.\n\n";
+  } else if (wordings.size > 0) {
+    // Mixed evidence: the same rule the daemon's own classifier applies — one
+    // real failure outranks any number of infra ones. Said out loud rather than
+    // left silent, because a worker reading a 429 in job 2 would otherwise
+    // claim a waiver the classifier never granted.
+    note =
+      `MIXED: ${wordings.size} of the failing jobs name runner infrastructure ` +
+      `(${[...wordings].map((w) => `"${w}"`).join(", ")}), but at least one failed for a reason that is not ` +
+      "infrastructure. A real failure outranks any number of infra ones, so this red counts: fix the " +
+      "non-infrastructure failure below.\n\n";
+  }
+  // Nothing matched: silence. An unclassifiable red is charged, and a reply that
+  // hedged here would teach a worker to argue for a waiver from any red log.
+  return allow(
+    `${prUrl} at ${verification.headSha}: ${verification.status} — ${verification.reason}${omitted}\n\n${note}${rendered}`,
+    undefined,
+    issue,
+  );
+}
+
 /**
  * Update one pull request's title and/or body — nothing else.
  *

package/src/wake.ts

@@ -20,9 +20,13 @@
  *
  * Every failure is honest and non-fatal: the caller's own mutation already
  * committed, so a missing or unreachable daemon degrades to "the next scheduled
- * pass will claim" rather than an error that suggests the mutation failed.
+ * pass will claim" rather than an error that suggests the mutation failed. That
+ * now includes an unreadable bearer token: `POST /wake` is authenticated (Phase
+ * 4), and a wake nobody can authenticate is one more reason the pass waits —
+ * never an exception thrown through a mutation that already landed.
  */
 
+import { httpAuthHeader, httpTokenPath } from "./http-token.ts";
 import { livingDaemon } from "./lifecycle.ts";
 
 /** What the wake attempt did, as the line a human reads after the mutation. */
@@ -31,11 +35,24 @@ export async function wakeDispatch(projectName: string): Promise<string> {
   if (daemon === undefined) {
     return "daemon not running — claiming starts when the daemon next starts or ticks";
   }
+  // The daemon mints this at start, and `livingDaemon()` just said one is
+  // running — so an absent token is a real anomaly (a different
+  // `$OMP_CONDUCTOR_HOME`, or a hand-deleted file), not the ordinary "no daemon
+  // has ever run here" that `missingHttpTokenMessage()` describes. Say what is
+  // missing and where, and degrade exactly like an unreachable daemon: the
+  // mutation stands, the pass is merely not brought forward.
+  const auth = httpAuthHeader();
+  if (auth === undefined) {
+    return (
+      `daemon wake failed — no readable HTTP token at ${httpTokenPath()}; ` +
+      "the next scheduled pass will claim"
+    );
+  }
   let response: Response;
   try {
     response = await fetch(`http://127.0.0.1:${daemon.port}/wake`, {
       method: "POST",
-      headers: { "content-type": "application/json" },
+      headers: { ...auth, "content-type": "application/json" },
       body: JSON.stringify({ project: projectName }),
     });
   } catch {