omp-conductor 0.19.7 → 0.20.0

package/src/worker.ts

@@ -12,8 +12,10 @@
 
 import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
+import { knowledgeSection } from "./knowledge.ts";
 import { createSession, disposeSession, SessionAdmissionClosedError, type AgentSessionLike } from "./omp.ts";
 import type { GateShape, ReleaseBlockContext } from "./release-policy.ts";
+import { TO_SPEC_SCHEMA } from "./to-spec.ts";
 import type { Caps, GraphToolsObservation, ResolvedGrants, RunState } from "./types.ts";
 
 /** Structured evidence fields from the worker's final report. */
@@ -53,6 +55,17 @@ export interface WorkerSettlement {
   blockers?: string[];
   /** Commands run as evidence, exactly as executed. */
   proof?: string[];
+  /**
+   * Durable facts about the REPOSITORY this run learned — where a subsystem is
+   * wired, which command actually proves a change, a test that looks like it
+   * asserts something and does not (Phase 3 knowledge overlay). Appended to the
+   * fleet's per-repo overlay at settlement and shown to every later run in the
+   * same repo, which is why they must be about the repo and not about this
+   * task: "I fixed the parser" helps nobody, "the parser's only caller is
+   * cli.ts:route" saves the next run a search. Optional, and every settlement
+   * that predates it stays valid.
+   */
+  discoveries?: string[];
 }
 
 /**
@@ -98,6 +111,16 @@ export const WORKER_SETTLEMENT_SCHEMA = {
       items: { type: "string" },
       description: "The commands you ran as evidence, exactly as executed.",
     },
+    discoveries: {
+      type: "array",
+      items: { type: "string" },
+      description:
+        "Optional. Durable facts about THIS REPOSITORY that cost you turns to find and would cost " +
+        "the next run the same: where a subsystem is wired, which command actually proves a change, " +
+        "a test that looks like it asserts something and does not. Not a summary of your task — " +
+        "these are shown to every later run in this repo, so leave the field out rather than " +
+        "filling it with what you did.",
+    },
   },
 } as const;
 
@@ -172,6 +195,219 @@ export function renderReviewRevisionPrompt(findings: string, round: number): str
   return `${REVIEW_REVISION_PROMPT}\n\nReview round ${round} — blocking findings:\n\n${findings}`;
 }
 
+/**
+ * The template one fresh correction's brief is rendered from.
+ *
+ * Read per correction rather than cached, exactly like the dispatcher's own
+ * `buildBrief`: editing the brief then takes effect on the next correction
+ * instead of needing a daemon restart.
+ */
+const CORRECTION_BRIEF_PATH = join(import.meta.dir, "briefs", "correction.md");
+
+/**
+ * How much of the diff one fresh correction brief carries (#1046).
+ *
+ * The same bound, and the same reasoning, as {@link ADJUDICATION_DIFF_LIMIT}: a
+ * correction needs the change it is correcting, not every generated line, and an
+ * enormous PR must not crowd out the findings and criteria that follow it. The
+ * marker says exactly what was withheld, so the session reads the rest off the
+ * branch it already has checked out rather than assuming it saw everything.
+ */
+export const CORRECTION_DIFF_LIMIT = 60_000;
+
+/**
+ * Everything one fresh correction is told (#1046).
+ *
+ * Assembled by the caller from durable rows and live tracker reads, never from a
+ * worker's own claims — and, unlike a resumed revision, this is ALL the session
+ * gets. The exhausted implementation transcript is not replayed, so a field
+ * missing here is context the correction simply does not have.
+ */
+export interface CorrectionEvidence {
+  /** Canonical `owner/repo` of the code repo, or the routing name when the
+   *  clone URL yields none. Also the key the knowledge overlay is read under. */
+  repo: string;
+  issue: number;
+  issueTitle?: string;
+  /** The issue's acceptance criteria, verbatim. */
+  issueBody: string;
+  /** The pull request this correction must push onto. Never a second PR. */
+  prUrl: string;
+  /** The exact head the findings were written against. */
+  headSha: string;
+  /** The branch and worktree that already exist for this run. */
+  branch: string;
+  worktree: string;
+  /** Live check state at that head, as the tracker reported it. */
+  checks: string;
+  /** The diff at that head, bounded by {@link CORRECTION_DIFF_LIMIT}. */
+  diff: string;
+  round: number;
+  /** The review-round ceiling, when the caller knows it. */
+  maxRounds?: number;
+  /** This round's blocking findings, delivered verbatim. */
+  findings: string;
+  /** Earlier rounds and what became of them. A finding without its outcome
+   *  reads as an outstanding complaint even when it was fixed, which is how a
+   *  correction session re-does work that already landed. */
+  priorRounds: readonly AdjudicationRound[];
+  /**
+   * Why this round is a fresh session rather than a resume, in plain words —
+   * "the implementation session was killed by the turns cap", "the fleet's
+   * worker model changed after the rejected run". It goes in the brief because a
+   * session that does not know its transcript is gone will look for it: #1035's
+   * whole failure was correction rounds spent replaying an exhausted session,
+   * and a session told nothing behaves as if it were still that one.
+   */
+  reason: string;
+  /** The gate commands this correction must run before pushing. */
+  proofCommands: readonly string[];
+  /**
+   * The fleet's accumulated repo knowledge, pre-rendered by
+   * {@link knowledgeSection}. Omitted, it is read here from {@link repo} — the
+   * overlay exists to reach every session launched against a repo, so it must
+   * not depend on each caller remembering to pass it. Pass `""` to render none.
+   */
+  knowledge?: string;
+}
+
+/**
+ * The full opening prompt for one fresh correction round (#1046).
+ *
+ * Exported so a test can pin the exact payload without standing up a session:
+ * the brief IS the deliverable here. `REVIEW_REVISION_PROMPT` can be three
+ * sentences because the session it prompts already holds the issue, the plan and
+ * the diff; this one holds nothing, so every one of those facts has to arrive in
+ * the text — and a test that only asserted "a session was launched" would pass on
+ * an empty brief.
+ */
+export function renderCorrectionBrief(evidence: CorrectionEvidence): string {
+  const template = readFileSync(CORRECTION_BRIEF_PATH, "utf8");
+  const rounds =
+    evidence.priorRounds.length === 0
+      ? "_(none recorded — this is the first review round of this run)_"
+      : evidence.priorRounds
+          .map((round) => `Round ${String(round.round)} (${round.outcome ?? "never settled"}):\n${round.findings.trim()}`)
+          .join("\n\n");
+  const diff =
+    evidence.diff.length <= CORRECTION_DIFF_LIMIT
+      ? evidence.diff
+      : `${evidence.diff.slice(0, CORRECTION_DIFF_LIMIT)}\n` +
+        `[diff truncated at ${String(CORRECTION_DIFF_LIMIT)} of ${String(evidence.diff.length)} characters — ` +
+        "read the remainder from your own checkout with `git diff origin/HEAD...HEAD`]";
+  const proof =
+    evidence.proofCommands.length === 0
+      ? "_The project names no gate commands. Run the repo's own tests and say exactly what you ran._"
+      : ["```sh", ...evidence.proofCommands, "```"].join("\n");
+  // The overlay's own trailing blank line separates it from the heading that
+  // follows; an absent overlay renders nothing at all, so the brief closes up
+  // around it rather than leaving an empty section behind.
+  const knowledge = evidence.knowledge ?? knowledgeSection(evidence.repo);
+  return renderBrief(template, {
+    ISSUE_NUMBER: String(evidence.issue),
+    ISSUE_TITLE: evidence.issueTitle ?? "(title unavailable at dispatch)",
+    REPO: evidence.repo,
+    PR_URL: evidence.prUrl,
+    HEAD_SHA: evidence.headSha,
+    CHECKS: evidence.checks,
+    WORKTREE: evidence.worktree,
+    BRANCH: evidence.branch,
+    ROUND:
+      evidence.maxRounds === undefined
+        ? String(evidence.round)
+        : `${String(evidence.round)} of ${String(evidence.maxRounds)} allowed`,
+    REASON: evidence.reason,
+    FINDINGS: evidence.findings.trim(),
+    ACCEPTANCE_CRITERIA: evidence.issueBody.trim(),
+    PRIOR_ROUNDS: rounds,
+    DIFF: diff,
+    PROOF_COMMANDS: proof,
+    KNOWLEDGE: knowledge,
+  });
+}
+
+/**
+ * What one fresh correction is launched with (#1046).
+ *
+ * Deliberately {@link WorkerOpts} minus two keys. `brief` is gone because this
+ * transport renders its own from `briefs/correction.md` — a caller that could
+ * pass a brief could pass the findings-only resume prompt, which is the failure
+ * this exists to end. `resume` is gone because it is not a decision available
+ * here: a "fresh correction" that resumes is #1046's named silent fake, and the
+ * cheapest way to make it unavailable is to make it inexpressible.
+ *
+ * Everything else — worktree, caps, sockets, grants, meters, pause control — is
+ * inherited unchanged, because a correction is an ordinary mutable worker run on
+ * an existing branch. The caller owns the run's identity and passes it through
+ * here: this transport creates no worktree, opens no pull request, and derives no
+ * issue or run of its own.
+ */
+export interface CorrectionWorkerOpts extends Omit<WorkerOpts, "brief" | "resume"> {
+  /** The evidence the brief is rendered from. */
+  evidence: CorrectionEvidence;
+  /**
+   * The transcript of the session this correction replaces, when the caller
+   * recorded one. Used for exactly one thing: proving the lineage actually
+   * changed (see {@link CorrectionResult.freshSession}).
+   */
+  originSessionRef?: string;
+}
+
+/** What one fresh correction produced (#1046). */
+export interface CorrectionResult extends WorkerResult {
+  /**
+   * The transcript this correction opened — the lineage #1045 persists as
+   * `correctionSessionRef`. Absent when the session opened none.
+   */
+  sessionRef?: string;
+  /**
+   * Whether this really was a new session. False when the harness handed back
+   * {@link CorrectionWorkerOpts.originSessionRef} — the exact silent fake #1046
+   * names, a new model string over the old session file — and false when no
+   * transcript was opened at all, because neither case proves a fresh lineage.
+   * A caller must not record a fresh-correction provenance it did not get.
+   */
+  freshSession: boolean;
+}
+
+/**
+ * Launch one fresh correction on an existing branch, worktree and pull request
+ * (#1046).
+ *
+ * The failure this ends (#1035): a capped implementation session kept receiving
+ * correction rounds by `resume: true` after it had exhausted its turns and after
+ * the fleet had switched worker models, so every round was spent replaying a dead
+ * transcript under a model that was no longer configured.
+ *
+ * So this composes {@link runWorker} — the same session primitive, the same caps,
+ * the same verb surface — with exactly two things different: the brief is
+ * self-contained ({@link renderCorrectionBrief}) and `resume` is never passed, so
+ * the harness opens a new transcript under whatever model the caller resolved.
+ * A second worker runtime would be a second set of cap, settlement and pause
+ * semantics to keep in sync, which is how the two paths drift apart.
+ *
+ * What it deliberately does NOT do: nothing here touches the run's identity. The
+ * branch, worktree, pull request, issue, run id and verb channel all arrive from
+ * the caller and leave unchanged, so a correction cannot open a second PR, create
+ * a second worktree, or start a new attempt against the issue.
+ */
+export async function runCorrectionWorker(
+  o: CorrectionWorkerOpts,
+  deps: RunWorkerDeps = { createSession },
+): Promise<CorrectionResult> {
+  const { evidence, originSessionRef, ...worker } = o;
+  // No `resume` key exists on `worker` to forward — the type forbids one, so
+  // `runWorker` takes its fresh-session path and the harness opens a new
+  // transcript in the run's own session directory.
+  const result = await runWorker({ ...worker, brief: renderCorrectionBrief(evidence) }, deps);
+  const sessionRef = result.sessionFile;
+  return {
+    ...result,
+    ...(sessionRef === undefined ? {} : { sessionRef }),
+    freshSession: sessionRef !== undefined && sessionRef !== originSessionRef,
+  };
+}
+
 
 /**
  * What the review-ceiling adjudicator is told (#877).
@@ -508,6 +744,19 @@ export interface WorkerResult {
    * "the tool was missing" apart.
    */
   graphTools?: GraphToolsObservation;
+  /**
+   * The repo facts this run reported in its structured settlement
+   * ({@link WorkerSettlement.discoveries}), verbatim and unjudged — the
+   * daemon appends them to the repo's knowledge overlay at settlement.
+   *
+   * Surfaced here rather than parsed back out of {@link report}: the rendered
+   * report is prose for a human, and re-deriving a list from it would be a
+   * second, weaker contract beside the schema the worker actually answered.
+   * Absent whenever the run yielded no structured settlement or named nothing
+   * — never an empty array, which would read as "the fleet learned nothing"
+   * rather than "the worker was not asked".
+   */
+  discoveries?: string[];
 }
 
 /**
@@ -619,7 +868,7 @@ export function structuredSettlement(content: unknown): WorkerSettlement | undef
       const value = record[key];
       if (typeof value === "string" && value !== "") settlement[key] = value;
     }
-    for (const key of ["blockers", "proof"] as const) {
+    for (const key of ["blockers", "proof", "discoveries"] as const) {
       const value = record[key];
       if (Array.isArray(value) && value.every((item) => typeof item === "string")) {
         settlement[key] = value;
@@ -807,6 +1056,199 @@ export async function runAdjudicator(
   };
 }
 
+/**
+ * The turn ceiling for one to-spec grooming scout (#1041).
+ *
+ * Half the adjudicator's 40, and for the opposite reason: an adjudication is
+ * handed its whole evidence in the brief (issue, diff, checks, every prior
+ * round) and spends its turns reasoning, while a scout is handed a candidate
+ * and must go and READ — the source repo at a ref, the entry points, the
+ * existing tests, the later work that may have retired the premise. That is
+ * search, and search is where a session bounded only by a worker's budget
+ * quietly spends a worker's budget. Twenty read/answer turns is enough for a
+ * bounded repo walk plus the structured answer; a scout still going after
+ * them has stopped grooming one candidate and started exploring the
+ * repository, which is exactly the cost the mechanical selection bounds
+ * everywhere else.
+ */
+export const TO_SPEC_MAX_TURNS = 20;
+
+/** What one to-spec scout produced (#1041). */
+export interface ToSpecScoutResult {
+  /**
+   * The agent's answer, verbatim and unjudged — the structured payload it
+   * yielded, serialized, or its own last text when it yielded nothing. This
+   * string is `recordToSpecGrooming`'s `input`, and nothing here reads a field
+   * of it: conductor re-validates every result through the strict parser at
+   * persistence time, so a runner that pre-parsed would be a second, weaker
+   * contract. Empty when the session produced neither.
+   */
+  raw: string;
+  /** The session's last words, kept whether or not it answered: for a scout
+   *  that produced nothing this text IS the evidence of what went wrong. */
+  report: string;
+  /** What actually ran it, for the durable provenance (#875). */
+  model?: string;
+  provider?: string;
+  turns: number;
+  spendUsd: number;
+}
+
+export interface ToSpecScoutOpts {
+  /** The rendered `briefs/to-spec.md` for exactly one candidate. */
+  brief: string;
+  /** A directory the session may read: everything about the candidate is in
+   *  the brief, and a scout has no branch — so this is the session directory,
+   *  never a run worktree. */
+  cwd: string;
+  /** The model pattern `project.groomRole` resolved to, in omp's model/role
+   *  syntax. Omitted leaves the harness to choose. */
+  model?: string;
+  sessionDir?: string;
+  ompSettingsFile?: string;
+  socketPath?: string;
+  onSpawn?: (pid: number) => void;
+  onChildLog?: (line: string) => void;
+  maySpawn?: () => boolean;
+  /** Hard turn ceiling; the daemon passes {@link TO_SPEC_MAX_TURNS}. */
+  maxTurns: number;
+}
+
+/**
+ * Run one bounded to-spec grooming pass (#1041).
+ *
+ * The adjudicator's twin, and read-only by the same construction: no
+ * `verbSocketPath`, so no conductor mutation verb — no label change, no
+ * promotion, no push — is reachable from this session. That absence is the
+ * whole safety argument for letting the daemon launch grooming unattended:
+ * the scout cannot queue the candidate it is grooming, only describe it. The
+ * queue label is added later, by the daemon, and only through the ready gate.
+ *
+ * Two deliberate differences from {@link runAdjudicator}. It carries
+ * {@link TO_SPEC_SCHEMA} with `outputSchemaMode: "strict"` — the to-spec
+ * contract has always been invoked strictly (#772), and a permissive schema
+ * would let a half-filled result through the harness only for the parser to
+ * refuse it as malformed one layer later, spending the batch to learn nothing.
+ * And it returns the answer UNPARSED: `recordToSpecGrooming` is the single
+ * persistence path and the only validator, so a second decode here would be
+ * a second contract to keep in sync.
+ */
+export async function runToSpecScout(
+  o: ToSpecScoutOpts,
+  deps: RunWorkerDeps = { createSession },
+): Promise<ToSpecScoutResult> {
+  let session: AgentSessionLike;
+  try {
+    session = await deps.createSession({
+      cwd: o.cwd,
+      ...(o.sessionDir === undefined ? {} : { sessionDir: o.sessionDir }),
+      ...(o.model === undefined ? {} : { model: o.model }),
+      ...(o.ompSettingsFile === undefined ? {} : { ompSettingsFile: o.ompSettingsFile }),
+      ...(o.socketPath === undefined ? {} : { socketPath: o.socketPath }),
+      ...(o.onSpawn === undefined ? {} : { onSpawn: o.onSpawn }),
+      ...(o.onChildLog === undefined ? {} : { onChildLog: o.onChildLog }),
+      ...(o.maySpawn === undefined ? {} : { maySpawn: o.maySpawn }),
+      // Worker role, and deliberately no `verbSocketPath` — see the doc
+      // comment: a scout reads and answers, it never mutates.
+      role: "worker",
+      outputSchema: TO_SPEC_SCHEMA,
+      outputSchemaMode: "strict",
+      requireYieldTool: true,
+    });
+  } catch (err) {
+    // The pre-spawn admission gate closed, or the harness could not start.
+    // Both are "no answer", and the caller settles the grooming row on the
+    // failure text rather than leaving the in-flight marker to expire.
+    return {
+      raw: "",
+      report: err instanceof Error ? err.message : String(err),
+      turns: 0,
+      spendUsd: 0,
+    };
+  }
+
+  let turns = 0;
+  let spendUsd = 0;
+  let report = "";
+  let yielded = "";
+  let model: string | undefined;
+  let provider: string | undefined;
+  const finished = Promise.withResolvers<void>();
+  session.on("turn_start", () => {
+    turns += 1;
+    if (turns > o.maxTurns) session.abort();
+  });
+  session.on("message_end", (event) => {
+    const message = field(event, "message");
+    if (field(message, "role") !== "assistant") return;
+    const content = field(message, "content");
+    const payload = yieldedPayload(content);
+    if (payload !== undefined) yielded = payload;
+    const text = reportText(content);
+    if (text !== "") report = text;
+    const usage = field(message, "usage");
+    const cost = usage === null ? null : field(usage, "cost");
+    const total = cost === null ? null : field(cost, "total");
+    if (typeof total === "number") spendUsd += total;
+    const messageModel = field(message, "model");
+    if (typeof messageModel === "string" && messageModel !== "") model = messageModel;
+    const messageProvider = field(message, "provider");
+    if (typeof messageProvider === "string" && messageProvider !== "") provider = messageProvider;
+  });
+  session.on("agent_end", (event) => {
+    // Same rule as every other session runner: a non-terminal `agent_end`
+    // means the harness will resume, so completing there would read a
+    // mid-flight snapshot as the answer.
+    if (shouldComplete(event as { isTerminal?: boolean })) finished.resolve();
+  });
+  session.on("session_exit", () => finished.resolve());
+  try {
+    await session.prompt(o.brief);
+    await finished.promise;
+  } finally {
+    await disposeSession(session);
+  }
+  return {
+    // The yielded payload first: under `requireYieldTool` that IS the answer,
+    // and a scout's prose is usually a sentence about it rather than the
+    // contract. The text stands in when nothing was yielded, so a session
+    // that answered in a fenced JSON block still persists as a verdict and
+    // one that only chatted persists as `blocked(malformed)` — which is the
+    // honest record of a spent pass, not a discard.
+    raw: yielded !== "" ? yielded : report,
+    report,
+    ...(model === undefined ? {} : { model }),
+    ...(provider === undefined ? {} : { provider }),
+    turns,
+    spendUsd,
+  };
+}
+
+/**
+ * The `yield` call's payload, serialized, or `undefined` when this message
+ * carried none (#1041).
+ *
+ * Transport, not decoding: it walks to `arguments.result.data` and stringifies
+ * whatever object is there without reading one field of it. That is the
+ * difference from {@link structuredVerdict} and {@link structuredSettlement},
+ * which validate because their callers act on the parse — here the strict
+ * to-spec parser at persistence time is the only validator, and a shape this
+ * function rejected would be a verdict silently downgraded to "no answer".
+ */
+function yieldedPayload(content: unknown): string | undefined {
+  if (!Array.isArray(content)) return undefined;
+  let payload: string | undefined;
+  for (const block of content) {
+    if (field(block, "type") !== "toolCall") continue;
+    if (field(block, "name") !== "yield") continue;
+    const result = field(field(block, "arguments"), "result");
+    const data = field(result, "data");
+    if (data === null || data === undefined || typeof data !== "object") continue;
+    payload = JSON.stringify(data);
+  }
+  return payload;
+}
+
 /**
  * Render a parsed {@link WorkerSettlement} as the run's stored report (#540).
  *
@@ -832,6 +1274,13 @@ export function renderSettlement(settlement: WorkerSettlement): string {
     lines.push("", "proof:");
     for (const item of settlement.proof) lines.push(`  - ${item}`);
   }
+  // Rendered like the other list fields so the stored report shows what was
+  // appended to the repo's knowledge overlay: an entry a reader disagrees with
+  // is traceable to the run that claimed it.
+  if (settlement.discoveries !== undefined && settlement.discoveries.length > 0) {
+    lines.push("", "discoveries:");
+    for (const item of settlement.discoveries) lines.push(`  - ${item}`);
+  }
   return lines.join("\n");
 }
 
@@ -1385,6 +1834,12 @@ export async function runWorker(
       spendUsd,
       provider429Count,
       report: renderSettlement(structured),
+      // The only exit that can carry them: a killed or unparseable run yielded
+      // no settlement, and inventing an empty list there would tell the
+      // knowledge overlay that the fleet asked and learned nothing.
+      ...(structured.discoveries === undefined || structured.discoveries.length === 0
+        ? {}
+        : { discoveries: structured.discoveries }),
     }));
   }