@cat-factory/executor-harness 1.58.0 → 1.62.0

package/src/host-markdown.ts

@@ -0,0 +1,155 @@
+// ---------------------------------------------------------------------------
+// The TEXT BOUNDARY for agent-authored text the harness writes onto a VCS host.
+//
+// A pull-request description is NOT an inert string sink. The host parses it: `#123` becomes
+// an issue link, `@name` notifies a real person, a closing keyword in front of an issue
+// reference CLOSES that issue when the PR merges, and an unbalanced code fence swallows
+// everything rendered after it — including the fenced JSON block the engine later appends as
+// the verification report's machine-readable contract.
+//
+// The agent's reviewer briefing (`pr-description.ts`) is model-authored prose that lands
+// verbatim on that surface, so it crosses this boundary first. "This closes #42" is idiomatic
+// for a briefing to emit and must not close issue 42; "@alice owns the rounding rule" is
+// idiomatic and must not page whoever holds that handle.
+//
+// This is a deliberate COPY of `hostMarkdown` in `@cat-factory/kernel`
+// (`src/shared/host-markdown.logic.ts`), for the same reason `isSafeTestPath` is copied: the
+// container image is built from `src/` plus typescript alone (the Dockerfile cannot resolve a
+// `workspace:*` dependency), so the harness carries no runtime dependency on any package here.
+// `test/host-markdown.conformity.test.ts` pins the two implementations to byte-identical
+// output over a shared corpus, so the copy cannot drift — change one, change the other.
+// ---------------------------------------------------------------------------
+
+/**
+ * The host's closing keywords. A PR body carrying one of these in front of an issue reference
+ * closes that issue on merge — a side effect the harness must never trigger on the agent's
+ * behalf. Same list on GitHub and GitLab.
+ */
+const CLOSING_KEYWORDS =
+  'close[sd]?|closing|fix|fixe[sd]|fixing|resolve[sd]?|resolving|implement(?:s|ed)?|implementing'
+
+/** An issue/MR URL on either host, in the form a closing keyword can reference. */
+const ISSUE_URL = String.raw`https?://\S+?/(?:issues|-/issues|merge_requests|pull)/\d+`
+
+/**
+ * Every auto-linking trigger, in ONE alternation.
+ *
+ * Deliberately a single pass rather than chained `.replace()` calls: each escape EMITS a `#`,
+ * so a later rule would re-escape the output of an earlier one (`@` → `@` → `@`).
+ * One regex means the replacement text is never rescanned.
+ */
+const AUTO_LINK_TRIGGERS = new RegExp(
+  [
+    // A closing keyword in front of an issue/MR URL. The URL form survives the character
+    // escapes below (nothing in it is a trigger), so the KEYWORD is what gets defused.
+    String.raw`(?<keyword>\b(?:${CLOSING_KEYWORDS}))(?=\s*:?\s+${ISSUE_URL})`,
+    // `@name` / `@org/team` — a mention notifies a real account.
+    String.raw`(?<at>@(?=[A-Za-z0-9]))`,
+    // `#123` and `owner/repo#123` — an issue/PR cross-reference.
+    String.raw`(?<hash>#(?=\d))`,
+    // `!123` — GitLab's merge-request reference.
+    String.raw`(?<bang>!(?=\d))`,
+  ].join('|'),
+  'gi',
+)
+
+/**
+ * Neutralise the host's auto-linking triggers in ONE line of untrusted text, leaving inline
+ * code spans alone (the host does not auto-link inside them, so escaping there would only
+ * show the reader a literal `&#35;`).
+ *
+ * The escapes are numeric HTML entities, which render as the original character but are
+ * invisible to the reference parser — so the reader sees exactly what the agent wrote while
+ * the mention/close side effects are defused.
+ */
+function inertLine(line: string): string {
+  return mapOutsideCodeSpans(line, (text) =>
+    text.replace(AUTO_LINK_TRIGGERS, (match, ...args) => {
+      const groups = args[args.length - 1] as Record<string, string | undefined>
+      // Entity-escaping the FIRST character is enough to break the parser's match while
+      // rendering identically — which matters most for the keyword, whose remaining letters
+      // are ordinary prose the reader should still see.
+      return `&#${match.charCodeAt(0)};${groups.keyword ? match.slice(1) : ''}`
+    }),
+  )
+}
+
+/**
+ * Apply `fn` to the parts of `line` that are NOT inline code spans. Code spans are matched by
+ * a backtick run and its matching closer, which is CommonMark's rule and — more to the point
+ * — the rule the host renderer applies when deciding where to auto-link.
+ */
+function mapOutsideCodeSpans(line: string, fn: (text: string) => string): string {
+  const out: string[] = []
+  let index = 0
+  const span = /(`+)[\s\S]*?\1/g
+  let match: RegExpExecArray | null
+  while ((match = span.exec(line)) !== null) {
+    out.push(fn(line.slice(index, match.index)), match[0])
+    index = match.index + match[0].length
+  }
+  return out.join('') + fn(line.slice(index))
+}
+
+/** A line that opens or closes a fenced code block, with the fence it uses. */
+function fenceAt(line: string): { char: string; length: number; info: boolean } | null {
+  const match = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line)
+  if (!match) return null
+  const fence = match[1]!
+  // A ``` fence's info string may not contain a backtick (CommonMark), which is what stops an
+  // inline span from being read as a fence.
+  if (fence.startsWith('`') && match[2]!.includes('`')) return null
+  return { char: fence[0]!, length: fence.length, info: match[2]!.trim().length > 0 }
+}
+
+/**
+ * Walk `lines`, tracking fenced-code state, and hand each line to `visit` together with
+ * whether it sits INSIDE a fenced block. Returns the fence still open at the end, if any.
+ *
+ * One shared walker so the three things that care about fences — leaving code untouched,
+ * closing what the text left open, and finding the briefing's title heading — can never
+ * disagree about where a block starts and ends.
+ */
+export function walkFences(
+  lines: readonly string[],
+  visit: (line: string, insideFence: boolean) => void,
+): { char: string; length: number } | null {
+  let open: { char: string; length: number } | null = null
+  for (const line of lines) {
+    const fence = fenceAt(line)
+    // The fence line itself belongs to the code block, so it is never rewritten.
+    visit(line, open !== null || fence !== null)
+    if (!fence) continue
+    if (!open) open = { char: fence.char, length: fence.length }
+    else if (fence.char === open.char && fence.length >= open.length && !fence.info) open = null
+  }
+  return open
+}
+
+/**
+ * Render untrusted multi-line markdown safe to send to a host: auto-link triggers defused
+ * outside fenced code, and any fence the text leaves open closed again.
+ *
+ * Unlike kernel's `hostMarkdown.prose` this does NOT cap the length — the caller
+ * ({@link import('./pr-description.js')}) applies its own budget with its own visible note
+ * BEFORE calling here, so an escape entity can never be sliced in half. With that one
+ * difference the output is identical, which the conformity test pins.
+ */
+export function inertMarkdown(text: string): string {
+  const normalised = text.replace(/\r\n?/g, '\n')
+  const rewritten: string[] = []
+  const open = walkFences(normalised.split('\n'), (line, insideFence) => {
+    rewritten.push(insideFence ? line : inertLine(line))
+  })
+  const joined = rewritten.join('\n')
+  return open ? `${joined}\n${open.char.repeat(open.length)}` : joined
+}
+
+/**
+ * Render untrusted text INLINE (a pull-request title): newlines folded to spaces because the
+ * surrounding line has its own meaning, and auto-link triggers defused. The caller caps the
+ * length first, for the same reason as {@link inertMarkdown}.
+ */
+export function inertInline(text: string): string {
+  return inertLine(text.replace(/\s+/g, ' '))
+}

package/src/job.ts

@@ -2,7 +2,16 @@ import type { HarnessCallMetric, PiRunStats } from './pi.js'
 import type { HarnessKind } from './pi-workspace.js'
 import type { FailureCause } from './failure.js'
 import type { EffortReport } from './effort.js'
-import type { ValidationChecksSpec, ValidationReport } from './validation-checks.js'
+import {
+  parseValidationChecksSpec,
+  type ValidationChecksSpec,
+  type ValidationReport,
+} from './validation-checks.js'
+import {
+  parseReproductionSpec,
+  type ReproductionReport,
+  type ReproductionSpec,
+} from './reproduction-proof.js'
 
 // The job the Worker's ContainerAgentExecutor POSTs to /run. Kept as plain
 // types with a hand-rolled validator so the image needs no schema dependency.
@@ -175,38 +184,6 @@ function parseValidationSpec(value: unknown): ValidationSpec | undefined {
   }
 }
 
-/**
- * Parse the optional PRE-PR VALIDATION CHECKS spec (see
- * docs/initiatives/pre-pr-validation.md): the service's ordered `{ label, command }` pairs and
- * the repair-round budget. Every entry needs a non-empty command; entries without one are
- * dropped, and a spec that ends up with no usable check returns `undefined` — so a malformed
- * body degrades to the exact pre-feature behaviour (no loop, PR opens as before) rather than
- * failing an otherwise-good coding run. `maxAttempts` is clamped to a sane range so a bad body
- * can't make a container loop forever.
- */
-function parseValidationChecksSpec(value: unknown): ValidationChecksSpec | undefined {
-  if (typeof value !== 'object' || value === null) return undefined
-  const o = value as Record<string, unknown>
-  if (!Array.isArray(o.checks)) return undefined
-  const checks: { label: string; command: string }[] = []
-  for (const raw of o.checks) {
-    if (typeof raw !== 'object' || raw === null) continue
-    const c = raw as Record<string, unknown>
-    if (typeof c.command !== 'string' || c.command.trim() === '') continue
-    const label = typeof c.label === 'string' && c.label.trim() ? c.label.trim() : c.command
-    checks.push({ label, command: c.command })
-  }
-  if (checks.length === 0) return undefined
-  const parsed = posInt(o.maxAttempts)
-  return {
-    checks,
-    maxAttempts: Math.min(
-      parsed ?? VALIDATION_DEFAULT_MAX_ATTEMPTS,
-      VALIDATION_MAX_ATTEMPTS_CEILING,
-    ),
-  }
-}
-
 /**
  * Parse the shared per-job auth fields, validating per harness: a subscription
  * harness (`claude-code` / `codex`) requires `subscriptionToken`; the default Pi
@@ -872,21 +849,18 @@ export interface AgentJob extends HarnessAuthFields {
    * job DATA, not the agent kind. See {@link ValidationChecksSpec}.
    */
   validationChecks?: ValidationChecksSpec
+  /**
+   * Coding mode: the run's BUGFIX REPRODUCTION PROOF — the declared reproduction command, the
+   * test file(s) that constitute it, and an optional setup command. When set, the harness runs
+   * that command against the pre-fix tree AND the tree the PR will open from, and reports both
+   * exit codes: only red-then-green is proof. Present only on a dispatch that opens a PR and
+   * whose run carries a reproduction declaration; absent ⇒ the run behaves exactly as before.
+   * Deliberately keyed off job DATA, not the agent kind. See
+   * `docs/initiatives/bugfix-reproduction-proof.md`.
+   */
+  reproduction?: ReproductionSpec
 }
 
-/**
- * The ceiling the harness clamps a body-supplied `validationChecks.maxAttempts` to, and the
- * default it applies when the body omits one.
- *
- * DELIBERATE DUPLICATES of `VALIDATION_MAX_ATTEMPTS_CEILING` / `VALIDATION_DEFAULT_MAX_ATTEMPTS`
- * in `@cat-factory/contracts` — the published image takes no schema dependency, so the harness
- * cannot import them. Keep the two in step: the API validates writes against the contracts
- * values, so a harness clamping to a DIFFERENT ceiling would silently cap a budget an operator
- * was allowed to save, with nothing to flag the mismatch.
- */
-export const VALIDATION_MAX_ATTEMPTS_CEILING = 10
-export const VALIDATION_DEFAULT_MAX_ATTEMPTS = 3
-
 /** Per-job, per-knob progress-guard overrides (see {@link AgentJob.guardLimits}). */
 export interface GuardLimitsSpec {
   maxToolCallsWithoutEdit?: number
@@ -938,6 +912,14 @@ export interface AgentResult {
    * Absent when the job carried no {@link AgentJob.validationChecks}.
    */
   validationReport?: ValidationReport
+  /**
+   * The BUGFIX REPRODUCTION PROOF: the declared reproduction command's verdict across the pre-fix
+   * tree and the final tree, computed by the harness from exit codes. Present on every outcome of
+   * a job that carried {@link AgentJob.reproduction} — a verdict is evidence, not a gate, so an
+   * `inconclusive` one accompanies the opened PR exactly like a `reproduced` one does. Absent
+   * when the job carried no reproduction declaration.
+   */
+  reproductionReport?: ReproductionReport
   /**
    * Preview mode: the in-container URL the built app is served at (e.g. `http://localhost:4173`).
    * This is NOT host-reachable on its own — the container runtime publishes the serve port to an
@@ -1345,6 +1327,7 @@ export function parseAgentJob(input: unknown): AgentJob {
     guardLimits: parseGuardLimits(o.guardLimits),
     validation: parseValidationSpec(o.validation),
     validationChecks: parseValidationChecksSpec(o.validationChecks),
+    reproduction: parseReproductionSpec(o.reproduction),
     reviewPrNumber: posInt(o.reviewPrNumber),
   })
   assertAllowedHost(job.repo.cloneUrl, 'repo.cloneUrl')
@@ -1383,6 +1366,7 @@ interface ParsedAgentJobParts {
   guardLimits: ReturnType<typeof parseGuardLimits>
   validation: ReturnType<typeof parseValidationSpec>
   validationChecks: ReturnType<typeof parseValidationChecksSpec>
+  reproduction: ReturnType<typeof parseReproductionSpec>
   reviewPrNumber: number | undefined
 }
 
@@ -1436,6 +1420,7 @@ function assembleAgentJob(
     guardLimits,
     validation,
     validationChecks,
+    reproduction,
     reviewPrNumber,
   } = parts
   const repo = (o.repo ?? {}) as Record<string, unknown>
@@ -1465,6 +1450,7 @@ function assembleAgentJob(
     ...(guardLimits ? { guardLimits } : {}),
     ...(validation ? { validation } : {}),
     ...(validationChecks ? { validationChecks } : {}),
+    ...(reproduction ? { reproduction } : {}),
   }
 }
 

package/src/pr-description.ts

@@ -0,0 +1,171 @@
+import { readFile, rm } from 'node:fs/promises'
+import { join } from 'node:path'
+import { inertInline, inertMarkdown, walkFences } from './host-markdown.js'
+import { redactSecrets } from './redact.js'
+
+// ---------------------------------------------------------------------------
+// The agent-authored pull-request description side channel. A coding agent whose
+// dispatch opens a PR is asked (via the backend-composed system prompt) to end its
+// run by writing a reviewer briefing — the problem, the decisions made, what to
+// look out for — to a sentinel file at the root of the checkout the PR belongs to.
+// The harness reads it after the agent settles, removes it (so it never lands in a
+// commit), and uses it as the PR body in place of the generic dispatch-time text
+// the job body carries. Absent or unusable ⇒ the dispatch-time fallback, unchanged.
+//
+// The briefing is MODEL-AUTHORED text landing verbatim on a host-parsed surface, so
+// it crosses `host-markdown.ts` (auto-link triggers defused, open code fences closed)
+// on the way out — see that module for why a PR body is not an inert string sink.
+//
+// The filename is kept in sync with `PR_DESCRIPTION_FILE` in `@cat-factory/agents`
+// (the executor-harness has no dependency on that package), exactly like the
+// effort-report and follow-ups sentinels.
+// ---------------------------------------------------------------------------
+
+/** The sentinel file the agent writes its PR description to (relative to the checkout root). */
+export const PR_DESCRIPTION_FILE = '.cat-pr-description.md'
+
+/**
+ * Ceiling on the agent-authored body.
+ *
+ * The engine appends its verification report to the SAME body later, and that section carries
+ * its own 50,000-character ceiling (`MAX_SECTION_CHARS` in kernel's `hostMarkdown`). GitHub
+ * rejects a body over 65,536 with a 422, and the report publisher swallows its own failures —
+ * so a briefing budget that does not leave the report room would surface as a report that
+ * silently never publishes. 15,000 + 50,000 stays under the limit with room to join them.
+ */
+const MAX_PR_BODY_CHARS = 15_000
+
+/** Ceiling on an agent-supplied title (GitHub truncates around 256; a title should be short). */
+const MAX_PR_TITLE_CHARS = 160
+
+/** Opens the engine-managed region of a PR body (kept in sync with `kernel/domain/pr-report.ts`). */
+export const PR_REPORT_MARKER_START = '<!-- cat-factory:verification-report:start -->'
+/** Closes the engine-managed region of a PR body. */
+export const PR_REPORT_MARKER_END = '<!-- cat-factory:verification-report:end -->'
+
+/**
+ * A marker inside the agent-authored briefing would make the engine's splice treat part of the
+ * briefing as its own managed region and rewrite it, so any occurrence is stripped up front.
+ * Deliberately laxer than the exact constants above (whitespace-tolerant), so a near-miss the
+ * splice itself would not match cannot survive here either.
+ */
+const MANAGED_SECTION_MARKER = /<!--\s*cat-factory:verification-report:(?:start|end)\s*-->/g
+
+/** An agent-authored PR description: an optional title plus the briefing body. */
+export interface AgentPrDescription {
+  title?: string
+  body?: string
+}
+
+/**
+ * Read + parse + REMOVE the agent's PR-description sentinel from `dir`. Lenient: returns
+ * undefined when the file is absent (the agent wrote none) or carries nothing usable. Never
+ * throws — a bad description must never fail an otherwise-good run; the caller falls back to
+ * the dispatch-time text.
+ *
+ * A SINGLE `# <title>` heading on the first line sets the PR title; everything after it is the
+ * body (see {@link splitTitle} for why a LONE heading is required). The whole text is
+ * secret-scrubbed, an over-budget body is truncated WITH a visible note (a silent cut would
+ * read as the complete briefing), and both halves are made inert for the host.
+ *
+ * On scrubbing: `redactSecrets`'s credential-assignment rule is deliberately eager, so a
+ * briefing sentence like "the token: handling changed" loses its next word. That is the right
+ * trade for a surface this public — the rule is shared with every other redaction path, and
+ * narrowing it so prose reads better would weaken all of them.
+ */
+export async function readPrDescription(dir: string): Promise<AgentPrDescription | undefined> {
+  const path = join(dir, PR_DESCRIPTION_FILE)
+  let raw: string
+  try {
+    raw = await readFile(path, 'utf8')
+  } catch {
+    return undefined // no description written — the fallback body applies
+  }
+  // Remove it so it never lands in a commit (defence in depth; the checkout also excludes it).
+  await rm(path, { force: true }).catch(() => {})
+  const text = redactSecrets(raw).replace(MANAGED_SECTION_MARKER, '').trim()
+  if (!text) return undefined
+
+  const split = splitTitle(text)
+  // Cap BEFORE the escapes on both halves, so a numeric entity can never be sliced in half.
+  const title = split.title ? inertInline(capTitle(split.title)) : undefined
+  const body = split.body ? inertMarkdown(capBody(split.body)) : undefined
+  if (!title && !body) return undefined
+  return { ...(title ? { title } : {}), ...(body ? { body } : {}) }
+}
+
+/**
+ * Split a leading `# <title>` heading off the briefing.
+ *
+ * The heading becomes the title ONLY when it is the single level-1 heading in the whole file,
+ * which is exactly what the prompt asks for ("a single `# <title>` heading line"). An agent
+ * that instead uses `#` for its section headings — `# Problem`, `# Decisions`, entirely
+ * idiomatic for the briefing the prompt describes — would otherwise have its first section
+ * silently become the pull request's title, replacing `<block> (<pipeline>)` with the word
+ * "Problem". Headings inside fenced code are not headings and are skipped, or a briefing
+ * quoting a shell snippet (`# rebuild the image`) would lose its title to the snippet.
+ */
+function splitTitle(text: string): { title?: string; body: string } {
+  const lines = text.split('\n')
+  const headings: number[] = []
+  let index = 0
+  walkFences(lines, (line, insideFence) => {
+    if (!insideFence && /^#\s+\S/.test(line)) headings.push(index)
+    index += 1
+  })
+  if (headings.length !== 1 || headings[0] !== 0) return { body: text }
+  const title = lines[0]!.replace(/^#\s+/, '').trim()
+  if (!title) return { body: text }
+  return { title, body: lines.slice(1).join('\n').trim() }
+}
+
+/** Cut an over-long title at a word boundary when one is near, marking the cut. */
+function capTitle(value: string): string {
+  const collapsed = value.trim()
+  if (collapsed.length <= MAX_PR_TITLE_CHARS) return collapsed
+  const head = collapsed.slice(0, MAX_PR_TITLE_CHARS - 1)
+  const space = head.lastIndexOf(' ')
+  const kept = space > MAX_PR_TITLE_CHARS * 0.6 ? head.slice(0, space) : head
+  return `${kept.trimEnd()}…`
+}
+
+/** Cut an over-budget body, marking the cut so it is never read as the whole briefing. */
+function capBody(value: string): string {
+  if (value.length <= MAX_PR_BODY_CHARS) return value
+  return (
+    value.slice(0, MAX_PR_BODY_CHARS).trimEnd() +
+    '\n\n_Truncated by the platform: the description exceeded the size budget._'
+  )
+}
+
+/**
+ * Fold an agent-authored description over the dispatch-time fallback the job body carries.
+ * Field-wise: the agent's title/body each win when present, so a body-only briefing keeps the
+ * backend-composed title and vice versa.
+ */
+export function applyPrDescription(
+  fallback: { title: string; body: string },
+  agent: AgentPrDescription | undefined,
+): { title: string; body: string } {
+  if (!agent) return fallback
+  return { title: agent.title ?? fallback.title, body: agent.body ?? fallback.body }
+}
+
+/**
+ * The body to PATCH onto an ALREADY-OPEN pull request when a resumed run produced a fresh
+ * briefing: the new description followed by whatever the engine's managed verification-report
+ * region currently holds.
+ *
+ * Carrying the region across is what makes the refresh safe. The engine re-publishes the report
+ * on every step settlement, so dropping it here would usually self-heal — but "usually" is not
+ * a property to rest the one artefact a reviewer reads on, and a run that settles no further
+ * step (the work is already merged, the run failed after its push) would never restore it.
+ */
+export function preserveManagedSection(currentBody: string | undefined, nextBody: string): string {
+  const existing = currentBody ?? ''
+  const start = existing.indexOf(PR_REPORT_MARKER_START)
+  const end = existing.indexOf(PR_REPORT_MARKER_END)
+  if (start === -1 || end <= start) return nextBody
+  const region = existing.slice(start, end + PR_REPORT_MARKER_END.length)
+  return `${nextBody.trim()}\n\n${region}\n`
+}