@cat-factory/executor-harness 1.54.0 → 1.56.0

package/README.md

@@ -59,7 +59,13 @@ The implementation job (`POST /run`) is the canonical sequence:
    Pi reads and concatenates both), and point Pi at the Worker's LLM proxy via
    `~/.pi/agent/models.json` (provider `proxy`, `api: openai-completions`),
 3. **run Pi** non-interactively (`pi -p --mode json --model proxy/<model> --approve`),
-4. **commit, push** a branch and **open a PR**, returning `{ prUrl, branch, summary }`.
+4. **validate** the checkout, when the job body carries `validationChecks` — the service's
+   configured check commands (install/lint/test/build) run with `sh -c` in the checkout, and
+   while they fail and the attempt budget remains the agent is re-run with the captured output
+   as its instruction (see [pre-PR validation](../../../docs/initiatives/pre-pr-validation.md)),
+5. **commit, push** a branch and **open a PR**, returning `{ prUrl, branch, summary }` — but
+   ONLY if step 4 ended green. A spent budget returns an error result with the validation report
+   and opens no PR. Absent `validationChecks`, step 4 does not happen at all.
 
 Bootstrap differs at the ends — it may start from an empty dir, and **resets
 history to one commit and force-pushes** the default branch instead of opening a
@@ -127,6 +133,7 @@ Kimi / DeepSeek) and meters spend. The provider key never enters the container.
 | `src/package-registries.ts` | Private-registry (npm) auth: renders the job's allowlisted entries into an npmrc — the user `~/.npmrc` in a container, a per-job file pointed at by `npm_config_userconfig` for a native job. |
 | `src/agent-runner.ts` | The subscription-harness runners (`runClaudeCode` / `runCodex`) — talk direct to the vendor with a leased OAuth token, lift per-turn usage/telemetry off the CLI event stream. |
 | `src/transcript-retention.ts` | Lifts the CLI session transcripts (`projects/` / `sessions/`) out of the isolated, credential-bearing config home before it is deleted, and prunes them on a TTL (debugging artifact retention). |
+| `src/validation-checks.ts` | Pre-PR validation: runs the job's check commands in the checkout (bounded, secret-scrubbed capture, per-command watchdog) and drives the retry-until-green loop that gates the PR. Generic — keyed off the job body, never the agent kind. |
 | `src/logger.ts`    | Structured logging.                                                                                     |
 
 ## Runner lifecycle knobs
@@ -139,6 +146,7 @@ runner):
 | `PORT`                | `8080`          | HTTP port the harness listens on.                           |
 | `JOB_MAX_DURATION_MS` | `3600000` (60m) | Hard ceiling on a job's wall-clock time; force-fails after. |
 | `JOB_INACTIVITY_MS`   | `600000` (10m)  | Kills a hung agent that produces no output for this long.   |
+| `VALIDATION_COMMAND_TIMEOUT_MS` | `900000` (15m) | Per-command watchdog for a pre-PR validation check; a timeout counts as a failure (exit 124) so one hung command can't wedge the loop. |
 | `HARNESS_TRANSCRIPT_TTL_MS` | `259200000` (3d) | How long lifted subscription-CLI session transcripts are kept before the retention sweep prunes them. |
 | `HARNESS_TRANSCRIPT_ROOT`   | `<tmpdir>/cf-agent-transcripts` | Where retained session transcripts are moved to (one dir per run). Meaningful only on a reused (warm-pool) container; a per-run container is torn down with the job. The TTL sweep deletes only dirs it created (each carries a `.cf-retained` marker), so pointing this at a shared directory never touches unrelated content — though a dedicated dir is still recommended. An override on a different filesystem than the config home falls back to copy-then-remove. |
 

package/dist/agent.js

@@ -8,6 +8,7 @@ import { configurePackageRegistries } from './package-registries.js';
 import { captureRedactedOutput, redactSecrets, registerKnownSecrets } from './redact.js';
 import { cloneRepo, commitAll, conflictDiff, fetchPullRequestHead, fetchReferenceBranches, hasAgentChanges, headCommit, inferVcsProvider, mergeBranch, openPullRequest, prepareExistingCheckout, pushBranch, reinitAndPush, unmergedPaths, } from './git.js';
 import { makeDirClaimer, noChangesReason, runCodingAgent, runMultiRepoCoding, } from './coding-agent.js';
+import { validationFailureMessage } from './validation-checks.js';
 import { acquireRepoCheckout, agentNeverActed, agentOutputTail, NEVER_ACTED_CAUSE, runAgentInWorkspace, unusableFinalAnswerCause, withWorkspace, } from './pi-workspace.js';
 import { diagnosticsSuffix, resolveStructuredOutput, } from './structured-output.js';
 import { log } from './logger.js';
@@ -802,6 +803,11 @@ function buildSingleRepoCodingSpec(job, pushBranch) {
                 },
             }
             : {}),
+        // Pre-PR validation: the service's check commands, run against the checkout BEFORE the PR
+        // opens with failures fed back to the agent (see docs/initiatives/pre-pr-validation.md).
+        // Forwarded straight off the job body — the loop is generic machinery keyed on the data, not
+        // on the agent kind.
+        ...(job.validationChecks ? { validationChecks: job.validationChecks } : {}),
     };
 }
 /**
@@ -812,12 +818,38 @@ function buildSingleRepoCodingSpec(job, pushBranch) {
  */
 async function runSingleRepoCoding(job, opts) {
     const pushBranch = job.pushBranch ?? job.newBranch ?? job.branch;
-    const { summary, stats, stderrTail, pushed, usage, callMetrics, validation, effortReport } = await runCodingAgent(buildSingleRepoCodingSpec(job, pushBranch), opts);
+    const { summary, stats, stderrTail, pushed, usage, callMetrics, validation, validationReport, effortReport, } = await runCodingAgent(buildSingleRepoCodingSpec(job, pushBranch), opts);
     // Ralph loop: the harness-computed validation verdict, forwarded onto the coding result as
     // `ralphVerdict` so the backend's `toRunResult` lifts it onto `AgentRunResult.ralphVerdict`.
     const ralphVerdict = validation ? { ralphVerdict: validation } : {};
     // The agent's effort self-assessment, spread onto every result path below (mirrors ralphVerdict).
     const effort = effortReport ? { effortReport } : {};
+    // The pre-PR validation report, spread onto every result path below: on the passing path it is
+    // the captured proof the checkout was green when the PR opened; on the exhausted path it is the
+    // evidence behind the failure below. Absent when the service configured no checks.
+    const validationFields = validationReport ? { validationReport } : {};
+    // Pre-PR validation spent its attempt budget with the checkout still red. FAIL the job — do
+    // NOT open a pull request, and do not pretend the push succeeded as a deliverable. The work is
+    // still on the branch (a retry resumes on it); the report carries each failing command's exit
+    // code and captured output so the step's failure detail says exactly what broke.
+    if (validationReport && !validationReport.passed) {
+        return {
+            // The work IS on the branch (the loop only runs for a pass that produced some, and the
+            // harness pushes it) — a retry resumes on top of it. `error` is what marks the job failed;
+            // reporting `pushed: false` here would misdescribe the branch state in the harness's own
+            // result for no benefit.
+            pushed,
+            branch: pushBranch,
+            summary,
+            stats,
+            error: validationFailureMessage(validationReport),
+            failureCause: 'agent',
+            ...(usage ? { usage } : {}),
+            ...(callMetrics ? { callMetrics } : {}),
+            ...validationFields,
+            ...effort,
+        };
+    }
     if (!pushed) {
         // A no-op: a failure for the implementer, a clean non-event for the fixers.
         if (job.noChangesIsError === false) {
@@ -829,6 +861,7 @@ async function runSingleRepoCoding(job, opts) {
                 ...(usage ? { usage } : {}),
                 ...(callMetrics ? { callMetrics } : {}),
                 ...ralphVerdict,
+                ...validationFields,
                 ...effort,
             };
         }
@@ -841,6 +874,7 @@ async function runSingleRepoCoding(job, opts) {
             failureCause: 'no-changes',
             ...(usage ? { usage } : {}),
             ...(callMetrics ? { callMetrics } : {}),
+            ...validationFields,
             ...effort,
         };
     }
@@ -874,6 +908,7 @@ async function runSingleRepoCoding(job, opts) {
                     stats,
                     ...(usage ? { usage } : {}),
                     ...(callMetrics ? { callMetrics } : {}),
+                    ...validationFields,
                     ...effort,
                 };
             }
@@ -886,6 +921,7 @@ async function runSingleRepoCoding(job, opts) {
                 failureCause: 'no-changes',
                 ...(usage ? { usage } : {}),
                 ...(callMetrics ? { callMetrics } : {}),
+                ...validationFields,
                 ...effort,
             };
         }
@@ -898,6 +934,7 @@ async function runSingleRepoCoding(job, opts) {
             ...(usage ? { usage } : {}),
             ...(callMetrics ? { callMetrics } : {}),
             ...ralphVerdict,
+            ...validationFields,
             ...effort,
         };
     }
@@ -909,6 +946,7 @@ async function runSingleRepoCoding(job, opts) {
         ...(usage ? { usage } : {}),
         ...(callMetrics ? { callMetrics } : {}),
         ...ralphVerdict,
+        ...validationFields,
         ...effort,
     };
 }

package/dist/coding-agent.js

@@ -8,6 +8,7 @@ import { FOLLOW_UPS_FILENAME, FollowUpTailer } from './follow-ups.js';
 import { EFFORT_REPORT_FILE } from './effort.js';
 import { acquireRepoCheckout, agentNeverActed, agentOutputTail, runAgentInWorkspace, withWorkspace, } from './pi-workspace.js';
 import { log } from './logger.js';
+import { runValidationLoop, } from './validation-checks.js';
 /**
  * How often the harness checkpoints the agent's work mid-run by pushing the branch.
  * A per-run container can be evicted at any moment; pushing the agent's commits
@@ -128,28 +129,57 @@ export async function runCodingAgent(spec, opts = {}) {
             }, followUpPollIntervalMs());
             followUpTick.unref?.();
         }
+        // One agent pass over this checkout, parameterised only by the prompt — so the pre-PR
+        // validation loop below can re-run the agent with a repair instruction without
+        // re-deriving (or drifting from) the dispatch's own settings.
+        const runAgentPass = (userPrompt) => runAgentInWorkspace({
+            dir: workDir,
+            systemPrompt: spec.systemPrompt,
+            userPrompt,
+            model: spec.model,
+            harness: spec.harness,
+            subscriptionToken: spec.subscriptionToken,
+            subscriptionBaseUrl: spec.subscriptionBaseUrl,
+            ambientAuth: spec.ambientAuth,
+            proxyBaseUrl: spec.proxyBaseUrl,
+            sessionToken: spec.sessionToken,
+            serviceDirectory,
+            webToolsGuidance: spec.webToolsGuidance,
+            webSearchProxy: spec.webSearchProxy,
+            guardLimits: spec.guardLimits,
+            ...(spec.skill ? { skill: spec.skill } : {}),
+        }, opts);
         let outcome;
         try {
             opts.onPhase?.('agent');
             logger.info('coding-agent: running agent', { serviceDirectory });
-            const agentRun = await runAgentInWorkspace({
-                dir: workDir,
-                systemPrompt: spec.systemPrompt,
-                userPrompt: spec.userPrompt,
-                model: spec.model,
-                harness: spec.harness,
-                subscriptionToken: spec.subscriptionToken,
-                subscriptionBaseUrl: spec.subscriptionBaseUrl,
-                ambientAuth: spec.ambientAuth,
-                proxyBaseUrl: spec.proxyBaseUrl,
-                sessionToken: spec.sessionToken,
-                serviceDirectory,
-                webToolsGuidance: spec.webToolsGuidance,
-                webSearchProxy: spec.webSearchProxy,
-                guardLimits: spec.guardLimits,
-                ...(spec.skill ? { skill: spec.skill } : {}),
-            }, opts);
+            let agentRun = await runAgentPass(spec.userPrompt);
+            // PRE-PR VALIDATION: run the service's configured checks against the checkout and, while
+            // they fail and budget remains, hand the captured output back to the agent and run it
+            // again. Sits BETWEEN the agent and the finalize/push/PR step so a red checkout never
+            // reaches `openPullRequest` — the whole point of the feature. Keyed purely off the job
+            // body carrying checks (no agent-kind switch); absent ⇒ this is a no-op and the flow
+            // below is byte-for-byte what it was.
+            const validationChecks = spec.validationChecks;
+            let validationReport;
+            if (validationChecks && (await producedWork(dir, spec, baseSha, resumed, opts))) {
+                validationReport = await runValidationLoop({
+                    workDir,
+                    spec: validationChecks,
+                    logger,
+                    opts,
+                    runAgentPass,
+                    onAgentPass: (run) => {
+                        agentRun = mergeAgentPasses(agentRun, run);
+                    },
+                    // The checks run against the WORKING TREE, but only tracked edits are staged for the
+                    // push — so a repair round can go green on a new file the PR would never contain.
+                    // Name those files in the next repair prompt so the agent adds them.
+                    listUncommittedNewFiles: () => listUntrackedFiles(workDir, opts.signal),
+                });
+            }
             outcome = await finalizeCodingRun({
+                validationReport,
                 dir,
                 spec,
                 logger,
@@ -284,7 +314,7 @@ async function prepareCodingCheckout(dir, spec, logger, opts) {
  * {@link runCodingAgent} so its body stays small; returns the built {@link CodingAgentOutcome}.
  */
 async function finalizeCodingRun(args) {
-    const { dir, spec, logger, opts, baseSha, resumed, workDir, checkpoint, followUpTick, followUpTailer, pushWorkOnce, inFlightPush, agentRun, } = args;
+    const { validationReport, dir, spec, logger, opts, baseSha, resumed, workDir, checkpoint, followUpTick, followUpTailer, pushWorkOnce, inFlightPush, agentRun, } = args;
     const { signal } = opts;
     const { summary, stats, stderrTail, usage, callMetrics, effortReport } = agentRun;
     let outcome;
@@ -368,8 +398,63 @@ async function finalizeCodingRun(args) {
     if (spec.validation) {
         outcome.validation = await runRalphValidation(workDir, spec.validation, logger, opts);
     }
+    // Pre-PR validation: the loop already ran (before this finalize, so a red checkout never
+    // reaches the PR-opening caller); attach its verdict for the caller to gate on and for the
+    // backend to record on the step.
+    if (validationReport)
+        outcome.validationReport = validationReport;
     return outcome;
 }
+/**
+ * Whether this pass produced anything worth VALIDATING — i.e. the branch advanced past its
+ * pre-run tip (or the run resumed an earlier one's pushed work). Gates the pre-PR validation
+ * loop, for two reasons: a run that changed nothing has nothing to check, and its real failure
+ * is "the agent produced no file changes" — reporting a red BASE branch instead would blame the
+ * run for a pre-existing condition it never touched (and burn the whole repair budget re-running
+ * an agent that already declined to act).
+ *
+ * Commits forgotten edits to tracked files first, exactly as {@link finalizeCodingRun} does, so
+ * an agent that edited-but-didn't-commit still counts as work. That call is idempotent, so
+ * finalize repeating it later is a no-op. Uncommitted NEW files are invisible here — but they
+ * are equally invisible to finalize, so a run whose only product is an uncommitted new file is
+ * a no-op on both paths, and the checks would have nothing to gate anyway.
+ */
+async function producedWork(dir, spec, baseSha, resumed, opts) {
+    await commitTrackedEdits(dir, spec.commitMessage, opts.signal);
+    return resumed || (await branchHasCommitsSince(dir, baseSha, opts.signal));
+}
+/**
+ * Fold a pre-PR validation REPAIR pass's run into the accumulated agent outcome, so a looped run
+ * reports what every round actually spent rather than only the first. Counts and telemetry are
+ * summed/concatenated; the single-valued fields (the summary the backend renders, the effort
+ * report, the diagnostics that judge the FINAL answer) take the LATEST pass, which is the one
+ * whose state the PR is opened from.
+ */
+function mergeAgentPasses(previous, next) {
+    return {
+        ...next,
+        stats: {
+            toolCalls: (previous.stats?.toolCalls ?? 0) + (next.stats?.toolCalls ?? 0),
+            assistantChars: (previous.stats?.assistantChars ?? 0) + (next.stats?.assistantChars ?? 0),
+        },
+        ...(previous.usage || next.usage
+            ? {
+                usage: {
+                    inputTokens: (previous.usage?.inputTokens ?? 0) + (next.usage?.inputTokens ?? 0),
+                    outputTokens: (previous.usage?.outputTokens ?? 0) + (next.usage?.outputTokens ?? 0),
+                },
+            }
+            : {}),
+        ...(previous.callMetrics || next.callMetrics
+            ? { callMetrics: [...(previous.callMetrics ?? []), ...(next.callMetrics ?? [])] }
+            : {}),
+        // The repair pass's own effort report wins when it wrote one; otherwise keep the first
+        // pass's rather than losing the assessment entirely.
+        ...((next.effortReport ?? previous.effortReport)
+            ? { effortReport: next.effortReport ?? previous.effortReport }
+            : {}),
+    };
+}
 /**
  * The Ralph-loop validation watchdog: the longest a completion command may run before it is
  * killed and treated as a failure (a hung `pnpm test` must never block the loop forever).

package/dist/job.js

@@ -66,6 +66,39 @@ function parseValidationSpec(value) {
         ...(iteration !== undefined ? { iteration } : {}),
     };
 }
+/**
+ * Parse the optional PRE-PR VALIDATION CHECKS spec (see
+ * docs/initiatives/pre-pr-validation.md): the service's ordered `{ label, command }` pairs and
+ * the repair-round budget. Every entry needs a non-empty command; entries without one are
+ * dropped, and a spec that ends up with no usable check returns `undefined` — so a malformed
+ * body degrades to the exact pre-feature behaviour (no loop, PR opens as before) rather than
+ * failing an otherwise-good coding run. `maxAttempts` is clamped to a sane range so a bad body
+ * can't make a container loop forever.
+ */
+function parseValidationChecksSpec(value) {
+    if (typeof value !== 'object' || value === null)
+        return undefined;
+    const o = value;
+    if (!Array.isArray(o.checks))
+        return undefined;
+    const checks = [];
+    for (const raw of o.checks) {
+        if (typeof raw !== 'object' || raw === null)
+            continue;
+        const c = raw;
+        if (typeof c.command !== 'string' || c.command.trim() === '')
+            continue;
+        const label = typeof c.label === 'string' && c.label.trim() ? c.label.trim() : c.command;
+        checks.push({ label, command: c.command });
+    }
+    if (checks.length === 0)
+        return undefined;
+    const parsed = posInt(o.maxAttempts);
+    return {
+        checks,
+        maxAttempts: Math.min(parsed ?? VALIDATION_DEFAULT_MAX_ATTEMPTS, VALIDATION_MAX_ATTEMPTS_CEILING),
+    };
+}
 /**
  * Parse the shared per-job auth fields, validating per harness: a subscription
  * harness (`claude-code` / `codex`) requires `subscriptionToken`; the default Pi
@@ -353,6 +386,18 @@ export function parseTestSecrets(value) {
     }
     return entries;
 }
+/**
+ * The ceiling the harness clamps a body-supplied `validationChecks.maxAttempts` to, and the
+ * default it applies when the body omits one.
+ *
+ * DELIBERATE DUPLICATES of `VALIDATION_MAX_ATTEMPTS_CEILING` / `VALIDATION_DEFAULT_MAX_ATTEMPTS`
+ * in `@cat-factory/contracts` — the published image takes no schema dependency, so the harness
+ * cannot import them. Keep the two in step: the API validates writes against the contracts
+ * values, so a harness clamping to a DIFFERENT ceiling would silently cap a budget an operator
+ * was allowed to save, with nothing to flag the mismatch.
+ */
+export const VALIDATION_MAX_ATTEMPTS_CEILING = 10;
+export const VALIDATION_DEFAULT_MAX_ATTEMPTS = 3;
 /** Parse the coding-mode bootstrap spec, or undefined when absent. Validates the target. */
 function parseAgentBootstrapSpec(value) {
     if (typeof value !== 'object' || value === null)
@@ -671,6 +716,7 @@ export function parseAgentJob(input) {
         testSecrets: parseTestSecrets(o.testSecrets),
         guardLimits: parseGuardLimits(o.guardLimits),
         validation: parseValidationSpec(o.validation),
+        validationChecks: parseValidationChecksSpec(o.validationChecks),
         reviewPrNumber: posInt(o.reviewPrNumber),
     });
     assertAllowedHost(job.repo.cloneUrl, 'repo.cloneUrl');
@@ -727,7 +773,7 @@ function parseAgentPrSpec(raw) {
  * literal doesn't blow the complexity budget; behaviour is byte-identical (spread order preserved).
  */
 function assembleAgentJob(o, mode, agentField, parts) {
-    const { output, pr, infra, peerRepos, referenceRepos, referenceBranches, bootstrap, contextFiles, packageRegistries, skill, testSecrets, guardLimits, validation, reviewPrNumber, } = parts;
+    const { output, pr, infra, peerRepos, referenceRepos, referenceBranches, bootstrap, contextFiles, packageRegistries, skill, testSecrets, guardLimits, validation, validationChecks, reviewPrNumber, } = parts;
     const repo = (o.repo ?? {});
     return {
         jobId: str(o.jobId, 'jobId'),
@@ -754,6 +800,7 @@ function assembleAgentJob(o, mode, agentField, parts) {
         ...(reviewPrNumber !== undefined ? { reviewPrNumber } : {}),
         ...(guardLimits ? { guardLimits } : {}),
         ...(validation ? { validation } : {}),
+        ...(validationChecks ? { validationChecks } : {}),
     };
 }
 /**

package/dist/runner.js

@@ -213,6 +213,9 @@ export class JobRegistry {
                 onFollowUp: (items) => {
                     entry.followUpBuffer.push(...items);
                 },
+                onValidationReport: (report) => {
+                    entry.validationReport = report;
+                },
                 onCallMetric: (call) => {
                     // Stamp the job-scoped sequence on the metric OBJECT: the handler keeps the same
                     // instance for its terminal result, so both channels carry the same `seq` and the