@cat-factory/executor-harness 1.43.6 → 1.45.0

package/dist/agent.js

@@ -654,6 +654,16 @@ async function runMultiRepoExplore(job, opts) {
         }, { infraSetupFields: {}, logger, signal: opts.signal });
     });
 }
+/**
+ * Whether a Ralph iteration ({@link AgentJob.validation} set) landed on a MULTI-REPO job (writable
+ * peer repos or read-only reference repos). The post-commit validation command is only wired into
+ * the single-repo flow, so a multi-repo run would silently skip it and degenerate the loop into a
+ * one-shot with no completion gate — multi-repo ralph is out of scope for v1 (see
+ * backend/docs/ralph-loop.md), so {@link runCodingMode} fails loudly on this instead.
+ */
+export function ralphUnsupportedOnMultiRepo(job) {
+    return Boolean(job.validation) && Boolean(job.peerRepos?.length || job.referenceRepos?.length);
+}
 /**
  * Edit-and-push coding, dispatching on job DATA: repo-bootstrap (force-push a fresh history to a
  * separate target repo), conflict-resolution (merge the base in, resolve, push back), multi-repo
@@ -678,7 +688,19 @@ async function runCodingMode(job, opts) {
     // all of them. Keyed off job DATA, not the agent kind — set for the implementer's writable
     // peer repos (service-connections phase 3, `peerRepos`) OR the doc-writer's READ-ONLY
     // reference repos (`referenceRepos`, cloned but never pushed).
-    const result = job.peerRepos?.length || job.referenceRepos?.length
+    const multiRepo = Boolean(job.peerRepos?.length || job.referenceRepos?.length);
+    // Ralph loop (v1): the post-commit validation command is only wired into the single-repo
+    // flow, so a multi-repo run would silently skip it and the loop would degenerate into a
+    // one-shot with no completion gate. Multi-repo ralph is deliberately out of scope for v1
+    // (see backend/docs/ralph-loop.md), so FAIL LOUDLY rather than run a validation-less pass.
+    if (ralphUnsupportedOnMultiRepo(job)) {
+        return {
+            error: 'Ralph loop is not supported on a multi-repo task (connected service repos). ' +
+                'Its validation command runs only in the single primary-repo checkout. ' +
+                'Run the Ralph loop on a task scoped to a single repo.',
+        };
+    }
+    const result = multiRepo
         ? await runMultiRepoCoding(job, opts)
         : await runSingleRepoCoding(job, opts);
     // Structured coding kind (repro-test): fold the final reply's JSON onto `custom` so the
@@ -700,7 +722,7 @@ async function runCodingMode(job, opts) {
  */
 async function runSingleRepoCoding(job, opts) {
     const pushBranch = job.pushBranch ?? job.newBranch ?? job.branch;
-    const { summary, stats, stderrTail, pushed, usage, callMetrics } = await runCodingAgent({
+    const { summary, stats, stderrTail, pushed, usage, callMetrics, validation } = await runCodingAgent({
         kind: 'agent',
         jobId: job.jobId,
         repo: job.repo,
@@ -724,7 +746,21 @@ async function runSingleRepoCoding(job, opts) {
         ...(job.persistentCheckout ? { persistentCheckout: true } : {}),
         ...(job.streamFollowUps ? { streamFollowUps: true } : {}),
         ...(job.referenceBranches?.length ? { referenceBranches: job.referenceBranches } : {}),
+        // Ralph loop: run the completion command after the agent commits and report its verdict.
+        ...(job.validation
+            ? {
+                validation: {
+                    command: job.validation.command,
+                    ...(job.validation.iteration !== undefined
+                        ? { iteration: job.validation.iteration }
+                        : {}),
+                },
+            }
+            : {}),
     }, opts);
+    // Ralph loop: the harness-computed validation verdict, forwarded onto the coding result as
+    // `ralphVerdict` so the backend's `toRunResult` lifts it onto `AgentRunResult.ralphVerdict`.
+    const ralphVerdict = validation ? { ralphVerdict: validation } : {};
     if (!pushed) {
         // A no-op: a failure for the implementer, a clean non-event for the fixers.
         if (job.noChangesIsError === false) {
@@ -735,6 +771,7 @@ async function runSingleRepoCoding(job, opts) {
                 stats,
                 ...(usage ? { usage } : {}),
                 ...(callMetrics ? { callMetrics } : {}),
+                ...ralphVerdict,
             };
         }
         return {
@@ -799,6 +836,7 @@ async function runSingleRepoCoding(job, opts) {
             stats,
             ...(usage ? { usage } : {}),
             ...(callMetrics ? { callMetrics } : {}),
+            ...ralphVerdict,
         };
     }
     return {
@@ -808,6 +846,7 @@ async function runSingleRepoCoding(job, opts) {
         stats,
         ...(usage ? { usage } : {}),
         ...(callMetrics ? { callMetrics } : {}),
+        ...ralphVerdict,
     };
 }
 /**

package/dist/coding-agent.js

@@ -1,5 +1,8 @@
 import { mkdir } from 'node:fs/promises';
 import { join } from 'node:path';
+import { spawn } from 'node:child_process';
+import { killChildProcess, spawnDetached } from './process.js';
+import { MAX_CAPTURED_OUTPUT_CHARS, redactSecrets } from './redact.js';
 import { branchAheadOfBase, branchHasCommitsSince, cloneExistingBranch, cloneRepo, commitTrackedEdits, createBranch, excludeFromGit, fetchReferenceBranches, headCommit, listUntrackedFiles, openPullRequest, prepareExistingCheckout, pushBranch, refreshFromBaseIfClean, remoteBranchExists, } from './git.js';
 import { FOLLOW_UPS_FILENAME, FollowUpTailer } from './follow-ups.js';
 import { acquireRepoCheckout, agentNeverActed, agentOutputTail, runAgentInWorkspace, withWorkspace, } from './pi-workspace.js';
@@ -299,6 +302,13 @@ export async function runCodingAgent(spec, opts = {}) {
                     ...(callMetrics ? { callMetrics } : {}),
                 };
             }
+            // Ralph loop: run the programmatic completion command against the pushed/committed
+            // state and attach its verdict (exit code = the loop's authoritative done signal).
+            // Runs regardless of whether this pass pushed — a no-op iteration must still be able
+            // to report that the criterion is (already) met. The harness runs it, never the model.
+            if (spec.validation) {
+                outcome.validation = await runRalphValidation(workDir, spec.validation, logger, opts);
+            }
         }
         finally {
             // Safety net for the throw path (the happy path already cleared these above).
@@ -309,6 +319,83 @@ export async function runCodingAgent(spec, opts = {}) {
         return outcome;
     });
 }
+/**
+ * The Ralph-loop validation watchdog: the longest a completion command may run before it is
+ * killed and treated as a failure (a hung `pnpm test` must never block the loop forever).
+ * Overridable via env for tests; defaults to 15 minutes.
+ */
+function ralphValidationTimeoutMs() {
+    const n = Number(process.env.RALPH_VALIDATION_TIMEOUT_MS);
+    return Number.isFinite(n) && n > 0 ? Math.floor(n) : 15 * 60_000;
+}
+/**
+ * Ralph loop: run the programmatic completion command in the checkout and return its exit
+ * code plus a bounded, redacted tail of its output. The EXIT CODE is the loop's authoritative
+ * done signal (0 = the criterion is met) — computed here by the harness, never self-reported
+ * by the model, which is the whole point of a programmatic exit condition. Runs
+ * `sh -c <command>` in `cwd`; a watchdog kills the whole process tree on timeout (a hung
+ * command counts as a failure so the loop is never blocked), and an aborted run resolves to a
+ * non-zero code too. The command runs INSIDE the sandboxed run container (the same trust
+ * boundary as the coding agent) — there is no host/backend execution.
+ */
+async function runRalphValidation(cwd, validation, logger, opts) {
+    const timeoutMs = ralphValidationTimeoutMs();
+    logger.info('coding-agent(ralph): running validation command', {
+        iteration: validation.iteration,
+    });
+    return new Promise((resolve) => {
+        let out = '';
+        let settled = false;
+        const child = spawn('sh', ['-c', validation.command], {
+            cwd,
+            detached: spawnDetached,
+            stdio: ['ignore', 'pipe', 'pipe'],
+        });
+        // Keep only the tail; guard against unbounded buffering on a chatty command.
+        const capture = (chunk) => {
+            out = (out + chunk.toString('utf8')).slice(-MAX_CAPTURED_OUTPUT_CHARS);
+        };
+        child.stdout?.on('data', capture);
+        child.stderr?.on('data', capture);
+        const finish = (exitCode) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            opts.signal?.removeEventListener('abort', onAbort);
+            const trimmed = out.trim();
+            const tail = trimmed ? redactSecrets(trimmed) : undefined;
+            logger.info('coding-agent(ralph): validation finished', {
+                exitCode,
+                iteration: validation.iteration,
+            });
+            resolve({
+                validationPassed: exitCode === 0,
+                exitCode,
+                ...(tail ? { validationOutputTail: tail } : {}),
+                ...(validation.iteration !== undefined ? { iteration: validation.iteration } : {}),
+            });
+        };
+        const timer = setTimeout(() => {
+            logger.warn('coding-agent(ralph): validation command timed out', { timeoutMs });
+            killChildProcess(child, undefined, logger);
+            finish(124); // conventional timeout exit code (a non-zero fail)
+        }, timeoutMs);
+        timer.unref?.();
+        const onAbort = () => {
+            killChildProcess(child, undefined, logger);
+            finish(130); // aborted (a non-zero fail)
+        };
+        opts.signal?.addEventListener('abort', onAbort, { once: true });
+        child.on('error', (err) => {
+            logger.warn('coding-agent(ralph): validation command failed to spawn', {
+                error: err instanceof Error ? err.message : String(err),
+            });
+            finish(127); // spawn error / command not found (a non-zero fail)
+        });
+        child.on('close', (code) => finish(code ?? 1));
+    });
+}
 /** Sanitise an owner/name into a safe single path segment for a sibling checkout directory. */
 export function safeDirSegment(value) {
     return value.replace(/[^A-Za-z0-9._-]/g, '-') || '_';

package/dist/job.js

@@ -45,6 +45,27 @@ function parseGuardLimits(value) {
         spec.maxConsecutiveWebCalls = web;
     return Object.keys(spec).length > 0 ? spec : undefined;
 }
+/**
+ * Parse the optional Ralph-loop validation spec. Requires a non-empty `command` string (the
+ * completion criterion the harness runs); `progressPath`/`iteration` are optional metadata.
+ * Returns undefined when absent or malformed (a coding run then behaves like any other — no
+ * post-commit validation). See {@link ValidationSpec}.
+ */
+function parseValidationSpec(value) {
+    if (typeof value !== 'object' || value === null)
+        return undefined;
+    const o = value;
+    if (typeof o.command !== 'string' || o.command.trim() === '')
+        return undefined;
+    const iteration = posInt(o.iteration);
+    return {
+        command: o.command,
+        ...(typeof o.progressPath === 'string' && o.progressPath
+            ? { progressPath: o.progressPath }
+            : {}),
+        ...(iteration !== undefined ? { iteration } : {}),
+    };
+}
 /**
  * Parse the shared per-job auth fields, validating per harness: a subscription
  * harness (`claude-code` / `codex`) requires `subscriptionToken`; the default Pi
@@ -589,6 +610,7 @@ export function parseAgentJob(input) {
     const packageRegistries = parsePackageRegistries(o.packageRegistries);
     const testSecrets = parseTestSecrets(o.testSecrets);
     const guardLimits = parseGuardLimits(o.guardLimits);
+    const validation = parseValidationSpec(o.validation);
     const job = {
         jobId: str(o.jobId, 'jobId'),
         mode,
@@ -623,6 +645,7 @@ export function parseAgentJob(input) {
         ...(o.persistentCheckout === true ? { persistentCheckout: true } : {}),
         ...(o.streamFollowUps === true ? { streamFollowUps: true } : {}),
         ...(guardLimits ? { guardLimits } : {}),
+        ...(validation ? { validation } : {}),
     };
     assertAllowedHost(job.repo.cloneUrl, 'repo.cloneUrl');
     if (job.githubApiBase)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cat-factory/executor-harness",
-  "version": "1.43.6",
+  "version": "1.45.0",
   "description": "Container payload: a thin TypeScript wrapper that runs the Pi coding agent against a cloned repo and opens a PR. Runs in the Cloudflare Container (and, in local native mode, as a host process); carries no secrets.",
   "repository": {
     "type": "git",
@@ -24,10 +24,10 @@
     "@hono/node-server": "^2.0.8",
     "@types/node": "^26.1.1",
     "hono": "^4.12.29",
-    "typescript": "^6.0.3",
+    "typescript": "7.0.2",
     "vitest": "^4.1.10",
-    "@cat-factory/server": "0.113.8",
-    "@cat-factory/spend": "0.12.26"
+    "@cat-factory/server": "0.126.0",
+    "@cat-factory/spend": "0.12.37"
   },
   "scripts": {
     "build": "tsc -p tsconfig.json",

package/src/agent.ts

@@ -797,6 +797,19 @@ async function runMultiRepoExplore(job: AgentJob, opts: RunOptions): Promise<Age
   })
 }
 
+/**
+ * Whether a Ralph iteration ({@link AgentJob.validation} set) landed on a MULTI-REPO job (writable
+ * peer repos or read-only reference repos). The post-commit validation command is only wired into
+ * the single-repo flow, so a multi-repo run would silently skip it and degenerate the loop into a
+ * one-shot with no completion gate — multi-repo ralph is out of scope for v1 (see
+ * backend/docs/ralph-loop.md), so {@link runCodingMode} fails loudly on this instead.
+ */
+export function ralphUnsupportedOnMultiRepo(
+  job: Pick<AgentJob, 'validation' | 'peerRepos' | 'referenceRepos'>,
+): boolean {
+  return Boolean(job.validation) && Boolean(job.peerRepos?.length || job.referenceRepos?.length)
+}
+
 /**
  * Edit-and-push coding, dispatching on job DATA: repo-bootstrap (force-push a fresh history to a
  * separate target repo), conflict-resolution (merge the base in, resolve, push back), multi-repo
@@ -819,10 +832,22 @@ async function runCodingMode(job: AgentJob, opts: RunOptions): Promise<AgentResu
   // all of them. Keyed off job DATA, not the agent kind — set for the implementer's writable
   // peer repos (service-connections phase 3, `peerRepos`) OR the doc-writer's READ-ONLY
   // reference repos (`referenceRepos`, cloned but never pushed).
-  const result =
-    job.peerRepos?.length || job.referenceRepos?.length
-      ? await runMultiRepoCoding(job, opts)
-      : await runSingleRepoCoding(job, opts)
+  const multiRepo = Boolean(job.peerRepos?.length || job.referenceRepos?.length)
+  // Ralph loop (v1): the post-commit validation command is only wired into the single-repo
+  // flow, so a multi-repo run would silently skip it and the loop would degenerate into a
+  // one-shot with no completion gate. Multi-repo ralph is deliberately out of scope for v1
+  // (see backend/docs/ralph-loop.md), so FAIL LOUDLY rather than run a validation-less pass.
+  if (ralphUnsupportedOnMultiRepo(job)) {
+    return {
+      error:
+        'Ralph loop is not supported on a multi-repo task (connected service repos). ' +
+        'Its validation command runs only in the single primary-repo checkout. ' +
+        'Run the Ralph loop on a task scoped to a single repo.',
+    }
+  }
+  const result = multiRepo
+    ? await runMultiRepoCoding(job, opts)
+    : await runSingleRepoCoding(job, opts)
 
   // Structured coding kind (repro-test): fold the final reply's JSON onto `custom` so the
   // backend post-completion resolver records the outcome. Skipped on a failed run (its `error`
@@ -843,34 +868,49 @@ async function runCodingMode(job: AgentJob, opts: RunOptions): Promise<AgentResu
  */
 async function runSingleRepoCoding(job: AgentJob, opts: RunOptions): Promise<AgentResult> {
   const pushBranch = job.pushBranch ?? job.newBranch ?? job.branch
-  const { summary, stats, stderrTail, pushed, usage, callMetrics } = await runCodingAgent(
-    {
-      kind: 'agent',
-      jobId: job.jobId,
-      repo: job.repo,
-      cloneBranch: job.branch,
-      ...(job.newBranch ? { newBranch: job.newBranch } : {}),
-      pushBranch,
-      ghToken: job.ghToken,
-      systemPrompt: job.systemPrompt,
-      userPrompt: job.userPrompt,
-      model: job.model,
-      harness: job.harness,
-      subscriptionToken: job.subscriptionToken,
-      subscriptionBaseUrl: job.subscriptionBaseUrl,
-      ambientAuth: job.ambientAuth,
-      proxyBaseUrl: job.proxyBaseUrl,
-      sessionToken: job.sessionToken,
-      commitMessage: job.commitMessage ?? job.pr?.title ?? 'Agent changes',
-      webToolsGuidance: job.webToolsGuidance,
-      webSearchProxy: job.webSearch,
-      guardLimits: job.guardLimits,
-      ...(job.persistentCheckout ? { persistentCheckout: true } : {}),
-      ...(job.streamFollowUps ? { streamFollowUps: true } : {}),
-      ...(job.referenceBranches?.length ? { referenceBranches: job.referenceBranches } : {}),
-    },
-    opts,
-  )
+  const { summary, stats, stderrTail, pushed, usage, callMetrics, validation } =
+    await runCodingAgent(
+      {
+        kind: 'agent',
+        jobId: job.jobId,
+        repo: job.repo,
+        cloneBranch: job.branch,
+        ...(job.newBranch ? { newBranch: job.newBranch } : {}),
+        pushBranch,
+        ghToken: job.ghToken,
+        systemPrompt: job.systemPrompt,
+        userPrompt: job.userPrompt,
+        model: job.model,
+        harness: job.harness,
+        subscriptionToken: job.subscriptionToken,
+        subscriptionBaseUrl: job.subscriptionBaseUrl,
+        ambientAuth: job.ambientAuth,
+        proxyBaseUrl: job.proxyBaseUrl,
+        sessionToken: job.sessionToken,
+        commitMessage: job.commitMessage ?? job.pr?.title ?? 'Agent changes',
+        webToolsGuidance: job.webToolsGuidance,
+        webSearchProxy: job.webSearch,
+        guardLimits: job.guardLimits,
+        ...(job.persistentCheckout ? { persistentCheckout: true } : {}),
+        ...(job.streamFollowUps ? { streamFollowUps: true } : {}),
+        ...(job.referenceBranches?.length ? { referenceBranches: job.referenceBranches } : {}),
+        // Ralph loop: run the completion command after the agent commits and report its verdict.
+        ...(job.validation
+          ? {
+              validation: {
+                command: job.validation.command,
+                ...(job.validation.iteration !== undefined
+                  ? { iteration: job.validation.iteration }
+                  : {}),
+              },
+            }
+          : {}),
+      },
+      opts,
+    )
+  // Ralph loop: the harness-computed validation verdict, forwarded onto the coding result as
+  // `ralphVerdict` so the backend's `toRunResult` lifts it onto `AgentRunResult.ralphVerdict`.
+  const ralphVerdict = validation ? { ralphVerdict: validation } : {}
 
   if (!pushed) {
     // A no-op: a failure for the implementer, a clean non-event for the fixers.
@@ -882,6 +922,7 @@ async function runSingleRepoCoding(job: AgentJob, opts: RunOptions): Promise<Age
         stats,
         ...(usage ? { usage } : {}),
         ...(callMetrics ? { callMetrics } : {}),
+        ...ralphVerdict,
       }
     }
     return {
@@ -951,6 +992,7 @@ async function runSingleRepoCoding(job: AgentJob, opts: RunOptions): Promise<Age
       stats,
       ...(usage ? { usage } : {}),
       ...(callMetrics ? { callMetrics } : {}),
+      ...ralphVerdict,
     }
   }
   return {
@@ -960,6 +1002,7 @@ async function runSingleRepoCoding(job: AgentJob, opts: RunOptions): Promise<Age
     stats,
     ...(usage ? { usage } : {}),
     ...(callMetrics ? { callMetrics } : {}),
+    ...ralphVerdict,
   }
 }
 

package/src/coding-agent.ts

@@ -1,5 +1,8 @@
 import { mkdir } from 'node:fs/promises'
 import { join } from 'node:path'
+import { spawn } from 'node:child_process'
+import { killChildProcess, spawnDetached } from './process.js'
+import { MAX_CAPTURED_OUTPUT_CHARS, redactSecrets } from './redact.js'
 import type {
   AgentJob,
   AgentResult,
@@ -36,7 +39,7 @@ import {
 } from './pi-workspace.js'
 import type { ProgressGuardLimits } from './pi.js'
 import type { RunOptions } from './runner.js'
-import { log } from './logger.js'
+import { log, type Logger } from './logger.js'
 
 // The shared skeleton for the container coding agents that clone a repo, run Pi
 // against it and push the result on a branch. The implementation (`/run`) and
@@ -92,6 +95,12 @@ export interface CodingAgentSpec extends HarnessAuthFields {
    * them. Best-effort per branch. Absent/empty ⇒ none fetched.
    */
   referenceBranches?: string[]
+  /**
+   * Ralph loop: run this programmatic completion command in the checkout AFTER the agent
+   * commits + pushes, capturing its exit code + a bounded output tail (the loop's exit
+   * condition — computed by the harness, never the model). Absent for every non-`ralph` run.
+   */
+  validation?: { command: string; iteration?: number }
 }
 
 /** The outcome of a coding agent run, before each caller maps it to its own result shape. */
@@ -107,6 +116,17 @@ export interface CodingAgentOutcome {
   usage?: { inputTokens: number; outputTokens: number }
   /** Per-model-call telemetry from a subscription harness's CLI stream (absent for Pi). */
   callMetrics?: HarnessCallMetric[]
+  /**
+   * Ralph loop: the verdict of the post-commit validation command (whether it exited 0, the
+   * exit code, and a bounded/redacted output tail). Present only when {@link CodingAgentSpec.validation}
+   * was set. The exit code is the loop's authoritative completion signal.
+   */
+  validation?: {
+    validationPassed: boolean
+    exitCode: number
+    validationOutputTail?: string
+    iteration?: number
+  }
 }
 
 /**
@@ -425,6 +445,14 @@ export async function runCodingAgent(
             ...(callMetrics ? { callMetrics } : {}),
           }
         }
+
+        // Ralph loop: run the programmatic completion command against the pushed/committed
+        // state and attach its verdict (exit code = the loop's authoritative done signal).
+        // Runs regardless of whether this pass pushed — a no-op iteration must still be able
+        // to report that the criterion is (already) met. The harness runs it, never the model.
+        if (spec.validation) {
+          outcome.validation = await runRalphValidation(workDir, spec.validation, logger, opts)
+        }
       } finally {
         // Safety net for the throw path (the happy path already cleared these above).
         clearInterval(checkpoint)
@@ -435,6 +463,94 @@ export async function runCodingAgent(
   )
 }
 
+/**
+ * The Ralph-loop validation watchdog: the longest a completion command may run before it is
+ * killed and treated as a failure (a hung `pnpm test` must never block the loop forever).
+ * Overridable via env for tests; defaults to 15 minutes.
+ */
+function ralphValidationTimeoutMs(): number {
+  const n = Number(process.env.RALPH_VALIDATION_TIMEOUT_MS)
+  return Number.isFinite(n) && n > 0 ? Math.floor(n) : 15 * 60_000
+}
+
+/**
+ * Ralph loop: run the programmatic completion command in the checkout and return its exit
+ * code plus a bounded, redacted tail of its output. The EXIT CODE is the loop's authoritative
+ * done signal (0 = the criterion is met) — computed here by the harness, never self-reported
+ * by the model, which is the whole point of a programmatic exit condition. Runs
+ * `sh -c <command>` in `cwd`; a watchdog kills the whole process tree on timeout (a hung
+ * command counts as a failure so the loop is never blocked), and an aborted run resolves to a
+ * non-zero code too. The command runs INSIDE the sandboxed run container (the same trust
+ * boundary as the coding agent) — there is no host/backend execution.
+ */
+async function runRalphValidation(
+  cwd: string,
+  validation: { command: string; iteration?: number },
+  logger: Logger,
+  opts: RunOptions,
+): Promise<{
+  validationPassed: boolean
+  exitCode: number
+  validationOutputTail?: string
+  iteration?: number
+}> {
+  const timeoutMs = ralphValidationTimeoutMs()
+  logger.info('coding-agent(ralph): running validation command', {
+    iteration: validation.iteration,
+  })
+  return new Promise((resolve) => {
+    let out = ''
+    let settled = false
+    const child = spawn('sh', ['-c', validation.command], {
+      cwd,
+      detached: spawnDetached,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    })
+    // Keep only the tail; guard against unbounded buffering on a chatty command.
+    const capture = (chunk: Buffer): void => {
+      out = (out + chunk.toString('utf8')).slice(-MAX_CAPTURED_OUTPUT_CHARS)
+    }
+    child.stdout?.on('data', capture)
+    child.stderr?.on('data', capture)
+    const finish = (exitCode: number): void => {
+      if (settled) return
+      settled = true
+      clearTimeout(timer)
+      opts.signal?.removeEventListener('abort', onAbort)
+      const trimmed = out.trim()
+      const tail = trimmed ? redactSecrets(trimmed) : undefined
+      logger.info('coding-agent(ralph): validation finished', {
+        exitCode,
+        iteration: validation.iteration,
+      })
+      resolve({
+        validationPassed: exitCode === 0,
+        exitCode,
+        ...(tail ? { validationOutputTail: tail } : {}),
+        ...(validation.iteration !== undefined ? { iteration: validation.iteration } : {}),
+      })
+    }
+    const timer = setTimeout(() => {
+      logger.warn('coding-agent(ralph): validation command timed out', { timeoutMs })
+      killChildProcess(child, undefined, logger)
+      finish(124) // conventional timeout exit code (a non-zero fail)
+    }, timeoutMs)
+    timer.unref?.()
+    const onAbort = (): void => {
+      killChildProcess(child, undefined, logger)
+      finish(130) // aborted (a non-zero fail)
+    }
+    opts.signal?.addEventListener('abort', onAbort, { once: true })
+    child.on('error', (err) => {
+      logger.warn('coding-agent(ralph): validation command failed to spawn', {
+        error: err instanceof Error ? err.message : String(err),
+      })
+      finish(127) // spawn error / command not found (a non-zero fail)
+    })
+    child.on('close', (code) => finish(code ?? 1))
+  })
+}
+
 /** Sanitise an owner/name into a safe single path segment for a sibling checkout directory. */
 export function safeDirSegment(value: string): string {
   return value.replace(/[^A-Za-z0-9._-]/g, '-') || '_'

package/src/job.ts

@@ -153,6 +153,26 @@ function parseGuardLimits(value: unknown): GuardLimitsSpec | undefined {
   return Object.keys(spec).length > 0 ? spec : undefined
 }
 
+/**
+ * Parse the optional Ralph-loop validation spec. Requires a non-empty `command` string (the
+ * completion criterion the harness runs); `progressPath`/`iteration` are optional metadata.
+ * Returns undefined when absent or malformed (a coding run then behaves like any other — no
+ * post-commit validation). See {@link ValidationSpec}.
+ */
+function parseValidationSpec(value: unknown): ValidationSpec | undefined {
+  if (typeof value !== 'object' || value === null) return undefined
+  const o = value as Record<string, unknown>
+  if (typeof o.command !== 'string' || o.command.trim() === '') return undefined
+  const iteration = posInt(o.iteration)
+  return {
+    command: o.command,
+    ...(typeof o.progressPath === 'string' && o.progressPath
+      ? { progressPath: o.progressPath }
+      : {}),
+    ...(iteration !== undefined ? { iteration } : {}),
+  }
+}
+
 /**
  * Parse the shared per-job auth fields, validating per harness: a subscription
  * harness (`claude-code` / `codex`) requires `subscriptionToken`; the default Pi
@@ -627,6 +647,23 @@ export interface AgentOutputSpec {
  * RUNNING — no agent runs and the serve is deliberately not torn down when the job returns
  * (see {@link AgentResult.preview}).
  */
+/**
+ * Coding mode (Ralph loop): the programmatic completion criterion. After the coding agent
+ * commits + pushes, the harness runs {@link command} in the checkout and reports its exit
+ * code back on {@link AgentResult.ralphVerdict} — exit 0 means the loop is done. This is the
+ * whole point of a Ralph loop's exit condition being a REAL check: the harness runs it, not
+ * the model. The command runs only inside the sandboxed run container (same trust boundary
+ * as the coding agent). Absent for every non-`ralph` coding run.
+ */
+export interface ValidationSpec {
+  /** The shell command the harness runs against the checkout (exit 0 = the criterion is met). */
+  command: string
+  /** Repo-relative progress-log path the agent maintains (informational; the harness doesn't write it). */
+  progressPath?: string
+  /** 1-based iteration number, echoed back on the verdict for the engine's attempt log. */
+  iteration?: number
+}
+
 export interface AgentJob extends HarnessAuthFields {
   jobId: string
   mode: AgentMode
@@ -753,6 +790,11 @@ export interface AgentJob extends HarnessAuthFields {
    * killed for a kind's normal working pattern. Absent ⇒ env/default for all knobs.
    */
   guardLimits?: GuardLimitsSpec
+  /**
+   * Coding mode (Ralph loop): the programmatic completion command the harness runs after the
+   * agent commits + pushes. Present only for a `ralph` iteration. See {@link ValidationSpec}.
+   */
+  validation?: ValidationSpec
 }
 
 /** Per-job, per-knob progress-guard overrides (see {@link AgentJob.guardLimits}). */
@@ -809,6 +851,18 @@ export interface AgentResult {
   pushed?: boolean
   prUrl?: string
   branch?: string
+  /**
+   * Coding mode (Ralph loop): the harness-computed verdict of the post-commit validation
+   * command — whether it exited 0, its exit code, and a bounded, redacted output tail. The
+   * engine reads this (never a model self-report) to decide whether the loop is done or must
+   * iterate again. Present only for a `ralph` iteration ({@link AgentJob.validation} set).
+   */
+  ralphVerdict?: {
+    validationPassed: boolean
+    exitCode: number
+    validationOutputTail?: string
+    iteration?: number
+  }
   /**
    * Coding mode (multi-repo): the PRs opened in the connected services' PEER repos, one per
    * repo the run actually changed (service-connections phase 3). Beside the own-service
@@ -1130,6 +1184,7 @@ export function parseAgentJob(input: unknown): AgentJob {
   const packageRegistries = parsePackageRegistries(o.packageRegistries)
   const testSecrets = parseTestSecrets(o.testSecrets)
   const guardLimits = parseGuardLimits(o.guardLimits)
+  const validation = parseValidationSpec(o.validation)
   const job: AgentJob = {
     jobId: str(o.jobId, 'jobId'),
     mode,
@@ -1164,6 +1219,7 @@ export function parseAgentJob(input: unknown): AgentJob {
     ...(o.persistentCheckout === true ? { persistentCheckout: true } : {}),
     ...(o.streamFollowUps === true ? { streamFollowUps: true } : {}),
     ...(guardLimits ? { guardLimits } : {}),
+    ...(validation ? { validation } : {}),
   }
   assertAllowedHost(job.repo.cloneUrl, 'repo.cloneUrl')
   if (job.githubApiBase) assertAllowedHost(job.githubApiBase, 'githubApiBase')