@cat-factory/executor-harness 1.50.14 → 1.50.18

package/src/coding-agent.ts

@@ -31,6 +31,7 @@ import {
 } from './git.js'
 import { FOLLOW_UPS_FILENAME, FollowUpTailer } from './follow-ups.js'
 import type { HarnessCallMetric, PiRunStats } from './pi.js'
+import { EFFORT_REPORT_FILE, type EffortReport } from './effort.js'
 import {
   acquireRepoCheckout,
   agentNeverActed,
@@ -123,6 +124,8 @@ export interface CodingAgentOutcome {
   usage?: { inputTokens: number; outputTokens: number }
   /** Per-model-call telemetry from a subscription harness's CLI stream (absent for Pi). */
   callMetrics?: HarnessCallMetric[]
+  /** The agent's effort self-assessment, lifted from its sentinel file (absent when it wrote none). */
+  effortReport?: EffortReport
   /**
    * Ralph loop: the verdict of the post-commit validation command (whether it exited 0, the
    * exit code, and a bounded/redacted output tail). Present only when {@link CodingAgentSpec.validation}
@@ -242,6 +245,14 @@ export async function runCodingAgent(
       const workDir = serviceDirectory ? join(dir, serviceDirectory) : dir
       if (serviceDirectory) await mkdir(workDir, { recursive: true })
 
+      // Every container agent is asked to write its effort self-assessment to `.cat-effort.json`
+      // in its cwd (the backend appends EFFORT_REPORT_GUIDANCE to every container prompt). Locally
+      // exclude it from git — exactly like the follow-ups sentinel below — so the agent's own
+      // `git add` can never stage it into the PR. `readEffortReport` also removes it after the run,
+      // but that cannot un-stage a mid-run commit; the per-clone exclude is what prevents it. A bare
+      // filename pattern matches the file in any subdirectory, so it covers a monorepo `workDir` too.
+      await excludeFromGit(dir, EFFORT_REPORT_FILE, signal)
+
       // Follow-up companion: tail the Coder's sentinel file and stream new items out on the
       // job view. Locally exclude it from git first so the agent's own `git add` can never
       // stage it and it never surfaces as an untracked leftover or in the PR. The sentinel
@@ -462,7 +473,7 @@ async function finalizeCodingRun(args: {
     agentRun,
   } = args
   const { signal } = opts
-  const { summary, stats, stderrTail, usage, callMetrics } = agentRun
+  const { summary, stats, stderrTail, usage, callMetrics, effortReport } = agentRun
   let outcome: CodingAgentOutcome
 
   // Stop tailing the follow-up sentinel and flush any items written after the last
@@ -521,6 +532,7 @@ async function finalizeCodingRun(args: {
       ...(stderrTail ? { stderrTail } : {}),
       ...(usage ? { usage } : {}),
       ...(callMetrics ? { callMetrics } : {}),
+      ...(effortReport ? { effortReport } : {}),
     }
   } else {
     opts.onPhase?.('push')
@@ -534,6 +546,7 @@ async function finalizeCodingRun(args: {
       ...(stderrTail ? { stderrTail } : {}),
       ...(usage ? { usage } : {}),
       ...(callMetrics ? { callMetrics } : {}),
+      ...(effortReport ? { effortReport } : {}),
     }
   }
 
@@ -765,26 +778,27 @@ export async function runMultiRepoCoding(
     // note + the backend system-prompt section explain the layout.
     opts.onPhase?.('agent')
     logger.info('multi-repo: running agent', { repos: legs.map((l) => l.dirName) })
-    const { summary, stats, stderrTail, usage, callMetrics } = await runAgentInWorkspace(
-      {
-        dir: root,
-        systemPrompt: job.systemPrompt,
-        userPrompt: job.userPrompt,
-        model: job.model,
-        harness: job.harness,
-        subscriptionToken: job.subscriptionToken,
-        subscriptionBaseUrl: job.subscriptionBaseUrl,
-        ambientAuth: job.ambientAuth,
-        proxyBaseUrl: job.proxyBaseUrl,
-        sessionToken: job.sessionToken,
-        webToolsGuidance: job.webToolsGuidance,
-        webSearchProxy: job.webSearch,
-        guardLimits: job.guardLimits,
-        ...(job.contextFiles ? { contextFiles: job.contextFiles } : {}),
-        multiRepo: true,
-      },
-      opts,
-    )
+    const { summary, stats, stderrTail, usage, callMetrics, effortReport } =
+      await runAgentInWorkspace(
+        {
+          dir: root,
+          systemPrompt: job.systemPrompt,
+          userPrompt: job.userPrompt,
+          model: job.model,
+          harness: job.harness,
+          subscriptionToken: job.subscriptionToken,
+          subscriptionBaseUrl: job.subscriptionBaseUrl,
+          ambientAuth: job.ambientAuth,
+          proxyBaseUrl: job.proxyBaseUrl,
+          sessionToken: job.sessionToken,
+          webToolsGuidance: job.webToolsGuidance,
+          webSearchProxy: job.webSearch,
+          guardLimits: job.guardLimits,
+          ...(job.contextFiles ? { contextFiles: job.contextFiles } : {}),
+          multiRepo: true,
+        },
+        opts,
+      )
 
     // Commit forgotten tracked edits, then push + open a PR for each repo the run actually changed.
     const { primaryPushed, primaryPrUrl, peerPullRequests } = await pushMultiRepoLegs(
@@ -807,6 +821,7 @@ export async function runMultiRepoCoding(
           stats,
           ...(usage ? { usage } : {}),
           ...(callMetrics ? { callMetrics } : {}),
+          ...(effortReport ? { effortReport } : {}),
         }
       }
       return {
@@ -822,6 +837,7 @@ export async function runMultiRepoCoding(
         failureCause: 'no-changes',
         ...(usage ? { usage } : {}),
         ...(callMetrics ? { callMetrics } : {}),
+        ...(effortReport ? { effortReport } : {}),
       }
     }
     logger.info('multi-repo: complete', {
@@ -838,6 +854,7 @@ export async function runMultiRepoCoding(
       stats,
       ...(usage ? { usage } : {}),
       ...(callMetrics ? { callMetrics } : {}),
+      ...(effortReport ? { effortReport } : {}),
     }
   })
 }

package/src/effort.ts

@@ -0,0 +1,99 @@
+import { readFile, rm } from 'node:fs/promises'
+import { join } from 'node:path'
+
+// ---------------------------------------------------------------------------
+// The agent effort self-assessment side channel. Every container agent is asked
+// (via the backend-composed system prompt) to end its run by writing a short JSON
+// self-assessment — how hard the work was, what reduced its effectiveness, the key
+// obstacles — to a sentinel file in its working directory. The harness reads it after
+// the agent finishes, removes it (so it never lands in a commit), and forwards it on
+// the job result; the backend records it on the step and surfaces it in run details.
+//
+// The filename is kept in sync with `EFFORT_REPORT_FILE` in `@cat-factory/agents`
+// (the executor-harness has no dependency on that package), exactly like CONTEXT_DIR
+// and the follow-ups sentinel. The shape mirrors the contracts `AgentEffortReport`.
+// ---------------------------------------------------------------------------
+
+/** The sentinel file the agent writes its effort self-assessment to (relative to its cwd). */
+export const EFFORT_REPORT_FILE = '.cat-effort.json'
+
+/** A container agent's self-assessment of the work it just did. */
+export interface EffortReport {
+  /** How hard the work was: 1 (trivial) .. 10 (extremely hard). */
+  difficulty: number
+  /** One or two sentences on how hard/easy the work was and why. */
+  summary?: string
+  /** What reduced the agent's effectiveness. */
+  reducedEffectiveness?: string
+  /** The key obstacles the agent hit. */
+  obstacles?: string[]
+}
+
+/**
+ * Read + parse + REMOVE the agent's effort sentinel file from `cwd`. Lenient: returns undefined
+ * when the file is absent (the agent wrote none), unreadable, not JSON, or carries nothing
+ * meaningful. Never throws — a malformed self-report must never fail an otherwise-good run.
+ */
+export async function readEffortReport(cwd: string): Promise<EffortReport | undefined> {
+  const path = join(cwd, EFFORT_REPORT_FILE)
+  let raw: string
+  try {
+    raw = await readFile(path, 'utf8')
+  } catch {
+    return undefined // no report written — the common case
+  }
+  // Remove it so it never lands in a commit (defence in depth; the backend also excludes it).
+  await rm(path, { force: true }).catch(() => {})
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(raw)
+  } catch {
+    return undefined
+  }
+  return coerceEffort(parsed)
+}
+
+/** Coerce arbitrary parsed JSON into a clean {@link EffortReport}, or undefined when it carries nothing. */
+function coerceEffort(value: unknown): EffortReport | undefined {
+  if (typeof value !== 'object' || value === null) return undefined
+  const o = value as Record<string, unknown>
+  const report: EffortReport = { difficulty: clampDifficulty(o.difficulty) }
+  if (typeof o.summary === 'string' && o.summary.trim()) {
+    report.summary = o.summary.trim().slice(0, 2000)
+  }
+  if (typeof o.reducedEffectiveness === 'string' && o.reducedEffectiveness.trim()) {
+    report.reducedEffectiveness = o.reducedEffectiveness.trim().slice(0, 2000)
+  }
+  if (Array.isArray(o.obstacles)) {
+    const obstacles = o.obstacles
+      .filter((x): x is string => typeof x === 'string' && x.trim().length > 0)
+      .map((x) => x.trim().slice(0, 500))
+      .slice(0, 20)
+    if (obstacles.length) report.obstacles = obstacles
+  }
+  // Nothing beyond a defaulted difficulty ⇒ the agent didn't really report anything; drop it so
+  // run details don't show an empty "5/10, no detail" card for a stray/blank file.
+  if (
+    report.summary === undefined &&
+    report.reducedEffectiveness === undefined &&
+    report.obstacles === undefined &&
+    !isFiniteNumber(o.difficulty)
+  ) {
+    return undefined
+  }
+  return report
+}
+
+function clampDifficulty(v: unknown): number {
+  const n = isFiniteNumber(v)
+    ? v
+    : typeof v === 'string' && v.trim() !== ''
+      ? Number(v)
+      : Number.NaN
+  if (!Number.isFinite(n)) return 5
+  return Math.min(10, Math.max(1, Math.round(n)))
+}
+
+function isFiniteNumber(v: unknown): v is number {
+  return typeof v === 'number' && Number.isFinite(v)
+}

package/src/job.ts

@@ -1,6 +1,7 @@
 import type { HarnessCallMetric, PiRunStats } from './pi.js'
 import type { HarnessKind } from './pi-workspace.js'
 import type { FailureCause } from './failure.js'
+import type { EffortReport } from './effort.js'
 
 // The job the Worker's ContainerAgentExecutor POSTs to /run. Kept as plain
 // types with a hand-rolled validator so the image needs no schema dependency.
@@ -921,6 +922,12 @@ export interface AgentResult {
    * {@link HarnessCallMetric}.
    */
   callMetrics?: HarnessCallMetric[]
+  /**
+   * The agent's effort self-assessment (how hard the work was, what reduced its effectiveness,
+   * the key obstacles), lifted from its sentinel file after the run. The backend forwards it onto
+   * the job result and records it on the step for run details. Absent when the agent wrote none.
+   */
+  effortReport?: EffortReport
 }
 
 /** Parse the coding-mode bootstrap spec, or undefined when absent. Validates the target. */
@@ -1117,6 +1124,24 @@ function isReservedEnvName(key: string): boolean {
   return RESERVED_ENV_PREFIXES.some((p) => lower.startsWith(p))
 }
 
+/**
+ * Collect only string→string entries from a raw `env` bag. A non-string value is dropped so a
+ * malformed binding can't inject `[object Object]` (or undefined) as an upstream URL. Reserved
+ * names that would break the toolchain or enable injection (PATH, NODE_OPTIONS, LD_PRELOAD, …) are
+ * dropped too: they are spread over `process.env` at build time, so a binding named `PATH` would
+ * replace it with a URL and the build would no longer find its tools. Extracted from the infra
+ * parsers to keep their cyclomatic complexity down.
+ */
+function parseInfraEnv(raw: unknown): Record<string, string> {
+  const env: Record<string, string> = {}
+  if (typeof raw === 'object' && raw !== null) {
+    for (const [key, val] of Object.entries(raw as Record<string, unknown>)) {
+      if (key && !isReservedEnvName(key) && typeof val === 'string') env[key] = val
+    }
+  }
+  return env
+}
+
 /** Parse the frontend UI-test infra spec (`kind: 'frontend'`), tolerating missing knobs. */
 function parseFrontendInfraSpec(o: Record<string, unknown>): FrontendInfraSpec {
   const packageManager =
@@ -1126,17 +1151,7 @@ function parseFrontendInfraSpec(o: Record<string, unknown>): FrontendInfraSpec {
   const serveMode = o.serveMode === 'static' || o.serveMode === 'command' ? o.serveMode : undefined
   const envInjection =
     o.envInjection === 'build' || o.envInjection === 'runtime' ? o.envInjection : undefined
-  // Only string→string entries survive; a non-string value is dropped so a malformed
-  // binding can't inject `[object Object]` (or undefined) as an upstream URL. Reserved names
-  // that would break the toolchain or enable injection (PATH, NODE_OPTIONS, LD_PRELOAD, …) are
-  // dropped too: they are spread over `process.env` at build time, so a binding named `PATH`
-  // would replace it with a URL and the build would no longer find its tools.
-  const env: Record<string, string> = {}
-  if (typeof o.env === 'object' && o.env !== null) {
-    for (const [key, val] of Object.entries(o.env as Record<string, unknown>)) {
-      if (key && !isReservedEnvName(key) && typeof val === 'string') env[key] = val
-    }
-  }
+  const env = parseInfraEnv(o.env)
   const servePort = port(o.servePort)
   const wiremockPort = port(o.wiremockPort)
   // The app's monorepo subdirectory becomes the install/build/serve cwd, so it goes through the
@@ -1369,11 +1384,7 @@ function assembleAgentJob(
     ghToken: str(o.ghToken, 'ghToken'),
     repo: parseRepoSpec(repo),
     branch: str(o.branch, 'branch'),
-    ...(typeof o.githubApiBase === 'string' ? { githubApiBase: o.githubApiBase } : {}),
-    ...(typeof o.webToolsGuidance === 'string' ? { webToolsGuidance: o.webToolsGuidance } : {}),
-    ...(o.webSearch === true ? { webSearch: true } : {}),
-    ...(o.full === true ? { full: true } : {}),
-    ...(typeof o.mergeBase === 'string' && o.mergeBase ? { mergeBase: o.mergeBase } : {}),
+    ...collectOptionalRequestFields(o),
     ...(bootstrap ? { bootstrap } : {}),
     ...(output ? { output } : {}),
     ...(contextFiles.length ? { contextFiles } : {}),
@@ -1381,20 +1392,36 @@ function assembleAgentJob(
     ...(skill ? { skill } : {}),
     ...(testSecrets.length ? { testSecrets } : {}),
     ...(infra ? { infra } : {}),
-    ...(typeof o.newBranch === 'string' && o.newBranch ? { newBranch: o.newBranch } : {}),
-    ...(typeof o.pushBranch === 'string' && o.pushBranch ? { pushBranch: o.pushBranch } : {}),
-    ...(typeof o.commitMessage === 'string' && o.commitMessage
-      ? { commitMessage: o.commitMessage }
-      : {}),
     ...(pr ? { pr } : {}),
     ...(peerRepos.length ? { peerRepos } : {}),
     ...(referenceRepos.length ? { referenceRepos } : {}),
     ...(referenceBranches.length ? { referenceBranches } : {}),
     ...(reviewPrNumber !== undefined ? { reviewPrNumber } : {}),
+    ...(guardLimits ? { guardLimits } : {}),
+    ...(validation ? { validation } : {}),
+  }
+}
+
+/**
+ * The optional {@link AgentJob} fields read directly off the request `o` (booleans + trimmed
+ * strings). Extracted from {@link assembleAgentJob} to keep its cyclomatic complexity down; every
+ * key is unique so grouping the conditional spreads is behaviour-neutral (spread order is
+ * irrelevant with no colliding keys).
+ */
+function collectOptionalRequestFields(o: Record<string, unknown>): Partial<AgentJob> {
+  return {
+    ...(typeof o.githubApiBase === 'string' ? { githubApiBase: o.githubApiBase } : {}),
+    ...(typeof o.webToolsGuidance === 'string' ? { webToolsGuidance: o.webToolsGuidance } : {}),
+    ...(o.webSearch === true ? { webSearch: true } : {}),
+    ...(o.full === true ? { full: true } : {}),
+    ...(typeof o.mergeBase === 'string' && o.mergeBase ? { mergeBase: o.mergeBase } : {}),
+    ...(typeof o.newBranch === 'string' && o.newBranch ? { newBranch: o.newBranch } : {}),
+    ...(typeof o.pushBranch === 'string' && o.pushBranch ? { pushBranch: o.pushBranch } : {}),
+    ...(typeof o.commitMessage === 'string' && o.commitMessage
+      ? { commitMessage: o.commitMessage }
+      : {}),
     ...(o.noChangesIsError === false ? { noChangesIsError: false } : {}),
     ...(o.persistentCheckout === true ? { persistentCheckout: true } : {}),
     ...(o.streamFollowUps === true ? { streamFollowUps: true } : {}),
-    ...(guardLimits ? { guardLimits } : {}),
-    ...(validation ? { validation } : {}),
   }
 }

package/src/pi-workspace.ts

@@ -2,6 +2,7 @@ import { mkdir, mkdtemp, rm } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import type { RepoSpec, SkillSpec } from './job.js'
+import { readEffortReport } from './effort.js'
 import { log } from './logger.js'
 import {
   type ContextFileInfo,
@@ -258,7 +259,7 @@ export async function runAgentInWorkspace(
     if (!spec.ambientAuth && !spec.subscriptionToken) {
       throw new Error(`The ${spec.harness} harness requires a subscription token`)
     }
-    return runSubscriptionHarness(spec.harness, {
+    const subOutcome = await runSubscriptionHarness(spec.harness, {
       cwd: spec.dir,
       model: spec.model,
       systemPrompt: subscriptionSystemPrompt(spec.systemPrompt, contextFiles),
@@ -272,6 +273,7 @@ export async function runAgentInWorkspace(
       onProgress: opts.onProgress,
       ...(opts.log ? { log: opts.log } : {}),
     })
+    return withEffortReport(spec.dir, subOutcome)
   }
   if (!spec.proxyBaseUrl || !spec.sessionToken) {
     throw new Error('The Pi harness requires proxyBaseUrl and sessionToken')
@@ -301,7 +303,7 @@ export async function runAgentInWorkspace(
   })
   await writePiModelsConfig({ model: spec.model, proxyBaseUrl })
   const { signal, onActivity, onProgress, onSpan } = opts
-  return runPi({
+  const piOutcome = await runPi({
     cwd: spec.dir,
     model: spec.model,
     userPrompt: spec.userPrompt,
@@ -316,6 +318,17 @@ export async function runAgentInWorkspace(
     guardLimits: mergeGuardLimits(progressGuardLimitsFromEnv(), spec.guardLimits),
     extraEnv,
   })
+  return withEffortReport(spec.dir, piOutcome)
+}
+
+/**
+ * Lift the agent's effort self-assessment off its sentinel file in `dir` and fold it onto the
+ * run outcome. Shared by both harness paths so EVERY container agent's effort report is captured
+ * in one place. Never throws (a bad/absent report just yields no `effortReport`).
+ */
+async function withEffortReport(dir: string, outcome: PiRunOutcome): Promise<PiRunOutcome> {
+  const effortReport = await readEffortReport(dir)
+  return effortReport ? { ...outcome, effortReport } : outcome
 }
 
 /**

package/src/pi.ts

@@ -7,6 +7,7 @@ import { pathExists } from './fs-utils.js'
 import { redactSecrets } from './redact.js'
 import { HarnessFailure } from './failure.js'
 import { log } from './logger.js'
+import type { EffortReport } from './effort.js'
 
 // Drives the Pi coding-agent CLI. Pi is pointed at the Worker's OpenAI-compatible
 // proxy via a custom provider in ~/.pi/agent/models.json, authenticated with the
@@ -534,6 +535,12 @@ export interface PiRunOutcome {
   callMetrics?: HarnessCallMetric[]
   /** Output-quality signals (truncation / empty final answer); see {@link RunDiagnostics}. */
   diagnostics?: RunDiagnostics
+  /**
+   * The agent's effort self-assessment, lifted from its sentinel file after the run (how hard the
+   * work was, what reduced its effectiveness, the key obstacles). Absent when the agent wrote none.
+   * See {@link EffortReport}.
+   */
+  effortReport?: EffortReport
 }
 
 /**