@cat-factory/executor-harness 1.52.2 → 1.56.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -1
- package/dist/agent-runner.js +14 -11
- package/dist/agent.js +96 -43
- package/dist/coding-agent.js +107 -18
- package/dist/frontend-infra.js +9 -2
- package/dist/job.js +48 -1
- package/dist/package-registries.js +78 -13
- package/dist/pi-workspace.js +24 -8
- package/dist/pi.js +4 -3
- package/dist/runner.js +3 -0
- package/dist/validation-checks.js +300 -0
- package/package.json +3 -3
- package/src/agent-runner.ts +25 -13
- package/src/agent.ts +107 -42
- package/src/coding-agent.ts +134 -8
- package/src/frontend-infra.ts +10 -3
- package/src/job.ts +66 -0
- package/src/package-registries.ts +95 -15
- package/src/pi-workspace.ts +27 -8
- package/src/pi.ts +4 -3
- package/src/runner.ts +29 -0
- package/src/validation-checks.ts +395 -0
package/src/agent-runner.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process'
|
|
2
2
|
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
|
|
3
|
-
import {
|
|
3
|
+
import { tmpdir } from 'node:os'
|
|
4
4
|
import { dirname, join } from 'node:path'
|
|
5
5
|
import {
|
|
6
6
|
claudeAssistantContent,
|
|
@@ -81,8 +81,9 @@ export interface SubscriptionRunOptions {
|
|
|
81
81
|
/**
|
|
82
82
|
* A repo-sourced Claude Skill to install natively before launch (repo-sourced Claude Skills,
|
|
83
83
|
* slice 2). The claude-code runner writes it to `CLAUDE_CONFIG_DIR/skills/<name>/SKILL.md`
|
|
84
|
-
* (+ resource files) so the CLI loads it
|
|
85
|
-
*
|
|
84
|
+
* (+ resource files) so the CLI loads it — but ONLY when it owns an isolated config home, i.e.
|
|
85
|
+
* NOT under `ambientAuth`. The codex runner ignores it outright. Every case that skips the
|
|
86
|
+
* native install reads the checkout's `.cat-context/skill/`, materialised by the caller.
|
|
86
87
|
*/
|
|
87
88
|
skill?: {
|
|
88
89
|
name: string
|
|
@@ -90,6 +91,14 @@ export interface SubscriptionRunOptions {
|
|
|
90
91
|
instructions: string
|
|
91
92
|
resources: { relPath: string; content: string }[]
|
|
92
93
|
}
|
|
94
|
+
/**
|
|
95
|
+
* Extra environment for the CLI child, scoped to this job (the tester's secrets, a
|
|
96
|
+
* private-registry npmrc pointer). Merged over the inherited `process.env` at spawn, so the
|
|
97
|
+
* agent and its shell tools see them without the harness mutating its OWN environment — which
|
|
98
|
+
* is shared by every concurrent job under the native host-process transport. See
|
|
99
|
+
* `RunOptions.agentEnv`.
|
|
100
|
+
*/
|
|
101
|
+
extraEnv?: Record<string, string>
|
|
93
102
|
/** Aborting this kills the CLI (the job's inactivity/max-duration watchdog). */
|
|
94
103
|
signal?: AbortSignal
|
|
95
104
|
/** Called on every chunk of CLI output, so the watchdog sees the agent is alive. */
|
|
@@ -449,14 +458,14 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
449
458
|
}
|
|
450
459
|
|
|
451
460
|
// Repo-sourced Claude Skill (slice 2): install it as a native skill under the config dir's
|
|
452
|
-
// `skills/<name>/` so the CLI discovers and can invoke it.
|
|
453
|
-
//
|
|
454
|
-
//
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
await writeNativeSkill(
|
|
461
|
+
// `skills/<name>/` so the CLI discovers and can invoke it. ONLY into the isolated per-run config
|
|
462
|
+
// home — never the developer's own `~/.claude` (ambient/native mode), where it would persist in
|
|
463
|
+
// their personal setup after the run and two concurrent jobs carrying same-named skills from
|
|
464
|
+
// different repos would clobber each other. An ambient run reads the skill from the checkout
|
|
465
|
+
// instead (`.cat-context/skill/`, materialised by the caller). Best-effort: a write failure must
|
|
466
|
+
// not wedge the run — the prompt still names the skill.
|
|
467
|
+
if (opts.skill && configHome) {
|
|
468
|
+
await writeNativeSkill(join(configHome, 'skills'), opts.skill).catch(() => {})
|
|
460
469
|
}
|
|
461
470
|
|
|
462
471
|
const env = buildClaudeEnv(opts, configHome)
|
|
@@ -542,8 +551,11 @@ function buildClaudeEnv(
|
|
|
542
551
|
opts: SubscriptionRunOptions,
|
|
543
552
|
configHome: string | undefined,
|
|
544
553
|
): Record<string, string> {
|
|
545
|
-
|
|
554
|
+
// The job-scoped env rides along in BOTH modes; the credential/config vars below are what
|
|
555
|
+
// ambient mode drops (the developer's own logged-in `~/.claude` is used instead).
|
|
556
|
+
if (opts.ambientAuth) return { ...opts.extraEnv }
|
|
546
557
|
return {
|
|
558
|
+
...opts.extraEnv,
|
|
547
559
|
CLAUDE_CONFIG_DIR: configHome!,
|
|
548
560
|
...(opts.subscriptionBaseUrl
|
|
549
561
|
? {
|
|
@@ -738,7 +750,7 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
|
|
|
738
750
|
},
|
|
739
751
|
prompt,
|
|
740
752
|
opts,
|
|
741
|
-
codexHome ? { CODEX_HOME: codexHome } : {},
|
|
753
|
+
{ ...opts.extraEnv, ...(codexHome ? { CODEX_HOME: codexHome } : {}) },
|
|
742
754
|
opts.subscriptionToken ? secretsToRedact(opts.subscriptionToken) : [],
|
|
743
755
|
onEvent,
|
|
744
756
|
)
|
package/src/agent.ts
CHANGED
|
@@ -38,6 +38,7 @@ import {
|
|
|
38
38
|
runCodingAgent,
|
|
39
39
|
runMultiRepoCoding,
|
|
40
40
|
} from './coding-agent.js'
|
|
41
|
+
import { validationFailureMessage } from './validation-checks.js'
|
|
41
42
|
import {
|
|
42
43
|
acquireRepoCheckout,
|
|
43
44
|
agentNeverActed,
|
|
@@ -155,8 +156,7 @@ async function manageInfra(
|
|
|
155
156
|
dir: string,
|
|
156
157
|
workDir: string,
|
|
157
158
|
infra: AgentInfraSpec,
|
|
158
|
-
|
|
159
|
-
onActivity: (() => void) | undefined,
|
|
159
|
+
opts: RunOptions,
|
|
160
160
|
logger: Logger,
|
|
161
161
|
): Promise<{
|
|
162
162
|
note?: string
|
|
@@ -168,7 +168,7 @@ async function manageInfra(
|
|
|
168
168
|
// `onActivity` feeds the inactivity watchdog through the frontend build/serve stand-up,
|
|
169
169
|
// which (unlike docker-compose's 5-min-capped `up`) can run past the inactivity window.
|
|
170
170
|
// Runs in `workDir` so a monorepo frontend builds/serves from its own package subtree.
|
|
171
|
-
const fe = await standUpFrontend(workDir, infra,
|
|
171
|
+
const fe = await standUpFrontend(workDir, infra, opts, logger)
|
|
172
172
|
return {
|
|
173
173
|
...(fe.note ? { note: fe.note } : {}),
|
|
174
174
|
...(fe.serveUrl ? { serveUrl: fe.serveUrl } : {}),
|
|
@@ -176,7 +176,7 @@ async function manageInfra(
|
|
|
176
176
|
cleanup: () => tearDownFrontend(fe.processes, logger),
|
|
177
177
|
}
|
|
178
178
|
}
|
|
179
|
-
const standUp = await standUpInfra(dir, infra, signal, logger)
|
|
179
|
+
const standUp = await standUpInfra(dir, infra, opts.signal, logger)
|
|
180
180
|
return {
|
|
181
181
|
...(standUp.note ? { note: standUp.note } : {}),
|
|
182
182
|
...(standUp.record ? { record: standUp.record } : {}),
|
|
@@ -324,13 +324,39 @@ function mergeEffort(result: AgentResult, effortReport: EffortReport | undefined
|
|
|
324
324
|
|
|
325
325
|
/** Run one generic agent job end to end, dispatching on `mode`. */
|
|
326
326
|
export async function handleAgent(job: AgentJob, opts: RunOptions = {}): Promise<AgentResult> {
|
|
327
|
-
//
|
|
328
|
-
//
|
|
329
|
-
//
|
|
330
|
-
//
|
|
331
|
-
await
|
|
332
|
-
|
|
333
|
-
|
|
327
|
+
// An `ambientAuth` job runs in the SHARED native host process on the developer's own HOME
|
|
328
|
+
// (see `LocalProcessRunnerTransport`), so anything this job would otherwise write to a
|
|
329
|
+
// process- or HOME-global gets a per-job directory instead — it can't corrupt the
|
|
330
|
+
// developer's files, and concurrent jobs can't race on them.
|
|
331
|
+
const scopeDir = job.ambientAuth ? await mkdtemp(join(tmpdir(), 'cf-jobenv-')) : undefined
|
|
332
|
+
try {
|
|
333
|
+
// Private-registry auth first, before any mode runs: every mode with a checkout may
|
|
334
|
+
// install dependencies (the agent's own shell and the frontend-infra stand-up both
|
|
335
|
+
// inherit this env, so they all read the written npmrc). In a container a job with no
|
|
336
|
+
// entries clears any stale ~/.npmrc from a prior job on a reused (warm-pool) container.
|
|
337
|
+
const registryEnv = await configurePackageRegistries(
|
|
338
|
+
job.packageRegistries,
|
|
339
|
+
scopeDir ? { isolatedDir: scopeDir } : {},
|
|
340
|
+
)
|
|
341
|
+
const scoped = withAgentEnv(opts, registryEnv)
|
|
342
|
+
if (job.mode === 'preview') return await runPreviewMode(job, scoped)
|
|
343
|
+
return job.mode === 'coding'
|
|
344
|
+
? await runCodingMode(job, scoped)
|
|
345
|
+
: await runExploreMode(job, scoped)
|
|
346
|
+
} finally {
|
|
347
|
+
if (scopeDir) await rm(scopeDir, { recursive: true, force: true }).catch(() => {})
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* Layer extra child-process env onto a job's {@link RunOptions}. The agent CLI is spawned with
|
|
353
|
+
* `{...process.env, ...agentEnv}`, so this is how per-job values reach the agent (and the shell
|
|
354
|
+
* tools it spawns) WITHOUT mutating the harness's own `process.env` — which is shared by every
|
|
355
|
+
* concurrent job when the harness runs as a native host process. Empty `env` ⇒ `opts` unchanged.
|
|
356
|
+
*/
|
|
357
|
+
function withAgentEnv(opts: RunOptions, env: Record<string, string>): RunOptions {
|
|
358
|
+
if (Object.keys(env).length === 0) return opts
|
|
359
|
+
return { ...opts, agentEnv: { ...opts.agentEnv, ...env } }
|
|
334
360
|
}
|
|
335
361
|
|
|
336
362
|
/**
|
|
@@ -391,7 +417,7 @@ async function runPreviewMode(job: AgentJob, opts: RunOptions): Promise<AgentRes
|
|
|
391
417
|
logger.info('agent(preview): building + serving', {
|
|
392
418
|
serviceDirectory: job.repo.serviceDirectory,
|
|
393
419
|
})
|
|
394
|
-
const fe = await standUpFrontend(workDir, infra, opts
|
|
420
|
+
const fe = await standUpFrontend(workDir, infra, opts, logger)
|
|
395
421
|
const infraSetupFields: { infraSetup?: InfraSetupRecord } = fe.record
|
|
396
422
|
? { infraSetup: fe.record }
|
|
397
423
|
: {}
|
|
@@ -423,27 +449,22 @@ async function runPreviewMode(job: AgentJob, opts: RunOptions): Promise<AgentRes
|
|
|
423
449
|
}
|
|
424
450
|
|
|
425
451
|
/**
|
|
426
|
-
*
|
|
427
|
-
*
|
|
428
|
-
*
|
|
429
|
-
*
|
|
430
|
-
*
|
|
431
|
-
*
|
|
452
|
+
* Build the env carrying the tester's sensitive secrets, so the agent's shell tools (spawned as
|
|
453
|
+
* child processes that inherit it) can read `$KEY` — the out-of-band delivery channel. Each value
|
|
454
|
+
* is registered for redaction so it can't leak into captured output/logs. Reserved/toolchain env
|
|
455
|
+
* names were already dropped at parse. No secrets ⇒ an empty env.
|
|
456
|
+
*
|
|
457
|
+
* Returned as EXPLICIT child env rather than written onto `process.env`: a process-global
|
|
458
|
+
* set/restore is only safe when the process runs one job, which the native host-process transport
|
|
459
|
+
* breaks (it serves every concurrent ambient job from one process). There, two overlapping tester
|
|
460
|
+
* runs would read each other's secrets, and whichever finished first would delete the other's
|
|
461
|
+
* mid-run. Scoping them to the spawn env makes the delivery correct under concurrency and drops
|
|
462
|
+
* the restore step entirely.
|
|
432
463
|
*/
|
|
433
|
-
function
|
|
434
|
-
if (!secrets?.length) return
|
|
464
|
+
export function testSecretEnv(secrets: TestSecretSpec[] | undefined): Record<string, string> {
|
|
465
|
+
if (!secrets?.length) return {}
|
|
435
466
|
registerKnownSecrets(secrets.map((s) => s.value))
|
|
436
|
-
|
|
437
|
-
for (const { key, value } of secrets) {
|
|
438
|
-
previous.set(key, process.env[key])
|
|
439
|
-
process.env[key] = value
|
|
440
|
-
}
|
|
441
|
-
return () => {
|
|
442
|
-
for (const [key, prior] of previous) {
|
|
443
|
-
if (prior === undefined) delete process.env[key]
|
|
444
|
-
else process.env[key] = prior
|
|
445
|
-
}
|
|
446
|
-
}
|
|
467
|
+
return Object.fromEntries(secrets.map(({ key, value }) => [key, value]))
|
|
447
468
|
}
|
|
448
469
|
|
|
449
470
|
/**
|
|
@@ -536,9 +557,7 @@ async function runExploreMode(job: AgentJob, opts: RunOptions): Promise<AgentRes
|
|
|
536
557
|
// The run-mode guidance itself lives in the backend-composed system/user prompt; the
|
|
537
558
|
// harness only manages the lifecycle + this dynamic stand-up note.
|
|
538
559
|
const infra = job.infra
|
|
539
|
-
const managed = infra
|
|
540
|
-
? await manageInfra(dir, workDir, infra, opts.signal, opts.onActivity, logger)
|
|
541
|
-
: undefined
|
|
560
|
+
const managed = infra ? await manageInfra(dir, workDir, infra, opts, logger) : undefined
|
|
542
561
|
// Fold the stand-up outcome into the agent prompt: a stand-up problem (build/compose
|
|
543
562
|
// failure) is flagged as a concern; a frontend serve URL points the UI tester at the
|
|
544
563
|
// app it just built + served (the backend env resolution already reached the harness).
|
|
@@ -553,10 +572,10 @@ async function runExploreMode(job: AgentJob, opts: RunOptions): Promise<AgentRes
|
|
|
553
572
|
? { infraSetup: managed.record }
|
|
554
573
|
: {}
|
|
555
574
|
|
|
556
|
-
//
|
|
557
|
-
// shell can read them as `$KEY
|
|
558
|
-
//
|
|
559
|
-
const
|
|
575
|
+
// Hand the tester's sensitive secrets to the agent's child process (out of band) so its
|
|
576
|
+
// shell can read them as `$KEY`. Scoped to this job's env, so a concurrent job in the same
|
|
577
|
+
// harness process never sees them. A no-op for non-tester runs (no `testSecrets`).
|
|
578
|
+
const agentOpts = withAgentEnv(opts, testSecretEnv(job.testSecrets))
|
|
560
579
|
|
|
561
580
|
try {
|
|
562
581
|
opts.onPhase?.('agent')
|
|
@@ -590,7 +609,7 @@ async function runExploreMode(job: AgentJob, opts: RunOptions): Promise<AgentRes
|
|
|
590
609
|
contextFiles: job.contextFiles,
|
|
591
610
|
guardLimits: job.guardLimits,
|
|
592
611
|
},
|
|
593
|
-
|
|
612
|
+
agentOpts,
|
|
594
613
|
)
|
|
595
614
|
|
|
596
615
|
return mergeEffort(
|
|
@@ -602,7 +621,6 @@ async function runExploreMode(job: AgentJob, opts: RunOptions): Promise<AgentRes
|
|
|
602
621
|
effortReport,
|
|
603
622
|
)
|
|
604
623
|
} finally {
|
|
605
|
-
restoreSecrets()
|
|
606
624
|
if (managed) await managed.cleanup()
|
|
607
625
|
}
|
|
608
626
|
},
|
|
@@ -949,6 +967,11 @@ function buildSingleRepoCodingSpec(
|
|
|
949
967
|
},
|
|
950
968
|
}
|
|
951
969
|
: {}),
|
|
970
|
+
// Pre-PR validation: the service's check commands, run against the checkout BEFORE the PR
|
|
971
|
+
// opens with failures fed back to the agent (see docs/initiatives/pre-pr-validation.md).
|
|
972
|
+
// Forwarded straight off the job body — the loop is generic machinery keyed on the data, not
|
|
973
|
+
// on the agent kind.
|
|
974
|
+
...(job.validationChecks ? { validationChecks: job.validationChecks } : {}),
|
|
952
975
|
}
|
|
953
976
|
}
|
|
954
977
|
|
|
@@ -960,13 +983,49 @@ function buildSingleRepoCodingSpec(
|
|
|
960
983
|
*/
|
|
961
984
|
async function runSingleRepoCoding(job: AgentJob, opts: RunOptions): Promise<AgentResult> {
|
|
962
985
|
const pushBranch = job.pushBranch ?? job.newBranch ?? job.branch
|
|
963
|
-
const {
|
|
964
|
-
|
|
986
|
+
const {
|
|
987
|
+
summary,
|
|
988
|
+
stats,
|
|
989
|
+
stderrTail,
|
|
990
|
+
pushed,
|
|
991
|
+
usage,
|
|
992
|
+
callMetrics,
|
|
993
|
+
validation,
|
|
994
|
+
validationReport,
|
|
995
|
+
effortReport,
|
|
996
|
+
} = await runCodingAgent(buildSingleRepoCodingSpec(job, pushBranch), opts)
|
|
965
997
|
// Ralph loop: the harness-computed validation verdict, forwarded onto the coding result as
|
|
966
998
|
// `ralphVerdict` so the backend's `toRunResult` lifts it onto `AgentRunResult.ralphVerdict`.
|
|
967
999
|
const ralphVerdict = validation ? { ralphVerdict: validation } : {}
|
|
968
1000
|
// The agent's effort self-assessment, spread onto every result path below (mirrors ralphVerdict).
|
|
969
1001
|
const effort = effortReport ? { effortReport } : {}
|
|
1002
|
+
// The pre-PR validation report, spread onto every result path below: on the passing path it is
|
|
1003
|
+
// the captured proof the checkout was green when the PR opened; on the exhausted path it is the
|
|
1004
|
+
// evidence behind the failure below. Absent when the service configured no checks.
|
|
1005
|
+
const validationFields = validationReport ? { validationReport } : {}
|
|
1006
|
+
|
|
1007
|
+
// Pre-PR validation spent its attempt budget with the checkout still red. FAIL the job — do
|
|
1008
|
+
// NOT open a pull request, and do not pretend the push succeeded as a deliverable. The work is
|
|
1009
|
+
// still on the branch (a retry resumes on it); the report carries each failing command's exit
|
|
1010
|
+
// code and captured output so the step's failure detail says exactly what broke.
|
|
1011
|
+
if (validationReport && !validationReport.passed) {
|
|
1012
|
+
return {
|
|
1013
|
+
// The work IS on the branch (the loop only runs for a pass that produced some, and the
|
|
1014
|
+
// harness pushes it) — a retry resumes on top of it. `error` is what marks the job failed;
|
|
1015
|
+
// reporting `pushed: false` here would misdescribe the branch state in the harness's own
|
|
1016
|
+
// result for no benefit.
|
|
1017
|
+
pushed,
|
|
1018
|
+
branch: pushBranch,
|
|
1019
|
+
summary,
|
|
1020
|
+
stats,
|
|
1021
|
+
error: validationFailureMessage(validationReport),
|
|
1022
|
+
failureCause: 'agent',
|
|
1023
|
+
...(usage ? { usage } : {}),
|
|
1024
|
+
...(callMetrics ? { callMetrics } : {}),
|
|
1025
|
+
...validationFields,
|
|
1026
|
+
...effort,
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
970
1029
|
|
|
971
1030
|
if (!pushed) {
|
|
972
1031
|
// A no-op: a failure for the implementer, a clean non-event for the fixers.
|
|
@@ -979,6 +1038,7 @@ async function runSingleRepoCoding(job: AgentJob, opts: RunOptions): Promise<Age
|
|
|
979
1038
|
...(usage ? { usage } : {}),
|
|
980
1039
|
...(callMetrics ? { callMetrics } : {}),
|
|
981
1040
|
...ralphVerdict,
|
|
1041
|
+
...validationFields,
|
|
982
1042
|
...effort,
|
|
983
1043
|
}
|
|
984
1044
|
}
|
|
@@ -991,6 +1051,7 @@ async function runSingleRepoCoding(job: AgentJob, opts: RunOptions): Promise<Age
|
|
|
991
1051
|
failureCause: 'no-changes',
|
|
992
1052
|
...(usage ? { usage } : {}),
|
|
993
1053
|
...(callMetrics ? { callMetrics } : {}),
|
|
1054
|
+
...validationFields,
|
|
994
1055
|
...effort,
|
|
995
1056
|
}
|
|
996
1057
|
}
|
|
@@ -1025,6 +1086,7 @@ async function runSingleRepoCoding(job: AgentJob, opts: RunOptions): Promise<Age
|
|
|
1025
1086
|
stats,
|
|
1026
1087
|
...(usage ? { usage } : {}),
|
|
1027
1088
|
...(callMetrics ? { callMetrics } : {}),
|
|
1089
|
+
...validationFields,
|
|
1028
1090
|
...effort,
|
|
1029
1091
|
}
|
|
1030
1092
|
}
|
|
@@ -1041,6 +1103,7 @@ async function runSingleRepoCoding(job: AgentJob, opts: RunOptions): Promise<Age
|
|
|
1041
1103
|
failureCause: 'no-changes',
|
|
1042
1104
|
...(usage ? { usage } : {}),
|
|
1043
1105
|
...(callMetrics ? { callMetrics } : {}),
|
|
1106
|
+
...validationFields,
|
|
1044
1107
|
...effort,
|
|
1045
1108
|
}
|
|
1046
1109
|
}
|
|
@@ -1053,6 +1116,7 @@ async function runSingleRepoCoding(job: AgentJob, opts: RunOptions): Promise<Age
|
|
|
1053
1116
|
...(usage ? { usage } : {}),
|
|
1054
1117
|
...(callMetrics ? { callMetrics } : {}),
|
|
1055
1118
|
...ralphVerdict,
|
|
1119
|
+
...validationFields,
|
|
1056
1120
|
...effort,
|
|
1057
1121
|
}
|
|
1058
1122
|
}
|
|
@@ -1064,6 +1128,7 @@ async function runSingleRepoCoding(job: AgentJob, opts: RunOptions): Promise<Age
|
|
|
1064
1128
|
...(usage ? { usage } : {}),
|
|
1065
1129
|
...(callMetrics ? { callMetrics } : {}),
|
|
1066
1130
|
...ralphVerdict,
|
|
1131
|
+
...validationFields,
|
|
1067
1132
|
...effort,
|
|
1068
1133
|
}
|
|
1069
1134
|
}
|
package/src/coding-agent.ts
CHANGED
|
@@ -42,6 +42,11 @@ import {
|
|
|
42
42
|
import type { ProgressGuardLimits } from './pi.js'
|
|
43
43
|
import type { RunOptions } from './runner.js'
|
|
44
44
|
import { log, type Logger } from './logger.js'
|
|
45
|
+
import {
|
|
46
|
+
runValidationLoop,
|
|
47
|
+
type ValidationChecksSpec,
|
|
48
|
+
type ValidationReport,
|
|
49
|
+
} from './validation-checks.js'
|
|
45
50
|
|
|
46
51
|
// The shared skeleton for the container coding agents that clone a repo, run Pi
|
|
47
52
|
// against it and push the result on a branch. The implementation (`/run`) and
|
|
@@ -103,10 +108,20 @@ export interface CodingAgentSpec extends HarnessAuthFields {
|
|
|
103
108
|
* condition — computed by the harness, never the model). Absent for every non-`ralph` run.
|
|
104
109
|
*/
|
|
105
110
|
validation?: { command: string; iteration?: number }
|
|
111
|
+
/**
|
|
112
|
+
* PRE-PR VALIDATION: the service's configured check commands + repair-round budget. When set,
|
|
113
|
+
* the harness runs them against the checkout after the agent settles and, while they fail and
|
|
114
|
+
* budget remains, re-runs the agent with the captured output as its instruction. A red checkout
|
|
115
|
+
* at the end means the caller opens NO pull request and fails the job. Set only for a dispatch
|
|
116
|
+
* that opens a PR and whose service configured checks; absent everywhere else. See
|
|
117
|
+
* `docs/initiatives/pre-pr-validation.md`.
|
|
118
|
+
*/
|
|
119
|
+
validationChecks?: ValidationChecksSpec
|
|
106
120
|
/**
|
|
107
121
|
* A repo-sourced Claude Skill to make available for this run (a `skill` step, slice 2). Threaded
|
|
108
|
-
* into {@link runAgentInWorkspace}, which installs it harness-aware
|
|
109
|
-
* for claude-code, `.cat-context/skill/` for
|
|
122
|
+
* into {@link runAgentInWorkspace}, which installs it harness-aware: natively under the ISOLATED
|
|
123
|
+
* `CLAUDE_CONFIG_DIR` for a leased-credential claude-code run, `.cat-context/skill/` for everything
|
|
124
|
+
* else (Pi, codex, and ambient claude-code, which has no isolated config dir). Absent ⇒ no skill.
|
|
110
125
|
*/
|
|
111
126
|
skill?: SkillSpec
|
|
112
127
|
}
|
|
@@ -137,6 +152,12 @@ export interface CodingAgentOutcome {
|
|
|
137
152
|
validationOutputTail?: string
|
|
138
153
|
iteration?: number
|
|
139
154
|
}
|
|
155
|
+
/**
|
|
156
|
+
* The pre-PR validation loop's LAST attempt (present only when {@link CodingAgentSpec.validationChecks}
|
|
157
|
+
* was set). `passed: false` means the attempt budget was spent with the checkout still red —
|
|
158
|
+
* the caller must open no PR and fail the job with this as the evidence.
|
|
159
|
+
*/
|
|
160
|
+
validationReport?: ValidationReport
|
|
140
161
|
}
|
|
141
162
|
|
|
142
163
|
/**
|
|
@@ -270,15 +291,17 @@ export async function runCodingAgent(
|
|
|
270
291
|
followUpTick.unref?.()
|
|
271
292
|
}
|
|
272
293
|
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
294
|
+
// One agent pass over this checkout, parameterised only by the prompt — so the pre-PR
|
|
295
|
+
// validation loop below can re-run the agent with a repair instruction without
|
|
296
|
+
// re-deriving (or drifting from) the dispatch's own settings.
|
|
297
|
+
const runAgentPass = (
|
|
298
|
+
userPrompt: string,
|
|
299
|
+
): Promise<Awaited<ReturnType<typeof runAgentInWorkspace>>> =>
|
|
300
|
+
runAgentInWorkspace(
|
|
278
301
|
{
|
|
279
302
|
dir: workDir,
|
|
280
303
|
systemPrompt: spec.systemPrompt,
|
|
281
|
-
userPrompt
|
|
304
|
+
userPrompt,
|
|
282
305
|
model: spec.model,
|
|
283
306
|
harness: spec.harness,
|
|
284
307
|
subscriptionToken: spec.subscriptionToken,
|
|
@@ -294,7 +317,38 @@ export async function runCodingAgent(
|
|
|
294
317
|
},
|
|
295
318
|
opts,
|
|
296
319
|
)
|
|
320
|
+
|
|
321
|
+
let outcome: CodingAgentOutcome
|
|
322
|
+
try {
|
|
323
|
+
opts.onPhase?.('agent')
|
|
324
|
+
logger.info('coding-agent: running agent', { serviceDirectory })
|
|
325
|
+
let agentRun = await runAgentPass(spec.userPrompt)
|
|
326
|
+
// PRE-PR VALIDATION: run the service's configured checks against the checkout and, while
|
|
327
|
+
// they fail and budget remains, hand the captured output back to the agent and run it
|
|
328
|
+
// again. Sits BETWEEN the agent and the finalize/push/PR step so a red checkout never
|
|
329
|
+
// reaches `openPullRequest` — the whole point of the feature. Keyed purely off the job
|
|
330
|
+
// body carrying checks (no agent-kind switch); absent ⇒ this is a no-op and the flow
|
|
331
|
+
// below is byte-for-byte what it was.
|
|
332
|
+
const validationChecks = spec.validationChecks
|
|
333
|
+
let validationReport: ValidationReport | undefined
|
|
334
|
+
if (validationChecks && (await producedWork(dir, spec, baseSha, resumed, opts))) {
|
|
335
|
+
validationReport = await runValidationLoop({
|
|
336
|
+
workDir,
|
|
337
|
+
spec: validationChecks,
|
|
338
|
+
logger,
|
|
339
|
+
opts,
|
|
340
|
+
runAgentPass,
|
|
341
|
+
onAgentPass: (run) => {
|
|
342
|
+
agentRun = mergeAgentPasses(agentRun, run)
|
|
343
|
+
},
|
|
344
|
+
// The checks run against the WORKING TREE, but only tracked edits are staged for the
|
|
345
|
+
// push — so a repair round can go green on a new file the PR would never contain.
|
|
346
|
+
// Name those files in the next repair prompt so the agent adds them.
|
|
347
|
+
listUncommittedNewFiles: () => listUntrackedFiles(workDir, opts.signal),
|
|
348
|
+
})
|
|
349
|
+
}
|
|
297
350
|
outcome = await finalizeCodingRun({
|
|
351
|
+
validationReport,
|
|
298
352
|
dir,
|
|
299
353
|
spec,
|
|
300
354
|
logger,
|
|
@@ -443,6 +497,8 @@ async function prepareCodingCheckout(
|
|
|
443
497
|
* {@link runCodingAgent} so its body stays small; returns the built {@link CodingAgentOutcome}.
|
|
444
498
|
*/
|
|
445
499
|
async function finalizeCodingRun(args: {
|
|
500
|
+
/** The pre-PR validation loop's last attempt, attached to the outcome (absent when unconfigured). */
|
|
501
|
+
validationReport?: ValidationReport
|
|
446
502
|
dir: string
|
|
447
503
|
spec: CodingAgentSpec
|
|
448
504
|
logger: Logger
|
|
@@ -458,6 +514,7 @@ async function finalizeCodingRun(args: {
|
|
|
458
514
|
agentRun: Awaited<ReturnType<typeof runAgentInWorkspace>>
|
|
459
515
|
}): Promise<CodingAgentOutcome> {
|
|
460
516
|
const {
|
|
517
|
+
validationReport,
|
|
461
518
|
dir,
|
|
462
519
|
spec,
|
|
463
520
|
logger,
|
|
@@ -557,9 +614,74 @@ async function finalizeCodingRun(args: {
|
|
|
557
614
|
if (spec.validation) {
|
|
558
615
|
outcome.validation = await runRalphValidation(workDir, spec.validation, logger, opts)
|
|
559
616
|
}
|
|
617
|
+
// Pre-PR validation: the loop already ran (before this finalize, so a red checkout never
|
|
618
|
+
// reaches the PR-opening caller); attach its verdict for the caller to gate on and for the
|
|
619
|
+
// backend to record on the step.
|
|
620
|
+
if (validationReport) outcome.validationReport = validationReport
|
|
560
621
|
return outcome
|
|
561
622
|
}
|
|
562
623
|
|
|
624
|
+
/**
|
|
625
|
+
* Whether this pass produced anything worth VALIDATING — i.e. the branch advanced past its
|
|
626
|
+
* pre-run tip (or the run resumed an earlier one's pushed work). Gates the pre-PR validation
|
|
627
|
+
* loop, for two reasons: a run that changed nothing has nothing to check, and its real failure
|
|
628
|
+
* is "the agent produced no file changes" — reporting a red BASE branch instead would blame the
|
|
629
|
+
* run for a pre-existing condition it never touched (and burn the whole repair budget re-running
|
|
630
|
+
* an agent that already declined to act).
|
|
631
|
+
*
|
|
632
|
+
* Commits forgotten edits to tracked files first, exactly as {@link finalizeCodingRun} does, so
|
|
633
|
+
* an agent that edited-but-didn't-commit still counts as work. That call is idempotent, so
|
|
634
|
+
* finalize repeating it later is a no-op. Uncommitted NEW files are invisible here — but they
|
|
635
|
+
* are equally invisible to finalize, so a run whose only product is an uncommitted new file is
|
|
636
|
+
* a no-op on both paths, and the checks would have nothing to gate anyway.
|
|
637
|
+
*/
|
|
638
|
+
async function producedWork(
|
|
639
|
+
dir: string,
|
|
640
|
+
spec: CodingAgentSpec,
|
|
641
|
+
baseSha: string,
|
|
642
|
+
resumed: boolean,
|
|
643
|
+
opts: RunOptions,
|
|
644
|
+
): Promise<boolean> {
|
|
645
|
+
await commitTrackedEdits(dir, spec.commitMessage, opts.signal)
|
|
646
|
+
return resumed || (await branchHasCommitsSince(dir, baseSha, opts.signal))
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
/**
|
|
650
|
+
* Fold a pre-PR validation REPAIR pass's run into the accumulated agent outcome, so a looped run
|
|
651
|
+
* reports what every round actually spent rather than only the first. Counts and telemetry are
|
|
652
|
+
* summed/concatenated; the single-valued fields (the summary the backend renders, the effort
|
|
653
|
+
* report, the diagnostics that judge the FINAL answer) take the LATEST pass, which is the one
|
|
654
|
+
* whose state the PR is opened from.
|
|
655
|
+
*/
|
|
656
|
+
function mergeAgentPasses<T extends Awaited<ReturnType<typeof runAgentInWorkspace>>>(
|
|
657
|
+
previous: T,
|
|
658
|
+
next: T,
|
|
659
|
+
): T {
|
|
660
|
+
return {
|
|
661
|
+
...next,
|
|
662
|
+
stats: {
|
|
663
|
+
toolCalls: (previous.stats?.toolCalls ?? 0) + (next.stats?.toolCalls ?? 0),
|
|
664
|
+
assistantChars: (previous.stats?.assistantChars ?? 0) + (next.stats?.assistantChars ?? 0),
|
|
665
|
+
},
|
|
666
|
+
...(previous.usage || next.usage
|
|
667
|
+
? {
|
|
668
|
+
usage: {
|
|
669
|
+
inputTokens: (previous.usage?.inputTokens ?? 0) + (next.usage?.inputTokens ?? 0),
|
|
670
|
+
outputTokens: (previous.usage?.outputTokens ?? 0) + (next.usage?.outputTokens ?? 0),
|
|
671
|
+
},
|
|
672
|
+
}
|
|
673
|
+
: {}),
|
|
674
|
+
...(previous.callMetrics || next.callMetrics
|
|
675
|
+
? { callMetrics: [...(previous.callMetrics ?? []), ...(next.callMetrics ?? [])] }
|
|
676
|
+
: {}),
|
|
677
|
+
// The repair pass's own effort report wins when it wrote one; otherwise keep the first
|
|
678
|
+
// pass's rather than losing the assessment entirely.
|
|
679
|
+
...((next.effortReport ?? previous.effortReport)
|
|
680
|
+
? { effortReport: next.effortReport ?? previous.effortReport }
|
|
681
|
+
: {}),
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
|
|
563
685
|
/**
|
|
564
686
|
* The Ralph-loop validation watchdog: the longest a completion command may run before it is
|
|
565
687
|
* killed and treated as a failure (a hung `pnpm test` must never block the loop forever).
|
|
@@ -602,6 +724,10 @@ async function runRalphValidation(
|
|
|
602
724
|
cwd,
|
|
603
725
|
detached: spawnDetached,
|
|
604
726
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
727
|
+
// The job's own env (see `RunOptions.agentEnv`): a validation command typically installs
|
|
728
|
+
// before it tests, and this is spawned by the HARNESS rather than the agent, so it does not
|
|
729
|
+
// otherwise inherit the job's private-registry npmrc pointer on the native path.
|
|
730
|
+
env: { ...process.env, ...opts.agentEnv },
|
|
605
731
|
})
|
|
606
732
|
// Keep only the tail; guard against unbounded buffering on a chatty command.
|
|
607
733
|
const capture = (chunk: Buffer): void => {
|
package/src/frontend-infra.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { promisify } from 'node:util'
|
|
|
3
3
|
import { writeFile } from 'node:fs/promises'
|
|
4
4
|
import { join } from 'node:path'
|
|
5
5
|
import type { FrontendInfraSpec, InfraSetupRecord } from './job.js'
|
|
6
|
+
import type { RunOptions } from './runner.js'
|
|
6
7
|
import { killChildProcess } from './process.js'
|
|
7
8
|
import { pathExists } from './fs-utils.js'
|
|
8
9
|
import { captureRedactedOutput, redactSecrets } from './redact.js'
|
|
@@ -77,10 +78,10 @@ function guardProcess(child: ChildProcess, label: string, logger: Logger): Child
|
|
|
77
78
|
export async function standUpFrontend(
|
|
78
79
|
dir: string,
|
|
79
80
|
infra: FrontendInfraSpec,
|
|
80
|
-
|
|
81
|
-
onActivity: (() => void) | undefined,
|
|
81
|
+
run: Pick<RunOptions, 'signal' | 'onActivity' | 'agentEnv'>,
|
|
82
82
|
logger: Logger = log,
|
|
83
83
|
): Promise<FrontendStandUp> {
|
|
84
|
+
const { signal, onActivity } = run
|
|
84
85
|
const startedAt = Date.now()
|
|
85
86
|
const processes: ChildProcess[] = []
|
|
86
87
|
// The frontend app's directory: the checkout root, or a monorepo subdirectory when the config
|
|
@@ -118,6 +119,11 @@ export async function standUpFrontend(
|
|
|
118
119
|
|
|
119
120
|
const buildEnv =
|
|
120
121
|
(infra.envInjection ?? DEFAULTS.envInjection) === 'build' ? (infra.env ?? {}) : {}
|
|
122
|
+
// The job's own env (see `RunOptions.agentEnv`) — today the private-registry npmrc pointer.
|
|
123
|
+
// The stand-up is spawned by the HARNESS, not by the agent, so it does not inherit whatever the
|
|
124
|
+
// agent's CLI child was given: without this the install here would miss the job's registry auth
|
|
125
|
+
// on the native path, where the npmrc is per-job rather than the process's `~/.npmrc`.
|
|
126
|
+
const jobEnv = run.agentEnv ?? {}
|
|
121
127
|
|
|
122
128
|
try {
|
|
123
129
|
// 1) Install dependencies.
|
|
@@ -128,6 +134,7 @@ export async function standUpFrontend(
|
|
|
128
134
|
signal,
|
|
129
135
|
timeout: 8 * 60_000,
|
|
130
136
|
maxBuffer: 16 * 1024 * 1024,
|
|
137
|
+
env: { ...process.env, ...jobEnv },
|
|
131
138
|
})
|
|
132
139
|
pushOutput(installed.stdout, installed.stderr)
|
|
133
140
|
|
|
@@ -140,7 +147,7 @@ export async function standUpFrontend(
|
|
|
140
147
|
signal,
|
|
141
148
|
timeout: 12 * 60_000,
|
|
142
149
|
maxBuffer: 16 * 1024 * 1024,
|
|
143
|
-
env: { ...process.env, ...buildEnv },
|
|
150
|
+
env: { ...process.env, ...jobEnv, ...buildEnv },
|
|
144
151
|
})
|
|
145
152
|
pushOutput(built.stdout, built.stderr)
|
|
146
153
|
|