@phi-code-admin/phi-code 0.88.0 → 0.89.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -0
- package/docs/design/plan-debug-build.md +266 -0
- package/extensions/phi/orchestrator.ts +358 -7
- package/extensions/phi/providers/acceptance.ts +102 -0
- package/extensions/phi/providers/build-loop.ts +89 -0
- package/extensions/phi/providers/candidate-select.ts +112 -0
- package/extensions/phi/providers/debug-build-commands.ts +129 -0
- package/extensions/phi/providers/debug-contract.ts +131 -0
- package/extensions/phi/providers/execution.ts +75 -0
- package/extensions/phi/providers/redteam.ts +115 -0
- package/extensions/phi/providers/triage.ts +96 -0
- package/package.json +1 -1
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimality guard — pick the best of several candidate fixes.
|
|
3
|
+
*
|
|
4
|
+
* Lesson from the SWE-bench-lite head-to-head: /plan's extra deliberation made
|
|
5
|
+
* it produce a MORE elaborate (and wrong) fix than a single shot on the missed
|
|
6
|
+
* instance, while its diff was better-formed (it applied; the baseline's did
|
|
7
|
+
* not). The winning move is to generate more than one candidate and keep the
|
|
8
|
+
* SIMPLEST one that actually passes an executable acceptance repro — combining
|
|
9
|
+
* /plan's well-formed diffs with single-shot's tendency to stay minimal.
|
|
10
|
+
*
|
|
11
|
+
* This module is the pure decision core (no I/O): the caller runs each
|
|
12
|
+
* candidate against the same acceptance repro and reports `passedRepro`; this
|
|
13
|
+
* picks the winner. "Simplest" = fewest changed source lines in the unified
|
|
14
|
+
* diff (a robust, language-agnostic proxy for over-engineering).
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
export interface FixCandidate {
|
|
18
|
+
/** Where it came from, for diagnostics: "single-shot", "plan", "plan-fix", ... */
|
|
19
|
+
source: string;
|
|
20
|
+
/** Unified diff. */
|
|
21
|
+
patch: string;
|
|
22
|
+
/**
|
|
23
|
+
* Did this candidate pass the model's own falsify-first acceptance repro
|
|
24
|
+
* (the repro that FAILS on the pre-fix code and must PASS after)? Undefined
|
|
25
|
+
* when no repro was available to gate on.
|
|
26
|
+
*/
|
|
27
|
+
passedRepro?: boolean;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface CandidateSelection {
|
|
31
|
+
chosen: FixCandidate | null;
|
|
32
|
+
/** Human-readable rationale (surfaced to the user and asserted in tests). */
|
|
33
|
+
reason: string;
|
|
34
|
+
/** Changed-source-line count of the chosen patch (0 when none). */
|
|
35
|
+
chosenSize: number;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Count changed source lines in a unified diff: added/removed lines, excluding
|
|
40
|
+
* diff/hunk headers (---, +++, @@, diff --git, index) so the number reflects
|
|
41
|
+
* real edits, not file boilerplate.
|
|
42
|
+
*/
|
|
43
|
+
export function diffChangedLines(patch: string): number {
|
|
44
|
+
if (!patch) return 0;
|
|
45
|
+
let n = 0;
|
|
46
|
+
for (const line of patch.split("\n")) {
|
|
47
|
+
if (line.startsWith("+++") || line.startsWith("---")) continue;
|
|
48
|
+
if ((line.startsWith("+") || line.startsWith("-")) && line.length > 0) n++;
|
|
49
|
+
}
|
|
50
|
+
return n;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const hasPatch = (c: FixCandidate) => c.patch.trim().length > 0;
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Select the best candidate:
|
|
57
|
+
* 1. Among candidates that PASSED the acceptance repro, pick the smallest diff
|
|
58
|
+
* (ties → the earliest in the list, so callers can order by preference).
|
|
59
|
+
* 2. If none is known to pass but some were never gated (passedRepro undefined),
|
|
60
|
+
* pick the smallest non-empty patch among those (best effort).
|
|
61
|
+
* 3. If a repro existed and every candidate FAILED it, choose nothing — the
|
|
62
|
+
* honest outcome is "no candidate is verified", not "ship the least-bad".
|
|
63
|
+
* 4. No non-empty patches → nothing.
|
|
64
|
+
*/
|
|
65
|
+
export function selectMinimalPassingCandidate(candidates: FixCandidate[]): CandidateSelection {
|
|
66
|
+
const nonEmpty = candidates.filter(hasPatch);
|
|
67
|
+
if (nonEmpty.length === 0) {
|
|
68
|
+
return { chosen: null, reason: "no candidate produced a non-empty patch", chosenSize: 0 };
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const passed = nonEmpty.filter((c) => c.passedRepro === true);
|
|
72
|
+
if (passed.length > 0) {
|
|
73
|
+
const chosen = smallest(passed);
|
|
74
|
+
const others = passed.filter((c) => c !== chosen);
|
|
75
|
+
const note = others.length
|
|
76
|
+
? ` (over ${others.map((c) => `${c.source}:${diffChangedLines(c.patch)}`).join(", ")})`
|
|
77
|
+
: "";
|
|
78
|
+
return {
|
|
79
|
+
chosen,
|
|
80
|
+
reason: `smallest patch that passes the acceptance repro: ${chosen.source} (${diffChangedLines(chosen.patch)} changed lines)${note}`,
|
|
81
|
+
chosenSize: diffChangedLines(chosen.patch),
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const anyGated = nonEmpty.some((c) => c.passedRepro === true || c.passedRepro === false);
|
|
86
|
+
if (anyGated) {
|
|
87
|
+
// A repro existed and nothing passed it — do not pretend a candidate is verified.
|
|
88
|
+
return { chosen: null, reason: "an acceptance repro existed but no candidate passed it", chosenSize: 0 };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// No repro to gate on: fall back to the smallest non-empty patch (best effort).
|
|
92
|
+
const chosen = smallest(nonEmpty);
|
|
93
|
+
return {
|
|
94
|
+
chosen,
|
|
95
|
+
reason: `no acceptance repro to gate on; smallest non-empty patch: ${chosen.source} (${diffChangedLines(chosen.patch)} changed lines)`,
|
|
96
|
+
chosenSize: diffChangedLines(chosen.patch),
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Smallest by changed-line count; ties keep list order (stable). */
|
|
101
|
+
function smallest(cs: FixCandidate[]): FixCandidate {
|
|
102
|
+
let best = cs[0];
|
|
103
|
+
let bestSize = diffChangedLines(best.patch);
|
|
104
|
+
for (let i = 1; i < cs.length; i++) {
|
|
105
|
+
const s = diffChangedLines(cs[i].patch);
|
|
106
|
+
if (s < bestSize) {
|
|
107
|
+
best = cs[i];
|
|
108
|
+
bestSize = s;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return best;
|
|
112
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Instruction builders for the /debug and /build phase pipelines.
|
|
3
|
+
*
|
|
4
|
+
* These are PURE text builders (no fs, no Pi) so the exact protocol each phase
|
|
5
|
+
* agent receives is unit-testable. They encode the contracts from
|
|
6
|
+
* docs/design/plan-debug-build.md: /debug turns a REAL failure green through
|
|
7
|
+
* REPRODUCE → LOCALIZE → FIX → VERIFY with execution as the only oracle; /build
|
|
8
|
+
* adds an execution-grounded verify (run recipe + acceptance + executable
|
|
9
|
+
* red-team) that routes real failures back to the /debug protocol.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import type { FailingState } from "./debug-contract.js";
|
|
13
|
+
|
|
14
|
+
/** Render a failing state as a compact, unambiguous block for an instruction. */
|
|
15
|
+
export function formatFailingState(state: FailingState): string {
|
|
16
|
+
const lines: string[] = [];
|
|
17
|
+
if (state.failingTest?.trim()) lines.push(`- Failing test: \`${state.failingTest.trim()}\``);
|
|
18
|
+
if (state.reproCommand?.trim()) lines.push(`- Repro command: \`${state.reproCommand.trim()}\``);
|
|
19
|
+
if (state.expected?.trim()) lines.push(`- Expected behaviour: ${state.expected.trim()}`);
|
|
20
|
+
if (state.trace?.trim()) lines.push(`- Trace / error:\n\`\`\`\n${state.trace.trim()}\n\`\`\``);
|
|
21
|
+
if (state.cwd?.trim()) lines.push(`- Working directory: \`${state.cwd.trim()}\``);
|
|
22
|
+
return lines.length ? lines.join("\n") : "- (no structured failing state supplied)";
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface DebugInstructions {
|
|
26
|
+
reproduce: string;
|
|
27
|
+
localize: string;
|
|
28
|
+
fix: string;
|
|
29
|
+
verify: string;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const DEBUG_RULES = `
|
|
33
|
+
---
|
|
34
|
+
## /debug operating rules (non-negotiable)
|
|
35
|
+
- **Execution is the only oracle.** No verdict without a real run whose output you paste. Never write FIXED because the code "looks right".
|
|
36
|
+
- **No fabricated PASS.** If you cannot run the reproduction (missing env/deps), emit \`BLOCKED: no executable environment\` — do NOT reconstruct a mock and grade your own reconstruction.
|
|
37
|
+
- **Minimal fix wins.** Prefer the smallest change; every added guard/condition is a liability that can hide the bug (an over-clever guard is exactly how these fixes go wrong).
|
|
38
|
+
- **Root cause, not workaround.** No skipped tests, no \`--no-verify\`, no mock that hides the failure.
|
|
39
|
+
- The user does NOT answer during these phases. Act autonomously; do not end with a question.`;
|
|
40
|
+
|
|
41
|
+
/** The four /debug phase instructions, specialised to the concrete failing state. */
|
|
42
|
+
export function debugPhaseInstructions(state: FailingState): DebugInstructions {
|
|
43
|
+
const failing = formatFailingState(state);
|
|
44
|
+
const repro = state.failingTest?.trim() || state.reproCommand?.trim() || "(the reproduction command)";
|
|
45
|
+
|
|
46
|
+
return {
|
|
47
|
+
reproduce:
|
|
48
|
+
`You are the REPRODUCE agent (phase 1 of /debug). Your only job is to CONFIRM the failure is real.
|
|
49
|
+
|
|
50
|
+
**The reported failing state:**
|
|
51
|
+
${failing}
|
|
52
|
+
|
|
53
|
+
**Do exactly this:**
|
|
54
|
+
1. Run the reproduction on the CURRENT, unmodified code: \`${repro}\`.
|
|
55
|
+
2. Paste the exact command and its full output.
|
|
56
|
+
3. Decide:
|
|
57
|
+
- If it FAILS as reported → capture the precise symptom (assertion, exception, exit code) and hand off to LOCALIZE.
|
|
58
|
+
- If it PASSES → STOP. Write \`BLOCKED: cannot reproduce — passes on current code\` with the run pasted. Do not invent a bug.
|
|
59
|
+
- If it cannot be run at all (missing deps/env) → STOP. Write \`BLOCKED: no executable environment\` with what failed.
|
|
60
|
+
4. Do NOT edit any source yet. This phase only observes.
|
|
61
|
+
5. **Last action:** call \`phase_result\` — \`verdict: PASS\` if it reproduced (proceed to LOCALIZE), or \`verdict: BLOCKED\` with the reason if you stopped.` +
|
|
62
|
+
DEBUG_RULES,
|
|
63
|
+
|
|
64
|
+
localize: `You are the LOCALIZE agent (phase 2 of /debug). Drive from the REAL symptom the previous phase captured.
|
|
65
|
+
|
|
66
|
+
**The failing state:**
|
|
67
|
+
${failing}
|
|
68
|
+
|
|
69
|
+
**Do exactly this:**
|
|
70
|
+
1. Read the traceback / error from the reproduction. Follow it to the exact frame.
|
|
71
|
+
2. Read the implicated source and the symbols it touches (grep the nearby/changed identifiers).
|
|
72
|
+
3. Name the fault site precisely — file:line and the wrong assumption. Localization, not more review, is the lever here.
|
|
73
|
+
4. Hand off a crisp fault description; do NOT fix yet.${DEBUG_RULES}`,
|
|
74
|
+
|
|
75
|
+
fix:
|
|
76
|
+
`You are the FIX agent (phase 3 of /debug). Produce the MINIMAL change that addresses the located root cause.
|
|
77
|
+
|
|
78
|
+
**The failing state:**
|
|
79
|
+
${failing}
|
|
80
|
+
|
|
81
|
+
**Do exactly this:**
|
|
82
|
+
1. Write the smallest patch that fixes the root cause at the fault site LOCALIZE named.
|
|
83
|
+
2. If two approaches are plausible, prefer the one that adds the least surface (fewest new branches/guards).
|
|
84
|
+
3. Do NOT run the suite to "confirm" here by eye — VERIFY re-runs everything. Just make the change and state precisely what you changed and why it addresses the cause.` +
|
|
85
|
+
DEBUG_RULES,
|
|
86
|
+
|
|
87
|
+
verify: `You are the VERIFY agent (phase 4 of /debug) — the oracle. A green verdict requires TWO real runs.
|
|
88
|
+
|
|
89
|
+
**The failing state:**
|
|
90
|
+
${failing}
|
|
91
|
+
|
|
92
|
+
**Do exactly this, pasting every command's output:**
|
|
93
|
+
1. Re-run the reproduction: \`${repro}\`. It MUST now pass.
|
|
94
|
+
2. Run the existing test suite (the project's test command). It MUST NOT regress.
|
|
95
|
+
3. Verdict:
|
|
96
|
+
- Both green → \`FIXED\`, and paste the before(fail)/after(pass) reproduction runs and the green suite as evidence.
|
|
97
|
+
- Reproduction still fails, or the suite regresses → \`BLOCKED\` with the closest diagnostic. Do NOT ship the least-bad patch; a wrong fix is worse than an honest BLOCKED.
|
|
98
|
+
4. Write the final verdict block, then call \`phase_result\` with \`verdict: PASS\` (FIXED) or \`verdict: BLOCKED\`, plus a one-line handoff:
|
|
99
|
+
\`\`\`
|
|
100
|
+
VERDICT: FIXED | BLOCKED
|
|
101
|
+
Evidence: reproBefore=fail reproAfter=pass suite=green|skipped
|
|
102
|
+
Reason: <only when BLOCKED>
|
|
103
|
+
\`\`\`${DEBUG_RULES}`,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* The /build execution-grounded verify phase: run the recipe, check acceptance,
|
|
109
|
+
* red-team the boundary the change touched, and route real failures to /debug.
|
|
110
|
+
*/
|
|
111
|
+
export function buildVerifyInstruction(spec: string): string {
|
|
112
|
+
return `You are the BUILD-VERIFY agent — the execution oracle for /build. The code has been written; now PROVE it runs and meets the spec, or report honestly what still fails.
|
|
113
|
+
|
|
114
|
+
**Original spec:** ${spec}
|
|
115
|
+
|
|
116
|
+
**Do exactly this, pasting every command's output:**
|
|
117
|
+
1. **Run recipe.** From the brief's \`## Run Recipe\` (or package.json / Makefile / Dockerfile), build and start the app. Distinguish a real failure from a stale launch recipe.
|
|
118
|
+
2. **Acceptance.** For each acceptance criterion derived from the SPEC (not the code), run a concrete check that exits 0 iff it holds. Mark each ✅ (ran+passed), ❌ (ran+failed), or ❔ (could not be executed — NEVER count ❔ as passing).
|
|
119
|
+
3. **Executable red-team.** Attack the specific input regimes the change touched (empty, null, boundary, wrong-type, and buffered-vs-streaming / malformed / auth as relevant). Each attack must be a RUNNABLE test that goes RED if it breaks the code — an opinion is not a finding.
|
|
120
|
+
4. **Route real failures.** For every ❌ criterion and every red-team break, treat it as a concrete failing state (failing test / repro command / expected) and fix it with the /debug protocol: REPRODUCE → LOCALIZE → minimal FIX → VERIFY (re-run the reproduction AND the suite).
|
|
121
|
+
5. **Honest verdict.** When all checkable criteria pass and the red-team finds no break, write \`BUILD: SUCCESS\`. If rounds/budget run out with failures open, write \`BUILD: PARTIAL\` and LIST exactly which criteria still fail — never a confident-wrong SUCCESS. Then call \`phase_result\` with \`verdict: PASS\` (SUCCESS) or \`verdict: FAIL\`/\`BLOCKED\` and the handoff.
|
|
122
|
+
\`\`\`
|
|
123
|
+
VERDICT: PASS | FAIL | BLOCKED
|
|
124
|
+
Handoff:
|
|
125
|
+
State: <what runs and is proven by a real run>
|
|
126
|
+
Open Risks: <criteria still ❔ unverified or ❌ failing>
|
|
127
|
+
Next: <the single most important remaining action>
|
|
128
|
+
\`\`\`${DEBUG_RULES}`;
|
|
129
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* /debug contract — turn a REAL failing state green, or honestly say BLOCKED.
|
|
3
|
+
*
|
|
4
|
+
* The whole point (docs/design/plan-debug-build.md): /debug never guesses what
|
|
5
|
+
* is wrong. It is given a reproducible failure, confirms it fails on the current
|
|
6
|
+
* code, fixes it, and VERIFIES by re-running — repro passes AND the existing
|
|
7
|
+
* suite does not regress — selecting the minimal passing candidate. There is no
|
|
8
|
+
* path to FIXED without execution.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { type FixCandidate, selectMinimalPassingCandidate } from "./candidate-select.js";
|
|
12
|
+
import { type CommandResult, passed } from "./execution.js";
|
|
13
|
+
|
|
14
|
+
/** A concrete, reproducible failing state — /debug's input. */
|
|
15
|
+
export interface FailingState {
|
|
16
|
+
/** A test command, e.g. "pytest tests/x.py::test_y". */
|
|
17
|
+
failingTest?: string;
|
|
18
|
+
/** A stack trace / error output pasted from a real run. */
|
|
19
|
+
trace?: string;
|
|
20
|
+
/** A command that exhibits the bug. */
|
|
21
|
+
reproCommand?: string;
|
|
22
|
+
/** What should happen instead (from the user/spec). */
|
|
23
|
+
expected?: string;
|
|
24
|
+
cwd?: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** How to run the existing suite, to catch regressions. */
|
|
28
|
+
export interface SuiteRecipe {
|
|
29
|
+
test?: string;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export type DebugVerdict = "FIXED" | "BLOCKED";
|
|
33
|
+
|
|
34
|
+
export interface DebugOutcome {
|
|
35
|
+
verdict: DebugVerdict;
|
|
36
|
+
patch?: string;
|
|
37
|
+
evidence?: { reproBefore: "fail"; reproAfter: "pass"; suite: "green" | "skipped" };
|
|
38
|
+
reason?: string;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** True when the input actually gives something runnable to reproduce. */
|
|
42
|
+
export function hasReproducibleFailure(state: FailingState): boolean {
|
|
43
|
+
return Boolean(state.failingTest?.trim() || state.reproCommand?.trim());
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** The command that reproduces the failure (test preferred over generic repro). */
|
|
47
|
+
export function reproCommand(state: FailingState): string | undefined {
|
|
48
|
+
return state.failingTest?.trim() || state.reproCommand?.trim() || undefined;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export type ReproduceDecision = { action: "proceed"; symptom: string } | { action: "blocked"; reason: string };
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* REPRODUCE gate: the failing state must FAIL on the current code. If there is
|
|
55
|
+
* nothing runnable, or it already passes, /debug stops — it will not fabricate a
|
|
56
|
+
* bug or a repro from imagination.
|
|
57
|
+
*/
|
|
58
|
+
export function decideReproduce(state: FailingState, runOnCurrentCode: CommandResult | null): ReproduceDecision {
|
|
59
|
+
if (!hasReproducibleFailure(state)) {
|
|
60
|
+
return { action: "blocked", reason: "no reproducible failing state (need a failing test or a repro command)" };
|
|
61
|
+
}
|
|
62
|
+
if (runOnCurrentCode === null) {
|
|
63
|
+
return { action: "blocked", reason: "could not run the reproduction (no executable environment)" };
|
|
64
|
+
}
|
|
65
|
+
if (passed(runOnCurrentCode)) {
|
|
66
|
+
return { action: "blocked", reason: "the reported failure does not reproduce — it passes on the current code" };
|
|
67
|
+
}
|
|
68
|
+
return { action: "proceed", symptom: `${runOnCurrentCode.command} exited ${runOnCurrentCode.exitCode}` };
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** A fix candidate paired with its verification runs. */
|
|
72
|
+
export interface VerifiedCandidate {
|
|
73
|
+
source: string;
|
|
74
|
+
patch: string;
|
|
75
|
+
/** The reproduction re-run WITH this candidate applied. */
|
|
76
|
+
reproAfter: CommandResult | null;
|
|
77
|
+
/** The existing suite re-run WITH this candidate applied (null = no suite). */
|
|
78
|
+
suite: CommandResult | null;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* VERIFY: a candidate is accepted only if the repro now PASSES and the suite (if
|
|
83
|
+
* any) is green. Among accepted candidates, pick the minimal diff. If none is
|
|
84
|
+
* accepted, BLOCKED — never ship the least-bad.
|
|
85
|
+
*/
|
|
86
|
+
export function decideVerify(candidates: VerifiedCandidate[], hasSuite: boolean): DebugOutcome {
|
|
87
|
+
const fixCandidates: FixCandidate[] = candidates.map((c) => {
|
|
88
|
+
const reproOk = c.reproAfter !== null && passed(c.reproAfter);
|
|
89
|
+
const suiteOk = !hasSuite || (c.suite !== null && passed(c.suite));
|
|
90
|
+
return { source: c.source, patch: c.patch, passedRepro: reproOk && suiteOk };
|
|
91
|
+
});
|
|
92
|
+
const selection = selectMinimalPassingCandidate(fixCandidates);
|
|
93
|
+
if (!selection.chosen) {
|
|
94
|
+
return {
|
|
95
|
+
verdict: "BLOCKED",
|
|
96
|
+
reason: fixCandidates.some((c) => c.patch.trim())
|
|
97
|
+
? "no candidate makes the reproduction pass without regressing the suite"
|
|
98
|
+
: "no candidate produced a patch",
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
return {
|
|
102
|
+
verdict: "FIXED",
|
|
103
|
+
patch: selection.chosen.patch,
|
|
104
|
+
evidence: { reproBefore: "fail", reproAfter: "pass", suite: hasSuite ? "green" : "skipped" },
|
|
105
|
+
reason: selection.reason,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Parse a /debug argument string / structured input into a FailingState. */
|
|
110
|
+
export function parseFailingState(arg: string, structured?: Partial<FailingState>): FailingState {
|
|
111
|
+
const a = arg.trim();
|
|
112
|
+
const state: FailingState = { ...structured };
|
|
113
|
+
// Heuristic: a bare test-runner invocation is a failing test; anything else
|
|
114
|
+
// with a command shape is a repro command; free prose is `expected`.
|
|
115
|
+
if (!state.failingTest && !state.reproCommand && a) {
|
|
116
|
+
// Test-runner invocations → a failing test. Bare `node x.js` / `python x.py`
|
|
117
|
+
// / `./x` are script runs → a repro command. Free prose → expected.
|
|
118
|
+
if (
|
|
119
|
+
/^(pytest|jest|vitest|mocha|go test|cargo test|npm (run )?test|npx (vitest|jest|mocha|playwright)|python -m pytest)\b/.test(
|
|
120
|
+
a,
|
|
121
|
+
)
|
|
122
|
+
) {
|
|
123
|
+
state.failingTest = a;
|
|
124
|
+
} else if (/^(node|python|python3|\.\/|bash|sh|deno|bun|ruby|php)\b|[|&;><]/.test(a)) {
|
|
125
|
+
state.reproCommand = a;
|
|
126
|
+
} else {
|
|
127
|
+
state.expected = state.expected || a;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
return state;
|
|
131
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Execution oracle — run real commands and report objective results.
|
|
3
|
+
*
|
|
4
|
+
* This is the ground truth the /plan → /debug → /build design turns on: an
|
|
5
|
+
* accept/reject decision must come from RUNNING code, not from a model reviewing
|
|
6
|
+
* its own output (see docs/design/plan-debug-build.md). Everything here is thin
|
|
7
|
+
* and deterministic so the result interpretation can be unit-tested; the spawn
|
|
8
|
+
* itself is exercised with trivial real commands.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { spawnSync } from "node:child_process";
|
|
12
|
+
|
|
13
|
+
export interface CommandResult {
|
|
14
|
+
command: string;
|
|
15
|
+
exitCode: number | null;
|
|
16
|
+
stdout: string;
|
|
17
|
+
stderr: string;
|
|
18
|
+
durationMs: number;
|
|
19
|
+
timedOut: boolean;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** A command passed iff it exited 0 and did not time out. */
|
|
23
|
+
export function passed(result: CommandResult): boolean {
|
|
24
|
+
return result.exitCode === 0 && !result.timedOut;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** One-line summary for a phase report / handoff. */
|
|
28
|
+
export function summarize(result: CommandResult): string {
|
|
29
|
+
if (result.timedOut) return `\`${result.command}\` → TIMED OUT after ${Math.round(result.durationMs / 1000)}s`;
|
|
30
|
+
return `\`${result.command}\` → exit ${result.exitCode ?? "?"} (${Math.round(result.durationMs / 1000)}s)`;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Last N lines of the combined output — what a model needs to see to react. */
|
|
34
|
+
export function tail(result: CommandResult, lines = 40): string {
|
|
35
|
+
const combined = `${result.stdout}\n${result.stderr}`.trimEnd();
|
|
36
|
+
const all = combined.split("\n");
|
|
37
|
+
return all.slice(Math.max(0, all.length - lines)).join("\n");
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface RunOptions {
|
|
41
|
+
cwd?: string;
|
|
42
|
+
timeoutMs?: number;
|
|
43
|
+
env?: NodeJS.ProcessEnv;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000;
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Run a shell command and capture its result. Never throws — a non-zero exit,
|
|
50
|
+
* a spawn error, or a timeout all come back as a CommandResult so callers make
|
|
51
|
+
* decisions from data, not exceptions.
|
|
52
|
+
*/
|
|
53
|
+
export function runCommand(command: string, options: RunOptions = {}): CommandResult {
|
|
54
|
+
const start = Date.now();
|
|
55
|
+
const res = spawnSync(command, {
|
|
56
|
+
cwd: options.cwd,
|
|
57
|
+
timeout: options.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
58
|
+
env: options.env ?? process.env,
|
|
59
|
+
shell: true,
|
|
60
|
+
encoding: "utf-8",
|
|
61
|
+
maxBuffer: 32 * 1024 * 1024,
|
|
62
|
+
});
|
|
63
|
+
const durationMs = Date.now() - start;
|
|
64
|
+
// spawnSync sets error with code "ETIMEDOUT" on timeout, and signal SIGTERM.
|
|
65
|
+
const timedOut = res.error !== undefined && (res.error as NodeJS.ErrnoException).code === "ETIMEDOUT";
|
|
66
|
+
const spawnFailed = res.error !== undefined && !timedOut;
|
|
67
|
+
return {
|
|
68
|
+
command,
|
|
69
|
+
exitCode: spawnFailed ? null : (res.status ?? (timedOut ? null : null)),
|
|
70
|
+
stdout: res.stdout ?? "",
|
|
71
|
+
stderr: spawnFailed ? `${(res.error as Error).message}\n${res.stderr ?? ""}` : (res.stderr ?? ""),
|
|
72
|
+
durationMs,
|
|
73
|
+
timedOut,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Executable red-team — the adversary's deliverable is a FAILING RUN, not an
|
|
3
|
+
* opinion (docs/design/plan-debug-build.md). This is the pure decision core: it
|
|
4
|
+
* turns the outcome of each adversarial attempt into a loop step, so a "breaking
|
|
5
|
+
* case" is only ever recorded when a real test went RED. Prose can't lie its way
|
|
6
|
+
* to a finding here — execution is the oracle.
|
|
7
|
+
*
|
|
8
|
+
* Why this exists: in the 3362 measurement an over-clever guard survived a
|
|
9
|
+
* different model's careful written review because the reviewer shared the
|
|
10
|
+
* misconception. It would NOT have survived the assertion being *run*. So the
|
|
11
|
+
* adversary must attack the specific input regime the diff touched, and express
|
|
12
|
+
* the attack as an executed test.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import type { FailingState } from "./debug-contract.js";
|
|
16
|
+
import { type CommandResult, passed, tail } from "./execution.js";
|
|
17
|
+
|
|
18
|
+
/** One adversarial attempt: a runnable test aimed at a specific input regime. */
|
|
19
|
+
export interface RedTeamAttempt {
|
|
20
|
+
/** Which boundary/regime it targets (e.g. "buffered vs streaming"). */
|
|
21
|
+
regime: string;
|
|
22
|
+
/** The runnable breaking test (a command). */
|
|
23
|
+
test: string;
|
|
24
|
+
/** The execution of that test (null = it could not be run). */
|
|
25
|
+
result: CommandResult | null;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** A confirmed break: an attempt whose test actually ran RED. */
|
|
29
|
+
export interface BreakingCase {
|
|
30
|
+
regime: string;
|
|
31
|
+
test: string;
|
|
32
|
+
symptom: string;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface RedTeamState {
|
|
36
|
+
/** Consecutive rounds that failed to break the code. */
|
|
37
|
+
dry: number;
|
|
38
|
+
attemptsUsed: number;
|
|
39
|
+
breakingCases: BreakingCase[];
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface RedTeamConfig {
|
|
43
|
+
/** Stop after this many consecutive rounds that could not break the code. */
|
|
44
|
+
dryRoundsToStop: number;
|
|
45
|
+
/** Hard budget on attempts. */
|
|
46
|
+
maxAttempts: number;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export const DEFAULT_REDTEAM_CONFIG: RedTeamConfig = { dryRoundsToStop: 2, maxAttempts: 8 };
|
|
50
|
+
|
|
51
|
+
export function initRedTeam(): RedTeamState {
|
|
52
|
+
return { dry: 0, attemptsUsed: 0, breakingCases: [] };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Keep going only while under budget and not yet K dry rounds in a row. */
|
|
56
|
+
export function shouldContinueRedTeam(state: RedTeamState, config: RedTeamConfig): boolean {
|
|
57
|
+
return state.attemptsUsed < config.maxAttempts && state.dry < config.dryRoundsToStop;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Fold one attempt into the state. A RED run (test failed) is a breaking case
|
|
62
|
+
* and resets the dry counter; a GREEN run (or one that could not be executed at
|
|
63
|
+
* all) counts as a dry round — you cannot claim a break you did not run.
|
|
64
|
+
*/
|
|
65
|
+
export function recordAttempt(state: RedTeamState, attempt: RedTeamAttempt, config: RedTeamConfig): RedTeamState {
|
|
66
|
+
const attemptsUsed = state.attemptsUsed + 1;
|
|
67
|
+
const ran = attempt.result !== null;
|
|
68
|
+
const broke = ran && !passed(attempt.result as CommandResult);
|
|
69
|
+
|
|
70
|
+
if (broke) {
|
|
71
|
+
const result = attempt.result as CommandResult;
|
|
72
|
+
return {
|
|
73
|
+
dry: 0,
|
|
74
|
+
attemptsUsed,
|
|
75
|
+
breakingCases: [
|
|
76
|
+
...state.breakingCases,
|
|
77
|
+
{ regime: attempt.regime, test: attempt.test, symptom: tail(result, 20) || `exit ${result.exitCode}` },
|
|
78
|
+
],
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
// Green, or unrunnable: this round produced no finding.
|
|
82
|
+
void config;
|
|
83
|
+
return { dry: state.dry + 1, attemptsUsed, breakingCases: state.breakingCases };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Turn confirmed breaks into failing states that /debug can consume directly. */
|
|
87
|
+
export function breakingCasesToFailingStates(cases: BreakingCase[], cwd?: string): FailingState[] {
|
|
88
|
+
return cases.map((c) => ({
|
|
89
|
+
reproCommand: c.test,
|
|
90
|
+
trace: c.symptom,
|
|
91
|
+
expected: `input regime "${c.regime}" must not break the change`,
|
|
92
|
+
cwd,
|
|
93
|
+
}));
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const STREAMING_HINT = /\b(stream|iter|chunk|read|io|http|response|decode|encode|buffer|socket|pipe)\b/i;
|
|
97
|
+
const PARSE_HINT = /\b(parse|json|yaml|xml|deserialize|decode|token|lexer)\b/i;
|
|
98
|
+
const NUMERIC_HINT = /\b(sum|count|index|offset|length|size|range|price|amount|math|float|int)\b/i;
|
|
99
|
+
const AUTH_HINT = /\b(auth|login|token|session|password|permission|role|jwt)\b/i;
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Cheap enumeration of the input regimes worth attacking, given hints from the
|
|
103
|
+
* change (file names + keywords). Always includes the universal boundaries;
|
|
104
|
+
* adds domain-specific ones when the hints match. The adversary attacks these,
|
|
105
|
+
* not the whole app.
|
|
106
|
+
*/
|
|
107
|
+
export function enumerateInputRegimes(hints: { changedFiles?: string[]; keywords?: string } = {}): string[] {
|
|
108
|
+
const blob = `${(hints.changedFiles ?? []).join(" ")} ${hints.keywords ?? ""}`;
|
|
109
|
+
const regimes = ["empty input", "null/undefined", "boundary value", "wrong type", "large input"];
|
|
110
|
+
if (STREAMING_HINT.test(blob)) regimes.push("buffered vs streaming");
|
|
111
|
+
if (PARSE_HINT.test(blob)) regimes.push("malformed/partial input", "non-ASCII / unicode");
|
|
112
|
+
if (NUMERIC_HINT.test(blob)) regimes.push("zero / negative", "overflow / very large number");
|
|
113
|
+
if (AUTH_HINT.test(blob)) regimes.push("missing credentials", "expired / tampered token");
|
|
114
|
+
return regimes;
|
|
115
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Triage / adaptive depth — pick the CHEAPEST mode consistent with the signals.
|
|
3
|
+
*
|
|
4
|
+
* The measured lesson (docs/design/plan-debug-build.md): the 6–14× overhead of
|
|
5
|
+
* the full pipeline is only worth paying when a single shot fails the real
|
|
6
|
+
* oracle. So this classifier defaults to the least machinery and escalates only
|
|
7
|
+
* on concrete signals — a supplied failing state routes to /debug, a large or
|
|
8
|
+
* under-specified build routes to /build, everything else is a single shot.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
export type Route = "debug" | "single-shot" | "build" | "plan";
|
|
12
|
+
|
|
13
|
+
export interface TriageSignals {
|
|
14
|
+
/** A concrete failing test / trace / repro command was supplied. */
|
|
15
|
+
hasFailingState?: boolean;
|
|
16
|
+
/** Rough number of files the task will touch (undefined = estimate from text). */
|
|
17
|
+
estimatedFiles?: number;
|
|
18
|
+
/** The task text, for cheap keyword signals. */
|
|
19
|
+
text?: string;
|
|
20
|
+
/** Caller forces a specific mode (an explicit /debug, /build, /plan). */
|
|
21
|
+
forced?: Route;
|
|
22
|
+
/** Caller wants only a plan artifact, not the execution loop. */
|
|
23
|
+
planOnly?: boolean;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface TriageDecision {
|
|
27
|
+
route: Route;
|
|
28
|
+
/** "minimal" = do the least; "full" = the multi-phase pipeline is justified. */
|
|
29
|
+
depth: "minimal" | "full";
|
|
30
|
+
reason: string;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const BUILD_KEYWORDS =
|
|
34
|
+
/\b(build|scaffold|application|app|feature|end[- ]?to[- ]?end|full[- ]?stack|from scratch|new (project|service|module|api|endpoint|component|page)|several files|multiple files)\b/i;
|
|
35
|
+
|
|
36
|
+
/** At/above this many touched files, a single shot is unlikely to stay coherent. */
|
|
37
|
+
export const MULTI_FILE_THRESHOLD = 3;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Cheap file-count estimate from a request: count distinct path-like tokens.
|
|
41
|
+
* Deliberately conservative — defaults to 1 when nothing looks like a path.
|
|
42
|
+
*/
|
|
43
|
+
export function estimateFiles(text: string): number {
|
|
44
|
+
if (!text.trim()) return 1;
|
|
45
|
+
const paths = new Set<string>();
|
|
46
|
+
for (const m of text.matchAll(/\b[\w./-]+\.[a-z]{1,6}\b/gi)) {
|
|
47
|
+
paths.add(m[0].toLowerCase());
|
|
48
|
+
}
|
|
49
|
+
return Math.max(1, paths.size);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Map signals to a route + depth. Pure and deterministic: same signals in, same
|
|
54
|
+
* decision out. Order matters — a forced mode wins, then a real failing state
|
|
55
|
+
* (cheapest useful oracle), then build-scale, else single shot.
|
|
56
|
+
*/
|
|
57
|
+
export function triage(signals: TriageSignals): TriageDecision {
|
|
58
|
+
if (signals.forced) {
|
|
59
|
+
return {
|
|
60
|
+
route: signals.forced,
|
|
61
|
+
depth: signals.forced === "single-shot" ? "minimal" : "full",
|
|
62
|
+
reason: `explicit /${signals.forced}`,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
if (signals.hasFailingState) {
|
|
67
|
+
return {
|
|
68
|
+
route: "debug",
|
|
69
|
+
depth: "minimal",
|
|
70
|
+
reason: "a reproducible failing state was supplied — fix it directly, skip planning",
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const text = signals.text ?? "";
|
|
75
|
+
const files = signals.estimatedFiles ?? estimateFiles(text);
|
|
76
|
+
const buildy = BUILD_KEYWORDS.test(text);
|
|
77
|
+
const large = files >= MULTI_FILE_THRESHOLD;
|
|
78
|
+
|
|
79
|
+
if (buildy || large) {
|
|
80
|
+
const why = large ? `~${files} files touched (≥ ${MULTI_FILE_THRESHOLD})` : "build-scale request";
|
|
81
|
+
if (signals.planOnly) {
|
|
82
|
+
return {
|
|
83
|
+
route: "plan",
|
|
84
|
+
depth: "full",
|
|
85
|
+
reason: `${why}; plan-only requested — decompose without the run loop`,
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
return { route: "build", depth: "full", reason: `${why} — decompose and run the build→verify→debug loop` };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
return {
|
|
92
|
+
route: "single-shot",
|
|
93
|
+
depth: "minimal",
|
|
94
|
+
reason: "small, self-contained — try one shot and verify before escalating",
|
|
95
|
+
};
|
|
96
|
+
}
|