@phi-code-admin/phi-code 0.92.1 → 0.93.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,52 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.93.1] - 2026-07-12
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
|
|
7
|
+
- /fix on PROSE input now has a real oracle: the single shot is instructed to
|
|
8
|
+
WRITE a literal reproduction (from the issue's exact snippets) and declare
|
|
9
|
+
it on a REPRO-CMD: handoff line; the driver oracle re-runs it. Without this,
|
|
10
|
+
prose inputs (e.g. SWE-bench problem statements) finished UNVERIFIED with
|
|
11
|
+
nothing to check. +2 tests.
|
|
12
|
+
|
|
13
|
+
## [0.93.0] - 2026-07-12
|
|
14
|
+
|
|
15
|
+
### Added — the escalation architecture the n=13 measurement demanded
|
|
16
|
+
|
|
17
|
+
- **`/fix` — single-shot first, oracle next, escalate only on red.** One direct
|
|
18
|
+
attempt at baseline cost, then the DRIVER runs the reproduction and the
|
|
19
|
+
project suite in the real sandbox (deterministic, zero model tokens): green →
|
|
20
|
+
done; red → the full /debug pipeline takes over, seeded with the exact red
|
|
21
|
+
run (command + trace, never a paraphrase). Nothing runnable → honestly
|
|
22
|
+
labelled UNVERIFIED. By construction never worse than the single shot
|
|
23
|
+
(measured: baseline 7/13 vs pipeline 6/13 at ~2.5×).
|
|
24
|
+
- **`/debug --candidates N` — diversity proposes, the oracle disposes.** N
|
|
25
|
+
independent FIX attempts on DIVERSE model families (routing-derived), each
|
|
26
|
+
patch captured and the tree reset between attempts; then deterministic
|
|
27
|
+
arbitration applies each candidate and runs the reproduction (from the input
|
|
28
|
+
or the REPRODUCE handoff's machine-readable REPRO-CMD line) + suite in the
|
|
29
|
+
sandbox; the minimal passing candidate wins (candidate-select). This is the
|
|
30
|
+
adversarial review that works — the one we measured (opinion panels rubber-
|
|
31
|
+
stamp shared misconceptions; a red run does not). Refuses to run over a dirty
|
|
32
|
+
tree (non-deletion policy) and falls back to single-candidate.
|
|
33
|
+
- **Never a blank page (lesson sympy-11870).** When no candidate passes
|
|
34
|
+
arbitration, the smallest non-empty candidate stays applied, clearly labelled
|
|
35
|
+
UNVERIFIED; a confirmed-BLOCKED halt now reports any draft left in the tree
|
|
36
|
+
(git diff --stat) instead of silently discarding work.
|
|
37
|
+
- **Literal reproductions (lesson requests-2148).** REPRODUCE must build the
|
|
38
|
+
repro from the EXACT snippets/values quoted in the issue (a reproduction of
|
|
39
|
+
your interpretation validates your interpretation, not the bug) and must
|
|
40
|
+
emit a REPRO-CMD: line for deterministic re-runs. LOCALIZE now consults and
|
|
41
|
+
feeds project memory (memory_search/memory_write).
|
|
42
|
+
- **Run telemetry.** Every /fix //debug //build run appends one JSONL line to
|
|
43
|
+
.phi/runs.jsonl (mode, phases with model+verdict+retries, sandbox exec count,
|
|
44
|
+
duration, outcome) — continuous measurement instead of campaigns.
|
|
45
|
+
|
|
46
|
+
30 new tests (escalation core 13, telemetry 6, instructions 5, integration 6 —
|
|
47
|
+
including a real-git multi-candidate arbitration that applies the minimal
|
|
48
|
+
passing patch and a real-run /fix escalation); full suite green (1469).
|
|
49
|
+
|
|
3
50
|
## [0.92.1] - 2026-07-12
|
|
4
51
|
|
|
5
52
|
### Docs
|
package/README.md
CHANGED
|
@@ -124,8 +124,9 @@ code, even across model families). Full design:
|
|
|
124
124
|
|
|
125
125
|
| Command | Job | Pipeline |
|
|
126
126
|
|---|---|---|
|
|
127
|
+
| `/fix <failing test \| repro \| description>` | Fix at single-shot cost, verified | SINGLE SHOT → sandbox oracle → green: done / red: escalate to /debug |
|
|
127
128
|
| `/plan <spec>` | Plan AND build a project | EXPLORE → PLAN → CODE → TEST → REVIEW |
|
|
128
|
-
| `/debug
|
|
129
|
+
| `/debug <…> [--candidates N]` | Turn a REAL failure green | REPRODUCE → LOCALIZE → FIX (×N diverse models) → real-run arbitration / VERIFY |
|
|
129
130
|
| `/build <spec>` | Build until it runs | EXPLORE → PLAN → CODE → BUILD-VERIFY (run recipe + acceptance + executable red-team) |
|
|
130
131
|
| `/sandbox [status\|prepare\|run <cmd>]` | Inspect the guaranteed environment | — |
|
|
131
132
|
|
|
@@ -151,9 +152,11 @@ with automatic fallback on transient provider errors.
|
|
|
151
152
|
|
|
152
153
|
**Measured, honestly:** on a 13-instance SWE-bench-lite slice (official harness),
|
|
153
154
|
a strong single-shot baseline resolved 7/13 vs `/debug` 6/13 at ~2.5× the time —
|
|
154
|
-
with `/debug` producing smaller patches and zero toxic ones.
|
|
155
|
-
|
|
156
|
-
|
|
155
|
+
with `/debug` producing smaller patches and zero toxic ones. `/fix` is the
|
|
156
|
+
composition that measurement demanded: by construction never worse than the
|
|
157
|
+
single shot, escalating exactly where a real red run proves the shot failed.
|
|
158
|
+
Every run appends a telemetry line to `.phi/runs.jsonl` (phases, models,
|
|
159
|
+
verdicts, sandbox executions, duration) so behaviour is measured continuously.
|
|
157
160
|
|
|
158
161
|
---
|
|
159
162
|
|
|
@@ -23,9 +23,20 @@ import { join } from "node:path";
|
|
|
23
23
|
import { Type } from "@sinclair/typebox";
|
|
24
24
|
import type { ExtensionAPI } from "phi-code";
|
|
25
25
|
import { type AgentDef, loadAgentDef } from "./providers/agent-def.js";
|
|
26
|
-
import {
|
|
27
|
-
import {
|
|
28
|
-
|
|
26
|
+
import { diffChangedLines } from "./providers/candidate-select.js";
|
|
27
|
+
import {
|
|
28
|
+
buildVerifyInstruction,
|
|
29
|
+
debugPhaseInstructions,
|
|
30
|
+
singleShotInstruction,
|
|
31
|
+
} from "./providers/debug-build-commands.js";
|
|
32
|
+
import {
|
|
33
|
+
decideVerify,
|
|
34
|
+
type FailingState,
|
|
35
|
+
parseFailingState,
|
|
36
|
+
type VerifiedCandidate,
|
|
37
|
+
} from "./providers/debug-contract.js";
|
|
38
|
+
import { decideEscalation, parseReproCmd, pickCandidateModels, type RoutingLike } from "./providers/escalation.js";
|
|
39
|
+
import { passed, runCommand, tail } from "./providers/execution.js";
|
|
29
40
|
import { defaultExplorerSpecs, READONLY_EXPLORER_TOOLS, runExploreFanout } from "./providers/explore-fanout.js";
|
|
30
41
|
import {
|
|
31
42
|
analyzePhaseMessages,
|
|
@@ -35,6 +46,7 @@ import {
|
|
|
35
46
|
type StructuredPhaseResult,
|
|
36
47
|
} from "./providers/phase-machine.js";
|
|
37
48
|
import { resolveSandbox, type Sandbox } from "./providers/sandbox.js";
|
|
49
|
+
import { appendRunRecord, buildRunRecord, type PhaseRecord } from "./providers/telemetry.js";
|
|
38
50
|
import { triage } from "./providers/triage.js";
|
|
39
51
|
|
|
40
52
|
// ─── Types ───────────────────────────────────────────────────────────────
|
|
@@ -300,9 +312,33 @@ export default function orchestratorExtension(pi: ExtensionAPI) {
|
|
|
300
312
|
let phaseQueue: OrchestratorPhase[] = [];
|
|
301
313
|
let orchestrationActive = false;
|
|
302
314
|
// Which pipeline is running. "plan" keeps the existing 5-phase agent_end path
|
|
303
|
-
// untouched; "debug"/"build" are driven by the separate linear generic
|
|
304
|
-
// so /plan's fragile review-fix + "/5" logic is never engaged for them.
|
|
305
|
-
let orchestrationMode: "plan" | "debug" | "build" = "plan";
|
|
315
|
+
// untouched; "debug"/"build"/"fix" are driven by the separate linear generic
|
|
316
|
+
// driver so /plan's fragile review-fix + "/5" logic is never engaged for them.
|
|
317
|
+
let orchestrationMode: "plan" | "debug" | "build" | "fix" = "plan";
|
|
318
|
+
// /fix escalation state: the failing state the command started from, whether
|
|
319
|
+
// the driver-level oracle already ran (it must run exactly once, after the
|
|
320
|
+
// single-shot phase), and the reproduction the shot itself declared via its
|
|
321
|
+
// REPRO-CMD handoff line (prose inputs have no runnable check otherwise).
|
|
322
|
+
let fixContext: { state: FailingState; oracleRan: boolean; reproFromShot?: string } | null = null;
|
|
323
|
+
// /debug --candidates N state: each FIX candidate's captured patch, the
|
|
324
|
+
// reproduction command used for deterministic arbitration (from the input or
|
|
325
|
+
// the REPRODUCE handoff's REPRO-CMD line), and whether arbitration ran.
|
|
326
|
+
let candidateContext: {
|
|
327
|
+
total: number;
|
|
328
|
+
reproCmd?: string;
|
|
329
|
+
list: { source: string; patch: string }[];
|
|
330
|
+
arbitrated: boolean;
|
|
331
|
+
} | null = null;
|
|
332
|
+
// Telemetry: per-phase records for the current generic run + sandbox_run
|
|
333
|
+
// invocation count (reset at orchestration start, flushed at finish).
|
|
334
|
+
let runPhaseRecords: PhaseRecord[] = [];
|
|
335
|
+
let sandboxExecCount = 0;
|
|
336
|
+
|
|
337
|
+
/** Run a git command in cwd; returns stdout ("" on any failure). */
|
|
338
|
+
function gitIn(cwd: string, args: string): string {
|
|
339
|
+
const r = runCommand(`git ${args}`, { cwd, timeoutMs: 60_000 });
|
|
340
|
+
return passed(r) ? r.stdout : "";
|
|
341
|
+
}
|
|
306
342
|
let activeAgentPrompt: string | null = null;
|
|
307
343
|
let _activeAgentTools: string[] | null = null;
|
|
308
344
|
let savedTools: string[] | null = null;
|
|
@@ -430,6 +466,7 @@ export default function orchestratorExtension(pi: ExtensionAPI) {
|
|
|
430
466
|
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
431
467
|
const p = params as { command: string; timeoutSeconds?: number };
|
|
432
468
|
const cwd = ctx?.cwd || process.cwd();
|
|
469
|
+
if (orchestrationActive) sandboxExecCount++;
|
|
433
470
|
const sandbox = getSessionSandbox(cwd);
|
|
434
471
|
const result = sandbox.exec(p.command, {
|
|
435
472
|
timeoutMs: Math.max(1, Math.min(1800, p.timeoutSeconds ?? 300)) * 1000,
|
|
@@ -1211,16 +1248,50 @@ Tag the note with relevant keywords for vector search.
|
|
|
1211
1248
|
|
|
1212
1249
|
// /debug: REPRODUCE → LOCALIZE → FIX → VERIFY. Each phase routes to a model
|
|
1213
1250
|
// that does NOT share the coder's blind spot (verify on the test family).
|
|
1214
|
-
function buildDebugPhases(state: FailingState): OrchestratorPhase[] {
|
|
1251
|
+
function buildDebugPhases(state: FailingState, candidates = 1): OrchestratorPhase[] {
|
|
1215
1252
|
const ins = debugPhaseInstructions(state);
|
|
1253
|
+
if (candidates <= 1) {
|
|
1254
|
+
return [
|
|
1255
|
+
genericPhase("reproduce", "🔴 Phase 1 — REPRODUCE", "test", "test", ins.reproduce),
|
|
1256
|
+
genericPhase("localize", "🔎 Phase 2 — LOCALIZE", "explore", "explore", ins.localize),
|
|
1257
|
+
genericPhase("fix", "🔧 Phase 3 — FIX", "code", "code", ins.fix),
|
|
1258
|
+
genericPhase("verify", "✅ Phase 4 — VERIFY", "test", "test", ins.verify),
|
|
1259
|
+
];
|
|
1260
|
+
}
|
|
1261
|
+
// Multi-candidate: N independent FIX attempts on DIVERSE model families
|
|
1262
|
+
// (diversity proposes), then the driver arbitrates each candidate with a
|
|
1263
|
+
// REAL run (the oracle disposes) — no VERIFY agent phase, no vote.
|
|
1264
|
+
const models = pickCandidateModels(readRoutingConfig(), candidates);
|
|
1265
|
+
const codeRoute = routeFor("code");
|
|
1266
|
+
const fixPhases = Array.from({ length: candidates }, (_, i) => {
|
|
1267
|
+
const ref = models[i % Math.max(1, models.length)] ?? codeRoute.preferred;
|
|
1268
|
+
const phase = genericPhase(
|
|
1269
|
+
"fix-cand",
|
|
1270
|
+
`🔧 Phase ${3 + i} — FIX candidate ${i + 1}/${candidates}`,
|
|
1271
|
+
"code",
|
|
1272
|
+
"code",
|
|
1273
|
+
`${ins.fix}\n\n**Candidate protocol:** you are INDEPENDENT candidate ${i + 1} of ${candidates}. Other candidates attempt the same fault separately; a REAL run of the reproduction and the suite arbitrates between the resulting patches. Make your own best minimal fix — do not hedge for the others.`,
|
|
1274
|
+
);
|
|
1275
|
+
phase.model = ref;
|
|
1276
|
+
phase.fallback = codeRoute.fallback;
|
|
1277
|
+
return phase;
|
|
1278
|
+
});
|
|
1216
1279
|
return [
|
|
1217
1280
|
genericPhase("reproduce", "🔴 Phase 1 — REPRODUCE", "test", "test", ins.reproduce),
|
|
1218
1281
|
genericPhase("localize", "🔎 Phase 2 — LOCALIZE", "explore", "explore", ins.localize),
|
|
1219
|
-
|
|
1220
|
-
genericPhase("verify", "✅ Phase 4 — VERIFY", "test", "test", ins.verify),
|
|
1282
|
+
...fixPhases,
|
|
1221
1283
|
];
|
|
1222
1284
|
}
|
|
1223
1285
|
|
|
1286
|
+
/** Read ~/.phi/agent/routing.json (shape shared with routeFor). */
|
|
1287
|
+
function readRoutingConfig(): RoutingLike {
|
|
1288
|
+
try {
|
|
1289
|
+
return JSON.parse(readFileSync(join(homedir(), ".phi", "agent", "routing.json"), "utf-8")) as RoutingLike;
|
|
1290
|
+
} catch {
|
|
1291
|
+
return {};
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
|
|
1224
1295
|
// /build: reuse /plan's EXPLORE→PLAN→CODE, then an execution-grounded
|
|
1225
1296
|
// BUILD-VERIFY that runs the recipe, checks acceptance, red-teams the
|
|
1226
1297
|
// boundary, and routes real failures to the /debug protocol.
|
|
@@ -1241,9 +1312,25 @@ Tag the note with relevant keywords for vector search.
|
|
|
1241
1312
|
const minutes = Math.floor(elapsed / 60);
|
|
1242
1313
|
const seconds = elapsed % 60;
|
|
1243
1314
|
const mode = orchestrationMode;
|
|
1315
|
+
// Telemetry: one JSONL line per run — continuous measurement for free.
|
|
1316
|
+
appendRunRecord(
|
|
1317
|
+
ctx.cwd || process.cwd(),
|
|
1318
|
+
buildRunRecord({
|
|
1319
|
+
mode,
|
|
1320
|
+
startedAtMs: phaseStartTime,
|
|
1321
|
+
endedAtMs: Date.now(),
|
|
1322
|
+
phases: runPhaseRecords,
|
|
1323
|
+
completedPhases,
|
|
1324
|
+
skippedPhases,
|
|
1325
|
+
sandboxExecs: sandboxExecCount,
|
|
1326
|
+
outcome: headline,
|
|
1327
|
+
}),
|
|
1328
|
+
);
|
|
1244
1329
|
setOrchestrationActive(false);
|
|
1245
1330
|
phasePending = false;
|
|
1246
1331
|
orchestrationMode = "plan";
|
|
1332
|
+
fixContext = null;
|
|
1333
|
+
candidateContext = null;
|
|
1247
1334
|
deactivateAgent();
|
|
1248
1335
|
if (phaseTimeoutId) {
|
|
1249
1336
|
clearTimeout(phaseTimeoutId);
|
|
@@ -1264,10 +1351,149 @@ Tag the note with relevant keywords for vector search.
|
|
|
1264
1351
|
);
|
|
1265
1352
|
}
|
|
1266
1353
|
|
|
1354
|
+
// /fix driver-level oracle: after the single shot, run the reproduction and
|
|
1355
|
+
// the project suite in the REAL sandbox. Green → finish at baseline cost.
|
|
1356
|
+
// Red → escalate: enqueue the full /debug pipeline seeded with the red run.
|
|
1357
|
+
// Deterministic — zero model tokens; the verdict comes from exit codes.
|
|
1358
|
+
function runFixOracleAndMaybeEscalate(ctx: any): boolean {
|
|
1359
|
+
if (orchestrationMode !== "fix" || !fixContext || fixContext.oracleRan) return false;
|
|
1360
|
+
fixContext.oracleRan = true;
|
|
1361
|
+
const cwd = ctx.cwd || process.cwd();
|
|
1362
|
+
const sandbox = getSessionSandbox(cwd);
|
|
1363
|
+
const reproCmd =
|
|
1364
|
+
fixContext.state.failingTest?.trim() || fixContext.state.reproCommand?.trim() || fixContext.reproFromShot;
|
|
1365
|
+
const suiteCmd = sandbox.recipe.test?.trim();
|
|
1366
|
+
|
|
1367
|
+
if (!sandbox.available() && (reproCmd || suiteCmd)) {
|
|
1368
|
+
ctx.ui.notify(
|
|
1369
|
+
`\n⚠️ **/fix oracle: sandbox unavailable** (${sandbox.reason}) — the single shot cannot be verified. Treating it as UNVERIFIED.`,
|
|
1370
|
+
"warning",
|
|
1371
|
+
);
|
|
1372
|
+
finishGenericOrchestration(ctx, `⚠️ **/fix finished UNVERIFIED — no executable environment for the oracle.**`);
|
|
1373
|
+
return true;
|
|
1374
|
+
}
|
|
1375
|
+
|
|
1376
|
+
ctx.ui.notify(
|
|
1377
|
+
`\n🧪 **/fix oracle** — running ${[reproCmd && "the reproduction", suiteCmd && "the suite"].filter(Boolean).join(" and ") || "nothing (no runnable check)"} in ${sandbox.describe()}…`,
|
|
1378
|
+
"info",
|
|
1379
|
+
);
|
|
1380
|
+
const repro = reproCmd ? sandbox.exec(reproCmd, { timeoutMs: 10 * 60 * 1000 }) : null;
|
|
1381
|
+
const suite = suiteCmd ? sandbox.exec(suiteCmd, { timeoutMs: 20 * 60 * 1000 }) : null;
|
|
1382
|
+
const decision = decideEscalation(fixContext.state, { repro, suite });
|
|
1383
|
+
|
|
1384
|
+
if (decision.action === "done-green") {
|
|
1385
|
+
finishGenericOrchestration(
|
|
1386
|
+
ctx,
|
|
1387
|
+
`✅ **/fix finished GREEN at single-shot cost** — oracle evidence: ${decision.evidence}.`,
|
|
1388
|
+
);
|
|
1389
|
+
return true;
|
|
1390
|
+
}
|
|
1391
|
+
if (decision.action === "done-unverified") {
|
|
1392
|
+
finishGenericOrchestration(
|
|
1393
|
+
ctx,
|
|
1394
|
+
`⚠️ **/fix finished UNVERIFIED** — ${decision.reason}. Provide a failing test or a repro command for a guaranteed verdict.`,
|
|
1395
|
+
);
|
|
1396
|
+
return true;
|
|
1397
|
+
}
|
|
1398
|
+
// escalate: the pipeline inherits the RED run as its concrete failing state.
|
|
1399
|
+
ctx.ui.notify(
|
|
1400
|
+
`\n🔺 **/fix escalating to the full /debug pipeline** — ${decision.diagnostic}. The single shot was not enough; the pipeline inherits the red run as its reproduction.`,
|
|
1401
|
+
"warning",
|
|
1402
|
+
);
|
|
1403
|
+
phaseQueue.push(...buildDebugPhases(decision.failing));
|
|
1404
|
+
return false; // fall through: sendNextGenericPhase will dispatch REPRODUCE
|
|
1405
|
+
}
|
|
1406
|
+
|
|
1407
|
+
// Multi-candidate arbitration — the oracle disposes. Apply each captured
|
|
1408
|
+
// candidate in turn, run the reproduction (and the suite when known) in the
|
|
1409
|
+
// REAL sandbox, select the minimal passing candidate (candidate-select), and
|
|
1410
|
+
// leave the WINNER applied. If none passes: never a blank page — the smallest
|
|
1411
|
+
// non-empty candidate is left applied, clearly labelled UNVERIFIED.
|
|
1412
|
+
function runCandidateArbitration(ctx: any): boolean {
|
|
1413
|
+
if (!candidateContext || candidateContext.arbitrated) return false;
|
|
1414
|
+
candidateContext.arbitrated = true;
|
|
1415
|
+
const cc = candidateContext;
|
|
1416
|
+
const cwd = ctx.cwd || process.cwd();
|
|
1417
|
+
const sandbox = getSessionSandbox(cwd);
|
|
1418
|
+
const suiteCmd = sandbox.recipe.test?.trim();
|
|
1419
|
+
const nonEmpty = cc.list.filter((c) => c.patch.trim());
|
|
1420
|
+
|
|
1421
|
+
if (nonEmpty.length === 0) {
|
|
1422
|
+
finishGenericOrchestration(ctx, `⏸️ **/${orchestrationMode} BLOCKED — no candidate produced a patch.**`);
|
|
1423
|
+
return true;
|
|
1424
|
+
}
|
|
1425
|
+
|
|
1426
|
+
const applyPatch = (patch: string): boolean => {
|
|
1427
|
+
try {
|
|
1428
|
+
const f = join(cwd, ".phi", "candidate.patch");
|
|
1429
|
+
writeFileSync(f, patch, "utf-8");
|
|
1430
|
+
const r = runCommand(`git apply --whitespace=nowarn "${f}"`, { cwd, timeoutMs: 60_000 });
|
|
1431
|
+
try {
|
|
1432
|
+
unlinkSync(f);
|
|
1433
|
+
} catch {
|
|
1434
|
+
/* best effort */
|
|
1435
|
+
}
|
|
1436
|
+
return passed(r);
|
|
1437
|
+
} catch {
|
|
1438
|
+
return false;
|
|
1439
|
+
}
|
|
1440
|
+
};
|
|
1441
|
+
|
|
1442
|
+
ctx.ui.notify(
|
|
1443
|
+
`\n⚖️ **Arbitrating ${nonEmpty.length} candidate(s)** with real runs${cc.reproCmd ? ` — repro \`${cc.reproCmd}\`` : ""}${suiteCmd ? `, suite \`${suiteCmd}\`` : ""} in ${sandbox.describe()}…`,
|
|
1444
|
+
"info",
|
|
1445
|
+
);
|
|
1446
|
+
|
|
1447
|
+
const verified: VerifiedCandidate[] = [];
|
|
1448
|
+
for (const cand of nonEmpty) {
|
|
1449
|
+
if (!applyPatch(cand.patch)) {
|
|
1450
|
+
verified.push({ source: cand.source, patch: cand.patch, reproAfter: null, suite: null });
|
|
1451
|
+
continue;
|
|
1452
|
+
}
|
|
1453
|
+
const reproAfter =
|
|
1454
|
+
cc.reproCmd && sandbox.available() ? sandbox.exec(cc.reproCmd, { timeoutMs: 10 * 60 * 1000 }) : null;
|
|
1455
|
+
const suite = suiteCmd && sandbox.available() ? sandbox.exec(suiteCmd, { timeoutMs: 20 * 60 * 1000 }) : null;
|
|
1456
|
+
verified.push({ source: cand.source, patch: cand.patch, reproAfter, suite });
|
|
1457
|
+
gitIn(cwd, "checkout -- .");
|
|
1458
|
+
const rp = reproAfter
|
|
1459
|
+
? `repro exit ${reproAfter.exitCode}${reproAfter.timedOut ? " TIMEOUT" : ""}`
|
|
1460
|
+
: "repro n/a";
|
|
1461
|
+
const st = suite ? `suite exit ${suite.exitCode}${suite.timedOut ? " TIMEOUT" : ""}` : "suite n/a";
|
|
1462
|
+
ctx.ui.notify(` ${cand.source}: ${rp}, ${st}`, "info");
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1465
|
+
const outcome = decideVerify(verified, Boolean(suiteCmd));
|
|
1466
|
+
if (outcome.verdict === "FIXED" && outcome.patch) {
|
|
1467
|
+
applyPatch(outcome.patch);
|
|
1468
|
+
finishGenericOrchestration(
|
|
1469
|
+
ctx,
|
|
1470
|
+
`✅ **/${orchestrationMode} FIXED by candidate arbitration** — ${outcome.reason}. Evidence: repro passes${suiteCmd ? ", suite green" : ""} (real runs above). The winning patch is applied.`,
|
|
1471
|
+
);
|
|
1472
|
+
return true;
|
|
1473
|
+
}
|
|
1474
|
+
|
|
1475
|
+
// Honest failure — but never a blank page: leave the smallest non-empty
|
|
1476
|
+
// candidate applied, clearly labelled unverified (lesson: sympy-11870).
|
|
1477
|
+
const smallest = [...nonEmpty].sort((a, b) => diffChangedLines(a.patch) - diffChangedLines(b.patch))[0];
|
|
1478
|
+
const draftApplied = applyPatch(smallest.patch);
|
|
1479
|
+
finishGenericOrchestration(
|
|
1480
|
+
ctx,
|
|
1481
|
+
`⏸️ **/${orchestrationMode} BLOCKED — no candidate passed arbitration** (${outcome.reason ?? "see runs above"}).` +
|
|
1482
|
+
(draftApplied
|
|
1483
|
+
? ` An **UNVERIFIED draft** (smallest candidate, ${smallest.source}) is left in the working tree — \`git diff\` to inspect, \`git checkout -- .\` to discard.`
|
|
1484
|
+
: ""),
|
|
1485
|
+
);
|
|
1486
|
+
return true;
|
|
1487
|
+
}
|
|
1488
|
+
|
|
1267
1489
|
function sendNextGenericPhase(ctx: any) {
|
|
1268
1490
|
if (phaseQueue.length === 0) {
|
|
1269
|
-
|
|
1270
|
-
return;
|
|
1491
|
+
if (runCandidateArbitration(ctx)) return;
|
|
1492
|
+
if (runFixOracleAndMaybeEscalate(ctx)) return;
|
|
1493
|
+
if (phaseQueue.length === 0) {
|
|
1494
|
+
finishGenericOrchestration(ctx, `✅ **/${orchestrationMode} finished.**`);
|
|
1495
|
+
return;
|
|
1496
|
+
}
|
|
1271
1497
|
}
|
|
1272
1498
|
const phase = phaseQueue.shift()!;
|
|
1273
1499
|
phasePending = true;
|
|
@@ -1320,7 +1546,9 @@ Tag the note with relevant keywords for vector search.
|
|
|
1320
1546
|
}
|
|
1321
1547
|
|
|
1322
1548
|
if (!phasePending) {
|
|
1323
|
-
|
|
1549
|
+
// Route through sendNextGenericPhase so the /fix oracle still runs on
|
|
1550
|
+
// this (duplicate-event) completion path instead of being bypassed.
|
|
1551
|
+
if (phaseQueue.length === 0) sendNextGenericPhase(ctx);
|
|
1324
1552
|
return;
|
|
1325
1553
|
}
|
|
1326
1554
|
if (phaseTimeoutId) {
|
|
@@ -1342,6 +1570,18 @@ Tag the note with relevant keywords for vector search.
|
|
|
1342
1570
|
maxToolCallsPerPhase: MAX_TOOL_CALLS_PER_PHASE,
|
|
1343
1571
|
});
|
|
1344
1572
|
|
|
1573
|
+
// Telemetry: one record per phase attempt (retries appear as extra rows).
|
|
1574
|
+
if (currentPhase) {
|
|
1575
|
+
runPhaseRecords.push({
|
|
1576
|
+
key: currentPhase.key,
|
|
1577
|
+
label: currentPhase.label,
|
|
1578
|
+
model: currentPhase.useFallback ? currentPhase.fallback : currentPhase.model,
|
|
1579
|
+
verdict: outcome.verdict,
|
|
1580
|
+
retried: Boolean(currentPhase.retried),
|
|
1581
|
+
blockedRetried: Boolean(currentPhase.blockedRetried),
|
|
1582
|
+
});
|
|
1583
|
+
}
|
|
1584
|
+
|
|
1345
1585
|
if (decision.action === "stop") {
|
|
1346
1586
|
ctx.ui.notify(
|
|
1347
1587
|
`\n❌ **/${orchestrationMode} aborted:** API authentication error (401). Check your API key.`,
|
|
@@ -1387,10 +1627,55 @@ Tag the note with relevant keywords for vector search.
|
|
|
1387
1627
|
`\n⏸️ **${currentPhase.label} reported BLOCKED (confirmed on retry).** ${outcome.handoff || "No reproducible/verifiable result."}`,
|
|
1388
1628
|
"warning",
|
|
1389
1629
|
);
|
|
1390
|
-
|
|
1630
|
+
// Never a silent blank page: if the run left changes in the tree (an
|
|
1631
|
+
// unverified FIX attempt), say so instead of pretending nothing exists.
|
|
1632
|
+
const draft = gitIn(ctx.cwd || process.cwd(), "diff --stat").trim();
|
|
1633
|
+
finishGenericOrchestration(
|
|
1634
|
+
ctx,
|
|
1635
|
+
`⏸️ **/${orchestrationMode} stopped: BLOCKED at ${currentPhase.label}.**` +
|
|
1636
|
+
(draft
|
|
1637
|
+
? `\n📝 An **UNVERIFIED draft** remains in the working tree (\`git diff\` to inspect, \`git checkout -- .\` to discard):\n\`\`\`\n${draft.slice(0, 600)}\n\`\`\``
|
|
1638
|
+
: ""),
|
|
1639
|
+
);
|
|
1391
1640
|
return;
|
|
1392
1641
|
}
|
|
1393
1642
|
|
|
1643
|
+
// Multi-candidate bookkeeping (mode debug, --candidates N):
|
|
1644
|
+
// - after REPRODUCE: learn the exact repro command from the REPRO-CMD
|
|
1645
|
+
// handoff line (used later for deterministic arbitration);
|
|
1646
|
+
// - after each FIX candidate: capture its patch (git diff of tracked
|
|
1647
|
+
// files) and RESET the tree so the next candidate starts clean.
|
|
1648
|
+
// /fix: the single shot may have WRITTEN its own reproduction (prose
|
|
1649
|
+
// inputs) and declared it via REPRO-CMD — the driver oracle re-runs it.
|
|
1650
|
+
if (fixContext && currentPhase?.key === "shot" && !fixContext.reproFromShot) {
|
|
1651
|
+
fixContext.reproFromShot = parseReproCmd(outcome.handoff);
|
|
1652
|
+
if (fixContext.reproFromShot) {
|
|
1653
|
+
ctx.ui.notify(`\n🧷 Shot-declared reproduction registered: \`${fixContext.reproFromShot}\``, "info");
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
|
|
1657
|
+
if (candidateContext && currentPhase) {
|
|
1658
|
+
const cwd = ctx.cwd || process.cwd();
|
|
1659
|
+
if (currentPhase.key === "reproduce" && !candidateContext.reproCmd) {
|
|
1660
|
+
candidateContext.reproCmd = parseReproCmd(outcome.handoff);
|
|
1661
|
+
if (candidateContext.reproCmd) {
|
|
1662
|
+
ctx.ui.notify(`\n🧷 Arbitration reproduction registered: \`${candidateContext.reproCmd}\``, "info");
|
|
1663
|
+
}
|
|
1664
|
+
}
|
|
1665
|
+
if (currentPhase.key === "fix-cand") {
|
|
1666
|
+
const patch = gitIn(cwd, "diff").trim();
|
|
1667
|
+
candidateContext.list.push({
|
|
1668
|
+
source: `${currentPhase.label} (${currentPhase.useFallback ? currentPhase.fallback : currentPhase.model})`,
|
|
1669
|
+
patch: patch ? `${patch}\n` : "",
|
|
1670
|
+
});
|
|
1671
|
+
gitIn(cwd, "checkout -- .");
|
|
1672
|
+
ctx.ui.notify(
|
|
1673
|
+
`\n📦 Candidate ${candidateContext.list.length}/${candidateContext.total} captured (${patch ? `${patch.length} bytes` : "EMPTY"}) — tree reset for the next candidate.`,
|
|
1674
|
+
"info",
|
|
1675
|
+
);
|
|
1676
|
+
}
|
|
1677
|
+
}
|
|
1678
|
+
|
|
1394
1679
|
// Propagate a concise handoff to the next phase, like /plan does.
|
|
1395
1680
|
const nextBrief = buildNextBrief(
|
|
1396
1681
|
analysis,
|
|
@@ -1408,13 +1693,18 @@ Tag the note with relevant keywords for vector search.
|
|
|
1408
1693
|
}
|
|
1409
1694
|
|
|
1410
1695
|
function startGenericOrchestration(
|
|
1411
|
-
mode: "debug" | "build",
|
|
1696
|
+
mode: "debug" | "build" | "fix",
|
|
1412
1697
|
phases: OrchestratorPhase[],
|
|
1413
1698
|
ctx: any,
|
|
1414
1699
|
headline: string,
|
|
1415
1700
|
) {
|
|
1416
1701
|
phaseQueue = phases.slice(1);
|
|
1417
1702
|
orchestrationMode = mode;
|
|
1703
|
+
// Defensive: stale /fix or candidate state must never leak across runs.
|
|
1704
|
+
if (mode !== "fix") fixContext = null;
|
|
1705
|
+
if (mode !== "debug") candidateContext = null;
|
|
1706
|
+
runPhaseRecords = [];
|
|
1707
|
+
sandboxExecCount = 0;
|
|
1418
1708
|
setOrchestrationActive(true);
|
|
1419
1709
|
phasePending = true;
|
|
1420
1710
|
originalModel = ctx.model || null;
|
|
@@ -1911,7 +2201,15 @@ It runs REPRODUCE → LOCALIZE → FIX → VERIFY and only reports FIXED with a
|
|
|
1911
2201
|
return;
|
|
1912
2202
|
}
|
|
1913
2203
|
|
|
1914
|
-
|
|
2204
|
+
// Opt-in multi-candidate FIX: `--candidates N` (2..4). Diversity
|
|
2205
|
+
// proposes (N model families patch independently), the oracle disposes
|
|
2206
|
+
// (a real run arbitrates). Cost-disciplined: default stays 1.
|
|
2207
|
+
const candMatch = raw.match(/(^|\s)--candidates[= ](\d)\b/);
|
|
2208
|
+
let candidates = candMatch ? Math.max(1, Math.min(4, Number(candMatch[2]))) : 1;
|
|
2209
|
+
const cleaned = raw.replace(/(^|\s)--candidates[= ]\d\b/g, " ").trim();
|
|
2210
|
+
const cwd = ctx.cwd || process.cwd();
|
|
2211
|
+
|
|
2212
|
+
const state: FailingState = parseFailingState(cleaned, { cwd });
|
|
1915
2213
|
// Honest gate at the boundary: /debug needs a reproducible failure. A bare
|
|
1916
2214
|
// prose description with no test/command still runs (REPRODUCE will try to
|
|
1917
2215
|
// derive one), but we tell the user the oracle depends on a real run.
|
|
@@ -1922,13 +2220,96 @@ It runs REPRODUCE → LOCALIZE → FIX → VERIFY and only reports FIXED with a
|
|
|
1922
2220
|
);
|
|
1923
2221
|
}
|
|
1924
2222
|
|
|
2223
|
+
if (candidates > 1) {
|
|
2224
|
+
// Candidate capture RESETS the tree between attempts — refuse to do
|
|
2225
|
+
// that over a user's uncommitted work (non-deletion policy).
|
|
2226
|
+
const porcelain = runCommand("git status --porcelain", { cwd, timeoutMs: 30_000 });
|
|
2227
|
+
if (!passed(porcelain)) {
|
|
2228
|
+
ctx.ui.notify(
|
|
2229
|
+
"⚠️ `--candidates` needs a git repository (candidate patches are captured and arbitrated via git). Falling back to the single-candidate pipeline.",
|
|
2230
|
+
"warning",
|
|
2231
|
+
);
|
|
2232
|
+
candidates = 1;
|
|
2233
|
+
} else if (porcelain.stdout.trim()) {
|
|
2234
|
+
ctx.ui.notify(
|
|
2235
|
+
"⚠️ Your working tree has uncommitted changes — candidate arbitration resets the tree between attempts and will NOT run over them. Commit/stash first for multi-candidate. Falling back to the single-candidate pipeline.",
|
|
2236
|
+
"warning",
|
|
2237
|
+
);
|
|
2238
|
+
candidates = 1;
|
|
2239
|
+
}
|
|
2240
|
+
}
|
|
2241
|
+
|
|
1925
2242
|
await ensurePlansDir();
|
|
1926
|
-
const phases = buildDebugPhases(state);
|
|
2243
|
+
const phases = buildDebugPhases(state, candidates);
|
|
2244
|
+
if (candidates > 1) {
|
|
2245
|
+
candidateContext = {
|
|
2246
|
+
total: candidates,
|
|
2247
|
+
reproCmd: state.failingTest?.trim() || state.reproCommand?.trim() || undefined,
|
|
2248
|
+
list: [],
|
|
2249
|
+
arbitrated: false,
|
|
2250
|
+
};
|
|
2251
|
+
}
|
|
1927
2252
|
startGenericOrchestration(
|
|
1928
2253
|
"debug",
|
|
1929
2254
|
phases,
|
|
1930
2255
|
ctx,
|
|
1931
|
-
|
|
2256
|
+
candidates > 1
|
|
2257
|
+
? `🐛 **/debug started (multi-candidate ×${candidates})** — REPRODUCE → LOCALIZE → ${candidates} independent FIXes → real-run arbitration\n`
|
|
2258
|
+
: `🐛 **/debug started** — 4 execution-grounded phases (verdict requires a real run)\n`,
|
|
2259
|
+
);
|
|
2260
|
+
},
|
|
2261
|
+
});
|
|
2262
|
+
|
|
2263
|
+
// ─── /fix Command — single-shot first, oracle next, escalate on red ──
|
|
2264
|
+
// Measured (n=13, official SWE-bench harness): the single shot resolved 7/13
|
|
2265
|
+
// vs the full pipeline's 6/13 at ~2.5× the time, and both arms converged on
|
|
2266
|
+
// 11/13. /fix is the composition: by construction never worse than the
|
|
2267
|
+
// single shot, and it inherits the pipeline exactly where a REAL red run
|
|
2268
|
+
// proves the shot failed.
|
|
2269
|
+
|
|
2270
|
+
pi.registerCommand("fix", {
|
|
2271
|
+
description:
|
|
2272
|
+
"Fix at single-shot cost, verified by the sandbox oracle — escalates to the full /debug pipeline only if a real run stays red",
|
|
2273
|
+
handler: async (args, ctx) => {
|
|
2274
|
+
if (orchestrationActive) {
|
|
2275
|
+
ctx.ui.notify("An orchestration is already running. Let it finish, or restart the session.", "warning");
|
|
2276
|
+
return;
|
|
2277
|
+
}
|
|
2278
|
+
const raw = args.trim();
|
|
2279
|
+
if (!raw) {
|
|
2280
|
+
ctx.ui.notify(
|
|
2281
|
+
`**Usage:** \`/fix <failing test | repro command | description>\`
|
|
2282
|
+
|
|
2283
|
+
/fix = one direct attempt, then the ORACLE (reproduction + suite in the sandbox):
|
|
2284
|
+
green → done at single-shot cost;
|
|
2285
|
+
red → the full /debug pipeline takes over, seeded with the red run.
|
|
2286
|
+
**Examples:**
|
|
2287
|
+
/fix pytest tests/test_auth.py::test_login_returns_jwt
|
|
2288
|
+
/fix node repro.js (crashes on empty input, should return [])
|
|
2289
|
+
/fix the /login route 500s when the body is missing
|
|
2290
|
+
|
|
2291
|
+
With no runnable check at all, the result is honestly labelled UNVERIFIED.`,
|
|
2292
|
+
"info",
|
|
2293
|
+
);
|
|
2294
|
+
return;
|
|
2295
|
+
}
|
|
2296
|
+
|
|
2297
|
+
const state: FailingState = parseFailingState(raw, { cwd: ctx.cwd || process.cwd() });
|
|
2298
|
+
if (!state.failingTest && !state.reproCommand) {
|
|
2299
|
+
ctx.ui.notify(
|
|
2300
|
+
`⚠️ No failing test or repro command detected — the /fix oracle will rely on the project suite alone (or report UNVERIFIED if none is known). A concrete repro gives a guaranteed verdict.`,
|
|
2301
|
+
"warning",
|
|
2302
|
+
);
|
|
2303
|
+
}
|
|
2304
|
+
|
|
2305
|
+
await ensurePlansDir();
|
|
2306
|
+
fixContext = { state, oracleRan: false };
|
|
2307
|
+
const shot = genericPhase("shot", "🎯 Phase 1 — SINGLE SHOT", "code", "code", singleShotInstruction(state));
|
|
2308
|
+
startGenericOrchestration(
|
|
2309
|
+
"fix",
|
|
2310
|
+
[shot],
|
|
2311
|
+
ctx,
|
|
2312
|
+
`🎯 **/fix started** — single shot, then the sandbox oracle decides (green = done, red = full pipeline)\n`,
|
|
1932
2313
|
);
|
|
1933
2314
|
},
|
|
1934
2315
|
});
|
|
@@ -53,6 +53,7 @@ ${failing}
|
|
|
53
53
|
|
|
54
54
|
**Do exactly this:**
|
|
55
55
|
1. Run the reproduction on the CURRENT, unmodified code with the \`sandbox_run\` tool: \`sandbox_run ${repro}\`.
|
|
56
|
+
- If you must CONSTRUCT the reproduction (only a description was given): build it **literally from the issue** — copy the exact code snippets, inputs and expected values QUOTED in the issue text into a runnable script/test. Do NOT paraphrase the issue into your own interpretation; a reproduction of your interpretation validates your interpretation, not the bug (measured failure mode).
|
|
56
57
|
2. Paste the exact command and its full output.
|
|
57
58
|
3. Decide from what \`sandbox_run\` returned:
|
|
58
59
|
- If it FAILS as reported → capture the precise symptom (assertion, exception, exit code) and hand off to LOCALIZE.
|
|
@@ -60,8 +61,10 @@ ${failing}
|
|
|
60
61
|
- If \`sandbox_run\` reports \`SANDBOX UNAVAILABLE\` → STOP. Write \`BLOCKED: no executable environment\`.
|
|
61
62
|
4. Do NOT edit any source yet. This phase only observes.
|
|
62
63
|
5. **BLOCKED is a last resort, not a first reaction.** If a run fails for an infrastructure-looking reason (tool error, missing file you guessed wrong, transient failure), adjust and retry at least once — e.g. locate the real test paths, try an alternative reproduction — before concluding. Only report BLOCKED after a genuine attempt showed the state is not reproducible or not runnable.
|
|
63
|
-
6. **Last action:** call \`phase_result\` — \`verdict: PASS\` if it reproduced (proceed to LOCALIZE), or \`verdict: BLOCKED\` with the reason if you stopped
|
|
64
|
-
|
|
64
|
+
6. **Last action:** call \`phase_result\` — \`verdict: PASS\` if it reproduced (proceed to LOCALIZE), or \`verdict: BLOCKED\` with the reason if you stopped. Your handoff MUST contain a line with the EXACT command that reproduces the failure, in this machine-readable form (the orchestrator re-runs it to arbitrate candidate fixes):
|
|
65
|
+
\`\`\`
|
|
66
|
+
REPRO-CMD: <the exact command, e.g. python /testbed/repro_issue.py>
|
|
67
|
+
\`\`\`` + DEBUG_RULES,
|
|
65
68
|
|
|
66
69
|
localize: `You are the LOCALIZE agent (phase 2 of /debug). Drive from the REAL symptom the previous phase captured.
|
|
67
70
|
|
|
@@ -69,10 +72,11 @@ ${failing}
|
|
|
69
72
|
${failing}
|
|
70
73
|
|
|
71
74
|
**Do exactly this:**
|
|
72
|
-
1.
|
|
73
|
-
2. Read the
|
|
74
|
-
3.
|
|
75
|
-
4.
|
|
75
|
+
1. Call \`memory_search\` with the failing symbol/module names — a previous run may have already localized this area or a related failure.
|
|
76
|
+
2. Read the traceback / error from the reproduction. Follow it to the exact frame.
|
|
77
|
+
3. Read the implicated source and the symbols it touches (grep the nearby/changed identifiers).
|
|
78
|
+
4. Name the fault site precisely — file:line and the wrong assumption. Localization, not more review, is the lever here.
|
|
79
|
+
5. Hand off a crisp fault description; do NOT fix yet. Call \`memory_write\` with the fault site so future runs on this project start ahead.${DEBUG_RULES}`,
|
|
76
80
|
|
|
77
81
|
fix:
|
|
78
82
|
`You are the FIX agent (phase 3 of /debug). Produce the MINIMAL change that addresses the located root cause.
|
|
@@ -107,6 +111,33 @@ Reason: <only when BLOCKED>
|
|
|
107
111
|
};
|
|
108
112
|
}
|
|
109
113
|
|
|
114
|
+
/**
|
|
115
|
+
* The /fix single-shot phase — the measured-cheapest first attempt. One agent
|
|
116
|
+
* fixes directly (baseline cost); the DRIVER then oracle-checks the result
|
|
117
|
+
* deterministically (sandbox repro + suite) and escalates to the full /debug
|
|
118
|
+
* pipeline only if a real run is red. Measured rationale: the single shot
|
|
119
|
+
* resolved 7/13 vs the full pipeline's 6/13 at ~2.5× the time — so pay the
|
|
120
|
+
* pipeline only when a real run proves the shot failed.
|
|
121
|
+
*/
|
|
122
|
+
export function singleShotInstruction(state: FailingState): string {
|
|
123
|
+
const failing = formatFailingState(state);
|
|
124
|
+
return `You are the FIX agent (single shot — /fix phase 1). Fix the problem directly, with the MINIMAL change.
|
|
125
|
+
|
|
126
|
+
**The problem:**
|
|
127
|
+
${failing}
|
|
128
|
+
|
|
129
|
+
**Do exactly this:**
|
|
130
|
+
1. Read the relevant code and locate the root cause.
|
|
131
|
+
2. Make the smallest change that addresses it. Do NOT edit tests. Every added guard/branch is a liability.
|
|
132
|
+
3. If NO runnable check was provided above, WRITE a minimal reproduction script derived **literally from the issue** (copy its exact snippets/expected values — not your paraphrase) into an untracked file (e.g. \`repro_issue.py\`), and confirm with \`sandbox_run\` that it fails before your change / passes after.
|
|
133
|
+
4. You MAY use \`sandbox_run\` to check your work at any point (the project's real environment).
|
|
134
|
+
5. After your change, the orchestrator re-runs the reproduction (and the suite when known) in the sandbox itself: your work is judged by those REAL runs, not by your confidence. If they are red, a full diagnostic pipeline takes over from your change.
|
|
135
|
+
6. **Last action:** call \`phase_result\` with \`verdict: PASS\` and a handoff describing what you changed — and, when you wrote a reproduction, its exact command on a machine-readable line:
|
|
136
|
+
\`\`\`
|
|
137
|
+
REPRO-CMD: <the exact command, e.g. python repro_issue.py>
|
|
138
|
+
\`\`\`${DEBUG_RULES}`;
|
|
139
|
+
}
|
|
140
|
+
|
|
110
141
|
/**
|
|
111
142
|
* The /build execution-grounded verify phase: run the recipe, check acceptance,
|
|
112
143
|
* red-team the boundary the change touched, and route real failures to /debug.
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Escalation core — the pure decisions behind /fix (single-shot first, oracle
|
|
3
|
+
* next, escalate to /debug only on red).
|
|
4
|
+
*
|
|
5
|
+
* Measured rationale (n=13, official SWE-bench harness): the single-shot
|
|
6
|
+
* baseline resolved 7/13 while the full /debug pipeline resolved 6/13 at ~2.5×
|
|
7
|
+
* the time — and on 11/13 instances both arms converged to the same verdict.
|
|
8
|
+
* So the right architecture is neither: pay the single-shot price always, pay
|
|
9
|
+
* the pipeline price only when a REAL run says the single shot failed. By
|
|
10
|
+
* construction /fix is never worse than the baseline, and inherits /debug's
|
|
11
|
+
* upside exactly where it can matter.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import type { FailingState } from "./debug-contract.js";
|
|
15
|
+
import { type CommandResult, passed, tail } from "./execution.js";
|
|
16
|
+
|
|
17
|
+
export type EscalationDecision =
|
|
18
|
+
| { action: "done-green"; evidence: string }
|
|
19
|
+
| { action: "escalate"; failing: FailingState; diagnostic: string }
|
|
20
|
+
| { action: "done-unverified"; reason: string };
|
|
21
|
+
|
|
22
|
+
export interface OracleRuns {
|
|
23
|
+
/** The reproduction run (null = no runnable reproduction was available). */
|
|
24
|
+
repro: CommandResult | null;
|
|
25
|
+
/** The project suite run (null = no suite command known). */
|
|
26
|
+
suite: CommandResult | null;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Decide after the single shot: green oracle → done at baseline cost; any red
|
|
31
|
+
* run → escalate to /debug with the red run as the concrete failing state
|
|
32
|
+
* (never a paraphrase — the exact command and its tail). Nothing runnable →
|
|
33
|
+
* done, honestly labelled unverified.
|
|
34
|
+
*/
|
|
35
|
+
export function decideEscalation(state: FailingState, runs: OracleRuns): EscalationDecision {
|
|
36
|
+
const { repro, suite } = runs;
|
|
37
|
+
|
|
38
|
+
if (repro !== null && !passed(repro)) {
|
|
39
|
+
return {
|
|
40
|
+
action: "escalate",
|
|
41
|
+
failing: {
|
|
42
|
+
...state,
|
|
43
|
+
reproCommand: state.failingTest?.trim() || state.reproCommand?.trim() || repro.command,
|
|
44
|
+
trace: tail(repro, 30),
|
|
45
|
+
},
|
|
46
|
+
diagnostic: `reproduction still red after the single shot: \`${repro.command}\` → exit ${repro.exitCode ?? "?"}${repro.timedOut ? " (TIMEOUT)" : ""}`,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
if (suite !== null && !passed(suite)) {
|
|
51
|
+
return {
|
|
52
|
+
action: "escalate",
|
|
53
|
+
failing: {
|
|
54
|
+
...state,
|
|
55
|
+
failingTest: suite.command,
|
|
56
|
+
trace: tail(suite, 30),
|
|
57
|
+
},
|
|
58
|
+
diagnostic: `the suite is red after the single shot: \`${suite.command}\` → exit ${suite.exitCode ?? "?"}${suite.timedOut ? " (TIMEOUT)" : ""}`,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
if (repro === null && suite === null) {
|
|
63
|
+
return {
|
|
64
|
+
action: "done-unverified",
|
|
65
|
+
reason: "no runnable reproduction and no known suite command — the single shot could not be oracle-checked",
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const parts: string[] = [];
|
|
70
|
+
if (repro) parts.push(`repro \`${repro.command}\` → exit 0`);
|
|
71
|
+
if (suite) parts.push(`suite \`${suite.command}\` → exit 0`);
|
|
72
|
+
return { action: "done-green", evidence: parts.join("; ") };
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Parse the `REPRO-CMD: <command>` line the REPRODUCE phase is instructed to
|
|
77
|
+
* put in its handoff, so the driver can re-run the exact reproduction
|
|
78
|
+
* deterministically (multi-candidate arbitration, /fix oracle). Returns
|
|
79
|
+
* undefined when absent/malformed — callers must degrade gracefully.
|
|
80
|
+
*/
|
|
81
|
+
export function parseReproCmd(handoff: string | null | undefined): string | undefined {
|
|
82
|
+
if (!handoff) return undefined;
|
|
83
|
+
const m = handoff.match(/^\s*REPRO-CMD:\s*(.+)\s*$/im);
|
|
84
|
+
const cmd = m?.[1]?.trim();
|
|
85
|
+
return cmd && cmd.length > 1 ? cmd : undefined;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Minimal shape of routing.json this module needs (kept structural). */
|
|
89
|
+
export interface RoutingLike {
|
|
90
|
+
routes?: Record<string, { preferredModel?: string; fallback?: string } | undefined>;
|
|
91
|
+
default?: { model?: string };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Pick up to `n` DISTINCT model refs for candidate generation, favouring family
|
|
96
|
+
* diversity: the coder first (it knows the task), then reviewer/test/explore
|
|
97
|
+
* families, then fallbacks. Diversity proposes — the oracle disposes.
|
|
98
|
+
*/
|
|
99
|
+
export function pickCandidateModels(routing: RoutingLike, n: number): string[] {
|
|
100
|
+
const routes = routing.routes ?? {};
|
|
101
|
+
const ordered = [
|
|
102
|
+
routes.code?.preferredModel,
|
|
103
|
+
routes.review?.preferredModel,
|
|
104
|
+
routes.test?.preferredModel,
|
|
105
|
+
routes.explore?.preferredModel,
|
|
106
|
+
routes.code?.fallback,
|
|
107
|
+
routes.review?.fallback,
|
|
108
|
+
routes.test?.fallback,
|
|
109
|
+
routing.default?.model,
|
|
110
|
+
];
|
|
111
|
+
const out: string[] = [];
|
|
112
|
+
for (const ref of ordered) {
|
|
113
|
+
if (typeof ref === "string" && ref.trim() && !out.includes(ref)) out.push(ref);
|
|
114
|
+
if (out.length >= n) break;
|
|
115
|
+
}
|
|
116
|
+
return out.slice(0, Math.max(1, n));
|
|
117
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Run telemetry — one JSONL line per orchestration run (.phi/runs.jsonl), so
|
|
3
|
+
* "how do /fix //debug //build actually behave over time?" is answered by
|
|
4
|
+
* appending to a file on every run instead of mounting a measurement campaign.
|
|
5
|
+
* Pure record building here; the single append call-site lives in the driver.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { appendFileSync, mkdirSync } from "node:fs";
|
|
9
|
+
import { dirname, join } from "node:path";
|
|
10
|
+
|
|
11
|
+
export interface PhaseRecord {
|
|
12
|
+
key: string;
|
|
13
|
+
label: string;
|
|
14
|
+
model: string;
|
|
15
|
+
verdict: string | null;
|
|
16
|
+
retried: boolean;
|
|
17
|
+
blockedRetried: boolean;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface RunRecord {
|
|
21
|
+
mode: string;
|
|
22
|
+
startedAt: string;
|
|
23
|
+
durationMs: number;
|
|
24
|
+
phases: PhaseRecord[];
|
|
25
|
+
completedPhases: number;
|
|
26
|
+
skippedPhases: number;
|
|
27
|
+
sandboxExecs: number;
|
|
28
|
+
outcome: string;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Build the run record (pure — timestamps/durations are supplied). */
|
|
32
|
+
export function buildRunRecord(input: {
|
|
33
|
+
mode: string;
|
|
34
|
+
startedAtMs: number | null;
|
|
35
|
+
endedAtMs: number;
|
|
36
|
+
phases: PhaseRecord[];
|
|
37
|
+
completedPhases: number;
|
|
38
|
+
skippedPhases: number;
|
|
39
|
+
sandboxExecs: number;
|
|
40
|
+
outcome: string;
|
|
41
|
+
}): RunRecord {
|
|
42
|
+
const started = input.startedAtMs ?? input.endedAtMs;
|
|
43
|
+
return {
|
|
44
|
+
mode: input.mode,
|
|
45
|
+
startedAt: new Date(started).toISOString(),
|
|
46
|
+
durationMs: Math.max(0, input.endedAtMs - started),
|
|
47
|
+
phases: input.phases,
|
|
48
|
+
completedPhases: input.completedPhases,
|
|
49
|
+
skippedPhases: input.skippedPhases,
|
|
50
|
+
sandboxExecs: input.sandboxExecs,
|
|
51
|
+
// Strip markdown noise; keep the headline single-line and bounded.
|
|
52
|
+
outcome: input.outcome
|
|
53
|
+
.replace(/\*\*|`/g, "")
|
|
54
|
+
.replace(/\s+/g, " ")
|
|
55
|
+
.trim()
|
|
56
|
+
.slice(0, 300),
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Serialize for JSONL (single line, newline-terminated). */
|
|
61
|
+
export function toJsonlLine(record: RunRecord): string {
|
|
62
|
+
return `${JSON.stringify(record)}\n`;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Append a run record to <cwd>/.phi/runs.jsonl. Best-effort — never throws. */
|
|
66
|
+
export function appendRunRecord(cwd: string, record: RunRecord): boolean {
|
|
67
|
+
try {
|
|
68
|
+
const file = join(cwd, ".phi", "runs.jsonl");
|
|
69
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
70
|
+
appendFileSync(file, toJsonlLine(record), "utf-8");
|
|
71
|
+
return true;
|
|
72
|
+
} catch {
|
|
73
|
+
return false;
|
|
74
|
+
}
|
|
75
|
+
}
|