@robmclarty/volley 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +248 -21
- package/dist/cli.js +3273 -1646
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +120 -11
- package/dist/index.js +1210 -394
- package/dist/index.js.map +1 -1
- package/package.json +3 -1
- package/src/critic/presets/reviewer.md +7 -0
- package/src/tester/presets/tester.md +29 -0
package/dist/index.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import { accessSync, constants, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs";
|
|
1
|
+
import { accessSync, constants, existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { basename, dirname, isAbsolute, join, posix, relative, resolve, sep } from "node:path";
|
|
3
3
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
4
|
-
import { createHmac, randomBytes, randomUUID } from "node:crypto";
|
|
4
|
+
import { createHash, createHmac, randomBytes, randomUUID } from "node:crypto";
|
|
5
5
|
import { spawn, spawnSync } from "node:child_process";
|
|
6
6
|
import { aborted_error, branch, claude_cli_reported, create_engine, fallback, loop, model_call, pipe, retry, run, sequence, step, throughput } from "fascicle";
|
|
7
|
+
import { z } from "zod";
|
|
7
8
|
import { homedir } from "node:os";
|
|
8
9
|
import { filesystem_logger, tee_logger } from "fascicle/adapters";
|
|
9
10
|
import { lookup } from "node:dns";
|
|
@@ -12,7 +13,6 @@ import ipaddr from "ipaddr.js";
|
|
|
12
13
|
import { parseHTML } from "linkedom";
|
|
13
14
|
import TurndownService from "turndown";
|
|
14
15
|
import { Agent, buildConnector, interceptors, request } from "undici";
|
|
15
|
-
import { z } from "zod";
|
|
16
16
|
import pc from "picocolors";
|
|
17
17
|
//#region src/types.ts
|
|
18
18
|
function config_error(message) {
|
|
@@ -530,6 +530,68 @@ function collect_changes(options) {
|
|
|
530
530
|
truncated: sorted.length > limit
|
|
531
531
|
};
|
|
532
532
|
}
|
|
533
|
+
/** A file's content hash under `root`, or null when it can't be read (gone,
|
|
534
|
+
* or a directory). */
|
|
535
|
+
function hash_file(root, path) {
|
|
536
|
+
try {
|
|
537
|
+
return createHash("sha256").update(readFileSync(join(root, path))).digest("hex");
|
|
538
|
+
} catch {
|
|
539
|
+
return null;
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
/** Every changed path in `root` since the baseline, by its content hash: what
|
|
543
|
+
* the build root holds before the tester writes, so what it writes can be told
|
|
544
|
+
* from a workspace that was dirty to begin with. Empty outside a git
|
|
545
|
+
* repository, where nothing is measured. */
|
|
546
|
+
function snapshot_changes(options) {
|
|
547
|
+
const changes = collect_changes({
|
|
548
|
+
...options,
|
|
549
|
+
limit: Number.POSITIVE_INFINITY
|
|
550
|
+
});
|
|
551
|
+
return new Map((changes?.files ?? []).map((file) => [file.path, hash_file(options.root, file.path)]));
|
|
552
|
+
}
|
|
553
|
+
/** What the tester wrote: the changes since the baseline whose content isn't
|
|
554
|
+
* what the snapshot taken before it held, as a change set of their own, and
|
|
555
|
+
* each written file's hash, which a later build is measured against (move 1's
|
|
556
|
+
* D9, D22). Null and none outside a git repository. */
|
|
557
|
+
function tester_written(options, before) {
|
|
558
|
+
const after = collect_changes({
|
|
559
|
+
...options,
|
|
560
|
+
limit: Number.POSITIVE_INFINITY
|
|
561
|
+
});
|
|
562
|
+
if (after === null) return {
|
|
563
|
+
changes: null,
|
|
564
|
+
hashes: {}
|
|
565
|
+
};
|
|
566
|
+
const hashes = {};
|
|
567
|
+
const files = after.files.filter((file) => {
|
|
568
|
+
const hash = hash_file(options.root, file.path);
|
|
569
|
+
if (before.has(file.path) && before.get(file.path) === hash) return false;
|
|
570
|
+
if (hash !== null) hashes[file.path] = hash;
|
|
571
|
+
return true;
|
|
572
|
+
});
|
|
573
|
+
return {
|
|
574
|
+
changes: {
|
|
575
|
+
baseline: after.baseline,
|
|
576
|
+
gate_edits: files.filter((file) => file.gate).map((file) => file.path),
|
|
577
|
+
total: files.length,
|
|
578
|
+
files,
|
|
579
|
+
truncated: false
|
|
580
|
+
},
|
|
581
|
+
hashes
|
|
582
|
+
};
|
|
583
|
+
}
|
|
584
|
+
/** `changes` with the tester's files taken out of its gate edits, unless the
|
|
585
|
+
* build changed one from the hash it had after the tester: the tester editing
|
|
586
|
+
* the gate is the point, and a builder that left its test alone didn't (D9).
|
|
587
|
+
* A file the build deleted no longer hashes as it did, so it stays an edit. */
|
|
588
|
+
function without_tester_edits(changes, root, hashes) {
|
|
589
|
+
if (changes === null || Object.keys(hashes).length === 0) return changes;
|
|
590
|
+
return {
|
|
591
|
+
...changes,
|
|
592
|
+
gate_edits: changes.gate_edits.filter((path) => hashes[path] === void 0 || hash_file(root, path) !== hashes[path])
|
|
593
|
+
};
|
|
594
|
+
}
|
|
533
595
|
//#endregion
|
|
534
596
|
//#region src/check/detect.ts
|
|
535
597
|
/**
|
|
@@ -575,6 +637,339 @@ function detect_containment(env) {
|
|
|
575
637
|
return value === "1" || value === "true";
|
|
576
638
|
}
|
|
577
639
|
//#endregion
|
|
640
|
+
//#region src/workspace_tools.ts
|
|
641
|
+
/**
|
|
642
|
+
* Shared workspace tools: `contain()` path confinement, the read-only tool set
|
|
643
|
+
* (`read_file`, `search_files`, `list_files`), and the two file writers
|
|
644
|
+
* (`write_file`, `edit_file`).
|
|
645
|
+
*
|
|
646
|
+
* The claude_cli builder/critic get file access from the CLI's own built-in
|
|
647
|
+
* tools. API-style providers (ollama, lmstudio, …) have no built-ins, so
|
|
648
|
+
* volley supplies its own. The critic uses the read-only set as-is via
|
|
649
|
+
* `src/critic/tools.ts`; the builder set in `src/builder/tools.ts` adds the
|
|
650
|
+
* writers and its exec surface; the tester's (`src/tester/tools.ts`) adds the
|
|
651
|
+
* writers held to the test and fixture paths, which is why they live here
|
|
652
|
+
* rather than in the builder's module. Every path is resolved and confined to
|
|
653
|
+
* the workspace root.
|
|
654
|
+
*/
|
|
655
|
+
const READ_FILE_MAX_BYTES = 2e5;
|
|
656
|
+
const WRITE_FILE_MAX_BYTES = 1e6;
|
|
657
|
+
const LIST_MAX_ENTRIES = 2e3;
|
|
658
|
+
/** Directories never worth handing a seat; they bury signal and blow the
|
|
659
|
+
* traversal budget. The acp seats' `list_files` and `search` skip the same
|
|
660
|
+
* names (`./acp_tools.ts`), so a seat sees one tree under acp and on the CLI. */
|
|
661
|
+
const IGNORED_DIRS = /* @__PURE__ */ new Set([
|
|
662
|
+
"node_modules",
|
|
663
|
+
".git",
|
|
664
|
+
".volley",
|
|
665
|
+
".check",
|
|
666
|
+
"dist",
|
|
667
|
+
".cache"
|
|
668
|
+
]);
|
|
669
|
+
/** Resolve `rel` inside `workspace`, rejecting absolute paths and any `..`
|
|
670
|
+
* escape. Returns the absolute path; throws on containment violation. The
|
|
671
|
+
* `workspace` root passed here is the build root, which is the per-run git
|
|
672
|
+
* worktree under `--worktree` (`build_root`), not necessarily the
|
|
673
|
+
* workspace the run was started on. */
|
|
674
|
+
function contain(workspace, rel) {
|
|
675
|
+
const root = resolve(workspace);
|
|
676
|
+
const candidate = isAbsolute(rel) ? resolve(rel) : resolve(root, rel);
|
|
677
|
+
if (!(candidate === root || candidate.startsWith(`${root}${sep}`))) throw new Error(`path escapes the workspace: ${rel}`);
|
|
678
|
+
return candidate;
|
|
679
|
+
}
|
|
680
|
+
function walk(root, start, limit) {
|
|
681
|
+
const out = [];
|
|
682
|
+
const stack = [start];
|
|
683
|
+
while (stack.length > 0 && out.length < limit) {
|
|
684
|
+
const dir = stack.pop();
|
|
685
|
+
let entries;
|
|
686
|
+
try {
|
|
687
|
+
entries = readdirSync(dir);
|
|
688
|
+
} catch {
|
|
689
|
+
continue;
|
|
690
|
+
}
|
|
691
|
+
for (const name of entries) {
|
|
692
|
+
const abs = join(dir, name);
|
|
693
|
+
let is_dir;
|
|
694
|
+
try {
|
|
695
|
+
is_dir = statSync(abs).isDirectory();
|
|
696
|
+
} catch {
|
|
697
|
+
continue;
|
|
698
|
+
}
|
|
699
|
+
if (is_dir) {
|
|
700
|
+
if (!IGNORED_DIRS.has(name)) stack.push(abs);
|
|
701
|
+
continue;
|
|
702
|
+
}
|
|
703
|
+
out.push({
|
|
704
|
+
rel: relative(root, abs),
|
|
705
|
+
abs
|
|
706
|
+
});
|
|
707
|
+
if (out.length >= limit) break;
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
return out;
|
|
711
|
+
}
|
|
712
|
+
/** The files a read tool's optional `path` covers: the whole tree, the tree
|
|
713
|
+
* under a directory, or the one file it names. A path that doesn't exist
|
|
714
|
+
* throws, so the seat reads why rather than an empty result that looks like
|
|
715
|
+
* "nothing matched". */
|
|
716
|
+
function scoped_files(workspace, path, limit) {
|
|
717
|
+
const root = resolve(workspace);
|
|
718
|
+
if (path === void 0) return walk(root, root, limit);
|
|
719
|
+
const start = contain(workspace, path);
|
|
720
|
+
let is_dir;
|
|
721
|
+
try {
|
|
722
|
+
is_dir = statSync(start).isDirectory();
|
|
723
|
+
} catch (err) {
|
|
724
|
+
throw new Error(`no such file or directory: ${path}`, { cause: err });
|
|
725
|
+
}
|
|
726
|
+
return is_dir ? walk(root, start, limit) : [{
|
|
727
|
+
rel: relative(root, start),
|
|
728
|
+
abs: start
|
|
729
|
+
}];
|
|
730
|
+
}
|
|
731
|
+
function build_regex(pattern, flags) {
|
|
732
|
+
try {
|
|
733
|
+
return new RegExp(pattern, flags);
|
|
734
|
+
} catch (err) {
|
|
735
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
736
|
+
throw new Error(`invalid regular expression: ${detail}`);
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
const read_file_input = z.object({ path: z.string().describe("Workspace-relative path to the file to read.") });
|
|
740
|
+
const search_files_input = z.object({
|
|
741
|
+
pattern: z.string().describe("JavaScript regular expression to match against each line."),
|
|
742
|
+
path: z.string().optional().describe("Optional workspace-relative directory or file to limit the search to."),
|
|
743
|
+
ignore_case: z.boolean().optional().describe("Case-insensitive match. Default false.")
|
|
744
|
+
});
|
|
745
|
+
const list_files_input = z.object({
|
|
746
|
+
path: z.string().optional().describe("Optional workspace-relative directory or file. Default: the workspace root."),
|
|
747
|
+
contains: z.string().optional().describe("Optional substring filter applied to each relative path.")
|
|
748
|
+
});
|
|
749
|
+
function read_only_tools(workspace) {
|
|
750
|
+
return [
|
|
751
|
+
{
|
|
752
|
+
name: "read_file",
|
|
753
|
+
read_only: true,
|
|
754
|
+
description: `Read a UTF-8 text file from the workspace. Returns the file content (truncated at ${String(READ_FILE_MAX_BYTES)} bytes).`,
|
|
755
|
+
input_schema: read_file_input,
|
|
756
|
+
execute: (raw) => {
|
|
757
|
+
const abs = contain(workspace, read_file_input.parse(raw).path);
|
|
758
|
+
const content = readFileSync(abs, "utf8");
|
|
759
|
+
return content.length > 2e5 ? `${content.slice(0, READ_FILE_MAX_BYTES)}\n… [truncated at ${String(READ_FILE_MAX_BYTES)} bytes]` : content;
|
|
760
|
+
}
|
|
761
|
+
},
|
|
762
|
+
{
|
|
763
|
+
name: "search_files",
|
|
764
|
+
read_only: true,
|
|
765
|
+
description: `Search workspace files for lines matching a regular expression. Returns up to ${String(200)} "relative/path:line: text" hits.`,
|
|
766
|
+
input_schema: search_files_input,
|
|
767
|
+
execute: (raw) => {
|
|
768
|
+
const input = search_files_input.parse(raw);
|
|
769
|
+
const regex = build_regex(input.pattern, input.ignore_case === true ? "i" : "");
|
|
770
|
+
const files = scoped_files(workspace, input.path, LIST_MAX_ENTRIES);
|
|
771
|
+
const hits = [];
|
|
772
|
+
for (const file of files) {
|
|
773
|
+
if (hits.length >= 200) break;
|
|
774
|
+
let content;
|
|
775
|
+
try {
|
|
776
|
+
content = readFileSync(file.abs, "utf8");
|
|
777
|
+
} catch {
|
|
778
|
+
continue;
|
|
779
|
+
}
|
|
780
|
+
const lines = content.split("\n");
|
|
781
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
782
|
+
if (hits.length >= 200) break;
|
|
783
|
+
const line = lines[i];
|
|
784
|
+
if (regex.test(line)) hits.push(`${file.rel}:${String(i + 1)}: ${line.trim()}`);
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
if (hits.length === 0) return "no matches";
|
|
788
|
+
const capped = hits.length >= 200 ? `\n… [capped at ${String(200)} matches]` : "";
|
|
789
|
+
return `${hits.join("\n")}${capped}`;
|
|
790
|
+
}
|
|
791
|
+
},
|
|
792
|
+
{
|
|
793
|
+
name: "list_files",
|
|
794
|
+
read_only: true,
|
|
795
|
+
description: "List files under a workspace directory (recursively; skips node_modules, .git, and other build/output directories).",
|
|
796
|
+
input_schema: list_files_input,
|
|
797
|
+
execute: (raw) => {
|
|
798
|
+
const input = list_files_input.parse(raw);
|
|
799
|
+
let files = scoped_files(workspace, input.path, LIST_MAX_ENTRIES).map((f) => f.rel);
|
|
800
|
+
if (input.contains !== void 0) {
|
|
801
|
+
const needle = input.contains;
|
|
802
|
+
files = files.filter((f) => f.includes(needle));
|
|
803
|
+
}
|
|
804
|
+
if (files.length === 0) return "no files";
|
|
805
|
+
const capped = files.length >= LIST_MAX_ENTRIES ? `\n… [capped at ${String(LIST_MAX_ENTRIES)} entries]` : "";
|
|
806
|
+
return `${files.join("\n")}${capped}`;
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
];
|
|
810
|
+
}
|
|
811
|
+
const write_file_input = z.object({
|
|
812
|
+
path: z.string().describe("Workspace-relative path to write. Parent directories are created."),
|
|
813
|
+
content: z.string().describe("Full UTF-8 content of the file. Overwrites any existing content.")
|
|
814
|
+
});
|
|
815
|
+
const edit_file_input = z.object({
|
|
816
|
+
path: z.string().describe("Workspace-relative path of the file to edit."),
|
|
817
|
+
old_str: z.string().min(1).describe("Exact text to replace. Must occur exactly once in the file."),
|
|
818
|
+
new_str: z.string().describe("Replacement text.")
|
|
819
|
+
});
|
|
820
|
+
function count_occurrences(haystack, needle) {
|
|
821
|
+
return haystack.split(needle).length - 1;
|
|
822
|
+
}
|
|
823
|
+
/** `write_file` and `edit_file`, confined to the workspace, and to whatever
|
|
824
|
+
* `guard` allows besides. A wrong-but-recoverable edit (0 or N matches) is
|
|
825
|
+
* returned for the model to read; a refused path or an over-cap write throws,
|
|
826
|
+
* as a containment violation does. */
|
|
827
|
+
function write_tools(workspace, guard) {
|
|
828
|
+
const target = (path) => {
|
|
829
|
+
const abs = contain(workspace, path);
|
|
830
|
+
guard?.(abs, path);
|
|
831
|
+
return abs;
|
|
832
|
+
};
|
|
833
|
+
return [{
|
|
834
|
+
name: "write_file",
|
|
835
|
+
description: `Write a UTF-8 text file in the workspace, creating parent directories and overwriting any existing content. Rejects content over ${String(WRITE_FILE_MAX_BYTES)} bytes.`,
|
|
836
|
+
input_schema: write_file_input,
|
|
837
|
+
execute: (raw) => {
|
|
838
|
+
const input = write_file_input.parse(raw);
|
|
839
|
+
const abs = target(input.path);
|
|
840
|
+
const bytes = Buffer.byteLength(input.content, "utf8");
|
|
841
|
+
if (bytes > 1e6) throw new Error(`content is ${String(bytes)} bytes, over the ${String(WRITE_FILE_MAX_BYTES)}-byte write cap`);
|
|
842
|
+
mkdirSync(dirname(abs), { recursive: true });
|
|
843
|
+
writeFileSync(abs, input.content, "utf8");
|
|
844
|
+
return `wrote ${String(bytes)} bytes to ${input.path}`;
|
|
845
|
+
}
|
|
846
|
+
}, {
|
|
847
|
+
name: "edit_file",
|
|
848
|
+
description: "Replace one exact occurrence of old_str with new_str in a workspace file. old_str must match the file content exactly once — on 0 or multiple matches nothing is changed and an error is returned; read the file and retry with a longer, unique old_str.",
|
|
849
|
+
input_schema: edit_file_input,
|
|
850
|
+
execute: (raw) => {
|
|
851
|
+
const input = edit_file_input.parse(raw);
|
|
852
|
+
const abs = target(input.path);
|
|
853
|
+
const content = readFileSync(abs, "utf8");
|
|
854
|
+
const matches = count_occurrences(content, input.old_str);
|
|
855
|
+
if (matches === 0) return `error: old_str not found in ${input.path} (0 matches). No change made. Read the file and retry with an exact excerpt of its current content.`;
|
|
856
|
+
if (matches > 1) return `error: old_str matches ${String(matches)} times in ${input.path}. No change made. Include more surrounding context to make old_str unique.`;
|
|
857
|
+
writeFileSync(abs, content.replace(input.old_str, input.new_str), "utf8");
|
|
858
|
+
return `edit applied to ${input.path}`;
|
|
859
|
+
}
|
|
860
|
+
}];
|
|
861
|
+
}
|
|
862
|
+
/** A `paths + sizes` listing of the workspace tree, no file contents — the
|
|
863
|
+
* grounding a tool-less critic gets when its provider's tool parser died and the
|
|
864
|
+
* degradation ladder traded read access for survival. Reuses the same
|
|
865
|
+
* `walk` + `IGNORED_DIRS` traversal as the read tools, so the inventory matches
|
|
866
|
+
* what a working `list_files` would have surfaced; caps at `limit` entries. */
|
|
867
|
+
function workspace_inventory(root, limit = LIST_MAX_ENTRIES) {
|
|
868
|
+
const base = resolve(root);
|
|
869
|
+
const files = walk(base, base, limit);
|
|
870
|
+
if (files.length === 0) return "no files";
|
|
871
|
+
const lines = files.map((file) => {
|
|
872
|
+
let size;
|
|
873
|
+
try {
|
|
874
|
+
size = statSync(file.abs).size;
|
|
875
|
+
} catch {
|
|
876
|
+
size = 0;
|
|
877
|
+
}
|
|
878
|
+
return `${file.rel} (${String(size)} bytes)`;
|
|
879
|
+
});
|
|
880
|
+
const capped = files.length >= limit ? `\n… [capped at ${String(limit)} entries]` : "";
|
|
881
|
+
return `${lines.join("\n")}${capped}`;
|
|
882
|
+
}
|
|
883
|
+
//#endregion
|
|
884
|
+
//#region src/tester/tools.ts
|
|
885
|
+
/**
|
|
886
|
+
* The tester's tools on any provider but `claude_cli`: the shared reads, and
|
|
887
|
+
* `write_file` and `edit_file` held to the test and fixture gate paths.
|
|
888
|
+
*
|
|
889
|
+
* The tester writes a failing test from the criteria before the first build,
|
|
890
|
+
* and its whole job is the test (move 1's D5). So a write must match a gate
|
|
891
|
+
* pattern, and must not touch the gate's other half: `package.json`, a
|
|
892
|
+
* `*.config.*` file, or anything under `.github/`. A tester that could rewrite
|
|
893
|
+
* the check's configuration could make its own result red or green (D20). It
|
|
894
|
+
* has no `bash` either, though the check runs what it writes, which is why a
|
|
895
|
+
* tester on a local model is contained as a builder is (D14).
|
|
896
|
+
*
|
|
897
|
+
* A path that reaches through a symbolic link is refused too: `contain()` is
|
|
898
|
+
* lexical, so `test/` linked to `src/` would otherwise let a write the
|
|
899
|
+
* patterns allow land in the source.
|
|
900
|
+
*/
|
|
901
|
+
/** Why the tester may not write `rel` (a `/`-separated workspace path) though
|
|
902
|
+
* a gate pattern matches it, or null when it may. */
|
|
903
|
+
function excluded_reason(rel) {
|
|
904
|
+
const segments = rel.split("/");
|
|
905
|
+
const name = segments.at(-1) ?? "";
|
|
906
|
+
if (name === "package.json") return "package.json decides which scripts and tests run";
|
|
907
|
+
if (name.includes(".config.")) return "a *.config.* file configures the check";
|
|
908
|
+
if (segments.includes(".github")) return "a path under .github/ is CI configuration";
|
|
909
|
+
return null;
|
|
910
|
+
}
|
|
911
|
+
/** Whether a gate pattern can only ever name files the tester may not write:
|
|
912
|
+
* `**\/package.json`, `**\/vitest.config.*`, `.github/workflows/**`. */
|
|
913
|
+
function names_only_excluded(pattern) {
|
|
914
|
+
return excluded_reason(pattern.replace(/\*+$/, "x")) !== null;
|
|
915
|
+
}
|
|
916
|
+
/** The gate patterns the tester may write under: the gate's tests and
|
|
917
|
+
* fixtures, its check configuration and manifest left out. Its prompt names
|
|
918
|
+
* these, and a `claude_cli` tester's `Edit` rules are made from them. */
|
|
919
|
+
function writable_gate_patterns(gate_paths) {
|
|
920
|
+
return gate_paths.filter((pattern) => !names_only_excluded(pattern));
|
|
921
|
+
}
|
|
922
|
+
/** Whether `path` exists as itself, a dangling link included. */
|
|
923
|
+
function lexists(path) {
|
|
924
|
+
try {
|
|
925
|
+
lstatSync(path);
|
|
926
|
+
return true;
|
|
927
|
+
} catch {
|
|
928
|
+
return false;
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
/** Whether `abs` (inside `root`) resolves somewhere other than where it reads:
|
|
932
|
+
* its nearest existing ancestor's real path isn't that ancestor's place under
|
|
933
|
+
* the root's real path, or can't be read (a dangling link). */
|
|
934
|
+
function resolves_elsewhere(root, abs) {
|
|
935
|
+
let probe = abs;
|
|
936
|
+
while (!lexists(probe) && probe !== root) probe = dirname(probe);
|
|
937
|
+
try {
|
|
938
|
+
return realpathSync(probe) !== join(realpathSync(root), relative(root, probe));
|
|
939
|
+
} catch {
|
|
940
|
+
return true;
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
/** Why the tester may not write `abs` (named `path` by the model) in
|
|
944
|
+
* `workspace`, or null when it may. */
|
|
945
|
+
function tester_write_refusal(workspace, gate_paths, abs) {
|
|
946
|
+
const root = resolve(workspace);
|
|
947
|
+
const rel = relative(root, abs).split(sep).join("/");
|
|
948
|
+
if (rel === "" || !matches_any(rel, [...gate_paths])) {
|
|
949
|
+
const writable = writable_gate_patterns(gate_paths);
|
|
950
|
+
return writable.length === 0 ? "no gate pattern names a test or fixture path" : `it matches none of the test and fixture gate patterns (${writable.join(", ")})`;
|
|
951
|
+
}
|
|
952
|
+
const excluded = excluded_reason(rel);
|
|
953
|
+
if (excluded !== null) return excluded;
|
|
954
|
+
if (resolves_elsewhere(root, abs)) return "it reaches through a symbolic link";
|
|
955
|
+
return null;
|
|
956
|
+
}
|
|
957
|
+
/** The error a tester's write to `path` (named `path`, contained at `abs`)
|
|
958
|
+
* is refused with, or null when it may write there. */
|
|
959
|
+
function refusal_message(workspace, gate_paths, abs, path) {
|
|
960
|
+
const refusal = tester_write_refusal(workspace, gate_paths, abs);
|
|
961
|
+
return refusal === null ? null : `the tester can't write ${path}: ${refusal}. It writes tests and fixtures only.`;
|
|
962
|
+
}
|
|
963
|
+
/** The tester's tool set: the read trio, then `write_file` and `edit_file`
|
|
964
|
+
* confined to the test and fixture gate paths. A refused write throws, naming
|
|
965
|
+
* the path and why; fascicle feeds that back to the model. */
|
|
966
|
+
function tester_tools(workspace, gate_paths) {
|
|
967
|
+
return [...read_only_tools(workspace), ...write_tools(workspace, (abs, path) => {
|
|
968
|
+
const refusal = refusal_message(workspace, gate_paths, abs, path);
|
|
969
|
+
if (refusal !== null) throw new Error(refusal);
|
|
970
|
+
})];
|
|
971
|
+
}
|
|
972
|
+
//#endregion
|
|
578
973
|
//#region src/trust.ts
|
|
579
974
|
/**
|
|
580
975
|
* Which run configs a host-side resume may trust. A resume runs what a run's
|
|
@@ -737,9 +1132,13 @@ function write_resolved_config(config, env = process.env) {
|
|
|
737
1132
|
builder_max_tool_calls: config.builder_max_tool_calls,
|
|
738
1133
|
builder_max_output_tokens: config.builder_max_output_tokens,
|
|
739
1134
|
builder_trim_history: config.builder_trim_history,
|
|
1135
|
+
effort_ladder: config.effort_ladder,
|
|
740
1136
|
critic_model: config.critic_model,
|
|
741
1137
|
critic_provider: config.critic_provider,
|
|
742
1138
|
critic_url: config.critic_url,
|
|
1139
|
+
critic_effort: config.critic_effort,
|
|
1140
|
+
tester: config.tester,
|
|
1141
|
+
tester_model: config.tester_model,
|
|
743
1142
|
builder_permission_mode: config.builder_permission_mode,
|
|
744
1143
|
critic_preset: config.critic_preset,
|
|
745
1144
|
critic_prompt_path: config.critic_prompt_path,
|
|
@@ -905,6 +1304,25 @@ const CRITIC_PRESETS = [
|
|
|
905
1304
|
"researcher"
|
|
906
1305
|
];
|
|
907
1306
|
const PERMISSION_MODES = ["acceptEdits", "bypassPermissions"];
|
|
1307
|
+
/** The efforts a rung or the critic may take, lowest first. */
|
|
1308
|
+
const EFFORT_RUNGS = [
|
|
1309
|
+
"low",
|
|
1310
|
+
"medium",
|
|
1311
|
+
"high",
|
|
1312
|
+
"xhigh",
|
|
1313
|
+
"max"
|
|
1314
|
+
];
|
|
1315
|
+
/** The critic's effort when a ladder is set and nothing names one: reading a
|
|
1316
|
+
* diff costs far less than writing it, and the critic is what makes a cheap
|
|
1317
|
+
* build safe. */
|
|
1318
|
+
const DEFAULT_LADDER_CRITIC_EFFORT = "high";
|
|
1319
|
+
/** Builder providers whose fascicle transport drops `effort` and records
|
|
1320
|
+
* `effort_ignored`, so every rung of a ladder runs the same there. */
|
|
1321
|
+
const EFFORT_IGNORED_PROVIDERS = ["ollama", "lmstudio"];
|
|
1322
|
+
/** Whether a builder on `provider` ignores the effort a rung sets. */
|
|
1323
|
+
function ignores_effort(provider) {
|
|
1324
|
+
return EFFORT_IGNORED_PROVIDERS.includes(provider);
|
|
1325
|
+
}
|
|
908
1326
|
/** `@path/to/file` reads the file; any other string passes through. */
|
|
909
1327
|
function expand_at_file(value, cwd) {
|
|
910
1328
|
if (!value.startsWith("@")) return value;
|
|
@@ -962,6 +1380,9 @@ const CRITIQUE_REFUSED = [
|
|
|
962
1380
|
["builder_max_tool_calls", "--builder-max-tool-calls"],
|
|
963
1381
|
["builder_max_output_tokens", "--builder-max-output-tokens"],
|
|
964
1382
|
["builder_trim_history", "--builder-trim-history"],
|
|
1383
|
+
["effort_ladder", "--effort-ladder"],
|
|
1384
|
+
["tester", "--tester"],
|
|
1385
|
+
["tester_model", "--tester-model"],
|
|
965
1386
|
["builder_permission_mode", "--builder-permission-mode"],
|
|
966
1387
|
["allow_unsandboxed_builder", "--allow-unsandboxed-builder"],
|
|
967
1388
|
["sandbox_image", "--sandbox-image"],
|
|
@@ -973,7 +1394,7 @@ const CRITIQUE_REFUSED = [
|
|
|
973
1394
|
["fail_on_gate_edit", "--fail-on-gate-edit"]
|
|
974
1395
|
];
|
|
975
1396
|
function refuse_builder_settings(raw) {
|
|
976
|
-
for (const [field, flag] of CRITIQUE_REFUSED) if (raw[field] !== void 0 && raw[field] !== false) throw config_error(`volley critique refuses ${flag} (${field}): a critique runs no builder and no loop`);
|
|
1397
|
+
for (const [field, flag] of CRITIQUE_REFUSED) if (raw[field] !== void 0 && raw[field] !== false && raw[field] !== "off") throw config_error(`volley critique refuses ${flag} (${field}): a critique runs no builder and no loop`);
|
|
977
1398
|
}
|
|
978
1399
|
const BUILDER_MAX_STEPS = {
|
|
979
1400
|
flag: "--builder-max-steps",
|
|
@@ -1020,6 +1441,84 @@ function positive_count(value, original, count) {
|
|
|
1020
1441
|
if (!Number.isInteger(value) || value < 1) throw config_error(`${count.flag} / ${count.env_var} must be a positive integer; got: ${String(original)}`);
|
|
1021
1442
|
return value;
|
|
1022
1443
|
}
|
|
1444
|
+
const EFFORT_LADDER = {
|
|
1445
|
+
flag: "--effort-ladder",
|
|
1446
|
+
field: "effort_ladder",
|
|
1447
|
+
env_var: "VOLLEY_EFFORT_LADDER"
|
|
1448
|
+
};
|
|
1449
|
+
const CRITIC_EFFORT = {
|
|
1450
|
+
flag: "--critic-effort",
|
|
1451
|
+
field: "critic_effort",
|
|
1452
|
+
env_var: "VOLLEY_CRITIC_EFFORT"
|
|
1453
|
+
};
|
|
1454
|
+
/** One effort, held to the five from `low` to `max`. `none` gets its own
|
|
1455
|
+
* reason, since it's fascicle's level and the likeliest to be tried. */
|
|
1456
|
+
function effort_rung(value, setting) {
|
|
1457
|
+
const rung = typeof value === "string" ? value.trim() : value;
|
|
1458
|
+
if (EFFORT_RUNGS.includes(rung)) return rung;
|
|
1459
|
+
const why = rung === "none" ? " (none sets no effort, and on claude_cli leaves Claude Code at its own default)" : "";
|
|
1460
|
+
throw config_error(`${setting.flag} (${setting.field}) / ${setting.env_var} takes ${EFFORT_RUNGS.join(", ")}; got: ${String(value)}${why}`);
|
|
1461
|
+
}
|
|
1462
|
+
/** `--effort-ladder` (`effort_ladder`), as a list or the flag's comma-separated
|
|
1463
|
+
* form, wins over `VOLLEY_EFFORT_LADDER`; an empty variable counts as unset.
|
|
1464
|
+
* Unset is null, which passes no effort. A value that names no rung is refused
|
|
1465
|
+
* rather than read as unset, since it was asked for. */
|
|
1466
|
+
function resolve_effort_ladder(value, env) {
|
|
1467
|
+
const env_value = env[EFFORT_LADDER.env_var];
|
|
1468
|
+
const given = value ?? (env_value === "" ? void 0 : env_value);
|
|
1469
|
+
if (given === void 0 || given === null) return null;
|
|
1470
|
+
const items = typeof given === "string" ? given.split(",") : given;
|
|
1471
|
+
if (!Array.isArray(items)) throw config_error(`${EFFORT_LADDER.flag} (${EFFORT_LADDER.field}) must be a list of rungs; got a ${typeof given}`);
|
|
1472
|
+
const named = items.filter((item) => typeof item !== "string" || item.trim() !== "");
|
|
1473
|
+
if (named.length === 0) throw config_error(`${EFFORT_LADDER.flag} (${EFFORT_LADDER.field}) / ${EFFORT_LADDER.env_var} names no rung; give one or more of ${EFFORT_RUNGS.join(", ")}, comma-separated`);
|
|
1474
|
+
return named.map((item) => effort_rung(item, EFFORT_LADDER));
|
|
1475
|
+
}
|
|
1476
|
+
/** `--critic-effort` (`critic_effort`) wins over `VOLLEY_CRITIC_EFFORT`; an
|
|
1477
|
+
* empty variable counts as unset. Unset is `high` when a ladder is set, else
|
|
1478
|
+
* null, so a run without a ladder sends the critic what it always has. */
|
|
1479
|
+
function resolve_critic_effort(value, env, ladder) {
|
|
1480
|
+
const env_value = env[CRITIC_EFFORT.env_var];
|
|
1481
|
+
const given = value ?? (env_value === "" ? void 0 : env_value);
|
|
1482
|
+
if (given !== void 0 && given !== null) return effort_rung(given, CRITIC_EFFORT);
|
|
1483
|
+
return ladder === null ? null : DEFAULT_LADDER_CRITIC_EFFORT;
|
|
1484
|
+
}
|
|
1485
|
+
/** `--tester` (`tester`) wins over `VOLLEY_TESTER`, and takes `on` or `off`;
|
|
1486
|
+
* an empty variable counts as unset, and unset is off (move 1's D8). */
|
|
1487
|
+
function resolve_tester(value, env) {
|
|
1488
|
+
const env_value = env["VOLLEY_TESTER"];
|
|
1489
|
+
const given = value ?? (env_value === "" ? void 0 : env_value);
|
|
1490
|
+
if (given === void 0 || given === null) return false;
|
|
1491
|
+
const setting = typeof given === "string" ? given.trim() : given;
|
|
1492
|
+
if (setting === "on" || setting === "off") return setting === "on";
|
|
1493
|
+
throw config_error(`--tester (tester) / VOLLEY_TESTER takes on or off; got: ${JSON.stringify(given)}`);
|
|
1494
|
+
}
|
|
1495
|
+
/** The tester's model with the tester on: `--tester-model` (`tester_model`),
|
|
1496
|
+
* else `VOLLEY_TESTER_MODEL`, else the critic's. With it off, null, and a
|
|
1497
|
+
* model named by flag or config is refused rather than ignored; the variable,
|
|
1498
|
+
* which may be set for every run, is left unread. */
|
|
1499
|
+
function resolve_tester_model(tester, value, env, critic_model) {
|
|
1500
|
+
const named = typeof value === "string" ? value.trim() : value;
|
|
1501
|
+
if (!tester) {
|
|
1502
|
+
if (named === void 0 || named === null || named === "") return null;
|
|
1503
|
+
throw config_error("--tester-model (tester_model) applies only with --tester on");
|
|
1504
|
+
}
|
|
1505
|
+
if (named !== void 0 && named !== null && named !== "") {
|
|
1506
|
+
if (typeof named !== "string") throw config_error(`--tester-model (tester_model) must be a model name; got a ${typeof named}`);
|
|
1507
|
+
return named;
|
|
1508
|
+
}
|
|
1509
|
+
const env_value = env["VOLLEY_TESTER_MODEL"]?.trim();
|
|
1510
|
+
return env_value === void 0 || env_value === "" ? critic_model : env_value;
|
|
1511
|
+
}
|
|
1512
|
+
/** Refuse a tester that can't do its job, or can't be contained, before any
|
|
1513
|
+
* spend (P1). It needs a check to go red (D18), a test or fixture gate pattern
|
|
1514
|
+
* to write under, and, on any provider but `claude_cli`, the containment a
|
|
1515
|
+
* builder needs, since the check runs what it writes (D14). Under acp, where
|
|
1516
|
+
* a person approves each edit, a local tester isn't refused here. */
|
|
1517
|
+
function refuse_unready_tester(check_resolved, gate_paths, critic_provider, guarded) {
|
|
1518
|
+
if (check_resolved === "none") throw config_error("--tester on needs a check for its test to fail: set --check to a command, or to auto in a workspace with checkride, rather than none");
|
|
1519
|
+
if (writable_gate_patterns(gate_paths).length === 0) throw config_error("--tester on needs a test or fixture gate pattern to write under; gate_paths names none");
|
|
1520
|
+
if (critic_provider !== "claude_cli" && !guarded) throw config_error(`a tester on '${critic_provider}' writes tests the check then runs, so it is contained as a builder is: launch volley inside its sandbox container (VOLLEY_CONTAINED=1), or pass --allow-unsandboxed-builder (VOLLEY_ALLOW_UNSANDBOXED_BUILDER=1). A claude_cli builder stays on the host, so with one, the opt-out is the only way.`);
|
|
1521
|
+
}
|
|
1023
1522
|
/** `--sandbox-image` (a merged flag/config value) wins over the
|
|
1024
1523
|
* `VOLLEY_SANDBOX_IMAGE` env var, which wins over the default image. Used only
|
|
1025
1524
|
* on the local-builder Docker sandbox path; the `claude_cli` path never
|
|
@@ -1146,6 +1645,8 @@ function resolve_config(raw, options = {}) {
|
|
|
1146
1645
|
if (!CRITIC_PROVIDERS.includes(critic_provider)) throw config_error(`--critic-provider must be one of ${CRITIC_PROVIDERS.join(", ")}; got: ${String(raw.critic_provider)}`);
|
|
1147
1646
|
const builder_url = resolve_seat_url("builder", raw.builder_url, builder_provider);
|
|
1148
1647
|
const critic_url = resolve_seat_url("critic", raw.critic_url, critic_provider);
|
|
1648
|
+
const effort_ladder = resolve_effort_ladder(raw.effort_ladder, builder_env);
|
|
1649
|
+
const critic_effort = resolve_critic_effort(raw.critic_effort, env, effort_ladder);
|
|
1149
1650
|
if (builder_provider === "openrouter" || critic_provider === "openrouter") openrouter_api_key(env);
|
|
1150
1651
|
if (contained) {
|
|
1151
1652
|
const claude_roles = [builder_provider === "claude_cli" && !critique ? "builder" : null, critic_provider === "claude_cli" ? "critic" : null].filter((role) => role !== null);
|
|
@@ -1159,12 +1660,17 @@ function resolve_config(raw, options = {}) {
|
|
|
1159
1660
|
if (discard_worktree && !worktree) throw config_error("--discard-worktree applies only to a --worktree run: there is no worktree to discard");
|
|
1160
1661
|
const gate_paths = raw.gate_paths ?? DEFAULT_GATE_PATTERNS;
|
|
1161
1662
|
if (!Array.isArray(gate_paths) || gate_paths.some((glob) => typeof glob !== "string")) throw config_error("gate_paths must be an array of glob strings");
|
|
1663
|
+
const check = raw.check ?? "auto";
|
|
1664
|
+
const check_resolved = resolve_check_runner(check, workspace);
|
|
1665
|
+
const tester = resolve_tester(raw.tester, builder_env);
|
|
1666
|
+
if (tester) refuse_unready_tester(check_resolved, gate_paths, critic_provider, options.uncontained_builder === "ask" || !unguarded_builder(raw, builder_env));
|
|
1667
|
+
const critic_model = raw.critic_model ?? DEFAULT_CRITIC_MODEL;
|
|
1668
|
+
const tester_model = resolve_tester_model(tester, raw.tester_model, builder_env, critic_model);
|
|
1162
1669
|
const sandbox_image = resolve_sandbox_image(raw, builder_env);
|
|
1163
1670
|
const state_dir = options.state_dir !== void 0 ? options.state_dir : resolve_state_dir(raw.state_dir, cwd, env);
|
|
1164
1671
|
if (state_dir === workspace) throw config_error(`--state-dir (state_dir) can't be the workspace itself: ${workspace}`);
|
|
1165
1672
|
const run_id = resolve_run_id(options.run_id);
|
|
1166
1673
|
if (state_dir !== null) validate_run_state_dir(join(state_dir, run_id), contained);
|
|
1167
|
-
const check = raw.check ?? "auto";
|
|
1168
1674
|
return {
|
|
1169
1675
|
version: 2,
|
|
1170
1676
|
run_id,
|
|
@@ -1173,7 +1679,7 @@ function resolve_config(raw, options = {}) {
|
|
|
1173
1679
|
prompt,
|
|
1174
1680
|
criteria,
|
|
1175
1681
|
check,
|
|
1176
|
-
check_resolved
|
|
1682
|
+
check_resolved,
|
|
1177
1683
|
builder_model: raw.builder_model ?? "opus",
|
|
1178
1684
|
builder_provider,
|
|
1179
1685
|
builder_url,
|
|
@@ -1181,10 +1687,14 @@ function resolve_config(raw, options = {}) {
|
|
|
1181
1687
|
builder_max_tool_calls,
|
|
1182
1688
|
builder_max_output_tokens,
|
|
1183
1689
|
builder_trim_history,
|
|
1690
|
+
effort_ladder,
|
|
1184
1691
|
allow_unsandboxed_builder,
|
|
1185
|
-
critic_model
|
|
1692
|
+
critic_model,
|
|
1186
1693
|
critic_provider,
|
|
1187
1694
|
critic_url,
|
|
1695
|
+
critic_effort,
|
|
1696
|
+
tester,
|
|
1697
|
+
tester_model,
|
|
1188
1698
|
builder_permission_mode,
|
|
1189
1699
|
critic_preset,
|
|
1190
1700
|
critic_prompt_path,
|
|
@@ -1410,290 +1920,64 @@ function stub(message, input, later) {
|
|
|
1410
1920
|
const detail = exit_code === null ? size : `${size}, exit ${String(exit_code)}`;
|
|
1411
1921
|
return `[volley] ${message.name} output (${detail}) elided from an earlier turn. Run it again if you need it.`;
|
|
1412
1922
|
}
|
|
1413
|
-
/**
|
|
1414
|
-
* The request with every tool result before `cutoff` that is long enough to
|
|
1415
|
-
* matter and didn't fail replaced by a stub. Every message keeps its place,
|
|
1416
|
-
* role, id, and name. A stub reads only what comes before `cutoff`, so the
|
|
1417
|
-
* same transcript prefix always gives the same bytes.
|
|
1418
|
-
*/
|
|
1419
|
-
function trim_view(messages, cutoff) {
|
|
1420
|
-
const window = messages.slice(0, cutoff);
|
|
1421
|
-
const inputs = call_inputs(window);
|
|
1422
|
-
const next_on_path = /* @__PURE__ */ new Map();
|
|
1423
|
-
const later_for = /* @__PURE__ */ new Map();
|
|
1424
|
-
for (let i = window.length - 1; i >= 0; i -= 1) {
|
|
1425
|
-
const message = window[i];
|
|
1426
|
-
if (message?.role !== "tool") continue;
|
|
1427
|
-
const input = inputs.get(message.tool_call_id);
|
|
1428
|
-
const path = input?.path ?? null;
|
|
1429
|
-
if (path === null) continue;
|
|
1430
|
-
if (message.name === "read_file") {
|
|
1431
|
-
const later = next_on_path.get(path);
|
|
1432
|
-
if (later !== void 0) later_for.set(i, later);
|
|
1433
|
-
}
|
|
1434
|
-
const supersedes = supersession_of(message, input);
|
|
1435
|
-
if (supersedes !== null) next_on_path.set(path, supersedes);
|
|
1436
|
-
}
|
|
1437
|
-
return messages.map((message, i) => {
|
|
1438
|
-
if (i >= cutoff || message.role !== "tool" || message.is_error === true) return message;
|
|
1439
|
-
if (message.content.length <= 200) return message;
|
|
1440
|
-
return {
|
|
1441
|
-
...message,
|
|
1442
|
-
content: stub(message, inputs.get(message.tool_call_id), later_for.get(i))
|
|
1443
|
-
};
|
|
1444
|
-
});
|
|
1445
|
-
}
|
|
1446
|
-
/**
|
|
1447
|
-
* Where to cut this turn's request, as a message index, or null to send it
|
|
1448
|
-
* whole. Boundaries are the starts of turns `TRIM_STRIDE_TURNS`,
|
|
1449
|
-
* `2 * TRIM_STRIDE_TURNS`, and so on, short of the last `TRIM_KEEP_TURNS`.
|
|
1450
|
-
* The cut moves to the next boundary only while the request, trimmed at the
|
|
1451
|
-
* last one, is still over `threshold`. A trimmed request only grows as the
|
|
1452
|
-
* session goes on, so a boundary once taken is kept.
|
|
1453
|
-
*/
|
|
1454
|
-
function trim_cutoff(messages, threshold) {
|
|
1455
|
-
const turn_starts = messages.flatMap((message, i) => message.role === "assistant" ? [i] : []);
|
|
1456
|
-
let cutoff = null;
|
|
1457
|
-
for (let turn = 8; turn <= turn_starts.length - 6; turn += 8) {
|
|
1458
|
-
if (estimate_tokens(cutoff === null ? messages : trim_view(messages, cutoff)) <= threshold) break;
|
|
1459
|
-
cutoff = turn_starts[turn] ?? null;
|
|
1460
|
-
}
|
|
1461
|
-
return cutoff;
|
|
1462
|
-
}
|
|
1463
|
-
/** The `prepare_step` hook that trims a request estimated above `threshold`
|
|
1464
|
-
* tokens. */
|
|
1465
|
-
function step_trim(threshold) {
|
|
1466
|
-
return ({ messages }) => {
|
|
1467
|
-
const cutoff = trim_cutoff(messages, threshold);
|
|
1468
|
-
return cutoff === null ? void 0 : { messages: trim_view(messages, cutoff) };
|
|
1469
|
-
};
|
|
1470
|
-
}
|
|
1471
|
-
//#endregion
|
|
1472
|
-
//#region src/workspace_tools.ts
|
|
1473
|
-
/**
|
|
1474
|
-
* Shared workspace tools: `contain()` path confinement plus the read-only
|
|
1475
|
-
* tool set (`read_file`, `search_files`, `list_files`).
|
|
1476
|
-
*
|
|
1477
|
-
* The claude_cli builder/critic get file access from the CLI's own built-in
|
|
1478
|
-
* tools. API-style providers (ollama, lmstudio, …) have no built-ins, so
|
|
1479
|
-
* volley supplies its own. The critic uses the read-only set as-is via
|
|
1480
|
-
* `src/critic/tools.ts`; the builder set in
|
|
1481
|
-
* `src/builder/tools.ts` reuses `contain()` and the read tools and adds its
|
|
1482
|
-
* write/exec surface. Every path is resolved and confined to the
|
|
1483
|
-
* workspace root.
|
|
1484
|
-
*/
|
|
1485
|
-
const READ_FILE_MAX_BYTES = 2e5;
|
|
1486
|
-
const LIST_MAX_ENTRIES = 2e3;
|
|
1487
|
-
/** Directories never worth handing a seat; they bury signal and blow the
|
|
1488
|
-
* traversal budget. The acp seats' `list_files` and `search` skip the same
|
|
1489
|
-
* names (`./acp_tools.ts`), so a seat sees one tree under acp and on the CLI. */
|
|
1490
|
-
const IGNORED_DIRS = /* @__PURE__ */ new Set([
|
|
1491
|
-
"node_modules",
|
|
1492
|
-
".git",
|
|
1493
|
-
".volley",
|
|
1494
|
-
".check",
|
|
1495
|
-
"dist",
|
|
1496
|
-
".cache"
|
|
1497
|
-
]);
|
|
1498
|
-
/** Resolve `rel` inside `workspace`, rejecting absolute paths and any `..`
|
|
1499
|
-
* escape. Returns the absolute path; throws on containment violation. The
|
|
1500
|
-
* `workspace` root passed here is the build root, which is the per-run git
|
|
1501
|
-
* worktree under `--worktree` (`build_root`), not necessarily the
|
|
1502
|
-
* workspace the run was started on. */
|
|
1503
|
-
function contain(workspace, rel) {
|
|
1504
|
-
const root = resolve(workspace);
|
|
1505
|
-
const candidate = isAbsolute(rel) ? resolve(rel) : resolve(root, rel);
|
|
1506
|
-
if (!(candidate === root || candidate.startsWith(`${root}${sep}`))) throw new Error(`path escapes the workspace: ${rel}`);
|
|
1507
|
-
return candidate;
|
|
1508
|
-
}
|
|
1509
|
-
function walk(root, start, limit) {
|
|
1510
|
-
const out = [];
|
|
1511
|
-
const stack = [start];
|
|
1512
|
-
while (stack.length > 0 && out.length < limit) {
|
|
1513
|
-
const dir = stack.pop();
|
|
1514
|
-
let entries;
|
|
1515
|
-
try {
|
|
1516
|
-
entries = readdirSync(dir);
|
|
1517
|
-
} catch {
|
|
1518
|
-
continue;
|
|
1519
|
-
}
|
|
1520
|
-
for (const name of entries) {
|
|
1521
|
-
const abs = join(dir, name);
|
|
1522
|
-
let is_dir;
|
|
1523
|
-
try {
|
|
1524
|
-
is_dir = statSync(abs).isDirectory();
|
|
1525
|
-
} catch {
|
|
1526
|
-
continue;
|
|
1527
|
-
}
|
|
1528
|
-
if (is_dir) {
|
|
1529
|
-
if (!IGNORED_DIRS.has(name)) stack.push(abs);
|
|
1530
|
-
continue;
|
|
1531
|
-
}
|
|
1532
|
-
out.push({
|
|
1533
|
-
rel: relative(root, abs),
|
|
1534
|
-
abs
|
|
1535
|
-
});
|
|
1536
|
-
if (out.length >= limit) break;
|
|
1537
|
-
}
|
|
1538
|
-
}
|
|
1539
|
-
return out;
|
|
1540
|
-
}
|
|
1541
|
-
/** The files a read tool's optional `path` covers: the whole tree, the tree
|
|
1542
|
-
* under a directory, or the one file it names. A path that doesn't exist
|
|
1543
|
-
* throws, so the seat reads why rather than an empty result that looks like
|
|
1544
|
-
* "nothing matched". */
|
|
1545
|
-
function scoped_files(workspace, path, limit) {
|
|
1546
|
-
const root = resolve(workspace);
|
|
1547
|
-
if (path === void 0) return walk(root, root, limit);
|
|
1548
|
-
const start = contain(workspace, path);
|
|
1549
|
-
let is_dir;
|
|
1550
|
-
try {
|
|
1551
|
-
is_dir = statSync(start).isDirectory();
|
|
1552
|
-
} catch (err) {
|
|
1553
|
-
throw new Error(`no such file or directory: ${path}`, { cause: err });
|
|
1554
|
-
}
|
|
1555
|
-
return is_dir ? walk(root, start, limit) : [{
|
|
1556
|
-
rel: relative(root, start),
|
|
1557
|
-
abs: start
|
|
1558
|
-
}];
|
|
1559
|
-
}
|
|
1560
|
-
function build_regex(pattern, flags) {
|
|
1561
|
-
try {
|
|
1562
|
-
return new RegExp(pattern, flags);
|
|
1563
|
-
} catch (err) {
|
|
1564
|
-
const detail = err instanceof Error ? err.message : String(err);
|
|
1565
|
-
throw new Error(`invalid regular expression: ${detail}`);
|
|
1566
|
-
}
|
|
1567
|
-
}
|
|
1568
|
-
const read_file_input = z.object({ path: z.string().describe("Workspace-relative path to the file to read.") });
|
|
1569
|
-
const search_files_input = z.object({
|
|
1570
|
-
pattern: z.string().describe("JavaScript regular expression to match against each line."),
|
|
1571
|
-
path: z.string().optional().describe("Optional workspace-relative directory or file to limit the search to."),
|
|
1572
|
-
ignore_case: z.boolean().optional().describe("Case-insensitive match. Default false.")
|
|
1573
|
-
});
|
|
1574
|
-
const list_files_input = z.object({
|
|
1575
|
-
path: z.string().optional().describe("Optional workspace-relative directory or file. Default: the workspace root."),
|
|
1576
|
-
contains: z.string().optional().describe("Optional substring filter applied to each relative path.")
|
|
1577
|
-
});
|
|
1578
|
-
function read_only_tools(workspace) {
|
|
1579
|
-
return [
|
|
1580
|
-
{
|
|
1581
|
-
name: "read_file",
|
|
1582
|
-
read_only: true,
|
|
1583
|
-
description: `Read a UTF-8 text file from the workspace. Returns the file content (truncated at ${String(READ_FILE_MAX_BYTES)} bytes).`,
|
|
1584
|
-
input_schema: read_file_input,
|
|
1585
|
-
execute: (raw) => {
|
|
1586
|
-
const abs = contain(workspace, read_file_input.parse(raw).path);
|
|
1587
|
-
const content = readFileSync(abs, "utf8");
|
|
1588
|
-
return content.length > 2e5 ? `${content.slice(0, READ_FILE_MAX_BYTES)}\n… [truncated at ${String(READ_FILE_MAX_BYTES)} bytes]` : content;
|
|
1589
|
-
}
|
|
1590
|
-
},
|
|
1591
|
-
{
|
|
1592
|
-
name: "search_files",
|
|
1593
|
-
read_only: true,
|
|
1594
|
-
description: `Search workspace files for lines matching a regular expression. Returns up to ${String(200)} "relative/path:line: text" hits.`,
|
|
1595
|
-
input_schema: search_files_input,
|
|
1596
|
-
execute: (raw) => {
|
|
1597
|
-
const input = search_files_input.parse(raw);
|
|
1598
|
-
const regex = build_regex(input.pattern, input.ignore_case === true ? "i" : "");
|
|
1599
|
-
const files = scoped_files(workspace, input.path, LIST_MAX_ENTRIES);
|
|
1600
|
-
const hits = [];
|
|
1601
|
-
for (const file of files) {
|
|
1602
|
-
if (hits.length >= 200) break;
|
|
1603
|
-
let content;
|
|
1604
|
-
try {
|
|
1605
|
-
content = readFileSync(file.abs, "utf8");
|
|
1606
|
-
} catch {
|
|
1607
|
-
continue;
|
|
1608
|
-
}
|
|
1609
|
-
const lines = content.split("\n");
|
|
1610
|
-
for (let i = 0; i < lines.length; i += 1) {
|
|
1611
|
-
if (hits.length >= 200) break;
|
|
1612
|
-
const line = lines[i];
|
|
1613
|
-
if (regex.test(line)) hits.push(`${file.rel}:${String(i + 1)}: ${line.trim()}`);
|
|
1614
|
-
}
|
|
1615
|
-
}
|
|
1616
|
-
if (hits.length === 0) return "no matches";
|
|
1617
|
-
const capped = hits.length >= 200 ? `\n… [capped at ${String(200)} matches]` : "";
|
|
1618
|
-
return `${hits.join("\n")}${capped}`;
|
|
1619
|
-
}
|
|
1620
|
-
},
|
|
1621
|
-
{
|
|
1622
|
-
name: "list_files",
|
|
1623
|
-
read_only: true,
|
|
1624
|
-
description: "List files under a workspace directory (recursively; skips node_modules, .git, and other build/output directories).",
|
|
1625
|
-
input_schema: list_files_input,
|
|
1626
|
-
execute: (raw) => {
|
|
1627
|
-
const input = list_files_input.parse(raw);
|
|
1628
|
-
let files = scoped_files(workspace, input.path, LIST_MAX_ENTRIES).map((f) => f.rel);
|
|
1629
|
-
if (input.contains !== void 0) {
|
|
1630
|
-
const needle = input.contains;
|
|
1631
|
-
files = files.filter((f) => f.includes(needle));
|
|
1632
|
-
}
|
|
1633
|
-
if (files.length === 0) return "no files";
|
|
1634
|
-
const capped = files.length >= LIST_MAX_ENTRIES ? `\n… [capped at ${String(LIST_MAX_ENTRIES)} entries]` : "";
|
|
1635
|
-
return `${files.join("\n")}${capped}`;
|
|
1636
|
-
}
|
|
1637
|
-
}
|
|
1638
|
-
];
|
|
1639
|
-
}
|
|
1640
|
-
/** A `paths + sizes` listing of the workspace tree, no file contents — the
|
|
1641
|
-
* grounding a tool-less critic gets when its provider's tool parser died and the
|
|
1642
|
-
* degradation ladder traded read access for survival. Reuses the same
|
|
1643
|
-
* `walk` + `IGNORED_DIRS` traversal as the read tools, so the inventory matches
|
|
1644
|
-
* what a working `list_files` would have surfaced; caps at `limit` entries. */
|
|
1645
|
-
function workspace_inventory(root, limit = LIST_MAX_ENTRIES) {
|
|
1646
|
-
const base = resolve(root);
|
|
1647
|
-
const files = walk(base, base, limit);
|
|
1648
|
-
if (files.length === 0) return "no files";
|
|
1649
|
-
const lines = files.map((file) => {
|
|
1650
|
-
let size;
|
|
1651
|
-
try {
|
|
1652
|
-
size = statSync(file.abs).size;
|
|
1653
|
-
} catch {
|
|
1654
|
-
size = 0;
|
|
1923
|
+
/**
|
|
1924
|
+
* The request with every tool result before `cutoff` that is long enough to
|
|
1925
|
+
* matter and didn't fail replaced by a stub. Every message keeps its place,
|
|
1926
|
+
* role, id, and name. A stub reads only what comes before `cutoff`, so the
|
|
1927
|
+
* same transcript prefix always gives the same bytes.
|
|
1928
|
+
*/
|
|
1929
|
+
function trim_view(messages, cutoff) {
|
|
1930
|
+
const window = messages.slice(0, cutoff);
|
|
1931
|
+
const inputs = call_inputs(window);
|
|
1932
|
+
const next_on_path = /* @__PURE__ */ new Map();
|
|
1933
|
+
const later_for = /* @__PURE__ */ new Map();
|
|
1934
|
+
for (let i = window.length - 1; i >= 0; i -= 1) {
|
|
1935
|
+
const message = window[i];
|
|
1936
|
+
if (message?.role !== "tool") continue;
|
|
1937
|
+
const input = inputs.get(message.tool_call_id);
|
|
1938
|
+
const path = input?.path ?? null;
|
|
1939
|
+
if (path === null) continue;
|
|
1940
|
+
if (message.name === "read_file") {
|
|
1941
|
+
const later = next_on_path.get(path);
|
|
1942
|
+
if (later !== void 0) later_for.set(i, later);
|
|
1655
1943
|
}
|
|
1656
|
-
|
|
1944
|
+
const supersedes = supersession_of(message, input);
|
|
1945
|
+
if (supersedes !== null) next_on_path.set(path, supersedes);
|
|
1946
|
+
}
|
|
1947
|
+
return messages.map((message, i) => {
|
|
1948
|
+
if (i >= cutoff || message.role !== "tool" || message.is_error === true) return message;
|
|
1949
|
+
if (message.content.length <= 200) return message;
|
|
1950
|
+
return {
|
|
1951
|
+
...message,
|
|
1952
|
+
content: stub(message, inputs.get(message.tool_call_id), later_for.get(i))
|
|
1953
|
+
};
|
|
1657
1954
|
});
|
|
1658
|
-
const capped = files.length >= limit ? `\n… [capped at ${String(limit)} entries]` : "";
|
|
1659
|
-
return `${lines.join("\n")}${capped}`;
|
|
1660
1955
|
}
|
|
1661
|
-
//#endregion
|
|
1662
|
-
//#region src/builder/tools.ts
|
|
1663
1956
|
/**
|
|
1664
|
-
*
|
|
1665
|
-
*
|
|
1666
|
-
*
|
|
1667
|
-
*
|
|
1668
|
-
*
|
|
1669
|
-
*
|
|
1670
|
-
* workspace via `contain()`.
|
|
1671
|
-
*
|
|
1672
|
-
* Error semantics: a wrong-but-recoverable input — `edit_file`
|
|
1673
|
-
* matching 0 or N places, a `bash` command exiting non-zero or timing out, a
|
|
1674
|
-
* `fetch` that is SSRF-blocked or fails — is *returned* as a tool result the
|
|
1675
|
-
* model reads and acts on. Tools throw only on a genuine harness fault
|
|
1676
|
-
* (containment violation, unreadable input, over-cap write).
|
|
1677
|
-
*
|
|
1678
|
-
* `fetch` is a native readability pipeline (`linkedom` + `@mozilla/
|
|
1679
|
-
* readability` + `turndown`) with SSRF protection at the undici connector, not
|
|
1680
|
-
* as a pre-flight URL check: a validating DNS `lookup` rejects hostnames that
|
|
1681
|
-
* resolve to a non-unicast address before the TCP connect, and the resolved
|
|
1682
|
-
* socket's peer address is re-checked after connect — which also catches
|
|
1683
|
-
* literal-IP URLs (Node skips `lookup` for those) and every redirect hop
|
|
1684
|
-
* (each re-dispatches through the same guarded connector). The byte cap is
|
|
1685
|
-
* enforced while reading the HTTP stream, so raw HTML is never returned and a
|
|
1686
|
-
* huge page cannot OOM the tool.
|
|
1687
|
-
*
|
|
1688
|
-
* Egress is defended in two independent layers: this tool's SSRF
|
|
1689
|
-
* deny-list (above) and the sandbox's container network posture (`--network none`
|
|
1690
|
-
* by default, or a host-collapsed allowlist bridge — see `SandboxNetwork` in
|
|
1691
|
-
* `src/sandbox.ts`). Under whole-process containment the whole volley
|
|
1692
|
-
* process — `fetch` included — runs inside the container, so both layers apply to
|
|
1693
|
-
* `fetch`'s egress: `--network none` leaves it no route and it returns a "could
|
|
1694
|
-
* not fetch" error result, and the loop continues.
|
|
1957
|
+
* Where to cut this turn's request, as a message index, or null to send it
|
|
1958
|
+
* whole. Boundaries are the starts of turns `TRIM_STRIDE_TURNS`,
|
|
1959
|
+
* `2 * TRIM_STRIDE_TURNS`, and so on, short of the last `TRIM_KEEP_TURNS`.
|
|
1960
|
+
* The cut moves to the next boundary only while the request, trimmed at the
|
|
1961
|
+
* last one, is still over `threshold`. A trimmed request only grows as the
|
|
1962
|
+
* session goes on, so a boundary once taken is kept.
|
|
1695
1963
|
*/
|
|
1696
|
-
|
|
1964
|
+
function trim_cutoff(messages, threshold) {
|
|
1965
|
+
const turn_starts = messages.flatMap((message, i) => message.role === "assistant" ? [i] : []);
|
|
1966
|
+
let cutoff = null;
|
|
1967
|
+
for (let turn = 8; turn <= turn_starts.length - 6; turn += 8) {
|
|
1968
|
+
if (estimate_tokens(cutoff === null ? messages : trim_view(messages, cutoff)) <= threshold) break;
|
|
1969
|
+
cutoff = turn_starts[turn] ?? null;
|
|
1970
|
+
}
|
|
1971
|
+
return cutoff;
|
|
1972
|
+
}
|
|
1973
|
+
/** The `prepare_step` hook that trims a request estimated above `threshold`
|
|
1974
|
+
* tokens. */
|
|
1975
|
+
function step_trim(threshold) {
|
|
1976
|
+
return ({ messages }) => {
|
|
1977
|
+
const cutoff = trim_cutoff(messages, threshold);
|
|
1978
|
+
return cutoff === null ? void 0 : { messages: trim_view(messages, cutoff) };
|
|
1979
|
+
};
|
|
1980
|
+
}
|
|
1697
1981
|
const BASH_MAX_OUTPUT_BYTES = 1e5;
|
|
1698
1982
|
const BASH_CAPTURE_MAX_BYTES = 1e7;
|
|
1699
1983
|
const FETCH_MAX_BYTES = 2e5;
|
|
@@ -1702,15 +1986,6 @@ const FETCH_MAX_REDIRECTS = 5;
|
|
|
1702
1986
|
const FETCH_TIMEOUT_MS = 3e4;
|
|
1703
1987
|
const FETCH_MIN_EXTRACT_CHARS = 200;
|
|
1704
1988
|
const FETCH_USER_AGENT = "volley-local-builder";
|
|
1705
|
-
const write_file_input = z.object({
|
|
1706
|
-
path: z.string().describe("Workspace-relative path to write. Parent directories are created."),
|
|
1707
|
-
content: z.string().describe("Full UTF-8 content of the file. Overwrites any existing content.")
|
|
1708
|
-
});
|
|
1709
|
-
const edit_file_input = z.object({
|
|
1710
|
-
path: z.string().describe("Workspace-relative path of the file to edit."),
|
|
1711
|
-
old_str: z.string().min(1).describe("Exact text to replace. Must occur exactly once in the file."),
|
|
1712
|
-
new_str: z.string().describe("Replacement text.")
|
|
1713
|
-
});
|
|
1714
1989
|
const bash_input = z.object({ command: z.string().describe("Shell command to run in the workspace directory.") });
|
|
1715
1990
|
const finish_input = z.object({ summary: z.string().describe("Short summary of what was accomplished.") });
|
|
1716
1991
|
const fetch_input = z.object({
|
|
@@ -1718,9 +1993,6 @@ const fetch_input = z.object({
|
|
|
1718
1993
|
max_chars: z.number().int().positive().optional().describe(`Maximum characters of extracted markdown to return (default ${String(FETCH_MAX_CHARS)}).`),
|
|
1719
1994
|
start_index: z.number().int().min(0).optional().describe("Character offset into the extracted markdown to start from, for paging a long page. Default 0.")
|
|
1720
1995
|
});
|
|
1721
|
-
function count_occurrences(haystack, needle) {
|
|
1722
|
-
return haystack.split(needle).length - 1;
|
|
1723
|
-
}
|
|
1724
1996
|
/** Truncate `text` to at most `max` bytes, appending a marker when clipped.
|
|
1725
1997
|
* Byte-based to honor the byte cap; a split multibyte char at the boundary
|
|
1726
1998
|
* renders as the replacement char, which is acceptable for tool output. */
|
|
@@ -1960,35 +2232,6 @@ function builder_tools(workspace, options = {}) {
|
|
|
1960
2232
|
const bash_timeout_ms = options.bash_timeout_ms ?? 6e5;
|
|
1961
2233
|
const bash_max_output_bytes = options.bash_max_output_bytes ?? BASH_MAX_OUTPUT_BYTES;
|
|
1962
2234
|
const bash_executor = options.bash_executor ?? host_bash_executor(workspace);
|
|
1963
|
-
const write_file = {
|
|
1964
|
-
name: "write_file",
|
|
1965
|
-
description: `Write a UTF-8 text file in the workspace, creating parent directories and overwriting any existing content. Rejects content over ${String(WRITE_FILE_MAX_BYTES)} bytes.`,
|
|
1966
|
-
input_schema: write_file_input,
|
|
1967
|
-
execute: (raw) => {
|
|
1968
|
-
const input = write_file_input.parse(raw);
|
|
1969
|
-
const abs = contain(workspace, input.path);
|
|
1970
|
-
const bytes = Buffer.byteLength(input.content, "utf8");
|
|
1971
|
-
if (bytes > 1e6) throw new Error(`content is ${String(bytes)} bytes, over the ${String(WRITE_FILE_MAX_BYTES)}-byte write cap`);
|
|
1972
|
-
mkdirSync(dirname(abs), { recursive: true });
|
|
1973
|
-
writeFileSync(abs, input.content, "utf8");
|
|
1974
|
-
return `wrote ${String(bytes)} bytes to ${input.path}`;
|
|
1975
|
-
}
|
|
1976
|
-
};
|
|
1977
|
-
const edit_file = {
|
|
1978
|
-
name: "edit_file",
|
|
1979
|
-
description: "Replace one exact occurrence of old_str with new_str in a workspace file. old_str must match the file content exactly once — on 0 or multiple matches nothing is changed and an error is returned; read the file and retry with a longer, unique old_str.",
|
|
1980
|
-
input_schema: edit_file_input,
|
|
1981
|
-
execute: (raw) => {
|
|
1982
|
-
const input = edit_file_input.parse(raw);
|
|
1983
|
-
const abs = contain(workspace, input.path);
|
|
1984
|
-
const content = readFileSync(abs, "utf8");
|
|
1985
|
-
const matches = count_occurrences(content, input.old_str);
|
|
1986
|
-
if (matches === 0) return `error: old_str not found in ${input.path} (0 matches). No change made. Read the file and retry with an exact excerpt of its current content.`;
|
|
1987
|
-
if (matches > 1) return `error: old_str matches ${String(matches)} times in ${input.path}. No change made. Include more surrounding context to make old_str unique.`;
|
|
1988
|
-
writeFileSync(abs, content.replace(input.old_str, input.new_str), "utf8");
|
|
1989
|
-
return `edit applied to ${input.path}`;
|
|
1990
|
-
}
|
|
1991
|
-
};
|
|
1992
2235
|
const bash = {
|
|
1993
2236
|
name: "bash",
|
|
1994
2237
|
description: `Run a shell command in the workspace and return its exit code, stdout, and stderr. Each call is independent — no shell state (working directory, environment, variables) carries over between commands. A non-zero exit is a normal result, not an error: read it and adjust. Each stream is truncated past ${String(bash_max_output_bytes)} bytes, and the command is killed if it runs longer than ${String(bash_timeout_ms)}ms.`,
|
|
@@ -2023,8 +2266,7 @@ function builder_tools(workspace, options = {}) {
|
|
|
2023
2266
|
};
|
|
2024
2267
|
return [
|
|
2025
2268
|
...read_only_tools(workspace),
|
|
2026
|
-
|
|
2027
|
-
edit_file,
|
|
2269
|
+
...write_tools(workspace),
|
|
2028
2270
|
bash,
|
|
2029
2271
|
fetch,
|
|
2030
2272
|
finish_tool()
|
|
@@ -2295,6 +2537,32 @@ const BUILDER_ALLOWED_TOOLS = [
|
|
|
2295
2537
|
];
|
|
2296
2538
|
const CRITIC_ALLOWED_TOOLS = ["Read(/**)"];
|
|
2297
2539
|
const CRITIC_DISALLOWED_TOOLS = "Write,Edit,MultiEdit,NotebookEdit,Bash";
|
|
2540
|
+
/** A gate pattern an `Edit(...)` rule can carry: one with no space, comma or
|
|
2541
|
+
* parenthesis, which the CLI would read as the rule list's own syntax. One
|
|
2542
|
+
* that can't be carried is left out, so the tester can write less, never
|
|
2543
|
+
* more. */
|
|
2544
|
+
function rule_safe(pattern) {
|
|
2545
|
+
return !/[\s,()]/.test(pattern);
|
|
2546
|
+
}
|
|
2547
|
+
/** A `claude_cli` tester's allowlist (move 1's D19): reads anywhere in the
|
|
2548
|
+
* build root, and one `Edit` rule per writable gate pattern
|
|
2549
|
+
* (`writable_gate_patterns`), anchored at the build root. The tester runs in
|
|
2550
|
+
* the CLI's default permission mode, where an edit no rule allows is refused,
|
|
2551
|
+
* since `-p` has no one to ask. */
|
|
2552
|
+
function tester_allowed_tools(writable) {
|
|
2553
|
+
return ["Read(/**)", ...writable.filter(rule_safe).map((pattern) => `Edit(/${pattern.replace(/^\/+/, "")})`)];
|
|
2554
|
+
}
|
|
2555
|
+
/** What a `claude_cli` tester is refused outright: a shell and notebooks, and
|
|
2556
|
+
* an edit to the files the gate patterns cover that configure the check rather
|
|
2557
|
+
* than test the work (D20). A deny rule wins over the allowlist in every
|
|
2558
|
+
* permission mode. */
|
|
2559
|
+
const TESTER_DISALLOWED_TOOLS = [
|
|
2560
|
+
"Bash",
|
|
2561
|
+
"NotebookEdit",
|
|
2562
|
+
"Edit(/**/package.json)",
|
|
2563
|
+
"Edit(/**/*.config.*)",
|
|
2564
|
+
"Edit(/**/.github/**)"
|
|
2565
|
+
].join(",");
|
|
2298
2566
|
const PERMISSION_BLOCKS = { permissions: {
|
|
2299
2567
|
blockWritesOutsideWorkingDirectories: true,
|
|
2300
2568
|
blockReadsOutsideWorkingDirectories: true
|
|
@@ -2435,6 +2703,7 @@ function accumulate(state, role, result, model, duration_ms) {
|
|
|
2435
2703
|
iteration_cost_usd: state.iteration_cost_usd + cost,
|
|
2436
2704
|
builder_cost_usd: state.builder_cost_usd + (role === "builder" ? cost : 0),
|
|
2437
2705
|
critic_cost_usd: state.critic_cost_usd + (role === "critic" ? cost : 0),
|
|
2706
|
+
tester_cost_usd: state.tester_cost_usd + (role === "tester" ? cost : 0),
|
|
2438
2707
|
cost_warned: state.cost_warned || record.cost_usd === null
|
|
2439
2708
|
};
|
|
2440
2709
|
}
|
|
@@ -2983,6 +3252,29 @@ function previous_check_section(check) {
|
|
|
2983
3252
|
if (excerpt.length > 0) lines.push("", excerpt);
|
|
2984
3253
|
return lines;
|
|
2985
3254
|
}
|
|
3255
|
+
/** The tester's test, in every build after it: its files, which the builder
|
|
3256
|
+
* makes pass rather than edits, and in the build it ran before, the check
|
|
3257
|
+
* that failed on them. A tester outside a git repository wrote files volley
|
|
3258
|
+
* can't name, so only the check is said. */
|
|
3259
|
+
function tester_section(tester) {
|
|
3260
|
+
if (tester === null || tester === void 0) return [];
|
|
3261
|
+
const lines = [
|
|
3262
|
+
"",
|
|
3263
|
+
"THE TESTER'S TEST",
|
|
3264
|
+
"-----------------"
|
|
3265
|
+
];
|
|
3266
|
+
if (tester.paths.length > 0) lines.push("Before the first build, a tester wrote these tests from the acceptance", "criteria:", ...tester.paths.map((path) => ` ${path}`), "Do the work that makes them pass. Never weaken, skip or delete one: an edit", "to any of them is reported as a gate edit.");
|
|
3267
|
+
else lines.push("Before the first build, a tester wrote a test from the acceptance criteria.");
|
|
3268
|
+
const check = tester.check;
|
|
3269
|
+
if (check !== null && check.ran && !check.ok) {
|
|
3270
|
+
const exit = check.exit_code === null ? "no exit code" : `exit ${String(check.exit_code)}`;
|
|
3271
|
+
lines.push("", `The check fails on it now, as it must (${exit}).`);
|
|
3272
|
+
if (check.failing_slots.length > 0) lines.push(`Failing slots: ${check.failing_slots.join(", ")}.`);
|
|
3273
|
+
const excerpt = check_excerpt(check);
|
|
3274
|
+
if (excerpt.length > 0) lines.push("", excerpt);
|
|
3275
|
+
}
|
|
3276
|
+
return lines;
|
|
3277
|
+
}
|
|
2986
3278
|
/** The line a later iteration's prompt carries when the last session ran out
|
|
2987
3279
|
* of steps: it never got to verify, so this one should check first. */
|
|
2988
3280
|
function step_limit_lines(step_limit_hit) {
|
|
@@ -3008,16 +3300,19 @@ function compose_builder_prompt(input) {
|
|
|
3008
3300
|
];
|
|
3009
3301
|
if (input.iteration > 1 && input.feedback !== null) parts.push("", "PREVIOUS CRITIC FEEDBACK", "------------------------", input.feedback, "", "Address the feedback above. The workspace already contains your prior work;", "read the current state, then make the necessary changes.");
|
|
3010
3302
|
if (input.iteration > 1 && input.previous !== null) parts.push(...previous_check_section(input.previous.check), ...step_limit_lines(input.previous.step_limit_hit));
|
|
3303
|
+
parts.push(...tester_section(input.tester));
|
|
3011
3304
|
if (input.checkride) parts.push("", "The definition of done for this workspace includes checkride exiting 0. Run", "`pnpm exec checkride --digest`, the check volley gates on, after each batch of", "edits and again before you finish. When it fails, read .check/digest.md", "first: it lists each failing slot's first findings. Open a slot's raw", "output under .check/ only if the digest isn't enough, then fix and re-run.");
|
|
3012
3305
|
return parts.join("\n");
|
|
3013
3306
|
}
|
|
3014
3307
|
function builder_max_steps_warning(max_steps) {
|
|
3015
3308
|
return `builder hit the ${String(max_steps)}-step limit without calling finish; handing the partial workspace to check + critic (raise --builder-max-steps if this recurs)`;
|
|
3016
3309
|
}
|
|
3017
|
-
/** Build the builder leaf for this run
|
|
3018
|
-
* the whole run, so it is wired once here: the provider's system
|
|
3019
|
-
* tool surface,
|
|
3020
|
-
|
|
3310
|
+
/** Build the builder leaf for this run at `effort`. Everything but the prompt
|
|
3311
|
+
* is fixed for the whole run, so it is wired once here: the provider's system
|
|
3312
|
+
* prompt, its tool surface, the live chunk stream to the renderer, and the
|
|
3313
|
+
* effort, which fascicle's `model_call` takes only as it's built. Null passes
|
|
3314
|
+
* no effort at all, so a run with no ladder makes the call it always has. */
|
|
3315
|
+
function make_builder_step(deps, effort = null) {
|
|
3021
3316
|
const { config } = deps;
|
|
3022
3317
|
return model_call({
|
|
3023
3318
|
engine: deps.engine,
|
|
@@ -3027,9 +3322,45 @@ function make_builder_step(deps) {
|
|
|
3027
3322
|
model: config.builder_model,
|
|
3028
3323
|
system: resolve_builder_system(config, deps.tool_set),
|
|
3029
3324
|
on_chunk: deps.on_chunk,
|
|
3325
|
+
...effort !== null ? { effort } : {},
|
|
3030
3326
|
...builder_tool_options(deps)
|
|
3031
3327
|
});
|
|
3032
3328
|
}
|
|
3329
|
+
function make_builder_ladder(deps) {
|
|
3330
|
+
const built = /* @__PURE__ */ new Map();
|
|
3331
|
+
const at = (rung) => {
|
|
3332
|
+
const cached = built.get(rung);
|
|
3333
|
+
if (cached !== void 0) return cached;
|
|
3334
|
+
const leaf = make_builder_step(deps, rung);
|
|
3335
|
+
built.set(rung, leaf);
|
|
3336
|
+
return leaf;
|
|
3337
|
+
};
|
|
3338
|
+
return {
|
|
3339
|
+
arm: at(deps.config.effort_ladder?.[0] ?? null),
|
|
3340
|
+
at
|
|
3341
|
+
};
|
|
3342
|
+
}
|
|
3343
|
+
/** The tester's test as `state` carries it, or null when no tester ran: its
|
|
3344
|
+
* files' hashes outlive the iteration it opened, its check doesn't. */
|
|
3345
|
+
function tester_test(state) {
|
|
3346
|
+
const paths = Object.keys(state.tester_hashes).toSorted();
|
|
3347
|
+
if (paths.length === 0 && state.tester_check === null) return null;
|
|
3348
|
+
return {
|
|
3349
|
+
paths,
|
|
3350
|
+
check: state.tester_check
|
|
3351
|
+
};
|
|
3352
|
+
}
|
|
3353
|
+
/** Mark the builder record as having asked for an effort its provider drops. */
|
|
3354
|
+
function with_effort_ignored(state, ignored) {
|
|
3355
|
+
if (!ignored || state.builder === null) return state;
|
|
3356
|
+
return {
|
|
3357
|
+
...state,
|
|
3358
|
+
builder: {
|
|
3359
|
+
...state.builder,
|
|
3360
|
+
effort_ignored: true
|
|
3361
|
+
}
|
|
3362
|
+
};
|
|
3363
|
+
}
|
|
3033
3364
|
/** Run one builder session on the opened iteration `state`, told what the
|
|
3034
3365
|
* iteration before it left behind (`previous`, null on the first). */
|
|
3035
3366
|
async function run_builder(deps, state, previous, ctx) {
|
|
@@ -3046,9 +3377,10 @@ async function run_builder(deps, state, previous, ctx) {
|
|
|
3046
3377
|
feedback: state.feedback,
|
|
3047
3378
|
iteration: state.iteration,
|
|
3048
3379
|
checkride: config.check_resolved === "checkride",
|
|
3049
|
-
previous
|
|
3380
|
+
previous,
|
|
3381
|
+
tester: tester_test(state)
|
|
3050
3382
|
}));
|
|
3051
|
-
const built = accumulate(state, "builder", result, config.builder_model);
|
|
3383
|
+
const built = with_effort_ignored(accumulate(state, "builder", result, config.builder_model), state.rung !== null && ignores_effort(config.builder_provider));
|
|
3052
3384
|
if (result.finish_reason !== "max_steps") return built;
|
|
3053
3385
|
deps.warn(builder_max_steps_warning(config.builder_max_steps));
|
|
3054
3386
|
return {
|
|
@@ -3104,6 +3436,26 @@ function place_iterations_dir(append, config) {
|
|
|
3104
3436
|
const outside = path === ".." || path.startsWith(`..${sep}`) || isAbsolute(path);
|
|
3105
3437
|
return append.split("\n").filter((line) => !(outside && line.includes(ITERATIONS_DIR))).join("\n").replaceAll(ITERATIONS_DIR, `${path}/`);
|
|
3106
3438
|
}
|
|
3439
|
+
/** The tester's test, for the critic to judge as well as the work: whether it
|
|
3440
|
+
* tests the criteria at all, since a check is only as good as what it runs.
|
|
3441
|
+
* Null when no tester wrote one. */
|
|
3442
|
+
function format_tester_section(paths) {
|
|
3443
|
+
if (paths === void 0 || paths.length === 0) return null;
|
|
3444
|
+
return [
|
|
3445
|
+
"THE TESTER'S TEST",
|
|
3446
|
+
"-----------------",
|
|
3447
|
+
"Before the first build, a tester wrote these tests from the acceptance",
|
|
3448
|
+
"criteria, and the check failed on them then:",
|
|
3449
|
+
"",
|
|
3450
|
+
...paths.map((path) => ` ${path}`),
|
|
3451
|
+
"",
|
|
3452
|
+
"Read them, and judge whether they test the criteria: a test a wrong",
|
|
3453
|
+
"implementation would pass, or one that leaves out a criterion a test could",
|
|
3454
|
+
"check, does not. Name each problem in test_findings, one per entry, and leave",
|
|
3455
|
+
"it empty when the test is sound. Judge the work against the criteria either",
|
|
3456
|
+
"way, not against the test alone."
|
|
3457
|
+
].join("\n");
|
|
3458
|
+
}
|
|
3107
3459
|
/** One-letter status column, so a hundred-path list stays readable. */
|
|
3108
3460
|
const STATUS_MARKS = {
|
|
3109
3461
|
added: "A",
|
|
@@ -3165,6 +3517,7 @@ function format_check_section(check) {
|
|
|
3165
3517
|
}
|
|
3166
3518
|
function compose_critic_prompt(input) {
|
|
3167
3519
|
const changes = format_changes_section(input.changes);
|
|
3520
|
+
const tester = format_tester_section(input.tester_paths);
|
|
3168
3521
|
return [
|
|
3169
3522
|
"ACCEPTANCE CRITERIA",
|
|
3170
3523
|
"-------------------",
|
|
@@ -3174,6 +3527,7 @@ function compose_critic_prompt(input) {
|
|
|
3174
3527
|
"",
|
|
3175
3528
|
format_check_section(input.check),
|
|
3176
3529
|
...changes === null ? [] : ["", changes],
|
|
3530
|
+
...tester === null ? [] : ["", tester],
|
|
3177
3531
|
"",
|
|
3178
3532
|
"Review the workspace against the acceptance criteria and return your",
|
|
3179
3533
|
"structured verdict."
|
|
@@ -3196,7 +3550,8 @@ function compose_critic_prompt(input) {
|
|
|
3196
3550
|
const verdict_schema = z.object({
|
|
3197
3551
|
verdict: z.enum(["approved", "changes_requested"]),
|
|
3198
3552
|
feedback: z.string().nullable().transform((feedback) => feedback ?? "").describe("Free-form markdown for the builder. Concrete and actionable. Will be passed verbatim into the next iteration."),
|
|
3199
|
-
unmet_criteria: z.array(z.string()).nullable().transform((unmet) => unmet ?? []).describe("The specific acceptance criteria judged unmet, verbatim. Empty when approved.")
|
|
3553
|
+
unmet_criteria: z.array(z.string()).nullable().transform((unmet) => unmet ?? []).describe("The specific acceptance criteria judged unmet, verbatim. Empty when approved."),
|
|
3554
|
+
test_findings: z.array(z.string()).nullable().optional().describe("Problems with the tester's test, one per entry: a criterion it leaves out, or an assertion a wrong implementation would pass. Empty or omitted when no tester ran or its test is sound.")
|
|
3200
3555
|
});
|
|
3201
3556
|
/** Read-only tool wiring per provider. The claude_cli critic is confined at
|
|
3202
3557
|
* the CLI permission layer (allowlist + explicit disallow, and the settings
|
|
@@ -3266,6 +3621,12 @@ function retry_cause_kind_of(err) {
|
|
|
3266
3621
|
const cause = err.cause_kind;
|
|
3267
3622
|
return cause === "provider_5xx" || cause === "network" ? cause : "unknown";
|
|
3268
3623
|
}
|
|
3624
|
+
/** The critic's findings about the tester's test, for its record, when it
|
|
3625
|
+
* named any. */
|
|
3626
|
+
function test_findings_of(verdict) {
|
|
3627
|
+
const findings = verdict.test_findings ?? [];
|
|
3628
|
+
return findings.length > 0 ? { test_findings: findings } : {};
|
|
3629
|
+
}
|
|
3269
3630
|
/** Fold a successful critic `generate` result into loop state, stamping the
|
|
3270
3631
|
* ladder bookkeeping onto the critic record. Shared by the tool-bearing path and
|
|
3271
3632
|
* the tool-less fallback so both shape the record identically (the fallback just
|
|
@@ -3279,7 +3640,8 @@ function finalize_critic(state, result, model, mark, duration_ms) {
|
|
|
3279
3640
|
retries: mark.retries,
|
|
3280
3641
|
...mark.retry_cause_kind !== void 0 ? { retry_cause_kind: mark.retry_cause_kind } : {},
|
|
3281
3642
|
...mark.critic_degraded === true ? { critic_degraded: true } : {},
|
|
3282
|
-
...mark.degrade_cause !== void 0 ? { degrade_cause: mark.degrade_cause } : {}
|
|
3643
|
+
...mark.degrade_cause !== void 0 ? { degrade_cause: mark.degrade_cause } : {},
|
|
3644
|
+
...test_findings_of(result.content)
|
|
3283
3645
|
},
|
|
3284
3646
|
verdict: result.content.verdict,
|
|
3285
3647
|
feedback: result.content.feedback,
|
|
@@ -3303,6 +3665,8 @@ function toolless_critic_prompt(config, tool_prompt) {
|
|
|
3303
3665
|
workspace_inventory(build_root(config.workspace, config.worktree))
|
|
3304
3666
|
].join("\n");
|
|
3305
3667
|
}
|
|
3668
|
+
/** What both of the critic's calls share, its effort included when one is set
|
|
3669
|
+
* (`critic_effort`); unset, no effort is passed, as before there was one. */
|
|
3306
3670
|
function critic_call_config(deps) {
|
|
3307
3671
|
const { config } = deps;
|
|
3308
3672
|
return {
|
|
@@ -3311,7 +3675,8 @@ function critic_call_config(deps) {
|
|
|
3311
3675
|
model: config.critic_model,
|
|
3312
3676
|
system: resolve_critic_prompt(config, deps.tool_set),
|
|
3313
3677
|
schema: verdict_schema,
|
|
3314
|
-
on_chunk: deps.on_chunk
|
|
3678
|
+
on_chunk: deps.on_chunk,
|
|
3679
|
+
...config.critic_effort !== null ? { effort: config.critic_effort } : {}
|
|
3315
3680
|
};
|
|
3316
3681
|
}
|
|
3317
3682
|
/** Rung 1's leaf: the critic with its read tools (or the CLI's read-only
|
|
@@ -3429,11 +3794,20 @@ async function run_critic(deps, state, ctx) {
|
|
|
3429
3794
|
criteria: config.criteria,
|
|
3430
3795
|
iteration: state.iteration,
|
|
3431
3796
|
check,
|
|
3432
|
-
changes: state.changes
|
|
3797
|
+
changes: state.changes,
|
|
3798
|
+
tester_paths: Object.keys(state.tester_hashes).toSorted()
|
|
3433
3799
|
});
|
|
3434
3800
|
const started = Date.now();
|
|
3435
3801
|
const { result, mark } = await ctx.call(deps.critic, prompt);
|
|
3436
|
-
|
|
3802
|
+
const critiqued = finalize_critic(state, result, config.critic_model, mark, Date.now() - started);
|
|
3803
|
+
if (config.critic_effort === null || !ignores_effort(config.critic_provider)) return critiqued;
|
|
3804
|
+
return critiqued.critic === null ? critiqued : {
|
|
3805
|
+
...critiqued,
|
|
3806
|
+
critic: {
|
|
3807
|
+
...critiqued.critic,
|
|
3808
|
+
effort_ignored: true
|
|
3809
|
+
}
|
|
3810
|
+
};
|
|
3437
3811
|
} catch (err) {
|
|
3438
3812
|
throw phase_error("critic", state.iteration, err);
|
|
3439
3813
|
}
|
|
@@ -3457,17 +3831,59 @@ function initial_state() {
|
|
|
3457
3831
|
changes: null,
|
|
3458
3832
|
builder: null,
|
|
3459
3833
|
critic: null,
|
|
3834
|
+
tester: null,
|
|
3835
|
+
tester_check: null,
|
|
3836
|
+
tester_changes: null,
|
|
3837
|
+
tester_hashes: {},
|
|
3460
3838
|
total_usage: EMPTY_USAGE,
|
|
3461
3839
|
total_cost_usd: 0,
|
|
3462
3840
|
builder_cost_usd: 0,
|
|
3463
3841
|
critic_cost_usd: 0,
|
|
3842
|
+
tester_cost_usd: 0,
|
|
3464
3843
|
check_duration_ms: 0,
|
|
3465
3844
|
builder_max_steps_hits: 0,
|
|
3845
|
+
builds: 0,
|
|
3846
|
+
rung: null,
|
|
3847
|
+
climbed: null,
|
|
3466
3848
|
iteration_cost_usd: 0,
|
|
3467
3849
|
halt: null,
|
|
3468
3850
|
cost_warned: false
|
|
3469
3851
|
};
|
|
3470
3852
|
}
|
|
3853
|
+
/** The effort build `build` (counted from 1) runs at: one rung further up the
|
|
3854
|
+
* ladder per build before it, the top rung repeating once it's reached. Null
|
|
3855
|
+
* with no ladder, which passes no effort at all. */
|
|
3856
|
+
function rung_for_build(ladder, build) {
|
|
3857
|
+
if (ladder === null || ladder.length === 0) return null;
|
|
3858
|
+
return ladder[Math.min(Math.max(build, 1), ladder.length) - 1] ?? null;
|
|
3859
|
+
}
|
|
3860
|
+
/** Why build `build` runs a rung above the build before it, read off the state
|
|
3861
|
+
* that build left (`previous`). That build didn't end the run, so its check
|
|
3862
|
+
* failed or the critic sent it back: a check that didn't fail means the critic,
|
|
3863
|
+
* which is also how a resumed run reads it, since a resume keeps the last
|
|
3864
|
+
* check but not the last verdict. Null when it doesn't climb, because it's the
|
|
3865
|
+
* first build, the ladder's top has been reached, or there is no ladder. */
|
|
3866
|
+
function climb_reason(ladder, build, previous) {
|
|
3867
|
+
if (ladder === null || build < 2 || build > ladder.length) return null;
|
|
3868
|
+
return previous.check?.ok === false ? "check_failed" : "critic_rejected";
|
|
3869
|
+
}
|
|
3870
|
+
/** Whether the tester opens the iteration after `s`: it's on, and this is the
|
|
3871
|
+
* run's first (move 1's D17). A run resumed before iteration 1 completed runs
|
|
3872
|
+
* it again. */
|
|
3873
|
+
function tester_opens(config, s) {
|
|
3874
|
+
return config.tester && s.iteration === 0;
|
|
3875
|
+
}
|
|
3876
|
+
/** Whether the test phase opened this iteration and no build has run in it
|
|
3877
|
+
* yet: the tester ran, and the run has started no build. The build then works
|
|
3878
|
+
* in the iteration the tester opened rather than opening the next. */
|
|
3879
|
+
function tester_opened(s) {
|
|
3880
|
+
return s.tester !== null && s.builds === 0;
|
|
3881
|
+
}
|
|
3882
|
+
/** Whether the check passed right after the tester, before any build. A check
|
|
3883
|
+
* skipped for the cost cap didn't run, so it never counts. */
|
|
3884
|
+
function tester_passed_early(s) {
|
|
3885
|
+
return tester_opened(s) && s.tester_check?.ran === true && s.tester_check.ok;
|
|
3886
|
+
}
|
|
3471
3887
|
/** How many rounds the loop may still run: the whole budget on a fresh run, what
|
|
3472
3888
|
* the resumed run's completed iterations left of it otherwise. */
|
|
3473
3889
|
function remaining_iterations(config, resume_from) {
|
|
@@ -3480,7 +3896,8 @@ function gate_edit_halt(config, state) {
|
|
|
3480
3896
|
return config.fail_on_gate_edit && (state.changes?.gate_edits.length ?? 0) > 0;
|
|
3481
3897
|
}
|
|
3482
3898
|
/** All stopping conditions live here: acceptance, cost cap, the gate
|
|
3483
|
-
* edit refusal,
|
|
3899
|
+
* edit refusal, a tester's test passing before any build, and — implicitly
|
|
3900
|
+
* via `max_rounds` — the iteration budget.
|
|
3484
3901
|
* Success wins when it and the cap land on the same iteration.
|
|
3485
3902
|
*
|
|
3486
3903
|
* A gate edit is the one condition that *beats* success rather than losing to
|
|
@@ -3491,9 +3908,10 @@ function gate(config, state) {
|
|
|
3491
3908
|
const gate_edit = gate_edit_halt(config, state);
|
|
3492
3909
|
const success = state.check?.ok === true && state.verdict === "approved" && !gate_edit;
|
|
3493
3910
|
const cap = cost_cap_hit(config, state);
|
|
3494
|
-
const
|
|
3911
|
+
const passed_early = tester_passed_early(state);
|
|
3912
|
+
const halt = gate_edit ? "gate_edit" : passed_early ? "tester_passed_early" : !success && cap ? "cost_cap" : null;
|
|
3495
3913
|
return {
|
|
3496
|
-
stop: success || cap || gate_edit,
|
|
3914
|
+
stop: success || cap || gate_edit || passed_early,
|
|
3497
3915
|
state: {
|
|
3498
3916
|
...state,
|
|
3499
3917
|
halt
|
|
@@ -3502,6 +3920,7 @@ function gate(config, state) {
|
|
|
3502
3920
|
}
|
|
3503
3921
|
function status_of(value, converged) {
|
|
3504
3922
|
if (value.halt === "gate_edit") return "gate_edit_blocked";
|
|
3923
|
+
if (value.halt === "tester_passed_early") return "tester_passed_early";
|
|
3505
3924
|
if (value.halt === "cost_cap") return "cost_cap_reached";
|
|
3506
3925
|
return converged ? "success" : "budget_exhausted";
|
|
3507
3926
|
}
|
|
@@ -3519,6 +3938,7 @@ function paint(role, text, color) {
|
|
|
3519
3938
|
switch (role) {
|
|
3520
3939
|
case "builder": return pc.cyan(text);
|
|
3521
3940
|
case "critic": return pc.yellow(text);
|
|
3941
|
+
case "tester": return pc.blue(text);
|
|
3522
3942
|
case "check": return pc.green(text);
|
|
3523
3943
|
case "cost": return pc.magenta(text);
|
|
3524
3944
|
case "error": return pc.bold(pc.red(text));
|
|
@@ -3531,6 +3951,11 @@ function truncate(text, max, pointer) {
|
|
|
3531
3951
|
if (text.length <= max) return text;
|
|
3532
3952
|
return `${text.slice(0, max)}\n… [truncated at ${String(max)} chars; ${pointer}]`;
|
|
3533
3953
|
}
|
|
3954
|
+
/** The rung a build runs at, and why it's above the last build's when it is:
|
|
3955
|
+
* `rung medium (climbed: check_failed)`. */
|
|
3956
|
+
function format_rung(rung, climbed) {
|
|
3957
|
+
return `rung ${rung}${climbed === null ? "" : ` (climbed: ${climbed})`}`;
|
|
3958
|
+
}
|
|
3534
3959
|
function format_usd(value) {
|
|
3535
3960
|
return value === null ? "$?" : `$${value.toFixed(3)}`;
|
|
3536
3961
|
}
|
|
@@ -3578,10 +4003,23 @@ function phase_summary(record) {
|
|
|
3578
4003
|
...record.retry_cause_kind !== void 0 ? { retry_cause_kind: record.retry_cause_kind } : {},
|
|
3579
4004
|
...record.critic_degraded === true ? { critic_degraded: true } : {},
|
|
3580
4005
|
...record.degrade_cause !== void 0 ? { degrade_cause: record.degrade_cause } : {},
|
|
4006
|
+
...record.test_findings !== void 0 ? { test_findings: record.test_findings } : {},
|
|
4007
|
+
...record.effort_ignored === true ? { effort_ignored: true } : {},
|
|
3581
4008
|
...record.throughput !== void 0 ? { throughput: record.throughput } : {},
|
|
3582
4009
|
...record.steps !== void 0 ? { steps: record.steps } : {}
|
|
3583
4010
|
};
|
|
3584
4011
|
}
|
|
4012
|
+
/** A check as the iteration archive records it, its artifacts aside. */
|
|
4013
|
+
function check_summary(check) {
|
|
4014
|
+
return check === null ? null : {
|
|
4015
|
+
ran: check.ran,
|
|
4016
|
+
runner: check.runner,
|
|
4017
|
+
ok: check.ok,
|
|
4018
|
+
exit_code: check.exit_code,
|
|
4019
|
+
duration_ms: check.duration_ms,
|
|
4020
|
+
failing_slots: check.failing_slots
|
|
4021
|
+
};
|
|
4022
|
+
}
|
|
3585
4023
|
/** Archive one completed iteration: verdict files, check artifacts, and the
|
|
3586
4024
|
* per-iteration summary. */
|
|
3587
4025
|
function archive_iteration(config, state) {
|
|
@@ -3610,16 +4048,17 @@ function archive_iteration(config, state) {
|
|
|
3610
4048
|
started_at: state.iteration_started_at,
|
|
3611
4049
|
completed_at,
|
|
3612
4050
|
duration_ms: Date.parse(completed_at) - Date.parse(state.iteration_started_at),
|
|
4051
|
+
builds: state.builds,
|
|
4052
|
+
rung: state.rung,
|
|
4053
|
+
climbed: state.climbed,
|
|
4054
|
+
effort_ignored: state.builder?.effort_ignored === true,
|
|
3613
4055
|
builder: phase_summary(state.builder),
|
|
3614
|
-
check: check
|
|
3615
|
-
ran: check.ran,
|
|
3616
|
-
runner: check.runner,
|
|
3617
|
-
ok: check.ok,
|
|
3618
|
-
exit_code: check.exit_code,
|
|
3619
|
-
duration_ms: check.duration_ms,
|
|
3620
|
-
failing_slots: check.failing_slots
|
|
3621
|
-
},
|
|
4056
|
+
check: check_summary(check),
|
|
3622
4057
|
critic: phase_summary(state.critic),
|
|
4058
|
+
tester: phase_summary(state.tester),
|
|
4059
|
+
tester_check: check_summary(state.tester_check),
|
|
4060
|
+
tester_changes: state.tester_changes,
|
|
4061
|
+
tester_hashes: state.tester_hashes,
|
|
3623
4062
|
changes: state.changes,
|
|
3624
4063
|
verdict: state.verdict,
|
|
3625
4064
|
unmet_criteria: state.unmet_criteria,
|
|
@@ -3638,6 +4077,7 @@ function run_result_from_state(config, state, status, completed_at) {
|
|
|
3638
4077
|
total_cost_usd: state.total_cost_usd,
|
|
3639
4078
|
builder_cost_usd: state.builder_cost_usd,
|
|
3640
4079
|
critic_cost_usd: state.critic_cost_usd,
|
|
4080
|
+
tester_cost_usd: state.tester_cost_usd,
|
|
3641
4081
|
check_duration_ms: state.check_duration_ms,
|
|
3642
4082
|
builder_max_steps_hits: state.builder_max_steps_hits,
|
|
3643
4083
|
final_verdict: state.verdict,
|
|
@@ -3721,6 +4161,18 @@ function all_gate_edits(archives) {
|
|
|
3721
4161
|
for (const archive of archives) for (const path of archive.changes?.gate_edits ?? []) edits.add(path);
|
|
3722
4162
|
return [...edits].toSorted();
|
|
3723
4163
|
}
|
|
4164
|
+
/** An archived build's rung, or null for none, or one this volley doesn't
|
|
4165
|
+
* know. */
|
|
4166
|
+
function archived_rung(archive) {
|
|
4167
|
+
const rung = archive.rung;
|
|
4168
|
+
return EFFORT_RUNGS.includes(rung) ? rung : null;
|
|
4169
|
+
}
|
|
4170
|
+
/** Whether every build ran at one effort: each build's rung, or the provider's
|
|
4171
|
+
* own where the provider ignored it, taking one value across the run. */
|
|
4172
|
+
function ladder_flat(archives) {
|
|
4173
|
+
const efforts = archives.map((archive) => archive.effort_ignored === true ? "ignored" : archived_rung(archive));
|
|
4174
|
+
return new Set(efforts).size <= 1;
|
|
4175
|
+
}
|
|
3724
4176
|
function salvage_stats(archives) {
|
|
3725
4177
|
let tool_calls = 0;
|
|
3726
4178
|
let salvaged_tool_calls = 0;
|
|
@@ -3750,13 +4202,17 @@ function build_run_summary(config, result) {
|
|
|
3750
4202
|
total_cost_usd: result.total_cost_usd,
|
|
3751
4203
|
builder_cost_usd: result.builder_cost_usd,
|
|
3752
4204
|
critic_cost_usd: result.critic_cost_usd,
|
|
4205
|
+
tester_cost_usd: result.tester_cost_usd,
|
|
4206
|
+
tester_passed_early: result.status === "tester_passed_early",
|
|
3753
4207
|
final_verdict: result.final_verdict,
|
|
3754
4208
|
check_trajectory: check_trajectory(archives),
|
|
3755
4209
|
gate_edits: all_gate_edits(archives),
|
|
3756
4210
|
unmet_criteria: archives.at(-1)?.unmet_criteria ?? [],
|
|
3757
4211
|
local_salvage: salvage_stats(archives),
|
|
3758
4212
|
critic_degraded: any_critic_degraded(archives),
|
|
3759
|
-
critic_degrade_causes: critic_degrade_causes(archives)
|
|
4213
|
+
critic_degrade_causes: critic_degrade_causes(archives),
|
|
4214
|
+
rungs: config.mode === "critique" ? [] : archives.map(archived_rung),
|
|
4215
|
+
ladder_flat: config.mode === "critique" || ladder_flat(archives)
|
|
3760
4216
|
}
|
|
3761
4217
|
};
|
|
3762
4218
|
}
|
|
@@ -3766,6 +4222,183 @@ function write_run_summary(config, result) {
|
|
|
3766
4222
|
writeFileSync(state_path(config, "summary.json"), `${JSON.stringify(build_run_summary(config, result), null, 2)}\n`);
|
|
3767
4223
|
}
|
|
3768
4224
|
//#endregion
|
|
4225
|
+
//#region src/tester/prompt.ts
|
|
4226
|
+
/**
|
|
4227
|
+
* The tester's prompts: its system prompt, the `tester.md` preset and a line
|
|
4228
|
+
* naming the tools its seat holds, and the prompt of its one call, composed
|
|
4229
|
+
* from the criteria, where it may write, the check that will run its test, and
|
|
4230
|
+
* the workspace inventory.
|
|
4231
|
+
*
|
|
4232
|
+
* The tester writes from the criteria rather than the task (move 1's Frame): a
|
|
4233
|
+
* critic can be argued with about prose criteria, a test can't, so the test
|
|
4234
|
+
* is the criteria made executable. The inventory saves its first turns on
|
|
4235
|
+
* finding where the tests live.
|
|
4236
|
+
*/
|
|
4237
|
+
/** Walk up from this module to the package root (works from src/ under tsx
|
|
4238
|
+
* and from dist/ in the published package, which ships src/tester/presets). */
|
|
4239
|
+
function tester_presets_dir() {
|
|
4240
|
+
let dir = dirname(fileURLToPath(import.meta.url));
|
|
4241
|
+
for (let depth = 0; depth < 6; depth += 1) {
|
|
4242
|
+
const candidate = join(dir, "src", "tester", "presets");
|
|
4243
|
+
if (existsSync(candidate)) return candidate;
|
|
4244
|
+
const local = join(dir, "presets");
|
|
4245
|
+
if (existsSync(join(local, "tester.md"))) return local;
|
|
4246
|
+
dir = dirname(dir);
|
|
4247
|
+
}
|
|
4248
|
+
throw config_error("tester presets directory not found");
|
|
4249
|
+
}
|
|
4250
|
+
/** The tools a tester on `provider` holds, as its system prompt names them:
|
|
4251
|
+
* the Claude CLI's own, under the tester's permission list, fascicle's under
|
|
4252
|
+
* `volley acp`, whose writes ask the person in the editor first, or
|
|
4253
|
+
* volley's. */
|
|
4254
|
+
function tools_paragraph(provider, tool_set) {
|
|
4255
|
+
if (provider === "claude_cli") return [
|
|
4256
|
+
"Read the workspace with Read, Grep and Glob. Write and Edit work only on the",
|
|
4257
|
+
"paths your prompt lists as writable, and are refused anywhere else. You",
|
|
4258
|
+
"cannot run commands."
|
|
4259
|
+
].join("\n");
|
|
4260
|
+
if (tool_set === "acp") return [
|
|
4261
|
+
"You have five tools, every path relative to the workspace root:",
|
|
4262
|
+
"- read_file, list_files and search read the workspace.",
|
|
4263
|
+
"- write_file(path, content) and edit_file(path, old_text, new_text) write",
|
|
4264
|
+
" only the paths your prompt lists as writable; anywhere else they refuse,",
|
|
4265
|
+
" naming why. The person in the editor may be asked before each write, and",
|
|
4266
|
+
" a write they decline is not made.",
|
|
4267
|
+
"You cannot run commands; no other tools are available to you."
|
|
4268
|
+
].join("\n");
|
|
4269
|
+
return [
|
|
4270
|
+
"You have five tools, every path workspace-relative:",
|
|
4271
|
+
"- read_file(path), search_files(pattern, path?, ignore_case?) and",
|
|
4272
|
+
" list_files(path?, contains?) read the workspace.",
|
|
4273
|
+
"- write_file(path, content) and edit_file(path, old_str, new_str) write",
|
|
4274
|
+
" only the paths your prompt lists as writable; anywhere else they refuse,",
|
|
4275
|
+
" naming why.",
|
|
4276
|
+
"You cannot run commands; no other tools are available to you."
|
|
4277
|
+
].join("\n");
|
|
4278
|
+
}
|
|
4279
|
+
/** The tester's system prompt for a seat on `provider` holding `tool_set`. */
|
|
4280
|
+
function resolve_tester_prompt(provider, tool_set = "volley") {
|
|
4281
|
+
return `${readFileSync(join(tester_presets_dir(), "tester.md"), "utf8").trimEnd()}\n\n${tools_paragraph(provider, tool_set)}`;
|
|
4282
|
+
}
|
|
4283
|
+
function compose_tester_prompt(input) {
|
|
4284
|
+
const check = input.check === "auto" ? "checkride, found by --check auto" : `\`${input.check}\``;
|
|
4285
|
+
return [
|
|
4286
|
+
"ACCEPTANCE CRITERIA",
|
|
4287
|
+
"-------------------",
|
|
4288
|
+
input.criteria,
|
|
4289
|
+
"",
|
|
4290
|
+
"WHERE YOU MAY WRITE",
|
|
4291
|
+
"-------------------",
|
|
4292
|
+
"Paths matching these patterns, except package.json, *.config.* files and",
|
|
4293
|
+
"anything under .github/:",
|
|
4294
|
+
...input.writable.map((pattern) => ` ${pattern}`),
|
|
4295
|
+
"",
|
|
4296
|
+
"THE CHECK",
|
|
4297
|
+
"---------",
|
|
4298
|
+
`${check} runs once you finish. It must fail then, on your test.`,
|
|
4299
|
+
"",
|
|
4300
|
+
"WORKSPACE FILE INVENTORY",
|
|
4301
|
+
"------------------------",
|
|
4302
|
+
input.inventory,
|
|
4303
|
+
"",
|
|
4304
|
+
"Write the failing test now."
|
|
4305
|
+
].join("\n");
|
|
4306
|
+
}
|
|
4307
|
+
//#endregion
|
|
4308
|
+
//#region src/tester/run.ts
|
|
4309
|
+
/**
|
|
4310
|
+
* The tester seat: one session before the first build that writes a failing
|
|
4311
|
+
* test from the criteria under the test and fixture gate paths (move 1's D5,
|
|
4312
|
+
* D20). It takes the critic's provider, URL and effort, and the critic's model
|
|
4313
|
+
* unless `--tester-model` names one (D7, D15), since writing the right test is
|
|
4314
|
+
* reading work and the critic seat is on every run.
|
|
4315
|
+
*
|
|
4316
|
+
* Confinement is per provider, as the critic's is. A `claude_cli` tester runs
|
|
4317
|
+
* under its own permission list (`tester_allowed_tools`,
|
|
4318
|
+
* `TESTER_DISALLOWED_TOOLS`) in the CLI's default mode; every other tester gets
|
|
4319
|
+
* volley's `tester_tools`, with no shell. This module builds the call and the
|
|
4320
|
+
* dry run's probe of it; the flow runs it.
|
|
4321
|
+
*/
|
|
4322
|
+
/** The tester's step budget on a tool-loop provider: reading the workspace,
|
|
4323
|
+
* then a test file or two. Its own constant, so a change to the builder's or
|
|
4324
|
+
* fascicle's default can't move it. */
|
|
4325
|
+
const TESTER_MAX_STEPS = 30;
|
|
4326
|
+
/** One write a turn, as a local builder makes by default: some model and
|
|
4327
|
+
* runtime pairs return parallel calls badly. Reads don't count. */
|
|
4328
|
+
const TESTER_MAX_TOOL_CALLS = 1;
|
|
4329
|
+
/** The model the tester runs: `--tester-model`, else the critic's. */
|
|
4330
|
+
function tester_model(config) {
|
|
4331
|
+
return config.tester_model ?? config.critic_model;
|
|
4332
|
+
}
|
|
4333
|
+
/** The tester's tool wiring for its provider. */
|
|
4334
|
+
function tester_tool_options(config, deps) {
|
|
4335
|
+
const { tools } = deps;
|
|
4336
|
+
if (config.critic_provider === "claude_cli") return { provider_options: { claude_cli: {
|
|
4337
|
+
allowed_tools: tester_allowed_tools(writable_gate_patterns(config.gate_paths)),
|
|
4338
|
+
extra_args: [
|
|
4339
|
+
"--disallowedTools",
|
|
4340
|
+
TESTER_DISALLOWED_TOOLS,
|
|
4341
|
+
...settings_args()
|
|
4342
|
+
]
|
|
4343
|
+
} } };
|
|
4344
|
+
return {
|
|
4345
|
+
tools: tools === void 0 ? tester_tools(build_root(config.workspace, config.worktree), config.gate_paths) : [...tools],
|
|
4346
|
+
...deps.on_tool_approval !== void 0 ? { on_tool_approval: deps.on_tool_approval } : {},
|
|
4347
|
+
max_steps: TESTER_MAX_STEPS,
|
|
4348
|
+
tool_error_policy: "feed_back",
|
|
4349
|
+
max_tool_calls_per_step: TESTER_MAX_TOOL_CALLS
|
|
4350
|
+
};
|
|
4351
|
+
}
|
|
4352
|
+
/** The tester's one call: its system prompt, its tools, the critic's effort. */
|
|
4353
|
+
function make_tester_step(deps) {
|
|
4354
|
+
const { config } = deps;
|
|
4355
|
+
return model_call({
|
|
4356
|
+
engine: deps.engine,
|
|
4357
|
+
id: "tester",
|
|
4358
|
+
description: "model call writing a failing test under the test and fixture gate paths",
|
|
4359
|
+
provider: config.critic_provider,
|
|
4360
|
+
model: tester_model(config),
|
|
4361
|
+
system: resolve_tester_prompt(config.critic_provider, deps.tool_set),
|
|
4362
|
+
on_chunk: deps.on_chunk,
|
|
4363
|
+
...config.critic_effort !== null ? { effort: config.critic_effort } : {},
|
|
4364
|
+
...tester_tool_options(config, deps)
|
|
4365
|
+
});
|
|
4366
|
+
}
|
|
4367
|
+
/** The prompt of the tester's call, from the run's criteria, writable
|
|
4368
|
+
* patterns, check, and the build root as it stands. */
|
|
4369
|
+
function tester_prompt(config) {
|
|
4370
|
+
return compose_tester_prompt({
|
|
4371
|
+
criteria: config.criteria,
|
|
4372
|
+
writable: writable_gate_patterns(config.gate_paths),
|
|
4373
|
+
check: config.check,
|
|
4374
|
+
inventory: workspace_inventory(build_root(config.workspace, config.worktree))
|
|
4375
|
+
});
|
|
4376
|
+
}
|
|
4377
|
+
/** Run the tester's session on the iteration `state` opened, folding its
|
|
4378
|
+
* record and spend into the state as `tester` (D21). A failure is a phase
|
|
4379
|
+
* error of its own, which exits as a provider error does. */
|
|
4380
|
+
async function run_tester(tester, config, state, ctx) {
|
|
4381
|
+
try {
|
|
4382
|
+
if (config.critic_provider === "ollama") await prewarm_ollama_model(seat_base_url("ollama", config.critic_url, process.env), tester_model(config), ctx.abort);
|
|
4383
|
+
const tested = accumulate(state, "tester", await ctx.call(tester, tester_prompt(config)), tester_model(config));
|
|
4384
|
+
if (config.critic_effort === null || !ignores_effort(config.critic_provider)) return tested;
|
|
4385
|
+
return tested.tester === null ? tested : {
|
|
4386
|
+
...tested,
|
|
4387
|
+
tester: {
|
|
4388
|
+
...tested.tester,
|
|
4389
|
+
effort_ignored: true
|
|
4390
|
+
}
|
|
4391
|
+
};
|
|
4392
|
+
} catch (err) {
|
|
4393
|
+
throw phase_error("tester", state.iteration, err);
|
|
4394
|
+
}
|
|
4395
|
+
}
|
|
4396
|
+
[
|
|
4397
|
+
"CANARY CHECK: a tiny wiring probe, not a real task. Write nothing.",
|
|
4398
|
+
"Call the read_file tool on the path \"package.json\", then, whatever it",
|
|
4399
|
+
"returns, reply with the one word \"canary\"."
|
|
4400
|
+
].join("\n");
|
|
4401
|
+
//#endregion
|
|
3769
4402
|
//#region src/phases.ts
|
|
3770
4403
|
/**
|
|
3771
4404
|
* The step bodies behind the nodes of `./flow.ts`, one function per node. Each
|
|
@@ -3879,7 +4512,10 @@ function next_iteration(s) {
|
|
|
3879
4512
|
critic: null,
|
|
3880
4513
|
check: null,
|
|
3881
4514
|
verdict: null,
|
|
3882
|
-
unmet_criteria: []
|
|
4515
|
+
unmet_criteria: [],
|
|
4516
|
+
tester: null,
|
|
4517
|
+
tester_check: null,
|
|
4518
|
+
tester_changes: null
|
|
3883
4519
|
};
|
|
3884
4520
|
}
|
|
3885
4521
|
/** What the builder of the iteration after `s` hears about it beyond the
|
|
@@ -3898,29 +4534,102 @@ function previous_round(config, s) {
|
|
|
3898
4534
|
* builder's work this way after every build, and a critique measures the
|
|
3899
4535
|
* change under review the same way, so the critic is shown one shape either
|
|
3900
4536
|
* way. */
|
|
3901
|
-
function measure_changes(deps, root) {
|
|
4537
|
+
function measure_changes(deps, root, tester_hashes = {}) {
|
|
3902
4538
|
const { config, renderer } = deps;
|
|
3903
|
-
const changes = collect_changes(
|
|
3904
|
-
root,
|
|
3905
|
-
baseline: deps.baseline,
|
|
3906
|
-
gate_patterns: config.gate_paths,
|
|
3907
|
-
excluded_dir: state_root_within(root, config.state_dir)
|
|
3908
|
-
});
|
|
4539
|
+
const changes = without_tester_edits(collect_changes(change_options(deps, root)), root, tester_hashes);
|
|
3909
4540
|
report_changes(config, renderer, changes);
|
|
3910
4541
|
return changes;
|
|
3911
4542
|
}
|
|
3912
|
-
/**
|
|
3913
|
-
*
|
|
3914
|
-
|
|
4543
|
+
/** How the build root's changes are collected: from the run's baseline,
|
|
4544
|
+
* against the gate, leaving out a state root inside it. */
|
|
4545
|
+
function change_options(deps, root) {
|
|
4546
|
+
return {
|
|
4547
|
+
root,
|
|
4548
|
+
baseline: deps.baseline,
|
|
4549
|
+
gate_patterns: deps.config.gate_paths,
|
|
4550
|
+
excluded_dir: state_root_within(root, deps.config.state_dir)
|
|
4551
|
+
};
|
|
4552
|
+
}
|
|
4553
|
+
/** The next build counted, with the rung it runs at and why that rung is
|
|
4554
|
+
* above the last build's, read off the state the last build left. */
|
|
4555
|
+
function climb(config, s, next) {
|
|
4556
|
+
const builds = s.builds + 1;
|
|
4557
|
+
return {
|
|
4558
|
+
...next,
|
|
4559
|
+
builds,
|
|
4560
|
+
rung: rung_for_build(config.effort_ladder, builds),
|
|
4561
|
+
climbed: climb_reason(config.effort_ladder, builds, s)
|
|
4562
|
+
};
|
|
4563
|
+
}
|
|
4564
|
+
/** `test`: the tester opens iteration 1 and writes its failing test, then
|
|
4565
|
+
* volley records what it wrote, by hash, and under `--git` checkpoints it, so
|
|
4566
|
+
* the build that follows is measured apart from it (move 1's D9, D22). */
|
|
4567
|
+
async function test_phase(deps, tester, s, ctx) {
|
|
3915
4568
|
const { config, renderer } = deps;
|
|
3916
4569
|
const next = next_iteration(s);
|
|
3917
4570
|
emit(ctx, {
|
|
3918
|
-
volley: "
|
|
4571
|
+
volley: "test_start",
|
|
3919
4572
|
round: next.iteration
|
|
3920
4573
|
});
|
|
4574
|
+
renderer.phase_start(next.iteration, "tester");
|
|
4575
|
+
const root = build_root(config.workspace, config.worktree);
|
|
4576
|
+
const before = snapshot_changes(change_options(deps, root));
|
|
4577
|
+
const tested = await run_tester(tester, config, next, ctx);
|
|
4578
|
+
renderer.phase_end(tested.iteration, "tester", true);
|
|
4579
|
+
renderer.cost_line(tested.iteration, "tester", tested.tester?.cost_usd ?? null, tested.total_cost_usd);
|
|
4580
|
+
if (tested.cost_warned && !s.cost_warned) renderer.warn("cost unavailable for this phase; recording null (run totals sum what is known)");
|
|
4581
|
+
const written = tester_written(change_options(deps, root), before);
|
|
4582
|
+
if (written.changes !== null) {
|
|
4583
|
+
const paths = written.changes.files.map((file) => file.path);
|
|
4584
|
+
renderer.info(paths.length === 0 ? "tester: wrote no file" : `tester: wrote ${String(paths.length)} file(s): ${paths.join(", ")}`);
|
|
4585
|
+
}
|
|
4586
|
+
if (config.git_checkpoints) git_checkpoint(root, `volley iter ${String(tested.iteration)}: test`, state_root_within(root, config.state_dir));
|
|
4587
|
+
return {
|
|
4588
|
+
...tested,
|
|
4589
|
+
tester_changes: written.changes,
|
|
4590
|
+
tester_hashes: written.hashes
|
|
4591
|
+
};
|
|
4592
|
+
}
|
|
4593
|
+
/** `red_check`: the check right after the tester, which must fail on its
|
|
4594
|
+
* test. It's recorded apart from the check after the build. A pass ends the
|
|
4595
|
+
* run before any build, at the loop guard (D6). */
|
|
4596
|
+
async function red_check_phase(deps, s, ctx) {
|
|
4597
|
+
const checked = await check_phase(deps, s, ctx);
|
|
4598
|
+
const result = checked.check;
|
|
4599
|
+
if (result?.ok === true) deps.renderer.warn("the check passed on the tester's test before any build: the test is vacuous, or the work is already done. Ending the run with no build (tester_passed_early, exit 2).");
|
|
4600
|
+
else deps.renderer.info("the check fails on the tester's test, as it must; building");
|
|
4601
|
+
return {
|
|
4602
|
+
...checked,
|
|
4603
|
+
check: null,
|
|
4604
|
+
tester_check: result
|
|
4605
|
+
};
|
|
4606
|
+
}
|
|
4607
|
+
/** `skip_red`: the tester alone crossed the cost cap, so the check is recorded
|
|
4608
|
+
* as skipped, and the guard halts the run on the cap with no build (D21). */
|
|
4609
|
+
function skip_red_phase(deps, s) {
|
|
4610
|
+
deps.renderer.warn("cost cap crossed during the tester; skipping the check, the build and the critic");
|
|
4611
|
+
return {
|
|
4612
|
+
...s,
|
|
4613
|
+
tester_check: skipped_check("cost_cap")
|
|
4614
|
+
};
|
|
4615
|
+
}
|
|
4616
|
+
/** `build`: one builder session at the build's rung edits the build root, then
|
|
4617
|
+
* volley measures what it changed, its tester's untouched test aside, and,
|
|
4618
|
+
* under `--git`, checkpoints it. It opens the iteration, unless the tester
|
|
4619
|
+
* already did. */
|
|
4620
|
+
async function build_phase(deps, builders, s, ctx) {
|
|
4621
|
+
const { config, renderer } = deps;
|
|
4622
|
+
const opened = tester_opened(s);
|
|
4623
|
+
const next = climb(config, s, opened ? s : next_iteration(s));
|
|
4624
|
+
emit(ctx, {
|
|
4625
|
+
volley: "round_start",
|
|
4626
|
+
round: next.iteration,
|
|
4627
|
+
...opened ? { after_test: true } : {}
|
|
4628
|
+
});
|
|
3921
4629
|
renderer.phase_start(next.iteration, "builder");
|
|
4630
|
+
if (next.rung !== null) renderer.rung(next.iteration, next.rung, next.climbed);
|
|
3922
4631
|
const built = await run_builder({
|
|
3923
|
-
builder,
|
|
4632
|
+
builder: builders.at(next.rung),
|
|
3924
4633
|
config,
|
|
3925
4634
|
warn: renderer.warn
|
|
3926
4635
|
}, next, previous_round(config, s), ctx);
|
|
@@ -3928,7 +4637,7 @@ async function build_phase(deps, builder, s, ctx) {
|
|
|
3928
4637
|
renderer.cost_line(built.iteration, "builder", built.builder?.cost_usd ?? null, built.total_cost_usd);
|
|
3929
4638
|
if (built.cost_warned && !s.cost_warned) renderer.warn("cost unavailable for this phase; recording null (run totals sum what is known)");
|
|
3930
4639
|
const root = build_root(config.workspace, config.worktree);
|
|
3931
|
-
const changes = measure_changes(deps, root);
|
|
4640
|
+
const changes = measure_changes(deps, root, built.tester_hashes);
|
|
3932
4641
|
if (config.git_checkpoints) git_checkpoint(root, `volley iter ${String(built.iteration)}: build`, state_root_within(root, config.state_dir));
|
|
3933
4642
|
return {
|
|
3934
4643
|
...built,
|
|
@@ -4077,6 +4786,12 @@ function record_phase(deps, s) {
|
|
|
4077
4786
|
* around the flow (workspace setup, the worktree lifecycle, `run`, the final
|
|
4078
4787
|
* summary) in `./orchestrator.ts`.
|
|
4079
4788
|
*
|
|
4789
|
+
* With `--tester on`, the run's first iteration opens with the test phase: the
|
|
4790
|
+
* tester writes a failing test, then the check must fail on it. A green check
|
|
4791
|
+
* there, or a tester that crossed the cost cap, holds the iteration with no
|
|
4792
|
+
* build, and the guard ends the run. Every other iteration, and every run with
|
|
4793
|
+
* the tester off, passes through `test` and `work` to today's build and verify.
|
|
4794
|
+
*
|
|
4080
4795
|
* A critique (`build_critique_flow`) is one round over a change already made,
|
|
4081
4796
|
* so it has no builder, no loop, and no guard: `measure`, which opens the
|
|
4082
4797
|
* round and measures the change from its baseline, then the same `check`,
|
|
@@ -4086,22 +4801,33 @@ function record_phase(deps, s) {
|
|
|
4086
4801
|
* `test/unit/flow_diagram.test.ts` holds it to the code: change a row by changing
|
|
4087
4802
|
* its step's `description`, then paste `pnpm diagram --prefix ' * '` over it.
|
|
4088
4803
|
*
|
|
4089
|
-
* volley
|
|
4090
|
-
* ├─ iteration
|
|
4091
|
-
* │ ├─
|
|
4092
|
-
* │ │
|
|
4093
|
-
* │ ├─
|
|
4094
|
-
* │ │
|
|
4804
|
+
* volley loop: one round per iteration, up to max_iterations
|
|
4805
|
+
* ├─ iteration sequence
|
|
4806
|
+
* │ ├─ test branch: does the tester open this iteration? (--tester on, the run's first)
|
|
4807
|
+
* │ │ ├─ then sequence
|
|
4808
|
+
* │ │ │ ├─ write_test open the iteration, run the tester, record what it wrote
|
|
4809
|
+
* │ │ │ │ └─ tester model call writing a failing test under the test and fixture gate paths
|
|
4810
|
+
* │ │ │ └─ red branch: did the tester alone cross the cost cap?
|
|
4811
|
+
* │ │ │ ├─ then skip_red record the check as skipped and build nothing
|
|
4812
|
+
* │ │ │ └─ else red_check the check, which must fail on the tester's test
|
|
4813
|
+
* │ │ └─ else no_test no tester this iteration
|
|
4814
|
+
* │ ├─ work branch: did the test phase end the run: a green check, or the cost cap?
|
|
4815
|
+
* │ │ ├─ then hold build nothing, and leave the run to the guard
|
|
4095
4816
|
* │ │ └─ else sequence
|
|
4096
|
-
* │ │ ├─
|
|
4097
|
-
* │ │ └─
|
|
4098
|
-
* │ │
|
|
4099
|
-
* │ │
|
|
4100
|
-
* │ │
|
|
4101
|
-
* │ │
|
|
4102
|
-
* │ │
|
|
4103
|
-
* │ └─
|
|
4104
|
-
*
|
|
4817
|
+
* │ │ ├─ build open the iteration unless the tester did, run the builder, measure the change
|
|
4818
|
+
* │ │ │ └─ builder model call: one agentic session in the build root
|
|
4819
|
+
* │ │ └─ verify branch: did the build alone cross the cost cap?
|
|
4820
|
+
* │ │ ├─ then skip_verify record the check as skipped and call no critic
|
|
4821
|
+
* │ │ └─ else sequence
|
|
4822
|
+
* │ │ ├─ check the deterministic gate: checkride, a command, or none
|
|
4823
|
+
* │ │ └─ critique the read-only critic's verdict and feedback
|
|
4824
|
+
* │ │ └─ critic fallback: judge without tools if a critic on any provider but claude_cli keeps dying or runs out of steps
|
|
4825
|
+
* │ │ ├─ retry one more try after a stream death, on any provider but claude_cli
|
|
4826
|
+
* │ │ │ └─ critic_tools model call with read-only workspace tools
|
|
4827
|
+
* │ │ └─ pipe mark the verdict degraded
|
|
4828
|
+
* │ │ └─ critic_toolless model call judging from the check output and a file list
|
|
4829
|
+
* │ └─ record archive the iteration and rewrite the run summary
|
|
4830
|
+
* └─ guard gate stop on approval over a green check, the cost cap, a gate edit, or a green check before any build
|
|
4105
4831
|
*/
|
|
4106
4832
|
/** The critic arm: the degradation ladder. A critic on any provider but
|
|
4107
4833
|
* `claude_cli` whose tool-bearing stream dies is retried at once (the death is
|
|
@@ -4143,17 +4869,52 @@ function critique_step(deps, seat) {
|
|
|
4143
4869
|
function record_step(deps) {
|
|
4144
4870
|
return step("record", (s) => record_phase(deps, s), { description: "archive the iteration and rewrite the run summary" });
|
|
4145
4871
|
}
|
|
4872
|
+
/** The test phase (move 1's D6, D17, D21): with the tester on, the run's
|
|
4873
|
+
* first iteration opens with the tester writing a failing test, then the
|
|
4874
|
+
* check, which must fail on it, unless the tester alone crossed the cost cap.
|
|
4875
|
+
* Every other iteration passes straight through. */
|
|
4876
|
+
function test_step(deps) {
|
|
4877
|
+
const { config } = deps;
|
|
4878
|
+
const tester = make_tester_step({
|
|
4879
|
+
...deps.seats.tester ?? { engine: deps.seats.critic.engine },
|
|
4880
|
+
config,
|
|
4881
|
+
on_chunk: deps.renderer.tester_chunk
|
|
4882
|
+
});
|
|
4883
|
+
const write_test = step("write_test", (s, ctx) => test_phase(deps, tester, s, ctx), {
|
|
4884
|
+
description: "open the iteration, run the tester, record what it wrote",
|
|
4885
|
+
arm: tester
|
|
4886
|
+
});
|
|
4887
|
+
const red = branch({
|
|
4888
|
+
name: "red",
|
|
4889
|
+
description: "did the tester alone cross the cost cap?",
|
|
4890
|
+
when: (s) => cost_cap_hit(config, s),
|
|
4891
|
+
then: step("skip_red", (s) => skip_red_phase(deps, s), { description: "record the check as skipped and build nothing" }),
|
|
4892
|
+
otherwise: step("red_check", (s, ctx) => red_check_phase(deps, s, ctx), { description: "the check, which must fail on the tester's test" })
|
|
4893
|
+
});
|
|
4894
|
+
return branch({
|
|
4895
|
+
name: "test",
|
|
4896
|
+
description: "does the tester open this iteration? (--tester on, the run's first)",
|
|
4897
|
+
when: (s) => tester_opens(config, s),
|
|
4898
|
+
then: sequence([write_test, red]),
|
|
4899
|
+
otherwise: step("no_test", (s) => s, { description: "no tester this iteration" })
|
|
4900
|
+
});
|
|
4901
|
+
}
|
|
4902
|
+
/** Whether the test phase ended the run before its build: the check passed on
|
|
4903
|
+
* the tester's test, or the tester crossed the cost cap. */
|
|
4904
|
+
function test_ended_run(config, s) {
|
|
4905
|
+
return tester_opened(s) && (tester_passed_early(s) || cost_cap_hit(config, s));
|
|
4906
|
+
}
|
|
4146
4907
|
function build_flow(deps, resume_from) {
|
|
4147
4908
|
const { config, renderer } = deps;
|
|
4148
|
-
const
|
|
4909
|
+
const builders = make_builder_ladder({
|
|
4149
4910
|
...deps.seats.builder,
|
|
4150
4911
|
config,
|
|
4151
4912
|
on_chunk: renderer.builder_chunk,
|
|
4152
4913
|
bash_executor: deps.bash_executor
|
|
4153
4914
|
});
|
|
4154
|
-
const build = step("build", (s, ctx) => build_phase(deps,
|
|
4155
|
-
description: "open the iteration, run the builder, measure the change",
|
|
4156
|
-
arm:
|
|
4915
|
+
const build = step("build", (s, ctx) => build_phase(deps, builders, s, ctx), {
|
|
4916
|
+
description: "open the iteration unless the tester did, run the builder, measure the change",
|
|
4917
|
+
arm: builders.arm
|
|
4157
4918
|
});
|
|
4158
4919
|
const skip_verify = step("skip_verify", (s) => skip_verify_phase(deps, s), { description: "record the check as skipped and call no critic" });
|
|
4159
4920
|
const check = check_step(deps);
|
|
@@ -4166,9 +4927,16 @@ function build_flow(deps, resume_from) {
|
|
|
4166
4927
|
then: skip_verify,
|
|
4167
4928
|
otherwise: sequence([check, critique])
|
|
4168
4929
|
});
|
|
4930
|
+
const work = branch({
|
|
4931
|
+
name: "work",
|
|
4932
|
+
description: "did the test phase end the run: a green check, or the cost cap?",
|
|
4933
|
+
when: (s) => test_ended_run(config, s),
|
|
4934
|
+
then: step("hold", (s) => s, { description: "build nothing, and leave the run to the guard" }),
|
|
4935
|
+
otherwise: sequence([build, verify])
|
|
4936
|
+
});
|
|
4169
4937
|
const iteration = sequence([
|
|
4170
|
-
|
|
4171
|
-
|
|
4938
|
+
test_step(deps),
|
|
4939
|
+
work,
|
|
4172
4940
|
record
|
|
4173
4941
|
], { name: "iteration" });
|
|
4174
4942
|
return loop({
|
|
@@ -4176,7 +4944,7 @@ function build_flow(deps, resume_from) {
|
|
|
4176
4944
|
description: "one round per iteration, up to max_iterations",
|
|
4177
4945
|
init: (input) => input.resume_from ?? initial_state(),
|
|
4178
4946
|
body: iteration,
|
|
4179
|
-
guard: step("gate", (s) => gate(config, s), { description: "stop on approval over a green check, the cost cap, or a
|
|
4947
|
+
guard: step("gate", (s) => gate(config, s), { description: "stop on approval over a green check, the cost cap, a gate edit, or a green check before any build" }),
|
|
4180
4948
|
finish: (s, { converged }) => ({
|
|
4181
4949
|
value: s,
|
|
4182
4950
|
converged
|
|
@@ -4274,6 +5042,10 @@ async function run_loop(config, deps, resume_from, bash_executor, baseline) {
|
|
|
4274
5042
|
critic: {
|
|
4275
5043
|
...deps.seats?.critic,
|
|
4276
5044
|
engine: engines.critic
|
|
5045
|
+
},
|
|
5046
|
+
tester: {
|
|
5047
|
+
...deps.seats?.tester,
|
|
5048
|
+
engine: engines.critic
|
|
4277
5049
|
}
|
|
4278
5050
|
},
|
|
4279
5051
|
config,
|
|
@@ -4402,12 +5174,28 @@ function create_renderer(options) {
|
|
|
4402
5174
|
const see_trajectory = `see ${options.trajectory_path ?? "the run's trajectory.jsonl"}`;
|
|
4403
5175
|
const silent = mode === "json";
|
|
4404
5176
|
const streaming = mode === "default" || mode === "verbose";
|
|
5177
|
+
const protocol = mode === "protocol";
|
|
4405
5178
|
let mid_line = false;
|
|
5179
|
+
let current_iteration = 0;
|
|
5180
|
+
let run_spend = 0;
|
|
4406
5181
|
const line = (text) => {
|
|
4407
5182
|
if (silent) return;
|
|
4408
5183
|
write(`${mid_line ? "\n" : ""}${text}\n`);
|
|
4409
5184
|
mid_line = false;
|
|
4410
5185
|
};
|
|
5186
|
+
/** One of runciter's typed stderr lines (its spec §5.4), which it reads into
|
|
5187
|
+
* the run's live state. */
|
|
5188
|
+
const typed = (value) => {
|
|
5189
|
+
line(JSON.stringify(value));
|
|
5190
|
+
};
|
|
5191
|
+
const progress = (n, message) => {
|
|
5192
|
+
typed({
|
|
5193
|
+
type: "runciter.progress",
|
|
5194
|
+
message: `iteration ${String(n)}: ${message}`,
|
|
5195
|
+
done: n - 1,
|
|
5196
|
+
...options.max_iterations !== void 0 ? { total: options.max_iterations } : {}
|
|
5197
|
+
});
|
|
5198
|
+
};
|
|
4411
5199
|
const chunk_renderer = (role) => (chunk) => {
|
|
4412
5200
|
if (!streaming) return;
|
|
4413
5201
|
switch (chunk.kind) {
|
|
@@ -4442,18 +5230,39 @@ function create_renderer(options) {
|
|
|
4442
5230
|
return {
|
|
4443
5231
|
builder_chunk: chunk_renderer("builder"),
|
|
4444
5232
|
critic_chunk: chunk_renderer("critic"),
|
|
4445
|
-
|
|
5233
|
+
tester_chunk: chunk_renderer("tester"),
|
|
5234
|
+
phase_start: (n, phase) => {
|
|
4446
5235
|
if (mode === "json") return;
|
|
4447
|
-
|
|
5236
|
+
if (protocol) {
|
|
5237
|
+
if (n > current_iteration && current_iteration > 0) typed({
|
|
5238
|
+
type: "runciter.usage",
|
|
5239
|
+
cost_usd: run_spend
|
|
5240
|
+
});
|
|
5241
|
+
current_iteration = Math.max(current_iteration, n);
|
|
5242
|
+
progress(n, phase);
|
|
5243
|
+
return;
|
|
5244
|
+
}
|
|
5245
|
+
line(paint(phase, `▶ [iter ${String(n)}] ${phase}`, color));
|
|
4448
5246
|
},
|
|
4449
|
-
phase_end: (
|
|
5247
|
+
phase_end: (n, phase, ok, detail) => {
|
|
4450
5248
|
if (mode === "json") return;
|
|
5249
|
+
if (protocol) {
|
|
5250
|
+
progress(n, `${phase} ${detail ?? (ok ? "done" : "failed")}`);
|
|
5251
|
+
return;
|
|
5252
|
+
}
|
|
4451
5253
|
const glyph = ok ? "✓" : "✗";
|
|
4452
5254
|
const suffix = detail !== void 0 ? ` — ${detail}` : "";
|
|
4453
|
-
line(paint(phase, `${glyph} [iter ${String(
|
|
5255
|
+
line(paint(phase, `${glyph} [iter ${String(n)}] ${phase}${suffix}`, color));
|
|
5256
|
+
},
|
|
5257
|
+
rung: (iteration, rung, climbed) => {
|
|
5258
|
+
line(paint("builder", ` [iter ${String(iteration)}] ${format_rung(rung, climbed)}`, color));
|
|
4454
5259
|
},
|
|
4455
5260
|
cost_line: (iteration, phase, phase_cost, run_total) => {
|
|
4456
5261
|
if (mode === "json") return;
|
|
5262
|
+
if (protocol) {
|
|
5263
|
+
run_spend = run_total;
|
|
5264
|
+
return;
|
|
5265
|
+
}
|
|
4457
5266
|
const cap = options.max_cost_usd !== null ? ` / ${format_usd(options.max_cost_usd)}` : "";
|
|
4458
5267
|
line(paint("cost", `${GLYPH_COST} [iter ${String(iteration)}] [${phase}] phase: ${format_usd(phase_cost)} | run: ${format_usd(run_total)}${cap}`, color));
|
|
4459
5268
|
},
|
|
@@ -4472,10 +5281,16 @@ function create_renderer(options) {
|
|
|
4472
5281
|
},
|
|
4473
5282
|
final_summary: (result) => {
|
|
4474
5283
|
if (silent) return;
|
|
5284
|
+
if (protocol) typed({
|
|
5285
|
+
type: "runciter.usage",
|
|
5286
|
+
cost_usd: result.total_cost_usd,
|
|
5287
|
+
input_tokens: result.total_usage.input_tokens,
|
|
5288
|
+
output_tokens: result.total_usage.output_tokens
|
|
5289
|
+
});
|
|
4475
5290
|
line("");
|
|
4476
5291
|
line(`run ${result.run_id}: ${result.status}`);
|
|
4477
5292
|
line(`iterations: ${String(result.iterations_completed)} | verdict: ${result.final_verdict ?? "n/a"}`);
|
|
4478
|
-
line(paint("cost", `${GLYPH_COST} total: ${format_usd(result.total_cost_usd)} (builder ${format_usd(result.builder_cost_usd)}, critic ${format_usd(result.critic_cost_usd)})`, color));
|
|
5293
|
+
line(paint("cost", `${GLYPH_COST} total: ${format_usd(result.total_cost_usd)} (builder ${format_usd(result.builder_cost_usd)}, critic ${format_usd(result.critic_cost_usd)}${result.tester_cost_usd > 0 ? `, tester ${format_usd(result.tester_cost_usd)}` : ""})`, color));
|
|
4479
5294
|
line(`tokens: ${String(result.total_usage.input_tokens)} in / ${String(result.total_usage.output_tokens)} out`);
|
|
4480
5295
|
line(`state: ${result.run_state_dir}`);
|
|
4481
5296
|
}
|
|
@@ -4679,7 +5494,8 @@ function exit_code_for_status(status) {
|
|
|
4679
5494
|
switch (status) {
|
|
4680
5495
|
case "success": return 0;
|
|
4681
5496
|
case "budget_exhausted":
|
|
4682
|
-
case "rejected":
|
|
5497
|
+
case "rejected":
|
|
5498
|
+
case "tester_passed_early": return 2;
|
|
4683
5499
|
case "cost_cap_reached": return 7;
|
|
4684
5500
|
case "gate_edit_blocked": return 8;
|
|
4685
5501
|
case "interrupted": return 130;
|