@cruxy/cli 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/approval/classify.js +21 -0
- package/dist/approval/policy.js +6 -0
- package/dist/approval/prompt.js +4 -2
- package/dist/approval/types.d.ts +5 -0
- package/dist/cli/commands/test.d.ts +9 -0
- package/dist/cli/commands/test.js +47 -0
- package/dist/cli/program.js +2 -0
- package/dist/cli/session-factory.js +6 -2
- package/dist/config/schema.d.ts +47 -0
- package/dist/config/schema.js +20 -0
- package/dist/errors/constructors.d.ts +5 -0
- package/dist/errors/constructors.js +16 -0
- package/dist/errors/types.d.ts +3 -0
- package/dist/errors/types.js +8 -0
- package/dist/testing/detect.d.ts +3 -0
- package/dist/testing/detect.js +44 -0
- package/dist/testing/index.d.ts +5 -0
- package/dist/testing/index.js +5 -0
- package/dist/testing/parse.d.ts +33 -0
- package/dist/testing/parse.js +137 -0
- package/dist/testing/run-tests-tool.d.ts +42 -0
- package/dist/testing/run-tests-tool.js +128 -0
- package/dist/testing/runner.d.ts +26 -0
- package/dist/testing/runner.js +124 -0
- package/dist/testing/types.d.ts +61 -0
- package/dist/testing/types.js +7 -0
- package/dist/tools/registry.js +3 -0
- package/dist/tools/types.d.ts +2 -2
- package/package.json +1 -1
|
@@ -16,6 +16,8 @@ export function classify(action, cwd) {
|
|
|
16
16
|
return fileRequest(action, patchHasDelete(action) ? "destructive" : "mutate", root);
|
|
17
17
|
case "shell":
|
|
18
18
|
return shellRequest(action, root);
|
|
19
|
+
case "test":
|
|
20
|
+
return testRequest(action, root);
|
|
19
21
|
case "vcs":
|
|
20
22
|
return vcsRequest(action, root);
|
|
21
23
|
case "rollback":
|
|
@@ -69,6 +71,25 @@ function shellRequest(action, root) {
|
|
|
69
71
|
cwd: root,
|
|
70
72
|
};
|
|
71
73
|
}
|
|
74
|
+
// ── test (run the project's test suite, C.13) ──────────────────────────────────
|
|
75
|
+
/**
|
|
76
|
+
* A test-suite execution. Destructive tier — a test script is arbitrary code
|
|
77
|
+
* from package.json — but grantable at the tightest possible scope: the exact
|
|
78
|
+
* command string. That is precisely what an edit→re-run iteration needs
|
|
79
|
+
* (approve once, re-run the same suite freely) without widening to a program
|
|
80
|
+
* prefix the way a shell grant would.
|
|
81
|
+
*/
|
|
82
|
+
function testRequest(action, root) {
|
|
83
|
+
const command = (action.command ?? "").trim();
|
|
84
|
+
return {
|
|
85
|
+
action,
|
|
86
|
+
tier: "destructive",
|
|
87
|
+
scope: command === "" ? { kind: "none" } : { kind: "shell-exact", command },
|
|
88
|
+
summary: `run tests: ${command}`,
|
|
89
|
+
targets: [],
|
|
90
|
+
cwd: root,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
72
93
|
// ── vcs (open pull request) ─────────────────────────────────────────────────────
|
|
73
94
|
/**
|
|
74
95
|
* A pull-request publish (C.15): branch → commit → push → open PR. Always
|
package/dist/approval/policy.js
CHANGED
|
@@ -38,6 +38,12 @@ export function scopeCovers(scope, request) {
|
|
|
38
38
|
const tokens = commandTokens(request.action.command ?? "");
|
|
39
39
|
return tokens !== null && tokens[0] === scope.token;
|
|
40
40
|
}
|
|
41
|
+
if (scope.kind === "shell-exact") {
|
|
42
|
+
// Test grants (C.13): the exact command string, test actions only — a
|
|
43
|
+
// grant for `pnpm test` can never cover run_command or any other command.
|
|
44
|
+
return (request.action.kind === "test" &&
|
|
45
|
+
(request.action.command ?? "").trim() === scope.command);
|
|
46
|
+
}
|
|
41
47
|
// file-subtree
|
|
42
48
|
return (request.targets.length > 0 &&
|
|
43
49
|
request.targets.every((t) => isInside(scope.root, t)));
|
package/dist/approval/prompt.js
CHANGED
|
@@ -45,9 +45,9 @@ export function render(request, color) {
|
|
|
45
45
|
lines.push(choices(request.scope, c));
|
|
46
46
|
return lines.filter((l) => l !== "").join("\n") + " ";
|
|
47
47
|
}
|
|
48
|
-
/** The action detail: a diff for file actions, the command + cwd for shell. */
|
|
48
|
+
/** The action detail: a diff for file actions, the command + cwd for shell/test. */
|
|
49
49
|
function detail(request, c) {
|
|
50
|
-
if (request.action.kind === "shell") {
|
|
50
|
+
if (request.action.kind === "shell" || request.action.kind === "test") {
|
|
51
51
|
return [
|
|
52
52
|
` ${c.dim("$")} ${request.action.command ?? ""}`,
|
|
53
53
|
` ${c.dim(`in ${request.cwd}`)}`,
|
|
@@ -67,6 +67,8 @@ function choices(scope, c) {
|
|
|
67
67
|
function scopeLabel(scope) {
|
|
68
68
|
if (scope.kind === "shell-prefix")
|
|
69
69
|
return `${scope.token} commands`;
|
|
70
|
+
if (scope.kind === "shell-exact")
|
|
71
|
+
return `re-runs of \`${scope.command}\``;
|
|
70
72
|
if (scope.kind === "file-subtree")
|
|
71
73
|
return `changes under ${path.basename(scope.root)}/`;
|
|
72
74
|
return null;
|
package/dist/approval/types.d.ts
CHANGED
|
@@ -17,6 +17,8 @@ export type RiskTier = "read" | "mutate" | "destructive";
|
|
|
17
17
|
* The tight scope a session grant is keyed by. Never blanket.
|
|
18
18
|
* - `shell-prefix` — a command's leading program token (e.g. `git`); only ever
|
|
19
19
|
* matches commands we can *positively* prove are simple (no shell features).
|
|
20
|
+
* - `shell-exact` — one exact command string, for `test` actions only (C.13):
|
|
21
|
+
* a grant covers re-runs of precisely that test command, nothing else.
|
|
20
22
|
* - `file-subtree` — an absolute directory (or, under the root-cap, an exact
|
|
21
23
|
* file path); matches targets that resolve inside it.
|
|
22
24
|
* - `none` — nothing safe to grant (e.g. a multi-file patch spanning the root).
|
|
@@ -24,6 +26,9 @@ export type RiskTier = "read" | "mutate" | "destructive";
|
|
|
24
26
|
export type Scope = {
|
|
25
27
|
readonly kind: "shell-prefix";
|
|
26
28
|
readonly token: string;
|
|
29
|
+
} | {
|
|
30
|
+
readonly kind: "shell-exact";
|
|
31
|
+
readonly command: string;
|
|
27
32
|
} | {
|
|
28
33
|
readonly kind: "file-subtree";
|
|
29
34
|
readonly root: string;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Command } from "commander";
|
|
2
|
+
/**
|
|
3
|
+
* `cruxy test` (C.13) — run the project's detected/configured test command
|
|
4
|
+
* once and print the structured result the agent would see. Directly
|
|
5
|
+
* user-invoked, so there is no approval gate (typing the command IS the
|
|
6
|
+
* consent — same as running the suite by hand); the process exit code mirrors
|
|
7
|
+
* the suite's pass/fail so scripts and CI can branch on it.
|
|
8
|
+
*/
|
|
9
|
+
export declare function testCommand(): Command;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { Command } from "commander";
|
|
2
|
+
import pc from "picocolors";
|
|
3
|
+
import { loadConfig } from "../../config/index.js";
|
|
4
|
+
import { testCommandNotFound } from "../../errors/index.js";
|
|
5
|
+
import { CommandTestRunner, detectTestCommand } from "../../testing/index.js";
|
|
6
|
+
import { logger } from "../../utils/logger.js";
|
|
7
|
+
/**
|
|
8
|
+
* `cruxy test` (C.13) — run the project's detected/configured test command
|
|
9
|
+
* once and print the structured result the agent would see. Directly
|
|
10
|
+
* user-invoked, so there is no approval gate (typing the command IS the
|
|
11
|
+
* consent — same as running the suite by hand); the process exit code mirrors
|
|
12
|
+
* the suite's pass/fail so scripts and CI can branch on it.
|
|
13
|
+
*/
|
|
14
|
+
export function testCommand() {
|
|
15
|
+
return new Command("test")
|
|
16
|
+
.description("run the project's test suite once and show the parsed result")
|
|
17
|
+
.action(async () => {
|
|
18
|
+
const { config } = loadConfig();
|
|
19
|
+
const cwd = process.cwd();
|
|
20
|
+
const detected = detectTestCommand(cwd, config);
|
|
21
|
+
if (detected === null)
|
|
22
|
+
throw testCommandNotFound();
|
|
23
|
+
logger.print(pc.dim(`running: ${detected.command} [${detected.source}]`));
|
|
24
|
+
const result = await new CommandTestRunner().run(detected.command, {
|
|
25
|
+
cwd,
|
|
26
|
+
timeoutMs: config.shell.timeoutMs,
|
|
27
|
+
captureBytes: config.test.captureBytes,
|
|
28
|
+
});
|
|
29
|
+
const seconds = (result.durationMs / 1000).toFixed(1);
|
|
30
|
+
if (result.passed) {
|
|
31
|
+
logger.print(`${pc.green("✓")} tests passed${result.total !== undefined ? ` (${result.total})` : ""} in ${seconds}s`);
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
logger.print(`${pc.red("✗")} tests failed (exit ${result.exitCode ?? "signal"}) in ${seconds}s`);
|
|
35
|
+
for (const failure of result.failures) {
|
|
36
|
+
const where = failure.file !== undefined
|
|
37
|
+
? pc.dim(` ${failure.file}${failure.line !== undefined ? `:${failure.line}` : ""}`)
|
|
38
|
+
: "";
|
|
39
|
+
logger.print(` ${pc.red("✗")} ${failure.name}${where}`);
|
|
40
|
+
}
|
|
41
|
+
if (result.failures.length === 0) {
|
|
42
|
+
// Nothing parseable — show the honest tail instead of fake structure.
|
|
43
|
+
logger.print(pc.dim(result.output.trimEnd()));
|
|
44
|
+
}
|
|
45
|
+
process.exitCode = 1;
|
|
46
|
+
});
|
|
47
|
+
}
|
package/dist/cli/program.js
CHANGED
|
@@ -12,6 +12,7 @@ import { loginCommand } from "./commands/login.js";
|
|
|
12
12
|
import { initCommand } from "./commands/init.js";
|
|
13
13
|
import { checkpointCommand } from "./commands/checkpoint.js";
|
|
14
14
|
import { rollbackCommand } from "./commands/rollback.js";
|
|
15
|
+
import { testCommand } from "./commands/test.js";
|
|
15
16
|
import { loadConfig } from "../config/index.js";
|
|
16
17
|
import { maybeRunOnboarding } from "./onboard.js";
|
|
17
18
|
export function buildProgram() {
|
|
@@ -40,6 +41,7 @@ export function buildProgram() {
|
|
|
40
41
|
program.addCommand(initCommand());
|
|
41
42
|
program.addCommand(checkpointCommand());
|
|
42
43
|
program.addCommand(rollbackCommand());
|
|
44
|
+
program.addCommand(testCommand());
|
|
43
45
|
// Default action: bare `cruxy` -> entrypoint. An unrecognized first operand
|
|
44
46
|
// means an unknown command (Commander runs the default action with it as an
|
|
45
47
|
// operand rather than erroring), so reject it as a usage error.
|
|
@@ -67,10 +67,14 @@ export function withCheckpointGate(requestApproval, checkpoints, cwd) {
|
|
|
67
67
|
if (request.tier === "read")
|
|
68
68
|
return decision;
|
|
69
69
|
await checkpoints.ensureCheckpoint();
|
|
70
|
-
|
|
70
|
+
// Shell AND test executions (C.13) can mutate files we can't attribute
|
|
71
|
+
// (scripts, snapshot writers) — record the lost attribution the same way.
|
|
72
|
+
if (action.kind === "shell" || action.kind === "test") {
|
|
71
73
|
await checkpoints.recordShellMutation();
|
|
72
|
-
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
73
76
|
await checkpoints.recordTouched([...request.targets]);
|
|
77
|
+
}
|
|
74
78
|
return decision;
|
|
75
79
|
};
|
|
76
80
|
}
|
package/dist/config/schema.d.ts
CHANGED
|
@@ -219,6 +219,27 @@ export declare const CheckpointConfigSchema: z.ZodObject<{
|
|
|
219
219
|
enabled?: boolean | undefined;
|
|
220
220
|
retention?: number | undefined;
|
|
221
221
|
}>;
|
|
222
|
+
/**
|
|
223
|
+
* Test-execution loop (C.13): how the agent runs the project's test suite and
|
|
224
|
+
* iterates on failures. The command is detected from package.json when unset;
|
|
225
|
+
* cruxy never invents one.
|
|
226
|
+
*/
|
|
227
|
+
export declare const TestConfigSchema: z.ZodObject<{
|
|
228
|
+
/** Explicit test command (overrides package.json detection). */
|
|
229
|
+
command: z.ZodOptional<z.ZodString>;
|
|
230
|
+
/** Consecutive failing runs before the edit→re-run loop trips its cap. */
|
|
231
|
+
maxIterations: z.ZodDefault<z.ZodNumber>;
|
|
232
|
+
/** Cap on captured test output bytes (tail-biased — failures come last). */
|
|
233
|
+
captureBytes: z.ZodDefault<z.ZodNumber>;
|
|
234
|
+
}, "strict", z.ZodTypeAny, {
|
|
235
|
+
maxIterations: number;
|
|
236
|
+
captureBytes: number;
|
|
237
|
+
command?: string | undefined;
|
|
238
|
+
}, {
|
|
239
|
+
maxIterations?: number | undefined;
|
|
240
|
+
command?: string | undefined;
|
|
241
|
+
captureBytes?: number | undefined;
|
|
242
|
+
}>;
|
|
222
243
|
/**
|
|
223
244
|
* Subagent orchestration (C.14): scoped child agents the main agent can spawn
|
|
224
245
|
* for bounded subtasks. Every cap here is a hard bound — a subagent can narrow
|
|
@@ -514,6 +535,22 @@ export declare const CruxyConfigSchema: z.ZodObject<{
|
|
|
514
535
|
timeoutMs?: number | undefined;
|
|
515
536
|
} | undefined;
|
|
516
537
|
}>>;
|
|
538
|
+
test: z.ZodDefault<z.ZodObject<{
|
|
539
|
+
/** Explicit test command (overrides package.json detection). */
|
|
540
|
+
command: z.ZodOptional<z.ZodString>;
|
|
541
|
+
/** Consecutive failing runs before the edit→re-run loop trips its cap. */
|
|
542
|
+
maxIterations: z.ZodDefault<z.ZodNumber>;
|
|
543
|
+
/** Cap on captured test output bytes (tail-biased — failures come last). */
|
|
544
|
+
captureBytes: z.ZodDefault<z.ZodNumber>;
|
|
545
|
+
}, "strict", z.ZodTypeAny, {
|
|
546
|
+
maxIterations: number;
|
|
547
|
+
captureBytes: number;
|
|
548
|
+
command?: string | undefined;
|
|
549
|
+
}, {
|
|
550
|
+
maxIterations?: number | undefined;
|
|
551
|
+
command?: string | undefined;
|
|
552
|
+
captureBytes?: number | undefined;
|
|
553
|
+
}>>;
|
|
517
554
|
mcpServers: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
518
555
|
command: z.ZodOptional<z.ZodString>;
|
|
519
556
|
args: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
@@ -592,6 +629,11 @@ export declare const CruxyConfigSchema: z.ZodObject<{
|
|
|
592
629
|
timeoutMs?: number | undefined;
|
|
593
630
|
};
|
|
594
631
|
};
|
|
632
|
+
test: {
|
|
633
|
+
maxIterations: number;
|
|
634
|
+
captureBytes: number;
|
|
635
|
+
command?: string | undefined;
|
|
636
|
+
};
|
|
595
637
|
mcpServers: Record<string, {
|
|
596
638
|
command?: string | undefined;
|
|
597
639
|
args?: string[] | undefined;
|
|
@@ -662,6 +704,11 @@ export declare const CruxyConfigSchema: z.ZodObject<{
|
|
|
662
704
|
timeoutMs?: number | undefined;
|
|
663
705
|
} | undefined;
|
|
664
706
|
} | undefined;
|
|
707
|
+
test?: {
|
|
708
|
+
maxIterations?: number | undefined;
|
|
709
|
+
command?: string | undefined;
|
|
710
|
+
captureBytes?: number | undefined;
|
|
711
|
+
} | undefined;
|
|
665
712
|
mcpServers?: Record<string, {
|
|
666
713
|
command?: string | undefined;
|
|
667
714
|
args?: string[] | undefined;
|
package/dist/config/schema.js
CHANGED
|
@@ -149,6 +149,25 @@ export const CheckpointConfigSchema = z
|
|
|
149
149
|
retention: z.number().int().positive().default(10),
|
|
150
150
|
})
|
|
151
151
|
.strict();
|
|
152
|
+
/**
|
|
153
|
+
* Test-execution loop (C.13): how the agent runs the project's test suite and
|
|
154
|
+
* iterates on failures. The command is detected from package.json when unset;
|
|
155
|
+
* cruxy never invents one.
|
|
156
|
+
*/
|
|
157
|
+
export const TestConfigSchema = z
|
|
158
|
+
.object({
|
|
159
|
+
/** Explicit test command (overrides package.json detection). */
|
|
160
|
+
command: z.string().min(1).optional(),
|
|
161
|
+
/** Consecutive failing runs before the edit→re-run loop trips its cap. */
|
|
162
|
+
maxIterations: z.number().int().positive().default(4),
|
|
163
|
+
/** Cap on captured test output bytes (tail-biased — failures come last). */
|
|
164
|
+
captureBytes: z
|
|
165
|
+
.number()
|
|
166
|
+
.int()
|
|
167
|
+
.positive()
|
|
168
|
+
.default(64 * 1024),
|
|
169
|
+
})
|
|
170
|
+
.strict();
|
|
152
171
|
/**
|
|
153
172
|
* Subagent orchestration (C.14): scoped child agents the main agent can spawn
|
|
154
173
|
* for bounded subtasks. Every cap here is a hard bound — a subagent can narrow
|
|
@@ -196,6 +215,7 @@ export const CruxyConfigSchema = z
|
|
|
196
215
|
index: IndexConfigSchema.default({}),
|
|
197
216
|
checkpoint: CheckpointConfigSchema.default({}),
|
|
198
217
|
subagent: SubagentConfigSchema.default({}),
|
|
218
|
+
test: TestConfigSchema.default({}),
|
|
199
219
|
mcpServers: z.record(z.string(), McpServerSchema).default({}),
|
|
200
220
|
logLevel: z.enum(LOG_LEVELS).default("info"),
|
|
201
221
|
})
|
|
@@ -85,6 +85,11 @@ export declare function subagentDepthExceeded(depth: number, maxDepth: number):
|
|
|
85
85
|
* reasons over; thrown only when the orchestrator itself cannot proceed.
|
|
86
86
|
*/
|
|
87
87
|
export declare function subagentFailed(underlying?: unknown): CruxyError;
|
|
88
|
+
/**
|
|
89
|
+
* No test command could be detected and none is configured (C.13). cruxy never
|
|
90
|
+
* invents a test command — the fix is always to declare one.
|
|
91
|
+
*/
|
|
92
|
+
export declare function testCommandNotFound(): CruxyError;
|
|
88
93
|
export declare function internal(underlying?: unknown): CruxyError;
|
|
89
94
|
/**
|
|
90
95
|
* Map a known provider/transport error (from `@cruxy/sdk`) to a typed
|
|
@@ -436,6 +436,22 @@ export function subagentFailed(underlying) {
|
|
|
436
436
|
underlying,
|
|
437
437
|
});
|
|
438
438
|
}
|
|
439
|
+
// ── testing (exit 2) ──────────────────────────────────────────────────────────
|
|
440
|
+
/**
|
|
441
|
+
* No test command could be detected and none is configured (C.13). cruxy never
|
|
442
|
+
* invents a test command — the fix is always to declare one.
|
|
443
|
+
*/
|
|
444
|
+
export function testCommandNotFound() {
|
|
445
|
+
return new CruxyError({
|
|
446
|
+
code: ErrorCode.TestCommandNotFound,
|
|
447
|
+
title: "no test command found for this project",
|
|
448
|
+
cause: "package.json has no usable `scripts.test` and `test.command` is not configured",
|
|
449
|
+
nextSteps: [
|
|
450
|
+
'set `test.command` in your cruxy config (e.g. `cruxy config set test.command "pnpm test"`)',
|
|
451
|
+
"or add a `test` script to package.json",
|
|
452
|
+
],
|
|
453
|
+
});
|
|
454
|
+
}
|
|
439
455
|
// ── internal (exit 1) ─────────────────────────────────────────────────────────
|
|
440
456
|
export function internal(underlying) {
|
|
441
457
|
return new CruxyError({
|
package/dist/errors/types.d.ts
CHANGED
|
@@ -49,6 +49,9 @@ export declare const ErrorCode: {
|
|
|
49
49
|
/** Carried inside a SubagentResult (informational) — never fatal by itself. */
|
|
50
50
|
readonly SubagentBudget: "CRUXY_E_SUBAGENT_BUDGET";
|
|
51
51
|
readonly SubagentFailed: "CRUXY_E_SUBAGENT_FAILED";
|
|
52
|
+
readonly TestCommandNotFound: "CRUXY_E_TEST_COMMAND_NOT_FOUND";
|
|
53
|
+
/** Carried inside a run_tests result (informational) — never fatal by itself. */
|
|
54
|
+
readonly TestIterationLimit: "CRUXY_E_TEST_ITERATION_LIMIT";
|
|
52
55
|
};
|
|
53
56
|
export type ErrorCode = (typeof ErrorCode)[keyof typeof ErrorCode];
|
|
54
57
|
/** The process exit code for an error code (defaults to 1 for safety). */
|
package/dist/errors/types.js
CHANGED
|
@@ -60,6 +60,10 @@ export const ErrorCode = {
|
|
|
60
60
|
/** Carried inside a SubagentResult (informational) — never fatal by itself. */
|
|
61
61
|
SubagentBudget: "CRUXY_E_SUBAGENT_BUDGET",
|
|
62
62
|
SubagentFailed: "CRUXY_E_SUBAGENT_FAILED",
|
|
63
|
+
// testing (exit 2 / 11)
|
|
64
|
+
TestCommandNotFound: "CRUXY_E_TEST_COMMAND_NOT_FOUND",
|
|
65
|
+
/** Carried inside a run_tests result (informational) — never fatal by itself. */
|
|
66
|
+
TestIterationLimit: "CRUXY_E_TEST_ITERATION_LIMIT",
|
|
63
67
|
};
|
|
64
68
|
/**
|
|
65
69
|
* Category exit codes. Distinct per category so a caller (CI, a script) can
|
|
@@ -104,6 +108,10 @@ const EXIT_CODES = {
|
|
|
104
108
|
[ErrorCode.SubagentDepthExceeded]: 2,
|
|
105
109
|
[ErrorCode.SubagentBudget]: 11,
|
|
106
110
|
[ErrorCode.SubagentFailed]: 11,
|
|
111
|
+
// No test command is a configuration gap (usage); the iteration limit
|
|
112
|
+
// surfaces inside a run_tests result and is never fatal by itself.
|
|
113
|
+
[ErrorCode.TestCommandNotFound]: 2,
|
|
114
|
+
[ErrorCode.TestIterationLimit]: 11,
|
|
107
115
|
};
|
|
108
116
|
/** The process exit code for an error code (defaults to 1 for safety). */
|
|
109
117
|
export function exitCodeFor(code) {
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
/**
|
|
4
|
+
* Resolve the project's test command (C.13). Order: explicit `test.command`
|
|
5
|
+
* config, then package.json `scripts.test`. Returns `null` when neither
|
|
6
|
+
* yields one — cruxy NEVER invents a test command; the caller surfaces the
|
|
7
|
+
* coded not-configured error instead.
|
|
8
|
+
*/
|
|
9
|
+
/** npm's scaffold placeholder is an error message, not a test suite. */
|
|
10
|
+
const NPM_PLACEHOLDER = /no test specified/i;
|
|
11
|
+
export function detectTestCommand(cwd, config) {
|
|
12
|
+
if (config.test.command) {
|
|
13
|
+
return { command: config.test.command, source: "config" };
|
|
14
|
+
}
|
|
15
|
+
const script = readTestScript(cwd);
|
|
16
|
+
if (script === null)
|
|
17
|
+
return null;
|
|
18
|
+
return { command: `${packageManager(cwd)} test`, source: "package-json" };
|
|
19
|
+
}
|
|
20
|
+
/** The package.json `scripts.test` value, or null if absent/placeholder/unreadable. */
|
|
21
|
+
function readTestScript(cwd) {
|
|
22
|
+
try {
|
|
23
|
+
const raw = readFileSync(path.join(cwd, "package.json"), "utf8");
|
|
24
|
+
const pkg = JSON.parse(raw);
|
|
25
|
+
const script = pkg.scripts?.test;
|
|
26
|
+
if (typeof script !== "string" || script.trim() === "")
|
|
27
|
+
return null;
|
|
28
|
+
if (NPM_PLACEHOLDER.test(script))
|
|
29
|
+
return null;
|
|
30
|
+
return script;
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
// No package.json / unparseable → not detected (never a crash).
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
/** Pick the package manager by lockfile; npm when nothing identifies one. */
|
|
38
|
+
function packageManager(cwd) {
|
|
39
|
+
if (existsSync(path.join(cwd, "pnpm-lock.yaml")))
|
|
40
|
+
return "pnpm";
|
|
41
|
+
if (existsSync(path.join(cwd, "yarn.lock")))
|
|
42
|
+
return "yarn";
|
|
43
|
+
return "npm";
|
|
44
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { FailureParser, TestFailure } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Best-effort failure extraction (C.13). Two conservative parsers ship —
|
|
4
|
+
* vitest-style and jest-style — behind the pluggable {@link FailureParser}
|
|
5
|
+
* seam. The contract: extract only what a pattern positively recognizes;
|
|
6
|
+
* when nothing matches, return NOTHING (the caller falls back to the raw
|
|
7
|
+
* tail). Parsers never decide pass/fail and never invent counts — `message`
|
|
8
|
+
* fields are verbatim runner output, not summaries we authored.
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* Vitest: per-test failure lines (`FAIL src/x.test.ts > suite > name`, also
|
|
12
|
+
* `×`/`✗` markers) and the `Tests 2 failed | 570 passed (572)` summary.
|
|
13
|
+
* Plain file-level `FAIL <file>` lines are deliberately left to the jest
|
|
14
|
+
* parser, which owns the file+bullet association.
|
|
15
|
+
*/
|
|
16
|
+
export declare const parseVitest: FailureParser;
|
|
17
|
+
/**
|
|
18
|
+
* Jest: `FAIL <file>` headers with `● <name>` bullets underneath (the bullet's
|
|
19
|
+
* following indented lines are its message, verbatim), and the
|
|
20
|
+
* `Tests: …, N total` summary. A FAIL header with no bullets (e.g. a suite
|
|
21
|
+
* that failed to load) becomes one file-level failure.
|
|
22
|
+
*/
|
|
23
|
+
export declare const parseJest: FailureParser;
|
|
24
|
+
/** Parser order: most-specific first. The pluggable seam for new frameworks. */
|
|
25
|
+
export declare const defaultParsers: readonly FailureParser[];
|
|
26
|
+
/**
|
|
27
|
+
* Run the parser chain; the first parser that recognizes anything wins.
|
|
28
|
+
* Nothing recognized → empty failures, no total — the raw tail is the result.
|
|
29
|
+
*/
|
|
30
|
+
export declare function parseFailures(output: string): {
|
|
31
|
+
failures: TestFailure[];
|
|
32
|
+
total?: number;
|
|
33
|
+
};
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Best-effort failure extraction (C.13). Two conservative parsers ship —
|
|
3
|
+
* vitest-style and jest-style — behind the pluggable {@link FailureParser}
|
|
4
|
+
* seam. The contract: extract only what a pattern positively recognizes;
|
|
5
|
+
* when nothing matches, return NOTHING (the caller falls back to the raw
|
|
6
|
+
* tail). Parsers never decide pass/fail and never invent counts — `message`
|
|
7
|
+
* fields are verbatim runner output, not summaries we authored.
|
|
8
|
+
*/
|
|
9
|
+
/**
|
|
10
|
+
* Vitest: per-test failure lines (`FAIL src/x.test.ts > suite > name`, also
|
|
11
|
+
* `×`/`✗` markers) and the `Tests 2 failed | 570 passed (572)` summary.
|
|
12
|
+
* Plain file-level `FAIL <file>` lines are deliberately left to the jest
|
|
13
|
+
* parser, which owns the file+bullet association.
|
|
14
|
+
*/
|
|
15
|
+
export const parseVitest = (output) => {
|
|
16
|
+
const failures = [];
|
|
17
|
+
for (const line of output.split("\n")) {
|
|
18
|
+
const match = /^\s*(?:FAIL|✗|×)\s+(\S+)\s+>\s+(.+?)\s*$/.exec(line);
|
|
19
|
+
if (match) {
|
|
20
|
+
failures.push({
|
|
21
|
+
name: match[2],
|
|
22
|
+
message: line.trim(),
|
|
23
|
+
file: match[1],
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
attachLines(failures, output);
|
|
28
|
+
const total = /^\s*Tests\s+.*\((\d+)\)\s*$/m.exec(output);
|
|
29
|
+
return {
|
|
30
|
+
failures,
|
|
31
|
+
...(total ? { total: Number(total[1]) } : {}),
|
|
32
|
+
};
|
|
33
|
+
};
|
|
34
|
+
/**
|
|
35
|
+
* Jest: `FAIL <file>` headers with `● <name>` bullets underneath (the bullet's
|
|
36
|
+
* following indented lines are its message, verbatim), and the
|
|
37
|
+
* `Tests: …, N total` summary. A FAIL header with no bullets (e.g. a suite
|
|
38
|
+
* that failed to load) becomes one file-level failure.
|
|
39
|
+
*/
|
|
40
|
+
export const parseJest = (output) => {
|
|
41
|
+
const failures = [];
|
|
42
|
+
const lines = output.split("\n");
|
|
43
|
+
let currentFile;
|
|
44
|
+
const filesWithBullets = new Set();
|
|
45
|
+
const bareFiles = [];
|
|
46
|
+
for (let i = 0; i < lines.length; i++) {
|
|
47
|
+
const fail = /^\s*FAIL\s+(\S+)\s*$/.exec(lines[i]);
|
|
48
|
+
if (fail) {
|
|
49
|
+
currentFile = fail[1];
|
|
50
|
+
bareFiles.push(fail[1]);
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
const bullet = /^\s*●\s+(.+?)\s*$/.exec(lines[i]);
|
|
54
|
+
if (bullet) {
|
|
55
|
+
failures.push({
|
|
56
|
+
name: bullet[1],
|
|
57
|
+
message: bulletMessage(lines, i),
|
|
58
|
+
...(currentFile ? { file: currentFile } : {}),
|
|
59
|
+
});
|
|
60
|
+
if (currentFile)
|
|
61
|
+
filesWithBullets.add(currentFile);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
// A failed suite with no per-test bullets is still one honest failure.
|
|
65
|
+
for (const file of bareFiles) {
|
|
66
|
+
if (!filesWithBullets.has(file)) {
|
|
67
|
+
failures.push({ name: file, message: `FAIL ${file}`, file });
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
attachLines(failures, output);
|
|
71
|
+
const total = /^Tests:.*?(\d+)\s+total\s*$/m.exec(output);
|
|
72
|
+
return {
|
|
73
|
+
failures,
|
|
74
|
+
...(total ? { total: Number(total[1]) } : {}),
|
|
75
|
+
};
|
|
76
|
+
};
|
|
77
|
+
/**
|
|
78
|
+
* Up to three non-blank lines under a jest bullet — its verbatim message.
|
|
79
|
+
* Jest separates the bullet from its detail with a blank line, so leading
|
|
80
|
+
* blanks are skipped; collection stops at the next blank, the next bullet,
|
|
81
|
+
* or the cap.
|
|
82
|
+
*/
|
|
83
|
+
function bulletMessage(lines, bulletIndex) {
|
|
84
|
+
const body = [];
|
|
85
|
+
for (let j = bulletIndex + 1; j < lines.length && body.length < 3; j++) {
|
|
86
|
+
const text = lines[j].trim();
|
|
87
|
+
if (text === "") {
|
|
88
|
+
if (body.length === 0)
|
|
89
|
+
continue; // the separator blank under the bullet
|
|
90
|
+
break;
|
|
91
|
+
}
|
|
92
|
+
if (/^\s*●\s+/.test(lines[j]))
|
|
93
|
+
break;
|
|
94
|
+
body.push(text);
|
|
95
|
+
}
|
|
96
|
+
return body.length > 0 ? body.join("\n") : lines[bulletIndex].trim();
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Attach `line` to failures whose file appears in a `file:line:col` stack
|
|
100
|
+
* reference anywhere in the output (vitest `❯ file:39:5`, jest
|
|
101
|
+
* `at … (file:12:15)`). First reference per file wins; no reference → no line.
|
|
102
|
+
*/
|
|
103
|
+
function attachLines(failures, output) {
|
|
104
|
+
if (failures.length === 0)
|
|
105
|
+
return;
|
|
106
|
+
const firstLineFor = new Map();
|
|
107
|
+
for (const match of output.matchAll(/([^\s():]+):(\d+):\d+/g)) {
|
|
108
|
+
if (!firstLineFor.has(match[1])) {
|
|
109
|
+
firstLineFor.set(match[1], Number(match[2]));
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
for (const failure of failures) {
|
|
113
|
+
if (failure.file === undefined || failure.line !== undefined)
|
|
114
|
+
continue;
|
|
115
|
+
const line = firstLineFor.get(failure.file);
|
|
116
|
+
if (line !== undefined)
|
|
117
|
+
failure.line = line;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
/** Parser order: most-specific first. The pluggable seam for new frameworks. */
|
|
121
|
+
export const defaultParsers = [
|
|
122
|
+
parseVitest,
|
|
123
|
+
parseJest,
|
|
124
|
+
];
|
|
125
|
+
/**
|
|
126
|
+
* Run the parser chain; the first parser that recognizes anything wins.
|
|
127
|
+
* Nothing recognized → empty failures, no total — the raw tail is the result.
|
|
128
|
+
*/
|
|
129
|
+
export function parseFailures(output) {
|
|
130
|
+
for (const parser of defaultParsers) {
|
|
131
|
+
const result = parser(output);
|
|
132
|
+
if (result.failures.length > 0 || result.total !== undefined) {
|
|
133
|
+
return result;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return { failures: [] };
|
|
137
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { Tool, ToolContext } from "../tools/types.js";
|
|
3
|
+
import type { TestCommand, TestRunner } from "./types.js";
|
|
4
|
+
/**
|
|
5
|
+
* The `run_tests` tool (C.13): execute the project's test suite and return a
|
|
6
|
+
* structured result the model can iterate on (edit → re-run → repeat). The
|
|
7
|
+
* iteration loop itself is the ordinary agent loop; this module contributes
|
|
8
|
+
* the two guarantees that make it trustworthy — honest green (exit-code-only)
|
|
9
|
+
* and a hard cap on consecutive failing runs.
|
|
10
|
+
*/
|
|
11
|
+
/**
|
|
12
|
+
* Counts consecutive FAILING test executions; a green run resets it. When the
|
|
13
|
+
* count reaches the cap, the next attempt is refused with the coded
|
|
14
|
+
* CRUXY_E_TEST_ITERATION_LIMIT result (nothing executes), and the counter
|
|
15
|
+
* resets — so the episode ends loudly, but a deliberate later attempt (a new
|
|
16
|
+
* user instruction) starts fresh rather than finding a permanently dead tool.
|
|
17
|
+
* Per-turn work stays bounded regardless via `agent.maxIterations`.
|
|
18
|
+
*/
|
|
19
|
+
export declare class TestIterationBudget {
|
|
20
|
+
private failedRuns;
|
|
21
|
+
/** Runs already spent in the current failing streak. */
|
|
22
|
+
get spent(): number;
|
|
23
|
+
/** True when the next run must be refused; resets the streak as it trips. */
|
|
24
|
+
trip(maxIterations: number): boolean;
|
|
25
|
+
record(passed: boolean): void;
|
|
26
|
+
}
|
|
27
|
+
declare const parameters: z.ZodObject<{
|
|
28
|
+
command: z.ZodOptional<z.ZodString>;
|
|
29
|
+
}, "strip", z.ZodTypeAny, {
|
|
30
|
+
command?: string | undefined;
|
|
31
|
+
}, {
|
|
32
|
+
command?: string | undefined;
|
|
33
|
+
}>;
|
|
34
|
+
export interface RunTestsToolDeps {
|
|
35
|
+
/** Execution seam (tests inject a fake; default spawns the real command). */
|
|
36
|
+
runner?: TestRunner;
|
|
37
|
+
/** Detection seam (defaults to config + package.json detection). */
|
|
38
|
+
detect?: (ctx: ToolContext) => TestCommand | null;
|
|
39
|
+
}
|
|
40
|
+
/** Build the `run_tests` tool. One instance = one session's iteration budget. */
|
|
41
|
+
export declare function makeRunTestsTool(deps?: RunTestsToolDeps): Tool<typeof parameters>;
|
|
42
|
+
export {};
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { ErrorCode } from "../errors/index.js";
|
|
3
|
+
import { detectTestCommand } from "./detect.js";
|
|
4
|
+
import { CommandTestRunner } from "./runner.js";
|
|
5
|
+
/**
|
|
6
|
+
* The `run_tests` tool (C.13): execute the project's test suite and return a
|
|
7
|
+
* structured result the model can iterate on (edit → re-run → repeat). The
|
|
8
|
+
* iteration loop itself is the ordinary agent loop; this module contributes
|
|
9
|
+
* the two guarantees that make it trustworthy — honest green (exit-code-only)
|
|
10
|
+
* and a hard cap on consecutive failing runs.
|
|
11
|
+
*/
|
|
12
|
+
/**
|
|
13
|
+
* Counts consecutive FAILING test executions; a green run resets it. When the
|
|
14
|
+
* count reaches the cap, the next attempt is refused with the coded
|
|
15
|
+
* CRUXY_E_TEST_ITERATION_LIMIT result (nothing executes), and the counter
|
|
16
|
+
* resets — so the episode ends loudly, but a deliberate later attempt (a new
|
|
17
|
+
* user instruction) starts fresh rather than finding a permanently dead tool.
|
|
18
|
+
* Per-turn work stays bounded regardless via `agent.maxIterations`.
|
|
19
|
+
*/
|
|
20
|
+
export class TestIterationBudget {
|
|
21
|
+
failedRuns = 0;
|
|
22
|
+
/** Runs already spent in the current failing streak. */
|
|
23
|
+
get spent() {
|
|
24
|
+
return this.failedRuns;
|
|
25
|
+
}
|
|
26
|
+
/** True when the next run must be refused; resets the streak as it trips. */
|
|
27
|
+
trip(maxIterations) {
|
|
28
|
+
if (this.failedRuns < maxIterations)
|
|
29
|
+
return false;
|
|
30
|
+
this.failedRuns = 0;
|
|
31
|
+
return true;
|
|
32
|
+
}
|
|
33
|
+
record(passed) {
|
|
34
|
+
this.failedRuns = passed ? 0 : this.failedRuns + 1;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
const parameters = z.object({
|
|
38
|
+
command: z
|
|
39
|
+
.string()
|
|
40
|
+
.min(1)
|
|
41
|
+
.optional()
|
|
42
|
+
.describe("Override the detected test command (still requires approval). " +
|
|
43
|
+
"Omit to use the project's configured/detected command."),
|
|
44
|
+
});
|
|
45
|
+
/** The wire shape fed back to the model — structured, bounded, honest. */
|
|
46
|
+
function renderResult(result, command, iteration) {
|
|
47
|
+
return JSON.stringify({
|
|
48
|
+
passed: result.passed,
|
|
49
|
+
exitCode: result.exitCode,
|
|
50
|
+
durationMs: result.durationMs,
|
|
51
|
+
command: command.command,
|
|
52
|
+
...(result.total !== undefined ? { total: result.total } : {}),
|
|
53
|
+
failures: result.failures,
|
|
54
|
+
iteration,
|
|
55
|
+
output: result.output,
|
|
56
|
+
outputTruncated: result.outputTruncated,
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
/** Build the `run_tests` tool. One instance = one session's iteration budget. */
|
|
60
|
+
export function makeRunTestsTool(deps = {}) {
|
|
61
|
+
const runner = deps.runner ?? new CommandTestRunner();
|
|
62
|
+
const detect = deps.detect ??
|
|
63
|
+
((ctx) => detectTestCommand(ctx.cwd, ctx.config));
|
|
64
|
+
const budget = new TestIterationBudget();
|
|
65
|
+
return {
|
|
66
|
+
name: "run_tests",
|
|
67
|
+
description: "Run the project's test suite and get a structured result: passed (from the exit code), " +
|
|
68
|
+
"extracted failures with file/line where recognizable, and the output tail. " +
|
|
69
|
+
"Use it to verify changes: run, read the failures, fix, re-run. The edit→re-run loop is " +
|
|
70
|
+
"capped — when the iteration limit trips, stop, summarize the remaining failures, and ask the user.",
|
|
71
|
+
parameters,
|
|
72
|
+
async execute(input, ctx) {
|
|
73
|
+
// Resolve the command first: detection failure needs no approval and
|
|
74
|
+
// must be a coded, actionable error — never an invented command.
|
|
75
|
+
let resolved;
|
|
76
|
+
if (input.command !== undefined) {
|
|
77
|
+
resolved = { command: input.command, source: "override" };
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
const detected = detect(ctx);
|
|
81
|
+
if (detected === null)
|
|
82
|
+
return { ok: false, error: NOT_FOUND };
|
|
83
|
+
resolved = detected;
|
|
84
|
+
}
|
|
85
|
+
// Iteration cap BEFORE the gate and the run: a refused attempt executes
|
|
86
|
+
// nothing and costs nothing.
|
|
87
|
+
const max = ctx.config.test.maxIterations;
|
|
88
|
+
if (budget.trip(max)) {
|
|
89
|
+
return {
|
|
90
|
+
ok: false,
|
|
91
|
+
error: `${ErrorCode.TestIterationLimit}: tests are still failing after ${max} run${max === 1 ? "" : "s"}. ` +
|
|
92
|
+
"Stop iterating. Summarize the remaining failures and what you tried, then ask the user how to proceed.",
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
// U.3 gate (destructive tier, exact-command grant scope). A thrown
|
|
96
|
+
// CRUXY_E_APPROVAL_REQUIRED (non-interactive) propagates — do not catch.
|
|
97
|
+
const decision = await ctx.requestApproval({
|
|
98
|
+
kind: "test",
|
|
99
|
+
command: resolved.command,
|
|
100
|
+
});
|
|
101
|
+
if (!decision.allow) {
|
|
102
|
+
return {
|
|
103
|
+
ok: false,
|
|
104
|
+
error: decision.feedback ?? "test run denied by the user",
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
const result = await runner.run(resolved.command, {
|
|
108
|
+
cwd: ctx.cwd,
|
|
109
|
+
timeoutMs: ctx.config.shell.timeoutMs,
|
|
110
|
+
captureBytes: ctx.config.test.captureBytes,
|
|
111
|
+
});
|
|
112
|
+
budget.record(result.passed);
|
|
113
|
+
const payload = renderResult(result, resolved, {
|
|
114
|
+
run: result.passed ? 0 : budget.spent,
|
|
115
|
+
max,
|
|
116
|
+
});
|
|
117
|
+
// Failing tests are an is_error result so the U.4 trail note reads ✗ —
|
|
118
|
+
// the payload is identical either way; the model reasons over both.
|
|
119
|
+
return result.passed
|
|
120
|
+
? { ok: true, output: payload }
|
|
121
|
+
: { ok: false, error: payload };
|
|
122
|
+
},
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
/** The coded not-configured message, actionable for model and user alike. */
|
|
126
|
+
const NOT_FOUND = `${ErrorCode.TestCommandNotFound}: no test command found — package.json has no usable ` +
|
|
127
|
+
"`scripts.test` and `test.command` is not configured. Ask the user to set `test.command` " +
|
|
128
|
+
'(e.g. `cruxy config set test.command "pnpm test"`); do not guess a command.';
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { TestRunner, TestRunOptions, TestRunResult } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* The shipped {@link TestRunner}: spawn the command via the system shell (the
|
|
4
|
+
* same detached-group + kill-tree discipline as `run_command`), capture a
|
|
5
|
+
* TAIL-biased, byte-capped transcript (failures live at the end of test
|
|
6
|
+
* output), and derive `passed` from the exit code — the only source of truth.
|
|
7
|
+
* A timeout, a signal kill, or a spawn error is a *failed result*, never a
|
|
8
|
+
* thrown exception and never a fabricated success.
|
|
9
|
+
*/
|
|
10
|
+
export declare class CommandTestRunner implements TestRunner {
|
|
11
|
+
run(command: string, opts: TestRunOptions): Promise<TestRunResult>;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* A rolling, tail-biased capture: whole chunks are dropped from the FRONT
|
|
15
|
+
* once the byte cap is exceeded, so the end of the output — where test
|
|
16
|
+
* runners print their failure summaries — always survives.
|
|
17
|
+
*/
|
|
18
|
+
export declare class TailCapture {
|
|
19
|
+
private readonly cap;
|
|
20
|
+
private chunks;
|
|
21
|
+
private bytes;
|
|
22
|
+
truncated: boolean;
|
|
23
|
+
constructor(cap: number);
|
|
24
|
+
push(buf: Buffer): void;
|
|
25
|
+
text(): string;
|
|
26
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { parseFailures } from "./parse.js";
|
|
3
|
+
/**
|
|
4
|
+
* The shipped {@link TestRunner}: spawn the command via the system shell (the
|
|
5
|
+
* same detached-group + kill-tree discipline as `run_command`), capture a
|
|
6
|
+
* TAIL-biased, byte-capped transcript (failures live at the end of test
|
|
7
|
+
* output), and derive `passed` from the exit code — the only source of truth.
|
|
8
|
+
* A timeout, a signal kill, or a spawn error is a *failed result*, never a
|
|
9
|
+
* thrown exception and never a fabricated success.
|
|
10
|
+
*/
|
|
11
|
+
export class CommandTestRunner {
|
|
12
|
+
run(command, opts) {
|
|
13
|
+
const startedAt = Date.now();
|
|
14
|
+
return new Promise((resolve) => {
|
|
15
|
+
const capture = new TailCapture(opts.captureBytes);
|
|
16
|
+
let child;
|
|
17
|
+
try {
|
|
18
|
+
child = spawn(command, {
|
|
19
|
+
shell: true,
|
|
20
|
+
cwd: opts.cwd,
|
|
21
|
+
detached: true,
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
catch (err) {
|
|
25
|
+
resolve(failed(null, err.message, startedAt));
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
child.stdout?.on("data", (buf) => capture.push(buf));
|
|
29
|
+
child.stderr?.on("data", (buf) => capture.push(buf));
|
|
30
|
+
let settled = false;
|
|
31
|
+
const timer = setTimeout(() => {
|
|
32
|
+
if (settled)
|
|
33
|
+
return;
|
|
34
|
+
settled = true;
|
|
35
|
+
killTree(child.pid);
|
|
36
|
+
resolve(failed(null, capture.text() +
|
|
37
|
+
`\n… [test run timed out after ${opts.timeoutMs}ms and was killed]`, startedAt, capture.truncated));
|
|
38
|
+
}, opts.timeoutMs);
|
|
39
|
+
child.on("error", (err) => {
|
|
40
|
+
if (settled)
|
|
41
|
+
return;
|
|
42
|
+
settled = true;
|
|
43
|
+
clearTimeout(timer);
|
|
44
|
+
resolve(failed(null, err.message, startedAt));
|
|
45
|
+
});
|
|
46
|
+
child.on("close", (code) => {
|
|
47
|
+
if (settled)
|
|
48
|
+
return;
|
|
49
|
+
settled = true;
|
|
50
|
+
clearTimeout(timer);
|
|
51
|
+
const output = capture.text();
|
|
52
|
+
// THE honest-green rule: exit code 0 and nothing else means passed.
|
|
53
|
+
// `code` is null on a signal kill — a failure, whatever the output says.
|
|
54
|
+
const passed = code === 0;
|
|
55
|
+
const parsed = passed ? { failures: [] } : parseFailures(output);
|
|
56
|
+
resolve({
|
|
57
|
+
passed,
|
|
58
|
+
exitCode: code,
|
|
59
|
+
durationMs: Date.now() - startedAt,
|
|
60
|
+
...(parsed.total !== undefined ? { total: parsed.total } : {}),
|
|
61
|
+
failures: parsed.failures,
|
|
62
|
+
output,
|
|
63
|
+
outputTruncated: capture.truncated,
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/** Shape a non-execution failure (spawn error, timeout) as a failed result. */
|
|
70
|
+
function failed(exitCode, output, startedAt, truncated = false) {
|
|
71
|
+
return {
|
|
72
|
+
passed: false,
|
|
73
|
+
exitCode,
|
|
74
|
+
durationMs: Date.now() - startedAt,
|
|
75
|
+
failures: [],
|
|
76
|
+
output,
|
|
77
|
+
outputTruncated: truncated,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* A rolling, tail-biased capture: whole chunks are dropped from the FRONT
|
|
82
|
+
* once the byte cap is exceeded, so the end of the output — where test
|
|
83
|
+
* runners print their failure summaries — always survives.
|
|
84
|
+
*/
|
|
85
|
+
export class TailCapture {
|
|
86
|
+
cap;
|
|
87
|
+
chunks = [];
|
|
88
|
+
bytes = 0;
|
|
89
|
+
truncated = false;
|
|
90
|
+
constructor(cap) {
|
|
91
|
+
this.cap = cap;
|
|
92
|
+
}
|
|
93
|
+
push(buf) {
|
|
94
|
+
this.chunks.push(buf);
|
|
95
|
+
this.bytes += buf.length;
|
|
96
|
+
// Drop head chunks while the REMAINDER still meets the cap.
|
|
97
|
+
while (this.chunks.length > 1 &&
|
|
98
|
+
this.bytes - this.chunks[0].length >= this.cap) {
|
|
99
|
+
this.bytes -= this.chunks[0].length;
|
|
100
|
+
this.chunks.shift();
|
|
101
|
+
this.truncated = true;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
text() {
|
|
105
|
+
let all = Buffer.concat(this.chunks);
|
|
106
|
+
if (all.length > this.cap) {
|
|
107
|
+
all = all.subarray(all.length - this.cap);
|
|
108
|
+
this.truncated = true;
|
|
109
|
+
}
|
|
110
|
+
const body = all.toString("utf8");
|
|
111
|
+
return this.truncated ? `… [earlier output truncated]\n${body}` : body;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
/** Kill the whole process group (POSIX; matches run_command's behavior). */
|
|
115
|
+
function killTree(pid) {
|
|
116
|
+
if (pid === undefined)
|
|
117
|
+
return;
|
|
118
|
+
try {
|
|
119
|
+
process.kill(-pid, "SIGKILL");
|
|
120
|
+
}
|
|
121
|
+
catch {
|
|
122
|
+
// Already exited, or no group — nothing to kill.
|
|
123
|
+
}
|
|
124
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types for the test-execution loop (C.13): run the project's test suite,
|
|
3
|
+
* parse what's honestly parseable, and let the agent iterate on failures
|
|
4
|
+
* under a hard cap. The cardinal rule lives in `runner.ts`: **passed is
|
|
5
|
+
* derived from the exit code and nothing else.**
|
|
6
|
+
*/
|
|
7
|
+
/** A resolved test command and where it came from — shown, never guessed. */
|
|
8
|
+
export interface TestCommand {
|
|
9
|
+
command: string;
|
|
10
|
+
source: "config" | "package-json";
|
|
11
|
+
}
|
|
12
|
+
/** One extracted failure. Every field beyond `name`/`message` is best-effort. */
|
|
13
|
+
export interface TestFailure {
|
|
14
|
+
name: string;
|
|
15
|
+
message: string;
|
|
16
|
+
file?: string;
|
|
17
|
+
line?: number;
|
|
18
|
+
}
|
|
19
|
+
/** The structured outcome of one test-suite execution. */
|
|
20
|
+
export interface TestRunResult {
|
|
21
|
+
/** `exitCode === 0`, full stop — never inferred from output text. */
|
|
22
|
+
passed: boolean;
|
|
23
|
+
/** The command's exit code; `null` when killed by signal or never spawned. */
|
|
24
|
+
exitCode: number | null;
|
|
25
|
+
/** Measured wall time of the execution. */
|
|
26
|
+
durationMs: number;
|
|
27
|
+
/** Total tests, only when a parser confidently extracted it. */
|
|
28
|
+
total?: number;
|
|
29
|
+
/** Best-effort extracted failures; may be empty even when `passed` is false. */
|
|
30
|
+
failures: TestFailure[];
|
|
31
|
+
/** Tail-biased captured stdout+stderr, capped at `test.captureBytes`. */
|
|
32
|
+
output: string;
|
|
33
|
+
/** Whether the head of the output was dropped to honor the byte cap. */
|
|
34
|
+
outputTruncated: boolean;
|
|
35
|
+
}
|
|
36
|
+
/** Execution bounds handed to a runner per run. */
|
|
37
|
+
export interface TestRunOptions {
|
|
38
|
+
cwd: string;
|
|
39
|
+
/** Kill the run (and its process tree) after this many ms. */
|
|
40
|
+
timeoutMs: number;
|
|
41
|
+
/** Cap on captured output bytes (tail-biased). */
|
|
42
|
+
captureBytes: number;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* The swappable execution seam (same discipline as VectorStore/ForgeProvider):
|
|
46
|
+
* the shipped {@link CommandTestRunner} spawns the command via the system
|
|
47
|
+
* shell; tests inject fakes, and a future framework-native runner (e.g. a
|
|
48
|
+
* vitest API runner) slots in without touching the tool.
|
|
49
|
+
*/
|
|
50
|
+
export interface TestRunner {
|
|
51
|
+
run(command: string, opts: TestRunOptions): Promise<TestRunResult>;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* A pluggable best-effort failure extractor: given the captured output,
|
|
55
|
+
* return whatever structure it can positively recognize — and nothing it
|
|
56
|
+
* can't. Parsers never touch `passed` and never fabricate counts.
|
|
57
|
+
*/
|
|
58
|
+
export type FailureParser = (output: string) => {
|
|
59
|
+
failures: TestFailure[];
|
|
60
|
+
total?: number;
|
|
61
|
+
};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types for the test-execution loop (C.13): run the project's test suite,
|
|
3
|
+
* parse what's honestly parseable, and let the agent iterate on failures
|
|
4
|
+
* under a hard cap. The cardinal rule lives in `runner.ts`: **passed is
|
|
5
|
+
* derived from the exit code and nothing else.**
|
|
6
|
+
*/
|
|
7
|
+
export {};
|
package/dist/tools/registry.js
CHANGED
|
@@ -3,6 +3,7 @@ import { listFilesTool } from "./list-files.js";
|
|
|
3
3
|
import { gitStatusTool } from "./git-status.js";
|
|
4
4
|
import { readFileTool, writeFileTool, editFileTool, applyPatchTool, globTool, grepFilesTool, } from "./file/index.js";
|
|
5
5
|
import { runCommandTool } from "./shell/index.js";
|
|
6
|
+
import { makeRunTestsTool } from "../testing/run-tests-tool.js";
|
|
6
7
|
import { searchCodebaseTool } from "./search-codebase.js";
|
|
7
8
|
import { listSkillsTool } from "./list-skills.js";
|
|
8
9
|
import { loadSkillTool } from "./load-skill.js";
|
|
@@ -63,6 +64,8 @@ export function buildDefaultRegistry() {
|
|
|
63
64
|
registry.register(grepFilesTool);
|
|
64
65
|
registry.register(gitStatusTool);
|
|
65
66
|
registry.register(runCommandTool);
|
|
67
|
+
// A fresh tool per registry — its iteration budget (C.13) is session-scoped.
|
|
68
|
+
registry.register(makeRunTestsTool());
|
|
66
69
|
registry.register(searchCodebaseTool);
|
|
67
70
|
registry.register(listSkillsTool);
|
|
68
71
|
registry.register(loadSkillTool);
|
package/dist/tools/types.d.ts
CHANGED
|
@@ -101,10 +101,10 @@ export type ActionPreview =
|
|
|
101
101
|
*/
|
|
102
102
|
export interface ApproveAction {
|
|
103
103
|
/** The category of side effect being requested. */
|
|
104
|
-
kind: "write" | "edit" | "shell" | "patch" | "vcs" | "rollback";
|
|
104
|
+
kind: "write" | "edit" | "shell" | "patch" | "vcs" | "rollback" | "test";
|
|
105
105
|
/** Absolute resolved path the action targets (write/edit). */
|
|
106
106
|
path?: string;
|
|
107
|
-
/** The command to run (shell). */
|
|
107
|
+
/** The command to run (shell / test). */
|
|
108
108
|
command?: string;
|
|
109
109
|
/** Exact-change preview rendered above the prompt (write/edit/patch/vcs/rollback). */
|
|
110
110
|
preview?: ActionPreview;
|