ccqa 0.3.8 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -217
- package/dist/bin/ccqa.mjs +1576 -97
- package/dist/package.json +1 -1
- package/package.json +1 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -1,26 +1,33 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
3
|
import { Command } from "commander";
|
|
4
|
-
import { accessSync, readFileSync } from "node:fs";
|
|
4
|
+
import { accessSync, readFileSync, statSync } from "node:fs";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { access, mkdir, mkdtemp, readFile, readdir, rm, stat, unlink, writeFile } from "node:fs/promises";
|
|
7
|
-
import { delimiter, dirname, join, resolve } from "node:path";
|
|
7
|
+
import { delimiter, dirname, join, relative, resolve } from "node:path";
|
|
8
8
|
import { query } from "@anthropic-ai/claude-agent-sdk";
|
|
9
9
|
import matter from "gray-matter";
|
|
10
|
-
import { spawn } from "node:child_process";
|
|
10
|
+
import { execFile, spawn } from "node:child_process";
|
|
11
11
|
import { createInterface } from "node:readline";
|
|
12
12
|
import { tmpdir } from "node:os";
|
|
13
|
+
import { createInterface as createInterface$1 } from "node:readline/promises";
|
|
14
|
+
import { z } from "zod";
|
|
15
|
+
import { promisify } from "node:util";
|
|
13
16
|
//#region src/prompts/trace.ts
|
|
14
17
|
function generateSessionName() {
|
|
15
18
|
return `ccqa-trace-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`;
|
|
16
19
|
}
|
|
17
20
|
function buildTraceSystemPrompt(spec, options) {
|
|
21
|
+
return buildTraceSystemPromptInner(spec, options, true);
|
|
22
|
+
}
|
|
23
|
+
function buildTraceSystemPromptInner(spec, options, emitRelatedPaths) {
|
|
18
24
|
const sessionName = options?.sessionName ?? generateSessionName();
|
|
19
25
|
const skipCookiesClear = options?.skipCookiesClear ?? false;
|
|
20
26
|
const stepsText = spec.steps.map((step) => `### ${step.id}: ${step.title}
|
|
21
27
|
- **Instruction**: ${step.instruction}
|
|
22
28
|
- **Expected**: ${step.expected}`).join("\n\n");
|
|
23
29
|
const prereqText = spec.prerequisites ? `## Prerequisites\n${spec.prerequisites}\n\n` : "";
|
|
30
|
+
const relatedPathsBlock = emitRelatedPaths ? buildRelatedPathsInstruction() : "";
|
|
24
31
|
return `You are an expert QA engineer executing a browser E2E test. Execute each step precisely and record every browser action as a structured log line.
|
|
25
32
|
|
|
26
33
|
## Session
|
|
@@ -106,6 +113,7 @@ For each step:
|
|
|
106
113
|
- Do NOT retry a selector without taking a fresh snapshot first
|
|
107
114
|
- Do NOT work around blockers (login walls, missing data, captchas) — stop and report
|
|
108
115
|
- **Do NOT suppress errors** — never use \`2>/dev/null\`, \`|| true\`, \`; other-command\`, or any other technique that hides agent-browser failures. Each \`agent-browser\` command must run standalone so failures are properly detected and recorded.
|
|
116
|
+
- **If \`agent-browser\` is not found, stop immediately.** Do not run \`which\`, \`find\`, \`npm ls\`, \`npm install\`, \`npx\`, \`brew\`, or any other discovery / installation command. Do not try alternate paths. The ccqa host already validates the binary before launching you, so if you see \`command not found\` it is a host-environment problem you cannot fix from inside the test run. Emit one line and terminate: \`ASSERTION_FAILED|step-XX|agent-browser binary not available in PATH\`.
|
|
109
117
|
|
|
110
118
|
## Source Code Reference
|
|
111
119
|
|
|
@@ -237,7 +245,7 @@ After each step (outside any code block):
|
|
|
237
245
|
ROUTE_STEP|<step-id>|<step-title>|ACTION:<what you did>|OBSERVATION:<what you verified>|STATUS:<PASSED|FAILED|SKIPPED>
|
|
238
246
|
\`\`\`
|
|
239
247
|
|
|
240
|
-
## Start
|
|
248
|
+
${relatedPathsBlock}## Start
|
|
241
249
|
|
|
242
250
|
${skipCookiesClear ? `A setup procedure has already been executed in this session. Do NOT clear cookies — keep the existing session state.
|
|
243
251
|
|
|
@@ -261,15 +269,49 @@ AB_ACTION|open|${spec.baseUrl}
|
|
|
261
269
|
|
|
262
270
|
Then emit \`STEP_START|step-01|...\` and begin.`;
|
|
263
271
|
}
|
|
272
|
+
function buildRelatedPathsInstruction() {
|
|
273
|
+
return `## Post-run: emit \`relatedPaths\` block
|
|
274
|
+
|
|
275
|
+
After all steps are complete (regardless of pass/fail) and **before** \`RUN_COMPLETED\`, you MUST emit a single \`RELATED_PATHS\` block. The host (not you) writes these paths into the spec's frontmatter — your only job is to emit the block.
|
|
276
|
+
|
|
277
|
+
\`relatedPaths\` is a list of glob patterns identifying the source files this spec depends on. CI uses them to decide whether a code change should trigger a drift check for this spec.
|
|
278
|
+
|
|
279
|
+
**Do NOT modify any source files.** You have only \`Read\`, \`Grep\`, and \`Glob\` for source inspection. The block you emit is the only output the host uses to update the spec.
|
|
280
|
+
|
|
281
|
+
**Inputs to consider:**
|
|
282
|
+
- The URLs you opened (\`AB_ACTION|open|...\`)
|
|
283
|
+
- The aria-labels, placeholders, and visible texts you clicked / filled / waited on
|
|
284
|
+
- The component / page / route files that render those strings (find them with \`Grep\`/\`Read\`/\`Glob\`)
|
|
285
|
+
|
|
286
|
+
**How to choose paths:**
|
|
287
|
+
1. For each URL the test navigates to, locate the route/page file and include it (e.g. \`src/app/tasks/page.tsx\`, \`src/pages/tasks/index.tsx\`).
|
|
288
|
+
2. For each unique aria-label / placeholder / visible text you interacted with, \`Grep\` the codebase, find the defining component, and include either the file or its parent feature directory.
|
|
289
|
+
3. Prefer **directory globs** (e.g. \`src/features/tasks/**\`) over individual files when several related components live in the same area. Otherwise list specific files.
|
|
290
|
+
4. Skip third-party files (\`node_modules/\`), build output (\`dist/\`, \`.next/\`), and generated code.
|
|
291
|
+
5. Be conservative — false positives (extra paths) are fine; false negatives (missing paths) cause drift to be missed in CI. When unsure whether a path is relevant, include it.
|
|
292
|
+
|
|
293
|
+
**Output format (STRICT — one line per path, no leading dashes, no commentary inside the block):**
|
|
294
|
+
|
|
295
|
+
\`\`\`
|
|
296
|
+
RELATED_PATHS_BEGIN
|
|
297
|
+
src/features/tasks/**
|
|
298
|
+
src/app/tasks/page.tsx
|
|
299
|
+
RELATED_PATHS_END
|
|
300
|
+
\`\`\`
|
|
301
|
+
|
|
302
|
+
Emit the block outside any other code block, on its own lines. If the test could not exercise the feature at all (e.g. blocked early), emit the block anyway with whatever paths you can identify; emit \`RELATED_PATHS_BEGIN\` immediately followed by \`RELATED_PATHS_END\` only if you genuinely could not identify any related file.
|
|
303
|
+
|
|
304
|
+
`;
|
|
305
|
+
}
|
|
264
306
|
function buildTracePrompt(spec) {
|
|
265
307
|
return `Execute the test for "${spec.title}" at ${spec.baseUrl}.`;
|
|
266
308
|
}
|
|
267
309
|
function buildSetupTraceSystemPrompt(spec) {
|
|
268
|
-
return
|
|
310
|
+
return buildTraceSystemPromptInner({
|
|
269
311
|
title: spec.title,
|
|
270
312
|
baseUrl: "about:blank",
|
|
271
313
|
steps: spec.steps
|
|
272
|
-
});
|
|
314
|
+
}, void 0, false);
|
|
273
315
|
}
|
|
274
316
|
function buildSetupTracePrompt(spec) {
|
|
275
317
|
return `Execute the setup procedure "${spec.title}". Follow each step precisely.`;
|
|
@@ -345,7 +387,7 @@ function resolveModel(explicit) {
|
|
|
345
387
|
return envModel && envModel.length > 0 ? envModel : void 0;
|
|
346
388
|
}
|
|
347
389
|
async function invokeClaudeStreaming(options, onEvent) {
|
|
348
|
-
const { prompt, systemPrompt, allowedTools, disableBuiltinTools = false, maxTurns, env, model, onAbAction, onAbActionFailed } = options;
|
|
390
|
+
const { prompt, systemPrompt, allowedTools, disableBuiltinTools = false, maxTurns, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false } = options;
|
|
349
391
|
const resolvedModel = resolveModel(model);
|
|
350
392
|
let lastAbToolUseId = null;
|
|
351
393
|
const sdkOptions = {
|
|
@@ -355,6 +397,7 @@ async function invokeClaudeStreaming(options, onEvent) {
|
|
|
355
397
|
permissionMode: "bypassPermissions",
|
|
356
398
|
allowDangerouslySkipPermissions: true,
|
|
357
399
|
...resolvedModel ? { model: resolvedModel } : {},
|
|
400
|
+
...cwd ? { cwd } : {},
|
|
358
401
|
...env ? { env: {
|
|
359
402
|
...process.env,
|
|
360
403
|
...env
|
|
@@ -397,7 +440,7 @@ async function invokeClaudeStreaming(options, onEvent) {
|
|
|
397
440
|
const q = await buildMessageStream(prompt, sdkOptions);
|
|
398
441
|
for await (const msg of q) {
|
|
399
442
|
onEvent(msg);
|
|
400
|
-
if (msg.type === "assistant") {
|
|
443
|
+
if (msg.type === "assistant" && !silenceBashLog) {
|
|
401
444
|
for (const block of msg.message.content ?? []) if (block.type === "tool_use" && block.name === "Bash") {
|
|
402
445
|
const cmd = block.input?.["command"];
|
|
403
446
|
if (typeof cmd === "string") bash(cmd);
|
|
@@ -518,20 +561,93 @@ async function* replayMockMessages(path) {
|
|
|
518
561
|
}
|
|
519
562
|
}
|
|
520
563
|
//#endregion
|
|
564
|
+
//#region src/spec/parser.ts
|
|
565
|
+
function parseTestSpec(content) {
|
|
566
|
+
const { data, content: body } = matter(content);
|
|
567
|
+
const steps = parseSteps(body);
|
|
568
|
+
const prerequisites = parsePrerequisites(body);
|
|
569
|
+
return {
|
|
570
|
+
title: String(data["title"] ?? "Untitled"),
|
|
571
|
+
baseUrl: String(data["baseUrl"] ?? "http://localhost:3000"),
|
|
572
|
+
prerequisites: prerequisites || void 0,
|
|
573
|
+
setups: parseSetupRefs(data["setups"]),
|
|
574
|
+
relatedPaths: parseRelatedPaths(data["relatedPaths"]),
|
|
575
|
+
steps
|
|
576
|
+
};
|
|
577
|
+
}
|
|
578
|
+
function parseRelatedPaths(raw) {
|
|
579
|
+
if (!Array.isArray(raw)) return void 0;
|
|
580
|
+
const paths = [];
|
|
581
|
+
for (const item of raw) if (typeof item === "string" && item.trim().length > 0) paths.push(item.trim());
|
|
582
|
+
return paths.length > 0 ? paths : void 0;
|
|
583
|
+
}
|
|
584
|
+
function parseSetupSpec(content) {
|
|
585
|
+
const { data, content: body } = matter(content);
|
|
586
|
+
const steps = parseSteps(body);
|
|
587
|
+
const placeholders = parsePlaceholders(data["placeholders"]);
|
|
588
|
+
return {
|
|
589
|
+
title: String(data["title"] ?? "Untitled"),
|
|
590
|
+
placeholders: Object.keys(placeholders).length > 0 ? placeholders : void 0,
|
|
591
|
+
steps
|
|
592
|
+
};
|
|
593
|
+
}
|
|
594
|
+
function parsePlaceholders(raw) {
|
|
595
|
+
if (!raw || typeof raw !== "object") return {};
|
|
596
|
+
const result = {};
|
|
597
|
+
for (const [key, val] of Object.entries(raw)) if (val && typeof val === "object" && "dummy" in val) {
|
|
598
|
+
const v = val;
|
|
599
|
+
result[key] = {
|
|
600
|
+
dummy: String(v["dummy"]),
|
|
601
|
+
description: v["description"] ? String(v["description"]) : void 0
|
|
602
|
+
};
|
|
603
|
+
}
|
|
604
|
+
return result;
|
|
605
|
+
}
|
|
606
|
+
function parseSetupRefs(raw) {
|
|
607
|
+
if (!Array.isArray(raw)) return void 0;
|
|
608
|
+
const refs = [];
|
|
609
|
+
for (const item of raw) if (typeof item === "object" && item !== null && "name" in item) {
|
|
610
|
+
const i = item;
|
|
611
|
+
refs.push({
|
|
612
|
+
name: String(i["name"]),
|
|
613
|
+
params: i["params"] && typeof i["params"] === "object" ? Object.fromEntries(Object.entries(i["params"]).map(([k, v]) => [k, String(v)])) : void 0
|
|
614
|
+
});
|
|
615
|
+
}
|
|
616
|
+
return refs.length > 0 ? refs : void 0;
|
|
617
|
+
}
|
|
618
|
+
function parsePrerequisites(body) {
|
|
619
|
+
const match = body.match(/##\s+Prerequisites\s+([\s\S]*?)(?=##|$)/);
|
|
620
|
+
if (!match || !match[1]) return null;
|
|
621
|
+
return match[1].trim();
|
|
622
|
+
}
|
|
623
|
+
function parseSteps(body) {
|
|
624
|
+
const stepBlocks = body.split(/###\s+Step\s+\d+:/);
|
|
625
|
+
const steps = [];
|
|
626
|
+
for (let i = 1; i < stepBlocks.length; i++) {
|
|
627
|
+
const block = stepBlocks[i];
|
|
628
|
+
if (!block) continue;
|
|
629
|
+
const titleMatch = block.match(/^(.+)/);
|
|
630
|
+
const instructionMatch = block.match(/\*\*Instruction\*\*:\s*(.+)/);
|
|
631
|
+
const expectedMatch = block.match(/\*\*Expected\*\*:\s*(.+)/);
|
|
632
|
+
if (!titleMatch || !instructionMatch || !expectedMatch) continue;
|
|
633
|
+
steps.push({
|
|
634
|
+
id: `step-${String(i).padStart(2, "0")}`,
|
|
635
|
+
title: titleMatch[1]?.trim() ?? "",
|
|
636
|
+
instruction: instructionMatch[1]?.trim() ?? "",
|
|
637
|
+
expected: expectedMatch[1]?.trim() ?? ""
|
|
638
|
+
});
|
|
639
|
+
}
|
|
640
|
+
return steps;
|
|
641
|
+
}
|
|
642
|
+
//#endregion
|
|
521
643
|
//#region src/store/index.ts
|
|
522
644
|
const CCQA_DIR = ".ccqa";
|
|
523
645
|
function getCcqaDir(cwd = process.cwd()) {
|
|
524
646
|
return join(cwd, CCQA_DIR);
|
|
525
647
|
}
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
* - "tasks/create-and-complete"
|
|
530
|
-
* - "features/tasks/test-cases/create-and-complete"
|
|
531
|
-
* - ".ccqa/features/tasks/test-cases/create-and-complete"
|
|
532
|
-
* All forms resolve to { featureName: "tasks", specName: "create-and-complete" }.
|
|
533
|
-
* Trailing slashes are tolerated.
|
|
534
|
-
*/
|
|
648
|
+
function specKey(ref) {
|
|
649
|
+
return `${ref.featureName}/${ref.specName}`;
|
|
650
|
+
}
|
|
535
651
|
function parseSpecPath(specPath) {
|
|
536
652
|
const parts = specPath.replace(/^\.\/+/, "").replace(/\/+$/, "").split("/").filter((p) => p.length > 0);
|
|
537
653
|
if (parts[0] === ".ccqa") parts.shift();
|
|
@@ -560,6 +676,32 @@ async function readSpecFile(featureName, specName, cwd) {
|
|
|
560
676
|
throw new Error(`Spec file not found: ${specPath}`);
|
|
561
677
|
});
|
|
562
678
|
}
|
|
679
|
+
async function tryReadSpecFile(featureName, specName, cwd) {
|
|
680
|
+
return readFile(join(getSpecDir(featureName, specName, cwd), "test-spec.md"), "utf-8").catch(() => null);
|
|
681
|
+
}
|
|
682
|
+
async function saveSpecFile(featureName, specName, content, cwd) {
|
|
683
|
+
const specDir = getSpecDir(featureName, specName, cwd);
|
|
684
|
+
await mkdir(specDir, { recursive: true });
|
|
685
|
+
const specPath = join(specDir, "test-spec.md");
|
|
686
|
+
await writeFile(specPath, content.endsWith("\n") ? content : content + "\n", "utf-8");
|
|
687
|
+
return specPath;
|
|
688
|
+
}
|
|
689
|
+
/**
|
|
690
|
+
* Replace (or insert) the `relatedPaths` key in the spec's YAML frontmatter.
|
|
691
|
+
* Preserves every other frontmatter key and the entire body. Returns the
|
|
692
|
+
* absolute path that was written, or null if the spec file does not exist.
|
|
693
|
+
*/
|
|
694
|
+
async function updateSpecRelatedPaths(featureName, specName, relatedPaths, cwd) {
|
|
695
|
+
const specPath = join(getSpecDir(featureName, specName, cwd), "test-spec.md");
|
|
696
|
+
const existing = await readFile(specPath, "utf-8").catch(() => null);
|
|
697
|
+
if (existing === null) return null;
|
|
698
|
+
const parsed = matter(existing);
|
|
699
|
+
const data = { ...parsed.data };
|
|
700
|
+
if (relatedPaths.length > 0) data["relatedPaths"] = relatedPaths;
|
|
701
|
+
else delete data["relatedPaths"];
|
|
702
|
+
await writeFile(specPath, matter.stringify(parsed.content, data), "utf-8");
|
|
703
|
+
return specPath;
|
|
704
|
+
}
|
|
563
705
|
async function saveRoute(featureName, specName, route, cwd) {
|
|
564
706
|
const specDir = getSpecDir(featureName, specName, cwd);
|
|
565
707
|
await mkdir(specDir, { recursive: true });
|
|
@@ -645,6 +787,44 @@ async function listAllSpecs(cwd) {
|
|
|
645
787
|
async function listSpecsForFeature(featureName, cwd) {
|
|
646
788
|
return readdir(join(getFeatureDir(featureName, cwd), "test-cases")).catch(() => []);
|
|
647
789
|
}
|
|
790
|
+
/**
|
|
791
|
+
* Lists every feature/spec dir under .ccqa/features/, regardless of whether
|
|
792
|
+
* the spec is fully drafted yet. Each spec file is read at most once: title
|
|
793
|
+
* and relatedPaths are both extracted from the same parse.
|
|
794
|
+
*/
|
|
795
|
+
async function listFeatureTree(cwd) {
|
|
796
|
+
const featuresDir = join(getCcqaDir(cwd), "features");
|
|
797
|
+
const featureDirs = await readdir(featuresDir).catch(() => []);
|
|
798
|
+
return Promise.all(featureDirs.map(async (featureName) => {
|
|
799
|
+
const testCasesDir = join(featuresDir, featureName, "test-cases");
|
|
800
|
+
const specDirs = await readdir(testCasesDir).catch(() => []);
|
|
801
|
+
return {
|
|
802
|
+
featureName,
|
|
803
|
+
specs: await Promise.all(specDirs.map(async (specName) => {
|
|
804
|
+
const content = await readFile(join(testCasesDir, specName, "test-spec.md"), "utf-8").catch(() => null);
|
|
805
|
+
if (content === null) return {
|
|
806
|
+
specName,
|
|
807
|
+
hasSpecFile: false
|
|
808
|
+
};
|
|
809
|
+
try {
|
|
810
|
+
const spec = parseTestSpec(content);
|
|
811
|
+
const entry = {
|
|
812
|
+
specName,
|
|
813
|
+
hasSpecFile: true
|
|
814
|
+
};
|
|
815
|
+
if (spec.title && spec.title !== "Untitled") entry.title = spec.title;
|
|
816
|
+
if (spec.relatedPaths) entry.relatedPaths = spec.relatedPaths;
|
|
817
|
+
return entry;
|
|
818
|
+
} catch {
|
|
819
|
+
return {
|
|
820
|
+
specName,
|
|
821
|
+
hasSpecFile: true
|
|
822
|
+
};
|
|
823
|
+
}
|
|
824
|
+
}))
|
|
825
|
+
};
|
|
826
|
+
}));
|
|
827
|
+
}
|
|
648
828
|
function routeToMarkdown(route) {
|
|
649
829
|
const lines = [
|
|
650
830
|
"---",
|
|
@@ -665,76 +845,28 @@ function routeToMarkdown(route) {
|
|
|
665
845
|
return lines.join("\n");
|
|
666
846
|
}
|
|
667
847
|
//#endregion
|
|
668
|
-
//#region src/
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
steps
|
|
689
|
-
};
|
|
690
|
-
}
|
|
691
|
-
function parsePlaceholders(raw) {
|
|
692
|
-
if (!raw || typeof raw !== "object") return {};
|
|
693
|
-
const result = {};
|
|
694
|
-
for (const [key, val] of Object.entries(raw)) if (val && typeof val === "object" && "dummy" in val) {
|
|
695
|
-
const v = val;
|
|
696
|
-
result[key] = {
|
|
697
|
-
dummy: String(v["dummy"]),
|
|
698
|
-
description: v["description"] ? String(v["description"]) : void 0
|
|
699
|
-
};
|
|
700
|
-
}
|
|
701
|
-
return result;
|
|
702
|
-
}
|
|
703
|
-
function parseSetupRefs(raw) {
|
|
704
|
-
if (!Array.isArray(raw)) return void 0;
|
|
705
|
-
const refs = [];
|
|
706
|
-
for (const item of raw) if (typeof item === "object" && item !== null && "name" in item) {
|
|
707
|
-
const i = item;
|
|
708
|
-
refs.push({
|
|
709
|
-
name: String(i["name"]),
|
|
710
|
-
params: i["params"] && typeof i["params"] === "object" ? Object.fromEntries(Object.entries(i["params"]).map(([k, v]) => [k, String(v)])) : void 0
|
|
711
|
-
});
|
|
712
|
-
}
|
|
713
|
-
return refs.length > 0 ? refs : void 0;
|
|
714
|
-
}
|
|
715
|
-
function parsePrerequisites(body) {
|
|
716
|
-
const match = body.match(/##\s+Prerequisites\s+([\s\S]*?)(?=##|$)/);
|
|
717
|
-
if (!match || !match[1]) return null;
|
|
718
|
-
return match[1].trim();
|
|
719
|
-
}
|
|
720
|
-
function parseSteps(body) {
|
|
721
|
-
const stepBlocks = body.split(/###\s+Step\s+\d+:/);
|
|
722
|
-
const steps = [];
|
|
723
|
-
for (let i = 1; i < stepBlocks.length; i++) {
|
|
724
|
-
const block = stepBlocks[i];
|
|
725
|
-
if (!block) continue;
|
|
726
|
-
const titleMatch = block.match(/^(.+)/);
|
|
727
|
-
const instructionMatch = block.match(/\*\*Instruction\*\*:\s*(.+)/);
|
|
728
|
-
const expectedMatch = block.match(/\*\*Expected\*\*:\s*(.+)/);
|
|
729
|
-
if (!titleMatch || !instructionMatch || !expectedMatch) continue;
|
|
730
|
-
steps.push({
|
|
731
|
-
id: `step-${String(i).padStart(2, "0")}`,
|
|
732
|
-
title: titleMatch[1]?.trim() ?? "",
|
|
733
|
-
instruction: instructionMatch[1]?.trim() ?? "",
|
|
734
|
-
expected: expectedMatch[1]?.trim() ?? ""
|
|
735
|
-
});
|
|
848
|
+
//#region src/drift/parse-related-paths.ts
|
|
849
|
+
/**
|
|
850
|
+
* Pull a `RELATED_PATHS_BEGIN ... RELATED_PATHS_END` block out of the trace
|
|
851
|
+
* agent's combined text output. Lines inside the block become entries; blank
|
|
852
|
+
* lines, bullet markers, and code fences are tolerated. Returns null when the
|
|
853
|
+
* agent did not emit a block at all so the caller can warn instead of silently
|
|
854
|
+
* clearing the spec's existing relatedPaths.
|
|
855
|
+
*/
|
|
856
|
+
function parseRelatedPathsBlock(text) {
|
|
857
|
+
const match = text.match(/RELATED_PATHS_BEGIN\s*\n?([\s\S]*?)\n?RELATED_PATHS_END/);
|
|
858
|
+
if (!match || match[1] === void 0) return null;
|
|
859
|
+
const seen = /* @__PURE__ */ new Set();
|
|
860
|
+
const out = [];
|
|
861
|
+
for (const raw of match[1].split("\n")) {
|
|
862
|
+
const line = raw.replace(/^```.*$/, "").trim();
|
|
863
|
+
if (!line) continue;
|
|
864
|
+
const cleaned = line.replace(/^[-*]\s+/, "").trim();
|
|
865
|
+
if (!cleaned || seen.has(cleaned)) continue;
|
|
866
|
+
seen.add(cleaned);
|
|
867
|
+
out.push(cleaned);
|
|
736
868
|
}
|
|
737
|
-
return
|
|
869
|
+
return out;
|
|
738
870
|
}
|
|
739
871
|
//#endregion
|
|
740
872
|
//#region src/runtime/bundled-config.ts
|
|
@@ -834,22 +966,46 @@ function waitExit(child) {
|
|
|
834
966
|
//#endregion
|
|
835
967
|
//#region src/runtime/agent-browser-bin.ts
|
|
836
968
|
const require$1 = createRequire(import.meta.url);
|
|
969
|
+
function hasAgentBrowserShim(dir) {
|
|
970
|
+
try {
|
|
971
|
+
statSync(join(dir, "agent-browser"));
|
|
972
|
+
return true;
|
|
973
|
+
} catch {
|
|
974
|
+
return false;
|
|
975
|
+
}
|
|
976
|
+
}
|
|
977
|
+
/**
|
|
978
|
+
* Walks up from `start` looking for a `node_modules/.bin/agent-browser` shim.
|
|
979
|
+
* Returns the .bin directory containing the shim, or null if none is found.
|
|
980
|
+
*/
|
|
981
|
+
function findNodeModulesBin(start) {
|
|
982
|
+
let cur = start;
|
|
983
|
+
while (true) {
|
|
984
|
+
const candidate = join(cur, "node_modules", ".bin");
|
|
985
|
+
if (hasAgentBrowserShim(candidate)) return candidate;
|
|
986
|
+
const parent = dirname(cur);
|
|
987
|
+
if (parent === cur) return null;
|
|
988
|
+
cur = parent;
|
|
989
|
+
}
|
|
990
|
+
}
|
|
837
991
|
/**
|
|
838
992
|
* Resolves the directory containing the `agent-browser` shim that npm/pnpm
|
|
839
993
|
* exposes on PATH for the peer-installed package. Used by `ccqa trace` to
|
|
840
994
|
* prepend this directory to PATH so the Claude subprocess can invoke
|
|
841
995
|
* `agent-browser ...` without requiring a global install.
|
|
842
996
|
*
|
|
843
|
-
* Returns null if agent-browser cannot be
|
|
997
|
+
* Returns null if agent-browser cannot be located.
|
|
844
998
|
*/
|
|
845
999
|
function resolveAgentBrowserBinDir() {
|
|
846
|
-
|
|
1000
|
+
const fromCwd = findNodeModulesBin(process.cwd());
|
|
1001
|
+
if (fromCwd) return fromCwd;
|
|
1002
|
+
const fromSelf = findNodeModulesBin(dirname(require$1.resolve("agent-browser/package.json")));
|
|
1003
|
+
if (fromSelf) return fromSelf;
|
|
847
1004
|
try {
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
return join(dirname(pkgJsonPath), "node_modules", ".bin");
|
|
1005
|
+
const candidate = join(dirname(require$1.resolve("agent-browser/package.json")), "node_modules", ".bin");
|
|
1006
|
+
if (hasAgentBrowserShim(candidate)) return candidate;
|
|
1007
|
+
} catch {}
|
|
1008
|
+
return null;
|
|
853
1009
|
}
|
|
854
1010
|
/**
|
|
855
1011
|
* Returns a PATH string with the agent-browser shim directory prepended,
|
|
@@ -863,6 +1019,48 @@ function pathWithAgentBrowserShim(currentPath) {
|
|
|
863
1019
|
if (path.split(delimiter).includes(dir)) return path;
|
|
864
1020
|
return dir + delimiter + path;
|
|
865
1021
|
}
|
|
1022
|
+
/**
|
|
1023
|
+
* Confirms before launching Claude that an `agent-browser` shim is reachable
|
|
1024
|
+
* via PATH. We do this up front so a missing peer dependency fails fast with
|
|
1025
|
+
* a clear message, instead of Claude burning tokens probing the system with
|
|
1026
|
+
* `which`, `find`, `npm install`, etc.
|
|
1027
|
+
*
|
|
1028
|
+
* The `resolver` argument is for tests; production calls take no args.
|
|
1029
|
+
*/
|
|
1030
|
+
function assertAgentBrowserAvailable(resolver = resolveAgentBrowserBinDir) {
|
|
1031
|
+
const dir = resolver();
|
|
1032
|
+
if (!dir) throw new AgentBrowserUnavailableError();
|
|
1033
|
+
const shim = join(dir, "agent-browser");
|
|
1034
|
+
try {
|
|
1035
|
+
const s = statSync(shim);
|
|
1036
|
+
if (!s.isFile() && !s.isSymbolicLink()) throw new AgentBrowserUnavailableError();
|
|
1037
|
+
} catch {
|
|
1038
|
+
throw new AgentBrowserUnavailableError();
|
|
1039
|
+
}
|
|
1040
|
+
return dir;
|
|
1041
|
+
}
|
|
1042
|
+
var AgentBrowserUnavailableError = class extends Error {
|
|
1043
|
+
constructor() {
|
|
1044
|
+
super("agent-browser binary not found on PATH");
|
|
1045
|
+
this.name = "AgentBrowserUnavailableError";
|
|
1046
|
+
}
|
|
1047
|
+
};
|
|
1048
|
+
/** Human-readable explanation shown to the user when the guard fires. */
|
|
1049
|
+
function formatAgentBrowserUnavailableMessage() {
|
|
1050
|
+
return [
|
|
1051
|
+
"agent-browser is not installed or not on PATH.",
|
|
1052
|
+
"",
|
|
1053
|
+
"ccqa drives the browser via the peer-installed `agent-browser` package.",
|
|
1054
|
+
"Install it in this project:",
|
|
1055
|
+
"",
|
|
1056
|
+
" pnpm add -D agent-browser",
|
|
1057
|
+
" # or",
|
|
1058
|
+
" npm install -D agent-browser",
|
|
1059
|
+
"",
|
|
1060
|
+
"If it is already installed, make sure you are running ccqa from the",
|
|
1061
|
+
"project root (or via your package runner, e.g. `pnpm exec ccqa ...`)."
|
|
1062
|
+
].join("\n");
|
|
1063
|
+
}
|
|
866
1064
|
//#endregion
|
|
867
1065
|
//#region src/runtime/env-vars.ts
|
|
868
1066
|
const ENV_VAR_RE = /\$\{([A-Z_][A-Z0-9_]*)\}|\$([A-Z_][A-Z0-9_]*)/g;
|
|
@@ -921,6 +1119,15 @@ const traceCommand = new Command("trace").argument("<feature/spec>", "Spec id in
|
|
|
921
1119
|
});
|
|
922
1120
|
async function runTrace(featureName, specName, model) {
|
|
923
1121
|
header("trace", `${featureName}/${specName}`);
|
|
1122
|
+
try {
|
|
1123
|
+
meta("agent-browser", assertAgentBrowserAvailable());
|
|
1124
|
+
} catch (e) {
|
|
1125
|
+
if (e instanceof AgentBrowserUnavailableError) {
|
|
1126
|
+
error(formatAgentBrowserUnavailableMessage());
|
|
1127
|
+
process.exit(1);
|
|
1128
|
+
}
|
|
1129
|
+
throw e;
|
|
1130
|
+
}
|
|
924
1131
|
await ensureCcqaDir();
|
|
925
1132
|
const spec = parseTestSpec(await readSpecFile(featureName, specName));
|
|
926
1133
|
const hasSetups = (spec.setups?.length ?? 0) > 0;
|
|
@@ -945,6 +1152,7 @@ async function runTrace(featureName, specName, model) {
|
|
|
945
1152
|
const routeSteps = [];
|
|
946
1153
|
let overallStatus = "passed";
|
|
947
1154
|
const traceActions = [];
|
|
1155
|
+
let relatedPathsBuffer = null;
|
|
948
1156
|
const { isError } = await invokeClaudeStreaming({
|
|
949
1157
|
prompt,
|
|
950
1158
|
systemPrompt,
|
|
@@ -971,6 +1179,11 @@ async function runTrace(featureName, specName, model) {
|
|
|
971
1179
|
for (const block of msg.message.content ?? []) {
|
|
972
1180
|
if (block.type !== "text" || !block.text) continue;
|
|
973
1181
|
const text = block.text;
|
|
1182
|
+
if (relatedPathsBuffer !== null) relatedPathsBuffer += text + "\n";
|
|
1183
|
+
else {
|
|
1184
|
+
const idx = text.indexOf("RELATED_PATHS_BEGIN");
|
|
1185
|
+
if (idx !== -1) relatedPathsBuffer = text.slice(idx) + "\n";
|
|
1186
|
+
}
|
|
974
1187
|
const statusLine = parseStatusLine(text);
|
|
975
1188
|
if (statusLine) step(statusLine.type, statusLine.stepId, statusLine.detail);
|
|
976
1189
|
for (const line of text.split("\n")) {
|
|
@@ -1001,6 +1214,11 @@ async function runTrace(featureName, specName, model) {
|
|
|
1001
1214
|
meta("saved", actionsPath);
|
|
1002
1215
|
meta("actions", traceActions.length);
|
|
1003
1216
|
meta("status", overallStatus.toUpperCase());
|
|
1217
|
+
const relatedPaths = relatedPathsBuffer !== null ? parseRelatedPathsBlock(relatedPathsBuffer) : null;
|
|
1218
|
+
if (relatedPaths !== null) {
|
|
1219
|
+
const written = await updateSpecRelatedPaths(featureName, specName, relatedPaths);
|
|
1220
|
+
if (written) meta("relatedPaths", `${relatedPaths.length} path(s) written to ${written}`);
|
|
1221
|
+
} else warn("trace did not emit a RELATED_PATHS block; drift --changed cannot scope this spec");
|
|
1004
1222
|
hint(`run 'ccqa generate ${featureName}/${specName}' to generate a test script`);
|
|
1005
1223
|
}
|
|
1006
1224
|
/**
|
|
@@ -1567,7 +1785,7 @@ async function diagnose(input, options = {}) {
|
|
|
1567
1785
|
reason: "diagnose returned no parseable diagnosis JSON"
|
|
1568
1786
|
},
|
|
1569
1787
|
confidence: 0,
|
|
1570
|
-
reasoning: truncate$
|
|
1788
|
+
reasoning: truncate$2(raw, 1e3)
|
|
1571
1789
|
},
|
|
1572
1790
|
raw,
|
|
1573
1791
|
sdkError: false
|
|
@@ -1624,7 +1842,7 @@ function extractJsonCandidates(raw) {
|
|
|
1624
1842
|
}
|
|
1625
1843
|
return out;
|
|
1626
1844
|
}
|
|
1627
|
-
function truncate$
|
|
1845
|
+
function truncate$2(s, max) {
|
|
1628
1846
|
return s.length <= max ? s : `${s.slice(0, max)}... [truncated, ${s.length - max} more chars]`;
|
|
1629
1847
|
}
|
|
1630
1848
|
function stripFence(raw) {
|
|
@@ -1852,11 +2070,11 @@ async function captureSnapshot(sessionName) {
|
|
|
1852
2070
|
resolve(null);
|
|
1853
2071
|
return;
|
|
1854
2072
|
}
|
|
1855
|
-
resolve(truncate(trimmed, MAX_OUTPUT_BYTES));
|
|
2073
|
+
resolve(truncate$1(trimmed, MAX_OUTPUT_BYTES));
|
|
1856
2074
|
});
|
|
1857
2075
|
});
|
|
1858
2076
|
}
|
|
1859
|
-
function truncate(s, maxBytes) {
|
|
2077
|
+
function truncate$1(s, maxBytes) {
|
|
1860
2078
|
if (s.length <= maxBytes) return s;
|
|
1861
2079
|
return `${s.slice(0, maxBytes)}\n... [truncated, ${s.length - maxBytes} more chars]`;
|
|
1862
2080
|
}
|
|
@@ -2426,6 +2644,15 @@ const traceSetupCommand = new Command("trace-setup").argument("<name>", "Setup n
|
|
|
2426
2644
|
});
|
|
2427
2645
|
async function runTraceSetup(name, model) {
|
|
2428
2646
|
header("trace-setup", name);
|
|
2647
|
+
try {
|
|
2648
|
+
meta("agent-browser", assertAgentBrowserAvailable());
|
|
2649
|
+
} catch (e) {
|
|
2650
|
+
if (e instanceof AgentBrowserUnavailableError) {
|
|
2651
|
+
error(formatAgentBrowserUnavailableMessage());
|
|
2652
|
+
process.exit(1);
|
|
2653
|
+
}
|
|
2654
|
+
throw e;
|
|
2655
|
+
}
|
|
2429
2656
|
await ensureCcqaDir();
|
|
2430
2657
|
const spec = parseSetupSpec(await readSetupSpecFile(name));
|
|
2431
2658
|
const resolvedSpec = replacePlaceholdersWithDummies(spec);
|
|
@@ -2695,6 +2922,1256 @@ async function cleanupActions(actions, model) {
|
|
|
2695
2922
|
return actions;
|
|
2696
2923
|
}
|
|
2697
2924
|
//#endregion
|
|
2925
|
+
//#region src/claude/extract-json.ts
|
|
2926
|
+
/**
|
|
2927
|
+
* Pulls a JSON object out of a Claude completion. Accepts either a fenced
|
|
2928
|
+
* ```json block or a bare `{...}` payload that constitutes the whole reply.
|
|
2929
|
+
* Returns null when neither shape is present.
|
|
2930
|
+
*/
|
|
2931
|
+
function extractJsonBlock(text) {
|
|
2932
|
+
const fenced = text.match(/```(?:json)?\s*\n([\s\S]*?)\n```/);
|
|
2933
|
+
if (fenced && fenced[1]) return fenced[1].trim();
|
|
2934
|
+
const trimmed = text.trim();
|
|
2935
|
+
if (trimmed.startsWith("{") && trimmed.endsWith("}")) return trimmed;
|
|
2936
|
+
return null;
|
|
2937
|
+
}
|
|
2938
|
+
//#endregion
|
|
2939
|
+
//#region src/prompts/draft.ts
|
|
2940
|
+
function buildNamingSystemPrompt() {
|
|
2941
|
+
return `You name a new ccqa test case based on the user's intent and the existing feature tree.
|
|
2942
|
+
|
|
2943
|
+
ccqa test cases live under \`.ccqa/features/<featureName>/test-cases/<specName>/test-spec.md\`.
|
|
2944
|
+
|
|
2945
|
+
## Naming rules
|
|
2946
|
+
|
|
2947
|
+
- featureName and specName are kebab-case ASCII (lowercase, words separated by '-').
|
|
2948
|
+
- featureName: a broad area (e.g. "tasks", "auth", "billing", "search").
|
|
2949
|
+
- specName: a short scenario name (e.g. "create-and-complete", "login-with-email", "search-by-tag").
|
|
2950
|
+
- Reuse existing featureName when the user's intent fits an existing area. Only invent a new featureName when the existing tree clearly does not cover the area.
|
|
2951
|
+
- specName must NOT collide with an existing spec under the chosen feature. If the natural name collides, pick a different one that distinguishes the new scenario from the existing ones.
|
|
2952
|
+
- Use the codebase (Read/Grep/Glob) sparingly to confirm domain vocabulary if helpful. Do not over-explore.
|
|
2953
|
+
|
|
2954
|
+
## Output (STRICT)
|
|
2955
|
+
|
|
2956
|
+
Output ONE fenced \`\`\`json block, nothing else outside it:
|
|
2957
|
+
|
|
2958
|
+
{
|
|
2959
|
+
"featureName": "<kebab-case>",
|
|
2960
|
+
"specName": "<kebab-case>",
|
|
2961
|
+
"reason": "<one short sentence: why this name and how it relates to existing specs>"
|
|
2962
|
+
}
|
|
2963
|
+
`;
|
|
2964
|
+
}
|
|
2965
|
+
function buildNamingPrompt(intent, tree) {
|
|
2966
|
+
return `## User intent
|
|
2967
|
+
|
|
2968
|
+
${intent}
|
|
2969
|
+
|
|
2970
|
+
## Existing feature tree
|
|
2971
|
+
|
|
2972
|
+
${tree.length === 0 ? "(no existing features yet)" : tree.map((f) => {
|
|
2973
|
+
const specLines = f.specs.length === 0 ? " (no specs yet)" : f.specs.map((s) => ` - ${s.specName}${s.title ? ` — ${s.title}` : ""}`).join("\n");
|
|
2974
|
+
return `- ${f.featureName}/\n${specLines}`;
|
|
2975
|
+
}).join("\n")}
|
|
2976
|
+
|
|
2977
|
+
## Task
|
|
2978
|
+
|
|
2979
|
+
Pick featureName and specName for the new test case. Follow the naming rules. Avoid colliding with any existing specName under the chosen feature.
|
|
2980
|
+
`;
|
|
2981
|
+
}
|
|
2982
|
+
function buildDraftSystemPrompt() {
|
|
2983
|
+
return `You are a QA engineer drafting and refining a ccqa test-spec.md.
|
|
2984
|
+
|
|
2985
|
+
The CLI runs you in a loop: each turn the user gives an intent (first run) or a refinement instruction (later runs). You read the codebase, validate the spec, and return a single JSON report. The CLI displays a diff and asks the user whether to apply.
|
|
2986
|
+
|
|
2987
|
+
## test-spec.md format (STRICT)
|
|
2988
|
+
|
|
2989
|
+
YAML frontmatter + Markdown body.
|
|
2990
|
+
|
|
2991
|
+
Frontmatter fields:
|
|
2992
|
+
- title: string (required)
|
|
2993
|
+
- baseUrl: string (required, e.g. http://localhost:3000)
|
|
2994
|
+
- prerequisites: string (optional, free text)
|
|
2995
|
+
- setups: array of { name: string, params?: Record<string,string> } (optional)
|
|
2996
|
+
- relatedPaths: array of string (optional) — glob patterns identifying source files this spec depends on. Used by \`ccqa drift --changed\` in CI to skip drift checks for unrelated changes.
|
|
2997
|
+
|
|
2998
|
+
Body must contain a \`## Steps\` section followed by step blocks:
|
|
2999
|
+
|
|
3000
|
+
\`\`\`
|
|
3001
|
+
### Step 1: <short title>
|
|
3002
|
+
- **Instruction**: <imperative, one sentence>
|
|
3003
|
+
- **Expected**: <observable outcome>
|
|
3004
|
+
|
|
3005
|
+
### Step 2: <short title>
|
|
3006
|
+
...
|
|
3007
|
+
\`\`\`
|
|
3008
|
+
|
|
3009
|
+
## Quality rules
|
|
3010
|
+
|
|
3011
|
+
- One user-facing action per step (login, click, fill, navigate, ...).
|
|
3012
|
+
- **Expected** must be assertion-friendly: visible text, URL pattern, element state.
|
|
3013
|
+
- Forbidden in **Expected**: timestamps, exact counts, session IDs, internal state.
|
|
3014
|
+
- 3–8 steps is typical. Fewer means too coarse; more means too fine.
|
|
3015
|
+
|
|
3016
|
+
## Workflow (use Read / Grep / Glob extensively)
|
|
3017
|
+
|
|
3018
|
+
1. Read the codebase under cwd to find concrete strings: routes, button labels, aria-labels, page titles, placeholders. Use those exact strings in **Expected**.
|
|
3019
|
+
2. If the spec references setups, Read \`.ccqa/setups/<name>/setup-spec.md\` and verify each \`params\` key matches the setup's \`placeholders\`.
|
|
3020
|
+
3. Populate \`relatedPaths\` in the frontmatter with **provisional** glob patterns pointing at the source files this spec touches: the route/page file for each URL the spec visits, plus the component files (or their parent feature directory) that render the aria-labels, placeholders, or visible texts the spec asserts on. Prefer directory globs (e.g. \`src/features/tasks/**\`) when several files in one area are involved. Be conservative — include a path if you're unsure rather than omit it. \`ccqa trace\` will refine this list later from real browser observations.
|
|
3021
|
+
4. Validate the (current or proposed) spec on four axes — emit one issue per finding:
|
|
3022
|
+
- **assertable**: each Expected can be verified against a string/URL/state that exists in code.
|
|
3023
|
+
- **setups**: referenced setup exists; params keys match placeholders.
|
|
3024
|
+
- **granularity**: not too coarse (multiple actions per step) nor too fine (snapshot-only steps); order is logical.
|
|
3025
|
+
- **unimplemented**: any feature mentioned in the spec that you cannot find in code.
|
|
3026
|
+
|
|
3027
|
+
## Output contract (STRICT)
|
|
3028
|
+
|
|
3029
|
+
Output exactly ONE fenced \`\`\`json code block, and nothing else outside it. No prose before or after.
|
|
3030
|
+
|
|
3031
|
+
Schema:
|
|
3032
|
+
|
|
3033
|
+
\`\`\`json
|
|
3034
|
+
{
|
|
3035
|
+
"issues": [
|
|
3036
|
+
{
|
|
3037
|
+
"severity": "OK" | "WARN" | "ERROR",
|
|
3038
|
+
"category": "assertable" | "setups" | "granularity" | "unimplemented",
|
|
3039
|
+
"stepId": "step-01" | null,
|
|
3040
|
+
"message": "<one-line summary>",
|
|
3041
|
+
"detail": "<optional, multiline explanation>"
|
|
3042
|
+
}
|
|
3043
|
+
],
|
|
3044
|
+
"patch": "<COMPLETE rewritten test-spec.md, or empty string if no changes>"
|
|
3045
|
+
}
|
|
3046
|
+
\`\`\`
|
|
3047
|
+
|
|
3048
|
+
## Patch rules
|
|
3049
|
+
|
|
3050
|
+
- \`patch\` must be the COMPLETE file content if non-empty (never a diff fragment).
|
|
3051
|
+
- The CLI replaces the file atomically with \`patch\`.
|
|
3052
|
+
- For **create** mode: produce a fresh spec from the user intent.
|
|
3053
|
+
- For **refine** mode with a non-empty user instruction: apply the user's request, plus fix any issues it introduces. Preserve the user's wording elsewhere.
|
|
3054
|
+
- For **refine** mode with an empty user instruction: only fix issues you find against the current spec; if everything is fine, return \`patch: ""\`.
|
|
3055
|
+
- If \`patch\` is the same as the current spec, return \`patch: ""\` instead.
|
|
3056
|
+
`;
|
|
3057
|
+
}
|
|
3058
|
+
function buildDraftPrompt(input) {
|
|
3059
|
+
const { mode, existing, userInput } = input;
|
|
3060
|
+
if (mode === "create") return `## Mode
|
|
3061
|
+
|
|
3062
|
+
create — no spec exists yet at the target path. Produce a fresh test-spec.md.
|
|
3063
|
+
|
|
3064
|
+
## User intent
|
|
3065
|
+
|
|
3066
|
+
${userInput}
|
|
3067
|
+
|
|
3068
|
+
## Task
|
|
3069
|
+
|
|
3070
|
+
Read the codebase under cwd. Discover concrete strings (routes, labels, titles). Produce a complete test-spec.md as the \`patch\` field, plus any issues you'd flag about your own draft.
|
|
3071
|
+
`;
|
|
3072
|
+
return `## Mode
|
|
3073
|
+
|
|
3074
|
+
refine — a spec already exists. Apply the user's instruction (if any) and validate against the codebase.
|
|
3075
|
+
|
|
3076
|
+
## Current spec
|
|
3077
|
+
|
|
3078
|
+
\`\`\`markdown
|
|
3079
|
+
${existing}\`\`\`
|
|
3080
|
+
|
|
3081
|
+
${userInput ? `## User refinement instruction\n\n${userInput}\n` : `## User refinement instruction\n\n(empty — re-validate the current spec against the codebase; only emit a non-empty patch if something is actually wrong)\n`}
|
|
3082
|
+
## Task
|
|
3083
|
+
|
|
3084
|
+
1. Read the codebase under cwd and any referenced setups.
|
|
3085
|
+
2. If the user's instruction is non-empty, apply it to the spec.
|
|
3086
|
+
3. Validate the resulting spec on the four axes. Emit issues.
|
|
3087
|
+
4. Return the complete updated spec as \`patch\`. If no changes are needed, return \`patch: ""\`.
|
|
3088
|
+
`;
|
|
3089
|
+
}
|
|
3090
|
+
//#endregion
|
|
3091
|
+
//#region src/types.ts
|
|
3092
|
+
const TestStepSchema = z.object({
|
|
3093
|
+
id: z.string(),
|
|
3094
|
+
title: z.string(),
|
|
3095
|
+
instruction: z.string(),
|
|
3096
|
+
expected: z.string()
|
|
3097
|
+
});
|
|
3098
|
+
const SetupRefSchema = z.object({
|
|
3099
|
+
name: z.string(),
|
|
3100
|
+
params: z.record(z.string(), z.string()).optional()
|
|
3101
|
+
});
|
|
3102
|
+
z.object({
|
|
3103
|
+
title: z.string(),
|
|
3104
|
+
baseUrl: z.string(),
|
|
3105
|
+
prerequisites: z.string().optional(),
|
|
3106
|
+
setups: z.array(SetupRefSchema).optional(),
|
|
3107
|
+
relatedPaths: z.array(z.string()).optional(),
|
|
3108
|
+
steps: z.array(TestStepSchema)
|
|
3109
|
+
});
|
|
3110
|
+
const PlaceholderDefSchema = z.object({
|
|
3111
|
+
dummy: z.string(),
|
|
3112
|
+
description: z.string().optional()
|
|
3113
|
+
});
|
|
3114
|
+
z.object({
|
|
3115
|
+
title: z.string(),
|
|
3116
|
+
placeholders: z.record(z.string(), PlaceholderDefSchema).optional(),
|
|
3117
|
+
steps: z.array(TestStepSchema)
|
|
3118
|
+
});
|
|
3119
|
+
const RouteStepSchema = z.object({
|
|
3120
|
+
title: z.string(),
|
|
3121
|
+
action: z.string(),
|
|
3122
|
+
observation: z.string(),
|
|
3123
|
+
status: z.enum([
|
|
3124
|
+
"PASSED",
|
|
3125
|
+
"FAILED",
|
|
3126
|
+
"SKIPPED"
|
|
3127
|
+
]),
|
|
3128
|
+
reason: z.string().optional()
|
|
3129
|
+
});
|
|
3130
|
+
z.object({
|
|
3131
|
+
specName: z.string(),
|
|
3132
|
+
timestamp: z.string(),
|
|
3133
|
+
status: z.enum(["passed", "failed"]),
|
|
3134
|
+
steps: z.array(RouteStepSchema)
|
|
3135
|
+
});
|
|
3136
|
+
const DraftIssueSchema = z.object({
|
|
3137
|
+
severity: z.enum([
|
|
3138
|
+
"OK",
|
|
3139
|
+
"WARN",
|
|
3140
|
+
"ERROR"
|
|
3141
|
+
]),
|
|
3142
|
+
category: z.enum([
|
|
3143
|
+
"assertable",
|
|
3144
|
+
"setups",
|
|
3145
|
+
"granularity",
|
|
3146
|
+
"unimplemented"
|
|
3147
|
+
]),
|
|
3148
|
+
stepId: z.string().nullable(),
|
|
3149
|
+
message: z.string(),
|
|
3150
|
+
detail: z.string().optional()
|
|
3151
|
+
});
|
|
3152
|
+
const DraftReportSchema = z.object({
|
|
3153
|
+
issues: z.array(DraftIssueSchema),
|
|
3154
|
+
patch: z.string()
|
|
3155
|
+
});
|
|
3156
|
+
const DraftNamingSchema = z.object({
|
|
3157
|
+
featureName: z.string().min(1),
|
|
3158
|
+
specName: z.string().min(1),
|
|
3159
|
+
reason: z.string().optional()
|
|
3160
|
+
});
|
|
3161
|
+
//#endregion
|
|
3162
|
+
//#region src/cli/draft.ts
|
|
3163
|
+
const CATEGORY_LABEL$1 = {
|
|
3164
|
+
assertable: "Assertability",
|
|
3165
|
+
setups: "Setup references",
|
|
3166
|
+
granularity: "Step granularity",
|
|
3167
|
+
unimplemented: "Unimplemented checks"
|
|
3168
|
+
};
|
|
3169
|
+
const draftCommand = new Command("draft").argument("[feature/spec]", "Optional spec path (e.g. tasks/create-and-complete). If omitted, Claude proposes one from your intent.").description("Interactively draft and refine a test-spec.md with Claude Code").option("--instruction <text>", "Non-interactive single-shot instruction (skips the interactive loop)").option("--apply", "Auto-apply each generated patch without [y/N] confirmation", false).action(async (specPath, opts) => {
|
|
3170
|
+
await ensureCcqaDir();
|
|
3171
|
+
let featureName;
|
|
3172
|
+
let specName;
|
|
3173
|
+
let prefilledIntent = null;
|
|
3174
|
+
if (specPath) ({featureName, specName} = parseSpecPath(specPath));
|
|
3175
|
+
else {
|
|
3176
|
+
const { naming, intent } = await proposeNaming(opts);
|
|
3177
|
+
featureName = naming.featureName;
|
|
3178
|
+
specName = naming.specName;
|
|
3179
|
+
prefilledIntent = intent;
|
|
3180
|
+
}
|
|
3181
|
+
await runDraft(featureName, specName, opts, prefilledIntent);
|
|
3182
|
+
});
|
|
3183
|
+
async function runDraft(featureName, specName, opts, prefilledIntent) {
|
|
3184
|
+
header("draft", `${featureName}/${specName}`);
|
|
3185
|
+
const oneShot = opts.instruction !== void 0;
|
|
3186
|
+
let useIntentOnce = prefilledIntent !== null && !oneShot;
|
|
3187
|
+
while (true) {
|
|
3188
|
+
const existing = await tryReadSpecFile(featureName, specName);
|
|
3189
|
+
const isFirstRun = existing === null;
|
|
3190
|
+
let userInput;
|
|
3191
|
+
if (oneShot) userInput = opts.instruction ?? "";
|
|
3192
|
+
else if (useIntentOnce && isFirstRun) {
|
|
3193
|
+
userInput = prefilledIntent ?? "";
|
|
3194
|
+
useIntentOnce = false;
|
|
3195
|
+
} else userInput = await prompt(isFirstRun ? "What do you want to test? > " : "How would you like to refine? (empty = re-validate) > ");
|
|
3196
|
+
if (isFirstRun && !userInput.trim()) {
|
|
3197
|
+
error("intent required for the first draft (no spec exists yet)");
|
|
3198
|
+
process.exit(1);
|
|
3199
|
+
}
|
|
3200
|
+
const turnResult = await runOneTurn({
|
|
3201
|
+
featureName,
|
|
3202
|
+
specName,
|
|
3203
|
+
existing,
|
|
3204
|
+
userInput: userInput.trim(),
|
|
3205
|
+
autoApply: opts.apply === true
|
|
3206
|
+
});
|
|
3207
|
+
if (oneShot) process.exit(turnResult.hasError && !turnResult.applied ? 1 : 0);
|
|
3208
|
+
blank();
|
|
3209
|
+
if (/^y/i.test(await prompt("Are you done with this draft? [y/N] "))) {
|
|
3210
|
+
info("draft session complete.");
|
|
3211
|
+
hint(`run 'ccqa trace ${featureName}/${specName}' to record actions`);
|
|
3212
|
+
process.exit(0);
|
|
3213
|
+
}
|
|
3214
|
+
}
|
|
3215
|
+
}
|
|
3216
|
+
async function runOneTurn(input) {
|
|
3217
|
+
const { featureName, specName, existing, userInput, autoApply } = input;
|
|
3218
|
+
const isFirstRun = existing === null;
|
|
3219
|
+
const systemPrompt = buildDraftSystemPrompt();
|
|
3220
|
+
const userPrompt = buildDraftPrompt({
|
|
3221
|
+
mode: isFirstRun ? "create" : "refine",
|
|
3222
|
+
existing: existing ?? "",
|
|
3223
|
+
userInput
|
|
3224
|
+
});
|
|
3225
|
+
info(isFirstRun ? "Reading codebase and drafting spec..." : "Re-validating spec against codebase...");
|
|
3226
|
+
const toolCounts = {};
|
|
3227
|
+
const startedAt = Date.now();
|
|
3228
|
+
const { result, isError } = await invokeClaudeStreaming({
|
|
3229
|
+
prompt: userPrompt,
|
|
3230
|
+
systemPrompt,
|
|
3231
|
+
allowedTools: [
|
|
3232
|
+
"Read",
|
|
3233
|
+
"Grep",
|
|
3234
|
+
"Glob"
|
|
3235
|
+
],
|
|
3236
|
+
silenceBashLog: true
|
|
3237
|
+
}, (msg) => {
|
|
3238
|
+
if (msg.type !== "assistant") return;
|
|
3239
|
+
for (const block of msg.message.content ?? []) if (block.type === "tool_use") toolCounts[block.name] = (toolCounts[block.name] ?? 0) + 1;
|
|
3240
|
+
});
|
|
3241
|
+
printToolSummary(toolCounts, Date.now() - startedAt);
|
|
3242
|
+
if (isError) {
|
|
3243
|
+
error("Claude returned an error result");
|
|
3244
|
+
return {
|
|
3245
|
+
hasError: true,
|
|
3246
|
+
applied: false
|
|
3247
|
+
};
|
|
3248
|
+
}
|
|
3249
|
+
const json = extractJsonBlock(result);
|
|
3250
|
+
if (!json) {
|
|
3251
|
+
error("Claude did not return a json block");
|
|
3252
|
+
warn(`raw tail: ${truncate(result, 200)}`);
|
|
3253
|
+
return {
|
|
3254
|
+
hasError: true,
|
|
3255
|
+
applied: false
|
|
3256
|
+
};
|
|
3257
|
+
}
|
|
3258
|
+
let report;
|
|
3259
|
+
try {
|
|
3260
|
+
report = DraftReportSchema.parse(JSON.parse(json));
|
|
3261
|
+
} catch (e) {
|
|
3262
|
+
error(`failed to parse draft report: ${e.message}`);
|
|
3263
|
+
return {
|
|
3264
|
+
hasError: true,
|
|
3265
|
+
applied: false
|
|
3266
|
+
};
|
|
3267
|
+
}
|
|
3268
|
+
const hasError = printReviewBlock(report.issues);
|
|
3269
|
+
const original = existing ?? "";
|
|
3270
|
+
if (!report.patch || report.patch === original) {
|
|
3271
|
+
blank();
|
|
3272
|
+
info("no changes proposed.");
|
|
3273
|
+
return {
|
|
3274
|
+
hasError,
|
|
3275
|
+
applied: false
|
|
3276
|
+
};
|
|
3277
|
+
}
|
|
3278
|
+
blank();
|
|
3279
|
+
info("--- proposed changes ---");
|
|
3280
|
+
printUnifiedDiff(original, report.patch);
|
|
3281
|
+
blank();
|
|
3282
|
+
if (!(autoApply ? true : /^y/i.test(await prompt("Apply this patch? [y/N] ")))) {
|
|
3283
|
+
info("aborted — no changes applied.");
|
|
3284
|
+
return {
|
|
3285
|
+
hasError,
|
|
3286
|
+
applied: false
|
|
3287
|
+
};
|
|
3288
|
+
}
|
|
3289
|
+
try {
|
|
3290
|
+
parseTestSpec(report.patch);
|
|
3291
|
+
} catch (e) {
|
|
3292
|
+
error(`refused to apply: patch failed validation (${e.message})`);
|
|
3293
|
+
return {
|
|
3294
|
+
hasError: true,
|
|
3295
|
+
applied: false
|
|
3296
|
+
};
|
|
3297
|
+
}
|
|
3298
|
+
meta("saved", await saveSpecFile(featureName, specName, report.patch));
|
|
3299
|
+
return {
|
|
3300
|
+
hasError,
|
|
3301
|
+
applied: true
|
|
3302
|
+
};
|
|
3303
|
+
}
|
|
3304
|
+
async function prompt(question) {
|
|
3305
|
+
const rl = createInterface$1({
|
|
3306
|
+
input: process.stdin,
|
|
3307
|
+
output: process.stdout
|
|
3308
|
+
});
|
|
3309
|
+
rl.on("SIGINT", () => {
|
|
3310
|
+
rl.close();
|
|
3311
|
+
process.exit(130);
|
|
3312
|
+
});
|
|
3313
|
+
try {
|
|
3314
|
+
return (await rl.question(question)).trim();
|
|
3315
|
+
} finally {
|
|
3316
|
+
rl.close();
|
|
3317
|
+
}
|
|
3318
|
+
}
|
|
3319
|
+
/** Aggregated tool-call counts shown after each Claude turn. */
|
|
3320
|
+
function formatToolSummary(counts, elapsedMs) {
|
|
3321
|
+
const entries = Object.entries(counts).filter(([, n]) => n > 0).sort((a, b) => b[1] - a[1]).map(([name, n]) => `${n} ${name}`);
|
|
3322
|
+
return ` ✓ ${entries.length === 0 ? "no tool calls" : entries.join(", ")} (${(elapsedMs / 1e3).toFixed(1)}s)`;
|
|
3323
|
+
}
|
|
3324
|
+
function printToolSummary(counts, elapsedMs) {
|
|
3325
|
+
process.stdout.write(`${formatToolSummary(counts, elapsedMs)}\n`);
|
|
3326
|
+
}
|
|
3327
|
+
/**
|
|
3328
|
+
* Renders the review report as a visually separated block, grouped by
|
|
3329
|
+
* severity. ERROR and WARN findings get full detail; OK findings collapse
|
|
3330
|
+
* to a one-line summary of category names. Returns whether any ERROR
|
|
3331
|
+
* severity was emitted.
|
|
3332
|
+
*/
|
|
3333
|
+
function printReviewBlock(issues) {
|
|
3334
|
+
const RULE = "─".repeat(67);
|
|
3335
|
+
const errors = issues.filter((i) => i.severity === "ERROR");
|
|
3336
|
+
const warnings = issues.filter((i) => i.severity === "WARN");
|
|
3337
|
+
const passed = issues.filter((i) => i.severity === "OK");
|
|
3338
|
+
const headerParts = [];
|
|
3339
|
+
if (errors.length) headerParts.push(`${errors.length} error${errors.length > 1 ? "s" : ""}`);
|
|
3340
|
+
if (warnings.length) headerParts.push(`${warnings.length} warning${warnings.length > 1 ? "s" : ""}`);
|
|
3341
|
+
if (passed.length) headerParts.push(`${passed.length} passed`);
|
|
3342
|
+
const headerSuffix = headerParts.length ? ` (${headerParts.join(", ")})` : "";
|
|
3343
|
+
const ruleLen = Math.max(0, 60 - headerSuffix.length);
|
|
3344
|
+
process.stdout.write(`\n── Review${headerSuffix} ${"─".repeat(ruleLen)}\n\n`);
|
|
3345
|
+
if (issues.length === 0) {
|
|
3346
|
+
process.stdout.write(" (no findings)\n");
|
|
3347
|
+
process.stdout.write(`\n${RULE}\n\n`);
|
|
3348
|
+
return false;
|
|
3349
|
+
}
|
|
3350
|
+
if (errors.length) {
|
|
3351
|
+
process.stdout.write(` ERRORS (${errors.length})\n`);
|
|
3352
|
+
for (const issue of errors) writeFinding$1(issue);
|
|
3353
|
+
process.stdout.write("\n");
|
|
3354
|
+
}
|
|
3355
|
+
if (warnings.length) {
|
|
3356
|
+
process.stdout.write(` WARNINGS (${warnings.length})\n`);
|
|
3357
|
+
for (const issue of warnings) writeFinding$1(issue);
|
|
3358
|
+
process.stdout.write("\n");
|
|
3359
|
+
}
|
|
3360
|
+
if (passed.length) {
|
|
3361
|
+
const names = passed.map((i) => CATEGORY_LABEL$1[i.category]).join(", ");
|
|
3362
|
+
process.stdout.write(` PASSED (${passed.length})\n ${names}\n`);
|
|
3363
|
+
}
|
|
3364
|
+
process.stdout.write(`\n${RULE}\n\n`);
|
|
3365
|
+
return errors.length > 0;
|
|
3366
|
+
}
|
|
3367
|
+
function writeFinding$1(issue) {
|
|
3368
|
+
const stepPart = issue.stepId ? ` ${issue.stepId}` : "";
|
|
3369
|
+
process.stdout.write(` ${CATEGORY_LABEL$1[issue.category]}${stepPart}\n`);
|
|
3370
|
+
process.stdout.write(` ${issue.message}\n`);
|
|
3371
|
+
if (issue.detail) process.stdout.write(` └ ${issue.detail.replace(/\n/g, "\n ")}\n`);
|
|
3372
|
+
}
|
|
3373
|
+
async function proposeNaming(opts) {
|
|
3374
|
+
const oneShot = opts.instruction !== void 0;
|
|
3375
|
+
const intent = oneShot ? opts.instruction ?? "" : await prompt("What do you want to test? > ");
|
|
3376
|
+
if (!intent.trim()) {
|
|
3377
|
+
error("intent required to propose a feature/spec name");
|
|
3378
|
+
process.exit(1);
|
|
3379
|
+
}
|
|
3380
|
+
const tree = await listFeatureTree();
|
|
3381
|
+
const treeForPrompt = tree.map((f) => ({
|
|
3382
|
+
featureName: f.featureName,
|
|
3383
|
+
specs: f.specs.map((s) => ({
|
|
3384
|
+
specName: s.specName,
|
|
3385
|
+
...s.title ? { title: s.title } : {}
|
|
3386
|
+
}))
|
|
3387
|
+
}));
|
|
3388
|
+
info("Proposing a feature/spec name based on your intent...");
|
|
3389
|
+
const { result, isError } = await invokeClaudeStreaming({
|
|
3390
|
+
silenceBashLog: true,
|
|
3391
|
+
prompt: buildNamingPrompt(intent.trim(), treeForPrompt),
|
|
3392
|
+
systemPrompt: buildNamingSystemPrompt(),
|
|
3393
|
+
allowedTools: [
|
|
3394
|
+
"Read",
|
|
3395
|
+
"Grep",
|
|
3396
|
+
"Glob"
|
|
3397
|
+
]
|
|
3398
|
+
}, () => {});
|
|
3399
|
+
if (isError) {
|
|
3400
|
+
error("Claude failed during naming");
|
|
3401
|
+
process.exit(1);
|
|
3402
|
+
}
|
|
3403
|
+
const json = extractJsonBlock(result);
|
|
3404
|
+
if (!json) {
|
|
3405
|
+
error("Claude did not return a json block for naming");
|
|
3406
|
+
process.exit(1);
|
|
3407
|
+
}
|
|
3408
|
+
let proposed;
|
|
3409
|
+
try {
|
|
3410
|
+
proposed = DraftNamingSchema.parse(JSON.parse(json));
|
|
3411
|
+
} catch (e) {
|
|
3412
|
+
error(`failed to parse naming response: ${e.message}`);
|
|
3413
|
+
process.exit(1);
|
|
3414
|
+
}
|
|
3415
|
+
const sanitized = {
|
|
3416
|
+
featureName: sanitizeNamePart(proposed.featureName),
|
|
3417
|
+
specName: sanitizeNamePart(proposed.specName)
|
|
3418
|
+
};
|
|
3419
|
+
if (!sanitized.featureName || !sanitized.specName) {
|
|
3420
|
+
error(`Claude returned an invalid name: ${proposed.featureName}/${proposed.specName}`);
|
|
3421
|
+
process.exit(1);
|
|
3422
|
+
}
|
|
3423
|
+
const final = ensureUnique(tree, sanitized.featureName, sanitized.specName);
|
|
3424
|
+
meta("proposed", `${final.featureName}/${final.specName}`);
|
|
3425
|
+
if (proposed.reason) meta("reason", proposed.reason);
|
|
3426
|
+
if (oneShot || opts.apply === true) return {
|
|
3427
|
+
naming: final,
|
|
3428
|
+
intent: intent.trim()
|
|
3429
|
+
};
|
|
3430
|
+
const answer = await prompt(`Use this name? [y/N/edit] > `);
|
|
3431
|
+
if (/^y/i.test(answer)) return {
|
|
3432
|
+
naming: final,
|
|
3433
|
+
intent: intent.trim()
|
|
3434
|
+
};
|
|
3435
|
+
if (/^e/i.test(answer)) {
|
|
3436
|
+
const manual = await prompt("Enter feature/spec (e.g. tasks/create-and-complete) > ");
|
|
3437
|
+
const parts = manual.split("/");
|
|
3438
|
+
if (parts.length !== 2 || !parts[0] || !parts[1]) {
|
|
3439
|
+
error(`invalid spec path: "${manual}". Expected "<feature>/<spec>"`);
|
|
3440
|
+
process.exit(1);
|
|
3441
|
+
}
|
|
3442
|
+
const featureName = sanitizeNamePart(parts[0]);
|
|
3443
|
+
const specName = sanitizeNamePart(parts[1]);
|
|
3444
|
+
if (!featureName || !specName) {
|
|
3445
|
+
error(`invalid characters in name: ${parts[0]}/${parts[1]}`);
|
|
3446
|
+
process.exit(1);
|
|
3447
|
+
}
|
|
3448
|
+
return {
|
|
3449
|
+
naming: {
|
|
3450
|
+
featureName,
|
|
3451
|
+
specName
|
|
3452
|
+
},
|
|
3453
|
+
intent: intent.trim()
|
|
3454
|
+
};
|
|
3455
|
+
}
|
|
3456
|
+
info("aborted — no draft created.");
|
|
3457
|
+
process.exit(0);
|
|
3458
|
+
}
|
|
3459
|
+
/**
|
|
3460
|
+
* Restrict to kebab-case-friendly characters: lowercase letters, digits, hyphen.
|
|
3461
|
+
* Anything else is dropped or replaced with '-'. Collapses repeated/edge hyphens.
|
|
3462
|
+
*/
|
|
3463
|
+
function sanitizeNamePart(raw) {
|
|
3464
|
+
return raw.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60);
|
|
3465
|
+
}
|
|
3466
|
+
function ensureUnique(tree, featureName, specName) {
|
|
3467
|
+
const feature = tree.find((f) => f.featureName === featureName);
|
|
3468
|
+
if (!feature) return {
|
|
3469
|
+
featureName,
|
|
3470
|
+
specName
|
|
3471
|
+
};
|
|
3472
|
+
const taken = new Set(feature.specs.map((s) => s.specName));
|
|
3473
|
+
if (!taken.has(specName)) return {
|
|
3474
|
+
featureName,
|
|
3475
|
+
specName
|
|
3476
|
+
};
|
|
3477
|
+
for (let i = 2; i < 100; i++) {
|
|
3478
|
+
const candidate = `${specName}-${i}`;
|
|
3479
|
+
if (!taken.has(candidate)) return {
|
|
3480
|
+
featureName,
|
|
3481
|
+
specName: candidate
|
|
3482
|
+
};
|
|
3483
|
+
}
|
|
3484
|
+
return {
|
|
3485
|
+
featureName,
|
|
3486
|
+
specName: `${specName}-${Date.now()}`
|
|
3487
|
+
};
|
|
3488
|
+
}
|
|
3489
|
+
function printUnifiedDiff(before, after) {
|
|
3490
|
+
const lines = computeLineDiff(before.split("\n"), after.split("\n"));
|
|
3491
|
+
for (const line of lines) process.stdout.write(line + "\n");
|
|
3492
|
+
}
|
|
3493
|
+
function computeLineDiff(a, b) {
|
|
3494
|
+
const n = a.length;
|
|
3495
|
+
const m = b.length;
|
|
3496
|
+
const dp = Array.from({ length: n + 1 }, () => new Array(m + 1).fill(0));
|
|
3497
|
+
for (let i = n - 1; i >= 0; i--) for (let j = m - 1; j >= 0; j--) dp[i][j] = a[i] === b[j] ? dp[i + 1][j + 1] + 1 : Math.max(dp[i + 1][j], dp[i][j + 1]);
|
|
3498
|
+
const out = [];
|
|
3499
|
+
let i = 0;
|
|
3500
|
+
let j = 0;
|
|
3501
|
+
while (i < n && j < m) if (a[i] === b[j]) {
|
|
3502
|
+
out.push({
|
|
3503
|
+
kind: "ctx",
|
|
3504
|
+
text: a[i]
|
|
3505
|
+
});
|
|
3506
|
+
i++;
|
|
3507
|
+
j++;
|
|
3508
|
+
} else if (dp[i + 1][j] >= dp[i][j + 1]) {
|
|
3509
|
+
out.push({
|
|
3510
|
+
kind: "del",
|
|
3511
|
+
text: a[i]
|
|
3512
|
+
});
|
|
3513
|
+
i++;
|
|
3514
|
+
} else {
|
|
3515
|
+
out.push({
|
|
3516
|
+
kind: "add",
|
|
3517
|
+
text: b[j]
|
|
3518
|
+
});
|
|
3519
|
+
j++;
|
|
3520
|
+
}
|
|
3521
|
+
while (i < n) out.push({
|
|
3522
|
+
kind: "del",
|
|
3523
|
+
text: a[i++]
|
|
3524
|
+
});
|
|
3525
|
+
while (j < m) out.push({
|
|
3526
|
+
kind: "add",
|
|
3527
|
+
text: b[j++]
|
|
3528
|
+
});
|
|
3529
|
+
return out.map((l) => l.kind === "add" ? `+ ${l.text}` : l.kind === "del" ? `- ${l.text}` : ` ${l.text}`);
|
|
3530
|
+
}
|
|
3531
|
+
function truncate(s, n) {
|
|
3532
|
+
if (s.length <= n) return s;
|
|
3533
|
+
return s.slice(s.length - n);
|
|
3534
|
+
}
|
|
3535
|
+
//#endregion
|
|
3536
|
+
//#region src/prompts/drift.ts
|
|
3537
|
+
function buildDriftSystemPrompt() {
|
|
3538
|
+
return `${buildDraftSystemPrompt()}
|
|
3539
|
+
|
|
3540
|
+
## Drift mode
|
|
3541
|
+
|
|
3542
|
+
You are running non-interactively in CI. The user will not see or apply the patch — only the \`issues\` array.
|
|
3543
|
+
|
|
3544
|
+
- Always set \`patch\` to "" in your response.
|
|
3545
|
+
- Focus issue messages on what is **out of sync** between the spec and the current codebase: missing aria-labels, renamed routes, removed buttons, placeholders that no longer exist, setup references that point to non-existent files.
|
|
3546
|
+
- Do NOT raise issues about stylistic preferences in the spec wording.
|
|
3547
|
+
- Treat \`category: unimplemented\` as the primary signal for drift: anything the spec asserts that you cannot find in code is a drift finding.
|
|
3548
|
+
|
|
3549
|
+
## Drift severity policy (STRICT)
|
|
3550
|
+
|
|
3551
|
+
The CLI exits non-zero when any issue has \`severity: "ERROR"\` (default) or — with \`--severity warn\` — when any \`WARN\` is present. Pick severity by **whether a deterministic replay of this spec would fail today**, not by how confident you are in your own analysis.
|
|
3552
|
+
|
|
3553
|
+
Use **ERROR** when the spec would break on replay:
|
|
3554
|
+
- A selector the spec relies on (\`aria-label\`, \`placeholder\`, \`data-testid\`, button text) **does not exist anywhere in the source**.
|
|
3555
|
+
- A URL / route the spec navigates to is no longer defined.
|
|
3556
|
+
- An **Expected** asserts a string or visible text that is no longer rendered by the relevant component.
|
|
3557
|
+
- A \`setups[].name\` does not resolve to \`.ccqa/setups/<name>/setup-spec.md\`, or a \`params\` key is not declared in that setup's \`placeholders\`.
|
|
3558
|
+
- The spec references a feature/page that has been removed from the codebase.
|
|
3559
|
+
|
|
3560
|
+
Use **WARN** when the spec is still likely to work, but quality could improve:
|
|
3561
|
+
- The Expected is vague ("a message appears") when a precise string exists in code.
|
|
3562
|
+
- A step bundles multiple actions, or a needed intermediate verification step is missing.
|
|
3563
|
+
- Stable signals exist that the spec could leverage but currently doesn't.
|
|
3564
|
+
- You are unsure whether a referenced string exists (give the user the benefit of the doubt; do not hard-fail CI on uncertainty).
|
|
3565
|
+
|
|
3566
|
+
Use **OK** for axes you actively verified and found no issue.
|
|
3567
|
+
|
|
3568
|
+
If you cannot decide between ERROR and WARN, choose WARN. Reserve ERROR for findings you can back up with a specific file path or grep result that proves the drift.
|
|
3569
|
+
`;
|
|
3570
|
+
}
|
|
3571
|
+
function buildDriftUserPrompt(existing) {
|
|
3572
|
+
return buildDraftPrompt({
|
|
3573
|
+
mode: "refine",
|
|
3574
|
+
existing,
|
|
3575
|
+
userInput: ""
|
|
3576
|
+
});
|
|
3577
|
+
}
|
|
3578
|
+
//#endregion
|
|
3579
|
+
//#region src/drift/affected.ts
|
|
3580
|
+
const execFileP = promisify(execFile);
|
|
3581
|
+
/**
|
|
3582
|
+
* Resolve the base ref to diff against for `ccqa drift --changed`.
|
|
3583
|
+
* Precedence: explicit override > GITHUB_BASE_REF > origin/main.
|
|
3584
|
+
*/
|
|
3585
|
+
function resolveBaseRef(explicit) {
|
|
3586
|
+
if (explicit && explicit.length > 0) return explicit;
|
|
3587
|
+
const ghBase = process.env["GITHUB_BASE_REF"];
|
|
3588
|
+
if (ghBase && ghBase.length > 0) return ghBase.startsWith("origin/") ? ghBase : `origin/${ghBase}`;
|
|
3589
|
+
return "origin/main";
|
|
3590
|
+
}
|
|
3591
|
+
/**
|
|
3592
|
+
* Run `git diff --name-status base...HEAD` from `cwd` and return one entry per
|
|
3593
|
+
* changed file. Renames are reported under their NEW path with status
|
|
3594
|
+
* "renamed" — the OLD path is dropped because the spec mapping is against the
|
|
3595
|
+
* post-rename layout.
|
|
3596
|
+
*
|
|
3597
|
+
* Paths are re-rooted to be relative to `cwd`, not the git repo root. In a
|
|
3598
|
+
* monorepo where `cwd` is a sub-package (e.g. `js/apps/knowledge-webapp`),
|
|
3599
|
+
* git emits paths relative to the repo root, but specs declare relatedPaths
|
|
3600
|
+
* relative to their own package. Changes outside `cwd` are dropped so an
|
|
3601
|
+
* unrelated PR can never accidentally scope a sub-package's specs in.
|
|
3602
|
+
*/
|
|
3603
|
+
async function getChangedFiles(base, cwd) {
|
|
3604
|
+
const [{ stdout: rootOut }, { stdout: diffOut }] = await Promise.all([execFileP("git", ["rev-parse", "--show-toplevel"], { cwd }), execFileP("git", [
|
|
3605
|
+
"diff",
|
|
3606
|
+
"--name-status",
|
|
3607
|
+
"-M",
|
|
3608
|
+
`${base}...HEAD`
|
|
3609
|
+
], {
|
|
3610
|
+
cwd,
|
|
3611
|
+
maxBuffer: 32 * 1024 * 1024
|
|
3612
|
+
})]);
|
|
3613
|
+
return rerootChangedFiles(parseGitDiffOutput(diffOut), rootOut.trim(), cwd);
|
|
3614
|
+
}
|
|
3615
|
+
/**
|
|
3616
|
+
* Convert paths in `entries` from git-repo-root relative to `cwd` relative,
|
|
3617
|
+
* dropping anything outside `cwd`. Exported for unit tests.
|
|
3618
|
+
*/
|
|
3619
|
+
function rerootChangedFiles(entries, repoRoot, cwd) {
|
|
3620
|
+
const prefix = relative(repoRoot, cwd);
|
|
3621
|
+
if (!prefix) return entries;
|
|
3622
|
+
const out = [];
|
|
3623
|
+
for (const e of entries) {
|
|
3624
|
+
const rel = relative(prefix, e.path);
|
|
3625
|
+
if (rel.startsWith("..") || rel === "") continue;
|
|
3626
|
+
out.push({
|
|
3627
|
+
...e,
|
|
3628
|
+
path: rel
|
|
3629
|
+
});
|
|
3630
|
+
}
|
|
3631
|
+
return out;
|
|
3632
|
+
}
|
|
3633
|
+
function parseGitDiffOutput(stdout) {
|
|
3634
|
+
const out = [];
|
|
3635
|
+
for (const line of stdout.split("\n")) {
|
|
3636
|
+
if (!line.trim()) continue;
|
|
3637
|
+
const parts = line.split(" ");
|
|
3638
|
+
const code = parts[0];
|
|
3639
|
+
if (!code) continue;
|
|
3640
|
+
if (code.startsWith("R")) {
|
|
3641
|
+
const newPath = parts[2];
|
|
3642
|
+
if (newPath) out.push({
|
|
3643
|
+
path: newPath,
|
|
3644
|
+
status: "renamed"
|
|
3645
|
+
});
|
|
3646
|
+
continue;
|
|
3647
|
+
}
|
|
3648
|
+
if (code.startsWith("C")) {
|
|
3649
|
+
const newPath = parts[2];
|
|
3650
|
+
if (newPath) out.push({
|
|
3651
|
+
path: newPath,
|
|
3652
|
+
status: "added"
|
|
3653
|
+
});
|
|
3654
|
+
continue;
|
|
3655
|
+
}
|
|
3656
|
+
const path = parts[1];
|
|
3657
|
+
if (!path) continue;
|
|
3658
|
+
switch (code[0]) {
|
|
3659
|
+
case "A":
|
|
3660
|
+
out.push({
|
|
3661
|
+
path,
|
|
3662
|
+
status: "added"
|
|
3663
|
+
});
|
|
3664
|
+
break;
|
|
3665
|
+
case "M":
|
|
3666
|
+
case "T":
|
|
3667
|
+
out.push({
|
|
3668
|
+
path,
|
|
3669
|
+
status: "modified"
|
|
3670
|
+
});
|
|
3671
|
+
break;
|
|
3672
|
+
case "D":
|
|
3673
|
+
out.push({
|
|
3674
|
+
path,
|
|
3675
|
+
status: "deleted"
|
|
3676
|
+
});
|
|
3677
|
+
break;
|
|
3678
|
+
default: out.push({
|
|
3679
|
+
path,
|
|
3680
|
+
status: "modified"
|
|
3681
|
+
});
|
|
3682
|
+
}
|
|
3683
|
+
}
|
|
3684
|
+
return out;
|
|
3685
|
+
}
|
|
3686
|
+
function stripLeadingDotSlash(s) {
|
|
3687
|
+
return s.startsWith("./") ? s.slice(2) : s;
|
|
3688
|
+
}
|
|
3689
|
+
const REGEX_CACHE = /* @__PURE__ */ new Map();
|
|
3690
|
+
/** Compiles `pattern` to a RegExp, memoized so repeated `--changed` matches don't re-build. */
|
|
3691
|
+
function compileGlob(pattern) {
|
|
3692
|
+
const cached = REGEX_CACHE.get(pattern);
|
|
3693
|
+
if (cached) return cached;
|
|
3694
|
+
const compiled = globToRegExp(stripLeadingDotSlash(pattern));
|
|
3695
|
+
REGEX_CACHE.set(pattern, compiled);
|
|
3696
|
+
return compiled;
|
|
3697
|
+
}
|
|
3698
|
+
function globToRegExp(pattern) {
|
|
3699
|
+
let re = "^";
|
|
3700
|
+
let i = 0;
|
|
3701
|
+
while (i < pattern.length) {
|
|
3702
|
+
const ch = pattern[i];
|
|
3703
|
+
if (ch === "?") {
|
|
3704
|
+
re += "[^/]";
|
|
3705
|
+
i++;
|
|
3706
|
+
continue;
|
|
3707
|
+
}
|
|
3708
|
+
if (ch !== "*") {
|
|
3709
|
+
re += /[.+^${}()|[\]\\]/.test(ch) ? "\\" + ch : ch;
|
|
3710
|
+
i++;
|
|
3711
|
+
continue;
|
|
3712
|
+
}
|
|
3713
|
+
if (pattern[i + 1] !== "*") {
|
|
3714
|
+
re += "[^/]*";
|
|
3715
|
+
i++;
|
|
3716
|
+
continue;
|
|
3717
|
+
}
|
|
3718
|
+
const hasLeadingSlash = re.endsWith("/");
|
|
3719
|
+
const hasTrailingSlash = pattern[i + 2] === "/";
|
|
3720
|
+
if (hasLeadingSlash) re = re.slice(0, -1);
|
|
3721
|
+
if (hasLeadingSlash || hasTrailingSlash) re += "(?:/?.*)?";
|
|
3722
|
+
else re += ".*";
|
|
3723
|
+
i += hasTrailingSlash ? 3 : 2;
|
|
3724
|
+
}
|
|
3725
|
+
return new RegExp(re + "$");
|
|
3726
|
+
}
|
|
3727
|
+
/**
|
|
3728
|
+
* Returns true if `changedPath` is covered by any of `relatedPaths`. An empty
|
|
3729
|
+
* `relatedPaths` returns false — callers handle the "unscoped spec" case
|
|
3730
|
+
* separately (treat the spec as always-affected) before calling this.
|
|
3731
|
+
*/
|
|
3732
|
+
function isPathAffectedBy(changedPath, relatedPaths) {
|
|
3733
|
+
const stripped = stripLeadingDotSlash(changedPath);
|
|
3734
|
+
for (const pattern of relatedPaths) if (compileGlob(pattern).test(stripped)) return true;
|
|
3735
|
+
return false;
|
|
3736
|
+
}
|
|
3737
|
+
//#endregion
|
|
3738
|
+
//#region src/drift/route-new-files.ts
|
|
3739
|
+
/**
|
|
3740
|
+
* Lightweight Claude call: given a list of new files in the PR and the existing
|
|
3741
|
+
* specs (with their relatedPaths globs as a hint), return the spec keys (in
|
|
3742
|
+
* "<feature>/<spec>" form) that the new files plausibly affect.
|
|
3743
|
+
*
|
|
3744
|
+
* Conservative by design — false positives are safer than false negatives,
|
|
3745
|
+
* because a missed spec turns into undetected drift in CI. When the router
|
|
3746
|
+
* call itself fails, we log a warning rather than fail-close: the surrounding
|
|
3747
|
+
* glob match is the primary signal; the router only adds coverage for new
|
|
3748
|
+
* paths no glob captures.
|
|
3749
|
+
*/
|
|
3750
|
+
async function routeNewFilesToSpecs(input) {
|
|
3751
|
+
const { newFiles, specs, cwd, model } = input;
|
|
3752
|
+
const empty = /* @__PURE__ */ new Set();
|
|
3753
|
+
if (newFiles.length === 0 || specs.length === 0) return empty;
|
|
3754
|
+
const { result, isError } = await invokeClaudeStreaming({
|
|
3755
|
+
prompt: buildRouterPrompt(await Promise.all(newFiles.map(async (path) => ({
|
|
3756
|
+
path,
|
|
3757
|
+
head: await readHead(join(cwd, path))
|
|
3758
|
+
}))), specs),
|
|
3759
|
+
systemPrompt: buildRouterSystemPrompt(),
|
|
3760
|
+
allowedTools: [
|
|
3761
|
+
"Read",
|
|
3762
|
+
"Grep",
|
|
3763
|
+
"Glob"
|
|
3764
|
+
],
|
|
3765
|
+
silenceBashLog: true,
|
|
3766
|
+
cwd,
|
|
3767
|
+
...model ? { model } : {}
|
|
3768
|
+
}, (_msg) => {});
|
|
3769
|
+
if (isError) {
|
|
3770
|
+
warn("new-file router: Claude returned an error; skipping router signal");
|
|
3771
|
+
return empty;
|
|
3772
|
+
}
|
|
3773
|
+
const json = extractJsonBlock(result);
|
|
3774
|
+
if (!json) {
|
|
3775
|
+
warn("new-file router: no JSON block in response; skipping router signal");
|
|
3776
|
+
return empty;
|
|
3777
|
+
}
|
|
3778
|
+
let parsed;
|
|
3779
|
+
try {
|
|
3780
|
+
parsed = JSON.parse(json);
|
|
3781
|
+
} catch (e) {
|
|
3782
|
+
warn(`new-file router: failed to parse JSON (${e.message}); skipping router signal`);
|
|
3783
|
+
return empty;
|
|
3784
|
+
}
|
|
3785
|
+
const out = /* @__PURE__ */ new Set();
|
|
3786
|
+
const validKeys = new Set(specs.map((s) => `${s.featureName}/${s.specName}`));
|
|
3787
|
+
if (typeof parsed === "object" && parsed !== null && "affectedSpecs" in parsed) {
|
|
3788
|
+
const arr = parsed.affectedSpecs;
|
|
3789
|
+
if (Array.isArray(arr)) {
|
|
3790
|
+
for (const item of arr) if (typeof item === "string" && validKeys.has(item)) out.add(item);
|
|
3791
|
+
}
|
|
3792
|
+
}
|
|
3793
|
+
return out;
|
|
3794
|
+
}
|
|
3795
|
+
async function readHead(absPath) {
|
|
3796
|
+
const content = await readFile(absPath, "utf-8").catch(() => "");
|
|
3797
|
+
if (!content) return "";
|
|
3798
|
+
return content.split("\n").slice(0, 40).join("\n");
|
|
3799
|
+
}
|
|
3800
|
+
function buildRouterSystemPrompt() {
|
|
3801
|
+
return `You triage which ccqa test specs are potentially affected by NEW source files added in a pull request.
|
|
3802
|
+
|
|
3803
|
+
You will receive:
|
|
3804
|
+
- A list of new files (path + first ~40 lines of each)
|
|
3805
|
+
- A list of existing specs with their declared relatedPaths globs
|
|
3806
|
+
|
|
3807
|
+
Your job: return the spec keys (in "<feature>/<spec>" form) whose behaviour might depend on any of the new files.
|
|
3808
|
+
|
|
3809
|
+
## Rules
|
|
3810
|
+
|
|
3811
|
+
- Be **conservative**: when in doubt, include the spec. A spurious inclusion costs one extra drift check; a missed spec lets real drift slip through CI.
|
|
3812
|
+
- Use \`Read\`, \`Grep\`, \`Glob\` if you need to inspect the spec body or related code, but stay focused — this is a triage step, not a full review.
|
|
3813
|
+
- Ignore specs whose relatedPaths clearly point to a different area than every new file (e.g. \`src/auth/**\` specs vs new files only under \`src/billing/**\`).
|
|
3814
|
+
- Files like tests, generated code, build artifacts, vendor dirs typically do not affect any spec. Skip them.
|
|
3815
|
+
|
|
3816
|
+
## Output (STRICT)
|
|
3817
|
+
|
|
3818
|
+
Output ONE fenced \`\`\`json block, nothing else:
|
|
3819
|
+
|
|
3820
|
+
\`\`\`json
|
|
3821
|
+
{
|
|
3822
|
+
"affectedSpecs": ["feature/spec", "feature/spec"]
|
|
3823
|
+
}
|
|
3824
|
+
\`\`\`
|
|
3825
|
+
|
|
3826
|
+
Use exactly the keys you saw in the input ("<feature>/<spec>"). Return an empty array if no spec is affected.
|
|
3827
|
+
`;
|
|
3828
|
+
}
|
|
3829
|
+
function buildRouterPrompt(previews, specs) {
|
|
3830
|
+
return `## New files
|
|
3831
|
+
|
|
3832
|
+
${previews.map((p) => {
|
|
3833
|
+
const headBlock = p.head ? `\n\`\`\`\n${p.head}\n\`\`\`` : "\n(empty or unreadable)";
|
|
3834
|
+
return `### ${p.path}${headBlock}`;
|
|
3835
|
+
}).join("\n\n")}
|
|
3836
|
+
|
|
3837
|
+
## Existing specs
|
|
3838
|
+
|
|
3839
|
+
${specs.map((s) => {
|
|
3840
|
+
const title = s.title ? ` — ${s.title}` : "";
|
|
3841
|
+
const paths = s.relatedPaths.length === 0 ? " (no relatedPaths declared)" : s.relatedPaths.map((p) => ` - ${p}`).join("\n");
|
|
3842
|
+
return `- ${s.featureName}/${s.specName}${title}\n${paths}`;
|
|
3843
|
+
}).join("\n")}
|
|
3844
|
+
|
|
3845
|
+
## Task
|
|
3846
|
+
|
|
3847
|
+
Return the spec keys that might be affected by any of the new files. Conservative inclusion is preferred over missing real drift.
|
|
3848
|
+
`;
|
|
3849
|
+
}
|
|
3850
|
+
//#endregion
|
|
3851
|
+
//#region src/cli/drift.ts
|
|
3852
|
+
const DEFAULT_CONCURRENCY = 3;
|
|
3853
|
+
const driftCommand = new Command("drift").argument("[feature/spec]", "Optional spec id. If omitted, every spec under .ccqa/features/ is checked.").description("Check whether each test-spec.md is still in sync with the current codebase (CI-friendly, no patches applied).").option("--format <fmt>", "Output format: text | json | github", "text").option("--severity <level>", "Exit non-zero on this severity or higher: warn | error", "error").option("--concurrency <n>", `Parallel spec checks (default: ${DEFAULT_CONCURRENCY})`).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--cwd <path>", "Working directory used as both the .ccqa root and the codebase Claude reads. Useful for monorepos. Defaults to process.cwd().").option("--changed", "Restrict drift checks to specs whose relatedPaths intersect the git diff against --base (or, in CI, $GITHUB_BASE_REF, else origin/main). New files are routed to specs via a single lightweight Claude call.").option("--base <ref>", "Base ref to diff against when --changed is set. Defaults to $GITHUB_BASE_REF (CI) or origin/main.").action(async (specPath, opts) => {
|
|
3854
|
+
const format = parseFormat(opts.format);
|
|
3855
|
+
const threshold = parseSeverity(opts.severity);
|
|
3856
|
+
const concurrency = parseConcurrency(opts.concurrency);
|
|
3857
|
+
const cwd = opts.cwd ? resolve(opts.cwd) : process.cwd();
|
|
3858
|
+
await ensureCcqaDir(cwd);
|
|
3859
|
+
if (opts.changed && specPath) {
|
|
3860
|
+
error("--changed and an explicit spec id cannot be combined; --changed only applies to a full sweep");
|
|
3861
|
+
process.exit(2);
|
|
3862
|
+
}
|
|
3863
|
+
let targets = await collectTargets(specPath, cwd);
|
|
3864
|
+
if (targets.length === 0) exitWithNoSpecs(format, "no test specs found under .ccqa/features/");
|
|
3865
|
+
if (format === "text") {
|
|
3866
|
+
header("drift", specPath ?? `${targets.length} spec${targets.length > 1 ? "s" : ""}`);
|
|
3867
|
+
if (opts.cwd) meta("cwd", cwd);
|
|
3868
|
+
}
|
|
3869
|
+
if (opts.changed) {
|
|
3870
|
+
const total = targets.length;
|
|
3871
|
+
targets = await filterByChanged({
|
|
3872
|
+
targets,
|
|
3873
|
+
cwd,
|
|
3874
|
+
baseOverride: opts.base,
|
|
3875
|
+
format,
|
|
3876
|
+
model: opts.model
|
|
3877
|
+
});
|
|
3878
|
+
if (format === "text") meta("scoped", `${targets.length} of ${total} spec${total > 1 ? "s" : ""}`);
|
|
3879
|
+
if (targets.length === 0) exitWithNoSpecs(format, "no specs intersect the changed file set; nothing to check");
|
|
3880
|
+
}
|
|
3881
|
+
const results = await runChecks(targets, concurrency, opts.model, cwd, format);
|
|
3882
|
+
emitReport(results, format, cwd);
|
|
3883
|
+
process.exit(determineExitCode(results, threshold));
|
|
3884
|
+
});
|
|
3885
|
+
function exitWithNoSpecs(format, message) {
|
|
3886
|
+
if (format === "json") process.stdout.write(`${JSON.stringify({ specs: [] }, null, 2)}\n`);
|
|
3887
|
+
else if (format === "text") info(message);
|
|
3888
|
+
process.exit(0);
|
|
3889
|
+
}
|
|
3890
|
+
async function filterByChanged(input) {
|
|
3891
|
+
const { targets, cwd, baseOverride, format, model } = input;
|
|
3892
|
+
const base = resolveBaseRef(baseOverride);
|
|
3893
|
+
let changed;
|
|
3894
|
+
try {
|
|
3895
|
+
changed = await getChangedFiles(base, cwd);
|
|
3896
|
+
} catch (e) {
|
|
3897
|
+
error(`failed to run 'git diff' against ${base}: ${e.message}`);
|
|
3898
|
+
process.exit(2);
|
|
3899
|
+
}
|
|
3900
|
+
if (format === "text") {
|
|
3901
|
+
meta("changed-base", base);
|
|
3902
|
+
meta("changed-files", changed.length);
|
|
3903
|
+
}
|
|
3904
|
+
if (changed.length === 0) return [];
|
|
3905
|
+
const newFiles = changed.filter((f) => f.status === "added");
|
|
3906
|
+
const existingChanges = changed.filter((f) => f.status !== "added");
|
|
3907
|
+
const affected = /* @__PURE__ */ new Set();
|
|
3908
|
+
for (const t of targets) {
|
|
3909
|
+
if (!t.relatedPaths) {
|
|
3910
|
+
affected.add(specKey(t));
|
|
3911
|
+
continue;
|
|
3912
|
+
}
|
|
3913
|
+
if (existingChanges.some((f) => isPathAffectedBy(f.path, t.relatedPaths)) || newFiles.some((f) => isPathAffectedBy(f.path, t.relatedPaths))) affected.add(specKey(t));
|
|
3914
|
+
}
|
|
3915
|
+
if (newFiles.length > 0) {
|
|
3916
|
+
if (format === "text") info(`routing ${newFiles.length} new file(s) to specs via Claude...`);
|
|
3917
|
+
const routed = await routeNewFilesToSpecs({
|
|
3918
|
+
newFiles: newFiles.map((f) => f.path),
|
|
3919
|
+
specs: targets.filter((t) => t.relatedPaths).map((t) => ({
|
|
3920
|
+
featureName: t.featureName,
|
|
3921
|
+
specName: t.specName,
|
|
3922
|
+
title: t.title,
|
|
3923
|
+
relatedPaths: t.relatedPaths
|
|
3924
|
+
})),
|
|
3925
|
+
cwd,
|
|
3926
|
+
model
|
|
3927
|
+
});
|
|
3928
|
+
for (const key of routed) affected.add(key);
|
|
3929
|
+
}
|
|
3930
|
+
return targets.filter((t) => affected.has(specKey(t)));
|
|
3931
|
+
}
|
|
3932
|
+
async function collectTargets(specPath, cwd) {
|
|
3933
|
+
if (specPath) {
|
|
3934
|
+
const { featureName, specName } = parseSpecPath(specPath);
|
|
3935
|
+
if (await tryReadSpecFile(featureName, specName, cwd) === null) {
|
|
3936
|
+
error(`spec not found: ${featureName}/${specName} (under ${cwd})`);
|
|
3937
|
+
process.exit(1);
|
|
3938
|
+
}
|
|
3939
|
+
return [{
|
|
3940
|
+
featureName,
|
|
3941
|
+
specName
|
|
3942
|
+
}];
|
|
3943
|
+
}
|
|
3944
|
+
const tree = await listFeatureTree(cwd);
|
|
3945
|
+
const out = [];
|
|
3946
|
+
for (const feature of tree) for (const spec of feature.specs) {
|
|
3947
|
+
if (!spec.hasSpecFile) continue;
|
|
3948
|
+
const t = {
|
|
3949
|
+
featureName: feature.featureName,
|
|
3950
|
+
specName: spec.specName
|
|
3951
|
+
};
|
|
3952
|
+
if (spec.relatedPaths) t.relatedPaths = spec.relatedPaths;
|
|
3953
|
+
if (spec.title) t.title = spec.title;
|
|
3954
|
+
out.push(t);
|
|
3955
|
+
}
|
|
3956
|
+
return out;
|
|
3957
|
+
}
|
|
3958
|
+
async function runChecks(targets, concurrency, model, cwd, format) {
|
|
3959
|
+
const results = new Array(targets.length);
|
|
3960
|
+
let cursor = 0;
|
|
3961
|
+
const worker = async () => {
|
|
3962
|
+
while (true) {
|
|
3963
|
+
const idx = cursor++;
|
|
3964
|
+
if (idx >= targets.length) return;
|
|
3965
|
+
const target = targets[idx];
|
|
3966
|
+
results[idx] = await checkSpec(target, model, cwd, format);
|
|
3967
|
+
}
|
|
3968
|
+
};
|
|
3969
|
+
const pool = Array.from({ length: Math.min(concurrency, targets.length) }, () => worker());
|
|
3970
|
+
await Promise.all(pool);
|
|
3971
|
+
return results;
|
|
3972
|
+
}
|
|
3973
|
+
async function checkSpec(target, model, cwd, format) {
|
|
3974
|
+
const { featureName, specName } = target;
|
|
3975
|
+
const existing = await tryReadSpecFile(featureName, specName, cwd);
|
|
3976
|
+
if (existing === null) return {
|
|
3977
|
+
target,
|
|
3978
|
+
ok: false,
|
|
3979
|
+
issues: [],
|
|
3980
|
+
error: `spec file disappeared after enumeration: ${featureName}/${specName}`
|
|
3981
|
+
};
|
|
3982
|
+
if (format === "text") info(`checking ${featureName}/${specName}`);
|
|
3983
|
+
const { result, isError } = await invokeClaudeStreaming({
|
|
3984
|
+
prompt: buildDriftUserPrompt(existing),
|
|
3985
|
+
systemPrompt: buildDriftSystemPrompt(),
|
|
3986
|
+
allowedTools: [
|
|
3987
|
+
"Read",
|
|
3988
|
+
"Grep",
|
|
3989
|
+
"Glob"
|
|
3990
|
+
],
|
|
3991
|
+
silenceBashLog: true,
|
|
3992
|
+
cwd,
|
|
3993
|
+
...model ? { model } : {}
|
|
3994
|
+
}, (_msg) => {});
|
|
3995
|
+
if (isError) return {
|
|
3996
|
+
target,
|
|
3997
|
+
ok: false,
|
|
3998
|
+
issues: [],
|
|
3999
|
+
error: "Claude returned an error result"
|
|
4000
|
+
};
|
|
4001
|
+
const json = extractJsonBlock(result);
|
|
4002
|
+
if (!json) return {
|
|
4003
|
+
target,
|
|
4004
|
+
ok: false,
|
|
4005
|
+
issues: [],
|
|
4006
|
+
error: "Claude did not return a json block"
|
|
4007
|
+
};
|
|
4008
|
+
let report;
|
|
4009
|
+
try {
|
|
4010
|
+
report = DraftReportSchema.parse(JSON.parse(json));
|
|
4011
|
+
} catch (e) {
|
|
4012
|
+
return {
|
|
4013
|
+
target,
|
|
4014
|
+
ok: false,
|
|
4015
|
+
issues: [],
|
|
4016
|
+
error: `failed to parse drift report: ${e.message}`
|
|
4017
|
+
};
|
|
4018
|
+
}
|
|
4019
|
+
return {
|
|
4020
|
+
target,
|
|
4021
|
+
ok: true,
|
|
4022
|
+
issues: report.issues
|
|
4023
|
+
};
|
|
4024
|
+
}
|
|
4025
|
+
function emitReport(results, format, cwd) {
|
|
4026
|
+
if (format === "json") {
|
|
4027
|
+
emitJson(results);
|
|
4028
|
+
return;
|
|
4029
|
+
}
|
|
4030
|
+
if (format === "github") {
|
|
4031
|
+
emitGithub(results, cwd);
|
|
4032
|
+
return;
|
|
4033
|
+
}
|
|
4034
|
+
emitText(results);
|
|
4035
|
+
}
|
|
4036
|
+
const CATEGORY_LABEL = {
|
|
4037
|
+
assertable: "Assertability",
|
|
4038
|
+
setups: "Setup references",
|
|
4039
|
+
granularity: "Step granularity",
|
|
4040
|
+
unimplemented: "Unimplemented checks"
|
|
4041
|
+
};
|
|
4042
|
+
const HEAVY_RULE = "═".repeat(72);
|
|
4043
|
+
function emitText(results) {
|
|
4044
|
+
for (const r of results) {
|
|
4045
|
+
blank();
|
|
4046
|
+
const heading = `══ ${r.target.featureName}/${r.target.specName} `;
|
|
4047
|
+
const tail = "═".repeat(Math.max(3, 72 - heading.length));
|
|
4048
|
+
process.stdout.write(`${heading}${tail}\n`);
|
|
4049
|
+
if (r.error) {
|
|
4050
|
+
process.stdout.write(` ERROR ${r.error}\n`);
|
|
4051
|
+
continue;
|
|
4052
|
+
}
|
|
4053
|
+
const errors = r.issues.filter((i) => i.severity === "ERROR");
|
|
4054
|
+
const warnings = r.issues.filter((i) => i.severity === "WARN");
|
|
4055
|
+
const passed = r.issues.filter((i) => i.severity === "OK");
|
|
4056
|
+
if (errors.length === 0 && warnings.length === 0) {
|
|
4057
|
+
const label = passed.length === 1 ? "check" : "checks";
|
|
4058
|
+
const detail = passed.length > 0 ? `all ${passed.length} ${label} passed` : "no issues";
|
|
4059
|
+
process.stdout.write(` ✓ ${detail}\n`);
|
|
4060
|
+
continue;
|
|
4061
|
+
}
|
|
4062
|
+
for (const issue of errors) writeFinding("ERROR", issue);
|
|
4063
|
+
for (const issue of warnings) writeFinding("WARN", issue);
|
|
4064
|
+
if (passed.length > 0) {
|
|
4065
|
+
const names = passed.map((i) => CATEGORY_LABEL[i.category]).join(", ");
|
|
4066
|
+
process.stdout.write(`\n ✓ passed (${passed.length}): ${names}\n`);
|
|
4067
|
+
}
|
|
4068
|
+
}
|
|
4069
|
+
blank();
|
|
4070
|
+
process.stdout.write(`${HEAVY_RULE}\n`);
|
|
4071
|
+
const totals = summarize(results);
|
|
4072
|
+
meta("specs", `${results.length} (${totals.errored} errored)`);
|
|
4073
|
+
meta("findings", `${totals.error} error, ${totals.warn} warn, ${totals.ok} ok`);
|
|
4074
|
+
}
|
|
4075
|
+
function writeFinding(level, issue) {
|
|
4076
|
+
const stepPart = issue.stepId ? ` ${issue.stepId}` : "";
|
|
4077
|
+
process.stdout.write(`\n ${level} ${CATEGORY_LABEL[issue.category]}${stepPart}\n`);
|
|
4078
|
+
process.stdout.write(` ${issue.message}\n`);
|
|
4079
|
+
if (issue.detail) process.stdout.write(` └ ${issue.detail.replace(/\n/g, "\n ")}\n`);
|
|
4080
|
+
}
|
|
4081
|
+
function emitJson(results) {
|
|
4082
|
+
const payload = { specs: results.map((r) => ({
|
|
4083
|
+
feature: r.target.featureName,
|
|
4084
|
+
spec: r.target.specName,
|
|
4085
|
+
ok: r.ok,
|
|
4086
|
+
...r.error ? { error: r.error } : {},
|
|
4087
|
+
issues: r.issues.map((i) => ({
|
|
4088
|
+
severity: i.severity,
|
|
4089
|
+
category: i.category,
|
|
4090
|
+
stepId: i.stepId,
|
|
4091
|
+
message: i.message,
|
|
4092
|
+
...i.detail ? { detail: i.detail } : {}
|
|
4093
|
+
}))
|
|
4094
|
+
})) };
|
|
4095
|
+
process.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
|
|
4096
|
+
}
|
|
4097
|
+
function emitGithub(results, cwd) {
|
|
4098
|
+
const repoRoot = process.env["GITHUB_WORKSPACE"] ?? process.cwd();
|
|
4099
|
+
for (const r of results) {
|
|
4100
|
+
const file = githubRelPath(cwd, repoRoot, r.target.featureName, r.target.specName);
|
|
4101
|
+
if (r.error) {
|
|
4102
|
+
process.stdout.write(`::error file=${file}::${escapeGhMessage(r.error)}\n`);
|
|
4103
|
+
continue;
|
|
4104
|
+
}
|
|
4105
|
+
for (const issue of r.issues) {
|
|
4106
|
+
if (issue.severity === "OK") continue;
|
|
4107
|
+
const level = issue.severity === "ERROR" ? "error" : "warning";
|
|
4108
|
+
const title = `${r.target.featureName}/${r.target.specName} — ${issue.category}${issue.stepId ? ` (${issue.stepId})` : ""}`;
|
|
4109
|
+
const body = issue.detail ? `${issue.message}\n${issue.detail}` : issue.message;
|
|
4110
|
+
process.stdout.write(`::${level} file=${file},title=${escapeGhProp(title)}::${escapeGhMessage(body)}\n`);
|
|
4111
|
+
}
|
|
4112
|
+
}
|
|
4113
|
+
}
|
|
4114
|
+
function githubRelPath(cwd, repoRoot, featureName, specName) {
|
|
4115
|
+
const abs = resolve(cwd, ".ccqa", "features", featureName, "test-cases", specName, "test-spec.md");
|
|
4116
|
+
const rel = relative(repoRoot, abs);
|
|
4117
|
+
return rel.startsWith("..") ? abs : rel;
|
|
4118
|
+
}
|
|
4119
|
+
function escapeGhMessage(s) {
|
|
4120
|
+
return s.replace(/%/g, "%25").replace(/\r/g, "%0D").replace(/\n/g, "%0A");
|
|
4121
|
+
}
|
|
4122
|
+
function escapeGhProp(s) {
|
|
4123
|
+
return s.replace(/%/g, "%25").replace(/\r/g, "%0D").replace(/\n/g, "%0A").replace(/,/g, "%2C").replace(/:/g, "%3A");
|
|
4124
|
+
}
|
|
4125
|
+
function summarize(results) {
|
|
4126
|
+
let error = 0;
|
|
4127
|
+
let warn = 0;
|
|
4128
|
+
let ok = 0;
|
|
4129
|
+
let errored = 0;
|
|
4130
|
+
for (const r of results) {
|
|
4131
|
+
if (r.error) errored++;
|
|
4132
|
+
for (const issue of r.issues) if (issue.severity === "ERROR") error++;
|
|
4133
|
+
else if (issue.severity === "WARN") warn++;
|
|
4134
|
+
else ok++;
|
|
4135
|
+
}
|
|
4136
|
+
return {
|
|
4137
|
+
error,
|
|
4138
|
+
warn,
|
|
4139
|
+
ok,
|
|
4140
|
+
errored
|
|
4141
|
+
};
|
|
4142
|
+
}
|
|
4143
|
+
function determineExitCode(results, threshold) {
|
|
4144
|
+
for (const r of results) {
|
|
4145
|
+
if (r.error) return 1;
|
|
4146
|
+
for (const issue of r.issues) {
|
|
4147
|
+
if (issue.severity === "ERROR") return 1;
|
|
4148
|
+
if (threshold === "warn" && issue.severity === "WARN") return 1;
|
|
4149
|
+
}
|
|
4150
|
+
}
|
|
4151
|
+
return 0;
|
|
4152
|
+
}
|
|
4153
|
+
function parseFormat(raw) {
|
|
4154
|
+
const v = raw ?? "text";
|
|
4155
|
+
if (v === "text" || v === "json" || v === "github") return v;
|
|
4156
|
+
error(`invalid --format: ${v} (expected text|json|github)`);
|
|
4157
|
+
process.exit(2);
|
|
4158
|
+
}
|
|
4159
|
+
function parseSeverity(raw) {
|
|
4160
|
+
const v = raw ?? "error";
|
|
4161
|
+
if (v === "warn" || v === "error") return v;
|
|
4162
|
+
error(`invalid --severity: ${v} (expected warn|error)`);
|
|
4163
|
+
process.exit(2);
|
|
4164
|
+
}
|
|
4165
|
+
function parseConcurrency(raw) {
|
|
4166
|
+
if (raw === void 0) return DEFAULT_CONCURRENCY;
|
|
4167
|
+
const n = Number.parseInt(raw, 10);
|
|
4168
|
+
if (!Number.isFinite(n) || n < 1) {
|
|
4169
|
+
error(`invalid --concurrency: ${raw} (expected positive integer)`);
|
|
4170
|
+
process.exit(2);
|
|
4171
|
+
}
|
|
4172
|
+
return n;
|
|
4173
|
+
}
|
|
4174
|
+
//#endregion
|
|
2698
4175
|
//#region src/cli/index.ts
|
|
2699
4176
|
const packageJsonPath = resolvePackageJson();
|
|
2700
4177
|
const { version } = JSON.parse(readFileSync(packageJsonPath, "utf8"));
|
|
@@ -2710,6 +4187,8 @@ function resolvePackageJson() {
|
|
|
2710
4187
|
}
|
|
2711
4188
|
const program = new Command();
|
|
2712
4189
|
program.name("ccqa").description("E2E test CLI using Claude Code + agent-browser").version(version);
|
|
4190
|
+
program.addCommand(draftCommand);
|
|
4191
|
+
program.addCommand(driftCommand);
|
|
2713
4192
|
program.addCommand(traceCommand);
|
|
2714
4193
|
program.addCommand(generateCommand);
|
|
2715
4194
|
program.addCommand(runCommand);
|