ccqa 1.54.0 → 1.56.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -4
- package/dist/bin/ccqa.mjs +983 -1069
- package/dist/cases/contract.cjs +0 -0
- package/dist/cases/contract.d.cts +122 -0
- package/dist/cases/contract.d.mts +122 -0
- package/dist/cases/contract.mjs +1 -0
- package/dist/hub-client/index.d.cts +2 -2
- package/dist/hub-client/index.d.mts +2 -2
- package/dist/package.json +14 -1
- package/package.json +14 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -118,8 +118,6 @@ const TestSpecSchema = z.object({
|
|
|
118
118
|
message: `\`${key}\` only applies to the agent-browser target — remove it or drop \`target: ${spec.target}\``
|
|
119
119
|
});
|
|
120
120
|
});
|
|
121
|
-
/** Default mode when `mode:` is absent. */
|
|
122
|
-
const DEFAULT_SPEC_MODE = "deterministic";
|
|
123
121
|
/**
|
|
124
122
|
* A block param declaration. `required` defaults to true; only explicit
|
|
125
123
|
* `required: false` makes it optional. `secret: true` flags the value as
|
|
@@ -300,7 +298,7 @@ function evidenceLabels(overrides, language) {
|
|
|
300
298
|
*
|
|
301
299
|
* The template language is deliberately tiny: `{placeholder}` segments filled
|
|
302
300
|
* from a substitution table the caller owns. Today the table is the spec's own
|
|
303
|
-
* coordinates (`{feature}` / `{spec}`);
|
|
301
|
+
* coordinates (`{feature}` / `{spec}`); a case source that names its cases
|
|
304
302
|
* differently adds keys to the table without touching this module.
|
|
305
303
|
*/
|
|
306
304
|
const PLACEHOLDER$1 = /\{([^{}]*)\}/g;
|
|
@@ -404,10 +402,6 @@ function caseValues(caseId) {
|
|
|
404
402
|
function resolveCaseRecordingPath(target, targetConfig, caseId) {
|
|
405
403
|
return expandPathTemplate(recordingPathTemplate(templateFor(target, targetConfig)), caseValues(caseId));
|
|
406
404
|
}
|
|
407
|
-
/** {@link resolveTestPath}, resolved against the project root. */
|
|
408
|
-
function resolveTestPathAbs(target, targetConfig, ref, cwd) {
|
|
409
|
-
return resolve(cwd, resolveTestPath(target, targetConfig, ref));
|
|
410
|
-
}
|
|
411
405
|
//#endregion
|
|
412
406
|
//#region src/config/project-config.ts
|
|
413
407
|
/**
|
|
@@ -452,38 +446,6 @@ const ConventionsSchema = z.object({
|
|
|
452
446
|
operate: z.array(z.string().min(1)).default([])
|
|
453
447
|
}).strict();
|
|
454
448
|
/**
|
|
455
|
-
* Which sections of a markdown test case mean what. The keys are ccqa's
|
|
456
|
-
* vocabulary; the values are the headings the project actually writes, in the
|
|
457
|
-
* project's own language. Every one is optional — a case that states no
|
|
458
|
-
* cleanup simply has none — except `steps`, without which there is nothing to
|
|
459
|
-
* record.
|
|
460
|
-
*
|
|
461
|
-
* A heading no key names is not lost: it reaches the recorder as further
|
|
462
|
-
* information about the case, which is what a precondition usually is.
|
|
463
|
-
*/
|
|
464
|
-
const IntentFieldsSchema = z.object({
|
|
465
|
-
title: z.string().min(1).default("Title"),
|
|
466
|
-
precondition: z.string().min(1).default("Precondition"),
|
|
467
|
-
steps: z.string().min(1).default("Steps"),
|
|
468
|
-
expected: z.string().min(1).default("Expected"),
|
|
469
|
-
cleanup: z.string().min(1).default("Cleanup"),
|
|
470
|
-
priority: z.string().min(1).default("Priority"),
|
|
471
|
-
link: z.string().min(1).default("Link"),
|
|
472
|
-
mode: z.string().min(1).optional(),
|
|
473
|
-
outputPath: z.string().min(1).optional()
|
|
474
|
-
}).strict();
|
|
475
|
-
/**
|
|
476
|
-
* Where a target's test cases are written, when they are not ccqa's own
|
|
477
|
-
* `spec.yaml`. `root` is the directory the cases live under, and a case's id
|
|
478
|
-
* is its path below it without the extension — which is also what `{case}`
|
|
479
|
-
* expands to in `testPath`.
|
|
480
|
-
*/
|
|
481
|
-
const IntentSourceSchema = z.object({
|
|
482
|
-
kind: z.literal("markdown"),
|
|
483
|
-
root: z.string().min(1),
|
|
484
|
-
fields: IntentFieldsSchema.prefault({})
|
|
485
|
-
}).strict();
|
|
486
|
-
/**
|
|
487
449
|
* How a generated test names the unique values it creates. `${CCQA_RUN_ID}` is
|
|
488
450
|
* what the recording holds; a repo with its own helper for this says so here,
|
|
489
451
|
* and the emitter calls that instead of leaving an env read in the test.
|
|
@@ -493,7 +455,8 @@ const RunIdConfigSchema = z.object({
|
|
|
493
455
|
expression: z.string().min(1)
|
|
494
456
|
}).strict();
|
|
495
457
|
/**
|
|
496
|
-
* A tag the test's title ends with, taken from
|
|
458
|
+
* A tag the test's title ends with, taken from one of the case's `fields`.
|
|
459
|
+
* Mechanical on
|
|
497
460
|
* purpose: a title tag is a convention a reviewer greps, and a model that
|
|
498
461
|
* decides it per case gets it wrong in a way nobody notices.
|
|
499
462
|
*/
|
|
@@ -506,7 +469,7 @@ const TitleTagsSchema = z.object({
|
|
|
506
469
|
* Per-target settings.
|
|
507
470
|
*
|
|
508
471
|
* `testPath` is the template deciding where a case's generated test lands
|
|
509
|
-
* (`{feature}` / `{spec}` for `spec.yaml`, `{case}` for
|
|
472
|
+
* (`{feature}` / `{spec}` for `spec.yaml`, `{case}` for a case source);
|
|
510
473
|
* omitted, the target's own default applies. It is a template rather than a
|
|
511
474
|
* directory because every other command has to find that file without asking
|
|
512
475
|
* the generator — see src/targets/test-path.ts.
|
|
@@ -523,7 +486,8 @@ const TitleTagsSchema = z.object({
|
|
|
523
486
|
const TargetConfigSchema = z.object({
|
|
524
487
|
kind: z.literal("external").optional(),
|
|
525
488
|
framework: z.literal("playwright").optional(),
|
|
526
|
-
|
|
489
|
+
cases: z.string().min(1).optional(),
|
|
490
|
+
intent: z.never({ error: "`intent` is gone: ccqa reads its own `spec.yaml` and nothing else. Point `cases` at a module in this repository that reads your format and answers ccqa's case contract (`cases: ./ccqa/cases.mjs`) — see the `cases` section of docs/targets.md" }).optional(),
|
|
527
491
|
testPath: z.string().min(1).superRefine((template, ctx) => {
|
|
528
492
|
const error = validateTestPathTemplate(template);
|
|
529
493
|
if (error) ctx.addIssue({
|
|
@@ -640,7 +604,7 @@ const ProjectConfigSchema = z.object({
|
|
|
640
604
|
if (target.kind !== "external") {
|
|
641
605
|
for (const key of [
|
|
642
606
|
"framework",
|
|
643
|
-
"
|
|
607
|
+
"cases",
|
|
644
608
|
"runId",
|
|
645
609
|
"header",
|
|
646
610
|
"titleTags"
|
|
@@ -649,7 +613,7 @@ const ProjectConfigSchema = z.object({
|
|
|
649
613
|
}
|
|
650
614
|
if (target.framework === void 0) refuse(at("framework"), "a kind: external target must say which framework its tests are written for (framework: playwright)");
|
|
651
615
|
if (target.testPath === void 0) refuse(at("testPath"), "a kind: external target must say where its generated tests go (testPath)");
|
|
652
|
-
if (target.
|
|
616
|
+
if (target.cases !== void 0 && target.testPath !== void 0 && !target.testPath.includes("{case}")) refuse(at("testPath"), "a target reading its own case source addresses cases by their path, so its testPath must use {case} — {spec} is only the last segment, and two cases filed under the same name would collide");
|
|
653
617
|
}
|
|
654
618
|
if (config.targets["agent-browser"]?.testPath !== void 0) ctx.addIssue({
|
|
655
619
|
code: "custom",
|
|
@@ -1547,8 +1511,8 @@ function splitCaseId(id) {
|
|
|
1547
1511
|
specName
|
|
1548
1512
|
};
|
|
1549
1513
|
}
|
|
1550
|
-
/**
|
|
1551
|
-
function
|
|
1514
|
+
/** A case a project's own source names, by the path-shaped id it gives it. */
|
|
1515
|
+
function sourcedCase(id, cwd) {
|
|
1552
1516
|
return {
|
|
1553
1517
|
id,
|
|
1554
1518
|
dir: join(getCcqaDir(cwd), CASES_DIR, ...id.split("/"))
|
|
@@ -1566,7 +1530,7 @@ function intentCase(id, cwd) {
|
|
|
1566
1530
|
* that was not asking about the test.
|
|
1567
1531
|
*/
|
|
1568
1532
|
function caseRefFor(id, cwd, recordingPath) {
|
|
1569
|
-
const base = typeof id === "string" ?
|
|
1533
|
+
const base = typeof id === "string" ? sourcedCase(id, cwd) : specCase(id.featureName, id.specName, cwd);
|
|
1570
1534
|
return recordingPath === void 0 ? base : {
|
|
1571
1535
|
...base,
|
|
1572
1536
|
recordingPathAbs: resolve(cwd, recordingPath)
|
|
@@ -1576,14 +1540,20 @@ async function ensureCcqaDir(cwd) {
|
|
|
1576
1540
|
await mkdir(join(getCcqaDir(cwd), "features"), { recursive: true });
|
|
1577
1541
|
await mkdir(join(getCcqaDir(cwd), "blocks"), { recursive: true });
|
|
1578
1542
|
}
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1543
|
+
/**
|
|
1544
|
+
* Where one case's `spec.yaml` is.
|
|
1545
|
+
*
|
|
1546
|
+
* Reading it is not here: a case document is read through `src/cases/`, so
|
|
1547
|
+
* that no feature can reach one kind of case behind the reader's back. This
|
|
1548
|
+
* says where the file is, which the store owns because the `.ccqa` layout is
|
|
1549
|
+
* the store's.
|
|
1550
|
+
*/
|
|
1551
|
+
function specFilePath(featureName, specName, cwd) {
|
|
1552
|
+
return join(getSpecDir(featureName, specName, cwd), SPEC_FILE);
|
|
1584
1553
|
}
|
|
1585
|
-
|
|
1586
|
-
|
|
1554
|
+
/** For the enumerations below only — see `src/cases/spec-source.ts` for the public read. */
|
|
1555
|
+
async function readSpecYaml(featureName, specName, cwd) {
|
|
1556
|
+
return readFile(specFilePath(featureName, specName, cwd), "utf-8").catch(() => null);
|
|
1587
1557
|
}
|
|
1588
1558
|
async function saveSpecFile(featureName, specName, content, cwd) {
|
|
1589
1559
|
const specDir = getSpecDir(featureName, specName, cwd);
|
|
@@ -1987,7 +1957,7 @@ async function listAllSpecsWithSpecFile(cwd) {
|
|
|
1987
1957
|
async function listActiveSpecs(cwd) {
|
|
1988
1958
|
const all = await listAllSpecsWithSpecFile(cwd);
|
|
1989
1959
|
return (await Promise.all(all.map(async (ref) => {
|
|
1990
|
-
return tryParseTestSpec(await
|
|
1960
|
+
return tryParseTestSpec(await readSpecYaml(ref.featureName, ref.specName, cwd))?.disabled === true ? null : ref;
|
|
1991
1961
|
}))).filter((ref) => ref !== null);
|
|
1992
1962
|
}
|
|
1993
1963
|
async function listAllSpecsFilteredBy(requiredFilename, cwd) {
|
|
@@ -2021,41 +1991,6 @@ async function resolveSpecTargets(target, enumerateAll, cwd) {
|
|
|
2021
1991
|
}
|
|
2022
1992
|
return (await listActiveSpecs(cwd)).filter((ref) => ref.featureName === target);
|
|
2023
1993
|
}
|
|
2024
|
-
/**
|
|
2025
|
-
* Lists every feature/spec dir under .ccqa/features/, regardless of whether
|
|
2026
|
-
* the spec is fully drafted yet. Each spec file is read at most once.
|
|
2027
|
-
*/
|
|
2028
|
-
async function listFeatureTree(cwd) {
|
|
2029
|
-
const featuresDir = join(getCcqaDir(cwd), "features");
|
|
2030
|
-
const featureDirs = await readdir(featuresDir).catch(() => []);
|
|
2031
|
-
return Promise.all(featureDirs.map(async (featureName) => {
|
|
2032
|
-
const testCasesDir = join(featuresDir, featureName, "test-cases");
|
|
2033
|
-
const specDirs = await readdir(testCasesDir).catch(() => []);
|
|
2034
|
-
return {
|
|
2035
|
-
featureName,
|
|
2036
|
-
specs: await Promise.all(specDirs.map(async (specName) => {
|
|
2037
|
-
const specFile = join(testCasesDir, specName, SPEC_FILE);
|
|
2038
|
-
const content = await readFile(specFile, "utf-8").catch(() => null);
|
|
2039
|
-
if (content === null) return {
|
|
2040
|
-
specName,
|
|
2041
|
-
hasSpecFile: false
|
|
2042
|
-
};
|
|
2043
|
-
try {
|
|
2044
|
-
return {
|
|
2045
|
-
specName,
|
|
2046
|
-
hasSpecFile: true,
|
|
2047
|
-
includedBlocks: collectIncludedBlockNames(parseTestSpec(content, specFile))
|
|
2048
|
-
};
|
|
2049
|
-
} catch {
|
|
2050
|
-
return {
|
|
2051
|
-
specName,
|
|
2052
|
-
hasSpecFile: true
|
|
2053
|
-
};
|
|
2054
|
-
}
|
|
2055
|
-
}))
|
|
2056
|
-
};
|
|
2057
|
-
}));
|
|
2058
|
-
}
|
|
2059
1994
|
//#endregion
|
|
2060
1995
|
//#region src/runtime/bundled-config.ts
|
|
2061
1996
|
const CANDIDATES = [
|
|
@@ -5683,7 +5618,7 @@ async function collectSpecArtifacts(args) {
|
|
|
5683
5618
|
};
|
|
5684
5619
|
const dir = specArtifactsDir(args.reportDir, args.feature, args.spec);
|
|
5685
5620
|
const relPrefix = posix.join(ARTIFACTS_SUBDIR, `${args.feature}__${args.spec}`);
|
|
5686
|
-
const relFiles = await walkFiles$1(dir);
|
|
5621
|
+
const relFiles = (await walkFiles$1(dir)).filter((rel) => !args.passed || !isTraceArchive(rel));
|
|
5687
5622
|
relFiles.sort((a, b) => rowRank(a) - rowRank(b) || a.localeCompare(b));
|
|
5688
5623
|
const kept = [];
|
|
5689
5624
|
const dropped = [];
|
|
@@ -5744,200 +5679,7 @@ function formatBytes(n) {
|
|
|
5744
5679
|
return `${(n / (1024 * 1024)).toFixed(1)} MB`;
|
|
5745
5680
|
}
|
|
5746
5681
|
//#endregion
|
|
5747
|
-
//#region src/
|
|
5748
|
-
/**
|
|
5749
|
-
* Split on level-2 headings. Deeper headings belong to the section they sit
|
|
5750
|
-
* in — a case that structures its steps with `###` is still one case.
|
|
5751
|
-
*/
|
|
5752
|
-
function splitSections(source) {
|
|
5753
|
-
const sections = [];
|
|
5754
|
-
let current = null;
|
|
5755
|
-
for (const line of source.split(/\r?\n/)) {
|
|
5756
|
-
const heading = /^##\s+(.*\S)\s*$/.exec(line);
|
|
5757
|
-
if (heading) {
|
|
5758
|
-
if (current) sections.push(current);
|
|
5759
|
-
current = {
|
|
5760
|
-
heading: heading[1],
|
|
5761
|
-
body: ""
|
|
5762
|
-
};
|
|
5763
|
-
continue;
|
|
5764
|
-
}
|
|
5765
|
-
if (current) current.body += `${line}\n`;
|
|
5766
|
-
}
|
|
5767
|
-
if (current) sections.push(current);
|
|
5768
|
-
return sections.map((s) => ({
|
|
5769
|
-
heading: s.heading,
|
|
5770
|
-
body: s.body.trim()
|
|
5771
|
-
}));
|
|
5772
|
-
}
|
|
5773
|
-
/**
|
|
5774
|
-
* `1. text`, `1) text`, `1.text`, at the start of a line.
|
|
5775
|
-
*
|
|
5776
|
-
* The indent is bounded to three spaces because deeper means a nested list —
|
|
5777
|
-
* a sub-item of the step above, not a step. And the literal digits are not the
|
|
5778
|
-
* step's number: markdown lets every item be written `1.` and leaves the
|
|
5779
|
-
* counting to the renderer, so ccqa counts the same way a reader does.
|
|
5780
|
-
*/
|
|
5781
|
-
const NUMBERED = /^ {0,3}(\d+)\s*[.).]\s*(.+)$/;
|
|
5782
|
-
/** `- text`, `* text`, `・text` — the bullet characters people actually use. */
|
|
5783
|
-
const BULLET = /^\s*(?:[-*+]|・)\s*(.+)$/;
|
|
5784
|
-
function numberedItems(body) {
|
|
5785
|
-
const steps = [];
|
|
5786
|
-
for (const line of body.split("\n")) {
|
|
5787
|
-
const m = NUMBERED.exec(line);
|
|
5788
|
-
if (m) steps.push({
|
|
5789
|
-
number: steps.length + 1,
|
|
5790
|
-
text: m[2].trim()
|
|
5791
|
-
});
|
|
5792
|
-
else if (steps.length > 0 && /^\s+\S/.test(line)) {
|
|
5793
|
-
const last = steps[steps.length - 1];
|
|
5794
|
-
last.text = `${last.text} ${line.trim()}`;
|
|
5795
|
-
}
|
|
5796
|
-
}
|
|
5797
|
-
return steps;
|
|
5798
|
-
}
|
|
5799
|
-
/**
|
|
5800
|
-
* A section written as a list, or as a paragraph.
|
|
5801
|
-
*
|
|
5802
|
-
* The list form is the useful one — each item is placed against the step that
|
|
5803
|
-
* makes it true — but a case whose author wrote a sentence has still stated an
|
|
5804
|
-
* expectation, and dropping it because of its punctuation would be ccqa losing
|
|
5805
|
-
* what the case says. So prose is one item, and nothing is lost either way.
|
|
5806
|
-
*/
|
|
5807
|
-
function listOrProse(body) {
|
|
5808
|
-
const items = bulletItems(body);
|
|
5809
|
-
if (items.length > 0) return items;
|
|
5810
|
-
const prose = body.trim();
|
|
5811
|
-
return prose.length > 0 ? [prose] : [];
|
|
5812
|
-
}
|
|
5813
|
-
/** Cleanup, likewise: numbered when numbered, otherwise the paragraph as one step. */
|
|
5814
|
-
function cleanupSteps(body) {
|
|
5815
|
-
const numbered = numberedItems(body);
|
|
5816
|
-
if (numbered.length > 0) return numbered;
|
|
5817
|
-
const prose = body.trim();
|
|
5818
|
-
return prose.length > 0 ? [{
|
|
5819
|
-
number: 1,
|
|
5820
|
-
text: prose
|
|
5821
|
-
}] : [];
|
|
5822
|
-
}
|
|
5823
|
-
/** `- text` at the left margin — not an indented bullet, which continues the step above it. */
|
|
5824
|
-
const TOP_LEVEL_BULLET = /^ {0,3}(?:[-*+]|・)\s*(.+)$/;
|
|
5825
|
-
/**
|
|
5826
|
-
* What a cleanup section says besides its steps. A case that numbers its
|
|
5827
|
-
* teardown and then lists what should be true afterwards is stating two
|
|
5828
|
-
* different things under one heading, and only the numbered half is a step.
|
|
5829
|
-
* An unnumbered section is prose the whole way down and states no separate
|
|
5830
|
-
* expectation.
|
|
5831
|
-
*/
|
|
5832
|
-
function cleanupExpectations(body) {
|
|
5833
|
-
if (numberedItems(body).length === 0) return [];
|
|
5834
|
-
return body.split("\n").map((line) => TOP_LEVEL_BULLET.exec(line)?.[1]?.trim()).filter((text) => text !== void 0 && text.length > 0);
|
|
5835
|
-
}
|
|
5836
|
-
function bulletItems(body) {
|
|
5837
|
-
return body.split("\n").map((line) => BULLET.exec(line)?.[1]?.trim()).filter((text) => text !== void 0 && text.length > 0);
|
|
5838
|
-
}
|
|
5839
|
-
/**
|
|
5840
|
-
* A labelled bullet's value: `URL: https://…` / `URL:https://…`. Both colons,
|
|
5841
|
-
* because a case written in Japanese uses the full-width one and nobody should
|
|
5842
|
-
* have to think about it.
|
|
5843
|
-
*/
|
|
5844
|
-
function labelled(items, label) {
|
|
5845
|
-
for (const item of items) {
|
|
5846
|
-
const m = /^([^::]+)[::]\s*(.*)$/.exec(item);
|
|
5847
|
-
if (m && m[1].trim().toLowerCase() === label.toLowerCase()) {
|
|
5848
|
-
const value = m[2].trim();
|
|
5849
|
-
if (value.length > 0) return value;
|
|
5850
|
-
}
|
|
5851
|
-
}
|
|
5852
|
-
}
|
|
5853
|
-
/**
|
|
5854
|
-
* Read one markdown file as a case. A heading the field map names becomes that
|
|
5855
|
-
* field; everything else is carried through untouched. The only hard
|
|
5856
|
-
* requirement is a numbered list of steps: a case with none describes nothing
|
|
5857
|
-
* to record, and failing here beats a recording of nothing.
|
|
5858
|
-
*/
|
|
5859
|
-
function parseMarkdownCase(input) {
|
|
5860
|
-
const { fields } = input;
|
|
5861
|
-
const sections = splitSections(input.source);
|
|
5862
|
-
const named = new Map(sections.map((s) => [s.heading, s]));
|
|
5863
|
-
const body = (heading) => named.get(heading)?.body ?? "";
|
|
5864
|
-
const steps = numberedItems(body(fields.steps));
|
|
5865
|
-
if (steps.length === 0) throw new Error(`${input.id}: no steps — the "${fields.steps}" section must hold a numbered list of what to do`);
|
|
5866
|
-
const linkItems = bulletItems(body(fields.link));
|
|
5867
|
-
const claimed = new Set(Object.values({
|
|
5868
|
-
...fields,
|
|
5869
|
-
precondition: ""
|
|
5870
|
-
}));
|
|
5871
|
-
claimed.delete("");
|
|
5872
|
-
const title = body(fields.title).trim();
|
|
5873
|
-
return {
|
|
5874
|
-
id: input.id,
|
|
5875
|
-
title: title.length > 0 ? firstLine(title) : input.id,
|
|
5876
|
-
steps,
|
|
5877
|
-
expected: listOrProse(body(fields.expected)),
|
|
5878
|
-
cleanup: cleanupSteps(body(fields.cleanup)),
|
|
5879
|
-
cleanupExpected: cleanupExpectations(body(fields.cleanup)),
|
|
5880
|
-
...body(fields.priority) ? { priority: firstLine(body(fields.priority)) } : {},
|
|
5881
|
-
...fields.mode && body(fields.mode) ? { mode: firstLine(body(fields.mode)) } : {},
|
|
5882
|
-
link: {
|
|
5883
|
-
...labelled(linkItems, "url") ? { url: labelled(linkItems, "url") } : {},
|
|
5884
|
-
...labelled(linkItems, "no") ? { ref: labelled(linkItems, "no") } : {}
|
|
5885
|
-
},
|
|
5886
|
-
other: sections.filter((s) => !claimed.has(s.heading) && s.body.length > 0),
|
|
5887
|
-
sections
|
|
5888
|
-
};
|
|
5889
|
-
}
|
|
5890
|
-
function firstLine(text) {
|
|
5891
|
-
return text.split("\n")[0].trim();
|
|
5892
|
-
}
|
|
5893
|
-
/**
|
|
5894
|
-
* The same file with one section's body replaced, and every other byte left
|
|
5895
|
-
* alone. Used to write a generated test's path back into the case that asked
|
|
5896
|
-
* for it — a case file is the project's, and ccqa rewrites exactly the line it
|
|
5897
|
-
* was given permission to.
|
|
5898
|
-
*/
|
|
5899
|
-
function replaceSectionBody(source, heading, body) {
|
|
5900
|
-
const eol = source.includes("\r\n") ? "\r\n" : "\n";
|
|
5901
|
-
const lines = source.split(/\r?\n/);
|
|
5902
|
-
const start = lines.findIndex((line) => /^##\s+(.*\S)\s*$/.exec(line)?.[1] === heading);
|
|
5903
|
-
if (start === -1) return null;
|
|
5904
|
-
let end = start + 1;
|
|
5905
|
-
while (end < lines.length && !/^##\s+/.test(lines[end])) end++;
|
|
5906
|
-
let tail = end;
|
|
5907
|
-
while (tail > start + 1 && lines[tail - 1].trim() === "") tail--;
|
|
5908
|
-
const rest = lines.slice(tail);
|
|
5909
|
-
return [
|
|
5910
|
-
...lines.slice(0, start + 1),
|
|
5911
|
-
"",
|
|
5912
|
-
body,
|
|
5913
|
-
...rest.length > 0 ? rest : [""]
|
|
5914
|
-
].join(eol);
|
|
5915
|
-
}
|
|
5916
|
-
//#endregion
|
|
5917
|
-
//#region src/intent/case.ts
|
|
5918
|
-
/**
|
|
5919
|
-
* A case's mode, from the heading the project mapped to `intent.fields.mode`.
|
|
5920
|
-
*
|
|
5921
|
-
* Only `live` turns the browser agent on; everything else is recorded and
|
|
5922
|
-
* generated. An unrecognised body is warned about rather than accepted
|
|
5923
|
-
* silently — a project that wrote `manual` there meant something by it, and
|
|
5924
|
-
* reading that as "record and generate" without a word is how a case ends up
|
|
5925
|
-
* automated that nobody meant to automate.
|
|
5926
|
-
*/
|
|
5927
|
-
function resolveMode(raw, id) {
|
|
5928
|
-
const value = raw?.trim().toLowerCase() ?? "";
|
|
5929
|
-
if (value === "live") return "live";
|
|
5930
|
-
if (value !== "" && !DETERMINISTIC_SPELLINGS.has(value)) warn(`${id}: mode "${raw.trim()}" is not a mode ccqa knows; recording and generating it`);
|
|
5931
|
-
return DEFAULT_SPEC_MODE;
|
|
5932
|
-
}
|
|
5933
|
-
/** Bodies that plainly mean "not live", so they pass without a warning. */
|
|
5934
|
-
const DETERMINISTIC_SPELLINGS = new Set([
|
|
5935
|
-
"deterministic",
|
|
5936
|
-
"recorded",
|
|
5937
|
-
"generated",
|
|
5938
|
-
"auto",
|
|
5939
|
-
"automated"
|
|
5940
|
-
]);
|
|
5682
|
+
//#region src/cases/case.ts
|
|
5941
5683
|
/**
|
|
5942
5684
|
* Step ids are the case's numbering, so evidence and diffs can cite them. The
|
|
5943
5685
|
* one spelling: `step-comment.ts` rebuilds an id from a comment's number, and
|
|
@@ -5952,11 +5694,12 @@ function cleanupId(n) {
|
|
|
5952
5694
|
/** ccqa's own spec, expanded — the shape every target has always consumed. */
|
|
5953
5695
|
function caseFromSpec(featureName, specName, yaml, blocks, cwd) {
|
|
5954
5696
|
const spec = parseTestSpec(yaml);
|
|
5697
|
+
const expanded = expandOrName(spec, blocks);
|
|
5955
5698
|
return {
|
|
5956
5699
|
ref: specCase(featureName, specName, cwd),
|
|
5957
5700
|
title: spec.title,
|
|
5958
5701
|
mode: spec.mode ?? "deterministic",
|
|
5959
|
-
steps:
|
|
5702
|
+
steps: expanded.steps,
|
|
5960
5703
|
cleanup: [],
|
|
5961
5704
|
expectations: [],
|
|
5962
5705
|
cleanupExpectations: [],
|
|
@@ -5965,137 +5708,337 @@ function caseFromSpec(featureName, specName, yaml, blocks, cwd) {
|
|
|
5965
5708
|
case: `${featureName}/${specName}`,
|
|
5966
5709
|
title: spec.title
|
|
5967
5710
|
},
|
|
5968
|
-
|
|
5969
|
-
|
|
5970
|
-
|
|
5971
|
-
|
|
5972
|
-
|
|
5711
|
+
disabled: spec.disabled === true,
|
|
5712
|
+
blocked: expanded.blocked,
|
|
5713
|
+
document: {
|
|
5714
|
+
path: specFilePath(featureName, specName, cwd),
|
|
5715
|
+
text: yaml
|
|
5716
|
+
},
|
|
5717
|
+
spec
|
|
5973
5718
|
};
|
|
5974
5719
|
}
|
|
5975
|
-
/**
|
|
5976
|
-
|
|
5977
|
-
|
|
5978
|
-
|
|
5979
|
-
|
|
5980
|
-
|
|
5981
|
-
|
|
5720
|
+
/**
|
|
5721
|
+
* The spec's steps with its blocks inlined — or, when a block cannot be
|
|
5722
|
+
* resolved, each step named as the document writes it and the reason carried
|
|
5723
|
+
* on `blocked`. Naming the block is what a reader of the document sees; it is
|
|
5724
|
+
* not a step anything may act on, which is what `blocked` says.
|
|
5725
|
+
*/
|
|
5726
|
+
function expandOrName(spec, blocks) {
|
|
5727
|
+
try {
|
|
5728
|
+
return {
|
|
5729
|
+
steps: expandSpec(spec, { blocks }),
|
|
5730
|
+
blocked: null
|
|
5731
|
+
};
|
|
5732
|
+
} catch (err) {
|
|
5733
|
+
return {
|
|
5734
|
+
steps: spec.steps.map((step, i) => {
|
|
5735
|
+
const body = isIncludeStep(step) ? {
|
|
5736
|
+
instruction: `include block: ${step.include}`,
|
|
5737
|
+
expected: ""
|
|
5738
|
+
} : step;
|
|
5739
|
+
return {
|
|
5740
|
+
id: stepId(i + 1),
|
|
5741
|
+
source: "case",
|
|
5742
|
+
...body
|
|
5743
|
+
};
|
|
5744
|
+
}),
|
|
5745
|
+
blocked: err instanceof Error ? err.message : String(err)
|
|
5746
|
+
};
|
|
5747
|
+
}
|
|
5748
|
+
}
|
|
5749
|
+
/**
|
|
5750
|
+
* Lift what a source answered (see `./contract.ts`) into the shape every
|
|
5751
|
+
* feature consumes.
|
|
5752
|
+
*/
|
|
5753
|
+
function caseFromDocument(read, cwd) {
|
|
5982
5754
|
return {
|
|
5983
|
-
ref:
|
|
5984
|
-
title:
|
|
5985
|
-
mode:
|
|
5986
|
-
steps:
|
|
5987
|
-
id: stepId(
|
|
5755
|
+
ref: sourcedCase(read.id, cwd),
|
|
5756
|
+
title: read.title,
|
|
5757
|
+
mode: read.mode,
|
|
5758
|
+
steps: read.steps.map((step, i) => ({
|
|
5759
|
+
id: stepId(i + 1),
|
|
5988
5760
|
source: "case",
|
|
5989
|
-
instruction: step.
|
|
5990
|
-
expected: ""
|
|
5761
|
+
instruction: step.instruction,
|
|
5762
|
+
expected: step.expected ?? ""
|
|
5991
5763
|
})),
|
|
5992
|
-
cleanup:
|
|
5993
|
-
id: cleanupId(
|
|
5764
|
+
cleanup: (read.cleanup ?? []).map((step, i) => ({
|
|
5765
|
+
id: cleanupId(i + 1),
|
|
5994
5766
|
source: "cleanup",
|
|
5995
|
-
instruction: step.
|
|
5996
|
-
expected: ""
|
|
5767
|
+
instruction: step.instruction,
|
|
5768
|
+
expected: step.expected ?? ""
|
|
5997
5769
|
})),
|
|
5998
|
-
expectations:
|
|
5999
|
-
cleanupExpectations:
|
|
6000
|
-
context:
|
|
5770
|
+
expectations: [...read.expectations ?? []],
|
|
5771
|
+
cleanupExpectations: [...read.cleanupExpectations ?? []],
|
|
5772
|
+
context: [...read.context ?? []],
|
|
6001
5773
|
fields: {
|
|
6002
|
-
case:
|
|
6003
|
-
title:
|
|
6004
|
-
...
|
|
6005
|
-
...parsed.link.url ? { "link.url": parsed.link.url } : {},
|
|
6006
|
-
...parsed.link.ref ? { "link.ref": parsed.link.ref } : {}
|
|
5774
|
+
case: read.id,
|
|
5775
|
+
title: read.title,
|
|
5776
|
+
...read.fields
|
|
6007
5777
|
},
|
|
6008
|
-
|
|
6009
|
-
|
|
6010
|
-
|
|
6011
|
-
|
|
6012
|
-
|
|
6013
|
-
}
|
|
5778
|
+
disabled: read.disabled === true,
|
|
5779
|
+
blocked: null,
|
|
5780
|
+
document: {
|
|
5781
|
+
path: read.path,
|
|
5782
|
+
text: read.text
|
|
5783
|
+
},
|
|
5784
|
+
spec: null
|
|
6014
5785
|
};
|
|
6015
5786
|
}
|
|
5787
|
+
//#endregion
|
|
5788
|
+
//#region src/cases/case-schema.ts
|
|
6016
5789
|
/**
|
|
6017
|
-
* The
|
|
6018
|
-
*
|
|
6019
|
-
*
|
|
6020
|
-
*
|
|
5790
|
+
* The published contract, as something ccqa can check an answer against.
|
|
5791
|
+
*
|
|
5792
|
+
* Internal on purpose: `ccqa/case-source` exports types only, so a reader
|
|
5793
|
+
* module never has to resolve ccqa's zod. The two are kept in step by the
|
|
5794
|
+
* assertions below rather than by one being derived from the other — the
|
|
5795
|
+
* public shape stays hand-written and readable, and a field added to either
|
|
5796
|
+
* without the other stops the build.
|
|
6021
5797
|
*/
|
|
6022
|
-
|
|
6023
|
-
|
|
6024
|
-
|
|
6025
|
-
|
|
6026
|
-
|
|
6027
|
-
|
|
6028
|
-
|
|
6029
|
-
|
|
6030
|
-
|
|
6031
|
-
|
|
6032
|
-
|
|
6033
|
-
|
|
6034
|
-
|
|
6035
|
-
|
|
5798
|
+
const CaseStepSchema = z.object({
|
|
5799
|
+
instruction: z.string().min(1),
|
|
5800
|
+
expected: z.string().optional()
|
|
5801
|
+
}).strict();
|
|
5802
|
+
const CaseSchema = z.object({
|
|
5803
|
+
id: z.string().min(1).regex(/^[^/\\][^\\]*$/, "a case id is a '/'-separated path with no leading slash and no backslashes").refine((id) => !id.endsWith("/") && !id.split("/").includes(".."), { message: "a case id must not end in '/' or contain '..'" }),
|
|
5804
|
+
path: z.string().min(1),
|
|
5805
|
+
text: z.string(),
|
|
5806
|
+
title: z.string().min(1),
|
|
5807
|
+
mode: z.enum(["deterministic", "live"]),
|
|
5808
|
+
steps: z.array(CaseStepSchema).min(1),
|
|
5809
|
+
cleanup: z.array(CaseStepSchema).optional(),
|
|
5810
|
+
expectations: z.array(z.string().min(1)).optional(),
|
|
5811
|
+
cleanupExpectations: z.array(z.string().min(1)).optional(),
|
|
5812
|
+
context: z.array(z.object({
|
|
5813
|
+
heading: z.string().min(1),
|
|
5814
|
+
body: z.string()
|
|
5815
|
+
}).strict()).optional(),
|
|
5816
|
+
fields: z.record(z.string().min(1), z.string()).optional(),
|
|
5817
|
+
disabled: z.boolean().optional()
|
|
5818
|
+
}).strict();
|
|
5819
|
+
//#endregion
|
|
5820
|
+
//#region src/cases/module-source.ts
|
|
6036
5821
|
/**
|
|
6037
|
-
*
|
|
5822
|
+
* The project's own case reader, loaded from the module `targets.<id>.cases`
|
|
5823
|
+
* names.
|
|
6038
5824
|
*
|
|
6039
|
-
*
|
|
6040
|
-
*
|
|
6041
|
-
*
|
|
6042
|
-
*
|
|
6043
|
-
* A `.md` that does not parse as a case is not one: the directory holds a
|
|
6044
|
-
* project's own documents, and a README or a template sitting beside the
|
|
6045
|
-
* cases must not be swept into a run or an audit. Naming such a file on the
|
|
6046
|
-
* command line still fails loudly — this only decides what a sweep picks up.
|
|
5825
|
+
* ccqa imports it and validates everything it returns. Nothing about the
|
|
5826
|
+
* project's format reaches ccqa — not a heading, not a bullet character, not a
|
|
5827
|
+
* punctuation mark — which is the point: a format-specific accommodation has
|
|
5828
|
+
* nowhere here to land.
|
|
6047
5829
|
*/
|
|
6048
|
-
|
|
6049
|
-
|
|
6050
|
-
|
|
6051
|
-
|
|
6052
|
-
|
|
6053
|
-
|
|
6054
|
-
|
|
6055
|
-
|
|
6056
|
-
|
|
6057
|
-
|
|
6058
|
-
|
|
6059
|
-
|
|
5830
|
+
/** Extensions Node can import on its own, on every version ccqa supports. */
|
|
5831
|
+
const LOADABLE = [
|
|
5832
|
+
".mjs",
|
|
5833
|
+
".js",
|
|
5834
|
+
".cjs"
|
|
5835
|
+
];
|
|
5836
|
+
function moduleCaseSource(modulePath, cwd, targetId) {
|
|
5837
|
+
const at = `targets.${targetId}.cases (${modulePath})`;
|
|
5838
|
+
let opened = null;
|
|
5839
|
+
const source = () => opened ??= open$1(modulePath, at, cwd);
|
|
5840
|
+
return {
|
|
5841
|
+
async list() {
|
|
5842
|
+
const listed = await (await source()).list();
|
|
5843
|
+
if (!Array.isArray(listed) || listed.some((id) => typeof id !== "string")) throw new RunUsageError(`${at}: list() must return an array of case ids`);
|
|
5844
|
+
return [...new Set(listed)].sort();
|
|
5845
|
+
},
|
|
5846
|
+
idFor: (ref) => ref,
|
|
5847
|
+
async read(ref) {
|
|
5848
|
+
const absent = (error) => ({
|
|
5849
|
+
id: ref,
|
|
5850
|
+
case: null,
|
|
5851
|
+
document: null,
|
|
5852
|
+
error
|
|
6060
5853
|
});
|
|
5854
|
+
let loaded;
|
|
5855
|
+
try {
|
|
5856
|
+
loaded = CaseSchema.parse(await (await source()).load(ref));
|
|
5857
|
+
} catch (err) {
|
|
5858
|
+
if (err instanceof RunUsageError) throw err;
|
|
5859
|
+
if (err instanceof ZodError) return absent(`${at}: load(${JSON.stringify(ref)}) returned ${describe(err)}`);
|
|
5860
|
+
return absent(errMessage(err));
|
|
5861
|
+
}
|
|
5862
|
+
return {
|
|
5863
|
+
id: loaded.id,
|
|
5864
|
+
case: caseFromDocument(loaded, cwd),
|
|
5865
|
+
document: {
|
|
5866
|
+
path: loaded.path,
|
|
5867
|
+
text: loaded.text
|
|
5868
|
+
},
|
|
5869
|
+
error: null
|
|
5870
|
+
};
|
|
6061
5871
|
}
|
|
6062
5872
|
};
|
|
6063
|
-
|
|
6064
|
-
|
|
6065
|
-
|
|
6066
|
-
|
|
6067
|
-
|
|
6068
|
-
|
|
6069
|
-
|
|
6070
|
-
|
|
6071
|
-
|
|
6072
|
-
|
|
6073
|
-
|
|
6074
|
-
|
|
6075
|
-
|
|
6076
|
-
|
|
6077
|
-
|
|
5873
|
+
}
|
|
5874
|
+
async function open$1(modulePath, at, cwd) {
|
|
5875
|
+
if (!LOADABLE.some((ext) => modulePath.endsWith(ext))) throw new RunUsageError(`${at}: cannot be loaded — ${notJavaScript()}`);
|
|
5876
|
+
let mod;
|
|
5877
|
+
try {
|
|
5878
|
+
mod = await import(pathToFileURL(modulePath).href);
|
|
5879
|
+
} catch (err) {
|
|
5880
|
+
const hint = err?.code === "ERR_UNKNOWN_FILE_EXTENSION" ? ` ${notJavaScript()}` : "";
|
|
5881
|
+
throw new RunUsageError(`${at}: could not be loaded — ${errMessage(err)}${hint}`);
|
|
5882
|
+
}
|
|
5883
|
+
if (typeof mod.default !== "function") throw new RunUsageError(`${at}: must default-export a function taking { cwd } and returning { list, load } — see https://github.com/shibukazu/ccqa/blob/main/docs/targets.md`);
|
|
5884
|
+
const opened = await mod.default({ cwd });
|
|
5885
|
+
if (typeof opened !== "object" || opened === null || typeof opened.list !== "function" || typeof opened.load !== "function") throw new RunUsageError(`${at}: the default export must return an object with list() and load()`);
|
|
5886
|
+
return opened;
|
|
5887
|
+
}
|
|
5888
|
+
/**
|
|
5889
|
+
* The mistake that is actually likely, in the terms it is made in. Consumers
|
|
5890
|
+
* write TypeScript, and an extension error alone reads as a broken install.
|
|
5891
|
+
*/
|
|
5892
|
+
function notJavaScript() {
|
|
5893
|
+
return `a case source is plain JavaScript (${LOADABLE.join(", ")}): ccqa imports it with no loader, so a TypeScript file cannot be read. Write it as .mjs — \`// @ts-check\` plus \`@type {import("ccqa/case-source").CaseSourceFactory}\` type-checks it without a build step — or point \`cases\` at compiled output.`;
|
|
5894
|
+
}
|
|
5895
|
+
/** A zod rejection as one line per offending field. */
|
|
5896
|
+
function describe(err) {
|
|
5897
|
+
return `a value that is not a case — ${err.issues.map((issue) => {
|
|
5898
|
+
return `${issue.path.join(".") || "(the case)"}: ${issue.message}`;
|
|
5899
|
+
}).join("; ")}`;
|
|
5900
|
+
}
|
|
5901
|
+
//#endregion
|
|
5902
|
+
//#region src/cases/spec-source.ts
|
|
5903
|
+
/**
|
|
5904
|
+
* ccqa's own `spec.yaml` as a case source.
|
|
5905
|
+
*
|
|
5906
|
+
* The functions that read a spec file live here rather than in the store, so
|
|
5907
|
+
* that a feature wanting a case document has no import that reaches one kind
|
|
5908
|
+
* of case directly. The store still says where the file is (`specFilePath`)
|
|
5909
|
+
* and still enumerates the tree — the layout is its business; reading a case
|
|
5910
|
+
* is not.
|
|
5911
|
+
*/
|
|
5912
|
+
/** One case's `spec.yaml`, null when there is no such file. */
|
|
5913
|
+
async function tryReadSpecFile(featureName, specName, cwd) {
|
|
5914
|
+
return readFile(specFilePath(featureName, specName, cwd), "utf-8").catch(() => null);
|
|
6078
5915
|
}
|
|
6079
5916
|
/**
|
|
6080
|
-
*
|
|
5917
|
+
* Lists every feature/spec dir under .ccqa/features/, regardless of whether
|
|
5918
|
+
* the spec is fully drafted yet. Each spec file is read at most once.
|
|
6081
5919
|
*
|
|
6082
|
-
*
|
|
6083
|
-
*
|
|
5920
|
+
* Only `ccqa draft` still asks for this: it authors ccqa's own format, so it
|
|
5921
|
+
* is the one command whose subject is the tree rather than the cases in it.
|
|
6084
5922
|
*/
|
|
6085
|
-
async function
|
|
6086
|
-
|
|
6087
|
-
|
|
5923
|
+
async function listFeatureTree(cwd) {
|
|
5924
|
+
const featuresDir = join(getCcqaDir(cwd), "features");
|
|
5925
|
+
const featureDirs = await readdir(featuresDir).catch(() => []);
|
|
5926
|
+
return Promise.all(featureDirs.map(async (featureName) => {
|
|
5927
|
+
const testCasesDir = join(featuresDir, featureName, "test-cases");
|
|
5928
|
+
const specDirs = await readdir(testCasesDir).catch(() => []);
|
|
5929
|
+
return {
|
|
5930
|
+
featureName,
|
|
5931
|
+
specs: await Promise.all(specDirs.map(async (specName) => {
|
|
5932
|
+
const specFile = join(testCasesDir, specName, "spec.yaml");
|
|
5933
|
+
const content = await readFile(specFile, "utf-8").catch(() => null);
|
|
5934
|
+
if (content === null) return {
|
|
5935
|
+
specName,
|
|
5936
|
+
hasSpecFile: false
|
|
5937
|
+
};
|
|
5938
|
+
try {
|
|
5939
|
+
return {
|
|
5940
|
+
specName,
|
|
5941
|
+
hasSpecFile: true,
|
|
5942
|
+
includedBlocks: collectIncludedBlockNames(parseTestSpec(content, specFile))
|
|
5943
|
+
};
|
|
5944
|
+
} catch {
|
|
5945
|
+
return {
|
|
5946
|
+
specName,
|
|
5947
|
+
hasSpecFile: true
|
|
5948
|
+
};
|
|
5949
|
+
}
|
|
5950
|
+
}))
|
|
5951
|
+
};
|
|
5952
|
+
}));
|
|
5953
|
+
}
|
|
5954
|
+
function specCaseSource(cwd) {
|
|
5955
|
+
let blocks = null;
|
|
5956
|
+
return {
|
|
5957
|
+
async list() {
|
|
5958
|
+
return (await listAllSpecsWithSpecFile(cwd)).map(specKey).sort();
|
|
5959
|
+
},
|
|
5960
|
+
idFor: (ref) => specKey(parseSpecPath(ref)),
|
|
5961
|
+
async read(id) {
|
|
5962
|
+
const { featureName, specName } = parseSpecPath(id);
|
|
5963
|
+
const path = specFilePath(featureName, specName, cwd);
|
|
5964
|
+
const yaml = await tryReadSpecFile(featureName, specName, cwd);
|
|
5965
|
+
if (yaml === null) return {
|
|
5966
|
+
id,
|
|
5967
|
+
case: null,
|
|
5968
|
+
document: null,
|
|
5969
|
+
error: `Spec file not found: ${path}`
|
|
5970
|
+
};
|
|
5971
|
+
const document = {
|
|
5972
|
+
path,
|
|
5973
|
+
text: yaml
|
|
5974
|
+
};
|
|
5975
|
+
blocks ??= loadAllBlocks(cwd);
|
|
5976
|
+
try {
|
|
5977
|
+
return {
|
|
5978
|
+
id,
|
|
5979
|
+
case: caseFromSpec(featureName, specName, yaml, await blocks, cwd),
|
|
5980
|
+
document,
|
|
5981
|
+
error: null
|
|
5982
|
+
};
|
|
5983
|
+
} catch (err) {
|
|
5984
|
+
return {
|
|
5985
|
+
id,
|
|
5986
|
+
case: null,
|
|
5987
|
+
document,
|
|
5988
|
+
error: errMessage(err)
|
|
5989
|
+
};
|
|
5990
|
+
}
|
|
5991
|
+
}
|
|
5992
|
+
};
|
|
6088
5993
|
}
|
|
5994
|
+
//#endregion
|
|
5995
|
+
//#region src/cases/reader.ts
|
|
6089
5996
|
/**
|
|
6090
|
-
*
|
|
6091
|
-
*
|
|
6092
|
-
*
|
|
6093
|
-
*
|
|
5997
|
+
* The target whose cases live in the project, when that is what we resolve to.
|
|
5998
|
+
*
|
|
5999
|
+
* Which of the two places a project keeps its cases in is settled by its
|
|
6000
|
+
* target, not by guessing at an argument's shape: a target that declares a
|
|
6001
|
+
* case source of its own reads the project's documents, and one that does not
|
|
6002
|
+
* reads ccqa's `spec.yaml`.
|
|
6094
6003
|
*/
|
|
6095
|
-
function
|
|
6096
|
-
const
|
|
6097
|
-
const
|
|
6098
|
-
|
|
6004
|
+
function caseTargetFor(config, cwd, targetOverride) {
|
|
6005
|
+
const id = targetOverride ?? config.defaultTarget;
|
|
6006
|
+
const targetConfig = config.targets[id];
|
|
6007
|
+
const module = targetConfig?.cases;
|
|
6008
|
+
if (targetConfig === void 0 || module === void 0) return null;
|
|
6009
|
+
return {
|
|
6010
|
+
id,
|
|
6011
|
+
targetConfig,
|
|
6012
|
+
module,
|
|
6013
|
+
modulePath: isAbsolute(module) ? module : resolve(cwd, module)
|
|
6014
|
+
};
|
|
6015
|
+
}
|
|
6016
|
+
function openCaseReader(config, cwd, opts = {}) {
|
|
6017
|
+
const target = caseTargetFor(config, cwd, opts.targetOverride);
|
|
6018
|
+
const source = target ? moduleCaseSource(target.modulePath, cwd, target.id) : specCaseSource(cwd);
|
|
6019
|
+
const byId = /* @__PURE__ */ new Map();
|
|
6020
|
+
const read = (ref) => {
|
|
6021
|
+
const key = source.idFor(ref);
|
|
6022
|
+
const cached = byId.get(key);
|
|
6023
|
+
if (cached !== void 0) return cached;
|
|
6024
|
+
const pending = source.read(key);
|
|
6025
|
+
byId.set(key, pending);
|
|
6026
|
+
pending.then((result) => {
|
|
6027
|
+
if (!byId.has(result.id)) byId.set(result.id, pending);
|
|
6028
|
+
}, () => {});
|
|
6029
|
+
return pending;
|
|
6030
|
+
};
|
|
6031
|
+
return {
|
|
6032
|
+
list: () => source.list(),
|
|
6033
|
+
read,
|
|
6034
|
+
async load(ref) {
|
|
6035
|
+
const result = await read(ref);
|
|
6036
|
+
if (result.case === null) throw new RunUsageError(result.error ?? `no test case named "${ref}"`);
|
|
6037
|
+
if (result.case.blocked !== null) throw new RunUsageError(result.case.blocked);
|
|
6038
|
+
return result.case;
|
|
6039
|
+
},
|
|
6040
|
+
target
|
|
6041
|
+
};
|
|
6099
6042
|
}
|
|
6100
6043
|
//#endregion
|
|
6101
6044
|
//#region src/targets/agent-browser/judge-steps.ts
|
|
@@ -13833,13 +13776,14 @@ async function loadEvidenceForSpec(evidenceDir, reportDir, descriptionByStepId)
|
|
|
13833
13776
|
const reportRoot = resolve(reportDir);
|
|
13834
13777
|
const jsonFiles = entries.filter((n) => n.endsWith(".json"));
|
|
13835
13778
|
const metas = (await Promise.all(jsonFiles.map((name) => readEvidenceMeta(join(evidenceDir, name), evidenceDir, reportRoot, descriptionByStepId)))).filter((m) => m !== null);
|
|
13836
|
-
metas.sort((a, b) =>
|
|
13837
|
-
if (a.stepId === "failure") return 1;
|
|
13838
|
-
if (b.stepId === "failure") return -1;
|
|
13839
|
-
return a.stepId.localeCompare(b.stepId);
|
|
13840
|
-
});
|
|
13779
|
+
metas.sort((a, b) => phase(a.stepId) - phase(b.stepId) || a.stepId.localeCompare(b.stepId));
|
|
13841
13780
|
return metas.length > 0 ? metas : null;
|
|
13842
13781
|
}
|
|
13782
|
+
/** Steps, then cleanup, then the failure capture. */
|
|
13783
|
+
function phase(stepId) {
|
|
13784
|
+
if (stepId === "failure") return 2;
|
|
13785
|
+
return stepId.startsWith("cleanup-") ? 1 : 0;
|
|
13786
|
+
}
|
|
13843
13787
|
async function readEvidenceMeta(metaPath, evidenceDir, reportRoot, descriptionByStepId) {
|
|
13844
13788
|
let raw;
|
|
13845
13789
|
try {
|
|
@@ -13856,7 +13800,7 @@ async function readEvidenceMeta(metaPath, evidenceDir, reportRoot, descriptionBy
|
|
|
13856
13800
|
if (typeof parsed !== "object" || parsed === null) return null;
|
|
13857
13801
|
const pngFile = parsed.pngFile;
|
|
13858
13802
|
if (typeof pngFile !== "string") return null;
|
|
13859
|
-
const relToReport = (file) => posix.relative(toPosix(reportRoot), toPosix(join(evidenceDir, file)));
|
|
13803
|
+
const relToReport = (file) => posix.relative(toPosix$1(reportRoot), toPosix$1(join(evidenceDir, file)));
|
|
13860
13804
|
const pngPath = relToReport(pngFile);
|
|
13861
13805
|
const beforePngFile = parsed.beforePngFile;
|
|
13862
13806
|
const beforePngPath = typeof beforePngFile === "string" ? relToReport(beforePngFile) : null;
|
|
@@ -13878,20 +13822,27 @@ async function readEvidenceMeta(metaPath, evidenceDir, reportRoot, descriptionBy
|
|
|
13878
13822
|
return result.success ? result.data : null;
|
|
13879
13823
|
}
|
|
13880
13824
|
/**
|
|
13881
|
-
* Build `step id →
|
|
13882
|
-
*
|
|
13883
|
-
*
|
|
13825
|
+
* Build `step id → caption` so the report can label each evidence screenshot.
|
|
13826
|
+
* Empty when the case could not be read — evidence still surfaces, just
|
|
13827
|
+
* without captions.
|
|
13828
|
+
*
|
|
13829
|
+
* What the step must make true, falling back to what it does. A case that
|
|
13830
|
+
* states its expectations once for the whole flow leaves every step's own
|
|
13831
|
+
* `expected` empty, and a filmstrip captioned with blanks says less than the
|
|
13832
|
+
* instructions the case's author wrote.
|
|
13833
|
+
*
|
|
13834
|
+
* Cleanup steps are captioned too: they have step ids of their own and leave
|
|
13835
|
+
* screenshots of their own, and an uncaptioned row reads as a step nobody
|
|
13836
|
+
* described rather than as the undo it is.
|
|
13884
13837
|
*/
|
|
13885
|
-
function
|
|
13886
|
-
if (!
|
|
13887
|
-
|
|
13888
|
-
|
|
13889
|
-
return
|
|
13890
|
-
}
|
|
13891
|
-
return /* @__PURE__ */ new Map();
|
|
13892
|
-
}
|
|
13838
|
+
function stepCaptions(testCase) {
|
|
13839
|
+
if (!testCase) return /* @__PURE__ */ new Map();
|
|
13840
|
+
return new Map([...testCase.steps, ...testCase.cleanup].map((s) => {
|
|
13841
|
+
if (isExpandedJudgeByLlmStep(s)) return [s.id, s.judgeByLlm.trim()];
|
|
13842
|
+
return [s.id, (s.expected.trim() || s.instruction).trim()];
|
|
13843
|
+
}));
|
|
13893
13844
|
}
|
|
13894
|
-
function toPosix(p) {
|
|
13845
|
+
function toPosix$1(p) {
|
|
13895
13846
|
return p.split(/[\\/]/).join("/");
|
|
13896
13847
|
}
|
|
13897
13848
|
//#endregion
|
|
@@ -13990,14 +13941,14 @@ function parseTraceEvents(jsonl) {
|
|
|
13990
13941
|
});
|
|
13991
13942
|
} else if (type === "action") {
|
|
13992
13943
|
if (typeof event.callId === "string" && typeof event.startTime === "number" && typeof event.endTime === "number") steps.push({
|
|
13993
|
-
title: titleOf(event),
|
|
13944
|
+
title: titleOf$1(event),
|
|
13994
13945
|
startTime: event.startTime,
|
|
13995
13946
|
endTime: event.endTime,
|
|
13996
13947
|
...errorOf(event)
|
|
13997
13948
|
});
|
|
13998
13949
|
} else if (type === "before") {
|
|
13999
13950
|
if (typeof event.callId === "string" && typeof event.startTime === "number") pendingByCallId.set(event.callId, {
|
|
14000
|
-
title: titleOf(event),
|
|
13951
|
+
title: titleOf$1(event),
|
|
14001
13952
|
startTime: event.startTime
|
|
14002
13953
|
});
|
|
14003
13954
|
} else if (type === "after") {
|
|
@@ -14035,7 +13986,7 @@ function errorOf(event) {
|
|
|
14035
13986
|
}
|
|
14036
13987
|
return {};
|
|
14037
13988
|
}
|
|
14038
|
-
function titleOf(event) {
|
|
13989
|
+
function titleOf$1(event) {
|
|
14039
13990
|
if (typeof event.apiName === "string") return event.apiName;
|
|
14040
13991
|
if (typeof event.title === "string") return event.title;
|
|
14041
13992
|
return "";
|
|
@@ -14208,12 +14159,11 @@ function substituteRunCommandFiles(runCommand, testFiles) {
|
|
|
14208
14159
|
*/
|
|
14209
14160
|
const runCommandRunner = { async run(specs, opts) {
|
|
14210
14161
|
const concurrency = Math.max(1, opts.concurrency);
|
|
14211
|
-
const blocks = opts.stepEvidence.supported ? await loadAllBlocks(opts.cwd) : /* @__PURE__ */ new Map();
|
|
14212
14162
|
return runPool(specs, concurrency, async (spec) => {
|
|
14213
14163
|
const key = `${spec.featureName}/${spec.specName}`;
|
|
14214
14164
|
let row;
|
|
14215
14165
|
try {
|
|
14216
|
-
row = await withBuffer(key, concurrency > 1, () => runOneSpec(spec, opts
|
|
14166
|
+
row = await withBuffer(key, concurrency > 1, () => runOneSpec(spec, opts));
|
|
14217
14167
|
} catch (err) {
|
|
14218
14168
|
const message = err instanceof Error ? err.message : String(err);
|
|
14219
14169
|
error(`${key}: runner error: ${message}`);
|
|
@@ -14237,11 +14187,10 @@ const runCommandRunner = { async run(specs, opts) {
|
|
|
14237
14187
|
return row;
|
|
14238
14188
|
}, { resources: opts.resources });
|
|
14239
14189
|
} };
|
|
14240
|
-
async function runOneSpec(ref, opts
|
|
14241
|
-
const { featureName, specName } = ref;
|
|
14242
|
-
const specYaml =
|
|
14243
|
-
const
|
|
14244
|
-
const title = parsedSpec?.title ?? null;
|
|
14190
|
+
async function runOneSpec(ref, opts) {
|
|
14191
|
+
const { featureName, specName, testCase } = ref;
|
|
14192
|
+
const specYaml = testCase?.document.text ?? null;
|
|
14193
|
+
const title = testCase?.title ?? null;
|
|
14245
14194
|
const failedRow = (detail) => ({
|
|
14246
14195
|
...emptySpecRow({
|
|
14247
14196
|
feature: featureName,
|
|
@@ -14267,9 +14216,9 @@ async function runOneSpec(ref, opts, blocks) {
|
|
|
14267
14216
|
const runCommand = opts.targetConfig.runCommand;
|
|
14268
14217
|
if (runCommand === void 0) return didNotExecute(`target "${opts.targetId}" has no runCommand configured in .ccqa/config.yaml`, "the target has no runCommand");
|
|
14269
14218
|
run(`${featureName}/${specName}`);
|
|
14270
|
-
const testFile =
|
|
14219
|
+
const testFile = resolveCaseTestPath(opts, opts.targetConfig, ref.caseId);
|
|
14271
14220
|
if (!await stat(resolve(opts.cwd, testFile)).then(() => true, () => false)) {
|
|
14272
|
-
const detail = `no generated test at ${testFile} — run 'ccqa generate ${
|
|
14221
|
+
const detail = `no generated test at ${testFile} — run 'ccqa generate ${ref.caseId}' first`;
|
|
14273
14222
|
error(detail);
|
|
14274
14223
|
return didNotExecute(detail, "no generated tests");
|
|
14275
14224
|
}
|
|
@@ -14368,6 +14317,7 @@ async function runOneSpec(ref, opts, blocks) {
|
|
|
14368
14317
|
reportDir: opts.reportDir,
|
|
14369
14318
|
feature: featureName,
|
|
14370
14319
|
spec: specName,
|
|
14320
|
+
passed: outcome.exitCode === 0,
|
|
14371
14321
|
warn
|
|
14372
14322
|
});
|
|
14373
14323
|
} catch (err) {
|
|
@@ -14378,8 +14328,7 @@ async function runOneSpec(ref, opts, blocks) {
|
|
|
14378
14328
|
opts,
|
|
14379
14329
|
evidence,
|
|
14380
14330
|
artifactsDir,
|
|
14381
|
-
|
|
14382
|
-
blocks,
|
|
14331
|
+
testCase,
|
|
14383
14332
|
...traceUnavailable !== void 0 ? { traceUnavailable } : {}
|
|
14384
14333
|
});
|
|
14385
14334
|
if (outcome.exitCode === 0) return {
|
|
@@ -14436,7 +14385,7 @@ async function loadStepEvidence(args) {
|
|
|
14436
14385
|
artifactsDir: args.artifactsDir,
|
|
14437
14386
|
evidenceDir
|
|
14438
14387
|
});
|
|
14439
|
-
const descriptions =
|
|
14388
|
+
const descriptions = stepCaptions(args.testCase);
|
|
14440
14389
|
const evidence = await loadEvidenceForSpec(evidenceDir, opts.reportDir, descriptions);
|
|
14441
14390
|
if (evidence) return { evidence };
|
|
14442
14391
|
return {
|
|
@@ -16374,90 +16323,122 @@ function resolveTargetFrom(spec, config, registry) {
|
|
|
16374
16323
|
return plugin;
|
|
16375
16324
|
}
|
|
16376
16325
|
//#endregion
|
|
16377
|
-
//#region src/
|
|
16378
|
-
async function resolveCase(argument, config, cwd, opts = {}) {
|
|
16379
|
-
const intentTarget = intentTargetFor(config, opts.targetOverride);
|
|
16380
|
-
if (intentTarget) {
|
|
16381
|
-
const { id, targetConfig, intent } = intentTarget;
|
|
16382
|
-
const target = registryFor(config).get(id);
|
|
16383
|
-
const testCase = await loadMarkdownCase(argument, intent, cwd);
|
|
16384
|
-
const testPath = resolveCaseTestPath(target, targetConfig, testCase.ref.id);
|
|
16385
|
-
const recordingPath = (opts.targetOverride === void 0 ? void 0 : ownIntentRecordingPath(config, intent, testCase.ref.id)) ?? resolveCaseRecordingPath(target, targetConfig, testCase.ref.id);
|
|
16386
|
-
const ref = caseRefFor(testCase.ref.id, cwd, recordingPath);
|
|
16387
|
-
return {
|
|
16388
|
-
testCase: {
|
|
16389
|
-
...testCase,
|
|
16390
|
-
ref
|
|
16391
|
-
},
|
|
16392
|
-
target,
|
|
16393
|
-
targetConfig,
|
|
16394
|
-
testPath
|
|
16395
|
-
};
|
|
16396
|
-
}
|
|
16397
|
-
const { featureName, specName } = parseSpecPath(argument);
|
|
16398
|
-
const testCase = caseFromSpec(featureName, specName, await readSpecFile(featureName, specName, cwd), await loadAllBlocks(cwd), cwd);
|
|
16399
|
-
const spec = testCase.source.kind === "spec" ? testCase.source.spec : null;
|
|
16400
|
-
const target = opts.targetOverride !== void 0 ? resolveTargetOverride(spec, opts.targetOverride, config) : resolveTarget(spec, config);
|
|
16401
|
-
const targetConfig = targetConfigFor(config, target.id);
|
|
16402
|
-
const specRef = {
|
|
16403
|
-
featureName,
|
|
16404
|
-
specName
|
|
16405
|
-
};
|
|
16406
|
-
const testPath = resolveTestPath(target, targetConfig, specRef);
|
|
16407
|
-
const ref = caseRefFor(specRef, cwd, opts.targetOverride === void 0 ? resolveRecordingPath(target, targetConfig, specRef) : ownRecordingPath(spec, config, specRef));
|
|
16408
|
-
return {
|
|
16409
|
-
testCase: {
|
|
16410
|
-
...testCase,
|
|
16411
|
-
ref
|
|
16412
|
-
},
|
|
16413
|
-
target,
|
|
16414
|
-
targetConfig,
|
|
16415
|
-
testPath
|
|
16416
|
-
};
|
|
16417
|
-
}
|
|
16326
|
+
//#region src/drift/renames.ts
|
|
16418
16327
|
/**
|
|
16419
|
-
*
|
|
16420
|
-
* `--target`. Undefined when that target no longer resolves: overriding the
|
|
16421
|
-
* target is how a spec with an unusable one is generated at all, so it must
|
|
16422
|
-
* not be what stops the command.
|
|
16328
|
+
* What the audit says was renamed, and which of a case's files still say it.
|
|
16423
16329
|
*
|
|
16424
|
-
*
|
|
16425
|
-
*
|
|
16426
|
-
*
|
|
16330
|
+
* A rename is the one finding a machine can act on without reading prose, but
|
|
16331
|
+
* only once it is known *where* the old string survives: the case's document
|
|
16332
|
+
* decides whether an edit is needed, and the saved recording decides whether
|
|
16333
|
+
* recompiling that document is enough (ADR-0035). Both answers are string
|
|
16334
|
+
* comparisons against text ccqa already holds, which is what keeps them out of
|
|
16335
|
+
* the model's hands.
|
|
16427
16336
|
*/
|
|
16428
|
-
|
|
16429
|
-
|
|
16430
|
-
|
|
16431
|
-
|
|
16432
|
-
|
|
16433
|
-
|
|
16434
|
-
|
|
16435
|
-
|
|
16337
|
+
/**
|
|
16338
|
+
* A string the test case still uses, and what the product renders in its place.
|
|
16339
|
+
*
|
|
16340
|
+
* Answered by the audit, never parsed back out of its prose: a quoted run is a
|
|
16341
|
+
* guess about one project's punctuation (ADR-0035).
|
|
16342
|
+
*/
|
|
16343
|
+
const RenameSchema = z.object({
|
|
16344
|
+
from: z.string(),
|
|
16345
|
+
to: z.string()
|
|
16346
|
+
});
|
|
16347
|
+
/**
|
|
16348
|
+
* The audit's renames, sanitized and checked against the document.
|
|
16349
|
+
*
|
|
16350
|
+
* A blank `from` matches everywhere and an identity pair edits nothing, so
|
|
16351
|
+
* neither is an instruction. Blank is tested after trimming and kept
|
|
16352
|
+
* untrimmed: a trimmed `from` is a string ccqa never checked the document for.
|
|
16353
|
+
* Two pairs naming the same `from` differently are the audit contradicting
|
|
16354
|
+
* itself, and both go — picking a winner would invent an answer nobody gave.
|
|
16355
|
+
*/
|
|
16356
|
+
function auditedRenames(document, renames) {
|
|
16357
|
+
const byFrom = /* @__PURE__ */ new Map();
|
|
16358
|
+
for (const r of renames) {
|
|
16359
|
+
if (r.from.trim() === "" || r.to.trim() === "" || r.from === r.to) continue;
|
|
16360
|
+
const kept = byFrom.get(r.from);
|
|
16361
|
+
if (kept === void 0) byFrom.set(r.from, r);
|
|
16362
|
+
else if (kept !== null && kept.to !== r.to) byFrom.set(r.from, null);
|
|
16363
|
+
}
|
|
16364
|
+
return [...byFrom.values()].filter((r) => r !== null).map((r) => ({
|
|
16365
|
+
...r,
|
|
16366
|
+
inDocument: document.includes(r.from)
|
|
16367
|
+
}));
|
|
16436
16368
|
}
|
|
16437
16369
|
/**
|
|
16438
|
-
*
|
|
16439
|
-
*
|
|
16370
|
+
* Whether the saved recording still names a string the audit says was renamed.
|
|
16371
|
+
*
|
|
16372
|
+
* It decides between the two machine repairs: a recording that names the old
|
|
16373
|
+
* string compiles it back in, so the case has to be recorded again rather than
|
|
16374
|
+
* regenerated. Missing one is not dangerous — the case routes to `regenerate`,
|
|
16375
|
+
* whose replay gate then refuses the dead recording — but it costs a browser
|
|
16376
|
+
* run to find that out.
|
|
16440
16377
|
*
|
|
16441
|
-
*
|
|
16442
|
-
*
|
|
16443
|
-
* describing a different case that happens to share an id, and its route is
|
|
16444
|
-
* not this case's.
|
|
16378
|
+
* Walked value by value rather than over a serialization: `JSON.stringify`
|
|
16379
|
+
* escapes quotes and backslashes, so a `from` holding one would silently miss.
|
|
16445
16380
|
*/
|
|
16446
|
-
function
|
|
16447
|
-
|
|
16448
|
-
|
|
16449
|
-
const plugin = registryFor(config).get(own.id);
|
|
16450
|
-
return plugin ? resolveCaseRecordingPath(plugin, own.targetConfig, caseId) : void 0;
|
|
16381
|
+
function recordingNamesRenamed(recording, renames) {
|
|
16382
|
+
if (renames.length === 0) return false;
|
|
16383
|
+
return [...recording.actions, ...recording.cleanup ?? []].some((action) => actionStrings(action).some((text) => renames.some((r) => text.includes(r.from))));
|
|
16451
16384
|
}
|
|
16452
|
-
/**
|
|
16453
|
-
|
|
16454
|
-
|
|
16455
|
-
|
|
16456
|
-
|
|
16457
|
-
|
|
16458
|
-
|
|
16459
|
-
|
|
16460
|
-
|
|
16385
|
+
/**
|
|
16386
|
+
* The strings a recorded action takes from the product: what it typed or
|
|
16387
|
+
* asserted, and how it addressed the element. What the recorder wrote about an
|
|
16388
|
+
* action — its observation, its step id, why it replayed badly — is ccqa's own
|
|
16389
|
+
* prose and would match on words the product never rendered.
|
|
16390
|
+
*/
|
|
16391
|
+
function actionStrings(action) {
|
|
16392
|
+
return [
|
|
16393
|
+
action.value,
|
|
16394
|
+
action.label,
|
|
16395
|
+
...locatorStrings(action.locator),
|
|
16396
|
+
...locatorStrings(action.target)
|
|
16397
|
+
].filter((s) => s !== void 0 && s !== "");
|
|
16398
|
+
}
|
|
16399
|
+
/**
|
|
16400
|
+
* A role locator is addressed by its accessible name; its `value` is the ARIA
|
|
16401
|
+
* role, which is that vocabulary rather than anything the product wrote.
|
|
16402
|
+
*/
|
|
16403
|
+
function locatorStrings(locator) {
|
|
16404
|
+
if (locator === void 0) return [];
|
|
16405
|
+
return locator.by === "role" ? [locator.name] : [locator.value];
|
|
16406
|
+
}
|
|
16407
|
+
//#endregion
|
|
16408
|
+
//#region src/drift/types.ts
|
|
16409
|
+
/** How everything downstream spells this case: its intent-source id, or `feature/spec`. */
|
|
16410
|
+
function caseIdOf(target) {
|
|
16411
|
+
return target.caseId ?? `${target.featureName}/${target.specName}`;
|
|
16412
|
+
}
|
|
16413
|
+
/**
|
|
16414
|
+
* What the audit says about one locator code could not find in the product.
|
|
16415
|
+
*
|
|
16416
|
+
* `drifted` says the test asks for something the product no longer renders;
|
|
16417
|
+
* `fine` says the miss has another explanation — a selector built at runtime, a
|
|
16418
|
+
* component this case never reaches, a file the scan did not read. Either is an
|
|
16419
|
+
* answer; silence is not, which is what the required list below is for.
|
|
16420
|
+
*/
|
|
16421
|
+
const LocatorVerdictSchema = z.object({
|
|
16422
|
+
id: z.string().min(1),
|
|
16423
|
+
verdict: z.enum(["drifted", "fine"]),
|
|
16424
|
+
note: z.string().default("")
|
|
16425
|
+
});
|
|
16426
|
+
/** The model's reply: a diagnosis, or `null` for "the spec still matches the code". */
|
|
16427
|
+
const DriftReplySchema = z.object({
|
|
16428
|
+
drift: DriftDiagnosisSchema.nullable(),
|
|
16429
|
+
locators: z.array(LocatorVerdictSchema).default([]),
|
|
16430
|
+
renames: z.array(RenameSchema.nullable().catch(null)).transform((rs) => rs.filter((r) => r !== null)).default([]).catch([])
|
|
16431
|
+
});
|
|
16432
|
+
/**
|
|
16433
|
+
* How a label reads against `--exit-on`. The threshold asks "would a
|
|
16434
|
+
* deterministic replay fail today". `TEST_DRIFT` and `SPEC_CHANGE` answer yes
|
|
16435
|
+
* and name the repair, so they hold the gate shut. The other three do not: a
|
|
16436
|
+
* suspected product bug or a dependency on data the source does not decide is
|
|
16437
|
+
* settled by running the case, not by refusing to run it, and `UNKNOWN` says
|
|
16438
|
+
* the audit could not tell.
|
|
16439
|
+
*/
|
|
16440
|
+
function driftSeverity(label) {
|
|
16441
|
+
return label === "TEST_DRIFT" || label === "SPEC_CHANGE" ? "error" : "warn";
|
|
16461
16442
|
}
|
|
16462
16443
|
//#endregion
|
|
16463
16444
|
//#region src/drift/artifacts.ts
|
|
@@ -16468,30 +16449,38 @@ function intentTargetFor(config, targetOverride) {
|
|
|
16468
16449
|
* is always named in `unaudited`.
|
|
16469
16450
|
*/
|
|
16470
16451
|
const MAX_GENERATED_BYTES = 64 * 1024;
|
|
16471
|
-
/**
|
|
16472
|
-
|
|
16452
|
+
/**
|
|
16453
|
+
* Read the sweep-wide inputs once, before the first spec. A caller that
|
|
16454
|
+
* already holds a reader passes it, so its cases are read once for the whole
|
|
16455
|
+
* command rather than once per phase.
|
|
16456
|
+
*/
|
|
16457
|
+
async function loadSpecArtifactsContext(cwd, reader) {
|
|
16473
16458
|
const [config, tsconfig] = await Promise.all([loadProjectConfig(cwd), loadTsconfigPaths(cwd)]);
|
|
16474
16459
|
return {
|
|
16475
16460
|
config,
|
|
16476
16461
|
tsconfig,
|
|
16477
|
-
|
|
16462
|
+
reader: reader ?? openCaseReader(config, cwd)
|
|
16478
16463
|
};
|
|
16479
16464
|
}
|
|
16480
16465
|
/**
|
|
16481
16466
|
* Read one case's documents. Throws when the document itself cannot be read —
|
|
16482
16467
|
* a case enumerated and then gone is an error to report, not an empty audit.
|
|
16468
|
+
*
|
|
16469
|
+
* A document that is there but will not parse is still audited: what its
|
|
16470
|
+
* author wrote is what the audit is for, and refusing to look at it would
|
|
16471
|
+
* clear a case nobody read.
|
|
16483
16472
|
*/
|
|
16484
16473
|
async function collectCaseArtifacts(target, cwd, ctx) {
|
|
16485
|
-
|
|
16486
|
-
}
|
|
16487
|
-
async function collectSpecCase(featureName, specName, cwd, ctx) {
|
|
16488
|
-
const specYaml = await readSpecFile(featureName, specName, cwd);
|
|
16474
|
+
const read = await ctx.reader.read(caseIdOf(target));
|
|
16475
|
+
if (read.document === null) throw new Error(read.error ?? `${caseIdOf(target)}: no such test case`);
|
|
16489
16476
|
const intent = {
|
|
16490
|
-
kind: "spec",
|
|
16491
|
-
path:
|
|
16492
|
-
body:
|
|
16477
|
+
kind: ctx.reader.target ? "project" : "spec",
|
|
16478
|
+
path: relative(cwd, read.document.path),
|
|
16479
|
+
body: read.document.text
|
|
16493
16480
|
};
|
|
16494
|
-
const
|
|
16481
|
+
const testCase = read.case;
|
|
16482
|
+
const live = testCase?.mode === "live";
|
|
16483
|
+
const title = testCase?.title ?? null;
|
|
16495
16484
|
if (live) return {
|
|
16496
16485
|
intent,
|
|
16497
16486
|
generated: [],
|
|
@@ -16502,114 +16491,109 @@ async function collectSpecCase(featureName, specName, cwd, ctx) {
|
|
|
16502
16491
|
};
|
|
16503
16492
|
return {
|
|
16504
16493
|
intent,
|
|
16505
|
-
...await
|
|
16494
|
+
...await readGenerated(testFileOf(testCase, ctx, cwd), cwd, ctx),
|
|
16506
16495
|
live,
|
|
16507
16496
|
title
|
|
16508
16497
|
};
|
|
16509
16498
|
}
|
|
16510
16499
|
/**
|
|
16511
|
-
* The generated half alone, for a caller that
|
|
16512
|
-
* failure classifier, which reads the same code the audit does
|
|
16513
|
-
* own account of what happened.
|
|
16500
|
+
* The generated half alone, for a caller that brings its own account of what
|
|
16501
|
+
* happened — the failure classifier, which reads the same code the audit does.
|
|
16514
16502
|
*/
|
|
16515
|
-
async function
|
|
16516
|
-
|
|
16503
|
+
async function collectGeneratedFor(caseId, cwd, ctx) {
|
|
16504
|
+
const read = await ctx.reader.read(caseId);
|
|
16505
|
+
if (read.case === null && read.error !== null) warn(`${caseId}: ${read.error}`);
|
|
16506
|
+
return readGenerated(testFileOf(read.case, ctx, cwd), cwd, ctx);
|
|
16517
16507
|
}
|
|
16518
|
-
|
|
16519
|
-
|
|
16520
|
-
|
|
16521
|
-
|
|
16522
|
-
|
|
16523
|
-
|
|
16524
|
-
|
|
16525
|
-
|
|
16526
|
-
|
|
16527
|
-
const
|
|
16528
|
-
|
|
16529
|
-
|
|
16530
|
-
|
|
16531
|
-
unaudited: [],
|
|
16532
|
-
reached: [],
|
|
16533
|
-
live,
|
|
16534
|
-
title: testCase.title
|
|
16535
|
-
};
|
|
16536
|
-
return {
|
|
16537
|
-
intent,
|
|
16538
|
-
...await readGenerated(resolve(cwd, resolveCaseTestPath(registryFor(ctx.config).get(ctx.intentTarget.id), targetConfig, caseId)), cwd, ctx),
|
|
16539
|
-
live,
|
|
16540
|
-
title: testCase.title
|
|
16508
|
+
/**
|
|
16509
|
+
* Where this case's files live, resolved through the target that owns them.
|
|
16510
|
+
*
|
|
16511
|
+
* All three together because one target resolution answers all three, and a
|
|
16512
|
+
* case that has one must not be reported as missing another. Every path is
|
|
16513
|
+
* null when nothing resolves — an unreadable case must cost its own paths,
|
|
16514
|
+
* not the sweep.
|
|
16515
|
+
*/
|
|
16516
|
+
async function caseFiles(target, ctx) {
|
|
16517
|
+
const none = {
|
|
16518
|
+
testPath: null,
|
|
16519
|
+
recordingPath: null,
|
|
16520
|
+
writeRoots: []
|
|
16541
16521
|
};
|
|
16522
|
+
const read = await ctx.reader.read(caseIdOf(target));
|
|
16523
|
+
if (read.case === null && read.error !== null) warn(`${caseIdOf(target)}: ${read.error}`);
|
|
16524
|
+
const testCase = read.case;
|
|
16525
|
+
const owner = testCase && targetOf$1(testCase, ctx);
|
|
16526
|
+
if (!testCase || !owner) return none;
|
|
16527
|
+
try {
|
|
16528
|
+
return {
|
|
16529
|
+
testPath: resolveCaseTestPath(owner.plugin, owner.targetConfig, testCase.ref.id),
|
|
16530
|
+
recordingPath: resolveCaseRecordingPath(owner.plugin, owner.targetConfig, testCase.ref.id),
|
|
16531
|
+
writeRoots: owner.targetConfig.writeRoots
|
|
16532
|
+
};
|
|
16533
|
+
} catch {
|
|
16534
|
+
return none;
|
|
16535
|
+
}
|
|
16542
16536
|
}
|
|
16543
16537
|
/**
|
|
16544
16538
|
* Where this case's generated test lives, project-relative — or null when it
|
|
16545
16539
|
* has none (a live case, or a document whose target cannot be resolved).
|
|
16546
16540
|
*/
|
|
16547
|
-
async function caseTestPath(target,
|
|
16548
|
-
|
|
16549
|
-
if (!ctx.intentTarget) return null;
|
|
16550
|
-
return resolveCaseTestPath(registryFor(ctx.config).get(ctx.intentTarget.id), ctx.intentTarget.targetConfig, target.caseId);
|
|
16551
|
-
}
|
|
16552
|
-
const specYaml = await readSpecFile(target.featureName, target.specName, cwd).catch(() => null);
|
|
16553
|
-
if (specYaml === null) return null;
|
|
16554
|
-
const abs = specTestFile(target.featureName, target.specName, specYaml, cwd, ctx.config);
|
|
16555
|
-
return abs === null ? null : relative(cwd, abs);
|
|
16541
|
+
async function caseTestPath(target, ctx) {
|
|
16542
|
+
return (await caseFiles(target, ctx)).testPath;
|
|
16556
16543
|
}
|
|
16557
16544
|
/**
|
|
16558
16545
|
* Where this case's recording lives, project-relative, resolved through the
|
|
16559
16546
|
* same target as its test — null on the same terms as {@link caseTestPath}.
|
|
16560
16547
|
*/
|
|
16561
|
-
async function caseRecordingPath(target,
|
|
16562
|
-
|
|
16563
|
-
if (!ctx.intentTarget) return null;
|
|
16564
|
-
return resolveCaseRecordingPath(registryFor(ctx.config).get(ctx.intentTarget.id), ctx.intentTarget.targetConfig, target.caseId);
|
|
16565
|
-
}
|
|
16566
|
-
const specYaml = await readSpecFile(target.featureName, target.specName, cwd).catch(() => null);
|
|
16567
|
-
if (specYaml === null) return null;
|
|
16568
|
-
return specTargetPaths(target.featureName, target.specName, specYaml, cwd, ctx.config)?.recordingPath ?? null;
|
|
16548
|
+
async function caseRecordingPath(target, ctx) {
|
|
16549
|
+
return (await caseFiles(target, ctx)).recordingPath;
|
|
16569
16550
|
}
|
|
16570
16551
|
/**
|
|
16571
|
-
* Where this case's test lands, and where its target lets ccqa write
|
|
16572
|
-
*
|
|
16573
|
-
* Together because they answer one question — which files a finding about this
|
|
16574
|
-
* case could be repaired by regenerating it — and because resolving the target
|
|
16575
|
-
* once is what keeps the two consistent for a spec that names a non-default one.
|
|
16552
|
+
* Where this case's test lands, and where its target lets ccqa write: the two
|
|
16553
|
+
* halves of "which files a finding about this case could be repaired in".
|
|
16576
16554
|
*/
|
|
16577
|
-
async function caseWriteArea(target,
|
|
16578
|
-
const testPath = await
|
|
16579
|
-
|
|
16580
|
-
testPath,
|
|
16581
|
-
writeRoots: ctx.intentTarget?.targetConfig.writeRoots ?? []
|
|
16582
|
-
};
|
|
16583
|
-
const specYaml = await readSpecFile(target.featureName, target.specName, cwd).catch(() => null);
|
|
16584
|
-
if (specYaml === null) return {
|
|
16555
|
+
async function caseWriteArea(target, ctx) {
|
|
16556
|
+
const { testPath, writeRoots } = await caseFiles(target, ctx);
|
|
16557
|
+
return {
|
|
16585
16558
|
testPath,
|
|
16586
|
-
writeRoots
|
|
16559
|
+
writeRoots
|
|
16587
16560
|
};
|
|
16561
|
+
}
|
|
16562
|
+
/**
|
|
16563
|
+
* The target that owns this case's generated files.
|
|
16564
|
+
*
|
|
16565
|
+
* A case the project states itself belongs to the target that declared the
|
|
16566
|
+
* source; a `spec.yaml` case names its own. Null when nothing resolves — an
|
|
16567
|
+
* unreadable target must cost this one case's paths, not the sweep.
|
|
16568
|
+
*/
|
|
16569
|
+
function targetOf$1(testCase, ctx) {
|
|
16570
|
+
if (ctx.reader.target) {
|
|
16571
|
+
const plugin = registryFor(ctx.config).get(ctx.reader.target.id);
|
|
16572
|
+
return plugin ? {
|
|
16573
|
+
plugin,
|
|
16574
|
+
targetConfig: ctx.reader.target.targetConfig
|
|
16575
|
+
} : null;
|
|
16576
|
+
}
|
|
16577
|
+
if (testCase.spec === null) return null;
|
|
16588
16578
|
try {
|
|
16589
|
-
const
|
|
16579
|
+
const plugin = resolveTarget(testCase.spec, ctx.config);
|
|
16590
16580
|
return {
|
|
16591
|
-
|
|
16592
|
-
|
|
16581
|
+
plugin,
|
|
16582
|
+
targetConfig: targetConfigFor(ctx.config, plugin.id)
|
|
16593
16583
|
};
|
|
16594
16584
|
} catch {
|
|
16595
|
-
return
|
|
16596
|
-
testPath,
|
|
16597
|
-
writeRoots: []
|
|
16598
|
-
};
|
|
16585
|
+
return null;
|
|
16599
16586
|
}
|
|
16600
16587
|
}
|
|
16601
|
-
|
|
16588
|
+
/** The generated test's absolute path, or null when this case compiles into none. */
|
|
16589
|
+
function testFileOf(testCase, ctx, cwd) {
|
|
16590
|
+
if (testCase === null) return null;
|
|
16591
|
+
const owner = targetOf$1(testCase, ctx);
|
|
16592
|
+
if (owner === null) return null;
|
|
16602
16593
|
try {
|
|
16603
|
-
|
|
16604
|
-
return {
|
|
16605
|
-
live: spec.mode === "live",
|
|
16606
|
-
title: spec.title ?? null
|
|
16607
|
-
};
|
|
16594
|
+
return resolve(cwd, resolveCaseTestPath(owner.plugin, owner.targetConfig, testCase.ref.id));
|
|
16608
16595
|
} catch {
|
|
16609
|
-
return
|
|
16610
|
-
live: false,
|
|
16611
|
-
title: null
|
|
16612
|
-
};
|
|
16596
|
+
return null;
|
|
16613
16597
|
}
|
|
16614
16598
|
}
|
|
16615
16599
|
/**
|
|
@@ -16657,35 +16641,6 @@ async function readGenerated(testAbs, cwd, ctx) {
|
|
|
16657
16641
|
reached
|
|
16658
16642
|
};
|
|
16659
16643
|
}
|
|
16660
|
-
/**
|
|
16661
|
-
* Where this spec's two files live, resolved through its target: the test, and
|
|
16662
|
-
* the recording it compiles from. A spec that will not parse, whose target
|
|
16663
|
-
* cannot be resolved, or whose `testPath` will not expand has neither to
|
|
16664
|
-
* derive — the audit then reads the spec alone rather than guessing at a file.
|
|
16665
|
-
*
|
|
16666
|
-
* Both together because one target resolution answers both, and a spec that
|
|
16667
|
-
* has one must not be reported as missing the other.
|
|
16668
|
-
*/
|
|
16669
|
-
function specTargetPaths(featureName, specName, specYaml, cwd, config) {
|
|
16670
|
-
try {
|
|
16671
|
-
const target = resolveTarget(parseTestSpec(specYaml), config);
|
|
16672
|
-
const targetConfig = targetConfigFor(config, target.id);
|
|
16673
|
-
const ref = {
|
|
16674
|
-
featureName,
|
|
16675
|
-
specName
|
|
16676
|
-
};
|
|
16677
|
-
return {
|
|
16678
|
-
testAbs: resolveTestPathAbs(target, targetConfig, ref, cwd),
|
|
16679
|
-
recordingPath: resolveRecordingPath(target, targetConfig, ref)
|
|
16680
|
-
};
|
|
16681
|
-
} catch {
|
|
16682
|
-
return null;
|
|
16683
|
-
}
|
|
16684
|
-
}
|
|
16685
|
-
/** Where this spec's test lives, resolved through its target. */
|
|
16686
|
-
function specTestFile(featureName, specName, specYaml, cwd, config) {
|
|
16687
|
-
return specTargetPaths(featureName, specName, specYaml, cwd, config)?.testAbs ?? null;
|
|
16688
|
-
}
|
|
16689
16644
|
//#endregion
|
|
16690
16645
|
//#region src/cli/colors.ts
|
|
16691
16646
|
/**
|
|
@@ -16768,7 +16723,7 @@ function createFailureAnalysisPass(deps) {
|
|
|
16768
16723
|
};
|
|
16769
16724
|
if (input.specYaml === null) return {
|
|
16770
16725
|
...fields,
|
|
16771
|
-
analysisSkipped: "
|
|
16726
|
+
analysisSkipped: "the case's own document could not be read"
|
|
16772
16727
|
};
|
|
16773
16728
|
const baselineMissing = specDiffResult.ok ? null : specDiffResult.skip;
|
|
16774
16729
|
info(`failure analysis: ${featureName}/${specName}${baselineMissing ? " (no baseline — classifying from current source)" : ""}`);
|
|
@@ -16856,9 +16811,9 @@ function needsAnalysis(row) {
|
|
|
16856
16811
|
* invents would land in the confusion matrix and the project's learned prompt.
|
|
16857
16812
|
* The recorded reason is both accurate and cheaper.
|
|
16858
16813
|
*/
|
|
16859
|
-
async function analyzeExternalRows(rows, run) {
|
|
16814
|
+
async function analyzeExternalRows(rows, run, caseIds = /* @__PURE__ */ new Map()) {
|
|
16860
16815
|
const { deps, pass } = run;
|
|
16861
|
-
const context = await loadSpecArtifactsContext(deps.cwd);
|
|
16816
|
+
const context = await loadSpecArtifactsContext(deps.cwd, deps.reader);
|
|
16862
16817
|
const out = [];
|
|
16863
16818
|
for (const row of rows) {
|
|
16864
16819
|
if (!needsAnalysis(row)) {
|
|
@@ -16869,9 +16824,10 @@ async function analyzeExternalRows(rows, run) {
|
|
|
16869
16824
|
featureName: row.feature,
|
|
16870
16825
|
specName: row.spec
|
|
16871
16826
|
};
|
|
16827
|
+
const caseId = caseIds.get(specKey(ref)) ?? specKey(ref);
|
|
16872
16828
|
const fields = await pass.analyze({
|
|
16873
16829
|
...ref,
|
|
16874
|
-
readScript: () => readGeneratedTestSources(
|
|
16830
|
+
readScript: () => readGeneratedTestSources(caseId, deps.cwd, context),
|
|
16875
16831
|
failureLog: row.failureLogExcerpt ?? "",
|
|
16876
16832
|
specYaml: row.specYaml,
|
|
16877
16833
|
parsedSpec: tryParseTestSpec(row.specYaml),
|
|
@@ -16902,9 +16858,8 @@ function readableArtifactsDir(ref, deps) {
|
|
|
16902
16858
|
* both surfaces read the same set. Best-effort: an ungenerated or unreadable
|
|
16903
16859
|
* spec just means less context, never a failed analysis.
|
|
16904
16860
|
*/
|
|
16905
|
-
async function readGeneratedTestSources(
|
|
16906
|
-
|
|
16907
|
-
const { generated, unaudited } = await collectSpecGenerated(ref.featureName, ref.specName, specYaml, cwd, context);
|
|
16861
|
+
async function readGeneratedTestSources(caseId, cwd, context) {
|
|
16862
|
+
const { generated, unaudited } = await collectGeneratedFor(caseId, cwd, context);
|
|
16908
16863
|
const parts = generated.map((f) => `// ${f.path}\n${f.content}`);
|
|
16909
16864
|
if (unaudited.length > 0) parts.push(`// [not shown: ${unaudited.join(", ")} — Read them for their full state]`);
|
|
16910
16865
|
return parts.join("\n\n");
|
|
@@ -18326,7 +18281,7 @@ function failureBody(row) {
|
|
|
18326
18281
|
}
|
|
18327
18282
|
/** A row's stored path (relative to `reportDir`), re-expressed relative to where the XML itself lives. */
|
|
18328
18283
|
function rewriteEvidencePath(storedPath, reportDir, junitDir) {
|
|
18329
|
-
return toPosix(relative(junitDir, resolve(reportDir, storedPath)));
|
|
18284
|
+
return toPosix$1(relative(junitDir, resolve(reportDir, storedPath)));
|
|
18330
18285
|
}
|
|
18331
18286
|
/** Step-boundary screenshot paths for one row, in capture order, for `<system-out>`. */
|
|
18332
18287
|
function evidencePaths(row, reportDir, junitDir) {
|
|
@@ -18391,35 +18346,22 @@ function currentReportCost() {
|
|
|
18391
18346
|
}
|
|
18392
18347
|
//#endregion
|
|
18393
18348
|
//#region src/run/spec-catalog.ts
|
|
18394
|
-
async function readSpecs(refs,
|
|
18349
|
+
async function readSpecs(refs, reader) {
|
|
18395
18350
|
const entries = await Promise.all(refs.map(async (ref) => {
|
|
18396
|
-
const
|
|
18397
|
-
|
|
18398
|
-
|
|
18399
|
-
yaml: null,
|
|
18400
|
-
error: null
|
|
18351
|
+
const read = await reader.read(specKey(ref));
|
|
18352
|
+
return [specKey(ref), {
|
|
18353
|
+
case: read.case,
|
|
18354
|
+
yaml: read.document?.text ?? null,
|
|
18355
|
+
error: read.document === null ? null : read.error
|
|
18401
18356
|
}];
|
|
18402
|
-
try {
|
|
18403
|
-
return [specKey(ref), {
|
|
18404
|
-
spec: parseTestSpec(yaml),
|
|
18405
|
-
yaml,
|
|
18406
|
-
error: null
|
|
18407
|
-
}];
|
|
18408
|
-
} catch (err) {
|
|
18409
|
-
return [specKey(ref), {
|
|
18410
|
-
spec: null,
|
|
18411
|
-
yaml,
|
|
18412
|
-
error: errMessage(err)
|
|
18413
|
-
}];
|
|
18414
|
-
}
|
|
18415
18357
|
}));
|
|
18416
18358
|
return new Map(entries);
|
|
18417
18359
|
}
|
|
18418
|
-
/**
|
|
18360
|
+
/** Case-declared `mode:` wins; otherwise `DEFAULT_SPEC_MODE`. */
|
|
18419
18361
|
function resolveSpecsModes(specs, catalog) {
|
|
18420
18362
|
return specs.map((s) => ({
|
|
18421
18363
|
...s,
|
|
18422
|
-
mode: catalog.get(specKey(s))?.
|
|
18364
|
+
mode: catalog.get(specKey(s))?.case?.mode ?? "deterministic"
|
|
18423
18365
|
}));
|
|
18424
18366
|
}
|
|
18425
18367
|
//#endregion
|
|
@@ -19994,7 +19936,7 @@ function runsLive(testCase) {
|
|
|
19994
19936
|
return testCase.mode === "live";
|
|
19995
19937
|
}
|
|
19996
19938
|
function liveCaseFrom(testCase, opts = {}) {
|
|
19997
|
-
const spec = testCase.
|
|
19939
|
+
const spec = testCase.spec;
|
|
19998
19940
|
const own = attachCaseExpectations(requireActionSteps(testCase.steps, testCase.ref.id, {
|
|
19999
19941
|
id: AGENT_BROWSER_TARGET,
|
|
20000
19942
|
reason: AGENT_BROWSER_JUDGE_STEPS.reason
|
|
@@ -20006,7 +19948,7 @@ function liveCaseFrom(testCase, opts = {}) {
|
|
|
20006
19948
|
title: testCase.title,
|
|
20007
19949
|
steps,
|
|
20008
19950
|
cleanupFrom: own.length,
|
|
20009
|
-
document: testCase.
|
|
19951
|
+
document: testCase.document.text,
|
|
20010
19952
|
blocks: spec ? collectIncludedBlockNames(spec) : [],
|
|
20011
19953
|
session: {
|
|
20012
19954
|
names: spec?.session ?? [],
|
|
@@ -20040,135 +19982,74 @@ function attachCaseExpectations(steps, expectations) {
|
|
|
20040
19982
|
//#endregion
|
|
20041
19983
|
//#region src/select/inventory.ts
|
|
20042
19984
|
/**
|
|
20043
|
-
* Read every test case the project states into the shape the selection
|
|
20044
|
-
*
|
|
20045
|
-
*
|
|
20046
|
-
*
|
|
20047
|
-
* (see `intentTargetFor`), so this reads one enumeration or the other.
|
|
19985
|
+
* Read every test case the project states into the shape the selection prompt
|
|
19986
|
+
* consumes — ccqa's own specs, or the documents the project writes itself,
|
|
19987
|
+
* whichever this project has. One enumeration, one read per case, through the
|
|
19988
|
+
* one reader: what the two kinds share is everything the prompt is shown.
|
|
20048
19989
|
*/
|
|
20049
19990
|
async function loadSpecInventory(cwd) {
|
|
20050
19991
|
const config = await loadProjectConfig(cwd);
|
|
20051
|
-
const
|
|
20052
|
-
|
|
20053
|
-
return
|
|
20054
|
-
|
|
20055
|
-
|
|
20056
|
-
|
|
20057
|
-
|
|
20058
|
-
|
|
20059
|
-
* Enumeration (`listAllSpecsWithSpecFile`, a directory walk) is kept separate
|
|
20060
|
-
* from reading and parsing each spec's own content, so every spec.yaml is
|
|
20061
|
-
* read and parsed exactly once — and the reads run in parallel.
|
|
20062
|
-
*/
|
|
20063
|
-
async function loadSpecFileInventory(config, cwd) {
|
|
20064
|
-
const [refs, blocks] = await Promise.all([listAllSpecsWithSpecFile(cwd), loadAllBlocks(cwd)]);
|
|
20065
|
-
return (await Promise.all(refs.map(async ({ featureName, specName }) => {
|
|
20066
|
-
const content = await tryReadSpecFile(featureName, specName, cwd);
|
|
20067
|
-
if (content === null) return null;
|
|
20068
|
-
const spec = parseTestSpec(content, `${featureName}/${specName}/spec.yaml`);
|
|
20069
|
-
if (spec.disabled) return null;
|
|
20070
|
-
return {
|
|
20071
|
-
featureName,
|
|
20072
|
-
specName,
|
|
20073
|
-
title: spec.title,
|
|
20074
|
-
steps: describeSteps(spec, blocks, `${featureName}/${specName}`),
|
|
20075
|
-
includedBlocks: collectIncludedBlockNames(spec),
|
|
20076
|
-
...resolveSpecPaths(spec, config, featureName, specName),
|
|
20077
|
-
sourcePath: `.ccqa/features/${featureName}/test-cases/${specName}/spec.yaml`
|
|
20078
|
-
};
|
|
20079
|
-
}))).filter((s) => s !== null);
|
|
20080
|
-
}
|
|
20081
|
-
/**
|
|
20082
|
-
* Same shape, for a project whose cases are its own markdown documents. No
|
|
20083
|
-
* `disabled` flag to drop here — that concept belongs to `spec.yaml`, and an
|
|
20084
|
-
* intent source has nothing that plays the same role.
|
|
20085
|
-
*/
|
|
20086
|
-
async function loadMarkdownInventory(intentTarget, config, cwd) {
|
|
20087
|
-
const { id: targetId, targetConfig, intent } = intentTarget;
|
|
20088
|
-
const target = registryFor(config).get(targetId);
|
|
20089
|
-
const ids = await listMarkdownCases(intent, cwd);
|
|
20090
|
-
return Promise.all(ids.map(async (caseId) => {
|
|
20091
|
-
const testCase = await loadMarkdownCase(caseId, intent, cwd);
|
|
19992
|
+
const reader = openCaseReader(config, cwd);
|
|
19993
|
+
const ids = await reader.list();
|
|
19994
|
+
return (await Promise.all(ids.map(async (id) => {
|
|
19995
|
+
const read = await reader.read(id);
|
|
19996
|
+
if (read.case === null) throw new Error(read.error ?? `${id}: could not be read`);
|
|
19997
|
+
const testCase = read.case;
|
|
19998
|
+
if (testCase.blocked !== null) warn(`${id}: ${testCase.blocked}`);
|
|
19999
|
+
if (testCase.disabled) return null;
|
|
20092
20000
|
const { featureName, specName } = splitCaseId(testCase.ref.id);
|
|
20093
20001
|
return {
|
|
20094
20002
|
featureName,
|
|
20095
20003
|
specName,
|
|
20096
20004
|
title: testCase.title,
|
|
20097
|
-
steps:
|
|
20098
|
-
includedBlocks: [],
|
|
20099
|
-
|
|
20100
|
-
|
|
20101
|
-
sourcePath: markdownSourcePath(testCase, caseId, cwd)
|
|
20005
|
+
steps: testCase.steps.map(describeStepBody),
|
|
20006
|
+
includedBlocks: testCase.spec ? collectIncludedBlockNames(testCase.spec) : [],
|
|
20007
|
+
...resolveCasePaths(testCase, config, reader),
|
|
20008
|
+
sourcePath: relative(cwd, testCase.document.path)
|
|
20102
20009
|
};
|
|
20103
|
-
}));
|
|
20104
|
-
}
|
|
20105
|
-
/** The case's own file, project-root-relative, or its id when the source somehow isn't markdown. */
|
|
20106
|
-
function markdownSourcePath(testCase, id, cwd) {
|
|
20107
|
-
return testCase.source.kind === "markdown" ? relative(cwd, testCase.source.path) : id;
|
|
20010
|
+
}))).filter((s) => s !== null);
|
|
20108
20011
|
}
|
|
20109
20012
|
/**
|
|
20110
20013
|
* `--format paths` needs a real file to hand a test runner, so a target that
|
|
20111
|
-
* can't be resolved (bad `target:`/`defaultTarget`) degrades this one
|
|
20014
|
+
* can't be resolved (bad `target:`/`defaultTarget`) degrades this one case's
|
|
20112
20015
|
* path to "" instead of failing the whole inventory — the other formats don't
|
|
20113
|
-
* depend on it. A live
|
|
20114
|
-
* from the
|
|
20016
|
+
* depend on it. A live case resolves to "" for the same reason: it is driven
|
|
20017
|
+
* from the document every run and has compiled nothing a runner could take.
|
|
20115
20018
|
*/
|
|
20116
|
-
function
|
|
20019
|
+
function resolveCasePaths(testCase, config, reader) {
|
|
20117
20020
|
const none = {
|
|
20118
20021
|
testPath: "",
|
|
20119
20022
|
recordingPath: ""
|
|
20120
20023
|
};
|
|
20121
|
-
if (
|
|
20024
|
+
if (runsLive(testCase)) return none;
|
|
20122
20025
|
try {
|
|
20123
|
-
const
|
|
20124
|
-
const
|
|
20125
|
-
const
|
|
20126
|
-
featureName,
|
|
20127
|
-
specName
|
|
20128
|
-
};
|
|
20026
|
+
const owner = reader.target;
|
|
20027
|
+
const target = owner ? registryFor(config).get(owner.id) : resolveTarget(testCase.spec, config);
|
|
20028
|
+
const targetConfig = owner ? owner.targetConfig : targetConfigFor(config, target.id);
|
|
20129
20029
|
return {
|
|
20130
|
-
testPath:
|
|
20131
|
-
recordingPath:
|
|
20030
|
+
testPath: resolveCaseTestPath(target, targetConfig, testCase.ref.id),
|
|
20031
|
+
recordingPath: resolveCaseRecordingPath(target, targetConfig, testCase.ref.id)
|
|
20132
20032
|
};
|
|
20133
20033
|
} catch (e) {
|
|
20134
|
-
warn(`${
|
|
20034
|
+
warn(`${testCase.ref.id}: could not resolve test path (${e.message})`);
|
|
20135
20035
|
return none;
|
|
20136
20036
|
}
|
|
20137
20037
|
}
|
|
20138
20038
|
/**
|
|
20139
|
-
* One line per step,
|
|
20140
|
-
* the selection prompt is told to weigh shared
|
|
20141
|
-
* against each
|
|
20142
|
-
* here rather than hidden behind a block name.
|
|
20039
|
+
* One line per step, in order. Include steps arrive already expanded to the
|
|
20040
|
+
* block's own steps — the selection prompt is told to weigh shared
|
|
20041
|
+
* login/navigation/layout against each case, which only works if those steps
|
|
20042
|
+
* are actually visible here rather than hidden behind a block name.
|
|
20143
20043
|
*
|
|
20144
|
-
*
|
|
20145
|
-
*
|
|
20146
|
-
*
|
|
20044
|
+
* A step with no `expected` of its own prints as the instruction alone: a case
|
|
20045
|
+
* that states its expectations once for the whole flow has nothing to put
|
|
20046
|
+
* after the arrow, and an empty one reads as a step that checks nothing.
|
|
20147
20047
|
*/
|
|
20148
|
-
function describeSteps(spec, blocks, specKey) {
|
|
20149
|
-
try {
|
|
20150
|
-
return expandSpec(spec, { blocks }).map(describeStepBody);
|
|
20151
|
-
} catch (e) {
|
|
20152
|
-
warn(`${specKey}: could not expand include steps (${e.message}) — showing block names instead`);
|
|
20153
|
-
return spec.steps.map(describeStep);
|
|
20154
|
-
}
|
|
20155
|
-
}
|
|
20156
|
-
function describeStep(step) {
|
|
20157
|
-
if (isIncludeStep(step)) return `include block: ${step.include}`;
|
|
20158
|
-
return describeStepBody(step);
|
|
20159
|
-
}
|
|
20160
20048
|
function describeStepBody(step) {
|
|
20161
20049
|
if (isJudgeBody(step)) return `judge: ${oneLine$1(step.judgeByLlm)}`;
|
|
20162
|
-
|
|
20163
|
-
|
|
20164
|
-
|
|
20165
|
-
* One line per step, in order — no `include:` to expand, since a markdown
|
|
20166
|
-
* case's steps are never block references. Unlike a spec step, a markdown
|
|
20167
|
-
* step carries no per-step `expected` (the case lists its expectations once,
|
|
20168
|
-
* for the whole flow), so only the instruction goes on the line.
|
|
20169
|
-
*/
|
|
20170
|
-
function describeMarkdownSteps(testCase) {
|
|
20171
|
-
return testCase.steps.map((step) => isJudgeBody(step) ? `judge: ${oneLine$1(step.judgeByLlm)}` : oneLine$1(step.instruction));
|
|
20050
|
+
const expected = oneLine$1(step.expected);
|
|
20051
|
+
const instruction = oneLine$1(step.instruction);
|
|
20052
|
+
return expected.length > 0 ? `${instruction} → ${expected}` : instruction;
|
|
20172
20053
|
}
|
|
20173
20054
|
function oneLine$1(text) {
|
|
20174
20055
|
return text.trim().replace(/\s+/g, " ");
|
|
@@ -21817,7 +21698,7 @@ async function copyEvidenceIntoReport(absPath, evidenceDir, reportDir) {
|
|
|
21817
21698
|
const dest = join(evidenceDir, basename(absPath));
|
|
21818
21699
|
try {
|
|
21819
21700
|
await cp(absPath, dest);
|
|
21820
|
-
return posix.relative(toPosix(resolve(reportDir)), toPosix(dest));
|
|
21701
|
+
return posix.relative(toPosix$1(resolve(reportDir)), toPosix$1(dest));
|
|
21821
21702
|
} catch (err) {
|
|
21822
21703
|
warn(`failed to copy live evidence ${absPath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
21823
21704
|
return null;
|
|
@@ -22294,6 +22175,10 @@ var CoverageInbox = class {
|
|
|
22294
22175
|
};
|
|
22295
22176
|
//#endregion
|
|
22296
22177
|
//#region src/run/target-dispatch.ts
|
|
22178
|
+
/** The title a report row carries for a case, or null when it could not be read. */
|
|
22179
|
+
function titleOf(spec) {
|
|
22180
|
+
return spec.testCase?.title ?? null;
|
|
22181
|
+
}
|
|
22297
22182
|
/**
|
|
22298
22183
|
* Resolve each spec's target and group. A spec with no spec.yaml at all falls
|
|
22299
22184
|
* through to the agent-browser path, whose runner surfaces the real error
|
|
@@ -22318,16 +22203,20 @@ function groupSpecsByTarget(specs, catalog, config, resolve = resolveTargetFor(c
|
|
|
22318
22203
|
const unresolved = [];
|
|
22319
22204
|
for (const ref of specs) {
|
|
22320
22205
|
const read = catalog.get(specKey(ref));
|
|
22206
|
+
const dispatched = () => ({
|
|
22207
|
+
...ref,
|
|
22208
|
+
caseId: specKey(ref),
|
|
22209
|
+
testCase: read?.case ?? null
|
|
22210
|
+
});
|
|
22321
22211
|
if (read?.error) {
|
|
22322
22212
|
unresolved.push({
|
|
22323
|
-
...
|
|
22324
|
-
title: null,
|
|
22213
|
+
...dispatched(),
|
|
22325
22214
|
reason: read.error,
|
|
22326
22215
|
targetId: null
|
|
22327
22216
|
});
|
|
22328
22217
|
continue;
|
|
22329
22218
|
}
|
|
22330
|
-
const spec = read?.spec ?? null;
|
|
22219
|
+
const spec = (read?.case ?? null)?.spec ?? null;
|
|
22331
22220
|
if (spec === null) {
|
|
22332
22221
|
agentBrowser.push(ref);
|
|
22333
22222
|
continue;
|
|
@@ -22337,8 +22226,7 @@ function groupSpecsByTarget(specs, catalog, config, resolve = resolveTargetFor(c
|
|
|
22337
22226
|
plugin = resolve(spec, config);
|
|
22338
22227
|
} catch (err) {
|
|
22339
22228
|
unresolved.push({
|
|
22340
|
-
...
|
|
22341
|
-
title: spec.title ?? null,
|
|
22229
|
+
...dispatched(),
|
|
22342
22230
|
reason: err instanceof Error ? err.message : String(err),
|
|
22343
22231
|
targetId: spec.target ?? config.defaultTarget ?? null
|
|
22344
22232
|
});
|
|
@@ -22348,10 +22236,7 @@ function groupSpecsByTarget(specs, catalog, config, resolve = resolveTargetFor(c
|
|
|
22348
22236
|
agentBrowser.push(ref);
|
|
22349
22237
|
continue;
|
|
22350
22238
|
}
|
|
22351
|
-
const entry =
|
|
22352
|
-
...ref,
|
|
22353
|
-
title: spec.title ?? null
|
|
22354
|
-
};
|
|
22239
|
+
const entry = dispatched();
|
|
22355
22240
|
const targetConfig = targetConfigFor(config, plugin.id);
|
|
22356
22241
|
const routed = externalRunnability(plugin, plugin.id, targetConfig);
|
|
22357
22242
|
if ("reason" in routed) skipped.push({
|
|
@@ -22403,16 +22288,16 @@ function externalGroupFor(plugin, targetConfig) {
|
|
|
22403
22288
|
};
|
|
22404
22289
|
}
|
|
22405
22290
|
/**
|
|
22406
|
-
* The external group for a project whose cases
|
|
22291
|
+
* The external group for a project whose cases come from its own source.
|
|
22407
22292
|
*
|
|
22408
22293
|
* Such a case is not dispatched by reading a `spec.yaml` `target:` — the
|
|
22409
|
-
* target that declares the
|
|
22294
|
+
* target that declares the case source already owns every case under it — so
|
|
22410
22295
|
* the routing is one lookup rather than a walk. What it shares with
|
|
22411
22296
|
* {@link groupSpecsByTarget} is the two conditions that decide whether ccqa
|
|
22412
22297
|
* can execute a generated test at all: the plugin has a runner, and the
|
|
22413
22298
|
* project configured the command to run it with.
|
|
22414
22299
|
*/
|
|
22415
|
-
function
|
|
22300
|
+
function groupOwnCases(cases, targetId, config) {
|
|
22416
22301
|
if (cases.length === 0) return {
|
|
22417
22302
|
external: [],
|
|
22418
22303
|
skipped: []
|
|
@@ -22453,7 +22338,7 @@ async function runExternalSpecs(dispatch, ctx) {
|
|
|
22453
22338
|
...emptySpecRow({
|
|
22454
22339
|
feature: u.featureName,
|
|
22455
22340
|
spec: u.specName,
|
|
22456
|
-
title: u
|
|
22341
|
+
title: titleOf(u),
|
|
22457
22342
|
status: "failed"
|
|
22458
22343
|
}),
|
|
22459
22344
|
...u.targetId ? { target: u.targetId } : {},
|
|
@@ -22467,7 +22352,7 @@ async function runExternalSpecs(dispatch, ctx) {
|
|
|
22467
22352
|
...emptySpecRow({
|
|
22468
22353
|
feature: s.featureName,
|
|
22469
22354
|
spec: s.specName,
|
|
22470
|
-
title: s
|
|
22355
|
+
title: titleOf(s),
|
|
22471
22356
|
status: "skipped"
|
|
22472
22357
|
}),
|
|
22473
22358
|
...s.targetId ? { target: s.targetId } : {},
|
|
@@ -22508,7 +22393,7 @@ async function runExternalSpecs(dispatch, ctx) {
|
|
|
22508
22393
|
...emptySpecRow({
|
|
22509
22394
|
feature: s.featureName,
|
|
22510
22395
|
spec: s.specName,
|
|
22511
|
-
title: s
|
|
22396
|
+
title: titleOf(s),
|
|
22512
22397
|
status: "failed"
|
|
22513
22398
|
}),
|
|
22514
22399
|
target: group.targetId,
|
|
@@ -23086,18 +22971,16 @@ function asHubReadError$1(err) {
|
|
|
23086
22971
|
throw new RunUsageError(`could not read from the hub: ${errMessage(err)}`);
|
|
23087
22972
|
}
|
|
23088
22973
|
/**
|
|
23089
|
-
* The cases a project's own
|
|
23090
|
-
* every one
|
|
23091
|
-
*
|
|
23092
|
-
*
|
|
22974
|
+
* The cases a project's own source holds, as this invocation named them:
|
|
22975
|
+
* every one the source holds, or exactly the ones the arguments name. An
|
|
22976
|
+
* argument is read both ways a person writes it — the path they see in their
|
|
22977
|
+
* editor, or the id everything else cites.
|
|
23093
22978
|
*/
|
|
23094
|
-
async function
|
|
23095
|
-
const
|
|
23096
|
-
|
|
23097
|
-
|
|
23098
|
-
|
|
23099
|
-
throw new RunUsageError(errMessage(err));
|
|
23100
|
-
}
|
|
22979
|
+
async function readOwnCases(targets, reader) {
|
|
22980
|
+
const named = targets.length > 0;
|
|
22981
|
+
const refs = named ? [...new Set(targets)] : await reader.list();
|
|
22982
|
+
const cases = await Promise.all(refs.map((ref) => reader.load(ref)));
|
|
22983
|
+
return named ? cases : cases.filter((c) => !c.disabled);
|
|
23101
22984
|
}
|
|
23102
22985
|
/** De-dupe by `featureName/specName`, keeping first-seen order. */
|
|
23103
22986
|
function dedupeSpecs(specs) {
|
|
@@ -23278,6 +23161,8 @@ async function executeRun(targets, opts) {
|
|
|
23278
23161
|
const triageUserPromptHash = triageUserPrompt ? hashTriageUserPrompt(triageUserPrompt) : null;
|
|
23279
23162
|
const reportDir = resolveReportDir(opts.reportDir, cwd);
|
|
23280
23163
|
const parsedBlocks = await loadAllBlocks(cwd);
|
|
23164
|
+
const projectConfig = await loadProjectConfig(cwd);
|
|
23165
|
+
const reader = openCaseReader(projectConfig, cwd);
|
|
23281
23166
|
const analysisDeps = {
|
|
23282
23167
|
diffProvider,
|
|
23283
23168
|
auth: diffProvider ? driftAuthAvailable() : {
|
|
@@ -23286,6 +23171,7 @@ async function executeRun(targets, opts) {
|
|
|
23286
23171
|
},
|
|
23287
23172
|
cwd,
|
|
23288
23173
|
reportDir,
|
|
23174
|
+
reader,
|
|
23289
23175
|
blocks: projectAvailableBlocks(parsedBlocks),
|
|
23290
23176
|
parsedBlocks,
|
|
23291
23177
|
...opts.model ? { model: opts.model } : {},
|
|
@@ -23293,18 +23179,15 @@ async function executeRun(targets, opts) {
|
|
|
23293
23179
|
customPrompt,
|
|
23294
23180
|
triageUserPrompt
|
|
23295
23181
|
};
|
|
23296
|
-
const projectConfig = await loadProjectConfig(cwd);
|
|
23297
23182
|
await loadEnvFiles(projectConfig.envFiles, cwd);
|
|
23298
|
-
const
|
|
23299
|
-
|
|
23300
|
-
|
|
23301
|
-
|
|
23302
|
-
cases: await readIntentCases(targets, intentTarget.intent, cwd)
|
|
23183
|
+
const ownRun = reader.target === null ? null : {
|
|
23184
|
+
id: reader.target.id,
|
|
23185
|
+
module: reader.target.module,
|
|
23186
|
+
cases: await readOwnCases(targets, reader)
|
|
23303
23187
|
};
|
|
23304
|
-
if (intentRun !== null && opts.onlyHubRerunNeeded) throw new RunUsageError("--only-hub-rerun-needed reads the hub's per-spec rerun verdicts, which a project whose cases are its own documents has none of — name the cases to run instead");
|
|
23305
23188
|
const enumerateAll = () => listActiveSpecs(cwd);
|
|
23306
|
-
const resolved =
|
|
23307
|
-
let specs =
|
|
23189
|
+
const resolved = ownRun ? [] : await Promise.all((targets.length ? targets : [void 0]).map((t) => resolveSpecTargets(t, enumerateAll, cwd)));
|
|
23190
|
+
let specs = ownRun ? ownRun.cases.map((c) => splitCaseId(c.ref.id)) : dedupeSpecs(resolved.flat());
|
|
23308
23191
|
if (filtering) {
|
|
23309
23192
|
const before = specs.length;
|
|
23310
23193
|
let inProgress = 0;
|
|
@@ -23336,16 +23219,16 @@ async function executeRun(targets, opts) {
|
|
|
23336
23219
|
}
|
|
23337
23220
|
}
|
|
23338
23221
|
const selectedKeys = new Set(specs.map(specKey));
|
|
23339
|
-
const selectedCases =
|
|
23222
|
+
const selectedCases = ownRun?.cases.filter((c) => selectedKeys.has(specKey(splitCaseId(c.ref.id)))) ?? [];
|
|
23340
23223
|
const liveCaseCount = selectedCases.filter(runsLive).length;
|
|
23341
23224
|
const generatedCases = opts.coverage === true || targets.length > 0 ? selectedCases.filter((c) => !runsLive(c)) : [];
|
|
23342
|
-
if (specs.length === 0 ||
|
|
23343
|
-
if (
|
|
23225
|
+
if (specs.length === 0 || ownRun !== null && liveCaseCount + generatedCases.length === 0) {
|
|
23226
|
+
if (ownRun === null) {
|
|
23344
23227
|
warn("no specs to run");
|
|
23345
|
-
if (Object.values(projectConfig.targets).some((t) => t.
|
|
23346
|
-
} else if (
|
|
23228
|
+
if (Object.values(projectConfig.targets).some((t) => t.cases)) hint(`this project has a target whose cases come from its own files, but its defaultTarget (${projectConfig.defaultTarget}) is not that target, so this run enumerated .ccqa/features/ instead`);
|
|
23229
|
+
} else if (ownRun.cases.length === 0) warn(`${ownRun.module} lists no test cases`);
|
|
23347
23230
|
else {
|
|
23348
|
-
warn(`nothing to run: ${
|
|
23231
|
+
warn(`nothing to run: ${ownRun.module} lists ${ownRun.cases.length} test case(s), none of them live`);
|
|
23349
23232
|
hint("ccqa runs a case itself only when its mode says `live`; every other case's test is executed by the project's own test command, which `ccqa select-specs --format paths` feeds");
|
|
23350
23233
|
hint("pass --coverage, or name the cases, to have ccqa run their generated tests through the target's runCommand — which is how a measurement observes what they reach");
|
|
23351
23234
|
}
|
|
@@ -23355,7 +23238,7 @@ async function executeRun(targets, opts) {
|
|
|
23355
23238
|
reportDir: null
|
|
23356
23239
|
};
|
|
23357
23240
|
}
|
|
23358
|
-
const catalog = await readSpecs(
|
|
23241
|
+
const catalog = await readSpecs(ownRun ? [] : specs, reader);
|
|
23359
23242
|
const actors = opts.coverage === true && forExecution ? await resolveActors(projectConfig.coverage?.actors ?? {}, cwd) : NO_ACTORS;
|
|
23360
23243
|
const resources = mergeGroups(await resolveSerialGroups(projectConfig.serialGroups, cwd), actorGroups(actors));
|
|
23361
23244
|
const declared = [...new Set(specs.flatMap(resources))];
|
|
@@ -23385,31 +23268,33 @@ async function executeRun(targets, opts) {
|
|
|
23385
23268
|
}
|
|
23386
23269
|
let dispatch;
|
|
23387
23270
|
try {
|
|
23388
|
-
dispatch = groupSpecsByTarget(
|
|
23389
|
-
if (
|
|
23390
|
-
const routed =
|
|
23271
|
+
dispatch = groupSpecsByTarget(ownRun ? [] : specs, catalog, projectConfig);
|
|
23272
|
+
if (ownRun !== null && generatedCases.length > 0) {
|
|
23273
|
+
const routed = groupOwnCases(generatedCases.map((c) => ({
|
|
23391
23274
|
...splitCaseId(c.ref.id),
|
|
23392
|
-
|
|
23393
|
-
|
|
23275
|
+
caseId: c.ref.id,
|
|
23276
|
+
testCase: c
|
|
23277
|
+
})), ownRun.id, projectConfig);
|
|
23394
23278
|
dispatch.external.push(...routed.external);
|
|
23395
23279
|
dispatch.skipped.push(...routed.skipped);
|
|
23396
23280
|
}
|
|
23397
23281
|
for (const { spec, groups } of waitingOnGroup) dispatch.skipped.push({
|
|
23398
23282
|
...spec,
|
|
23399
|
-
|
|
23283
|
+
caseId: specKey(spec),
|
|
23284
|
+
testCase: catalog.get(specKey(spec))?.case ?? null,
|
|
23400
23285
|
reason: `waiting on ${groups.join(", ")}, held by another job`,
|
|
23401
23286
|
targetId: null
|
|
23402
23287
|
});
|
|
23403
23288
|
} catch (err) {
|
|
23404
23289
|
throw new RunUsageError(errMessage(err));
|
|
23405
23290
|
}
|
|
23406
|
-
const withMode =
|
|
23291
|
+
const withMode = ownRun ? selectedCases.map((c) => ({
|
|
23407
23292
|
...splitCaseId(c.ref.id),
|
|
23408
23293
|
mode: c.mode
|
|
23409
23294
|
})) : resolveSpecsModes(dispatch.agentBrowser, catalog);
|
|
23410
|
-
const detSpecs =
|
|
23295
|
+
const detSpecs = ownRun ? [] : withMode.filter((s) => s.mode === "deterministic");
|
|
23411
23296
|
const liveSpecs = withMode.filter((s) => s.mode === "live");
|
|
23412
|
-
meta("modes",
|
|
23297
|
+
meta("modes", ownRun ? `${selectedCases.length} case(s) / ${liveSpecs.length} live / ${generatedCases.length} generated, run here / ${selectedCases.length - liveSpecs.length - generatedCases.length} for the project's own test command` : `${detSpecs.length} deterministic / ${liveSpecs.length} live`);
|
|
23413
23298
|
if (dispatch.external.length > 0) meta("external", dispatch.external.map((g) => `${g.targetId} ${g.specs.length}`).join(" / "));
|
|
23414
23299
|
const storedMaps = hubCtx !== null && deployedSha !== null ? createStoredSourceMaps(hubCtx, deployedSha) : void 0;
|
|
23415
23300
|
const coverage = opts.coverage && forExecution ? await startCoverage(cwd, projectConfig.coverage, actors, dispatch, opts.teardown, coverageInbox, storedMaps?.read, hubCtx !== null) : void 0;
|
|
@@ -23424,17 +23309,18 @@ async function executeRun(targets, opts) {
|
|
|
23424
23309
|
blank();
|
|
23425
23310
|
if (opts.dryRun) {
|
|
23426
23311
|
const runningHere = new Set(generatedCases.map((c) => specKey(splitCaseId(c.ref.id))));
|
|
23427
|
-
const notOurs = (
|
|
23312
|
+
const notOurs = (ownRun ? withMode.filter((s) => s.mode !== "live" && !runningHere.has(specKey(s))) : []).map((s) => ({
|
|
23428
23313
|
...s,
|
|
23429
|
-
|
|
23314
|
+
caseId: specKey(s),
|
|
23315
|
+
testCase: null,
|
|
23430
23316
|
reason: "run by the project's own test command",
|
|
23431
|
-
targetId:
|
|
23317
|
+
targetId: ownRun?.id ?? null
|
|
23432
23318
|
}));
|
|
23433
23319
|
const routed = {
|
|
23434
23320
|
...dispatch,
|
|
23435
23321
|
skipped: [...dispatch.skipped, ...notOurs]
|
|
23436
23322
|
};
|
|
23437
|
-
const ours =
|
|
23323
|
+
const ours = ownRun ? liveSpecs : withMode;
|
|
23438
23324
|
for (const line of formatDryRunLines(ours, routed, resources)) emitRaw(line + "\n");
|
|
23439
23325
|
blank();
|
|
23440
23326
|
info("dry run: nothing was executed and no report was written");
|
|
@@ -23531,10 +23417,10 @@ async function executeRun(targets, opts) {
|
|
|
23531
23417
|
report: incrementalReport,
|
|
23532
23418
|
conventions: targetConfigFor(projectConfig, AGENT_BROWSER_TARGET).conventions.operate
|
|
23533
23419
|
};
|
|
23534
|
-
const liveCases =
|
|
23535
|
-
const
|
|
23536
|
-
if (
|
|
23537
|
-
return liveCaseFrom(
|
|
23420
|
+
const liveCases = ownRun ? selectedCases.filter(runsLive).map((c) => liveCaseFrom(c, liveCaseOpts)) : liveSpecs.map((ref) => {
|
|
23421
|
+
const testCase = catalog.get(specKey(ref))?.case;
|
|
23422
|
+
if (testCase == null) throw new Error(`${specKey(ref)}: spec.yaml is no longer readable`);
|
|
23423
|
+
return liveCaseFrom(testCase, liveCaseOpts);
|
|
23538
23424
|
});
|
|
23539
23425
|
const live = await runLiveSpecs(liveCases, liveOpts);
|
|
23540
23426
|
let streamedEdges = {};
|
|
@@ -23556,8 +23442,8 @@ async function executeRun(targets, opts) {
|
|
|
23556
23442
|
featureName: r.feature,
|
|
23557
23443
|
specName: r.spec
|
|
23558
23444
|
}))], analysisDeps);
|
|
23559
|
-
const detResults = await analyzeDeterministicSummaries(det.summaries,
|
|
23560
|
-
const analyzedExternalRows = await analyzeExternalRows(externalRows, analysisRun);
|
|
23445
|
+
const detResults = await analyzeDeterministicSummaries(det.summaries, catalog, reportDir, analysisRun);
|
|
23446
|
+
const analyzedExternalRows = await analyzeExternalRows(externalRows, analysisRun, new Map(dispatch.external.flatMap((g) => g.specs.map((c) => [specKey(c), c.caseId]))));
|
|
23561
23447
|
const results = await rerunExplainedFailures([
|
|
23562
23448
|
...detResults,
|
|
23563
23449
|
...analyzedExternalRows,
|
|
@@ -23960,18 +23846,18 @@ function failedSpec(s) {
|
|
|
23960
23846
|
* specs share). Degrades — never throws — when Claude auth or the git diff
|
|
23961
23847
|
* aren't available. Caller writes report.json.
|
|
23962
23848
|
*/
|
|
23963
|
-
async function analyzeDeterministicSummaries(summaries,
|
|
23849
|
+
async function analyzeDeterministicSummaries(summaries, catalog, reportDir, { pass, deps }) {
|
|
23964
23850
|
const results = [];
|
|
23965
23851
|
for (const s of summaries) {
|
|
23966
23852
|
const assertions = collectAssertions(s);
|
|
23967
|
-
const
|
|
23968
|
-
const
|
|
23969
|
-
const
|
|
23970
|
-
const evidence = await loadEvidenceForSpec(s.evidenceDir, reportDir,
|
|
23853
|
+
const testCase = catalog.get(specKey(s))?.case ?? null;
|
|
23854
|
+
const specYaml = testCase?.document.text ?? null;
|
|
23855
|
+
const parsedSpec = testCase?.spec ?? null;
|
|
23856
|
+
const evidence = await loadEvidenceForSpec(s.evidenceDir, reportDir, stepCaptions(testCase));
|
|
23971
23857
|
const base = {
|
|
23972
23858
|
feature: s.featureName,
|
|
23973
23859
|
spec: s.specName,
|
|
23974
|
-
title:
|
|
23860
|
+
title: testCase?.title ?? null,
|
|
23975
23861
|
target: AGENT_BROWSER_TARGET,
|
|
23976
23862
|
startedAt: s.startedAt,
|
|
23977
23863
|
finishedAt: s.finishedAt,
|
|
@@ -24064,7 +23950,7 @@ function createRerunExecutor(ctx) {
|
|
|
24064
23950
|
}
|
|
24065
23951
|
const group = ctx.dispatch.external.find((g) => g.specs.some((s) => specKey(s) === key));
|
|
24066
23952
|
if (group === void 0) throw new Error(`${key} has no execution path to re-run`);
|
|
24067
|
-
const [row] = await group.runner.run([
|
|
23953
|
+
const [row] = await group.runner.run([group.specs.find((c) => specKey(c) === key)], {
|
|
24068
23954
|
cwd: ctx.cwd,
|
|
24069
23955
|
reportDir: scratch,
|
|
24070
23956
|
concurrency: 1,
|
|
@@ -24403,7 +24289,7 @@ function installTeardownSignalHandlers(teardown, onSignal) {
|
|
|
24403
24289
|
}
|
|
24404
24290
|
//#endregion
|
|
24405
24291
|
//#region src/cli/run.ts
|
|
24406
|
-
const runCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("run").argument("[targets...]", "Specs to run, space-separated: each '<feature>/<spec>', '<feature>', or omit for all. For a target
|
|
24292
|
+
const runCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("run").argument("[targets...]", "Specs to run, space-separated: each '<feature>/<spec>', '<feature>', or omit for all. For a target with a `cases` module, each is a case id or the path of its source file. Duplicates are de-duped.").description("Run specs, on any target. Agent-browser specs replay the recorded test.spec.ts under vitest (default), or, with spec.yaml `mode: live`, have Claude drive agent-browser live per step. External-target specs (playwright, runn) run through the target's configured `runCommand`. Where the project writes its cases as markdown, ccqa runs the ones whose mode says `live`; the rest are executed by the project's own test command (see `ccqa select-specs --format paths`). A structured report (report.json + evidence) is always written; use --report-to-hub to also stream it to a hub.").optionsGroup("Which specs to run:").option("--only-affected-by <ref>", "Only specs `ccqa select-specs` decides the diff against <ref> reaches (e.g. origin/main), by intersecting it with each spec's measured coverage from the hub. In pull_request CI, pass $GITHUB_BASE_REF. Cannot be combined with an explicit spec id.").option("--only-hub-rerun-needed", "Only specs the hub answers `rerunNeeded` for: the audit cleared them, and their last result does not cover what is deployed — including every spec the deploy log cannot place, which is assumed reached rather than skipped. A spec whose audit has not caught up answers `inProgress`, and one the audit rejected or whose last run failed answers `needsRepair`; neither is taken, because running them races the audit or repairs nothing. No git diff involved. Requires a hub connection and --hub-profile.").option("--dry-run", "Print the specs this invocation would run, then exit 0 without executing anything and without writing a report. Works with every selection flag.").optionsGroup("How to run them:").option("--concurrency <n>", "Run up to N specs in parallel within each phase (deterministic / external-target / live), never across phases. Default 1 (sequential). Specs in the same `serialGroups` entry of .ccqa/config.yaml still take turns. Live specs each get an isolated agent-browser session; high values spawn many headed Chrome instances.", parseConcurrency$1, 1).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--live-step-retry <n>", "(live only) Retry each failed step up to N more times before recording failure. This retries a step, not the whole spec — see --on-fail-explain-rerun for that.", (raw) => {
|
|
24407
24293
|
const n = Number(raw);
|
|
24408
24294
|
if (!Number.isFinite(n) || n < 0 || Math.floor(n) !== n) throw new Error(`--live-step-retry must be a non-negative integer, got "${raw}"`);
|
|
24409
24295
|
return n;
|
|
@@ -25661,7 +25547,7 @@ function stepsWithoutAsserts(stepIds, actions) {
|
|
|
25661
25547
|
async function runTrace(testCase, model, validationMode = "lenient", language, opts = {}) {
|
|
25662
25548
|
header("trace", testCase.ref.id);
|
|
25663
25549
|
await preflightAgentBrowserCommand();
|
|
25664
|
-
const specSteps = testCase.
|
|
25550
|
+
const specSteps = testCase.spec?.steps ?? [];
|
|
25665
25551
|
const expanded = [...testCase.steps, ...testCase.cleanup];
|
|
25666
25552
|
const steps = expanded.filter(isExpandedActionStep);
|
|
25667
25553
|
const sessionName = generateSessionName();
|
|
@@ -25675,7 +25561,7 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
|
|
|
25675
25561
|
meta("case", testCase.title);
|
|
25676
25562
|
meta("steps", steps.length);
|
|
25677
25563
|
if (testCase.cleanup.length > 0) meta("cleanup steps", testCase.cleanup.length);
|
|
25678
|
-
const includes = testCase.
|
|
25564
|
+
const includes = testCase.spec ? collectIncludedBlockNames(testCase.spec) : [];
|
|
25679
25565
|
if (includes.length > 0) meta("blocks", includes.join(", "));
|
|
25680
25566
|
blank();
|
|
25681
25567
|
opts.teardown?.trackSession(sessionName);
|
|
@@ -26264,6 +26150,85 @@ function dropList(drops) {
|
|
|
26264
26150
|
return drops.slice(0, 5).map((d) => ` - ${describeStepAction(d.action)} — ${d.reason}`).join("\n");
|
|
26265
26151
|
}
|
|
26266
26152
|
//#endregion
|
|
26153
|
+
//#region src/cli/resolve-case.ts
|
|
26154
|
+
async function resolveCase(argument, config, cwd, opts = {}) {
|
|
26155
|
+
const reader = openCaseReader(config, cwd, opts);
|
|
26156
|
+
const testCase = await reader.load(argument);
|
|
26157
|
+
if (reader.target) {
|
|
26158
|
+
const { id, targetConfig, modulePath } = reader.target;
|
|
26159
|
+
const target = registryFor(config).get(id);
|
|
26160
|
+
const testPath = resolveCaseTestPath(target, targetConfig, testCase.ref.id);
|
|
26161
|
+
const recordingPath = (opts.targetOverride === void 0 ? void 0 : ownCaseRecordingPath(config, cwd, modulePath, testCase.ref.id)) ?? resolveCaseRecordingPath(target, targetConfig, testCase.ref.id);
|
|
26162
|
+
const ref = caseRefFor(testCase.ref.id, cwd, recordingPath);
|
|
26163
|
+
return {
|
|
26164
|
+
testCase: {
|
|
26165
|
+
...testCase,
|
|
26166
|
+
ref
|
|
26167
|
+
},
|
|
26168
|
+
target,
|
|
26169
|
+
targetConfig,
|
|
26170
|
+
testPath
|
|
26171
|
+
};
|
|
26172
|
+
}
|
|
26173
|
+
const spec = testCase.spec;
|
|
26174
|
+
const specRef = splitSpecRef(testCase.ref.id);
|
|
26175
|
+
const target = opts.targetOverride !== void 0 ? resolveTargetOverride(spec, opts.targetOverride, config) : resolveTarget(spec, config);
|
|
26176
|
+
const targetConfig = targetConfigFor(config, target.id);
|
|
26177
|
+
const testPath = resolveTestPath(target, targetConfig, specRef);
|
|
26178
|
+
const ref = caseRefFor(specRef, cwd, opts.targetOverride === void 0 ? resolveRecordingPath(target, targetConfig, specRef) : ownRecordingPath(spec, config, specRef));
|
|
26179
|
+
return {
|
|
26180
|
+
testCase: {
|
|
26181
|
+
...testCase,
|
|
26182
|
+
ref
|
|
26183
|
+
},
|
|
26184
|
+
target,
|
|
26185
|
+
targetConfig,
|
|
26186
|
+
testPath
|
|
26187
|
+
};
|
|
26188
|
+
}
|
|
26189
|
+
/** A spec case's id is its two coordinates joined, and only ever those two. */
|
|
26190
|
+
function splitSpecRef(id) {
|
|
26191
|
+
const [featureName, specName] = id.split("/");
|
|
26192
|
+
return {
|
|
26193
|
+
featureName,
|
|
26194
|
+
specName
|
|
26195
|
+
};
|
|
26196
|
+
}
|
|
26197
|
+
/**
|
|
26198
|
+
* Where the spec's own target keeps this case's recording, ignoring any
|
|
26199
|
+
* `--target`. Undefined when that target no longer resolves: overriding the
|
|
26200
|
+
* target is how a spec with an unusable one is generated at all, so it must
|
|
26201
|
+
* not be what stops the command.
|
|
26202
|
+
*
|
|
26203
|
+
* Only the resolution is guarded. A target that resolved and then could not
|
|
26204
|
+
* expand its own `testPath` is a broken config, and swallowing that would put
|
|
26205
|
+
* the recording somewhere else without a word.
|
|
26206
|
+
*/
|
|
26207
|
+
function ownRecordingPath(spec, config, ref) {
|
|
26208
|
+
let target;
|
|
26209
|
+
try {
|
|
26210
|
+
target = resolveTarget(spec, config);
|
|
26211
|
+
} catch {
|
|
26212
|
+
return;
|
|
26213
|
+
}
|
|
26214
|
+
return resolveRecordingPath(target, targetConfigFor(config, target.id), ref);
|
|
26215
|
+
}
|
|
26216
|
+
/**
|
|
26217
|
+
* The case-source branch's half of the same rule: where the project's own
|
|
26218
|
+
* default target keeps this case's recording.
|
|
26219
|
+
*
|
|
26220
|
+
* Undefined when that target reads its cases through a different module than
|
|
26221
|
+
* the one this invocation resolved through — another module is describing a
|
|
26222
|
+
* different case that happens to share an id, and its route is not this
|
|
26223
|
+
* case's.
|
|
26224
|
+
*/
|
|
26225
|
+
function ownCaseRecordingPath(config, cwd, readingModule, caseId) {
|
|
26226
|
+
const own = caseTargetFor(config, cwd);
|
|
26227
|
+
if (own === null || own.modulePath !== readingModule) return void 0;
|
|
26228
|
+
const plugin = registryFor(config).get(own.id);
|
|
26229
|
+
return plugin ? resolveCaseRecordingPath(plugin, own.targetConfig, caseId) : void 0;
|
|
26230
|
+
}
|
|
26231
|
+
//#endregion
|
|
26267
26232
|
//#region src/cli/build-generate-run-summary.ts
|
|
26268
26233
|
/**
|
|
26269
26234
|
* Compact, prompt-friendly summary of one `ccqa generate` pass for the
|
|
@@ -26337,7 +26302,7 @@ async function runGenerate(resolved, opts) {
|
|
|
26337
26302
|
async function runGenerateLocked(resolved, opts, cwd) {
|
|
26338
26303
|
const { testCase, target, targetConfig, testPath } = resolved;
|
|
26339
26304
|
const { sessionState } = await loadProjectConfig(cwd);
|
|
26340
|
-
const spec = testCase.
|
|
26305
|
+
const spec = testCase.spec;
|
|
26341
26306
|
if (spec?.mode === "live") {
|
|
26342
26307
|
error(`this spec is 'mode: live' — a live spec runs without generated code. Run 'ccqa run ${testCase.ref.id}' instead`);
|
|
26343
26308
|
process.exit(2);
|
|
@@ -26394,7 +26359,7 @@ async function runGenerateLocked(resolved, opts, cwd) {
|
|
|
26394
26359
|
title: testCase.title,
|
|
26395
26360
|
steps: []
|
|
26396
26361
|
},
|
|
26397
|
-
specYaml:
|
|
26362
|
+
specYaml: spec ? testCase.document.text : "",
|
|
26398
26363
|
featureName,
|
|
26399
26364
|
specName,
|
|
26400
26365
|
ref: testCase.ref,
|
|
@@ -26425,7 +26390,6 @@ async function runGenerateLocked(resolved, opts, cwd) {
|
|
|
26425
26390
|
const written = await stampGeneratedTest(testCase.ref, testPathAbs);
|
|
26426
26391
|
if (written !== null && movedRecording !== void 0) info(`recording moved beside its test: ${written}`);
|
|
26427
26392
|
}
|
|
26428
|
-
if (result.passed) await writeBackOutputPath(testCase, targetConfig, testPath);
|
|
26429
26393
|
if (opts.updateAgentPrompt) await runGenerateAgentPromptUpdate(target, testCase.ref.id, result, opts, cwd);
|
|
26430
26394
|
if (!result.passed) warn("auto-fix exhausted; test still failing");
|
|
26431
26395
|
else hint(`run 'ccqa run ${testCase.ref.id}' to execute the test`);
|
|
@@ -26454,28 +26418,6 @@ async function runGenerateAgentPromptUpdate(target, caseId, result, opts, cwd) {
|
|
|
26454
26418
|
});
|
|
26455
26419
|
}
|
|
26456
26420
|
/**
|
|
26457
|
-
* Write the generated test's path into the case's own `outputPath` section.
|
|
26458
|
-
*
|
|
26459
|
-
* Only when the project named that section and the case actually has it: a
|
|
26460
|
-
* case file belongs to the project, and ccqa rewrites the one heading it was
|
|
26461
|
-
* given permission to rewrite, leaving every other byte alone.
|
|
26462
|
-
*/
|
|
26463
|
-
async function writeBackOutputPath(testCase, targetConfig, testPath) {
|
|
26464
|
-
const heading = targetConfig.intent?.fields.outputPath;
|
|
26465
|
-
if (!heading || testCase.source.kind !== "markdown") return;
|
|
26466
|
-
const { path } = testCase.source;
|
|
26467
|
-
const before = await readFile(path, "utf8").catch(() => null);
|
|
26468
|
-
if (before === null) return;
|
|
26469
|
-
const after = replaceSectionBody(before, heading, testPath);
|
|
26470
|
-
if (after === null) {
|
|
26471
|
-
warn(`the case has no "${heading}" section, so the generated test's path was not written back`);
|
|
26472
|
-
return;
|
|
26473
|
-
}
|
|
26474
|
-
if (after === before) return;
|
|
26475
|
-
await writeFile(path, after, "utf8");
|
|
26476
|
-
meta("wrote back", `${heading} in ${relative(process.cwd(), path) || path}`);
|
|
26477
|
-
}
|
|
26478
|
-
/**
|
|
26479
26421
|
* The reason regeneration is refused because the test on disk is not the one
|
|
26480
26422
|
* the last generation wrote — or null when it may proceed. A matching stamp
|
|
26481
26423
|
* means ccqa produced this file, so regenerating it costs nobody anything and
|
|
@@ -26522,13 +26464,11 @@ function staleRecordingWarning(caseId, editedAt, recordedAt) {
|
|
|
26522
26464
|
return `the case was edited after it was recorded (case ${editedAt.toISOString()}, recording ${recordedAt}). Generation compiles the recording, so anything added to the case since is not in it — run 'ccqa record ${caseId}' if the steps or the expected results changed.`;
|
|
26523
26465
|
}
|
|
26524
26466
|
async function warnIfCaseOutranRecording(testCase, recordedAt) {
|
|
26525
|
-
const
|
|
26526
|
-
if (path === void 0) return;
|
|
26527
|
-
const editedAt = await stat(path).then((st) => st.mtime, () => null);
|
|
26467
|
+
const editedAt = await stat(testCase.document.path).then((st) => st.mtime, () => null);
|
|
26528
26468
|
const warning = staleRecordingWarning(testCase.ref.id, editedAt, recordedAt);
|
|
26529
26469
|
if (warning !== null) warn(warning);
|
|
26530
26470
|
}
|
|
26531
|
-
const generateCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("generate").argument("<case>", "The case to generate from: a spec id ('<feature>/<spec>'), or — for a target
|
|
26471
|
+
const generateCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("generate").argument("<case>", "The case to generate from: a spec id ('<feature>/<spec>'), or — for a target with a `cases` module — a case id or the path of its source file").description("Generate test code from a case via its target. Recording-backed targets compile the case's existing recording (run `ccqa record` first); spec-input targets generate directly from the spec.").optionsGroup("How to generate:").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--target <id>", "Generate through this target instead of the spec's own — e.g. emit a Playwright spec from an agent-browser recording. The spec's `target:` stays the default for `ccqa run`.").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--no-replay", "Skip replaying the saved recording before regenerating from it. The replay drives your application for real — it performs the recorded actions, and whatever they create the recorded cleanup then tries to remove — and it needs a browser and the spec's variables. Skip it where that is unwelcome or unavailable, and accept that a route the application has outgrown regenerates into a test that cannot pass.").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--overwrite", "Replace an existing generated test without the y/N prompt (declines on a non-TTY)").optionsGroup("Learning:").option("--learn-hub-codegen-prompt", "After generation, ask Claude to refresh the target's \"<target>.agent\" learning prompt on the hub from a summary of the run. LLM-generating targets (playwright, runn) only; requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
26532
26472
|
await withCostReporting("generate", () => runGenerateCli(specPath, opts));
|
|
26533
26473
|
}));
|
|
26534
26474
|
async function runGenerateCli(caseArgument, opts) {
|
|
@@ -26598,7 +26538,7 @@ async function runGenerateCli(caseArgument, opts) {
|
|
|
26598
26538
|
//#endregion
|
|
26599
26539
|
//#region src/cli/record.ts
|
|
26600
26540
|
const VALIDATION_MODES = ["lenient", "strict"];
|
|
26601
|
-
const recordCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("record").argument("<case>", "The case to record: a spec id ('<feature>/<spec>'), or — for a target
|
|
26541
|
+
const recordCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("record").argument("<case>", "The case to record: a spec id ('<feature>/<spec>'), or — for a target with a `cases` module — a case id or the path of its source file").description("Record a test from a case: run agent-browser to collect actions (trace), then compile them into runnable code via the case's target (generate) — a vitest test.spec.ts for agent-browser, a @playwright/test spec for playwright and for a target the project defines. Recording-backed targets only; spec-input targets like runn have no trace step (use `ccqa generate`), and agent-browser live specs need no recording.").optionsGroup("How to record:").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--instruction <text>", "Extra guidance for the recording agent — e.g. the drift audit's finding when re-recording a drifted spec.").option("--trace-validation <mode>", "What to do with actions that fail post-trace validation: 'lenient' (default) tags them; 'strict' drops them.", (raw) => {
|
|
26602
26542
|
if (VALIDATION_MODES.includes(raw)) return raw;
|
|
26603
26543
|
throw new Error(`--trace-validation must be one of ${VALIDATION_MODES.join(" | ")}`);
|
|
26604
26544
|
}, "lenient").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--timeout <seconds>", "Abort the recording after this many seconds, wherever it is (trace, generate, auto-fix): reap the browser session, seal the open hub run (--report-to-hub) with a 'timed out' note, and exit 124. Prefer this over wrapping the command in an external `timeout`, whose SIGTERM may never reach this process.", parseTimeoutSeconds).option("--trace-only", "Stop after the trace step; do not generate test code").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--report-to-hub", "Leave a run (kind: record) on the hub saying this spec was recorded and what the recording spent on Claude, so a budget summed over the hub's runs sees it. It advances no ledger: a recording verifies nothing.").optionsGroup("Learning:").option("--learn-hub-trace-prompt", "After the trace finishes, ask Claude to refresh the \"record.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
@@ -26613,7 +26553,7 @@ async function runRecord(caseArgument, opts) {
|
|
|
26613
26553
|
const resolved = await resolveCase(caseArgument, config, cwdForProfile);
|
|
26614
26554
|
const { testCase, target } = resolved;
|
|
26615
26555
|
const caseId = testCase.ref.id;
|
|
26616
|
-
const spec = testCase.
|
|
26556
|
+
const spec = testCase.spec;
|
|
26617
26557
|
if (target.input === "spec") {
|
|
26618
26558
|
error(`target "${target.id}" does not use a browser recording — run 'ccqa generate ${caseId}' instead`);
|
|
26619
26559
|
process.exit(2);
|
|
@@ -27481,7 +27421,7 @@ These are the same definitions failure analysis uses on a case that actually fai
|
|
|
27481
27421
|
|
|
27482
27422
|
## What separates TEST_DRIFT from SPEC_CHANGE
|
|
27483
27423
|
|
|
27484
|
-
This is the distinction that matters, because the two lead to different actions: TEST_DRIFT gets the test
|
|
27424
|
+
This is the distinction that matters, because the two lead to different actions: TEST_DRIFT gets the stale string replaced wherever it is written and the test recompiled, SPEC_CHANGE gets a human to decide what the case should now say.
|
|
27485
27425
|
|
|
27486
27426
|
Ask whether the **intent** the step describes still exists in the product:
|
|
27487
27427
|
|
|
@@ -27576,12 +27516,17 @@ Drift found:
|
|
|
27576
27516
|
"evidence": [
|
|
27577
27517
|
{ "file": "<path:line>", "detail": "<what this proves>" }
|
|
27578
27518
|
]
|
|
27579
|
-
}
|
|
27519
|
+
},
|
|
27520
|
+
"renames": [
|
|
27521
|
+
{ "from": "<the string as the test case writes it>", "to": "<what the source renders in its place>" }
|
|
27522
|
+
]
|
|
27580
27523
|
}
|
|
27581
27524
|
\`\`\`
|
|
27582
27525
|
|
|
27583
27526
|
\`subDiagnosis\`: \`SELECTOR_DRIFT\` when a selector or string was renamed, \`OVER_ASSERTION\` when the case asserts something narrower than the product ever promised, \`NONE\` otherwise.
|
|
27584
27527
|
|
|
27528
|
+
\`renames\`: one entry per renamed string, when the label is \`TEST_DRIFT\` and \`subDiagnosis\` is \`SELECTOR_DRIFT\` — a finding that says a string was renamed and does not name the pair leaves the repair to a person. Omit the key otherwise. It sits beside \`drift\`, and beside \`locators\` where that was asked for. \`from\` is the string as the test case writes it — in the generated code, in the case's own document, or both — and \`to\` is what the source you cited renders in its place. Give each verbatim, with no quotes, ellipses or paraphrase of your own: each pair is looked up character for character in the file it claims to be about, and one that does not match is dropped. Naming a rename neither settles the label nor stands in for the citation the finding still needs.
|
|
27529
|
+
|
|
27585
27530
|
\`specChangeKind\`: set it only when the label is \`SPEC_CHANGE\`, and omit the field entirely otherwise. It says which repair the case needs — deleting it, or rewriting and re-recording it:
|
|
27586
27531
|
|
|
27587
27532
|
- \`FEATURE_REMOVED\` — the code no longer implements the behaviour at all: removed, moved elsewhere, or deliberately disabled. This is the stronger claim, so earn it: your evidence must point at where the implementation would be if it still existed.
|
|
@@ -27594,7 +27539,7 @@ function buildDriftUserPrompt(artifacts, sourceRoots = [], locators) {
|
|
|
27594
27539
|
const { kind, path, body } = artifacts.intent;
|
|
27595
27540
|
return `## ${kind === "spec" ? "spec.yaml" : `Test case document — ${path}`}
|
|
27596
27541
|
|
|
27597
|
-
\`\`\`${kind === "spec" ? "yaml" : "
|
|
27542
|
+
\`\`\`${kind === "spec" ? "yaml" : extname(path).replace(/^\./, "")}
|
|
27598
27543
|
${body}
|
|
27599
27544
|
\`\`\`
|
|
27600
27545
|
|
|
@@ -27637,9 +27582,9 @@ What renders a screen is often a template rather than a script — a \`.vue\`, \
|
|
|
27637
27582
|
|
|
27638
27583
|
`;
|
|
27639
27584
|
}
|
|
27640
|
-
/** What states the case
|
|
27585
|
+
/** What states the case: ccqa's own format, or one it does not know. */
|
|
27641
27586
|
function intentSurfaceBlock(kind) {
|
|
27642
|
-
if (kind === "
|
|
27587
|
+
if (kind === "project") return `- **the test case document** — always present. Written and owned by the project's own authors, in whatever format they keep their test cases in; ccqa does not parse it and neither should you assume a shape. Read it as a statement of intent: what the case does, and what must then be true. Its structure and vocabulary are theirs, not ccqa's.`;
|
|
27643
27588
|
return `- **spec.yaml** — always present. Pure YAML: \`title\`, then \`steps\`, each either an action (\`instruction\` + \`expected\`) or \`include: <block-name>\` with \`params\`. \`expected\` names something observable — visible text, an aria-label, a URL, an element state.`;
|
|
27644
27589
|
}
|
|
27645
27590
|
function generatedSection(artifacts) {
|
|
@@ -28076,41 +28021,6 @@ function repairTargetsOutside(evidence, area) {
|
|
|
28076
28021
|
return outside.length > 0 ? outside : null;
|
|
28077
28022
|
}
|
|
28078
28023
|
//#endregion
|
|
28079
|
-
//#region src/drift/types.ts
|
|
28080
|
-
/** How everything downstream spells this case: its intent-source id, or `feature/spec`. */
|
|
28081
|
-
function caseIdOf(target) {
|
|
28082
|
-
return target.caseId ?? `${target.featureName}/${target.specName}`;
|
|
28083
|
-
}
|
|
28084
|
-
/**
|
|
28085
|
-
* What the audit says about one locator code could not find in the product.
|
|
28086
|
-
*
|
|
28087
|
-
* `drifted` says the test asks for something the product no longer renders;
|
|
28088
|
-
* `fine` says the miss has another explanation — a selector built at runtime, a
|
|
28089
|
-
* component this case never reaches, a file the scan did not read. Either is an
|
|
28090
|
-
* answer; silence is not, which is what the required list below is for.
|
|
28091
|
-
*/
|
|
28092
|
-
const LocatorVerdictSchema = z.object({
|
|
28093
|
-
id: z.string().min(1),
|
|
28094
|
-
verdict: z.enum(["drifted", "fine"]),
|
|
28095
|
-
note: z.string().default("")
|
|
28096
|
-
});
|
|
28097
|
-
/** The model's reply: a diagnosis, or `null` for "the spec still matches the code". */
|
|
28098
|
-
const DriftReplySchema = z.object({
|
|
28099
|
-
drift: DriftDiagnosisSchema.nullable(),
|
|
28100
|
-
locators: z.array(LocatorVerdictSchema).default([])
|
|
28101
|
-
});
|
|
28102
|
-
/**
|
|
28103
|
-
* How a label reads against `--exit-on`. The threshold asks "would a
|
|
28104
|
-
* deterministic replay fail today". `TEST_DRIFT` and `SPEC_CHANGE` answer yes
|
|
28105
|
-
* and name the repair, so they hold the gate shut. The other three do not: a
|
|
28106
|
-
* suspected product bug or a dependency on data the source does not decide is
|
|
28107
|
-
* settled by running the case, not by refusing to run it, and `UNKNOWN` says
|
|
28108
|
-
* the audit could not tell.
|
|
28109
|
-
*/
|
|
28110
|
-
function driftSeverity(label) {
|
|
28111
|
-
return label === "TEST_DRIFT" || label === "SPEC_CHANGE" ? "error" : "warn";
|
|
28112
|
-
}
|
|
28113
|
-
//#endregion
|
|
28114
28024
|
//#region src/drift/analyze.ts
|
|
28115
28025
|
const DEFAULT_CONCURRENCY$1 = 3;
|
|
28116
28026
|
/**
|
|
@@ -28217,7 +28127,7 @@ async function checkSpec(target, opts) {
|
|
|
28217
28127
|
correctSurface(drift, {
|
|
28218
28128
|
cwd: opts.cwd,
|
|
28219
28129
|
sourceRoots: opts.sourceRoots.map((r) => r.abs),
|
|
28220
|
-
...await caseWriteArea(target, opts.
|
|
28130
|
+
...await caseWriteArea(target, opts.context)
|
|
28221
28131
|
});
|
|
28222
28132
|
}
|
|
28223
28133
|
return {
|
|
@@ -28225,7 +28135,9 @@ async function checkSpec(target, opts) {
|
|
|
28225
28135
|
ok: true,
|
|
28226
28136
|
drift,
|
|
28227
28137
|
live: artifacts.live,
|
|
28228
|
-
title: artifacts.title
|
|
28138
|
+
title: artifacts.title,
|
|
28139
|
+
documentPath: artifacts.intent.path,
|
|
28140
|
+
renames: auditedRenames(artifacts.intent.body, reply.renames)
|
|
28229
28141
|
};
|
|
28230
28142
|
} catch (e) {
|
|
28231
28143
|
lastError = `failed to parse drift reply: ${e.message}`;
|
|
@@ -28297,7 +28209,7 @@ function renderGithub(results, cwd) {
|
|
|
28297
28209
|
const repoRoot = process.env["GITHUB_WORKSPACE"] ?? process.cwd();
|
|
28298
28210
|
const lines = [];
|
|
28299
28211
|
for (const r of results) {
|
|
28300
|
-
const file = githubRelPath(cwd, repoRoot, r
|
|
28212
|
+
const file = githubRelPath(cwd, repoRoot, r);
|
|
28301
28213
|
if (r.error) {
|
|
28302
28214
|
lines.push(`::error file=${file}::${escapeGhMessage(r.error)}`);
|
|
28303
28215
|
continue;
|
|
@@ -28310,8 +28222,15 @@ function renderGithub(results, cwd) {
|
|
|
28310
28222
|
}
|
|
28311
28223
|
return lines.length === 0 ? "" : `${lines.join("\n")}\n`;
|
|
28312
28224
|
}
|
|
28313
|
-
|
|
28314
|
-
|
|
28225
|
+
/**
|
|
28226
|
+
* The file an annotation points at: the document this case is stated in.
|
|
28227
|
+
*
|
|
28228
|
+
* Falls back to the case's directory when the audit never got far enough to
|
|
28229
|
+
* read one — an annotation on a directory still lands a reviewer in the right
|
|
28230
|
+
* place, and one on a `spec.yaml` that a project does not use lands nowhere.
|
|
28231
|
+
*/
|
|
28232
|
+
function githubRelPath(cwd, repoRoot, result) {
|
|
28233
|
+
const abs = resolve(cwd, result.documentPath ?? join(".ccqa", "features", result.target.featureName, "test-cases", result.target.specName));
|
|
28315
28234
|
const rel = relative(repoRoot, abs);
|
|
28316
28235
|
return rel.startsWith("..") ? abs : rel;
|
|
28317
28236
|
}
|
|
@@ -28357,7 +28276,7 @@ function determineExitCode(results, threshold) {
|
|
|
28357
28276
|
//#endregion
|
|
28358
28277
|
//#region src/drift/to-report.ts
|
|
28359
28278
|
/** Tracks the drift prompt's own version — the two must never drift apart. */
|
|
28360
|
-
const DRIFT_REPORT_PROMPT_VERSION = "
|
|
28279
|
+
const DRIFT_REPORT_PROMPT_VERSION = "12";
|
|
28361
28280
|
/**
|
|
28362
28281
|
* Spec-level status under the given threshold, mirroring determineExitCode's
|
|
28363
28282
|
* per-spec logic (exit-code.ts) but scoped to a single SpecResult.
|
|
@@ -28437,7 +28356,11 @@ async function writeAuditBriefs(input) {
|
|
|
28437
28356
|
async function buildBrief(result, cwd, ctx) {
|
|
28438
28357
|
const drift = result.drift;
|
|
28439
28358
|
const id = caseIdOf(result.target);
|
|
28440
|
-
const test = result.live ? null : await caseTestPath(result.target,
|
|
28359
|
+
const test = result.live ? null : await caseTestPath(result.target, ctx);
|
|
28360
|
+
const rewrite = drift.label === "TEST_DRIFT" ? (result.renames ?? []).filter((r) => r.inDocument).map(({ from, to }) => ({
|
|
28361
|
+
from,
|
|
28362
|
+
to
|
|
28363
|
+
})) : [];
|
|
28441
28364
|
return {
|
|
28442
28365
|
case: id,
|
|
28443
28366
|
kind: drift.label,
|
|
@@ -28448,39 +28371,53 @@ async function buildBrief(result, cwd, ctx) {
|
|
|
28448
28371
|
reasoning: drift.reasoning,
|
|
28449
28372
|
evidence: drift.evidence,
|
|
28450
28373
|
test,
|
|
28451
|
-
|
|
28374
|
+
document: documentOf(result, cwd),
|
|
28375
|
+
repair: await buildRepair(result, test, rewrite, cwd, ctx)
|
|
28452
28376
|
};
|
|
28453
28377
|
}
|
|
28454
|
-
|
|
28378
|
+
/**
|
|
28379
|
+
* The case's document as something that can be opened from anywhere. A case
|
|
28380
|
+
* filed outside the project relativizes to `../…`, which resolves against
|
|
28381
|
+
* whatever directory the fix job happens to be in rather than this one.
|
|
28382
|
+
*/
|
|
28383
|
+
function documentOf(result, cwd) {
|
|
28384
|
+
const rel = result.documentPath;
|
|
28385
|
+
if (rel === void 0) return null;
|
|
28386
|
+
return rel.startsWith("..") ? resolve(cwd, rel) : rel;
|
|
28387
|
+
}
|
|
28388
|
+
async function buildRepair(result, test, rewrite, cwd, ctx) {
|
|
28455
28389
|
const drift = result.drift;
|
|
28456
|
-
|
|
28457
|
-
|
|
28458
|
-
reason: "this case runs live: the document is the test"
|
|
28459
|
-
};
|
|
28460
|
-
if (drift.label !== "TEST_DRIFT") return {
|
|
28461
|
-
route: "external",
|
|
28462
|
-
reason: `${drift.label} is not repaired by regenerating: the same document would compile to the same test`
|
|
28463
|
-
};
|
|
28464
|
-
if (drift.surface !== "generated") return {
|
|
28390
|
+
const renames = result.renames ?? [];
|
|
28391
|
+
const external = (reason) => ({
|
|
28465
28392
|
route: "external",
|
|
28466
|
-
reason
|
|
28393
|
+
reason,
|
|
28394
|
+
rewrite
|
|
28395
|
+
});
|
|
28396
|
+
if (drift.label !== "TEST_DRIFT") return external(`${drift.label} names no repair a machine can make: recompiling or re-recording reproduces the case as it stands`);
|
|
28397
|
+
if (result.live) return rewrite.length === 0 ? external("a live case is repaired by rewriting its document, and nothing the audit named is in it") : {
|
|
28398
|
+
route: "rewrite",
|
|
28399
|
+
reason: "a live case runs its document, so rewriting it is the whole repair — apply 'rewrite' to 'document', then run the case to verify.",
|
|
28400
|
+
rewrite
|
|
28467
28401
|
};
|
|
28402
|
+
if (test === null) return external("this case resolves to no generated test: there is nothing to regenerate");
|
|
28468
28403
|
const testAbs = resolve(cwd, test);
|
|
28469
|
-
const
|
|
28404
|
+
const recording = await getRecording(caseRefFor(result.target.caseId ?? {
|
|
28470
28405
|
featureName: result.target.featureName,
|
|
28471
28406
|
specName: result.target.specName
|
|
28472
|
-
}, cwd, await caseRecordingPath(result.target,
|
|
28473
|
-
if (!
|
|
28474
|
-
|
|
28475
|
-
|
|
28476
|
-
|
|
28477
|
-
|
|
28478
|
-
|
|
28479
|
-
|
|
28480
|
-
};
|
|
28407
|
+
}, cwd, await caseRecordingPath(result.target, ctx) ?? void 0)).catch(() => null);
|
|
28408
|
+
if (!recording?.generated) return external("no generation stamp: ccqa did not write this test, or it predates the stamp");
|
|
28409
|
+
if (!await matchesGenerationStamp(recording.generated, testAbs)) return external("the test has been edited since it was generated");
|
|
28410
|
+
const apply = (why) => rewrite.length > 0 ? ` Apply 'rewrite' to 'document' first: ${why}.` : "";
|
|
28411
|
+
if (recordingNamesRenamed(recording, renames)) return {
|
|
28412
|
+
route: "rerecord",
|
|
28413
|
+
reason: `the saved recording names a renamed string, so a regeneration would compile it back in — re-record and verify.${apply("a re-recording drives the case as its document states it")}`,
|
|
28414
|
+
rewrite
|
|
28415
|
+
};
|
|
28416
|
+
if (drift.surface !== "generated" && rewrite.length === 0) return external("the finding is not on the generated test, and nothing it named is in the document — it is on a file the test imports, which a regeneration only reads, or no replacement was named");
|
|
28481
28417
|
return {
|
|
28482
28418
|
route: "regenerate",
|
|
28483
|
-
reason: `test drift
|
|
28419
|
+
reason: `test drift, and the saved recording names no renamed string — regenerate and verify.${apply("regenerating alone recompiles the same document")}`,
|
|
28420
|
+
rewrite
|
|
28484
28421
|
};
|
|
28485
28422
|
}
|
|
28486
28423
|
//#endregion
|
|
@@ -28700,7 +28637,7 @@ function selectSpecsNeedingAudit(targets, report, stillDrifted = /* @__PURE__ */
|
|
|
28700
28637
|
//#endregion
|
|
28701
28638
|
//#region src/cli/audit.ts
|
|
28702
28639
|
const DEFAULT_CONCURRENCY = 3;
|
|
28703
|
-
const auditCommand = addProfileOption(addLanguageOption(new Command("audit").argument("[feature/spec]", "Optional spec id. If omitted, every spec under .ccqa/features/ is checked.").description("Read each spec against the code it describes and report where the two have drifted. Static: no browser is run, so this is the cheap check to put in front of `ccqa run`.").optionsGroup("Which specs to audit:").option("--only-affected-by <ref>", "Only specs `ccqa select-specs` decides the diff against <ref> reaches (e.g. origin/main), by intersecting it with each spec's measured coverage from the hub. In pull_request CI, pass $GITHUB_BASE_REF. Specs without a measurement are audited rather than skipped.").option("--only-hub-audit-needed", "Only specs the hub says a deploy has landed on since the audit last read them. A spec that was never audited is always included, one the hub cannot answer for is audited rather than skipped, and one whose drift entry is still open is always re-audited — a merged fix changes only the spec tree, which no deploy answer covers. No git diff involved. Requires a hub connection and --hub-profile.").optionsGroup("How to run it:").option("--concurrency <n>", `Parallel spec checks (default: ${DEFAULT_CONCURRENCY})`).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").optionsGroup("What to do with the results:").option("--report-format <fmt>", "Output format: text | json | github", "text").option("--report-to-hub", "Push the result to a ccqa hub as a run (kind: drift), which is what updates the drift ledger. A spec it finds drifted answers `needsRepair` to `ccqa run --only-hub-rerun-needed`, and is not run until a person repairs it.").option("--exit-on <level>", "Exit non-zero on this severity or higher: warn | error", "error").option("--brief <dir>", "Also write one JSON file per finding under <dir>, named by case id: the verdict, its citations,
|
|
28640
|
+
const auditCommand = addProfileOption(addLanguageOption(new Command("audit").argument("[feature/spec]", "Optional spec id. If omitted, every spec under .ccqa/features/ is checked.").description("Read each spec against the code it describes and report where the two have drifted. Static: no browser is run, so this is the cheap check to put in front of `ccqa run`.").optionsGroup("Which specs to audit:").option("--only-affected-by <ref>", "Only specs `ccqa select-specs` decides the diff against <ref> reaches (e.g. origin/main), by intersecting it with each spec's measured coverage from the hub. In pull_request CI, pass $GITHUB_BASE_REF. Specs without a measurement are audited rather than skipped.").option("--only-hub-audit-needed", "Only specs the hub says a deploy has landed on since the audit last read them. A spec that was never audited is always included, one the hub cannot answer for is audited rather than skipped, and one whose drift entry is still open is always re-audited — a merged fix changes only the spec tree, which no deploy answer covers. No git diff involved. Requires a hub connection and --hub-profile.").optionsGroup("How to run it:").option("--concurrency <n>", `Parallel spec checks (default: ${DEFAULT_CONCURRENCY})`).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").optionsGroup("What to do with the results:").option("--report-format <fmt>", "Output format: text | json | github", "text").option("--report-to-hub", "Push the result to a ccqa hub as a run (kind: drift), which is what updates the drift ledger. A spec it finds drifted answers `needsRepair` to `ccqa run --only-hub-rerun-needed`, and is not run until a person repairs it.").option("--exit-on <level>", "Exit non-zero on this severity or higher: warn | error", "error").option("--brief <dir>", "Also write one JSON file per finding under <dir>, named by case id: the verdict, its citations, which repair the case needs (regenerate, re-record, rewrite its document, or a person), and any renamed strings to apply to that document first. What reads them is outside ccqa.").option("--dump-inputs <dir>", "Also write what each case's audit was given — its document, every file it was handed and the import that reached it, the source roots, and the prompt — as one markdown file per case under <dir>. For telling 'this never reached the audit' apart from 'it did, and nothing was reported'.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory used as both the .ccqa root and the codebase Claude reads. Useful for monorepos. Defaults to process.cwd().").option("--project <name>", "Logical project name for the pushed run. Defaults to the current directory's name.").option(...hubUrlOption).option(...hubTokenOption).option(...hubHeaderOption))).action(withUsageErrors(async (specPath, opts) => {
|
|
28704
28641
|
await withCostReporting("audit", () => runAudit(specPath, opts));
|
|
28705
28642
|
}));
|
|
28706
28643
|
async function runAudit(specPath, opts) {
|
|
@@ -28738,10 +28675,10 @@ async function runAudit(specPath, opts) {
|
|
|
28738
28675
|
}
|
|
28739
28676
|
const artifactsContext = await loadSpecArtifactsContext(cwd);
|
|
28740
28677
|
const config = artifactsContext.config;
|
|
28741
|
-
const intent = artifactsContext.
|
|
28678
|
+
const intent = artifactsContext.reader.target;
|
|
28742
28679
|
const sourceRoots = await resolveSourceRoots(cwd, config.sourceRoots);
|
|
28743
|
-
let targets = await collectTargets(specPath, cwd,
|
|
28744
|
-
if (targets.length === 0) exitWithNoSpecs(format, "noSpecsFound", `no test cases found under ${intent ? intent.
|
|
28680
|
+
let targets = await collectTargets(specPath, cwd, artifactsContext.reader);
|
|
28681
|
+
if (targets.length === 0) exitWithNoSpecs(format, "noSpecsFound", `no test cases found under ${intent ? intent.module : ".ccqa/features/"}`);
|
|
28745
28682
|
if (format === "text") {
|
|
28746
28683
|
header("audit", specPath ?? `${targets.length} case${targets.length > 1 ? "s" : ""}`);
|
|
28747
28684
|
if (opts.cwd) meta("cwd", cwd);
|
|
@@ -28979,39 +28916,24 @@ function exitWithNoSpecs(format, reason, message) {
|
|
|
28979
28916
|
* The cases this sweep audits.
|
|
28980
28917
|
*
|
|
28981
28918
|
* Which of the two places a project keeps them in is settled by its target,
|
|
28982
|
-
* not guessed at
|
|
28983
|
-
*
|
|
28984
|
-
*
|
|
28985
|
-
* how `ccqa generate` reads the same argument.
|
|
28919
|
+
* not guessed at, and the reader already answers that — so this asks the same
|
|
28920
|
+
* question of both kinds. A case named on the command line is resolved the
|
|
28921
|
+
* same way, which is exactly how `ccqa generate` reads the same argument.
|
|
28986
28922
|
*/
|
|
28987
|
-
async function collectTargets(specPath, cwd,
|
|
28988
|
-
|
|
28923
|
+
async function collectTargets(specPath, cwd, reader) {
|
|
28924
|
+
const asTarget = (id) => reader.target === null ? splitCaseId(id) : {
|
|
28989
28925
|
...splitCaseId(id),
|
|
28990
28926
|
caseId: id
|
|
28991
|
-
}
|
|
28992
|
-
const tree = await listFeatureTree(cwd);
|
|
28927
|
+
};
|
|
28993
28928
|
if (specPath) {
|
|
28994
|
-
const
|
|
28995
|
-
if (
|
|
28996
|
-
error(`
|
|
28929
|
+
const read = await reader.read(specPath);
|
|
28930
|
+
if (read.document === null) {
|
|
28931
|
+
error(read.error ?? `test case not found: ${specPath} (under ${cwd})`);
|
|
28997
28932
|
process.exit(1);
|
|
28998
28933
|
}
|
|
28999
|
-
return [
|
|
29000
|
-
featureName,
|
|
29001
|
-
specName
|
|
29002
|
-
}];
|
|
29003
|
-
}
|
|
29004
|
-
const active = new Set((await listActiveSpecs(cwd)).map(specKey));
|
|
29005
|
-
const out = [];
|
|
29006
|
-
for (const feature of tree) for (const spec of feature.specs) {
|
|
29007
|
-
if (!spec.hasSpecFile) continue;
|
|
29008
|
-
const target = {
|
|
29009
|
-
featureName: feature.featureName,
|
|
29010
|
-
specName: spec.specName
|
|
29011
|
-
};
|
|
29012
|
-
if (active.has(specKey(target))) out.push(target);
|
|
28934
|
+
return [asTarget(read.id)];
|
|
29013
28935
|
}
|
|
29014
|
-
return
|
|
28936
|
+
return (await Promise.all((await reader.list()).map((id) => reader.read(id)))).filter((read) => read.case?.disabled !== true).map((read) => asTarget(read.id));
|
|
29015
28937
|
}
|
|
29016
28938
|
function parseFormat$1(raw) {
|
|
29017
28939
|
const v = raw ?? "text";
|
|
@@ -29124,29 +29046,41 @@ For each test case above, write a 1–2 sentence factual \`summary\` of what it
|
|
|
29124
29046
|
//#endregion
|
|
29125
29047
|
//#region src/spec/spec-changed-at.ts
|
|
29126
29048
|
/**
|
|
29127
|
-
* One `git log` over the
|
|
29128
|
-
*
|
|
29049
|
+
* One `git log` over the cases' own files, walked newest-first. The first time
|
|
29050
|
+
* a case's file appears is that case's last edit.
|
|
29051
|
+
*
|
|
29052
|
+
* Asked about the documents rather than a directory, because where a case is
|
|
29053
|
+
* filed is the project's business: ccqa's own specs live under `.ccqa`, and a
|
|
29054
|
+
* project that writes its own cases keeps them wherever it keeps them.
|
|
29129
29055
|
*
|
|
29130
29056
|
* Best-effort: outside a repository (or with no history) this returns an empty
|
|
29131
29057
|
* map and every caller falls back to what it did before. A missing timestamp
|
|
29132
|
-
* must never make a
|
|
29058
|
+
* must never make a case look fresher than it is.
|
|
29133
29059
|
*/
|
|
29134
|
-
async function
|
|
29060
|
+
async function readCaseChangedAt(cwd, documentPaths) {
|
|
29135
29061
|
const out = /* @__PURE__ */ new Map();
|
|
29062
|
+
const byRelPath = new Map(documentPaths.map((abs) => [toPosix(relative(cwd, abs)), abs]).filter(([rel]) => rel !== "" && !rel.startsWith("../")));
|
|
29063
|
+
const paths = [...byRelPath.keys()];
|
|
29064
|
+
for (let i = 0; i < paths.length; i += PATHSPEC_CHUNK) await readChunk(cwd, paths.slice(i, i + PATHSPEC_CHUNK), byRelPath, out);
|
|
29065
|
+
return out;
|
|
29066
|
+
}
|
|
29067
|
+
const PATHSPEC_CHUNK = 500;
|
|
29068
|
+
async function readChunk(cwd, paths, byRelPath, out) {
|
|
29136
29069
|
let stdout;
|
|
29137
29070
|
try {
|
|
29138
29071
|
({stdout} = await execFileP("git", [
|
|
29139
29072
|
"log",
|
|
29140
29073
|
"--pretty=format:%x00%cI",
|
|
29141
29074
|
"--name-only",
|
|
29075
|
+
"--relative",
|
|
29142
29076
|
"--",
|
|
29143
|
-
|
|
29077
|
+
...paths
|
|
29144
29078
|
], {
|
|
29145
29079
|
cwd,
|
|
29146
29080
|
maxBuffer: 64 * 1024 * 1024
|
|
29147
29081
|
}));
|
|
29148
29082
|
} catch {
|
|
29149
|
-
return
|
|
29083
|
+
return;
|
|
29150
29084
|
}
|
|
29151
29085
|
let when = "";
|
|
29152
29086
|
for (const line of stdout.split("\n")) {
|
|
@@ -29154,19 +29088,12 @@ async function readSpecChangedAt(cwd) {
|
|
|
29154
29088
|
when = line.slice(1).trim();
|
|
29155
29089
|
continue;
|
|
29156
29090
|
}
|
|
29157
|
-
const
|
|
29158
|
-
if (
|
|
29091
|
+
const abs = byRelPath.get(line.trim());
|
|
29092
|
+
if (abs && when && !out.has(abs)) out.set(abs, when);
|
|
29159
29093
|
}
|
|
29160
|
-
return out;
|
|
29161
29094
|
}
|
|
29162
|
-
|
|
29163
|
-
|
|
29164
|
-
* Every file in a case's directory counts: the generated code moving is as
|
|
29165
|
-
* much a change to the test as the spec.yaml moving.
|
|
29166
|
-
*/
|
|
29167
|
-
function specKeyOf(path) {
|
|
29168
|
-
const m = /^\.ccqa\/features\/([^/]+)\/test-cases\/([^/]+)\//.exec(path);
|
|
29169
|
-
return m ? `${m[1]}/${m[2]}` : null;
|
|
29095
|
+
function toPosix(p) {
|
|
29096
|
+
return p.split(/[\\/]/).join("/");
|
|
29170
29097
|
}
|
|
29171
29098
|
//#endregion
|
|
29172
29099
|
//#region src/cli/perspectives.ts
|
|
@@ -29185,7 +29112,7 @@ async function runPerspectivesCheck(opts) {
|
|
|
29185
29112
|
const hub = requireHubOrExit(opts);
|
|
29186
29113
|
const project = resolveProject(opts);
|
|
29187
29114
|
header("perspectives", `check (project: ${project})`);
|
|
29188
|
-
const skeleton = await buildSkeleton(await
|
|
29115
|
+
const skeleton = await buildSkeleton(await openLocalCases());
|
|
29189
29116
|
const localCount = skeleton.reduce((n, f) => n + f.specs.length, 0);
|
|
29190
29117
|
const existingDoc = await hub.getPerspectives(project);
|
|
29191
29118
|
if (existingDoc === null) {
|
|
@@ -29261,15 +29188,16 @@ async function runPerspectives(opts) {
|
|
|
29261
29188
|
const hub = requireHubOrExit(opts);
|
|
29262
29189
|
const project = resolveProject(opts);
|
|
29263
29190
|
header("perspectives", `project: ${project}`);
|
|
29264
|
-
const
|
|
29191
|
+
const local = await openLocalCases();
|
|
29192
|
+
const skeleton = await buildSkeleton(local);
|
|
29265
29193
|
const allSpecs = skeleton.flatMap((f) => f.specs);
|
|
29266
29194
|
if (allSpecs.length === 0) {
|
|
29267
|
-
info(
|
|
29195
|
+
info(`no test cases found under ${local.where} — nothing to inventory.`);
|
|
29268
29196
|
return;
|
|
29269
29197
|
}
|
|
29270
29198
|
const existingDoc = await hub.getPerspectives(project);
|
|
29271
29199
|
const noteMap = extractNotes(existingDoc);
|
|
29272
|
-
const specBodies =
|
|
29200
|
+
const specBodies = loadSpecBodies(skeleton, local);
|
|
29273
29201
|
meta("language", opts.language ?? "auto");
|
|
29274
29202
|
info(`Summarising ${allSpecs.length} test case(s) across ${skeleton.length} feature(s)...`);
|
|
29275
29203
|
const summaries = await requestSummaries(specBodies, opts);
|
|
@@ -29313,40 +29241,50 @@ async function cleanupLegacyLocalFiles() {
|
|
|
29313
29241
|
for (const path of removed) meta("removed legacy file", path);
|
|
29314
29242
|
}
|
|
29315
29243
|
/**
|
|
29316
|
-
* Turn the
|
|
29317
|
-
* transcribed from each
|
|
29318
|
-
* artifacts
|
|
29319
|
-
*
|
|
29320
|
-
*/
|
|
29321
|
-
async function buildSkeleton(
|
|
29322
|
-
const
|
|
29323
|
-
const
|
|
29324
|
-
|
|
29325
|
-
const
|
|
29326
|
-
|
|
29327
|
-
|
|
29328
|
-
|
|
29329
|
-
|
|
29330
|
-
|
|
29331
|
-
|
|
29332
|
-
|
|
29333
|
-
|
|
29334
|
-
|
|
29335
|
-
|
|
29336
|
-
|
|
29337
|
-
|
|
29338
|
-
...meta.disabled ? { disabled: true } : {}
|
|
29339
|
-
};
|
|
29340
|
-
}));
|
|
29341
|
-
return {
|
|
29342
|
-
featureName: feature.featureName,
|
|
29343
|
-
specs: built
|
|
29244
|
+
* Turn the project's cases into the skeleton perspectives features: title
|
|
29245
|
+
* transcribed from each case, status derived mechanically from on-disk
|
|
29246
|
+
* artifacts, grouped by the feature half of the case's id. `summary` is left
|
|
29247
|
+
* empty here; Claude fills it later.
|
|
29248
|
+
*/
|
|
29249
|
+
async function buildSkeleton(local) {
|
|
29250
|
+
const changedAt = await readCaseChangedAt(process.cwd(), local.cases.map((c) => c.document.path));
|
|
29251
|
+
const byFeature = /* @__PURE__ */ new Map();
|
|
29252
|
+
for (const read of local.cases) {
|
|
29253
|
+
const { featureName, specName } = splitCaseId(read.id);
|
|
29254
|
+
const testCase = read.case;
|
|
29255
|
+
if (testCase === null) warn(`${read.id}: ${read.error ?? "could not be read"}`);
|
|
29256
|
+
const steps = testCase ? transcribeSteps(testCase.spec?.steps ?? testCase.steps) : [];
|
|
29257
|
+
const lastEdit = changedAt.get(read.document.path);
|
|
29258
|
+
const built = {
|
|
29259
|
+
specName,
|
|
29260
|
+
title: testCase?.title ?? specName,
|
|
29261
|
+
summary: "",
|
|
29262
|
+
...steps.length > 0 ? { steps } : {},
|
|
29263
|
+
status: await deriveStatus(read.id, testCase, local),
|
|
29264
|
+
...lastEdit ? { changedAt: lastEdit } : {},
|
|
29265
|
+
...testCase?.disabled ? { disabled: true } : {}
|
|
29344
29266
|
};
|
|
29345
|
-
|
|
29346
|
-
|
|
29347
|
-
|
|
29267
|
+
const specs = byFeature.get(featureName);
|
|
29268
|
+
if (specs) specs.push(built);
|
|
29269
|
+
else byFeature.set(featureName, [built]);
|
|
29270
|
+
}
|
|
29271
|
+
return [...byFeature.entries()].map(([featureName, specs]) => ({
|
|
29272
|
+
featureName,
|
|
29273
|
+
specs: [...specs].sort((a, b) => a.specName.localeCompare(b.specName))
|
|
29348
29274
|
})).sort((a, b) => a.featureName.localeCompare(b.featureName));
|
|
29349
29275
|
}
|
|
29276
|
+
async function openLocalCases() {
|
|
29277
|
+
const cwd = process.cwd();
|
|
29278
|
+
const config = await loadProjectConfig(cwd).catch(() => ProjectConfigSchema.parse({}));
|
|
29279
|
+
const reader = openCaseReader(config, cwd);
|
|
29280
|
+
const ids = await reader.list();
|
|
29281
|
+
return {
|
|
29282
|
+
cases: (await Promise.all(ids.map((id) => reader.read(id)))).filter((r) => r.document !== null),
|
|
29283
|
+
config,
|
|
29284
|
+
reader,
|
|
29285
|
+
where: reader.target ? reader.target.module : ".ccqa/features"
|
|
29286
|
+
};
|
|
29287
|
+
}
|
|
29350
29288
|
/**
|
|
29351
29289
|
* `(featureName, specName)` → human note, extracted from the hub's current
|
|
29352
29290
|
* perspectives document. Notes are preserved across regeneration; everything
|
|
@@ -29404,39 +29342,16 @@ function noteKey(featureName, specName) {
|
|
|
29404
29342
|
return `${featureName}/${specName}`;
|
|
29405
29343
|
}
|
|
29406
29344
|
/**
|
|
29407
|
-
*
|
|
29408
|
-
*
|
|
29409
|
-
*
|
|
29410
|
-
|
|
29411
|
-
|
|
29412
|
-
|
|
29413
|
-
|
|
29414
|
-
|
|
29415
|
-
|
|
29416
|
-
|
|
29417
|
-
};
|
|
29418
|
-
if (specYaml === null) return defaults;
|
|
29419
|
-
try {
|
|
29420
|
-
const parsed = parse(specYaml);
|
|
29421
|
-
const title = typeof parsed.title === "string" && parsed.title.length > 0 ? parsed.title : specName;
|
|
29422
|
-
const modeResult = SpecModeSchema.safeParse(parsed.mode);
|
|
29423
|
-
return {
|
|
29424
|
-
title,
|
|
29425
|
-
mode: modeResult.success ? modeResult.data : DEFAULT_SPEC_MODE,
|
|
29426
|
-
steps: transcribeSteps(parsed.steps),
|
|
29427
|
-
disabled: tryParseTestSpec(specYaml)?.disabled === true
|
|
29428
|
-
};
|
|
29429
|
-
} catch {
|
|
29430
|
-
return defaults;
|
|
29431
|
-
}
|
|
29432
|
-
}
|
|
29433
|
-
/**
|
|
29434
|
-
* The spec's procedure, copied verbatim for the inventory: an include step
|
|
29435
|
-
* keeps only the block name (its params are wiring, not procedure), an
|
|
29436
|
-
* action step keeps its instruction/expected text, a judge step its claim.
|
|
29437
|
-
* Anything malformed is
|
|
29438
|
-
* skipped — the inventory never fails over one bad step, matching how the
|
|
29439
|
-
* rest of this sweep treats a broken spec.
|
|
29345
|
+
* The case's procedure, copied verbatim for the inventory: an include step
|
|
29346
|
+
* keeps only the block name (its params are wiring, not procedure), an action
|
|
29347
|
+
* step keeps its instruction/expected text, a judge step its claim.
|
|
29348
|
+
*
|
|
29349
|
+
* Read off the document rather than off the expanded steps, so a `spec.yaml`
|
|
29350
|
+
* case still shows the block it includes rather than that block's contents —
|
|
29351
|
+
* the inventory is a stock-take of what each case says, and inlining a shared
|
|
29352
|
+
* login into forty cases says the same thing forty times.
|
|
29353
|
+
*
|
|
29354
|
+
* Anything malformed is skipped: the inventory never fails over one bad step.
|
|
29440
29355
|
*/
|
|
29441
29356
|
function transcribeSteps(raw) {
|
|
29442
29357
|
if (!Array.isArray(raw)) return [];
|
|
@@ -29454,41 +29369,37 @@ function transcribeSteps(raw) {
|
|
|
29454
29369
|
return steps;
|
|
29455
29370
|
}
|
|
29456
29371
|
/**
|
|
29457
|
-
*
|
|
29458
|
-
*
|
|
29459
|
-
*
|
|
29460
|
-
*
|
|
29372
|
+
* The target that owns this case's generated files.
|
|
29373
|
+
*
|
|
29374
|
+
* A case the project states itself belongs to the target that declared the
|
|
29375
|
+
* source; a `spec.yaml` case names its own. Best-effort: an unresolvable
|
|
29376
|
+
* target falls back to agent-browser, so the inventory never fails over one
|
|
29377
|
+
* bad case.
|
|
29461
29378
|
*/
|
|
29462
|
-
function
|
|
29463
|
-
if (
|
|
29464
|
-
|
|
29465
|
-
if (!spec) return null;
|
|
29379
|
+
function targetOf(testCase, local) {
|
|
29380
|
+
if (local.reader.target) return registryFor(local.config).get(local.reader.target.id) ?? null;
|
|
29381
|
+
if (testCase?.spec == null) return null;
|
|
29466
29382
|
try {
|
|
29467
|
-
return resolveTarget(spec, config);
|
|
29383
|
+
return resolveTarget(testCase.spec, local.config);
|
|
29468
29384
|
} catch {
|
|
29469
29385
|
return null;
|
|
29470
29386
|
}
|
|
29471
29387
|
}
|
|
29472
29388
|
/**
|
|
29473
|
-
* Coverage facts for one
|
|
29474
|
-
* `PerspectiveStatusSchema`).
|
|
29475
|
-
* target (also the fallback for a spec whose target couldn't be resolved) —
|
|
29476
|
-
* every caller must resolve it (via `resolveSpecTarget`) and pass it, so the
|
|
29477
|
-
* write path and the `--check` comparison agree on the target dimension.
|
|
29389
|
+
* Coverage facts for one case, interpreted through its target (see
|
|
29390
|
+
* `PerspectiveStatusSchema`).
|
|
29478
29391
|
*/
|
|
29479
|
-
async function deriveStatus(
|
|
29392
|
+
async function deriveStatus(id, testCase, local) {
|
|
29480
29393
|
const cwd = process.cwd();
|
|
29481
|
-
const
|
|
29482
|
-
featureName,
|
|
29483
|
-
specName
|
|
29484
|
-
};
|
|
29394
|
+
const plugin = targetOf(testCase, local);
|
|
29485
29395
|
const target = plugin ?? agentBrowserTarget;
|
|
29486
|
-
const targetConfig = targetConfigFor(config, target.id);
|
|
29487
|
-
const generated = await exists(
|
|
29488
|
-
const hasRecording = await findRecordingPath(caseRefFor(
|
|
29396
|
+
const targetConfig = local.reader.target?.targetConfig ?? targetConfigFor(local.config, target.id);
|
|
29397
|
+
const generated = await exists(resolve(cwd, resolveCaseTestPath(target, targetConfig, id)));
|
|
29398
|
+
const hasRecording = await findRecordingPath(caseRefFor(id, cwd, resolveCaseRecordingPath(target, targetConfig, id))) !== null;
|
|
29399
|
+
const traced = target.input === "recording" ? hasRecording : true;
|
|
29489
29400
|
return {
|
|
29490
|
-
mode,
|
|
29491
|
-
traced
|
|
29401
|
+
mode: testCase?.mode ?? "deterministic",
|
|
29402
|
+
traced,
|
|
29492
29403
|
generated,
|
|
29493
29404
|
...plugin && plugin.id !== "agent-browser" ? { target: plugin.id } : {}
|
|
29494
29405
|
};
|
|
@@ -29496,15 +29407,18 @@ async function deriveStatus(featureName, specName, mode, plugin, config) {
|
|
|
29496
29407
|
function exists(path) {
|
|
29497
29408
|
return stat(path).then(() => true, () => false);
|
|
29498
29409
|
}
|
|
29499
|
-
|
|
29500
|
-
|
|
29501
|
-
|
|
29502
|
-
|
|
29503
|
-
|
|
29504
|
-
|
|
29505
|
-
|
|
29506
|
-
|
|
29507
|
-
|
|
29410
|
+
/** `splitCaseId` as the two positional arguments `noteKey` takes. */
|
|
29411
|
+
function splitPair(id) {
|
|
29412
|
+
const { featureName, specName } = splitCaseId(id);
|
|
29413
|
+
return [featureName, specName];
|
|
29414
|
+
}
|
|
29415
|
+
function loadSpecBodies(skeleton, local) {
|
|
29416
|
+
const byKey = new Map(local.cases.map((c) => [noteKey(...splitPair(c.id)), c.document?.text ?? ""]));
|
|
29417
|
+
return skeleton.flatMap((feature) => feature.specs.map((spec) => ({
|
|
29418
|
+
featureName: feature.featureName,
|
|
29419
|
+
specName: spec.specName,
|
|
29420
|
+
title: spec.title,
|
|
29421
|
+
specYaml: byKey.get(noteKey(feature.featureName, spec.specName)) ?? ""
|
|
29508
29422
|
})));
|
|
29509
29423
|
}
|
|
29510
29424
|
async function requestSummaries(specs, opts) {
|
|
@@ -29619,7 +29533,7 @@ async function runSelectSpecs(opts) {
|
|
|
29619
29533
|
}
|
|
29620
29534
|
const specs = specsResult.specs;
|
|
29621
29535
|
if (specs.length === 0) {
|
|
29622
|
-
error("
|
|
29536
|
+
error("this project states no test cases");
|
|
29623
29537
|
process.exit(1);
|
|
29624
29538
|
}
|
|
29625
29539
|
if (!changedResult.ok) {
|