ccqa 1.53.0 → 1.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -4
- package/dist/bin/ccqa.mjs +862 -1052
- package/dist/cases/contract.cjs +0 -0
- package/dist/cases/contract.d.cts +122 -0
- package/dist/cases/contract.d.mts +122 -0
- package/dist/cases/contract.mjs +1 -0
- package/dist/hub-client/index.d.cts +2 -2
- package/dist/hub-client/index.d.mts +2 -2
- package/dist/package.json +14 -1
- package/package.json +14 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -118,8 +118,6 @@ const TestSpecSchema = z.object({
|
|
|
118
118
|
message: `\`${key}\` only applies to the agent-browser target — remove it or drop \`target: ${spec.target}\``
|
|
119
119
|
});
|
|
120
120
|
});
|
|
121
|
-
/** Default mode when `mode:` is absent. */
|
|
122
|
-
const DEFAULT_SPEC_MODE = "deterministic";
|
|
123
121
|
/**
|
|
124
122
|
* A block param declaration. `required` defaults to true; only explicit
|
|
125
123
|
* `required: false` makes it optional. `secret: true` flags the value as
|
|
@@ -300,7 +298,7 @@ function evidenceLabels(overrides, language) {
|
|
|
300
298
|
*
|
|
301
299
|
* The template language is deliberately tiny: `{placeholder}` segments filled
|
|
302
300
|
* from a substitution table the caller owns. Today the table is the spec's own
|
|
303
|
-
* coordinates (`{feature}` / `{spec}`);
|
|
301
|
+
* coordinates (`{feature}` / `{spec}`); a case source that names its cases
|
|
304
302
|
* differently adds keys to the table without touching this module.
|
|
305
303
|
*/
|
|
306
304
|
const PLACEHOLDER$1 = /\{([^{}]*)\}/g;
|
|
@@ -404,10 +402,6 @@ function caseValues(caseId) {
|
|
|
404
402
|
function resolveCaseRecordingPath(target, targetConfig, caseId) {
|
|
405
403
|
return expandPathTemplate(recordingPathTemplate(templateFor(target, targetConfig)), caseValues(caseId));
|
|
406
404
|
}
|
|
407
|
-
/** {@link resolveTestPath}, resolved against the project root. */
|
|
408
|
-
function resolveTestPathAbs(target, targetConfig, ref, cwd) {
|
|
409
|
-
return resolve(cwd, resolveTestPath(target, targetConfig, ref));
|
|
410
|
-
}
|
|
411
405
|
//#endregion
|
|
412
406
|
//#region src/config/project-config.ts
|
|
413
407
|
/**
|
|
@@ -452,38 +446,6 @@ const ConventionsSchema = z.object({
|
|
|
452
446
|
operate: z.array(z.string().min(1)).default([])
|
|
453
447
|
}).strict();
|
|
454
448
|
/**
|
|
455
|
-
* Which sections of a markdown test case mean what. The keys are ccqa's
|
|
456
|
-
* vocabulary; the values are the headings the project actually writes, in the
|
|
457
|
-
* project's own language. Every one is optional — a case that states no
|
|
458
|
-
* cleanup simply has none — except `steps`, without which there is nothing to
|
|
459
|
-
* record.
|
|
460
|
-
*
|
|
461
|
-
* A heading no key names is not lost: it reaches the recorder as further
|
|
462
|
-
* information about the case, which is what a precondition usually is.
|
|
463
|
-
*/
|
|
464
|
-
const IntentFieldsSchema = z.object({
|
|
465
|
-
title: z.string().min(1).default("Title"),
|
|
466
|
-
precondition: z.string().min(1).default("Precondition"),
|
|
467
|
-
steps: z.string().min(1).default("Steps"),
|
|
468
|
-
expected: z.string().min(1).default("Expected"),
|
|
469
|
-
cleanup: z.string().min(1).default("Cleanup"),
|
|
470
|
-
priority: z.string().min(1).default("Priority"),
|
|
471
|
-
link: z.string().min(1).default("Link"),
|
|
472
|
-
mode: z.string().min(1).optional(),
|
|
473
|
-
outputPath: z.string().min(1).optional()
|
|
474
|
-
}).strict();
|
|
475
|
-
/**
|
|
476
|
-
* Where a target's test cases are written, when they are not ccqa's own
|
|
477
|
-
* `spec.yaml`. `root` is the directory the cases live under, and a case's id
|
|
478
|
-
* is its path below it without the extension — which is also what `{case}`
|
|
479
|
-
* expands to in `testPath`.
|
|
480
|
-
*/
|
|
481
|
-
const IntentSourceSchema = z.object({
|
|
482
|
-
kind: z.literal("markdown"),
|
|
483
|
-
root: z.string().min(1),
|
|
484
|
-
fields: IntentFieldsSchema.prefault({})
|
|
485
|
-
}).strict();
|
|
486
|
-
/**
|
|
487
449
|
* How a generated test names the unique values it creates. `${CCQA_RUN_ID}` is
|
|
488
450
|
* what the recording holds; a repo with its own helper for this says so here,
|
|
489
451
|
* and the emitter calls that instead of leaving an env read in the test.
|
|
@@ -493,7 +455,8 @@ const RunIdConfigSchema = z.object({
|
|
|
493
455
|
expression: z.string().min(1)
|
|
494
456
|
}).strict();
|
|
495
457
|
/**
|
|
496
|
-
* A tag the test's title ends with, taken from
|
|
458
|
+
* A tag the test's title ends with, taken from one of the case's `fields`.
|
|
459
|
+
* Mechanical on
|
|
497
460
|
* purpose: a title tag is a convention a reviewer greps, and a model that
|
|
498
461
|
* decides it per case gets it wrong in a way nobody notices.
|
|
499
462
|
*/
|
|
@@ -506,7 +469,7 @@ const TitleTagsSchema = z.object({
|
|
|
506
469
|
* Per-target settings.
|
|
507
470
|
*
|
|
508
471
|
* `testPath` is the template deciding where a case's generated test lands
|
|
509
|
-
* (`{feature}` / `{spec}` for `spec.yaml`, `{case}` for
|
|
472
|
+
* (`{feature}` / `{spec}` for `spec.yaml`, `{case}` for a case source);
|
|
510
473
|
* omitted, the target's own default applies. It is a template rather than a
|
|
511
474
|
* directory because every other command has to find that file without asking
|
|
512
475
|
* the generator — see src/targets/test-path.ts.
|
|
@@ -523,7 +486,8 @@ const TitleTagsSchema = z.object({
|
|
|
523
486
|
const TargetConfigSchema = z.object({
|
|
524
487
|
kind: z.literal("external").optional(),
|
|
525
488
|
framework: z.literal("playwright").optional(),
|
|
526
|
-
|
|
489
|
+
cases: z.string().min(1).optional(),
|
|
490
|
+
intent: z.never({ error: "`intent` is gone: ccqa reads its own `spec.yaml` and nothing else. Point `cases` at a module in this repository that reads your format and answers ccqa's case contract (`cases: ./ccqa/cases.mjs`) — see the `cases` section of docs/targets.md" }).optional(),
|
|
527
491
|
testPath: z.string().min(1).superRefine((template, ctx) => {
|
|
528
492
|
const error = validateTestPathTemplate(template);
|
|
529
493
|
if (error) ctx.addIssue({
|
|
@@ -640,7 +604,7 @@ const ProjectConfigSchema = z.object({
|
|
|
640
604
|
if (target.kind !== "external") {
|
|
641
605
|
for (const key of [
|
|
642
606
|
"framework",
|
|
643
|
-
"
|
|
607
|
+
"cases",
|
|
644
608
|
"runId",
|
|
645
609
|
"header",
|
|
646
610
|
"titleTags"
|
|
@@ -649,7 +613,7 @@ const ProjectConfigSchema = z.object({
|
|
|
649
613
|
}
|
|
650
614
|
if (target.framework === void 0) refuse(at("framework"), "a kind: external target must say which framework its tests are written for (framework: playwright)");
|
|
651
615
|
if (target.testPath === void 0) refuse(at("testPath"), "a kind: external target must say where its generated tests go (testPath)");
|
|
652
|
-
if (target.
|
|
616
|
+
if (target.cases !== void 0 && target.testPath !== void 0 && !target.testPath.includes("{case}")) refuse(at("testPath"), "a target reading its own case source addresses cases by their path, so its testPath must use {case} — {spec} is only the last segment, and two cases filed under the same name would collide");
|
|
653
617
|
}
|
|
654
618
|
if (config.targets["agent-browser"]?.testPath !== void 0) ctx.addIssue({
|
|
655
619
|
code: "custom",
|
|
@@ -1547,8 +1511,8 @@ function splitCaseId(id) {
|
|
|
1547
1511
|
specName
|
|
1548
1512
|
};
|
|
1549
1513
|
}
|
|
1550
|
-
/**
|
|
1551
|
-
function
|
|
1514
|
+
/** A case a project's own source names, by the path-shaped id it gives it. */
|
|
1515
|
+
function sourcedCase(id, cwd) {
|
|
1552
1516
|
return {
|
|
1553
1517
|
id,
|
|
1554
1518
|
dir: join(getCcqaDir(cwd), CASES_DIR, ...id.split("/"))
|
|
@@ -1566,7 +1530,7 @@ function intentCase(id, cwd) {
|
|
|
1566
1530
|
* that was not asking about the test.
|
|
1567
1531
|
*/
|
|
1568
1532
|
function caseRefFor(id, cwd, recordingPath) {
|
|
1569
|
-
const base = typeof id === "string" ?
|
|
1533
|
+
const base = typeof id === "string" ? sourcedCase(id, cwd) : specCase(id.featureName, id.specName, cwd);
|
|
1570
1534
|
return recordingPath === void 0 ? base : {
|
|
1571
1535
|
...base,
|
|
1572
1536
|
recordingPathAbs: resolve(cwd, recordingPath)
|
|
@@ -1576,14 +1540,20 @@ async function ensureCcqaDir(cwd) {
|
|
|
1576
1540
|
await mkdir(join(getCcqaDir(cwd), "features"), { recursive: true });
|
|
1577
1541
|
await mkdir(join(getCcqaDir(cwd), "blocks"), { recursive: true });
|
|
1578
1542
|
}
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1543
|
+
/**
|
|
1544
|
+
* Where one case's `spec.yaml` is.
|
|
1545
|
+
*
|
|
1546
|
+
* Reading it is not here: a case document is read through `src/cases/`, so
|
|
1547
|
+
* that no feature can reach one kind of case behind the reader's back. This
|
|
1548
|
+
* says where the file is, which the store owns because the `.ccqa` layout is
|
|
1549
|
+
* the store's.
|
|
1550
|
+
*/
|
|
1551
|
+
function specFilePath(featureName, specName, cwd) {
|
|
1552
|
+
return join(getSpecDir(featureName, specName, cwd), SPEC_FILE);
|
|
1584
1553
|
}
|
|
1585
|
-
|
|
1586
|
-
|
|
1554
|
+
/** For the enumerations below only — see `src/cases/spec-source.ts` for the public read. */
|
|
1555
|
+
async function readSpecYaml(featureName, specName, cwd) {
|
|
1556
|
+
return readFile(specFilePath(featureName, specName, cwd), "utf-8").catch(() => null);
|
|
1587
1557
|
}
|
|
1588
1558
|
async function saveSpecFile(featureName, specName, content, cwd) {
|
|
1589
1559
|
const specDir = getSpecDir(featureName, specName, cwd);
|
|
@@ -1987,7 +1957,7 @@ async function listAllSpecsWithSpecFile(cwd) {
|
|
|
1987
1957
|
async function listActiveSpecs(cwd) {
|
|
1988
1958
|
const all = await listAllSpecsWithSpecFile(cwd);
|
|
1989
1959
|
return (await Promise.all(all.map(async (ref) => {
|
|
1990
|
-
return tryParseTestSpec(await
|
|
1960
|
+
return tryParseTestSpec(await readSpecYaml(ref.featureName, ref.specName, cwd))?.disabled === true ? null : ref;
|
|
1991
1961
|
}))).filter((ref) => ref !== null);
|
|
1992
1962
|
}
|
|
1993
1963
|
async function listAllSpecsFilteredBy(requiredFilename, cwd) {
|
|
@@ -2021,41 +1991,6 @@ async function resolveSpecTargets(target, enumerateAll, cwd) {
|
|
|
2021
1991
|
}
|
|
2022
1992
|
return (await listActiveSpecs(cwd)).filter((ref) => ref.featureName === target);
|
|
2023
1993
|
}
|
|
2024
|
-
/**
|
|
2025
|
-
* Lists every feature/spec dir under .ccqa/features/, regardless of whether
|
|
2026
|
-
* the spec is fully drafted yet. Each spec file is read at most once.
|
|
2027
|
-
*/
|
|
2028
|
-
async function listFeatureTree(cwd) {
|
|
2029
|
-
const featuresDir = join(getCcqaDir(cwd), "features");
|
|
2030
|
-
const featureDirs = await readdir(featuresDir).catch(() => []);
|
|
2031
|
-
return Promise.all(featureDirs.map(async (featureName) => {
|
|
2032
|
-
const testCasesDir = join(featuresDir, featureName, "test-cases");
|
|
2033
|
-
const specDirs = await readdir(testCasesDir).catch(() => []);
|
|
2034
|
-
return {
|
|
2035
|
-
featureName,
|
|
2036
|
-
specs: await Promise.all(specDirs.map(async (specName) => {
|
|
2037
|
-
const specFile = join(testCasesDir, specName, SPEC_FILE);
|
|
2038
|
-
const content = await readFile(specFile, "utf-8").catch(() => null);
|
|
2039
|
-
if (content === null) return {
|
|
2040
|
-
specName,
|
|
2041
|
-
hasSpecFile: false
|
|
2042
|
-
};
|
|
2043
|
-
try {
|
|
2044
|
-
return {
|
|
2045
|
-
specName,
|
|
2046
|
-
hasSpecFile: true,
|
|
2047
|
-
includedBlocks: collectIncludedBlockNames(parseTestSpec(content, specFile))
|
|
2048
|
-
};
|
|
2049
|
-
} catch {
|
|
2050
|
-
return {
|
|
2051
|
-
specName,
|
|
2052
|
-
hasSpecFile: true
|
|
2053
|
-
};
|
|
2054
|
-
}
|
|
2055
|
-
}))
|
|
2056
|
-
};
|
|
2057
|
-
}));
|
|
2058
|
-
}
|
|
2059
1994
|
//#endregion
|
|
2060
1995
|
//#region src/runtime/bundled-config.ts
|
|
2061
1996
|
const CANDIDATES = [
|
|
@@ -5683,7 +5618,7 @@ async function collectSpecArtifacts(args) {
|
|
|
5683
5618
|
};
|
|
5684
5619
|
const dir = specArtifactsDir(args.reportDir, args.feature, args.spec);
|
|
5685
5620
|
const relPrefix = posix.join(ARTIFACTS_SUBDIR, `${args.feature}__${args.spec}`);
|
|
5686
|
-
const relFiles = await walkFiles$1(dir);
|
|
5621
|
+
const relFiles = (await walkFiles$1(dir)).filter((rel) => !args.passed || !isTraceArchive(rel));
|
|
5687
5622
|
relFiles.sort((a, b) => rowRank(a) - rowRank(b) || a.localeCompare(b));
|
|
5688
5623
|
const kept = [];
|
|
5689
5624
|
const dropped = [];
|
|
@@ -5744,200 +5679,7 @@ function formatBytes(n) {
|
|
|
5744
5679
|
return `${(n / (1024 * 1024)).toFixed(1)} MB`;
|
|
5745
5680
|
}
|
|
5746
5681
|
//#endregion
|
|
5747
|
-
//#region src/
|
|
5748
|
-
/**
|
|
5749
|
-
* Split on level-2 headings. Deeper headings belong to the section they sit
|
|
5750
|
-
* in — a case that structures its steps with `###` is still one case.
|
|
5751
|
-
*/
|
|
5752
|
-
function splitSections(source) {
|
|
5753
|
-
const sections = [];
|
|
5754
|
-
let current = null;
|
|
5755
|
-
for (const line of source.split(/\r?\n/)) {
|
|
5756
|
-
const heading = /^##\s+(.*\S)\s*$/.exec(line);
|
|
5757
|
-
if (heading) {
|
|
5758
|
-
if (current) sections.push(current);
|
|
5759
|
-
current = {
|
|
5760
|
-
heading: heading[1],
|
|
5761
|
-
body: ""
|
|
5762
|
-
};
|
|
5763
|
-
continue;
|
|
5764
|
-
}
|
|
5765
|
-
if (current) current.body += `${line}\n`;
|
|
5766
|
-
}
|
|
5767
|
-
if (current) sections.push(current);
|
|
5768
|
-
return sections.map((s) => ({
|
|
5769
|
-
heading: s.heading,
|
|
5770
|
-
body: s.body.trim()
|
|
5771
|
-
}));
|
|
5772
|
-
}
|
|
5773
|
-
/**
|
|
5774
|
-
* `1. text`, `1) text`, `1.text`, at the start of a line.
|
|
5775
|
-
*
|
|
5776
|
-
* The indent is bounded to three spaces because deeper means a nested list —
|
|
5777
|
-
* a sub-item of the step above, not a step. And the literal digits are not the
|
|
5778
|
-
* step's number: markdown lets every item be written `1.` and leaves the
|
|
5779
|
-
* counting to the renderer, so ccqa counts the same way a reader does.
|
|
5780
|
-
*/
|
|
5781
|
-
const NUMBERED = /^ {0,3}(\d+)\s*[.).]\s*(.+)$/;
|
|
5782
|
-
/** `- text`, `* text`, `・text` — the bullet characters people actually use. */
|
|
5783
|
-
const BULLET = /^\s*(?:[-*+]|・)\s*(.+)$/;
|
|
5784
|
-
function numberedItems(body) {
|
|
5785
|
-
const steps = [];
|
|
5786
|
-
for (const line of body.split("\n")) {
|
|
5787
|
-
const m = NUMBERED.exec(line);
|
|
5788
|
-
if (m) steps.push({
|
|
5789
|
-
number: steps.length + 1,
|
|
5790
|
-
text: m[2].trim()
|
|
5791
|
-
});
|
|
5792
|
-
else if (steps.length > 0 && /^\s+\S/.test(line)) {
|
|
5793
|
-
const last = steps[steps.length - 1];
|
|
5794
|
-
last.text = `${last.text} ${line.trim()}`;
|
|
5795
|
-
}
|
|
5796
|
-
}
|
|
5797
|
-
return steps;
|
|
5798
|
-
}
|
|
5799
|
-
/**
|
|
5800
|
-
* A section written as a list, or as a paragraph.
|
|
5801
|
-
*
|
|
5802
|
-
* The list form is the useful one — each item is placed against the step that
|
|
5803
|
-
* makes it true — but a case whose author wrote a sentence has still stated an
|
|
5804
|
-
* expectation, and dropping it because of its punctuation would be ccqa losing
|
|
5805
|
-
* what the case says. So prose is one item, and nothing is lost either way.
|
|
5806
|
-
*/
|
|
5807
|
-
function listOrProse(body) {
|
|
5808
|
-
const items = bulletItems(body);
|
|
5809
|
-
if (items.length > 0) return items;
|
|
5810
|
-
const prose = body.trim();
|
|
5811
|
-
return prose.length > 0 ? [prose] : [];
|
|
5812
|
-
}
|
|
5813
|
-
/** Cleanup, likewise: numbered when numbered, otherwise the paragraph as one step. */
|
|
5814
|
-
function cleanupSteps(body) {
|
|
5815
|
-
const numbered = numberedItems(body);
|
|
5816
|
-
if (numbered.length > 0) return numbered;
|
|
5817
|
-
const prose = body.trim();
|
|
5818
|
-
return prose.length > 0 ? [{
|
|
5819
|
-
number: 1,
|
|
5820
|
-
text: prose
|
|
5821
|
-
}] : [];
|
|
5822
|
-
}
|
|
5823
|
-
/** `- text` at the left margin — not an indented bullet, which continues the step above it. */
|
|
5824
|
-
const TOP_LEVEL_BULLET = /^ {0,3}(?:[-*+]|・)\s*(.+)$/;
|
|
5825
|
-
/**
|
|
5826
|
-
* What a cleanup section says besides its steps. A case that numbers its
|
|
5827
|
-
* teardown and then lists what should be true afterwards is stating two
|
|
5828
|
-
* different things under one heading, and only the numbered half is a step.
|
|
5829
|
-
* An unnumbered section is prose the whole way down and states no separate
|
|
5830
|
-
* expectation.
|
|
5831
|
-
*/
|
|
5832
|
-
function cleanupExpectations(body) {
|
|
5833
|
-
if (numberedItems(body).length === 0) return [];
|
|
5834
|
-
return body.split("\n").map((line) => TOP_LEVEL_BULLET.exec(line)?.[1]?.trim()).filter((text) => text !== void 0 && text.length > 0);
|
|
5835
|
-
}
|
|
5836
|
-
function bulletItems(body) {
|
|
5837
|
-
return body.split("\n").map((line) => BULLET.exec(line)?.[1]?.trim()).filter((text) => text !== void 0 && text.length > 0);
|
|
5838
|
-
}
|
|
5839
|
-
/**
|
|
5840
|
-
* A labelled bullet's value: `URL: https://…` / `URL:https://…`. Both colons,
|
|
5841
|
-
* because a case written in Japanese uses the full-width one and nobody should
|
|
5842
|
-
* have to think about it.
|
|
5843
|
-
*/
|
|
5844
|
-
function labelled(items, label) {
|
|
5845
|
-
for (const item of items) {
|
|
5846
|
-
const m = /^([^::]+)[::]\s*(.*)$/.exec(item);
|
|
5847
|
-
if (m && m[1].trim().toLowerCase() === label.toLowerCase()) {
|
|
5848
|
-
const value = m[2].trim();
|
|
5849
|
-
if (value.length > 0) return value;
|
|
5850
|
-
}
|
|
5851
|
-
}
|
|
5852
|
-
}
|
|
5853
|
-
/**
|
|
5854
|
-
* Read one markdown file as a case. A heading the field map names becomes that
|
|
5855
|
-
* field; everything else is carried through untouched. The only hard
|
|
5856
|
-
* requirement is a numbered list of steps: a case with none describes nothing
|
|
5857
|
-
* to record, and failing here beats a recording of nothing.
|
|
5858
|
-
*/
|
|
5859
|
-
function parseMarkdownCase(input) {
|
|
5860
|
-
const { fields } = input;
|
|
5861
|
-
const sections = splitSections(input.source);
|
|
5862
|
-
const named = new Map(sections.map((s) => [s.heading, s]));
|
|
5863
|
-
const body = (heading) => named.get(heading)?.body ?? "";
|
|
5864
|
-
const steps = numberedItems(body(fields.steps));
|
|
5865
|
-
if (steps.length === 0) throw new Error(`${input.id}: no steps — the "${fields.steps}" section must hold a numbered list of what to do`);
|
|
5866
|
-
const linkItems = bulletItems(body(fields.link));
|
|
5867
|
-
const claimed = new Set(Object.values({
|
|
5868
|
-
...fields,
|
|
5869
|
-
precondition: ""
|
|
5870
|
-
}));
|
|
5871
|
-
claimed.delete("");
|
|
5872
|
-
const title = body(fields.title).trim();
|
|
5873
|
-
return {
|
|
5874
|
-
id: input.id,
|
|
5875
|
-
title: title.length > 0 ? firstLine(title) : input.id,
|
|
5876
|
-
steps,
|
|
5877
|
-
expected: listOrProse(body(fields.expected)),
|
|
5878
|
-
cleanup: cleanupSteps(body(fields.cleanup)),
|
|
5879
|
-
cleanupExpected: cleanupExpectations(body(fields.cleanup)),
|
|
5880
|
-
...body(fields.priority) ? { priority: firstLine(body(fields.priority)) } : {},
|
|
5881
|
-
...fields.mode && body(fields.mode) ? { mode: firstLine(body(fields.mode)) } : {},
|
|
5882
|
-
link: {
|
|
5883
|
-
...labelled(linkItems, "url") ? { url: labelled(linkItems, "url") } : {},
|
|
5884
|
-
...labelled(linkItems, "no") ? { ref: labelled(linkItems, "no") } : {}
|
|
5885
|
-
},
|
|
5886
|
-
other: sections.filter((s) => !claimed.has(s.heading) && s.body.length > 0),
|
|
5887
|
-
sections
|
|
5888
|
-
};
|
|
5889
|
-
}
|
|
5890
|
-
function firstLine(text) {
|
|
5891
|
-
return text.split("\n")[0].trim();
|
|
5892
|
-
}
|
|
5893
|
-
/**
|
|
5894
|
-
* The same file with one section's body replaced, and every other byte left
|
|
5895
|
-
* alone. Used to write a generated test's path back into the case that asked
|
|
5896
|
-
* for it — a case file is the project's, and ccqa rewrites exactly the line it
|
|
5897
|
-
* was given permission to.
|
|
5898
|
-
*/
|
|
5899
|
-
function replaceSectionBody(source, heading, body) {
|
|
5900
|
-
const eol = source.includes("\r\n") ? "\r\n" : "\n";
|
|
5901
|
-
const lines = source.split(/\r?\n/);
|
|
5902
|
-
const start = lines.findIndex((line) => /^##\s+(.*\S)\s*$/.exec(line)?.[1] === heading);
|
|
5903
|
-
if (start === -1) return null;
|
|
5904
|
-
let end = start + 1;
|
|
5905
|
-
while (end < lines.length && !/^##\s+/.test(lines[end])) end++;
|
|
5906
|
-
let tail = end;
|
|
5907
|
-
while (tail > start + 1 && lines[tail - 1].trim() === "") tail--;
|
|
5908
|
-
const rest = lines.slice(tail);
|
|
5909
|
-
return [
|
|
5910
|
-
...lines.slice(0, start + 1),
|
|
5911
|
-
"",
|
|
5912
|
-
body,
|
|
5913
|
-
...rest.length > 0 ? rest : [""]
|
|
5914
|
-
].join(eol);
|
|
5915
|
-
}
|
|
5916
|
-
//#endregion
|
|
5917
|
-
//#region src/intent/case.ts
|
|
5918
|
-
/**
|
|
5919
|
-
* A case's mode, from the heading the project mapped to `intent.fields.mode`.
|
|
5920
|
-
*
|
|
5921
|
-
* Only `live` turns the browser agent on; everything else is recorded and
|
|
5922
|
-
* generated. An unrecognised body is warned about rather than accepted
|
|
5923
|
-
* silently — a project that wrote `manual` there meant something by it, and
|
|
5924
|
-
* reading that as "record and generate" without a word is how a case ends up
|
|
5925
|
-
* automated that nobody meant to automate.
|
|
5926
|
-
*/
|
|
5927
|
-
function resolveMode(raw, id) {
|
|
5928
|
-
const value = raw?.trim().toLowerCase() ?? "";
|
|
5929
|
-
if (value === "live") return "live";
|
|
5930
|
-
if (value !== "" && !DETERMINISTIC_SPELLINGS.has(value)) warn(`${id}: mode "${raw.trim()}" is not a mode ccqa knows; recording and generating it`);
|
|
5931
|
-
return DEFAULT_SPEC_MODE;
|
|
5932
|
-
}
|
|
5933
|
-
/** Bodies that plainly mean "not live", so they pass without a warning. */
|
|
5934
|
-
const DETERMINISTIC_SPELLINGS = new Set([
|
|
5935
|
-
"deterministic",
|
|
5936
|
-
"recorded",
|
|
5937
|
-
"generated",
|
|
5938
|
-
"auto",
|
|
5939
|
-
"automated"
|
|
5940
|
-
]);
|
|
5682
|
+
//#region src/cases/case.ts
|
|
5941
5683
|
/**
|
|
5942
5684
|
* Step ids are the case's numbering, so evidence and diffs can cite them. The
|
|
5943
5685
|
* one spelling: `step-comment.ts` rebuilds an id from a comment's number, and
|
|
@@ -5952,11 +5694,12 @@ function cleanupId(n) {
|
|
|
5952
5694
|
/** ccqa's own spec, expanded — the shape every target has always consumed. */
|
|
5953
5695
|
function caseFromSpec(featureName, specName, yaml, blocks, cwd) {
|
|
5954
5696
|
const spec = parseTestSpec(yaml);
|
|
5697
|
+
const expanded = expandOrName(spec, blocks);
|
|
5955
5698
|
return {
|
|
5956
5699
|
ref: specCase(featureName, specName, cwd),
|
|
5957
5700
|
title: spec.title,
|
|
5958
5701
|
mode: spec.mode ?? "deterministic",
|
|
5959
|
-
steps:
|
|
5702
|
+
steps: expanded.steps,
|
|
5960
5703
|
cleanup: [],
|
|
5961
5704
|
expectations: [],
|
|
5962
5705
|
cleanupExpectations: [],
|
|
@@ -5965,137 +5708,337 @@ function caseFromSpec(featureName, specName, yaml, blocks, cwd) {
|
|
|
5965
5708
|
case: `${featureName}/${specName}`,
|
|
5966
5709
|
title: spec.title
|
|
5967
5710
|
},
|
|
5968
|
-
|
|
5969
|
-
|
|
5970
|
-
|
|
5971
|
-
|
|
5972
|
-
|
|
5711
|
+
disabled: spec.disabled === true,
|
|
5712
|
+
blocked: expanded.blocked,
|
|
5713
|
+
document: {
|
|
5714
|
+
path: specFilePath(featureName, specName, cwd),
|
|
5715
|
+
text: yaml
|
|
5716
|
+
},
|
|
5717
|
+
spec
|
|
5973
5718
|
};
|
|
5974
5719
|
}
|
|
5975
|
-
/**
|
|
5976
|
-
|
|
5977
|
-
|
|
5978
|
-
|
|
5979
|
-
|
|
5980
|
-
|
|
5981
|
-
|
|
5720
|
+
/**
|
|
5721
|
+
* The spec's steps with its blocks inlined — or, when a block cannot be
|
|
5722
|
+
* resolved, each step named as the document writes it and the reason carried
|
|
5723
|
+
* on `blocked`. Naming the block is what a reader of the document sees; it is
|
|
5724
|
+
* not a step anything may act on, which is what `blocked` says.
|
|
5725
|
+
*/
|
|
5726
|
+
function expandOrName(spec, blocks) {
|
|
5727
|
+
try {
|
|
5728
|
+
return {
|
|
5729
|
+
steps: expandSpec(spec, { blocks }),
|
|
5730
|
+
blocked: null
|
|
5731
|
+
};
|
|
5732
|
+
} catch (err) {
|
|
5733
|
+
return {
|
|
5734
|
+
steps: spec.steps.map((step, i) => {
|
|
5735
|
+
const body = isIncludeStep(step) ? {
|
|
5736
|
+
instruction: `include block: ${step.include}`,
|
|
5737
|
+
expected: ""
|
|
5738
|
+
} : step;
|
|
5739
|
+
return {
|
|
5740
|
+
id: stepId(i + 1),
|
|
5741
|
+
source: "case",
|
|
5742
|
+
...body
|
|
5743
|
+
};
|
|
5744
|
+
}),
|
|
5745
|
+
blocked: err instanceof Error ? err.message : String(err)
|
|
5746
|
+
};
|
|
5747
|
+
}
|
|
5748
|
+
}
|
|
5749
|
+
/**
|
|
5750
|
+
* Lift what a source answered (see `./contract.ts`) into the shape every
|
|
5751
|
+
* feature consumes.
|
|
5752
|
+
*/
|
|
5753
|
+
function caseFromDocument(read, cwd) {
|
|
5982
5754
|
return {
|
|
5983
|
-
ref:
|
|
5984
|
-
title:
|
|
5985
|
-
mode:
|
|
5986
|
-
steps:
|
|
5987
|
-
id: stepId(
|
|
5755
|
+
ref: sourcedCase(read.id, cwd),
|
|
5756
|
+
title: read.title,
|
|
5757
|
+
mode: read.mode,
|
|
5758
|
+
steps: read.steps.map((step, i) => ({
|
|
5759
|
+
id: stepId(i + 1),
|
|
5988
5760
|
source: "case",
|
|
5989
|
-
instruction: step.
|
|
5990
|
-
expected: ""
|
|
5761
|
+
instruction: step.instruction,
|
|
5762
|
+
expected: step.expected ?? ""
|
|
5991
5763
|
})),
|
|
5992
|
-
cleanup:
|
|
5993
|
-
id: cleanupId(
|
|
5764
|
+
cleanup: (read.cleanup ?? []).map((step, i) => ({
|
|
5765
|
+
id: cleanupId(i + 1),
|
|
5994
5766
|
source: "cleanup",
|
|
5995
|
-
instruction: step.
|
|
5996
|
-
expected: ""
|
|
5767
|
+
instruction: step.instruction,
|
|
5768
|
+
expected: step.expected ?? ""
|
|
5997
5769
|
})),
|
|
5998
|
-
expectations:
|
|
5999
|
-
cleanupExpectations:
|
|
6000
|
-
context:
|
|
5770
|
+
expectations: [...read.expectations ?? []],
|
|
5771
|
+
cleanupExpectations: [...read.cleanupExpectations ?? []],
|
|
5772
|
+
context: [...read.context ?? []],
|
|
6001
5773
|
fields: {
|
|
6002
|
-
case:
|
|
6003
|
-
title:
|
|
6004
|
-
...
|
|
6005
|
-
...parsed.link.url ? { "link.url": parsed.link.url } : {},
|
|
6006
|
-
...parsed.link.ref ? { "link.ref": parsed.link.ref } : {}
|
|
5774
|
+
case: read.id,
|
|
5775
|
+
title: read.title,
|
|
5776
|
+
...read.fields
|
|
6007
5777
|
},
|
|
6008
|
-
|
|
6009
|
-
|
|
6010
|
-
|
|
6011
|
-
|
|
6012
|
-
|
|
6013
|
-
}
|
|
5778
|
+
disabled: read.disabled === true,
|
|
5779
|
+
blocked: null,
|
|
5780
|
+
document: {
|
|
5781
|
+
path: read.path,
|
|
5782
|
+
text: read.text
|
|
5783
|
+
},
|
|
5784
|
+
spec: null
|
|
6014
5785
|
};
|
|
6015
5786
|
}
|
|
5787
|
+
//#endregion
|
|
5788
|
+
//#region src/cases/case-schema.ts
|
|
6016
5789
|
/**
|
|
6017
|
-
* The
|
|
6018
|
-
*
|
|
6019
|
-
*
|
|
6020
|
-
*
|
|
5790
|
+
* The published contract, as something ccqa can check an answer against.
|
|
5791
|
+
*
|
|
5792
|
+
* Internal on purpose: `ccqa/case-source` exports types only, so a reader
|
|
5793
|
+
* module never has to resolve ccqa's zod. The two are kept in step by the
|
|
5794
|
+
* assertions below rather than by one being derived from the other — the
|
|
5795
|
+
* public shape stays hand-written and readable, and a field added to either
|
|
5796
|
+
* without the other stops the build.
|
|
6021
5797
|
*/
|
|
6022
|
-
|
|
6023
|
-
|
|
6024
|
-
|
|
6025
|
-
|
|
6026
|
-
|
|
6027
|
-
|
|
6028
|
-
|
|
6029
|
-
|
|
6030
|
-
|
|
6031
|
-
|
|
6032
|
-
|
|
6033
|
-
|
|
6034
|
-
|
|
6035
|
-
|
|
5798
|
+
const CaseStepSchema = z.object({
|
|
5799
|
+
instruction: z.string().min(1),
|
|
5800
|
+
expected: z.string().optional()
|
|
5801
|
+
}).strict();
|
|
5802
|
+
const CaseSchema = z.object({
|
|
5803
|
+
id: z.string().min(1).regex(/^[^/\\][^\\]*$/, "a case id is a '/'-separated path with no leading slash and no backslashes").refine((id) => !id.endsWith("/") && !id.split("/").includes(".."), { message: "a case id must not end in '/' or contain '..'" }),
|
|
5804
|
+
path: z.string().min(1),
|
|
5805
|
+
text: z.string(),
|
|
5806
|
+
title: z.string().min(1),
|
|
5807
|
+
mode: z.enum(["deterministic", "live"]),
|
|
5808
|
+
steps: z.array(CaseStepSchema).min(1),
|
|
5809
|
+
cleanup: z.array(CaseStepSchema).optional(),
|
|
5810
|
+
expectations: z.array(z.string().min(1)).optional(),
|
|
5811
|
+
cleanupExpectations: z.array(z.string().min(1)).optional(),
|
|
5812
|
+
context: z.array(z.object({
|
|
5813
|
+
heading: z.string().min(1),
|
|
5814
|
+
body: z.string()
|
|
5815
|
+
}).strict()).optional(),
|
|
5816
|
+
fields: z.record(z.string().min(1), z.string()).optional(),
|
|
5817
|
+
disabled: z.boolean().optional()
|
|
5818
|
+
}).strict();
|
|
5819
|
+
//#endregion
|
|
5820
|
+
//#region src/cases/module-source.ts
|
|
6036
5821
|
/**
|
|
6037
|
-
*
|
|
5822
|
+
* The project's own case reader, loaded from the module `targets.<id>.cases`
|
|
5823
|
+
* names.
|
|
6038
5824
|
*
|
|
6039
|
-
*
|
|
6040
|
-
*
|
|
6041
|
-
*
|
|
6042
|
-
*
|
|
6043
|
-
* A `.md` that does not parse as a case is not one: the directory holds a
|
|
6044
|
-
* project's own documents, and a README or a template sitting beside the
|
|
6045
|
-
* cases must not be swept into a run or an audit. Naming such a file on the
|
|
6046
|
-
* command line still fails loudly — this only decides what a sweep picks up.
|
|
5825
|
+
* ccqa imports it and validates everything it returns. Nothing about the
|
|
5826
|
+
* project's format reaches ccqa — not a heading, not a bullet character, not a
|
|
5827
|
+
* punctuation mark — which is the point: a format-specific accommodation has
|
|
5828
|
+
* nowhere here to land.
|
|
6047
5829
|
*/
|
|
6048
|
-
|
|
6049
|
-
|
|
6050
|
-
|
|
6051
|
-
|
|
6052
|
-
|
|
6053
|
-
|
|
6054
|
-
|
|
6055
|
-
|
|
6056
|
-
|
|
6057
|
-
|
|
6058
|
-
|
|
6059
|
-
|
|
5830
|
+
/** Extensions Node can import on its own, on every version ccqa supports. */
|
|
5831
|
+
const LOADABLE = [
|
|
5832
|
+
".mjs",
|
|
5833
|
+
".js",
|
|
5834
|
+
".cjs"
|
|
5835
|
+
];
|
|
5836
|
+
function moduleCaseSource(modulePath, cwd, targetId) {
|
|
5837
|
+
const at = `targets.${targetId}.cases (${modulePath})`;
|
|
5838
|
+
let opened = null;
|
|
5839
|
+
const source = () => opened ??= open$1(modulePath, at, cwd);
|
|
5840
|
+
return {
|
|
5841
|
+
async list() {
|
|
5842
|
+
const listed = await (await source()).list();
|
|
5843
|
+
if (!Array.isArray(listed) || listed.some((id) => typeof id !== "string")) throw new RunUsageError(`${at}: list() must return an array of case ids`);
|
|
5844
|
+
return [...new Set(listed)].sort();
|
|
5845
|
+
},
|
|
5846
|
+
idFor: (ref) => ref,
|
|
5847
|
+
async read(ref) {
|
|
5848
|
+
const absent = (error) => ({
|
|
5849
|
+
id: ref,
|
|
5850
|
+
case: null,
|
|
5851
|
+
document: null,
|
|
5852
|
+
error
|
|
6060
5853
|
});
|
|
5854
|
+
let loaded;
|
|
5855
|
+
try {
|
|
5856
|
+
loaded = CaseSchema.parse(await (await source()).load(ref));
|
|
5857
|
+
} catch (err) {
|
|
5858
|
+
if (err instanceof RunUsageError) throw err;
|
|
5859
|
+
if (err instanceof ZodError) return absent(`${at}: load(${JSON.stringify(ref)}) returned ${describe(err)}`);
|
|
5860
|
+
return absent(errMessage(err));
|
|
5861
|
+
}
|
|
5862
|
+
return {
|
|
5863
|
+
id: loaded.id,
|
|
5864
|
+
case: caseFromDocument(loaded, cwd),
|
|
5865
|
+
document: {
|
|
5866
|
+
path: loaded.path,
|
|
5867
|
+
text: loaded.text
|
|
5868
|
+
},
|
|
5869
|
+
error: null
|
|
5870
|
+
};
|
|
6061
5871
|
}
|
|
6062
5872
|
};
|
|
6063
|
-
|
|
6064
|
-
|
|
6065
|
-
|
|
6066
|
-
|
|
6067
|
-
|
|
6068
|
-
|
|
6069
|
-
|
|
6070
|
-
|
|
6071
|
-
|
|
6072
|
-
|
|
6073
|
-
|
|
6074
|
-
|
|
6075
|
-
|
|
6076
|
-
|
|
6077
|
-
|
|
5873
|
+
}
|
|
5874
|
+
async function open$1(modulePath, at, cwd) {
|
|
5875
|
+
if (!LOADABLE.some((ext) => modulePath.endsWith(ext))) throw new RunUsageError(`${at}: cannot be loaded — ${notJavaScript()}`);
|
|
5876
|
+
let mod;
|
|
5877
|
+
try {
|
|
5878
|
+
mod = await import(pathToFileURL(modulePath).href);
|
|
5879
|
+
} catch (err) {
|
|
5880
|
+
const hint = err?.code === "ERR_UNKNOWN_FILE_EXTENSION" ? ` ${notJavaScript()}` : "";
|
|
5881
|
+
throw new RunUsageError(`${at}: could not be loaded — ${errMessage(err)}${hint}`);
|
|
5882
|
+
}
|
|
5883
|
+
if (typeof mod.default !== "function") throw new RunUsageError(`${at}: must default-export a function taking { cwd } and returning { list, load } — see https://github.com/shibukazu/ccqa/blob/main/docs/targets.md`);
|
|
5884
|
+
const opened = await mod.default({ cwd });
|
|
5885
|
+
if (typeof opened !== "object" || opened === null || typeof opened.list !== "function" || typeof opened.load !== "function") throw new RunUsageError(`${at}: the default export must return an object with list() and load()`);
|
|
5886
|
+
return opened;
|
|
5887
|
+
}
|
|
5888
|
+
/**
|
|
5889
|
+
* The mistake that is actually likely, in the terms it is made in. Consumers
|
|
5890
|
+
* write TypeScript, and an extension error alone reads as a broken install.
|
|
5891
|
+
*/
|
|
5892
|
+
function notJavaScript() {
|
|
5893
|
+
return `a case source is plain JavaScript (${LOADABLE.join(", ")}): ccqa imports it with no loader, so a TypeScript file cannot be read. Write it as .mjs — \`// @ts-check\` plus \`@type {import("ccqa/case-source").CaseSourceFactory}\` type-checks it without a build step — or point \`cases\` at compiled output.`;
|
|
5894
|
+
}
|
|
5895
|
+
/** A zod rejection as one line per offending field. */
|
|
5896
|
+
function describe(err) {
|
|
5897
|
+
return `a value that is not a case — ${err.issues.map((issue) => {
|
|
5898
|
+
return `${issue.path.join(".") || "(the case)"}: ${issue.message}`;
|
|
5899
|
+
}).join("; ")}`;
|
|
5900
|
+
}
|
|
5901
|
+
//#endregion
|
|
5902
|
+
//#region src/cases/spec-source.ts
|
|
5903
|
+
/**
|
|
5904
|
+
* ccqa's own `spec.yaml` as a case source.
|
|
5905
|
+
*
|
|
5906
|
+
* The functions that read a spec file live here rather than in the store, so
|
|
5907
|
+
* that a feature wanting a case document has no import that reaches one kind
|
|
5908
|
+
* of case directly. The store still says where the file is (`specFilePath`)
|
|
5909
|
+
* and still enumerates the tree — the layout is its business; reading a case
|
|
5910
|
+
* is not.
|
|
5911
|
+
*/
|
|
5912
|
+
/** One case's `spec.yaml`, null when there is no such file. */
|
|
5913
|
+
async function tryReadSpecFile(featureName, specName, cwd) {
|
|
5914
|
+
return readFile(specFilePath(featureName, specName, cwd), "utf-8").catch(() => null);
|
|
6078
5915
|
}
|
|
6079
5916
|
/**
|
|
6080
|
-
*
|
|
5917
|
+
* Lists every feature/spec dir under .ccqa/features/, regardless of whether
|
|
5918
|
+
* the spec is fully drafted yet. Each spec file is read at most once.
|
|
6081
5919
|
*
|
|
6082
|
-
*
|
|
6083
|
-
*
|
|
5920
|
+
* Only `ccqa draft` still asks for this: it authors ccqa's own format, so it
|
|
5921
|
+
* is the one command whose subject is the tree rather than the cases in it.
|
|
6084
5922
|
*/
|
|
6085
|
-
async function
|
|
6086
|
-
|
|
6087
|
-
|
|
5923
|
+
async function listFeatureTree(cwd) {
|
|
5924
|
+
const featuresDir = join(getCcqaDir(cwd), "features");
|
|
5925
|
+
const featureDirs = await readdir(featuresDir).catch(() => []);
|
|
5926
|
+
return Promise.all(featureDirs.map(async (featureName) => {
|
|
5927
|
+
const testCasesDir = join(featuresDir, featureName, "test-cases");
|
|
5928
|
+
const specDirs = await readdir(testCasesDir).catch(() => []);
|
|
5929
|
+
return {
|
|
5930
|
+
featureName,
|
|
5931
|
+
specs: await Promise.all(specDirs.map(async (specName) => {
|
|
5932
|
+
const specFile = join(testCasesDir, specName, "spec.yaml");
|
|
5933
|
+
const content = await readFile(specFile, "utf-8").catch(() => null);
|
|
5934
|
+
if (content === null) return {
|
|
5935
|
+
specName,
|
|
5936
|
+
hasSpecFile: false
|
|
5937
|
+
};
|
|
5938
|
+
try {
|
|
5939
|
+
return {
|
|
5940
|
+
specName,
|
|
5941
|
+
hasSpecFile: true,
|
|
5942
|
+
includedBlocks: collectIncludedBlockNames(parseTestSpec(content, specFile))
|
|
5943
|
+
};
|
|
5944
|
+
} catch {
|
|
5945
|
+
return {
|
|
5946
|
+
specName,
|
|
5947
|
+
hasSpecFile: true
|
|
5948
|
+
};
|
|
5949
|
+
}
|
|
5950
|
+
}))
|
|
5951
|
+
};
|
|
5952
|
+
}));
|
|
5953
|
+
}
|
|
5954
|
+
function specCaseSource(cwd) {
|
|
5955
|
+
let blocks = null;
|
|
5956
|
+
return {
|
|
5957
|
+
async list() {
|
|
5958
|
+
return (await listAllSpecsWithSpecFile(cwd)).map(specKey).sort();
|
|
5959
|
+
},
|
|
5960
|
+
idFor: (ref) => specKey(parseSpecPath(ref)),
|
|
5961
|
+
async read(id) {
|
|
5962
|
+
const { featureName, specName } = parseSpecPath(id);
|
|
5963
|
+
const path = specFilePath(featureName, specName, cwd);
|
|
5964
|
+
const yaml = await tryReadSpecFile(featureName, specName, cwd);
|
|
5965
|
+
if (yaml === null) return {
|
|
5966
|
+
id,
|
|
5967
|
+
case: null,
|
|
5968
|
+
document: null,
|
|
5969
|
+
error: `Spec file not found: ${path}`
|
|
5970
|
+
};
|
|
5971
|
+
const document = {
|
|
5972
|
+
path,
|
|
5973
|
+
text: yaml
|
|
5974
|
+
};
|
|
5975
|
+
blocks ??= loadAllBlocks(cwd);
|
|
5976
|
+
try {
|
|
5977
|
+
return {
|
|
5978
|
+
id,
|
|
5979
|
+
case: caseFromSpec(featureName, specName, yaml, await blocks, cwd),
|
|
5980
|
+
document,
|
|
5981
|
+
error: null
|
|
5982
|
+
};
|
|
5983
|
+
} catch (err) {
|
|
5984
|
+
return {
|
|
5985
|
+
id,
|
|
5986
|
+
case: null,
|
|
5987
|
+
document,
|
|
5988
|
+
error: errMessage(err)
|
|
5989
|
+
};
|
|
5990
|
+
}
|
|
5991
|
+
}
|
|
5992
|
+
};
|
|
6088
5993
|
}
|
|
5994
|
+
//#endregion
|
|
5995
|
+
//#region src/cases/reader.ts
|
|
6089
5996
|
/**
|
|
6090
|
-
*
|
|
6091
|
-
*
|
|
6092
|
-
*
|
|
6093
|
-
*
|
|
5997
|
+
* The target whose cases live in the project, when that is what we resolve to.
|
|
5998
|
+
*
|
|
5999
|
+
* Which of the two places a project keeps its cases in is settled by its
|
|
6000
|
+
* target, not by guessing at an argument's shape: a target that declares a
|
|
6001
|
+
* case source of its own reads the project's documents, and one that does not
|
|
6002
|
+
* reads ccqa's `spec.yaml`.
|
|
6094
6003
|
*/
|
|
6095
|
-
function
|
|
6096
|
-
const
|
|
6097
|
-
const
|
|
6098
|
-
|
|
6004
|
+
function caseTargetFor(config, cwd, targetOverride) {
|
|
6005
|
+
const id = targetOverride ?? config.defaultTarget;
|
|
6006
|
+
const targetConfig = config.targets[id];
|
|
6007
|
+
const module = targetConfig?.cases;
|
|
6008
|
+
if (targetConfig === void 0 || module === void 0) return null;
|
|
6009
|
+
return {
|
|
6010
|
+
id,
|
|
6011
|
+
targetConfig,
|
|
6012
|
+
module,
|
|
6013
|
+
modulePath: isAbsolute(module) ? module : resolve(cwd, module)
|
|
6014
|
+
};
|
|
6015
|
+
}
|
|
6016
|
+
function openCaseReader(config, cwd, opts = {}) {
|
|
6017
|
+
const target = caseTargetFor(config, cwd, opts.targetOverride);
|
|
6018
|
+
const source = target ? moduleCaseSource(target.modulePath, cwd, target.id) : specCaseSource(cwd);
|
|
6019
|
+
const byId = /* @__PURE__ */ new Map();
|
|
6020
|
+
const read = (ref) => {
|
|
6021
|
+
const key = source.idFor(ref);
|
|
6022
|
+
const cached = byId.get(key);
|
|
6023
|
+
if (cached !== void 0) return cached;
|
|
6024
|
+
const pending = source.read(key);
|
|
6025
|
+
byId.set(key, pending);
|
|
6026
|
+
pending.then((result) => {
|
|
6027
|
+
if (!byId.has(result.id)) byId.set(result.id, pending);
|
|
6028
|
+
}, () => {});
|
|
6029
|
+
return pending;
|
|
6030
|
+
};
|
|
6031
|
+
return {
|
|
6032
|
+
list: () => source.list(),
|
|
6033
|
+
read,
|
|
6034
|
+
async load(ref) {
|
|
6035
|
+
const result = await read(ref);
|
|
6036
|
+
if (result.case === null) throw new RunUsageError(result.error ?? `no test case named "${ref}"`);
|
|
6037
|
+
if (result.case.blocked !== null) throw new RunUsageError(result.case.blocked);
|
|
6038
|
+
return result.case;
|
|
6039
|
+
},
|
|
6040
|
+
target
|
|
6041
|
+
};
|
|
6099
6042
|
}
|
|
6100
6043
|
//#endregion
|
|
6101
6044
|
//#region src/targets/agent-browser/judge-steps.ts
|
|
@@ -13833,13 +13776,14 @@ async function loadEvidenceForSpec(evidenceDir, reportDir, descriptionByStepId)
|
|
|
13833
13776
|
const reportRoot = resolve(reportDir);
|
|
13834
13777
|
const jsonFiles = entries.filter((n) => n.endsWith(".json"));
|
|
13835
13778
|
const metas = (await Promise.all(jsonFiles.map((name) => readEvidenceMeta(join(evidenceDir, name), evidenceDir, reportRoot, descriptionByStepId)))).filter((m) => m !== null);
|
|
13836
|
-
metas.sort((a, b) =>
|
|
13837
|
-
if (a.stepId === "failure") return 1;
|
|
13838
|
-
if (b.stepId === "failure") return -1;
|
|
13839
|
-
return a.stepId.localeCompare(b.stepId);
|
|
13840
|
-
});
|
|
13779
|
+
metas.sort((a, b) => phase(a.stepId) - phase(b.stepId) || a.stepId.localeCompare(b.stepId));
|
|
13841
13780
|
return metas.length > 0 ? metas : null;
|
|
13842
13781
|
}
|
|
13782
|
+
/** Steps, then cleanup, then the failure capture. */
|
|
13783
|
+
function phase(stepId) {
|
|
13784
|
+
if (stepId === "failure") return 2;
|
|
13785
|
+
return stepId.startsWith("cleanup-") ? 1 : 0;
|
|
13786
|
+
}
|
|
13843
13787
|
async function readEvidenceMeta(metaPath, evidenceDir, reportRoot, descriptionByStepId) {
|
|
13844
13788
|
let raw;
|
|
13845
13789
|
try {
|
|
@@ -13856,7 +13800,7 @@ async function readEvidenceMeta(metaPath, evidenceDir, reportRoot, descriptionBy
|
|
|
13856
13800
|
if (typeof parsed !== "object" || parsed === null) return null;
|
|
13857
13801
|
const pngFile = parsed.pngFile;
|
|
13858
13802
|
if (typeof pngFile !== "string") return null;
|
|
13859
|
-
const relToReport = (file) => posix.relative(toPosix(reportRoot), toPosix(join(evidenceDir, file)));
|
|
13803
|
+
const relToReport = (file) => posix.relative(toPosix$1(reportRoot), toPosix$1(join(evidenceDir, file)));
|
|
13860
13804
|
const pngPath = relToReport(pngFile);
|
|
13861
13805
|
const beforePngFile = parsed.beforePngFile;
|
|
13862
13806
|
const beforePngPath = typeof beforePngFile === "string" ? relToReport(beforePngFile) : null;
|
|
@@ -13878,20 +13822,27 @@ async function readEvidenceMeta(metaPath, evidenceDir, reportRoot, descriptionBy
|
|
|
13878
13822
|
return result.success ? result.data : null;
|
|
13879
13823
|
}
|
|
13880
13824
|
/**
|
|
13881
|
-
* Build `step id →
|
|
13882
|
-
*
|
|
13883
|
-
*
|
|
13825
|
+
* Build `step id → caption` so the report can label each evidence screenshot.
|
|
13826
|
+
* Empty when the case could not be read — evidence still surfaces, just
|
|
13827
|
+
* without captions.
|
|
13828
|
+
*
|
|
13829
|
+
* What the step must make true, falling back to what it does. A case that
|
|
13830
|
+
* states its expectations once for the whole flow leaves every step's own
|
|
13831
|
+
* `expected` empty, and a filmstrip captioned with blanks says less than the
|
|
13832
|
+
* instructions the case's author wrote.
|
|
13833
|
+
*
|
|
13834
|
+
* Cleanup steps are captioned too: they have step ids of their own and leave
|
|
13835
|
+
* screenshots of their own, and an uncaptioned row reads as a step nobody
|
|
13836
|
+
* described rather than as the undo it is.
|
|
13884
13837
|
*/
|
|
13885
|
-
function
|
|
13886
|
-
if (!
|
|
13887
|
-
|
|
13888
|
-
|
|
13889
|
-
return
|
|
13890
|
-
}
|
|
13891
|
-
return /* @__PURE__ */ new Map();
|
|
13892
|
-
}
|
|
13838
|
+
function stepCaptions(testCase) {
|
|
13839
|
+
if (!testCase) return /* @__PURE__ */ new Map();
|
|
13840
|
+
return new Map([...testCase.steps, ...testCase.cleanup].map((s) => {
|
|
13841
|
+
if (isExpandedJudgeByLlmStep(s)) return [s.id, s.judgeByLlm.trim()];
|
|
13842
|
+
return [s.id, (s.expected.trim() || s.instruction).trim()];
|
|
13843
|
+
}));
|
|
13893
13844
|
}
|
|
13894
|
-
function toPosix(p) {
|
|
13845
|
+
function toPosix$1(p) {
|
|
13895
13846
|
return p.split(/[\\/]/).join("/");
|
|
13896
13847
|
}
|
|
13897
13848
|
//#endregion
|
|
@@ -13990,14 +13941,14 @@ function parseTraceEvents(jsonl) {
|
|
|
13990
13941
|
});
|
|
13991
13942
|
} else if (type === "action") {
|
|
13992
13943
|
if (typeof event.callId === "string" && typeof event.startTime === "number" && typeof event.endTime === "number") steps.push({
|
|
13993
|
-
title: titleOf(event),
|
|
13944
|
+
title: titleOf$1(event),
|
|
13994
13945
|
startTime: event.startTime,
|
|
13995
13946
|
endTime: event.endTime,
|
|
13996
13947
|
...errorOf(event)
|
|
13997
13948
|
});
|
|
13998
13949
|
} else if (type === "before") {
|
|
13999
13950
|
if (typeof event.callId === "string" && typeof event.startTime === "number") pendingByCallId.set(event.callId, {
|
|
14000
|
-
title: titleOf(event),
|
|
13951
|
+
title: titleOf$1(event),
|
|
14001
13952
|
startTime: event.startTime
|
|
14002
13953
|
});
|
|
14003
13954
|
} else if (type === "after") {
|
|
@@ -14035,7 +13986,7 @@ function errorOf(event) {
|
|
|
14035
13986
|
}
|
|
14036
13987
|
return {};
|
|
14037
13988
|
}
|
|
14038
|
-
function titleOf(event) {
|
|
13989
|
+
function titleOf$1(event) {
|
|
14039
13990
|
if (typeof event.apiName === "string") return event.apiName;
|
|
14040
13991
|
if (typeof event.title === "string") return event.title;
|
|
14041
13992
|
return "";
|
|
@@ -14208,12 +14159,11 @@ function substituteRunCommandFiles(runCommand, testFiles) {
|
|
|
14208
14159
|
*/
|
|
14209
14160
|
const runCommandRunner = { async run(specs, opts) {
|
|
14210
14161
|
const concurrency = Math.max(1, opts.concurrency);
|
|
14211
|
-
const blocks = opts.stepEvidence.supported ? await loadAllBlocks(opts.cwd) : /* @__PURE__ */ new Map();
|
|
14212
14162
|
return runPool(specs, concurrency, async (spec) => {
|
|
14213
14163
|
const key = `${spec.featureName}/${spec.specName}`;
|
|
14214
14164
|
let row;
|
|
14215
14165
|
try {
|
|
14216
|
-
row = await withBuffer(key, concurrency > 1, () => runOneSpec(spec, opts
|
|
14166
|
+
row = await withBuffer(key, concurrency > 1, () => runOneSpec(spec, opts));
|
|
14217
14167
|
} catch (err) {
|
|
14218
14168
|
const message = err instanceof Error ? err.message : String(err);
|
|
14219
14169
|
error(`${key}: runner error: ${message}`);
|
|
@@ -14237,11 +14187,10 @@ const runCommandRunner = { async run(specs, opts) {
|
|
|
14237
14187
|
return row;
|
|
14238
14188
|
}, { resources: opts.resources });
|
|
14239
14189
|
} };
|
|
14240
|
-
async function runOneSpec(ref, opts
|
|
14241
|
-
const { featureName, specName } = ref;
|
|
14242
|
-
const specYaml =
|
|
14243
|
-
const
|
|
14244
|
-
const title = parsedSpec?.title ?? null;
|
|
14190
|
+
async function runOneSpec(ref, opts) {
|
|
14191
|
+
const { featureName, specName, testCase } = ref;
|
|
14192
|
+
const specYaml = testCase?.document.text ?? null;
|
|
14193
|
+
const title = testCase?.title ?? null;
|
|
14245
14194
|
const failedRow = (detail) => ({
|
|
14246
14195
|
...emptySpecRow({
|
|
14247
14196
|
feature: featureName,
|
|
@@ -14267,9 +14216,9 @@ async function runOneSpec(ref, opts, blocks) {
|
|
|
14267
14216
|
const runCommand = opts.targetConfig.runCommand;
|
|
14268
14217
|
if (runCommand === void 0) return didNotExecute(`target "${opts.targetId}" has no runCommand configured in .ccqa/config.yaml`, "the target has no runCommand");
|
|
14269
14218
|
run(`${featureName}/${specName}`);
|
|
14270
|
-
const testFile =
|
|
14219
|
+
const testFile = resolveCaseTestPath(opts, opts.targetConfig, ref.caseId);
|
|
14271
14220
|
if (!await stat(resolve(opts.cwd, testFile)).then(() => true, () => false)) {
|
|
14272
|
-
const detail = `no generated test at ${testFile} — run 'ccqa generate ${
|
|
14221
|
+
const detail = `no generated test at ${testFile} — run 'ccqa generate ${ref.caseId}' first`;
|
|
14273
14222
|
error(detail);
|
|
14274
14223
|
return didNotExecute(detail, "no generated tests");
|
|
14275
14224
|
}
|
|
@@ -14368,6 +14317,7 @@ async function runOneSpec(ref, opts, blocks) {
|
|
|
14368
14317
|
reportDir: opts.reportDir,
|
|
14369
14318
|
feature: featureName,
|
|
14370
14319
|
spec: specName,
|
|
14320
|
+
passed: outcome.exitCode === 0,
|
|
14371
14321
|
warn
|
|
14372
14322
|
});
|
|
14373
14323
|
} catch (err) {
|
|
@@ -14378,8 +14328,7 @@ async function runOneSpec(ref, opts, blocks) {
|
|
|
14378
14328
|
opts,
|
|
14379
14329
|
evidence,
|
|
14380
14330
|
artifactsDir,
|
|
14381
|
-
|
|
14382
|
-
blocks,
|
|
14331
|
+
testCase,
|
|
14383
14332
|
...traceUnavailable !== void 0 ? { traceUnavailable } : {}
|
|
14384
14333
|
});
|
|
14385
14334
|
if (outcome.exitCode === 0) return {
|
|
@@ -14436,7 +14385,7 @@ async function loadStepEvidence(args) {
|
|
|
14436
14385
|
artifactsDir: args.artifactsDir,
|
|
14437
14386
|
evidenceDir
|
|
14438
14387
|
});
|
|
14439
|
-
const descriptions =
|
|
14388
|
+
const descriptions = stepCaptions(args.testCase);
|
|
14440
14389
|
const evidence = await loadEvidenceForSpec(evidenceDir, opts.reportDir, descriptions);
|
|
14441
14390
|
if (evidence) return { evidence };
|
|
14442
14391
|
return {
|
|
@@ -16374,90 +16323,39 @@ function resolveTargetFrom(spec, config, registry) {
|
|
|
16374
16323
|
return plugin;
|
|
16375
16324
|
}
|
|
16376
16325
|
//#endregion
|
|
16377
|
-
//#region src/
|
|
16378
|
-
|
|
16379
|
-
|
|
16380
|
-
|
|
16381
|
-
const { id, targetConfig, intent } = intentTarget;
|
|
16382
|
-
const target = registryFor(config).get(id);
|
|
16383
|
-
const testCase = await loadMarkdownCase(argument, intent, cwd);
|
|
16384
|
-
const testPath = resolveCaseTestPath(target, targetConfig, testCase.ref.id);
|
|
16385
|
-
const recordingPath = (opts.targetOverride === void 0 ? void 0 : ownIntentRecordingPath(config, intent, testCase.ref.id)) ?? resolveCaseRecordingPath(target, targetConfig, testCase.ref.id);
|
|
16386
|
-
const ref = caseRefFor(testCase.ref.id, cwd, recordingPath);
|
|
16387
|
-
return {
|
|
16388
|
-
testCase: {
|
|
16389
|
-
...testCase,
|
|
16390
|
-
ref
|
|
16391
|
-
},
|
|
16392
|
-
target,
|
|
16393
|
-
targetConfig,
|
|
16394
|
-
testPath
|
|
16395
|
-
};
|
|
16396
|
-
}
|
|
16397
|
-
const { featureName, specName } = parseSpecPath(argument);
|
|
16398
|
-
const testCase = caseFromSpec(featureName, specName, await readSpecFile(featureName, specName, cwd), await loadAllBlocks(cwd), cwd);
|
|
16399
|
-
const spec = testCase.source.kind === "spec" ? testCase.source.spec : null;
|
|
16400
|
-
const target = opts.targetOverride !== void 0 ? resolveTargetOverride(spec, opts.targetOverride, config) : resolveTarget(spec, config);
|
|
16401
|
-
const targetConfig = targetConfigFor(config, target.id);
|
|
16402
|
-
const specRef = {
|
|
16403
|
-
featureName,
|
|
16404
|
-
specName
|
|
16405
|
-
};
|
|
16406
|
-
const testPath = resolveTestPath(target, targetConfig, specRef);
|
|
16407
|
-
const ref = caseRefFor(specRef, cwd, opts.targetOverride === void 0 ? resolveRecordingPath(target, targetConfig, specRef) : ownRecordingPath(spec, config, specRef));
|
|
16408
|
-
return {
|
|
16409
|
-
testCase: {
|
|
16410
|
-
...testCase,
|
|
16411
|
-
ref
|
|
16412
|
-
},
|
|
16413
|
-
target,
|
|
16414
|
-
targetConfig,
|
|
16415
|
-
testPath
|
|
16416
|
-
};
|
|
16326
|
+
//#region src/drift/types.ts
|
|
16327
|
+
/** How everything downstream spells this case: its intent-source id, or `feature/spec`. */
|
|
16328
|
+
function caseIdOf(target) {
|
|
16329
|
+
return target.caseId ?? `${target.featureName}/${target.specName}`;
|
|
16417
16330
|
}
|
|
16418
16331
|
/**
|
|
16419
|
-
*
|
|
16420
|
-
* `--target`. Undefined when that target no longer resolves: overriding the
|
|
16421
|
-
* target is how a spec with an unusable one is generated at all, so it must
|
|
16422
|
-
* not be what stops the command.
|
|
16332
|
+
* What the audit says about one locator code could not find in the product.
|
|
16423
16333
|
*
|
|
16424
|
-
*
|
|
16425
|
-
*
|
|
16426
|
-
*
|
|
16334
|
+
* `drifted` says the test asks for something the product no longer renders;
|
|
16335
|
+
* `fine` says the miss has another explanation — a selector built at runtime, a
|
|
16336
|
+
* component this case never reaches, a file the scan did not read. Either is an
|
|
16337
|
+
* answer; silence is not, which is what the required list below is for.
|
|
16427
16338
|
*/
|
|
16428
|
-
|
|
16429
|
-
|
|
16430
|
-
|
|
16431
|
-
|
|
16432
|
-
|
|
16433
|
-
|
|
16434
|
-
|
|
16435
|
-
|
|
16436
|
-
|
|
16339
|
+
const LocatorVerdictSchema = z.object({
|
|
16340
|
+
id: z.string().min(1),
|
|
16341
|
+
verdict: z.enum(["drifted", "fine"]),
|
|
16342
|
+
note: z.string().default("")
|
|
16343
|
+
});
|
|
16344
|
+
/** The model's reply: a diagnosis, or `null` for "the spec still matches the code". */
|
|
16345
|
+
const DriftReplySchema = z.object({
|
|
16346
|
+
drift: DriftDiagnosisSchema.nullable(),
|
|
16347
|
+
locators: z.array(LocatorVerdictSchema).default([])
|
|
16348
|
+
});
|
|
16437
16349
|
/**
|
|
16438
|
-
*
|
|
16439
|
-
*
|
|
16440
|
-
*
|
|
16441
|
-
*
|
|
16442
|
-
*
|
|
16443
|
-
*
|
|
16444
|
-
* not this case's.
|
|
16350
|
+
* How a label reads against `--exit-on`. The threshold asks "would a
|
|
16351
|
+
* deterministic replay fail today". `TEST_DRIFT` and `SPEC_CHANGE` answer yes
|
|
16352
|
+
* and name the repair, so they hold the gate shut. The other three do not: a
|
|
16353
|
+
* suspected product bug or a dependency on data the source does not decide is
|
|
16354
|
+
* settled by running the case, not by refusing to run it, and `UNKNOWN` says
|
|
16355
|
+
* the audit could not tell.
|
|
16445
16356
|
*/
|
|
16446
|
-
function
|
|
16447
|
-
|
|
16448
|
-
if (own === null || own.intent.root !== reading.root) return void 0;
|
|
16449
|
-
const plugin = registryFor(config).get(own.id);
|
|
16450
|
-
return plugin ? resolveCaseRecordingPath(plugin, own.targetConfig, caseId) : void 0;
|
|
16451
|
-
}
|
|
16452
|
-
/** The target whose cases live in the project, when that is what we resolve to. */
|
|
16453
|
-
function intentTargetFor(config, targetOverride) {
|
|
16454
|
-
const id = targetOverride ?? config.defaultTarget;
|
|
16455
|
-
const targetConfig = config.targets[id];
|
|
16456
|
-
return targetConfig?.intent ? {
|
|
16457
|
-
id,
|
|
16458
|
-
targetConfig,
|
|
16459
|
-
intent: targetConfig.intent
|
|
16460
|
-
} : null;
|
|
16357
|
+
function driftSeverity(label) {
|
|
16358
|
+
return label === "TEST_DRIFT" || label === "SPEC_CHANGE" ? "error" : "warn";
|
|
16461
16359
|
}
|
|
16462
16360
|
//#endregion
|
|
16463
16361
|
//#region src/drift/artifacts.ts
|
|
@@ -16468,30 +16366,38 @@ function intentTargetFor(config, targetOverride) {
|
|
|
16468
16366
|
* is always named in `unaudited`.
|
|
16469
16367
|
*/
|
|
16470
16368
|
const MAX_GENERATED_BYTES = 64 * 1024;
|
|
16471
|
-
/**
|
|
16472
|
-
|
|
16369
|
+
/**
|
|
16370
|
+
* Read the sweep-wide inputs once, before the first spec. A caller that
|
|
16371
|
+
* already holds a reader passes it, so its cases are read once for the whole
|
|
16372
|
+
* command rather than once per phase.
|
|
16373
|
+
*/
|
|
16374
|
+
async function loadSpecArtifactsContext(cwd, reader) {
|
|
16473
16375
|
const [config, tsconfig] = await Promise.all([loadProjectConfig(cwd), loadTsconfigPaths(cwd)]);
|
|
16474
16376
|
return {
|
|
16475
16377
|
config,
|
|
16476
16378
|
tsconfig,
|
|
16477
|
-
|
|
16379
|
+
reader: reader ?? openCaseReader(config, cwd)
|
|
16478
16380
|
};
|
|
16479
16381
|
}
|
|
16480
16382
|
/**
|
|
16481
16383
|
* Read one case's documents. Throws when the document itself cannot be read —
|
|
16482
16384
|
* a case enumerated and then gone is an error to report, not an empty audit.
|
|
16385
|
+
*
|
|
16386
|
+
* A document that is there but will not parse is still audited: what its
|
|
16387
|
+
* author wrote is what the audit is for, and refusing to look at it would
|
|
16388
|
+
* clear a case nobody read.
|
|
16483
16389
|
*/
|
|
16484
16390
|
async function collectCaseArtifacts(target, cwd, ctx) {
|
|
16485
|
-
|
|
16486
|
-
}
|
|
16487
|
-
async function collectSpecCase(featureName, specName, cwd, ctx) {
|
|
16488
|
-
const specYaml = await readSpecFile(featureName, specName, cwd);
|
|
16391
|
+
const read = await ctx.reader.read(caseIdOf(target));
|
|
16392
|
+
if (read.document === null) throw new Error(read.error ?? `${caseIdOf(target)}: no such test case`);
|
|
16489
16393
|
const intent = {
|
|
16490
|
-
kind: "spec",
|
|
16491
|
-
path:
|
|
16492
|
-
body:
|
|
16394
|
+
kind: ctx.reader.target ? "project" : "spec",
|
|
16395
|
+
path: relative(cwd, read.document.path),
|
|
16396
|
+
body: read.document.text
|
|
16493
16397
|
};
|
|
16494
|
-
const
|
|
16398
|
+
const testCase = read.case;
|
|
16399
|
+
const live = testCase?.mode === "live";
|
|
16400
|
+
const title = testCase?.title ?? null;
|
|
16495
16401
|
if (live) return {
|
|
16496
16402
|
intent,
|
|
16497
16403
|
generated: [],
|
|
@@ -16502,114 +16408,109 @@ async function collectSpecCase(featureName, specName, cwd, ctx) {
|
|
|
16502
16408
|
};
|
|
16503
16409
|
return {
|
|
16504
16410
|
intent,
|
|
16505
|
-
...await
|
|
16411
|
+
...await readGenerated(testFileOf(testCase, ctx, cwd), cwd, ctx),
|
|
16506
16412
|
live,
|
|
16507
16413
|
title
|
|
16508
16414
|
};
|
|
16509
16415
|
}
|
|
16510
16416
|
/**
|
|
16511
|
-
* The generated half alone, for a caller that
|
|
16512
|
-
* failure classifier, which reads the same code the audit does
|
|
16513
|
-
* own account of what happened.
|
|
16417
|
+
* The generated half alone, for a caller that brings its own account of what
|
|
16418
|
+
* happened — the failure classifier, which reads the same code the audit does.
|
|
16514
16419
|
*/
|
|
16515
|
-
async function
|
|
16516
|
-
|
|
16420
|
+
async function collectGeneratedFor(caseId, cwd, ctx) {
|
|
16421
|
+
const read = await ctx.reader.read(caseId);
|
|
16422
|
+
if (read.case === null && read.error !== null) warn(`${caseId}: ${read.error}`);
|
|
16423
|
+
return readGenerated(testFileOf(read.case, ctx, cwd), cwd, ctx);
|
|
16517
16424
|
}
|
|
16518
|
-
|
|
16519
|
-
|
|
16520
|
-
|
|
16521
|
-
|
|
16522
|
-
|
|
16523
|
-
|
|
16524
|
-
|
|
16525
|
-
|
|
16526
|
-
|
|
16527
|
-
const
|
|
16528
|
-
|
|
16529
|
-
|
|
16530
|
-
|
|
16531
|
-
unaudited: [],
|
|
16532
|
-
reached: [],
|
|
16533
|
-
live,
|
|
16534
|
-
title: testCase.title
|
|
16535
|
-
};
|
|
16536
|
-
return {
|
|
16537
|
-
intent,
|
|
16538
|
-
...await readGenerated(resolve(cwd, resolveCaseTestPath(registryFor(ctx.config).get(ctx.intentTarget.id), targetConfig, caseId)), cwd, ctx),
|
|
16539
|
-
live,
|
|
16540
|
-
title: testCase.title
|
|
16425
|
+
/**
|
|
16426
|
+
* Where this case's files live, resolved through the target that owns them.
|
|
16427
|
+
*
|
|
16428
|
+
* All three together because one target resolution answers all three, and a
|
|
16429
|
+
* case that has one must not be reported as missing another. Every path is
|
|
16430
|
+
* null when nothing resolves — an unreadable case must cost its own paths,
|
|
16431
|
+
* not the sweep.
|
|
16432
|
+
*/
|
|
16433
|
+
async function caseFiles(target, ctx) {
|
|
16434
|
+
const none = {
|
|
16435
|
+
testPath: null,
|
|
16436
|
+
recordingPath: null,
|
|
16437
|
+
writeRoots: []
|
|
16541
16438
|
};
|
|
16439
|
+
const read = await ctx.reader.read(caseIdOf(target));
|
|
16440
|
+
if (read.case === null && read.error !== null) warn(`${caseIdOf(target)}: ${read.error}`);
|
|
16441
|
+
const testCase = read.case;
|
|
16442
|
+
const owner = testCase && targetOf$1(testCase, ctx);
|
|
16443
|
+
if (!testCase || !owner) return none;
|
|
16444
|
+
try {
|
|
16445
|
+
return {
|
|
16446
|
+
testPath: resolveCaseTestPath(owner.plugin, owner.targetConfig, testCase.ref.id),
|
|
16447
|
+
recordingPath: resolveCaseRecordingPath(owner.plugin, owner.targetConfig, testCase.ref.id),
|
|
16448
|
+
writeRoots: owner.targetConfig.writeRoots
|
|
16449
|
+
};
|
|
16450
|
+
} catch {
|
|
16451
|
+
return none;
|
|
16452
|
+
}
|
|
16542
16453
|
}
|
|
16543
16454
|
/**
|
|
16544
16455
|
* Where this case's generated test lives, project-relative — or null when it
|
|
16545
16456
|
* has none (a live case, or a document whose target cannot be resolved).
|
|
16546
16457
|
*/
|
|
16547
|
-
async function caseTestPath(target,
|
|
16548
|
-
|
|
16549
|
-
if (!ctx.intentTarget) return null;
|
|
16550
|
-
return resolveCaseTestPath(registryFor(ctx.config).get(ctx.intentTarget.id), ctx.intentTarget.targetConfig, target.caseId);
|
|
16551
|
-
}
|
|
16552
|
-
const specYaml = await readSpecFile(target.featureName, target.specName, cwd).catch(() => null);
|
|
16553
|
-
if (specYaml === null) return null;
|
|
16554
|
-
const abs = specTestFile(target.featureName, target.specName, specYaml, cwd, ctx.config);
|
|
16555
|
-
return abs === null ? null : relative(cwd, abs);
|
|
16458
|
+
async function caseTestPath(target, ctx) {
|
|
16459
|
+
return (await caseFiles(target, ctx)).testPath;
|
|
16556
16460
|
}
|
|
16557
16461
|
/**
|
|
16558
16462
|
* Where this case's recording lives, project-relative, resolved through the
|
|
16559
16463
|
* same target as its test — null on the same terms as {@link caseTestPath}.
|
|
16560
16464
|
*/
|
|
16561
|
-
async function caseRecordingPath(target,
|
|
16562
|
-
|
|
16563
|
-
if (!ctx.intentTarget) return null;
|
|
16564
|
-
return resolveCaseRecordingPath(registryFor(ctx.config).get(ctx.intentTarget.id), ctx.intentTarget.targetConfig, target.caseId);
|
|
16565
|
-
}
|
|
16566
|
-
const specYaml = await readSpecFile(target.featureName, target.specName, cwd).catch(() => null);
|
|
16567
|
-
if (specYaml === null) return null;
|
|
16568
|
-
return specTargetPaths(target.featureName, target.specName, specYaml, cwd, ctx.config)?.recordingPath ?? null;
|
|
16465
|
+
async function caseRecordingPath(target, ctx) {
|
|
16466
|
+
return (await caseFiles(target, ctx)).recordingPath;
|
|
16569
16467
|
}
|
|
16570
16468
|
/**
|
|
16571
|
-
* Where this case's test lands, and where its target lets ccqa write
|
|
16572
|
-
*
|
|
16573
|
-
* Together because they answer one question — which files a finding about this
|
|
16574
|
-
* case could be repaired by regenerating it — and because resolving the target
|
|
16575
|
-
* once is what keeps the two consistent for a spec that names a non-default one.
|
|
16469
|
+
* Where this case's test lands, and where its target lets ccqa write: the two
|
|
16470
|
+
* halves of "which files a finding about this case could be repaired in".
|
|
16576
16471
|
*/
|
|
16577
|
-
async function caseWriteArea(target,
|
|
16578
|
-
const testPath = await
|
|
16579
|
-
|
|
16472
|
+
async function caseWriteArea(target, ctx) {
|
|
16473
|
+
const { testPath, writeRoots } = await caseFiles(target, ctx);
|
|
16474
|
+
return {
|
|
16580
16475
|
testPath,
|
|
16581
|
-
writeRoots
|
|
16582
|
-
};
|
|
16583
|
-
const specYaml = await readSpecFile(target.featureName, target.specName, cwd).catch(() => null);
|
|
16584
|
-
if (specYaml === null) return {
|
|
16585
|
-
testPath,
|
|
16586
|
-
writeRoots: []
|
|
16476
|
+
writeRoots
|
|
16587
16477
|
};
|
|
16478
|
+
}
|
|
16479
|
+
/**
|
|
16480
|
+
* The target that owns this case's generated files.
|
|
16481
|
+
*
|
|
16482
|
+
* A case the project states itself belongs to the target that declared the
|
|
16483
|
+
* source; a `spec.yaml` case names its own. Null when nothing resolves — an
|
|
16484
|
+
* unreadable target must cost this one case's paths, not the sweep.
|
|
16485
|
+
*/
|
|
16486
|
+
function targetOf$1(testCase, ctx) {
|
|
16487
|
+
if (ctx.reader.target) {
|
|
16488
|
+
const plugin = registryFor(ctx.config).get(ctx.reader.target.id);
|
|
16489
|
+
return plugin ? {
|
|
16490
|
+
plugin,
|
|
16491
|
+
targetConfig: ctx.reader.target.targetConfig
|
|
16492
|
+
} : null;
|
|
16493
|
+
}
|
|
16494
|
+
if (testCase.spec === null) return null;
|
|
16588
16495
|
try {
|
|
16589
|
-
const
|
|
16496
|
+
const plugin = resolveTarget(testCase.spec, ctx.config);
|
|
16590
16497
|
return {
|
|
16591
|
-
|
|
16592
|
-
|
|
16498
|
+
plugin,
|
|
16499
|
+
targetConfig: targetConfigFor(ctx.config, plugin.id)
|
|
16593
16500
|
};
|
|
16594
16501
|
} catch {
|
|
16595
|
-
return
|
|
16596
|
-
testPath,
|
|
16597
|
-
writeRoots: []
|
|
16598
|
-
};
|
|
16502
|
+
return null;
|
|
16599
16503
|
}
|
|
16600
16504
|
}
|
|
16601
|
-
|
|
16505
|
+
/** The generated test's absolute path, or null when this case compiles into none. */
|
|
16506
|
+
function testFileOf(testCase, ctx, cwd) {
|
|
16507
|
+
if (testCase === null) return null;
|
|
16508
|
+
const owner = targetOf$1(testCase, ctx);
|
|
16509
|
+
if (owner === null) return null;
|
|
16602
16510
|
try {
|
|
16603
|
-
|
|
16604
|
-
return {
|
|
16605
|
-
live: spec.mode === "live",
|
|
16606
|
-
title: spec.title ?? null
|
|
16607
|
-
};
|
|
16511
|
+
return resolve(cwd, resolveCaseTestPath(owner.plugin, owner.targetConfig, testCase.ref.id));
|
|
16608
16512
|
} catch {
|
|
16609
|
-
return
|
|
16610
|
-
live: false,
|
|
16611
|
-
title: null
|
|
16612
|
-
};
|
|
16513
|
+
return null;
|
|
16613
16514
|
}
|
|
16614
16515
|
}
|
|
16615
16516
|
/**
|
|
@@ -16657,35 +16558,6 @@ async function readGenerated(testAbs, cwd, ctx) {
|
|
|
16657
16558
|
reached
|
|
16658
16559
|
};
|
|
16659
16560
|
}
|
|
16660
|
-
/**
|
|
16661
|
-
* Where this spec's two files live, resolved through its target: the test, and
|
|
16662
|
-
* the recording it compiles from. A spec that will not parse, whose target
|
|
16663
|
-
* cannot be resolved, or whose `testPath` will not expand has neither to
|
|
16664
|
-
* derive — the audit then reads the spec alone rather than guessing at a file.
|
|
16665
|
-
*
|
|
16666
|
-
* Both together because one target resolution answers both, and a spec that
|
|
16667
|
-
* has one must not be reported as missing the other.
|
|
16668
|
-
*/
|
|
16669
|
-
function specTargetPaths(featureName, specName, specYaml, cwd, config) {
|
|
16670
|
-
try {
|
|
16671
|
-
const target = resolveTarget(parseTestSpec(specYaml), config);
|
|
16672
|
-
const targetConfig = targetConfigFor(config, target.id);
|
|
16673
|
-
const ref = {
|
|
16674
|
-
featureName,
|
|
16675
|
-
specName
|
|
16676
|
-
};
|
|
16677
|
-
return {
|
|
16678
|
-
testAbs: resolveTestPathAbs(target, targetConfig, ref, cwd),
|
|
16679
|
-
recordingPath: resolveRecordingPath(target, targetConfig, ref)
|
|
16680
|
-
};
|
|
16681
|
-
} catch {
|
|
16682
|
-
return null;
|
|
16683
|
-
}
|
|
16684
|
-
}
|
|
16685
|
-
/** Where this spec's test lives, resolved through its target. */
|
|
16686
|
-
function specTestFile(featureName, specName, specYaml, cwd, config) {
|
|
16687
|
-
return specTargetPaths(featureName, specName, specYaml, cwd, config)?.testAbs ?? null;
|
|
16688
|
-
}
|
|
16689
16561
|
//#endregion
|
|
16690
16562
|
//#region src/cli/colors.ts
|
|
16691
16563
|
/**
|
|
@@ -16768,7 +16640,7 @@ function createFailureAnalysisPass(deps) {
|
|
|
16768
16640
|
};
|
|
16769
16641
|
if (input.specYaml === null) return {
|
|
16770
16642
|
...fields,
|
|
16771
|
-
analysisSkipped: "
|
|
16643
|
+
analysisSkipped: "the case's own document could not be read"
|
|
16772
16644
|
};
|
|
16773
16645
|
const baselineMissing = specDiffResult.ok ? null : specDiffResult.skip;
|
|
16774
16646
|
info(`failure analysis: ${featureName}/${specName}${baselineMissing ? " (no baseline — classifying from current source)" : ""}`);
|
|
@@ -16856,9 +16728,9 @@ function needsAnalysis(row) {
|
|
|
16856
16728
|
* invents would land in the confusion matrix and the project's learned prompt.
|
|
16857
16729
|
* The recorded reason is both accurate and cheaper.
|
|
16858
16730
|
*/
|
|
16859
|
-
async function analyzeExternalRows(rows, run) {
|
|
16731
|
+
async function analyzeExternalRows(rows, run, caseIds = /* @__PURE__ */ new Map()) {
|
|
16860
16732
|
const { deps, pass } = run;
|
|
16861
|
-
const context = await loadSpecArtifactsContext(deps.cwd);
|
|
16733
|
+
const context = await loadSpecArtifactsContext(deps.cwd, deps.reader);
|
|
16862
16734
|
const out = [];
|
|
16863
16735
|
for (const row of rows) {
|
|
16864
16736
|
if (!needsAnalysis(row)) {
|
|
@@ -16869,9 +16741,10 @@ async function analyzeExternalRows(rows, run) {
|
|
|
16869
16741
|
featureName: row.feature,
|
|
16870
16742
|
specName: row.spec
|
|
16871
16743
|
};
|
|
16744
|
+
const caseId = caseIds.get(specKey(ref)) ?? specKey(ref);
|
|
16872
16745
|
const fields = await pass.analyze({
|
|
16873
16746
|
...ref,
|
|
16874
|
-
readScript: () => readGeneratedTestSources(
|
|
16747
|
+
readScript: () => readGeneratedTestSources(caseId, deps.cwd, context),
|
|
16875
16748
|
failureLog: row.failureLogExcerpt ?? "",
|
|
16876
16749
|
specYaml: row.specYaml,
|
|
16877
16750
|
parsedSpec: tryParseTestSpec(row.specYaml),
|
|
@@ -16902,9 +16775,8 @@ function readableArtifactsDir(ref, deps) {
|
|
|
16902
16775
|
* both surfaces read the same set. Best-effort: an ungenerated or unreadable
|
|
16903
16776
|
* spec just means less context, never a failed analysis.
|
|
16904
16777
|
*/
|
|
16905
|
-
async function readGeneratedTestSources(
|
|
16906
|
-
|
|
16907
|
-
const { generated, unaudited } = await collectSpecGenerated(ref.featureName, ref.specName, specYaml, cwd, context);
|
|
16778
|
+
async function readGeneratedTestSources(caseId, cwd, context) {
|
|
16779
|
+
const { generated, unaudited } = await collectGeneratedFor(caseId, cwd, context);
|
|
16908
16780
|
const parts = generated.map((f) => `// ${f.path}\n${f.content}`);
|
|
16909
16781
|
if (unaudited.length > 0) parts.push(`// [not shown: ${unaudited.join(", ")} — Read them for their full state]`);
|
|
16910
16782
|
return parts.join("\n\n");
|
|
@@ -18326,7 +18198,7 @@ function failureBody(row) {
|
|
|
18326
18198
|
}
|
|
18327
18199
|
/** A row's stored path (relative to `reportDir`), re-expressed relative to where the XML itself lives. */
|
|
18328
18200
|
function rewriteEvidencePath(storedPath, reportDir, junitDir) {
|
|
18329
|
-
return toPosix(relative(junitDir, resolve(reportDir, storedPath)));
|
|
18201
|
+
return toPosix$1(relative(junitDir, resolve(reportDir, storedPath)));
|
|
18330
18202
|
}
|
|
18331
18203
|
/** Step-boundary screenshot paths for one row, in capture order, for `<system-out>`. */
|
|
18332
18204
|
function evidencePaths(row, reportDir, junitDir) {
|
|
@@ -18391,35 +18263,22 @@ function currentReportCost() {
|
|
|
18391
18263
|
}
|
|
18392
18264
|
//#endregion
|
|
18393
18265
|
//#region src/run/spec-catalog.ts
|
|
18394
|
-
async function readSpecs(refs,
|
|
18266
|
+
async function readSpecs(refs, reader) {
|
|
18395
18267
|
const entries = await Promise.all(refs.map(async (ref) => {
|
|
18396
|
-
const
|
|
18397
|
-
|
|
18398
|
-
|
|
18399
|
-
yaml: null,
|
|
18400
|
-
error: null
|
|
18268
|
+
const read = await reader.read(specKey(ref));
|
|
18269
|
+
return [specKey(ref), {
|
|
18270
|
+
case: read.case,
|
|
18271
|
+
yaml: read.document?.text ?? null,
|
|
18272
|
+
error: read.document === null ? null : read.error
|
|
18401
18273
|
}];
|
|
18402
|
-
try {
|
|
18403
|
-
return [specKey(ref), {
|
|
18404
|
-
spec: parseTestSpec(yaml),
|
|
18405
|
-
yaml,
|
|
18406
|
-
error: null
|
|
18407
|
-
}];
|
|
18408
|
-
} catch (err) {
|
|
18409
|
-
return [specKey(ref), {
|
|
18410
|
-
spec: null,
|
|
18411
|
-
yaml,
|
|
18412
|
-
error: errMessage(err)
|
|
18413
|
-
}];
|
|
18414
|
-
}
|
|
18415
18274
|
}));
|
|
18416
18275
|
return new Map(entries);
|
|
18417
18276
|
}
|
|
18418
|
-
/**
|
|
18277
|
+
/** Case-declared `mode:` wins; otherwise `DEFAULT_SPEC_MODE`. */
|
|
18419
18278
|
function resolveSpecsModes(specs, catalog) {
|
|
18420
18279
|
return specs.map((s) => ({
|
|
18421
18280
|
...s,
|
|
18422
|
-
mode: catalog.get(specKey(s))?.
|
|
18281
|
+
mode: catalog.get(specKey(s))?.case?.mode ?? "deterministic"
|
|
18423
18282
|
}));
|
|
18424
18283
|
}
|
|
18425
18284
|
//#endregion
|
|
@@ -19994,7 +19853,7 @@ function runsLive(testCase) {
|
|
|
19994
19853
|
return testCase.mode === "live";
|
|
19995
19854
|
}
|
|
19996
19855
|
function liveCaseFrom(testCase, opts = {}) {
|
|
19997
|
-
const spec = testCase.
|
|
19856
|
+
const spec = testCase.spec;
|
|
19998
19857
|
const own = attachCaseExpectations(requireActionSteps(testCase.steps, testCase.ref.id, {
|
|
19999
19858
|
id: AGENT_BROWSER_TARGET,
|
|
20000
19859
|
reason: AGENT_BROWSER_JUDGE_STEPS.reason
|
|
@@ -20006,7 +19865,7 @@ function liveCaseFrom(testCase, opts = {}) {
|
|
|
20006
19865
|
title: testCase.title,
|
|
20007
19866
|
steps,
|
|
20008
19867
|
cleanupFrom: own.length,
|
|
20009
|
-
document: testCase.
|
|
19868
|
+
document: testCase.document.text,
|
|
20010
19869
|
blocks: spec ? collectIncludedBlockNames(spec) : [],
|
|
20011
19870
|
session: {
|
|
20012
19871
|
names: spec?.session ?? [],
|
|
@@ -20040,135 +19899,74 @@ function attachCaseExpectations(steps, expectations) {
|
|
|
20040
19899
|
//#endregion
|
|
20041
19900
|
//#region src/select/inventory.ts
|
|
20042
19901
|
/**
|
|
20043
|
-
* Read every test case the project states into the shape the selection
|
|
20044
|
-
*
|
|
20045
|
-
*
|
|
20046
|
-
*
|
|
20047
|
-
* (see `intentTargetFor`), so this reads one enumeration or the other.
|
|
19902
|
+
* Read every test case the project states into the shape the selection prompt
|
|
19903
|
+
* consumes — ccqa's own specs, or the documents the project writes itself,
|
|
19904
|
+
* whichever this project has. One enumeration, one read per case, through the
|
|
19905
|
+
* one reader: what the two kinds share is everything the prompt is shown.
|
|
20048
19906
|
*/
|
|
20049
19907
|
async function loadSpecInventory(cwd) {
|
|
20050
19908
|
const config = await loadProjectConfig(cwd);
|
|
20051
|
-
const
|
|
20052
|
-
|
|
20053
|
-
return
|
|
20054
|
-
|
|
20055
|
-
|
|
20056
|
-
|
|
20057
|
-
|
|
20058
|
-
|
|
20059
|
-
* Enumeration (`listAllSpecsWithSpecFile`, a directory walk) is kept separate
|
|
20060
|
-
* from reading and parsing each spec's own content, so every spec.yaml is
|
|
20061
|
-
* read and parsed exactly once — and the reads run in parallel.
|
|
20062
|
-
*/
|
|
20063
|
-
async function loadSpecFileInventory(config, cwd) {
|
|
20064
|
-
const [refs, blocks] = await Promise.all([listAllSpecsWithSpecFile(cwd), loadAllBlocks(cwd)]);
|
|
20065
|
-
return (await Promise.all(refs.map(async ({ featureName, specName }) => {
|
|
20066
|
-
const content = await tryReadSpecFile(featureName, specName, cwd);
|
|
20067
|
-
if (content === null) return null;
|
|
20068
|
-
const spec = parseTestSpec(content, `${featureName}/${specName}/spec.yaml`);
|
|
20069
|
-
if (spec.disabled) return null;
|
|
20070
|
-
return {
|
|
20071
|
-
featureName,
|
|
20072
|
-
specName,
|
|
20073
|
-
title: spec.title,
|
|
20074
|
-
steps: describeSteps(spec, blocks, `${featureName}/${specName}`),
|
|
20075
|
-
includedBlocks: collectIncludedBlockNames(spec),
|
|
20076
|
-
...resolveSpecPaths(spec, config, featureName, specName),
|
|
20077
|
-
sourcePath: `.ccqa/features/${featureName}/test-cases/${specName}/spec.yaml`
|
|
20078
|
-
};
|
|
20079
|
-
}))).filter((s) => s !== null);
|
|
20080
|
-
}
|
|
20081
|
-
/**
|
|
20082
|
-
* Same shape, for a project whose cases are its own markdown documents. No
|
|
20083
|
-
* `disabled` flag to drop here — that concept belongs to `spec.yaml`, and an
|
|
20084
|
-
* intent source has nothing that plays the same role.
|
|
20085
|
-
*/
|
|
20086
|
-
async function loadMarkdownInventory(intentTarget, config, cwd) {
|
|
20087
|
-
const { id: targetId, targetConfig, intent } = intentTarget;
|
|
20088
|
-
const target = registryFor(config).get(targetId);
|
|
20089
|
-
const ids = await listMarkdownCases(intent, cwd);
|
|
20090
|
-
return Promise.all(ids.map(async (caseId) => {
|
|
20091
|
-
const testCase = await loadMarkdownCase(caseId, intent, cwd);
|
|
19909
|
+
const reader = openCaseReader(config, cwd);
|
|
19910
|
+
const ids = await reader.list();
|
|
19911
|
+
return (await Promise.all(ids.map(async (id) => {
|
|
19912
|
+
const read = await reader.read(id);
|
|
19913
|
+
if (read.case === null) throw new Error(read.error ?? `${id}: could not be read`);
|
|
19914
|
+
const testCase = read.case;
|
|
19915
|
+
if (testCase.blocked !== null) warn(`${id}: ${testCase.blocked}`);
|
|
19916
|
+
if (testCase.disabled) return null;
|
|
20092
19917
|
const { featureName, specName } = splitCaseId(testCase.ref.id);
|
|
20093
19918
|
return {
|
|
20094
19919
|
featureName,
|
|
20095
19920
|
specName,
|
|
20096
19921
|
title: testCase.title,
|
|
20097
|
-
steps:
|
|
20098
|
-
includedBlocks: [],
|
|
20099
|
-
|
|
20100
|
-
|
|
20101
|
-
sourcePath: markdownSourcePath(testCase, caseId, cwd)
|
|
19922
|
+
steps: testCase.steps.map(describeStepBody),
|
|
19923
|
+
includedBlocks: testCase.spec ? collectIncludedBlockNames(testCase.spec) : [],
|
|
19924
|
+
...resolveCasePaths(testCase, config, reader),
|
|
19925
|
+
sourcePath: relative(cwd, testCase.document.path)
|
|
20102
19926
|
};
|
|
20103
|
-
}));
|
|
20104
|
-
}
|
|
20105
|
-
/** The case's own file, project-root-relative, or its id when the source somehow isn't markdown. */
|
|
20106
|
-
function markdownSourcePath(testCase, id, cwd) {
|
|
20107
|
-
return testCase.source.kind === "markdown" ? relative(cwd, testCase.source.path) : id;
|
|
19927
|
+
}))).filter((s) => s !== null);
|
|
20108
19928
|
}
|
|
20109
19929
|
/**
|
|
20110
19930
|
* `--format paths` needs a real file to hand a test runner, so a target that
|
|
20111
|
-
* can't be resolved (bad `target:`/`defaultTarget`) degrades this one
|
|
19931
|
+
* can't be resolved (bad `target:`/`defaultTarget`) degrades this one case's
|
|
20112
19932
|
* path to "" instead of failing the whole inventory — the other formats don't
|
|
20113
|
-
* depend on it. A live
|
|
20114
|
-
* from the
|
|
19933
|
+
* depend on it. A live case resolves to "" for the same reason: it is driven
|
|
19934
|
+
* from the document every run and has compiled nothing a runner could take.
|
|
20115
19935
|
*/
|
|
20116
|
-
function
|
|
19936
|
+
function resolveCasePaths(testCase, config, reader) {
|
|
20117
19937
|
const none = {
|
|
20118
19938
|
testPath: "",
|
|
20119
19939
|
recordingPath: ""
|
|
20120
19940
|
};
|
|
20121
|
-
if (
|
|
19941
|
+
if (runsLive(testCase)) return none;
|
|
20122
19942
|
try {
|
|
20123
|
-
const
|
|
20124
|
-
const
|
|
20125
|
-
const
|
|
20126
|
-
featureName,
|
|
20127
|
-
specName
|
|
20128
|
-
};
|
|
19943
|
+
const owner = reader.target;
|
|
19944
|
+
const target = owner ? registryFor(config).get(owner.id) : resolveTarget(testCase.spec, config);
|
|
19945
|
+
const targetConfig = owner ? owner.targetConfig : targetConfigFor(config, target.id);
|
|
20129
19946
|
return {
|
|
20130
|
-
testPath:
|
|
20131
|
-
recordingPath:
|
|
19947
|
+
testPath: resolveCaseTestPath(target, targetConfig, testCase.ref.id),
|
|
19948
|
+
recordingPath: resolveCaseRecordingPath(target, targetConfig, testCase.ref.id)
|
|
20132
19949
|
};
|
|
20133
19950
|
} catch (e) {
|
|
20134
|
-
warn(`${
|
|
19951
|
+
warn(`${testCase.ref.id}: could not resolve test path (${e.message})`);
|
|
20135
19952
|
return none;
|
|
20136
19953
|
}
|
|
20137
19954
|
}
|
|
20138
19955
|
/**
|
|
20139
|
-
* One line per step,
|
|
20140
|
-
* the selection prompt is told to weigh shared
|
|
20141
|
-
* against each
|
|
20142
|
-
* here rather than hidden behind a block name.
|
|
19956
|
+
* One line per step, in order. Include steps arrive already expanded to the
|
|
19957
|
+
* block's own steps — the selection prompt is told to weigh shared
|
|
19958
|
+
* login/navigation/layout against each case, which only works if those steps
|
|
19959
|
+
* are actually visible here rather than hidden behind a block name.
|
|
20143
19960
|
*
|
|
20144
|
-
*
|
|
20145
|
-
*
|
|
20146
|
-
*
|
|
19961
|
+
* A step with no `expected` of its own prints as the instruction alone: a case
|
|
19962
|
+
* that states its expectations once for the whole flow has nothing to put
|
|
19963
|
+
* after the arrow, and an empty one reads as a step that checks nothing.
|
|
20147
19964
|
*/
|
|
20148
|
-
function describeSteps(spec, blocks, specKey) {
|
|
20149
|
-
try {
|
|
20150
|
-
return expandSpec(spec, { blocks }).map(describeStepBody);
|
|
20151
|
-
} catch (e) {
|
|
20152
|
-
warn(`${specKey}: could not expand include steps (${e.message}) — showing block names instead`);
|
|
20153
|
-
return spec.steps.map(describeStep);
|
|
20154
|
-
}
|
|
20155
|
-
}
|
|
20156
|
-
function describeStep(step) {
|
|
20157
|
-
if (isIncludeStep(step)) return `include block: ${step.include}`;
|
|
20158
|
-
return describeStepBody(step);
|
|
20159
|
-
}
|
|
20160
19965
|
function describeStepBody(step) {
|
|
20161
19966
|
if (isJudgeBody(step)) return `judge: ${oneLine$1(step.judgeByLlm)}`;
|
|
20162
|
-
|
|
20163
|
-
|
|
20164
|
-
|
|
20165
|
-
* One line per step, in order — no `include:` to expand, since a markdown
|
|
20166
|
-
* case's steps are never block references. Unlike a spec step, a markdown
|
|
20167
|
-
* step carries no per-step `expected` (the case lists its expectations once,
|
|
20168
|
-
* for the whole flow), so only the instruction goes on the line.
|
|
20169
|
-
*/
|
|
20170
|
-
function describeMarkdownSteps(testCase) {
|
|
20171
|
-
return testCase.steps.map((step) => isJudgeBody(step) ? `judge: ${oneLine$1(step.judgeByLlm)}` : oneLine$1(step.instruction));
|
|
19967
|
+
const expected = oneLine$1(step.expected);
|
|
19968
|
+
const instruction = oneLine$1(step.instruction);
|
|
19969
|
+
return expected.length > 0 ? `${instruction} → ${expected}` : instruction;
|
|
20172
19970
|
}
|
|
20173
19971
|
function oneLine$1(text) {
|
|
20174
19972
|
return text.trim().replace(/\s+/g, " ");
|
|
@@ -21817,7 +21615,7 @@ async function copyEvidenceIntoReport(absPath, evidenceDir, reportDir) {
|
|
|
21817
21615
|
const dest = join(evidenceDir, basename(absPath));
|
|
21818
21616
|
try {
|
|
21819
21617
|
await cp(absPath, dest);
|
|
21820
|
-
return posix.relative(toPosix(resolve(reportDir)), toPosix(dest));
|
|
21618
|
+
return posix.relative(toPosix$1(resolve(reportDir)), toPosix$1(dest));
|
|
21821
21619
|
} catch (err) {
|
|
21822
21620
|
warn(`failed to copy live evidence ${absPath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
21823
21621
|
return null;
|
|
@@ -22294,6 +22092,10 @@ var CoverageInbox = class {
|
|
|
22294
22092
|
};
|
|
22295
22093
|
//#endregion
|
|
22296
22094
|
//#region src/run/target-dispatch.ts
|
|
22095
|
+
/** The title a report row carries for a case, or null when it could not be read. */
|
|
22096
|
+
function titleOf(spec) {
|
|
22097
|
+
return spec.testCase?.title ?? null;
|
|
22098
|
+
}
|
|
22297
22099
|
/**
|
|
22298
22100
|
* Resolve each spec's target and group. A spec with no spec.yaml at all falls
|
|
22299
22101
|
* through to the agent-browser path, whose runner surfaces the real error
|
|
@@ -22318,16 +22120,20 @@ function groupSpecsByTarget(specs, catalog, config, resolve = resolveTargetFor(c
|
|
|
22318
22120
|
const unresolved = [];
|
|
22319
22121
|
for (const ref of specs) {
|
|
22320
22122
|
const read = catalog.get(specKey(ref));
|
|
22123
|
+
const dispatched = () => ({
|
|
22124
|
+
...ref,
|
|
22125
|
+
caseId: specKey(ref),
|
|
22126
|
+
testCase: read?.case ?? null
|
|
22127
|
+
});
|
|
22321
22128
|
if (read?.error) {
|
|
22322
22129
|
unresolved.push({
|
|
22323
|
-
...
|
|
22324
|
-
title: null,
|
|
22130
|
+
...dispatched(),
|
|
22325
22131
|
reason: read.error,
|
|
22326
22132
|
targetId: null
|
|
22327
22133
|
});
|
|
22328
22134
|
continue;
|
|
22329
22135
|
}
|
|
22330
|
-
const spec = read?.spec ?? null;
|
|
22136
|
+
const spec = (read?.case ?? null)?.spec ?? null;
|
|
22331
22137
|
if (spec === null) {
|
|
22332
22138
|
agentBrowser.push(ref);
|
|
22333
22139
|
continue;
|
|
@@ -22337,8 +22143,7 @@ function groupSpecsByTarget(specs, catalog, config, resolve = resolveTargetFor(c
|
|
|
22337
22143
|
plugin = resolve(spec, config);
|
|
22338
22144
|
} catch (err) {
|
|
22339
22145
|
unresolved.push({
|
|
22340
|
-
...
|
|
22341
|
-
title: spec.title ?? null,
|
|
22146
|
+
...dispatched(),
|
|
22342
22147
|
reason: err instanceof Error ? err.message : String(err),
|
|
22343
22148
|
targetId: spec.target ?? config.defaultTarget ?? null
|
|
22344
22149
|
});
|
|
@@ -22348,10 +22153,7 @@ function groupSpecsByTarget(specs, catalog, config, resolve = resolveTargetFor(c
|
|
|
22348
22153
|
agentBrowser.push(ref);
|
|
22349
22154
|
continue;
|
|
22350
22155
|
}
|
|
22351
|
-
const entry =
|
|
22352
|
-
...ref,
|
|
22353
|
-
title: spec.title ?? null
|
|
22354
|
-
};
|
|
22156
|
+
const entry = dispatched();
|
|
22355
22157
|
const targetConfig = targetConfigFor(config, plugin.id);
|
|
22356
22158
|
const routed = externalRunnability(plugin, plugin.id, targetConfig);
|
|
22357
22159
|
if ("reason" in routed) skipped.push({
|
|
@@ -22403,16 +22205,16 @@ function externalGroupFor(plugin, targetConfig) {
|
|
|
22403
22205
|
};
|
|
22404
22206
|
}
|
|
22405
22207
|
/**
|
|
22406
|
-
* The external group for a project whose cases
|
|
22208
|
+
* The external group for a project whose cases come from its own source.
|
|
22407
22209
|
*
|
|
22408
22210
|
* Such a case is not dispatched by reading a `spec.yaml` `target:` — the
|
|
22409
|
-
* target that declares the
|
|
22211
|
+
* target that declares the case source already owns every case under it — so
|
|
22410
22212
|
* the routing is one lookup rather than a walk. What it shares with
|
|
22411
22213
|
* {@link groupSpecsByTarget} is the two conditions that decide whether ccqa
|
|
22412
22214
|
* can execute a generated test at all: the plugin has a runner, and the
|
|
22413
22215
|
* project configured the command to run it with.
|
|
22414
22216
|
*/
|
|
22415
|
-
function
|
|
22217
|
+
function groupOwnCases(cases, targetId, config) {
|
|
22416
22218
|
if (cases.length === 0) return {
|
|
22417
22219
|
external: [],
|
|
22418
22220
|
skipped: []
|
|
@@ -22453,7 +22255,7 @@ async function runExternalSpecs(dispatch, ctx) {
|
|
|
22453
22255
|
...emptySpecRow({
|
|
22454
22256
|
feature: u.featureName,
|
|
22455
22257
|
spec: u.specName,
|
|
22456
|
-
title: u
|
|
22258
|
+
title: titleOf(u),
|
|
22457
22259
|
status: "failed"
|
|
22458
22260
|
}),
|
|
22459
22261
|
...u.targetId ? { target: u.targetId } : {},
|
|
@@ -22467,7 +22269,7 @@ async function runExternalSpecs(dispatch, ctx) {
|
|
|
22467
22269
|
...emptySpecRow({
|
|
22468
22270
|
feature: s.featureName,
|
|
22469
22271
|
spec: s.specName,
|
|
22470
|
-
title: s
|
|
22272
|
+
title: titleOf(s),
|
|
22471
22273
|
status: "skipped"
|
|
22472
22274
|
}),
|
|
22473
22275
|
...s.targetId ? { target: s.targetId } : {},
|
|
@@ -22508,7 +22310,7 @@ async function runExternalSpecs(dispatch, ctx) {
|
|
|
22508
22310
|
...emptySpecRow({
|
|
22509
22311
|
feature: s.featureName,
|
|
22510
22312
|
spec: s.specName,
|
|
22511
|
-
title: s
|
|
22313
|
+
title: titleOf(s),
|
|
22512
22314
|
status: "failed"
|
|
22513
22315
|
}),
|
|
22514
22316
|
target: group.targetId,
|
|
@@ -23086,18 +22888,16 @@ function asHubReadError$1(err) {
|
|
|
23086
22888
|
throw new RunUsageError(`could not read from the hub: ${errMessage(err)}`);
|
|
23087
22889
|
}
|
|
23088
22890
|
/**
|
|
23089
|
-
* The cases a project's own
|
|
23090
|
-
* every one
|
|
23091
|
-
*
|
|
23092
|
-
*
|
|
22891
|
+
* The cases a project's own source holds, as this invocation named them:
|
|
22892
|
+
* every one the source holds, or exactly the ones the arguments name. An
|
|
22893
|
+
* argument is read both ways a person writes it — the path they see in their
|
|
22894
|
+
* editor, or the id everything else cites.
|
|
23093
22895
|
*/
|
|
23094
|
-
async function
|
|
23095
|
-
const
|
|
23096
|
-
|
|
23097
|
-
|
|
23098
|
-
|
|
23099
|
-
throw new RunUsageError(errMessage(err));
|
|
23100
|
-
}
|
|
22896
|
+
async function readOwnCases(targets, reader) {
|
|
22897
|
+
const named = targets.length > 0;
|
|
22898
|
+
const refs = named ? [...new Set(targets)] : await reader.list();
|
|
22899
|
+
const cases = await Promise.all(refs.map((ref) => reader.load(ref)));
|
|
22900
|
+
return named ? cases : cases.filter((c) => !c.disabled);
|
|
23101
22901
|
}
|
|
23102
22902
|
/** De-dupe by `featureName/specName`, keeping first-seen order. */
|
|
23103
22903
|
function dedupeSpecs(specs) {
|
|
@@ -23278,6 +23078,8 @@ async function executeRun(targets, opts) {
|
|
|
23278
23078
|
const triageUserPromptHash = triageUserPrompt ? hashTriageUserPrompt(triageUserPrompt) : null;
|
|
23279
23079
|
const reportDir = resolveReportDir(opts.reportDir, cwd);
|
|
23280
23080
|
const parsedBlocks = await loadAllBlocks(cwd);
|
|
23081
|
+
const projectConfig = await loadProjectConfig(cwd);
|
|
23082
|
+
const reader = openCaseReader(projectConfig, cwd);
|
|
23281
23083
|
const analysisDeps = {
|
|
23282
23084
|
diffProvider,
|
|
23283
23085
|
auth: diffProvider ? driftAuthAvailable() : {
|
|
@@ -23286,6 +23088,7 @@ async function executeRun(targets, opts) {
|
|
|
23286
23088
|
},
|
|
23287
23089
|
cwd,
|
|
23288
23090
|
reportDir,
|
|
23091
|
+
reader,
|
|
23289
23092
|
blocks: projectAvailableBlocks(parsedBlocks),
|
|
23290
23093
|
parsedBlocks,
|
|
23291
23094
|
...opts.model ? { model: opts.model } : {},
|
|
@@ -23293,18 +23096,15 @@ async function executeRun(targets, opts) {
|
|
|
23293
23096
|
customPrompt,
|
|
23294
23097
|
triageUserPrompt
|
|
23295
23098
|
};
|
|
23296
|
-
const projectConfig = await loadProjectConfig(cwd);
|
|
23297
23099
|
await loadEnvFiles(projectConfig.envFiles, cwd);
|
|
23298
|
-
const
|
|
23299
|
-
|
|
23300
|
-
|
|
23301
|
-
|
|
23302
|
-
cases: await readIntentCases(targets, intentTarget.intent, cwd)
|
|
23100
|
+
const ownRun = reader.target === null ? null : {
|
|
23101
|
+
id: reader.target.id,
|
|
23102
|
+
module: reader.target.module,
|
|
23103
|
+
cases: await readOwnCases(targets, reader)
|
|
23303
23104
|
};
|
|
23304
|
-
if (intentRun !== null && opts.onlyHubRerunNeeded) throw new RunUsageError("--only-hub-rerun-needed reads the hub's per-spec rerun verdicts, which a project whose cases are its own documents has none of — name the cases to run instead");
|
|
23305
23105
|
const enumerateAll = () => listActiveSpecs(cwd);
|
|
23306
|
-
const resolved =
|
|
23307
|
-
let specs =
|
|
23106
|
+
const resolved = ownRun ? [] : await Promise.all((targets.length ? targets : [void 0]).map((t) => resolveSpecTargets(t, enumerateAll, cwd)));
|
|
23107
|
+
let specs = ownRun ? ownRun.cases.map((c) => splitCaseId(c.ref.id)) : dedupeSpecs(resolved.flat());
|
|
23308
23108
|
if (filtering) {
|
|
23309
23109
|
const before = specs.length;
|
|
23310
23110
|
let inProgress = 0;
|
|
@@ -23336,16 +23136,16 @@ async function executeRun(targets, opts) {
|
|
|
23336
23136
|
}
|
|
23337
23137
|
}
|
|
23338
23138
|
const selectedKeys = new Set(specs.map(specKey));
|
|
23339
|
-
const selectedCases =
|
|
23139
|
+
const selectedCases = ownRun?.cases.filter((c) => selectedKeys.has(specKey(splitCaseId(c.ref.id)))) ?? [];
|
|
23340
23140
|
const liveCaseCount = selectedCases.filter(runsLive).length;
|
|
23341
23141
|
const generatedCases = opts.coverage === true || targets.length > 0 ? selectedCases.filter((c) => !runsLive(c)) : [];
|
|
23342
|
-
if (specs.length === 0 ||
|
|
23343
|
-
if (
|
|
23142
|
+
if (specs.length === 0 || ownRun !== null && liveCaseCount + generatedCases.length === 0) {
|
|
23143
|
+
if (ownRun === null) {
|
|
23344
23144
|
warn("no specs to run");
|
|
23345
|
-
if (Object.values(projectConfig.targets).some((t) => t.
|
|
23346
|
-
} else if (
|
|
23145
|
+
if (Object.values(projectConfig.targets).some((t) => t.cases)) hint(`this project has a target whose cases come from its own files, but its defaultTarget (${projectConfig.defaultTarget}) is not that target, so this run enumerated .ccqa/features/ instead`);
|
|
23146
|
+
} else if (ownRun.cases.length === 0) warn(`${ownRun.module} lists no test cases`);
|
|
23347
23147
|
else {
|
|
23348
|
-
warn(`nothing to run: ${
|
|
23148
|
+
warn(`nothing to run: ${ownRun.module} lists ${ownRun.cases.length} test case(s), none of them live`);
|
|
23349
23149
|
hint("ccqa runs a case itself only when its mode says `live`; every other case's test is executed by the project's own test command, which `ccqa select-specs --format paths` feeds");
|
|
23350
23150
|
hint("pass --coverage, or name the cases, to have ccqa run their generated tests through the target's runCommand — which is how a measurement observes what they reach");
|
|
23351
23151
|
}
|
|
@@ -23355,7 +23155,7 @@ async function executeRun(targets, opts) {
|
|
|
23355
23155
|
reportDir: null
|
|
23356
23156
|
};
|
|
23357
23157
|
}
|
|
23358
|
-
const catalog = await readSpecs(
|
|
23158
|
+
const catalog = await readSpecs(ownRun ? [] : specs, reader);
|
|
23359
23159
|
const actors = opts.coverage === true && forExecution ? await resolveActors(projectConfig.coverage?.actors ?? {}, cwd) : NO_ACTORS;
|
|
23360
23160
|
const resources = mergeGroups(await resolveSerialGroups(projectConfig.serialGroups, cwd), actorGroups(actors));
|
|
23361
23161
|
const declared = [...new Set(specs.flatMap(resources))];
|
|
@@ -23385,31 +23185,33 @@ async function executeRun(targets, opts) {
|
|
|
23385
23185
|
}
|
|
23386
23186
|
let dispatch;
|
|
23387
23187
|
try {
|
|
23388
|
-
dispatch = groupSpecsByTarget(
|
|
23389
|
-
if (
|
|
23390
|
-
const routed =
|
|
23188
|
+
dispatch = groupSpecsByTarget(ownRun ? [] : specs, catalog, projectConfig);
|
|
23189
|
+
if (ownRun !== null && generatedCases.length > 0) {
|
|
23190
|
+
const routed = groupOwnCases(generatedCases.map((c) => ({
|
|
23391
23191
|
...splitCaseId(c.ref.id),
|
|
23392
|
-
|
|
23393
|
-
|
|
23192
|
+
caseId: c.ref.id,
|
|
23193
|
+
testCase: c
|
|
23194
|
+
})), ownRun.id, projectConfig);
|
|
23394
23195
|
dispatch.external.push(...routed.external);
|
|
23395
23196
|
dispatch.skipped.push(...routed.skipped);
|
|
23396
23197
|
}
|
|
23397
23198
|
for (const { spec, groups } of waitingOnGroup) dispatch.skipped.push({
|
|
23398
23199
|
...spec,
|
|
23399
|
-
|
|
23200
|
+
caseId: specKey(spec),
|
|
23201
|
+
testCase: catalog.get(specKey(spec))?.case ?? null,
|
|
23400
23202
|
reason: `waiting on ${groups.join(", ")}, held by another job`,
|
|
23401
23203
|
targetId: null
|
|
23402
23204
|
});
|
|
23403
23205
|
} catch (err) {
|
|
23404
23206
|
throw new RunUsageError(errMessage(err));
|
|
23405
23207
|
}
|
|
23406
|
-
const withMode =
|
|
23208
|
+
const withMode = ownRun ? selectedCases.map((c) => ({
|
|
23407
23209
|
...splitCaseId(c.ref.id),
|
|
23408
23210
|
mode: c.mode
|
|
23409
23211
|
})) : resolveSpecsModes(dispatch.agentBrowser, catalog);
|
|
23410
|
-
const detSpecs =
|
|
23212
|
+
const detSpecs = ownRun ? [] : withMode.filter((s) => s.mode === "deterministic");
|
|
23411
23213
|
const liveSpecs = withMode.filter((s) => s.mode === "live");
|
|
23412
|
-
meta("modes",
|
|
23214
|
+
meta("modes", ownRun ? `${selectedCases.length} case(s) / ${liveSpecs.length} live / ${generatedCases.length} generated, run here / ${selectedCases.length - liveSpecs.length - generatedCases.length} for the project's own test command` : `${detSpecs.length} deterministic / ${liveSpecs.length} live`);
|
|
23413
23215
|
if (dispatch.external.length > 0) meta("external", dispatch.external.map((g) => `${g.targetId} ${g.specs.length}`).join(" / "));
|
|
23414
23216
|
const storedMaps = hubCtx !== null && deployedSha !== null ? createStoredSourceMaps(hubCtx, deployedSha) : void 0;
|
|
23415
23217
|
const coverage = opts.coverage && forExecution ? await startCoverage(cwd, projectConfig.coverage, actors, dispatch, opts.teardown, coverageInbox, storedMaps?.read, hubCtx !== null) : void 0;
|
|
@@ -23424,17 +23226,18 @@ async function executeRun(targets, opts) {
|
|
|
23424
23226
|
blank();
|
|
23425
23227
|
if (opts.dryRun) {
|
|
23426
23228
|
const runningHere = new Set(generatedCases.map((c) => specKey(splitCaseId(c.ref.id))));
|
|
23427
|
-
const notOurs = (
|
|
23229
|
+
const notOurs = (ownRun ? withMode.filter((s) => s.mode !== "live" && !runningHere.has(specKey(s))) : []).map((s) => ({
|
|
23428
23230
|
...s,
|
|
23429
|
-
|
|
23231
|
+
caseId: specKey(s),
|
|
23232
|
+
testCase: null,
|
|
23430
23233
|
reason: "run by the project's own test command",
|
|
23431
|
-
targetId:
|
|
23234
|
+
targetId: ownRun?.id ?? null
|
|
23432
23235
|
}));
|
|
23433
23236
|
const routed = {
|
|
23434
23237
|
...dispatch,
|
|
23435
23238
|
skipped: [...dispatch.skipped, ...notOurs]
|
|
23436
23239
|
};
|
|
23437
|
-
const ours =
|
|
23240
|
+
const ours = ownRun ? liveSpecs : withMode;
|
|
23438
23241
|
for (const line of formatDryRunLines(ours, routed, resources)) emitRaw(line + "\n");
|
|
23439
23242
|
blank();
|
|
23440
23243
|
info("dry run: nothing was executed and no report was written");
|
|
@@ -23531,10 +23334,10 @@ async function executeRun(targets, opts) {
|
|
|
23531
23334
|
report: incrementalReport,
|
|
23532
23335
|
conventions: targetConfigFor(projectConfig, AGENT_BROWSER_TARGET).conventions.operate
|
|
23533
23336
|
};
|
|
23534
|
-
const liveCases =
|
|
23535
|
-
const
|
|
23536
|
-
if (
|
|
23537
|
-
return liveCaseFrom(
|
|
23337
|
+
const liveCases = ownRun ? selectedCases.filter(runsLive).map((c) => liveCaseFrom(c, liveCaseOpts)) : liveSpecs.map((ref) => {
|
|
23338
|
+
const testCase = catalog.get(specKey(ref))?.case;
|
|
23339
|
+
if (testCase == null) throw new Error(`${specKey(ref)}: spec.yaml is no longer readable`);
|
|
23340
|
+
return liveCaseFrom(testCase, liveCaseOpts);
|
|
23538
23341
|
});
|
|
23539
23342
|
const live = await runLiveSpecs(liveCases, liveOpts);
|
|
23540
23343
|
let streamedEdges = {};
|
|
@@ -23556,8 +23359,8 @@ async function executeRun(targets, opts) {
|
|
|
23556
23359
|
featureName: r.feature,
|
|
23557
23360
|
specName: r.spec
|
|
23558
23361
|
}))], analysisDeps);
|
|
23559
|
-
const detResults = await analyzeDeterministicSummaries(det.summaries,
|
|
23560
|
-
const analyzedExternalRows = await analyzeExternalRows(externalRows, analysisRun);
|
|
23362
|
+
const detResults = await analyzeDeterministicSummaries(det.summaries, catalog, reportDir, analysisRun);
|
|
23363
|
+
const analyzedExternalRows = await analyzeExternalRows(externalRows, analysisRun, new Map(dispatch.external.flatMap((g) => g.specs.map((c) => [specKey(c), c.caseId]))));
|
|
23561
23364
|
const results = await rerunExplainedFailures([
|
|
23562
23365
|
...detResults,
|
|
23563
23366
|
...analyzedExternalRows,
|
|
@@ -23960,18 +23763,18 @@ function failedSpec(s) {
|
|
|
23960
23763
|
* specs share). Degrades — never throws — when Claude auth or the git diff
|
|
23961
23764
|
* aren't available. Caller writes report.json.
|
|
23962
23765
|
*/
|
|
23963
|
-
async function analyzeDeterministicSummaries(summaries,
|
|
23766
|
+
async function analyzeDeterministicSummaries(summaries, catalog, reportDir, { pass, deps }) {
|
|
23964
23767
|
const results = [];
|
|
23965
23768
|
for (const s of summaries) {
|
|
23966
23769
|
const assertions = collectAssertions(s);
|
|
23967
|
-
const
|
|
23968
|
-
const
|
|
23969
|
-
const
|
|
23970
|
-
const evidence = await loadEvidenceForSpec(s.evidenceDir, reportDir,
|
|
23770
|
+
const testCase = catalog.get(specKey(s))?.case ?? null;
|
|
23771
|
+
const specYaml = testCase?.document.text ?? null;
|
|
23772
|
+
const parsedSpec = testCase?.spec ?? null;
|
|
23773
|
+
const evidence = await loadEvidenceForSpec(s.evidenceDir, reportDir, stepCaptions(testCase));
|
|
23971
23774
|
const base = {
|
|
23972
23775
|
feature: s.featureName,
|
|
23973
23776
|
spec: s.specName,
|
|
23974
|
-
title:
|
|
23777
|
+
title: testCase?.title ?? null,
|
|
23975
23778
|
target: AGENT_BROWSER_TARGET,
|
|
23976
23779
|
startedAt: s.startedAt,
|
|
23977
23780
|
finishedAt: s.finishedAt,
|
|
@@ -24064,7 +23867,7 @@ function createRerunExecutor(ctx) {
|
|
|
24064
23867
|
}
|
|
24065
23868
|
const group = ctx.dispatch.external.find((g) => g.specs.some((s) => specKey(s) === key));
|
|
24066
23869
|
if (group === void 0) throw new Error(`${key} has no execution path to re-run`);
|
|
24067
|
-
const [row] = await group.runner.run([
|
|
23870
|
+
const [row] = await group.runner.run([group.specs.find((c) => specKey(c) === key)], {
|
|
24068
23871
|
cwd: ctx.cwd,
|
|
24069
23872
|
reportDir: scratch,
|
|
24070
23873
|
concurrency: 1,
|
|
@@ -24403,7 +24206,7 @@ function installTeardownSignalHandlers(teardown, onSignal) {
|
|
|
24403
24206
|
}
|
|
24404
24207
|
//#endregion
|
|
24405
24208
|
//#region src/cli/run.ts
|
|
24406
|
-
const runCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("run").argument("[targets...]", "Specs to run, space-separated: each '<feature>/<spec>', '<feature>', or omit for all. For a target
|
|
24209
|
+
const runCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("run").argument("[targets...]", "Specs to run, space-separated: each '<feature>/<spec>', '<feature>', or omit for all. For a target with a `cases` module, each is a case id or the path of its source file. Duplicates are de-duped.").description("Run specs, on any target. Agent-browser specs replay the recorded test.spec.ts under vitest (default), or, with spec.yaml `mode: live`, have Claude drive agent-browser live per step. External-target specs (playwright, runn) run through the target's configured `runCommand`. Where the project writes its cases as markdown, ccqa runs the ones whose mode says `live`; the rest are executed by the project's own test command (see `ccqa select-specs --format paths`). A structured report (report.json + evidence) is always written; use --report-to-hub to also stream it to a hub.").optionsGroup("Which specs to run:").option("--only-affected-by <ref>", "Only specs `ccqa select-specs` decides the diff against <ref> reaches (e.g. origin/main), by intersecting it with each spec's measured coverage from the hub. In pull_request CI, pass $GITHUB_BASE_REF. Cannot be combined with an explicit spec id.").option("--only-hub-rerun-needed", "Only specs the hub answers `rerunNeeded` for: the audit cleared them, and their last result does not cover what is deployed — including every spec the deploy log cannot place, which is assumed reached rather than skipped. A spec whose audit has not caught up answers `inProgress`, and one the audit rejected or whose last run failed answers `needsRepair`; neither is taken, because running them races the audit or repairs nothing. No git diff involved. Requires a hub connection and --hub-profile.").option("--dry-run", "Print the specs this invocation would run, then exit 0 without executing anything and without writing a report. Works with every selection flag.").optionsGroup("How to run them:").option("--concurrency <n>", "Run up to N specs in parallel within each phase (deterministic / external-target / live), never across phases. Default 1 (sequential). Specs in the same `serialGroups` entry of .ccqa/config.yaml still take turns. Live specs each get an isolated agent-browser session; high values spawn many headed Chrome instances.", parseConcurrency$1, 1).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--live-step-retry <n>", "(live only) Retry each failed step up to N more times before recording failure. This retries a step, not the whole spec — see --on-fail-explain-rerun for that.", (raw) => {
|
|
24407
24210
|
const n = Number(raw);
|
|
24408
24211
|
if (!Number.isFinite(n) || n < 0 || Math.floor(n) !== n) throw new Error(`--live-step-retry must be a non-negative integer, got "${raw}"`);
|
|
24409
24212
|
return n;
|
|
@@ -25661,7 +25464,7 @@ function stepsWithoutAsserts(stepIds, actions) {
|
|
|
25661
25464
|
async function runTrace(testCase, model, validationMode = "lenient", language, opts = {}) {
|
|
25662
25465
|
header("trace", testCase.ref.id);
|
|
25663
25466
|
await preflightAgentBrowserCommand();
|
|
25664
|
-
const specSteps = testCase.
|
|
25467
|
+
const specSteps = testCase.spec?.steps ?? [];
|
|
25665
25468
|
const expanded = [...testCase.steps, ...testCase.cleanup];
|
|
25666
25469
|
const steps = expanded.filter(isExpandedActionStep);
|
|
25667
25470
|
const sessionName = generateSessionName();
|
|
@@ -25675,7 +25478,7 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
|
|
|
25675
25478
|
meta("case", testCase.title);
|
|
25676
25479
|
meta("steps", steps.length);
|
|
25677
25480
|
if (testCase.cleanup.length > 0) meta("cleanup steps", testCase.cleanup.length);
|
|
25678
|
-
const includes = testCase.
|
|
25481
|
+
const includes = testCase.spec ? collectIncludedBlockNames(testCase.spec) : [];
|
|
25679
25482
|
if (includes.length > 0) meta("blocks", includes.join(", "));
|
|
25680
25483
|
blank();
|
|
25681
25484
|
opts.teardown?.trackSession(sessionName);
|
|
@@ -26264,6 +26067,85 @@ function dropList(drops) {
|
|
|
26264
26067
|
return drops.slice(0, 5).map((d) => ` - ${describeStepAction(d.action)} — ${d.reason}`).join("\n");
|
|
26265
26068
|
}
|
|
26266
26069
|
//#endregion
|
|
26070
|
+
//#region src/cli/resolve-case.ts
|
|
26071
|
+
async function resolveCase(argument, config, cwd, opts = {}) {
|
|
26072
|
+
const reader = openCaseReader(config, cwd, opts);
|
|
26073
|
+
const testCase = await reader.load(argument);
|
|
26074
|
+
if (reader.target) {
|
|
26075
|
+
const { id, targetConfig, modulePath } = reader.target;
|
|
26076
|
+
const target = registryFor(config).get(id);
|
|
26077
|
+
const testPath = resolveCaseTestPath(target, targetConfig, testCase.ref.id);
|
|
26078
|
+
const recordingPath = (opts.targetOverride === void 0 ? void 0 : ownCaseRecordingPath(config, cwd, modulePath, testCase.ref.id)) ?? resolveCaseRecordingPath(target, targetConfig, testCase.ref.id);
|
|
26079
|
+
const ref = caseRefFor(testCase.ref.id, cwd, recordingPath);
|
|
26080
|
+
return {
|
|
26081
|
+
testCase: {
|
|
26082
|
+
...testCase,
|
|
26083
|
+
ref
|
|
26084
|
+
},
|
|
26085
|
+
target,
|
|
26086
|
+
targetConfig,
|
|
26087
|
+
testPath
|
|
26088
|
+
};
|
|
26089
|
+
}
|
|
26090
|
+
const spec = testCase.spec;
|
|
26091
|
+
const specRef = splitSpecRef(testCase.ref.id);
|
|
26092
|
+
const target = opts.targetOverride !== void 0 ? resolveTargetOverride(spec, opts.targetOverride, config) : resolveTarget(spec, config);
|
|
26093
|
+
const targetConfig = targetConfigFor(config, target.id);
|
|
26094
|
+
const testPath = resolveTestPath(target, targetConfig, specRef);
|
|
26095
|
+
const ref = caseRefFor(specRef, cwd, opts.targetOverride === void 0 ? resolveRecordingPath(target, targetConfig, specRef) : ownRecordingPath(spec, config, specRef));
|
|
26096
|
+
return {
|
|
26097
|
+
testCase: {
|
|
26098
|
+
...testCase,
|
|
26099
|
+
ref
|
|
26100
|
+
},
|
|
26101
|
+
target,
|
|
26102
|
+
targetConfig,
|
|
26103
|
+
testPath
|
|
26104
|
+
};
|
|
26105
|
+
}
|
|
26106
|
+
/** A spec case's id is its two coordinates joined, and only ever those two. */
|
|
26107
|
+
function splitSpecRef(id) {
|
|
26108
|
+
const [featureName, specName] = id.split("/");
|
|
26109
|
+
return {
|
|
26110
|
+
featureName,
|
|
26111
|
+
specName
|
|
26112
|
+
};
|
|
26113
|
+
}
|
|
26114
|
+
/**
|
|
26115
|
+
* Where the spec's own target keeps this case's recording, ignoring any
|
|
26116
|
+
* `--target`. Undefined when that target no longer resolves: overriding the
|
|
26117
|
+
* target is how a spec with an unusable one is generated at all, so it must
|
|
26118
|
+
* not be what stops the command.
|
|
26119
|
+
*
|
|
26120
|
+
* Only the resolution is guarded. A target that resolved and then could not
|
|
26121
|
+
* expand its own `testPath` is a broken config, and swallowing that would put
|
|
26122
|
+
* the recording somewhere else without a word.
|
|
26123
|
+
*/
|
|
26124
|
+
function ownRecordingPath(spec, config, ref) {
|
|
26125
|
+
let target;
|
|
26126
|
+
try {
|
|
26127
|
+
target = resolveTarget(spec, config);
|
|
26128
|
+
} catch {
|
|
26129
|
+
return;
|
|
26130
|
+
}
|
|
26131
|
+
return resolveRecordingPath(target, targetConfigFor(config, target.id), ref);
|
|
26132
|
+
}
|
|
26133
|
+
/**
|
|
26134
|
+
* The case-source branch's half of the same rule: where the project's own
|
|
26135
|
+
* default target keeps this case's recording.
|
|
26136
|
+
*
|
|
26137
|
+
* Undefined when that target reads its cases through a different module than
|
|
26138
|
+
* the one this invocation resolved through — another module is describing a
|
|
26139
|
+
* different case that happens to share an id, and its route is not this
|
|
26140
|
+
* case's.
|
|
26141
|
+
*/
|
|
26142
|
+
function ownCaseRecordingPath(config, cwd, readingModule, caseId) {
|
|
26143
|
+
const own = caseTargetFor(config, cwd);
|
|
26144
|
+
if (own === null || own.modulePath !== readingModule) return void 0;
|
|
26145
|
+
const plugin = registryFor(config).get(own.id);
|
|
26146
|
+
return plugin ? resolveCaseRecordingPath(plugin, own.targetConfig, caseId) : void 0;
|
|
26147
|
+
}
|
|
26148
|
+
//#endregion
|
|
26267
26149
|
//#region src/cli/build-generate-run-summary.ts
|
|
26268
26150
|
/**
|
|
26269
26151
|
* Compact, prompt-friendly summary of one `ccqa generate` pass for the
|
|
@@ -26337,7 +26219,7 @@ async function runGenerate(resolved, opts) {
|
|
|
26337
26219
|
async function runGenerateLocked(resolved, opts, cwd) {
|
|
26338
26220
|
const { testCase, target, targetConfig, testPath } = resolved;
|
|
26339
26221
|
const { sessionState } = await loadProjectConfig(cwd);
|
|
26340
|
-
const spec = testCase.
|
|
26222
|
+
const spec = testCase.spec;
|
|
26341
26223
|
if (spec?.mode === "live") {
|
|
26342
26224
|
error(`this spec is 'mode: live' — a live spec runs without generated code. Run 'ccqa run ${testCase.ref.id}' instead`);
|
|
26343
26225
|
process.exit(2);
|
|
@@ -26394,7 +26276,7 @@ async function runGenerateLocked(resolved, opts, cwd) {
|
|
|
26394
26276
|
title: testCase.title,
|
|
26395
26277
|
steps: []
|
|
26396
26278
|
},
|
|
26397
|
-
specYaml:
|
|
26279
|
+
specYaml: spec ? testCase.document.text : "",
|
|
26398
26280
|
featureName,
|
|
26399
26281
|
specName,
|
|
26400
26282
|
ref: testCase.ref,
|
|
@@ -26425,7 +26307,6 @@ async function runGenerateLocked(resolved, opts, cwd) {
|
|
|
26425
26307
|
const written = await stampGeneratedTest(testCase.ref, testPathAbs);
|
|
26426
26308
|
if (written !== null && movedRecording !== void 0) info(`recording moved beside its test: ${written}`);
|
|
26427
26309
|
}
|
|
26428
|
-
if (result.passed) await writeBackOutputPath(testCase, targetConfig, testPath);
|
|
26429
26310
|
if (opts.updateAgentPrompt) await runGenerateAgentPromptUpdate(target, testCase.ref.id, result, opts, cwd);
|
|
26430
26311
|
if (!result.passed) warn("auto-fix exhausted; test still failing");
|
|
26431
26312
|
else hint(`run 'ccqa run ${testCase.ref.id}' to execute the test`);
|
|
@@ -26454,28 +26335,6 @@ async function runGenerateAgentPromptUpdate(target, caseId, result, opts, cwd) {
|
|
|
26454
26335
|
});
|
|
26455
26336
|
}
|
|
26456
26337
|
/**
|
|
26457
|
-
* Write the generated test's path into the case's own `outputPath` section.
|
|
26458
|
-
*
|
|
26459
|
-
* Only when the project named that section and the case actually has it: a
|
|
26460
|
-
* case file belongs to the project, and ccqa rewrites the one heading it was
|
|
26461
|
-
* given permission to rewrite, leaving every other byte alone.
|
|
26462
|
-
*/
|
|
26463
|
-
async function writeBackOutputPath(testCase, targetConfig, testPath) {
|
|
26464
|
-
const heading = targetConfig.intent?.fields.outputPath;
|
|
26465
|
-
if (!heading || testCase.source.kind !== "markdown") return;
|
|
26466
|
-
const { path } = testCase.source;
|
|
26467
|
-
const before = await readFile(path, "utf8").catch(() => null);
|
|
26468
|
-
if (before === null) return;
|
|
26469
|
-
const after = replaceSectionBody(before, heading, testPath);
|
|
26470
|
-
if (after === null) {
|
|
26471
|
-
warn(`the case has no "${heading}" section, so the generated test's path was not written back`);
|
|
26472
|
-
return;
|
|
26473
|
-
}
|
|
26474
|
-
if (after === before) return;
|
|
26475
|
-
await writeFile(path, after, "utf8");
|
|
26476
|
-
meta("wrote back", `${heading} in ${relative(process.cwd(), path) || path}`);
|
|
26477
|
-
}
|
|
26478
|
-
/**
|
|
26479
26338
|
* The reason regeneration is refused because the test on disk is not the one
|
|
26480
26339
|
* the last generation wrote — or null when it may proceed. A matching stamp
|
|
26481
26340
|
* means ccqa produced this file, so regenerating it costs nobody anything and
|
|
@@ -26522,13 +26381,11 @@ function staleRecordingWarning(caseId, editedAt, recordedAt) {
|
|
|
26522
26381
|
return `the case was edited after it was recorded (case ${editedAt.toISOString()}, recording ${recordedAt}). Generation compiles the recording, so anything added to the case since is not in it — run 'ccqa record ${caseId}' if the steps or the expected results changed.`;
|
|
26523
26382
|
}
|
|
26524
26383
|
async function warnIfCaseOutranRecording(testCase, recordedAt) {
|
|
26525
|
-
const
|
|
26526
|
-
if (path === void 0) return;
|
|
26527
|
-
const editedAt = await stat(path).then((st) => st.mtime, () => null);
|
|
26384
|
+
const editedAt = await stat(testCase.document.path).then((st) => st.mtime, () => null);
|
|
26528
26385
|
const warning = staleRecordingWarning(testCase.ref.id, editedAt, recordedAt);
|
|
26529
26386
|
if (warning !== null) warn(warning);
|
|
26530
26387
|
}
|
|
26531
|
-
const generateCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("generate").argument("<case>", "The case to generate from: a spec id ('<feature>/<spec>'), or — for a target
|
|
26388
|
+
const generateCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("generate").argument("<case>", "The case to generate from: a spec id ('<feature>/<spec>'), or — for a target with a `cases` module — a case id or the path of its source file").description("Generate test code from a case via its target. Recording-backed targets compile the case's existing recording (run `ccqa record` first); spec-input targets generate directly from the spec.").optionsGroup("How to generate:").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--target <id>", "Generate through this target instead of the spec's own — e.g. emit a Playwright spec from an agent-browser recording. The spec's `target:` stays the default for `ccqa run`.").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--no-replay", "Skip replaying the saved recording before regenerating from it. The replay drives your application for real — it performs the recorded actions, and whatever they create the recorded cleanup then tries to remove — and it needs a browser and the spec's variables. Skip it where that is unwelcome or unavailable, and accept that a route the application has outgrown regenerates into a test that cannot pass.").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--overwrite", "Replace an existing generated test without the y/N prompt (declines on a non-TTY)").optionsGroup("Learning:").option("--learn-hub-codegen-prompt", "After generation, ask Claude to refresh the target's \"<target>.agent\" learning prompt on the hub from a summary of the run. LLM-generating targets (playwright, runn) only; requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
26532
26389
|
await withCostReporting("generate", () => runGenerateCli(specPath, opts));
|
|
26533
26390
|
}));
|
|
26534
26391
|
async function runGenerateCli(caseArgument, opts) {
|
|
@@ -26598,7 +26455,7 @@ async function runGenerateCli(caseArgument, opts) {
|
|
|
26598
26455
|
//#endregion
|
|
26599
26456
|
//#region src/cli/record.ts
|
|
26600
26457
|
const VALIDATION_MODES = ["lenient", "strict"];
|
|
26601
|
-
const recordCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("record").argument("<case>", "The case to record: a spec id ('<feature>/<spec>'), or — for a target
|
|
26458
|
+
const recordCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("record").argument("<case>", "The case to record: a spec id ('<feature>/<spec>'), or — for a target with a `cases` module — a case id or the path of its source file").description("Record a test from a case: run agent-browser to collect actions (trace), then compile them into runnable code via the case's target (generate) — a vitest test.spec.ts for agent-browser, a @playwright/test spec for playwright and for a target the project defines. Recording-backed targets only; spec-input targets like runn have no trace step (use `ccqa generate`), and agent-browser live specs need no recording.").optionsGroup("How to record:").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--instruction <text>", "Extra guidance for the recording agent — e.g. the drift audit's finding when re-recording a drifted spec.").option("--trace-validation <mode>", "What to do with actions that fail post-trace validation: 'lenient' (default) tags them; 'strict' drops them.", (raw) => {
|
|
26602
26459
|
if (VALIDATION_MODES.includes(raw)) return raw;
|
|
26603
26460
|
throw new Error(`--trace-validation must be one of ${VALIDATION_MODES.join(" | ")}`);
|
|
26604
26461
|
}, "lenient").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--timeout <seconds>", "Abort the recording after this many seconds, wherever it is (trace, generate, auto-fix): reap the browser session, seal the open hub run (--report-to-hub) with a 'timed out' note, and exit 124. Prefer this over wrapping the command in an external `timeout`, whose SIGTERM may never reach this process.", parseTimeoutSeconds).option("--trace-only", "Stop after the trace step; do not generate test code").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--report-to-hub", "Leave a run (kind: record) on the hub saying this spec was recorded and what the recording spent on Claude, so a budget summed over the hub's runs sees it. It advances no ledger: a recording verifies nothing.").optionsGroup("Learning:").option("--learn-hub-trace-prompt", "After the trace finishes, ask Claude to refresh the \"record.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
@@ -26613,7 +26470,7 @@ async function runRecord(caseArgument, opts) {
|
|
|
26613
26470
|
const resolved = await resolveCase(caseArgument, config, cwdForProfile);
|
|
26614
26471
|
const { testCase, target } = resolved;
|
|
26615
26472
|
const caseId = testCase.ref.id;
|
|
26616
|
-
const spec = testCase.
|
|
26473
|
+
const spec = testCase.spec;
|
|
26617
26474
|
if (target.input === "spec") {
|
|
26618
26475
|
error(`target "${target.id}" does not use a browser recording — run 'ccqa generate ${caseId}' instead`);
|
|
26619
26476
|
process.exit(2);
|
|
@@ -27594,7 +27451,7 @@ function buildDriftUserPrompt(artifacts, sourceRoots = [], locators) {
|
|
|
27594
27451
|
const { kind, path, body } = artifacts.intent;
|
|
27595
27452
|
return `## ${kind === "spec" ? "spec.yaml" : `Test case document — ${path}`}
|
|
27596
27453
|
|
|
27597
|
-
\`\`\`${kind === "spec" ? "yaml" : "
|
|
27454
|
+
\`\`\`${kind === "spec" ? "yaml" : extname(path).replace(/^\./, "")}
|
|
27598
27455
|
${body}
|
|
27599
27456
|
\`\`\`
|
|
27600
27457
|
|
|
@@ -27637,9 +27494,9 @@ What renders a screen is often a template rather than a script — a \`.vue\`, \
|
|
|
27637
27494
|
|
|
27638
27495
|
`;
|
|
27639
27496
|
}
|
|
27640
|
-
/** What states the case
|
|
27497
|
+
/** What states the case: ccqa's own format, or one it does not know. */
|
|
27641
27498
|
function intentSurfaceBlock(kind) {
|
|
27642
|
-
if (kind === "
|
|
27499
|
+
if (kind === "project") return `- **the test case document** — always present. Written and owned by the project's own authors, in whatever format they keep their test cases in; ccqa does not parse it and neither should you assume a shape. Read it as a statement of intent: what the case does, and what must then be true. Its structure and vocabulary are theirs, not ccqa's.`;
|
|
27643
27500
|
return `- **spec.yaml** — always present. Pure YAML: \`title\`, then \`steps\`, each either an action (\`instruction\` + \`expected\`) or \`include: <block-name>\` with \`params\`. \`expected\` names something observable — visible text, an aria-label, a URL, an element state.`;
|
|
27644
27501
|
}
|
|
27645
27502
|
function generatedSection(artifacts) {
|
|
@@ -28076,41 +27933,6 @@ function repairTargetsOutside(evidence, area) {
|
|
|
28076
27933
|
return outside.length > 0 ? outside : null;
|
|
28077
27934
|
}
|
|
28078
27935
|
//#endregion
|
|
28079
|
-
//#region src/drift/types.ts
|
|
28080
|
-
/** How everything downstream spells this case: its intent-source id, or `feature/spec`. */
|
|
28081
|
-
function caseIdOf(target) {
|
|
28082
|
-
return target.caseId ?? `${target.featureName}/${target.specName}`;
|
|
28083
|
-
}
|
|
28084
|
-
/**
|
|
28085
|
-
* What the audit says about one locator code could not find in the product.
|
|
28086
|
-
*
|
|
28087
|
-
* `drifted` says the test asks for something the product no longer renders;
|
|
28088
|
-
* `fine` says the miss has another explanation — a selector built at runtime, a
|
|
28089
|
-
* component this case never reaches, a file the scan did not read. Either is an
|
|
28090
|
-
* answer; silence is not, which is what the required list below is for.
|
|
28091
|
-
*/
|
|
28092
|
-
const LocatorVerdictSchema = z.object({
|
|
28093
|
-
id: z.string().min(1),
|
|
28094
|
-
verdict: z.enum(["drifted", "fine"]),
|
|
28095
|
-
note: z.string().default("")
|
|
28096
|
-
});
|
|
28097
|
-
/** The model's reply: a diagnosis, or `null` for "the spec still matches the code". */
|
|
28098
|
-
const DriftReplySchema = z.object({
|
|
28099
|
-
drift: DriftDiagnosisSchema.nullable(),
|
|
28100
|
-
locators: z.array(LocatorVerdictSchema).default([])
|
|
28101
|
-
});
|
|
28102
|
-
/**
|
|
28103
|
-
* How a label reads against `--exit-on`. The threshold asks "would a
|
|
28104
|
-
* deterministic replay fail today". `TEST_DRIFT` and `SPEC_CHANGE` answer yes
|
|
28105
|
-
* and name the repair, so they hold the gate shut. The other three do not: a
|
|
28106
|
-
* suspected product bug or a dependency on data the source does not decide is
|
|
28107
|
-
* settled by running the case, not by refusing to run it, and `UNKNOWN` says
|
|
28108
|
-
* the audit could not tell.
|
|
28109
|
-
*/
|
|
28110
|
-
function driftSeverity(label) {
|
|
28111
|
-
return label === "TEST_DRIFT" || label === "SPEC_CHANGE" ? "error" : "warn";
|
|
28112
|
-
}
|
|
28113
|
-
//#endregion
|
|
28114
27936
|
//#region src/drift/analyze.ts
|
|
28115
27937
|
const DEFAULT_CONCURRENCY$1 = 3;
|
|
28116
27938
|
/**
|
|
@@ -28217,7 +28039,7 @@ async function checkSpec(target, opts) {
|
|
|
28217
28039
|
correctSurface(drift, {
|
|
28218
28040
|
cwd: opts.cwd,
|
|
28219
28041
|
sourceRoots: opts.sourceRoots.map((r) => r.abs),
|
|
28220
|
-
...await caseWriteArea(target, opts.
|
|
28042
|
+
...await caseWriteArea(target, opts.context)
|
|
28221
28043
|
});
|
|
28222
28044
|
}
|
|
28223
28045
|
return {
|
|
@@ -28225,7 +28047,8 @@ async function checkSpec(target, opts) {
|
|
|
28225
28047
|
ok: true,
|
|
28226
28048
|
drift,
|
|
28227
28049
|
live: artifacts.live,
|
|
28228
|
-
title: artifacts.title
|
|
28050
|
+
title: artifacts.title,
|
|
28051
|
+
documentPath: artifacts.intent.path
|
|
28229
28052
|
};
|
|
28230
28053
|
} catch (e) {
|
|
28231
28054
|
lastError = `failed to parse drift reply: ${e.message}`;
|
|
@@ -28297,7 +28120,7 @@ function renderGithub(results, cwd) {
|
|
|
28297
28120
|
const repoRoot = process.env["GITHUB_WORKSPACE"] ?? process.cwd();
|
|
28298
28121
|
const lines = [];
|
|
28299
28122
|
for (const r of results) {
|
|
28300
|
-
const file = githubRelPath(cwd, repoRoot, r
|
|
28123
|
+
const file = githubRelPath(cwd, repoRoot, r);
|
|
28301
28124
|
if (r.error) {
|
|
28302
28125
|
lines.push(`::error file=${file}::${escapeGhMessage(r.error)}`);
|
|
28303
28126
|
continue;
|
|
@@ -28310,8 +28133,15 @@ function renderGithub(results, cwd) {
|
|
|
28310
28133
|
}
|
|
28311
28134
|
return lines.length === 0 ? "" : `${lines.join("\n")}\n`;
|
|
28312
28135
|
}
|
|
28313
|
-
|
|
28314
|
-
|
|
28136
|
+
/**
|
|
28137
|
+
* The file an annotation points at: the document this case is stated in.
|
|
28138
|
+
*
|
|
28139
|
+
* Falls back to the case's directory when the audit never got far enough to
|
|
28140
|
+
* read one — an annotation on a directory still lands a reviewer in the right
|
|
28141
|
+
* place, and one on a `spec.yaml` that a project does not use lands nowhere.
|
|
28142
|
+
*/
|
|
28143
|
+
function githubRelPath(cwd, repoRoot, result) {
|
|
28144
|
+
const abs = resolve(cwd, result.documentPath ?? join(".ccqa", "features", result.target.featureName, "test-cases", result.target.specName));
|
|
28315
28145
|
const rel = relative(repoRoot, abs);
|
|
28316
28146
|
return rel.startsWith("..") ? abs : rel;
|
|
28317
28147
|
}
|
|
@@ -28357,7 +28187,7 @@ function determineExitCode(results, threshold) {
|
|
|
28357
28187
|
//#endregion
|
|
28358
28188
|
//#region src/drift/to-report.ts
|
|
28359
28189
|
/** Tracks the drift prompt's own version — the two must never drift apart. */
|
|
28360
|
-
const DRIFT_REPORT_PROMPT_VERSION = "
|
|
28190
|
+
const DRIFT_REPORT_PROMPT_VERSION = "11";
|
|
28361
28191
|
/**
|
|
28362
28192
|
* Spec-level status under the given threshold, mirroring determineExitCode's
|
|
28363
28193
|
* per-spec logic (exit-code.ts) but scoped to a single SpecResult.
|
|
@@ -28437,7 +28267,7 @@ async function writeAuditBriefs(input) {
|
|
|
28437
28267
|
async function buildBrief(result, cwd, ctx) {
|
|
28438
28268
|
const drift = result.drift;
|
|
28439
28269
|
const id = caseIdOf(result.target);
|
|
28440
|
-
const test = result.live ? null : await caseTestPath(result.target,
|
|
28270
|
+
const test = result.live ? null : await caseTestPath(result.target, ctx);
|
|
28441
28271
|
return {
|
|
28442
28272
|
case: id,
|
|
28443
28273
|
kind: drift.label,
|
|
@@ -28469,7 +28299,7 @@ async function repairRoute(result, test, cwd, ctx) {
|
|
|
28469
28299
|
const stamp = (await getRecording(caseRefFor(result.target.caseId ?? {
|
|
28470
28300
|
featureName: result.target.featureName,
|
|
28471
28301
|
specName: result.target.specName
|
|
28472
|
-
}, cwd, await caseRecordingPath(result.target,
|
|
28302
|
+
}, cwd, await caseRecordingPath(result.target, ctx) ?? void 0)).catch(() => null))?.generated;
|
|
28473
28303
|
if (!stamp) return {
|
|
28474
28304
|
route: "external",
|
|
28475
28305
|
reason: "no generation stamp: ccqa did not write this test, or it predates the stamp"
|
|
@@ -28738,10 +28568,10 @@ async function runAudit(specPath, opts) {
|
|
|
28738
28568
|
}
|
|
28739
28569
|
const artifactsContext = await loadSpecArtifactsContext(cwd);
|
|
28740
28570
|
const config = artifactsContext.config;
|
|
28741
|
-
const intent = artifactsContext.
|
|
28571
|
+
const intent = artifactsContext.reader.target;
|
|
28742
28572
|
const sourceRoots = await resolveSourceRoots(cwd, config.sourceRoots);
|
|
28743
|
-
let targets = await collectTargets(specPath, cwd,
|
|
28744
|
-
if (targets.length === 0) exitWithNoSpecs(format, "noSpecsFound", `no test cases found under ${intent ? intent.
|
|
28573
|
+
let targets = await collectTargets(specPath, cwd, artifactsContext.reader);
|
|
28574
|
+
if (targets.length === 0) exitWithNoSpecs(format, "noSpecsFound", `no test cases found under ${intent ? intent.module : ".ccqa/features/"}`);
|
|
28745
28575
|
if (format === "text") {
|
|
28746
28576
|
header("audit", specPath ?? `${targets.length} case${targets.length > 1 ? "s" : ""}`);
|
|
28747
28577
|
if (opts.cwd) meta("cwd", cwd);
|
|
@@ -28979,39 +28809,24 @@ function exitWithNoSpecs(format, reason, message) {
|
|
|
28979
28809
|
* The cases this sweep audits.
|
|
28980
28810
|
*
|
|
28981
28811
|
* Which of the two places a project keeps them in is settled by its target,
|
|
28982
|
-
* not guessed at
|
|
28983
|
-
*
|
|
28984
|
-
*
|
|
28985
|
-
* how `ccqa generate` reads the same argument.
|
|
28812
|
+
* not guessed at, and the reader already answers that — so this asks the same
|
|
28813
|
+
* question of both kinds. A case named on the command line is resolved the
|
|
28814
|
+
* same way, which is exactly how `ccqa generate` reads the same argument.
|
|
28986
28815
|
*/
|
|
28987
|
-
async function collectTargets(specPath, cwd,
|
|
28988
|
-
|
|
28816
|
+
async function collectTargets(specPath, cwd, reader) {
|
|
28817
|
+
const asTarget = (id) => reader.target === null ? splitCaseId(id) : {
|
|
28989
28818
|
...splitCaseId(id),
|
|
28990
28819
|
caseId: id
|
|
28991
|
-
}
|
|
28992
|
-
const tree = await listFeatureTree(cwd);
|
|
28820
|
+
};
|
|
28993
28821
|
if (specPath) {
|
|
28994
|
-
const
|
|
28995
|
-
if (
|
|
28996
|
-
error(`
|
|
28822
|
+
const read = await reader.read(specPath);
|
|
28823
|
+
if (read.document === null) {
|
|
28824
|
+
error(read.error ?? `test case not found: ${specPath} (under ${cwd})`);
|
|
28997
28825
|
process.exit(1);
|
|
28998
28826
|
}
|
|
28999
|
-
return [
|
|
29000
|
-
featureName,
|
|
29001
|
-
specName
|
|
29002
|
-
}];
|
|
28827
|
+
return [asTarget(read.id)];
|
|
29003
28828
|
}
|
|
29004
|
-
|
|
29005
|
-
const out = [];
|
|
29006
|
-
for (const feature of tree) for (const spec of feature.specs) {
|
|
29007
|
-
if (!spec.hasSpecFile) continue;
|
|
29008
|
-
const target = {
|
|
29009
|
-
featureName: feature.featureName,
|
|
29010
|
-
specName: spec.specName
|
|
29011
|
-
};
|
|
29012
|
-
if (active.has(specKey(target))) out.push(target);
|
|
29013
|
-
}
|
|
29014
|
-
return out;
|
|
28829
|
+
return (await Promise.all((await reader.list()).map((id) => reader.read(id)))).filter((read) => read.case?.disabled !== true).map((read) => asTarget(read.id));
|
|
29015
28830
|
}
|
|
29016
28831
|
function parseFormat$1(raw) {
|
|
29017
28832
|
const v = raw ?? "text";
|
|
@@ -29124,29 +28939,41 @@ For each test case above, write a 1–2 sentence factual \`summary\` of what it
|
|
|
29124
28939
|
//#endregion
|
|
29125
28940
|
//#region src/spec/spec-changed-at.ts
|
|
29126
28941
|
/**
|
|
29127
|
-
* One `git log` over the
|
|
29128
|
-
*
|
|
28942
|
+
* One `git log` over the cases' own files, walked newest-first. The first time
|
|
28943
|
+
* a case's file appears is that case's last edit.
|
|
28944
|
+
*
|
|
28945
|
+
* Asked about the documents rather than a directory, because where a case is
|
|
28946
|
+
* filed is the project's business: ccqa's own specs live under `.ccqa`, and a
|
|
28947
|
+
* project that writes its own cases keeps them wherever it keeps them.
|
|
29129
28948
|
*
|
|
29130
28949
|
* Best-effort: outside a repository (or with no history) this returns an empty
|
|
29131
28950
|
* map and every caller falls back to what it did before. A missing timestamp
|
|
29132
|
-
* must never make a
|
|
28951
|
+
* must never make a case look fresher than it is.
|
|
29133
28952
|
*/
|
|
29134
|
-
async function
|
|
28953
|
+
async function readCaseChangedAt(cwd, documentPaths) {
|
|
29135
28954
|
const out = /* @__PURE__ */ new Map();
|
|
28955
|
+
const byRelPath = new Map(documentPaths.map((abs) => [toPosix(relative(cwd, abs)), abs]).filter(([rel]) => rel !== "" && !rel.startsWith("../")));
|
|
28956
|
+
const paths = [...byRelPath.keys()];
|
|
28957
|
+
for (let i = 0; i < paths.length; i += PATHSPEC_CHUNK) await readChunk(cwd, paths.slice(i, i + PATHSPEC_CHUNK), byRelPath, out);
|
|
28958
|
+
return out;
|
|
28959
|
+
}
|
|
28960
|
+
const PATHSPEC_CHUNK = 500;
|
|
28961
|
+
async function readChunk(cwd, paths, byRelPath, out) {
|
|
29136
28962
|
let stdout;
|
|
29137
28963
|
try {
|
|
29138
28964
|
({stdout} = await execFileP("git", [
|
|
29139
28965
|
"log",
|
|
29140
28966
|
"--pretty=format:%x00%cI",
|
|
29141
28967
|
"--name-only",
|
|
28968
|
+
"--relative",
|
|
29142
28969
|
"--",
|
|
29143
|
-
|
|
28970
|
+
...paths
|
|
29144
28971
|
], {
|
|
29145
28972
|
cwd,
|
|
29146
28973
|
maxBuffer: 64 * 1024 * 1024
|
|
29147
28974
|
}));
|
|
29148
28975
|
} catch {
|
|
29149
|
-
return
|
|
28976
|
+
return;
|
|
29150
28977
|
}
|
|
29151
28978
|
let when = "";
|
|
29152
28979
|
for (const line of stdout.split("\n")) {
|
|
@@ -29154,19 +28981,12 @@ async function readSpecChangedAt(cwd) {
|
|
|
29154
28981
|
when = line.slice(1).trim();
|
|
29155
28982
|
continue;
|
|
29156
28983
|
}
|
|
29157
|
-
const
|
|
29158
|
-
if (
|
|
28984
|
+
const abs = byRelPath.get(line.trim());
|
|
28985
|
+
if (abs && when && !out.has(abs)) out.set(abs, when);
|
|
29159
28986
|
}
|
|
29160
|
-
return out;
|
|
29161
28987
|
}
|
|
29162
|
-
|
|
29163
|
-
|
|
29164
|
-
* Every file in a case's directory counts: the generated code moving is as
|
|
29165
|
-
* much a change to the test as the spec.yaml moving.
|
|
29166
|
-
*/
|
|
29167
|
-
function specKeyOf(path) {
|
|
29168
|
-
const m = /^\.ccqa\/features\/([^/]+)\/test-cases\/([^/]+)\//.exec(path);
|
|
29169
|
-
return m ? `${m[1]}/${m[2]}` : null;
|
|
28988
|
+
function toPosix(p) {
|
|
28989
|
+
return p.split(/[\\/]/).join("/");
|
|
29170
28990
|
}
|
|
29171
28991
|
//#endregion
|
|
29172
28992
|
//#region src/cli/perspectives.ts
|
|
@@ -29185,7 +29005,7 @@ async function runPerspectivesCheck(opts) {
|
|
|
29185
29005
|
const hub = requireHubOrExit(opts);
|
|
29186
29006
|
const project = resolveProject(opts);
|
|
29187
29007
|
header("perspectives", `check (project: ${project})`);
|
|
29188
|
-
const skeleton = await buildSkeleton(await
|
|
29008
|
+
const skeleton = await buildSkeleton(await openLocalCases());
|
|
29189
29009
|
const localCount = skeleton.reduce((n, f) => n + f.specs.length, 0);
|
|
29190
29010
|
const existingDoc = await hub.getPerspectives(project);
|
|
29191
29011
|
if (existingDoc === null) {
|
|
@@ -29261,15 +29081,16 @@ async function runPerspectives(opts) {
|
|
|
29261
29081
|
const hub = requireHubOrExit(opts);
|
|
29262
29082
|
const project = resolveProject(opts);
|
|
29263
29083
|
header("perspectives", `project: ${project}`);
|
|
29264
|
-
const
|
|
29084
|
+
const local = await openLocalCases();
|
|
29085
|
+
const skeleton = await buildSkeleton(local);
|
|
29265
29086
|
const allSpecs = skeleton.flatMap((f) => f.specs);
|
|
29266
29087
|
if (allSpecs.length === 0) {
|
|
29267
|
-
info(
|
|
29088
|
+
info(`no test cases found under ${local.where} — nothing to inventory.`);
|
|
29268
29089
|
return;
|
|
29269
29090
|
}
|
|
29270
29091
|
const existingDoc = await hub.getPerspectives(project);
|
|
29271
29092
|
const noteMap = extractNotes(existingDoc);
|
|
29272
|
-
const specBodies =
|
|
29093
|
+
const specBodies = loadSpecBodies(skeleton, local);
|
|
29273
29094
|
meta("language", opts.language ?? "auto");
|
|
29274
29095
|
info(`Summarising ${allSpecs.length} test case(s) across ${skeleton.length} feature(s)...`);
|
|
29275
29096
|
const summaries = await requestSummaries(specBodies, opts);
|
|
@@ -29313,40 +29134,50 @@ async function cleanupLegacyLocalFiles() {
|
|
|
29313
29134
|
for (const path of removed) meta("removed legacy file", path);
|
|
29314
29135
|
}
|
|
29315
29136
|
/**
|
|
29316
|
-
* Turn the
|
|
29317
|
-
* transcribed from each
|
|
29318
|
-
* artifacts
|
|
29319
|
-
*
|
|
29320
|
-
*/
|
|
29321
|
-
async function buildSkeleton(
|
|
29322
|
-
const
|
|
29323
|
-
const
|
|
29324
|
-
|
|
29325
|
-
const
|
|
29326
|
-
|
|
29327
|
-
|
|
29328
|
-
|
|
29329
|
-
|
|
29330
|
-
|
|
29331
|
-
|
|
29332
|
-
|
|
29333
|
-
|
|
29334
|
-
|
|
29335
|
-
|
|
29336
|
-
|
|
29337
|
-
|
|
29338
|
-
...meta.disabled ? { disabled: true } : {}
|
|
29339
|
-
};
|
|
29340
|
-
}));
|
|
29341
|
-
return {
|
|
29342
|
-
featureName: feature.featureName,
|
|
29343
|
-
specs: built
|
|
29137
|
+
* Turn the project's cases into the skeleton perspectives features: title
|
|
29138
|
+
* transcribed from each case, status derived mechanically from on-disk
|
|
29139
|
+
* artifacts, grouped by the feature half of the case's id. `summary` is left
|
|
29140
|
+
* empty here; Claude fills it later.
|
|
29141
|
+
*/
|
|
29142
|
+
async function buildSkeleton(local) {
|
|
29143
|
+
const changedAt = await readCaseChangedAt(process.cwd(), local.cases.map((c) => c.document.path));
|
|
29144
|
+
const byFeature = /* @__PURE__ */ new Map();
|
|
29145
|
+
for (const read of local.cases) {
|
|
29146
|
+
const { featureName, specName } = splitCaseId(read.id);
|
|
29147
|
+
const testCase = read.case;
|
|
29148
|
+
if (testCase === null) warn(`${read.id}: ${read.error ?? "could not be read"}`);
|
|
29149
|
+
const steps = testCase ? transcribeSteps(testCase.spec?.steps ?? testCase.steps) : [];
|
|
29150
|
+
const lastEdit = changedAt.get(read.document.path);
|
|
29151
|
+
const built = {
|
|
29152
|
+
specName,
|
|
29153
|
+
title: testCase?.title ?? specName,
|
|
29154
|
+
summary: "",
|
|
29155
|
+
...steps.length > 0 ? { steps } : {},
|
|
29156
|
+
status: await deriveStatus(read.id, testCase, local),
|
|
29157
|
+
...lastEdit ? { changedAt: lastEdit } : {},
|
|
29158
|
+
...testCase?.disabled ? { disabled: true } : {}
|
|
29344
29159
|
};
|
|
29345
|
-
|
|
29346
|
-
|
|
29347
|
-
|
|
29160
|
+
const specs = byFeature.get(featureName);
|
|
29161
|
+
if (specs) specs.push(built);
|
|
29162
|
+
else byFeature.set(featureName, [built]);
|
|
29163
|
+
}
|
|
29164
|
+
return [...byFeature.entries()].map(([featureName, specs]) => ({
|
|
29165
|
+
featureName,
|
|
29166
|
+
specs: [...specs].sort((a, b) => a.specName.localeCompare(b.specName))
|
|
29348
29167
|
})).sort((a, b) => a.featureName.localeCompare(b.featureName));
|
|
29349
29168
|
}
|
|
29169
|
+
async function openLocalCases() {
|
|
29170
|
+
const cwd = process.cwd();
|
|
29171
|
+
const config = await loadProjectConfig(cwd).catch(() => ProjectConfigSchema.parse({}));
|
|
29172
|
+
const reader = openCaseReader(config, cwd);
|
|
29173
|
+
const ids = await reader.list();
|
|
29174
|
+
return {
|
|
29175
|
+
cases: (await Promise.all(ids.map((id) => reader.read(id)))).filter((r) => r.document !== null),
|
|
29176
|
+
config,
|
|
29177
|
+
reader,
|
|
29178
|
+
where: reader.target ? reader.target.module : ".ccqa/features"
|
|
29179
|
+
};
|
|
29180
|
+
}
|
|
29350
29181
|
/**
|
|
29351
29182
|
* `(featureName, specName)` → human note, extracted from the hub's current
|
|
29352
29183
|
* perspectives document. Notes are preserved across regeneration; everything
|
|
@@ -29404,39 +29235,16 @@ function noteKey(featureName, specName) {
|
|
|
29404
29235
|
return `${featureName}/${specName}`;
|
|
29405
29236
|
}
|
|
29406
29237
|
/**
|
|
29407
|
-
*
|
|
29408
|
-
*
|
|
29409
|
-
*
|
|
29410
|
-
|
|
29411
|
-
|
|
29412
|
-
|
|
29413
|
-
|
|
29414
|
-
|
|
29415
|
-
|
|
29416
|
-
|
|
29417
|
-
};
|
|
29418
|
-
if (specYaml === null) return defaults;
|
|
29419
|
-
try {
|
|
29420
|
-
const parsed = parse(specYaml);
|
|
29421
|
-
const title = typeof parsed.title === "string" && parsed.title.length > 0 ? parsed.title : specName;
|
|
29422
|
-
const modeResult = SpecModeSchema.safeParse(parsed.mode);
|
|
29423
|
-
return {
|
|
29424
|
-
title,
|
|
29425
|
-
mode: modeResult.success ? modeResult.data : DEFAULT_SPEC_MODE,
|
|
29426
|
-
steps: transcribeSteps(parsed.steps),
|
|
29427
|
-
disabled: tryParseTestSpec(specYaml)?.disabled === true
|
|
29428
|
-
};
|
|
29429
|
-
} catch {
|
|
29430
|
-
return defaults;
|
|
29431
|
-
}
|
|
29432
|
-
}
|
|
29433
|
-
/**
|
|
29434
|
-
* The spec's procedure, copied verbatim for the inventory: an include step
|
|
29435
|
-
* keeps only the block name (its params are wiring, not procedure), an
|
|
29436
|
-
* action step keeps its instruction/expected text, a judge step its claim.
|
|
29437
|
-
* Anything malformed is
|
|
29438
|
-
* skipped — the inventory never fails over one bad step, matching how the
|
|
29439
|
-
* rest of this sweep treats a broken spec.
|
|
29238
|
+
* The case's procedure, copied verbatim for the inventory: an include step
|
|
29239
|
+
* keeps only the block name (its params are wiring, not procedure), an action
|
|
29240
|
+
* step keeps its instruction/expected text, a judge step its claim.
|
|
29241
|
+
*
|
|
29242
|
+
* Read off the document rather than off the expanded steps, so a `spec.yaml`
|
|
29243
|
+
* case still shows the block it includes rather than that block's contents —
|
|
29244
|
+
* the inventory is a stock-take of what each case says, and inlining a shared
|
|
29245
|
+
* login into forty cases says the same thing forty times.
|
|
29246
|
+
*
|
|
29247
|
+
* Anything malformed is skipped: the inventory never fails over one bad step.
|
|
29440
29248
|
*/
|
|
29441
29249
|
function transcribeSteps(raw) {
|
|
29442
29250
|
if (!Array.isArray(raw)) return [];
|
|
@@ -29454,41 +29262,37 @@ function transcribeSteps(raw) {
|
|
|
29454
29262
|
return steps;
|
|
29455
29263
|
}
|
|
29456
29264
|
/**
|
|
29457
|
-
*
|
|
29458
|
-
*
|
|
29459
|
-
*
|
|
29460
|
-
*
|
|
29265
|
+
* The target that owns this case's generated files.
|
|
29266
|
+
*
|
|
29267
|
+
* A case the project states itself belongs to the target that declared the
|
|
29268
|
+
* source; a `spec.yaml` case names its own. Best-effort: an unresolvable
|
|
29269
|
+
* target falls back to agent-browser, so the inventory never fails over one
|
|
29270
|
+
* bad case.
|
|
29461
29271
|
*/
|
|
29462
|
-
function
|
|
29463
|
-
if (
|
|
29464
|
-
|
|
29465
|
-
if (!spec) return null;
|
|
29272
|
+
function targetOf(testCase, local) {
|
|
29273
|
+
if (local.reader.target) return registryFor(local.config).get(local.reader.target.id) ?? null;
|
|
29274
|
+
if (testCase?.spec == null) return null;
|
|
29466
29275
|
try {
|
|
29467
|
-
return resolveTarget(spec, config);
|
|
29276
|
+
return resolveTarget(testCase.spec, local.config);
|
|
29468
29277
|
} catch {
|
|
29469
29278
|
return null;
|
|
29470
29279
|
}
|
|
29471
29280
|
}
|
|
29472
29281
|
/**
|
|
29473
|
-
* Coverage facts for one
|
|
29474
|
-
* `PerspectiveStatusSchema`).
|
|
29475
|
-
* target (also the fallback for a spec whose target couldn't be resolved) —
|
|
29476
|
-
* every caller must resolve it (via `resolveSpecTarget`) and pass it, so the
|
|
29477
|
-
* write path and the `--check` comparison agree on the target dimension.
|
|
29282
|
+
* Coverage facts for one case, interpreted through its target (see
|
|
29283
|
+
* `PerspectiveStatusSchema`).
|
|
29478
29284
|
*/
|
|
29479
|
-
async function deriveStatus(
|
|
29285
|
+
async function deriveStatus(id, testCase, local) {
|
|
29480
29286
|
const cwd = process.cwd();
|
|
29481
|
-
const
|
|
29482
|
-
featureName,
|
|
29483
|
-
specName
|
|
29484
|
-
};
|
|
29287
|
+
const plugin = targetOf(testCase, local);
|
|
29485
29288
|
const target = plugin ?? agentBrowserTarget;
|
|
29486
|
-
const targetConfig = targetConfigFor(config, target.id);
|
|
29487
|
-
const generated = await exists(
|
|
29488
|
-
const hasRecording = await findRecordingPath(caseRefFor(
|
|
29289
|
+
const targetConfig = local.reader.target?.targetConfig ?? targetConfigFor(local.config, target.id);
|
|
29290
|
+
const generated = await exists(resolve(cwd, resolveCaseTestPath(target, targetConfig, id)));
|
|
29291
|
+
const hasRecording = await findRecordingPath(caseRefFor(id, cwd, resolveCaseRecordingPath(target, targetConfig, id))) !== null;
|
|
29292
|
+
const traced = target.input === "recording" ? hasRecording : true;
|
|
29489
29293
|
return {
|
|
29490
|
-
mode,
|
|
29491
|
-
traced
|
|
29294
|
+
mode: testCase?.mode ?? "deterministic",
|
|
29295
|
+
traced,
|
|
29492
29296
|
generated,
|
|
29493
29297
|
...plugin && plugin.id !== "agent-browser" ? { target: plugin.id } : {}
|
|
29494
29298
|
};
|
|
@@ -29496,15 +29300,18 @@ async function deriveStatus(featureName, specName, mode, plugin, config) {
|
|
|
29496
29300
|
function exists(path) {
|
|
29497
29301
|
return stat(path).then(() => true, () => false);
|
|
29498
29302
|
}
|
|
29499
|
-
|
|
29500
|
-
|
|
29501
|
-
|
|
29502
|
-
|
|
29503
|
-
|
|
29504
|
-
|
|
29505
|
-
|
|
29506
|
-
|
|
29507
|
-
|
|
29303
|
+
/** `splitCaseId` as the two positional arguments `noteKey` takes. */
|
|
29304
|
+
function splitPair(id) {
|
|
29305
|
+
const { featureName, specName } = splitCaseId(id);
|
|
29306
|
+
return [featureName, specName];
|
|
29307
|
+
}
|
|
29308
|
+
function loadSpecBodies(skeleton, local) {
|
|
29309
|
+
const byKey = new Map(local.cases.map((c) => [noteKey(...splitPair(c.id)), c.document?.text ?? ""]));
|
|
29310
|
+
return skeleton.flatMap((feature) => feature.specs.map((spec) => ({
|
|
29311
|
+
featureName: feature.featureName,
|
|
29312
|
+
specName: spec.specName,
|
|
29313
|
+
title: spec.title,
|
|
29314
|
+
specYaml: byKey.get(noteKey(feature.featureName, spec.specName)) ?? ""
|
|
29508
29315
|
})));
|
|
29509
29316
|
}
|
|
29510
29317
|
async function requestSummaries(specs, opts) {
|
|
@@ -29619,7 +29426,7 @@ async function runSelectSpecs(opts) {
|
|
|
29619
29426
|
}
|
|
29620
29427
|
const specs = specsResult.specs;
|
|
29621
29428
|
if (specs.length === 0) {
|
|
29622
|
-
error("
|
|
29429
|
+
error("this project states no test cases");
|
|
29623
29430
|
process.exit(1);
|
|
29624
29431
|
}
|
|
29625
29432
|
if (!changedResult.ok) {
|
|
@@ -31531,18 +31338,21 @@ z.object({
|
|
|
31531
31338
|
//#endregion
|
|
31532
31339
|
//#region src/hub/api/auth.ts
|
|
31533
31340
|
/**
|
|
31534
|
-
*
|
|
31535
|
-
*
|
|
31536
|
-
*
|
|
31537
|
-
*
|
|
31538
|
-
* carry a header), a `?token=` query parameter — see docs/hub-api.md for
|
|
31539
|
-
* the security tradeoff that accepts.
|
|
31341
|
+
* Reads the token from `Authorization: Bearer <token>`, or — on GET only —
|
|
31342
|
+
* from `?token=`. A query token leaks through `Referer`, history and proxy
|
|
31343
|
+
* logs, so it is accepted only where a browser `<a>` can't send a header;
|
|
31344
|
+
* see docs/hub-api.md.
|
|
31540
31345
|
*/
|
|
31541
31346
|
function extractToken(req, url) {
|
|
31542
31347
|
const header = req.headers.authorization;
|
|
31543
31348
|
if (header?.startsWith("Bearer ")) return header.slice(7);
|
|
31349
|
+
if (req.method !== "GET") return null;
|
|
31544
31350
|
return url.searchParams.get("token");
|
|
31545
31351
|
}
|
|
31352
|
+
/**
|
|
31353
|
+
* Constant-time comparison against the hub's bearer token, so response
|
|
31354
|
+
* timing can't be used to guess the token character-by-character.
|
|
31355
|
+
*/
|
|
31546
31356
|
function isValidToken(provided, expected) {
|
|
31547
31357
|
if (provided === null) return false;
|
|
31548
31358
|
const a = Buffer.from(provided);
|