ccqa 1.52.0 → 1.53.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/ccqa.mjs +2267 -1451
- package/dist/{evidence-constants-C425F7ZG.mjs → evidence-constants-BLXD0LRY.mjs} +10 -8
- package/dist/{evidence-constants-BufWx8Bt.cjs → evidence-constants-BkPzGtzI.cjs} +10 -8
- package/dist/hub-client/index.d.cts +1 -1
- package/dist/hub-client/index.d.mts +1 -1
- package/dist/package.json +1 -1
- package/dist/runtime/step-evidence.cjs +1 -1
- package/dist/runtime/step-evidence.d.cts +11 -6
- package/dist/runtime/step-evidence.d.mts +11 -6
- package/dist/runtime/step-evidence.mjs +1 -1
- package/dist/runtime/test-helpers.cjs +1 -1
- package/dist/runtime/test-helpers.mjs +1 -1
- package/package.json +1 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -19,7 +19,7 @@ import { createInterface as createInterface$1 } from "node:readline/promises";
|
|
|
19
19
|
import { createServer } from "node:http";
|
|
20
20
|
import { connect, createServer as createServer$1 } from "node:net";
|
|
21
21
|
import { connect as connect$1 } from "node:tls";
|
|
22
|
-
import { gunzipSync, gzipSync } from "node:zlib";
|
|
22
|
+
import { gunzipSync, gzipSync, inflateRawSync } from "node:zlib";
|
|
23
23
|
import { pipeline } from "node:stream/promises";
|
|
24
24
|
//#region src/spec/yaml-schema.ts
|
|
25
25
|
/**
|
|
@@ -3876,6 +3876,17 @@ async function fireMockPreToolUseHooks(msg, options) {
|
|
|
3876
3876
|
* drift). Centralised so the prompts cannot drift apart on mechanics that
|
|
3877
3877
|
* must stay consistent across commands.
|
|
3878
3878
|
*/
|
|
3879
|
+
/**
|
|
3880
|
+
* `body` inside a fence the body itself cannot close. What prompts embed this
|
|
3881
|
+
* way — the project's guides, a markdown draft — carries fences of its own, and
|
|
3882
|
+
* a three-backtick wrapper around one ends at the first of them: everything
|
|
3883
|
+
* after it then reads as the prompt's own words.
|
|
3884
|
+
*/
|
|
3885
|
+
function fencedBlock(body) {
|
|
3886
|
+
const longest = Math.max(0, ...[...body.matchAll(/`+/g)].map((m) => m[0].length));
|
|
3887
|
+
const fence = "`".repeat(Math.max(3, longest + 1));
|
|
3888
|
+
return `${fence}\n${body}\n${fence}`;
|
|
3889
|
+
}
|
|
3879
3890
|
/** Prefix every line with its 1-based number, the form fix suggestions cite. */
|
|
3880
3891
|
function numberLines(script) {
|
|
3881
3892
|
return script.split("\n").map((l, i) => `${i + 1}: ${l}`).join("\n");
|
|
@@ -5616,6 +5627,12 @@ const ARTIFACTS_DIR_VAR = "{artifactsDir}";
|
|
|
5616
5627
|
const ARTIFACTS_DIR_ENV = "CCQA_ARTIFACTS_DIR";
|
|
5617
5628
|
/** The runner's full stdout+stderr capture, always written into the artifacts dir. */
|
|
5618
5629
|
const OUTPUT_LOG_FILE = "output.log";
|
|
5630
|
+
/** What Playwright names the trace it writes under the output dir, one per test. */
|
|
5631
|
+
const TRACE_ARCHIVE_FILE = "trace.zip";
|
|
5632
|
+
/** Whether a path under the artifacts dir names one of those traces. */
|
|
5633
|
+
function isTraceArchive(rel) {
|
|
5634
|
+
return rel === TRACE_ARCHIVE_FILE || rel.endsWith(`/${TRACE_ARCHIVE_FILE}`);
|
|
5635
|
+
}
|
|
5619
5636
|
/** `<reportDir>/artifacts/<feature>__<spec>` — created before the runCommand runs. */
|
|
5620
5637
|
function specArtifactsDir(reportDir, feature, spec) {
|
|
5621
5638
|
return join(reportDir, ARTIFACTS_SUBDIR, `${feature}__${spec}`);
|
|
@@ -5628,6 +5645,7 @@ function specArtifactsDir(reportDir, feature, spec) {
|
|
|
5628
5645
|
function substituteArtifactsDir(command, artifactsDir) {
|
|
5629
5646
|
return command.replaceAll(ARTIFACTS_DIR_VAR, quoteForShell(artifactsDir));
|
|
5630
5647
|
}
|
|
5648
|
+
/** Single quotes survive every shell metacharacter except themselves. */
|
|
5631
5649
|
function quoteForShell(s) {
|
|
5632
5650
|
return /^[A-Za-z0-9_./-]+$/.test(s) ? s : `'${s.replaceAll("'", `'\\''`)}'`;
|
|
5633
5651
|
}
|
|
@@ -5665,8 +5683,8 @@ async function collectSpecArtifacts(args) {
|
|
|
5665
5683
|
};
|
|
5666
5684
|
const dir = specArtifactsDir(args.reportDir, args.feature, args.spec);
|
|
5667
5685
|
const relPrefix = posix.join(ARTIFACTS_SUBDIR, `${args.feature}__${args.spec}`);
|
|
5668
|
-
const relFiles = await walkFiles$1(dir
|
|
5669
|
-
relFiles.sort((a, b) => a
|
|
5686
|
+
const relFiles = await walkFiles$1(dir);
|
|
5687
|
+
relFiles.sort((a, b) => rowRank(a) - rowRank(b) || a.localeCompare(b));
|
|
5670
5688
|
const kept = [];
|
|
5671
5689
|
const dropped = [];
|
|
5672
5690
|
let totalBytes = 0;
|
|
@@ -5691,8 +5709,21 @@ async function collectSpecArtifacts(args) {
|
|
|
5691
5709
|
}
|
|
5692
5710
|
return kept;
|
|
5693
5711
|
}
|
|
5712
|
+
/**
|
|
5713
|
+
* Where a file sits in the row order, and so which files the caps cut first.
|
|
5714
|
+
*
|
|
5715
|
+
* A Playwright trace is tens of megabytes and ccqa asks for one on every run
|
|
5716
|
+
* it can — taken before the rest of the budget it would be the whole of it,
|
|
5717
|
+
* and the screenshots and reports a reader actually opens would be the files
|
|
5718
|
+
* dropped. Ranked last it takes only what is left, and it is still on disk and
|
|
5719
|
+
* in the pushed bundle either way.
|
|
5720
|
+
*/
|
|
5721
|
+
function rowRank(rel) {
|
|
5722
|
+
if (rel === "output.log") return 0;
|
|
5723
|
+
return isTraceArchive(rel) ? 2 : 1;
|
|
5724
|
+
}
|
|
5694
5725
|
/** All files under `dir` as posix paths relative to it; missing dir → []. */
|
|
5695
|
-
async function walkFiles$1(dir, relBase) {
|
|
5726
|
+
async function walkFiles$1(dir, relBase = "") {
|
|
5696
5727
|
let entries;
|
|
5697
5728
|
try {
|
|
5698
5729
|
entries = await readdir(dir, { withFileTypes: true });
|
|
@@ -6361,12 +6392,28 @@ function askCdpUrl(session) {
|
|
|
6361
6392
|
* which step, and a format that only one side knew would silently produce a
|
|
6362
6393
|
* table with every row empty.
|
|
6363
6394
|
*/
|
|
6364
|
-
/**
|
|
6365
|
-
|
|
6366
|
-
|
|
6367
|
-
|
|
6368
|
-
|
|
6369
|
-
|
|
6395
|
+
/**
|
|
6396
|
+
* `step: step-01 [case]` — the identifier form, for a `spec.yaml` step.
|
|
6397
|
+
*
|
|
6398
|
+
* Anchored, and the bracket is read only as part of it: a cited step's text is
|
|
6399
|
+
* prose, and prose holds brackets of its own ("click the [Save] button") that
|
|
6400
|
+
* would otherwise be read as the step's source.
|
|
6401
|
+
*/
|
|
6402
|
+
const ID_FORM = /^\s*step:\s*(\S+?)\s*\[([^\]]+)\]/;
|
|
6403
|
+
/** `step 1: <text>` / `cleanup 1: <text>` — the English cited form. */
|
|
6404
|
+
const CITED_EN = /^\s*(step|cleanup)\s+(\d+)\s*:/;
|
|
6405
|
+
/** `1. <text>` / `後処理 1. <text>` — the Japanese cited form. */
|
|
6406
|
+
const CITED_JA = /^\s*(後処理\s*)?(\d+)\s*[..]\s/;
|
|
6407
|
+
/**
|
|
6408
|
+
* `await test.step("step 1: ...", async () => {` — the label as a step title.
|
|
6409
|
+
*
|
|
6410
|
+
* The title is the same label the comment form carries, so both are read by
|
|
6411
|
+
* the one parser below. The quote is captured and required back so an
|
|
6412
|
+
* apostrophe inside the title cannot end it early. Global so it can be run
|
|
6413
|
+
* over a whole file; `matchAll` and `replace` both leave `lastIndex` alone, so
|
|
6414
|
+
* the single instance is safe to share.
|
|
6415
|
+
*/
|
|
6416
|
+
const STEP_CALL = /\btest\.step\(\s*(["'`])((?:\\.|(?!\1).)*)\1/g;
|
|
6370
6417
|
/** Sources whose steps come from the project's own case document. */
|
|
6371
6418
|
const CASE_SOURCES = new Set(["case", "cleanup"]);
|
|
6372
6419
|
/** `step-03` → 3; `cleanup-01` → 1. Absent when the id is not numbered. */
|
|
@@ -6379,26 +6426,88 @@ function stepNumber(stepId) {
|
|
|
6379
6426
|
* no effect on a `spec.yaml` step, whose comment is an identifier either way.
|
|
6380
6427
|
*/
|
|
6381
6428
|
function renderStepComment(marker, japanese = false) {
|
|
6429
|
+
return `// ${renderStepLabel(marker, japanese)}`;
|
|
6430
|
+
}
|
|
6431
|
+
/**
|
|
6432
|
+
* The same label without the comment marker, for a `test.step` title.
|
|
6433
|
+
*
|
|
6434
|
+
* One renderer for both placements: the title is what the Playwright report
|
|
6435
|
+
* shows and what the evidence table reads back, and a second wording would
|
|
6436
|
+
* mean a test whose report and whose table disagree about a step's name.
|
|
6437
|
+
*/
|
|
6438
|
+
function renderStepLabel(marker, japanese = false) {
|
|
6382
6439
|
const number = CASE_SOURCES.has(marker.source) ? stepNumber(marker.stepId) : null;
|
|
6383
6440
|
const text = marker.text?.trim().split("\n")[0]?.trim();
|
|
6384
|
-
if (number === null || !text) return
|
|
6441
|
+
if (number === null || !text) return `step: ${marker.stepId} [${marker.source}]`;
|
|
6385
6442
|
const isCleanup = marker.source === "cleanup";
|
|
6386
|
-
if (japanese) return
|
|
6387
|
-
return
|
|
6443
|
+
if (japanese) return `${isCleanup ? "後処理 " : ""}${number}. ${text}`;
|
|
6444
|
+
return `${isCleanup ? "cleanup" : "step"} ${number}: ${text}`;
|
|
6388
6445
|
}
|
|
6389
|
-
/**
|
|
6446
|
+
/**
|
|
6447
|
+
* The step id a line opens, or null when it opens none. Both placements are
|
|
6448
|
+
* accepted — a `test.step` title for a Playwright test, a comment for the
|
|
6449
|
+
* agent-browser emitter and for every test generated before the titles.
|
|
6450
|
+
*/
|
|
6390
6451
|
function parseStepComment(line) {
|
|
6391
|
-
const
|
|
6392
|
-
if (
|
|
6393
|
-
const
|
|
6394
|
-
|
|
6395
|
-
|
|
6396
|
-
|
|
6452
|
+
const block = parseStepBlock(line);
|
|
6453
|
+
if (block !== null) return block;
|
|
6454
|
+
const comment = /^\s*\/\/\s*(.*)$/.exec(line);
|
|
6455
|
+
return comment ? parseStepLabel(comment[1])?.stepId ?? null : null;
|
|
6456
|
+
}
|
|
6457
|
+
/**
|
|
6458
|
+
* A file's lines, with a `test.step` header a formatter broke across lines
|
|
6459
|
+
* folded back onto the line it opens.
|
|
6460
|
+
*
|
|
6461
|
+
* Both readers of a step boundary walk lines — the evidence table attributes
|
|
6462
|
+
* the assertions under one, the generation gate collects the ids a file titles
|
|
6463
|
+
* — and neither can see a call whose title sits on the next line. Prettier
|
|
6464
|
+
* wraps `test.step` like any other call (it is not in its test-call
|
|
6465
|
+
* allowlist), and a line walk over the wrapped form finds no steps at all:
|
|
6466
|
+
* every step then reads as deciding nothing and every rewrite is rejected.
|
|
6467
|
+
*/
|
|
6468
|
+
function stepLines(source) {
|
|
6469
|
+
return source.replace(STEP_CALL, (call) => call.replace(/\s*\n\s*/g, " ")).split("\n");
|
|
6470
|
+
}
|
|
6471
|
+
/**
|
|
6472
|
+
* The step id a `test.step` title names — the block form only.
|
|
6473
|
+
*
|
|
6474
|
+
* Split out from {@link parseStepComment} because reading and writing want
|
|
6475
|
+
* different strictness. Anything ccqa has ever generated has to stay readable,
|
|
6476
|
+
* so attribution accepts the comment form too; a file ccqa writes today has no
|
|
6477
|
+
* reason to be in it, so the generation gate asks this narrower question and
|
|
6478
|
+
* rejects a rewrite that fell back to comments.
|
|
6479
|
+
*/
|
|
6480
|
+
function parseStepBlock(line) {
|
|
6481
|
+
const [call] = line.matchAll(STEP_CALL);
|
|
6482
|
+
return call ? parseStepLabel(call[2])?.stepId ?? null : null;
|
|
6483
|
+
}
|
|
6484
|
+
/**
|
|
6485
|
+
* The step a rendered label names, or null when it names none — the single
|
|
6486
|
+
* inverse of {@link renderStepLabel}, wherever a label is read back from: a
|
|
6487
|
+
* comment, a `test.step` title in the file, or the same title as a Playwright
|
|
6488
|
+
* trace recorded it.
|
|
6489
|
+
*/
|
|
6490
|
+
function parseStepLabel(label) {
|
|
6491
|
+
const byId = ID_FORM.exec(label);
|
|
6492
|
+
if (byId) return {
|
|
6493
|
+
stepId: byId[1],
|
|
6494
|
+
source: byId[2]
|
|
6495
|
+
};
|
|
6496
|
+
const en = CITED_EN.exec(label);
|
|
6497
|
+
if (en) return labelFor(en[1] === "cleanup", Number.parseInt(en[2], 10));
|
|
6498
|
+
const ja = CITED_JA.exec(label);
|
|
6499
|
+
if (ja) return labelFor(ja[1] !== void 0, Number.parseInt(ja[2], 10));
|
|
6397
6500
|
return null;
|
|
6398
6501
|
}
|
|
6399
6502
|
/** Built by the same functions that named the step, so the two cannot drift. */
|
|
6400
|
-
function
|
|
6401
|
-
return cleanup ?
|
|
6503
|
+
function labelFor(cleanup, number) {
|
|
6504
|
+
return cleanup ? {
|
|
6505
|
+
stepId: cleanupId(number),
|
|
6506
|
+
source: "cleanup"
|
|
6507
|
+
} : {
|
|
6508
|
+
stepId: stepId(number),
|
|
6509
|
+
source: "case"
|
|
6510
|
+
};
|
|
6402
6511
|
}
|
|
6403
6512
|
//#endregion
|
|
6404
6513
|
//#region src/codegen/actions-to-script.ts
|
|
@@ -7801,1314 +7910,1519 @@ function renderTitleTag(tags, values) {
|
|
|
7801
7910
|
return mapped === void 0 ? "" : ` ${tags.format.replaceAll("{value}", mapped)}`;
|
|
7802
7911
|
}
|
|
7803
7912
|
//#endregion
|
|
7804
|
-
//#region src/
|
|
7805
|
-
/**
|
|
7806
|
-
* Tool-neutral intermediate representation (IR) for recorded browser
|
|
7807
|
-
* interactions. A trace produces `RecordedAction[]` (persisted as `ir.json`);
|
|
7808
|
-
* emitters translate it into target-specific test code and the replay
|
|
7809
|
-
* validator re-executes it. The IR replaces the old dual addressing
|
|
7810
|
-
* (raw selector strings vs `find_*` + findLocator fields) with a single
|
|
7811
|
-
* `Locator` model.
|
|
7812
|
-
*/
|
|
7813
|
-
/**
|
|
7814
|
-
* The step id the recorder marks work that precedes the case's first step —
|
|
7815
|
-
* signing in, reaching the starting screen. Not a step of the case, and not
|
|
7816
|
-
* the same thing as an action it could not attribute at all.
|
|
7817
|
-
*/
|
|
7818
|
-
const SETUP_STEP_ID = "setup";
|
|
7819
|
-
//#endregion
|
|
7820
|
-
//#region src/ir/route-diff.ts
|
|
7913
|
+
//#region src/prompts/llm-gen.ts
|
|
7821
7914
|
/**
|
|
7822
|
-
*
|
|
7823
|
-
*
|
|
7824
|
-
*
|
|
7825
|
-
*
|
|
7826
|
-
*
|
|
7827
|
-
* says the copy moved; a step that gained operations says the flow grew. That
|
|
7828
|
-
* is the evidence a reviewer needs, and reading it out of two JSON files by eye
|
|
7829
|
-
* is exactly the work nobody does.
|
|
7830
|
-
*
|
|
7831
|
-
* The diff is per spec step, because a step is the unit the spec author wrote
|
|
7832
|
-
* and the unit a reviewer reasons in. Actions carry their `stepId` from the
|
|
7833
|
-
* trace, so the grouping is recorded, not guessed.
|
|
7915
|
+
* The reuse-first contract — the rules that make generated tests build on the
|
|
7916
|
+
* consumer repo's existing assets instead of re-implementing them. `draft*`
|
|
7917
|
+
* rules apply only when a mechanical draft is present; `draftInvariant`, when
|
|
7918
|
+
* supplied by the target, adds one more "do not drop this from the draft" rule
|
|
7919
|
+
* (e.g. Playwright's `test.step` blocks).
|
|
7834
7920
|
*/
|
|
7835
|
-
|
|
7836
|
-
|
|
7837
|
-
|
|
7838
|
-
|
|
7839
|
-
|
|
7840
|
-
|
|
7841
|
-
if (
|
|
7842
|
-
|
|
7843
|
-
|
|
7844
|
-
|
|
7921
|
+
function reuseFirstContract(hasDraft, draftInvariant) {
|
|
7922
|
+
const rules = [
|
|
7923
|
+
"1. **Reuse first.** Before writing any code, search the declared resources (path resources with Glob/Grep/Read; package resources under their resolved roots) and import every asset you can use: page objects, step helpers, fixtures, custom selectors, shared constants and regular expressions. Re-implementing an equivalent of an existing asset — including copy-pasting a constant or a regex — is forbidden.",
|
|
7924
|
+
"2. **Create missing parts as support files.** When a needed part does not exist (e.g. a page object for a screen no resource covers), create it following the conventions guides and the style of the existing resources, and output it with `\"kind\": \"support\"`. Package resources are read-only: never modify their contents — if a package lacks something, create the missing piece as a new support file in the repo and mention it in `summary`.",
|
|
7925
|
+
"3. **Match the examples.** Write in the same style as the conventions examples: naming, structure, assertion phrasing, import layout. Where you follow an existing file rather than the plain reading of a guideline — a waiting pattern, a locator shape, a teardown structure — say which file, by path, in the comment. \"Following the existing implementation\" without naming it cannot be checked by the next reader, and it is how a pattern that was right once outlives the reason for it."
|
|
7926
|
+
];
|
|
7927
|
+
if (hasDraft) {
|
|
7928
|
+
rules.push("4. **The draft is ground truth.** The mechanical draft below encodes the recorded route. Do not change the meaning of its operation sequence or its assertions — no reordering, dropping, or weakening. You may rewrite locators when the conventions or the sources you read justify a better one.");
|
|
7929
|
+
if (draftInvariant) rules.push(`5. ${draftInvariant}`);
|
|
7930
|
+
}
|
|
7931
|
+
return `## Reuse contract\n\n${rules.join("\n")}`;
|
|
7845
7932
|
}
|
|
7846
7933
|
/**
|
|
7847
|
-
* The
|
|
7848
|
-
*
|
|
7849
|
-
* positions in a list.
|
|
7934
|
+
* The JSON output contract shared by every LLM generation pass. Mirrors the
|
|
7935
|
+
* cleanup prompt's "output ONLY JSON" style so parsing stays uniform.
|
|
7850
7936
|
*/
|
|
7851
|
-
function
|
|
7852
|
-
return
|
|
7937
|
+
function outputContract(testPath, writeRoots) {
|
|
7938
|
+
return `## Output format
|
|
7939
|
+
|
|
7940
|
+
When you are done exploring, reply with ONLY a JSON object (no explanation, no markdown code fences):
|
|
7941
|
+
|
|
7942
|
+
{"files": [{"path": "<project-root-relative path>", "contents": "<full file contents>", "kind": "test" | "support"}], "summary": "<one-paragraph human-readable summary>"}
|
|
7943
|
+
|
|
7944
|
+
- \`kind\` MUST be exactly \`"test"\` or \`"support"\` (no other value): \`"test"\` marks an executable test; \`"support"\` marks a companion file (page object, helper, ...).
|
|
7945
|
+
- The \`"kind": "test"\` file goes to exactly \`${testPath}\`. That path is the project's, not yours: never move, rename, or split the test.
|
|
7946
|
+
- Every \`"kind": "support"\` file must stay under one of: ${[posix.dirname(testPath), ...writeRoots].map((r) => `\`${r}\``).join(", ")}.
|
|
7947
|
+
- Those roots are also yours to **correct**, not only to add to. A page object
|
|
7948
|
+
there that this case uses, and whose locator or waiting is wrong for what
|
|
7949
|
+
this case needs, is output again with the fix — as \`"kind": "support"\`, whole.
|
|
7950
|
+
Reusing something is not the same as leaving it as it is; anywhere else, and
|
|
7951
|
+
anything you only read, stays untouched.
|
|
7952
|
+
- Absolute paths, \`..\` segments, and anything under \`node_modules/\` are rejected.
|
|
7953
|
+
- Emit the complete contents of every file you output — no placeholders or elisions.`;
|
|
7954
|
+
}
|
|
7955
|
+
/** Corrective note appended when the previous reply failed the output contract. */
|
|
7956
|
+
function retryNote(error) {
|
|
7957
|
+
return `\n\n## Previous attempt rejected\n\nYour previous reply violated the output contract: ${error}\nReply again with ONLY the JSON object described in "Output format".`;
|
|
7958
|
+
}
|
|
7959
|
+
function formatStep(step) {
|
|
7960
|
+
if (isExpandedActionStep(step)) return `- ${step.id}: ${body(step.instruction)}\n expected: ${body(step.expected)}`;
|
|
7961
|
+
const source = step.from ? ` (read from \`${step.from}\`)` : "";
|
|
7962
|
+
return `- ${step.id}: judge by LLM${source}\n claim: ${body(step.judgeByLlm)}`;
|
|
7853
7963
|
}
|
|
7854
7964
|
/**
|
|
7855
|
-
*
|
|
7856
|
-
*
|
|
7857
|
-
* element as unchanged — the very thing it exists to show.
|
|
7965
|
+
* Step text is usually a block scalar, so it arrives with newlines. Left as-is
|
|
7966
|
+
* its later lines sit flush against the list and read as further steps.
|
|
7858
7967
|
*/
|
|
7859
|
-
function
|
|
7860
|
-
|
|
7861
|
-
|
|
7862
|
-
|
|
7863
|
-
|
|
7864
|
-
|
|
7968
|
+
function body(text) {
|
|
7969
|
+
return text.trim().split("\n").join("\n ");
|
|
7970
|
+
}
|
|
7971
|
+
function buildLlmGenPrompt(input) {
|
|
7972
|
+
const sections = [];
|
|
7973
|
+
sections.push(input.taskInstructions);
|
|
7974
|
+
const steps = input.steps.map(formatStep).join("\n");
|
|
7975
|
+
sections.push(`## Test spec\n\nTitle: ${input.specTitle}\n\nSteps:\n${steps}`);
|
|
7976
|
+
if (input.draft) sections.push(`## Mechanical draft (recorded ground truth)\n\nPath: ${input.draft.path}\n\n\`\`\`
|
|
7977
|
+
` + input.draft.contents + "\n```");
|
|
7978
|
+
if (input.resources.length > 0) {
|
|
7979
|
+
const lines = input.resources.map((r) => {
|
|
7980
|
+
return `- ${r.kind === "package" ? `npm package \`${r.ref}\`` : `repo code \`${r.ref}\``}${r.description ? ` — ${r.description}` : ""}\n explore under: ${r.root}`;
|
|
7981
|
+
});
|
|
7982
|
+
sections.push(`## Reusable resources\n\nExisting code assets the generated test MUST reuse (import path resources the way this repo does; import package resources by name):\n\n${lines.join("\n")}`);
|
|
7865
7983
|
}
|
|
7866
|
-
|
|
7984
|
+
if (input.conventionSections.length > 0) {
|
|
7985
|
+
const bodies = input.conventionSections.map((c) => `### ${c.path}\n\n${fencedBlock(c.body)}`).join("\n\n");
|
|
7986
|
+
sections.push(`## Conventions
|
|
7987
|
+
|
|
7988
|
+
These are the project's rules for the code you write, not background reading. Code that breaks one of them is defective: the review that follows this pass quotes the rule back at it and sends it here again to be fixed.\n\n${bodies}`);
|
|
7989
|
+
}
|
|
7990
|
+
if (input.promptBundle) sections.push(`## Project prompt guidance\n\n${input.promptBundle}`);
|
|
7991
|
+
sections.push(reuseFirstContract(input.draft !== void 0, input.draftInvariant));
|
|
7992
|
+
sections.push(outputContract(input.testPath, input.writeRoots));
|
|
7993
|
+
return sections.join("\n\n") + languageDirective(input.language);
|
|
7867
7994
|
}
|
|
7868
7995
|
/**
|
|
7869
|
-
*
|
|
7870
|
-
*
|
|
7871
|
-
* addressed a new way still pairs with its old self and reads as "changed"
|
|
7872
|
-
* rather than as one removal plus one addition.
|
|
7996
|
+
* Prompt for one iteration of the verification loop: the generated files
|
|
7997
|
+
* failed their run command; ask for corrected files under the same contract.
|
|
7873
7998
|
*/
|
|
7874
|
-
function
|
|
7999
|
+
function buildLlmFixPrompt(input) {
|
|
8000
|
+
const files = input.files.map((f) => `### ${f.path} (${f.kind})\n\n\`\`\`\n${f.contents}\n\`\`\``).join("\n\n");
|
|
7875
8001
|
return [
|
|
7876
|
-
|
|
7877
|
-
|
|
7878
|
-
|
|
7879
|
-
|
|
7880
|
-
}
|
|
7881
|
-
|
|
7882
|
-
const moved = findMoves(before, after);
|
|
7883
|
-
const movedForms = new Set(moved.map((m) => m.action));
|
|
7884
|
-
const stayed = (a) => !movedForms.has(describeAction(a));
|
|
7885
|
-
const inStep = (actions, step) => actions.filter((a) => (a.stepId ?? NO_STEP) === step && stayed(a));
|
|
7886
|
-
const changes = [];
|
|
7887
|
-
const unchangedSteps = [];
|
|
7888
|
-
for (const step of orderedSteps(before, after)) {
|
|
7889
|
-
const wasIn = inStep(before, step);
|
|
7890
|
-
const isIn = inStep(after, step);
|
|
7891
|
-
if (wasIn.length === 0 && isIn.length === 0) continue;
|
|
7892
|
-
const stepChanges = diffStep(step, wasIn, isIn);
|
|
7893
|
-
if (stepChanges.length === 0) unchangedSteps.push(step);
|
|
7894
|
-
else changes.push(...stepChanges);
|
|
7895
|
-
}
|
|
7896
|
-
return {
|
|
7897
|
-
changes,
|
|
7898
|
-
moved,
|
|
7899
|
-
unchangedSteps
|
|
7900
|
-
};
|
|
8002
|
+
`You generated ${input.targetId} test files for this project, but their verification run failed. Fix the files so the command passes. Keep the tested behaviour identical — fix locators, imports, syntax, and setup, never delete or weaken assertions to force a pass. You may Read/Grep/Glob the project to investigate.`,
|
|
8003
|
+
`## Failing command\n\n\`${input.command}\``,
|
|
8004
|
+
`## Command output (tail)\n\n\`\`\`\n${input.outputTail}\n\`\`\``,
|
|
8005
|
+
`## Current files\n\n${files}`,
|
|
8006
|
+
outputContract(input.testPath, input.writeRoots) + "\n- Output only the files that need changes; files you omit stay as they are.\n- If the failure is NOT caused by the generated files (an environment/setup issue) or the files are already correct, reply `{\"files\": [], \"summary\": \"<why no file change is needed>\"}` — verification is then simply re-run."
|
|
8007
|
+
].join("\n\n") + languageDirective(input.language);
|
|
7901
8008
|
}
|
|
7902
8009
|
/**
|
|
7903
|
-
*
|
|
7904
|
-
*
|
|
7905
|
-
* more than once in either recording is left to the per-step diff — which of
|
|
7906
|
-
* two identical operations moved is a question the recording cannot answer.
|
|
8010
|
+
* Playwright rewrite pass: turn the mechanical draft into a library-reusing
|
|
8011
|
+
* spec. The draft carries the recorded route; the resources carry the assets.
|
|
7907
8012
|
*/
|
|
7908
|
-
function
|
|
7909
|
-
|
|
7910
|
-
|
|
7911
|
-
|
|
7912
|
-
|
|
7913
|
-
|
|
7914
|
-
|
|
7915
|
-
|
|
7916
|
-
}
|
|
7917
|
-
for (const form of repeated) seen.delete(form);
|
|
7918
|
-
return seen;
|
|
7919
|
-
};
|
|
7920
|
-
const b = index(before);
|
|
7921
|
-
const moved = [];
|
|
7922
|
-
for (const [form, action] of index(after)) {
|
|
7923
|
-
const was = b.get(form);
|
|
7924
|
-
if (was === void 0) continue;
|
|
7925
|
-
const from = was.stepId ?? NO_STEP;
|
|
7926
|
-
const to = action.stepId ?? NO_STEP;
|
|
7927
|
-
if (from !== to) moved.push({
|
|
7928
|
-
action: form,
|
|
7929
|
-
from,
|
|
7930
|
-
to
|
|
7931
|
-
});
|
|
7932
|
-
}
|
|
7933
|
-
return moved;
|
|
7934
|
-
}
|
|
7935
|
-
/** Every step id either recording mentions, in the new recording's order first. */
|
|
7936
|
-
function orderedSteps(before, after) {
|
|
7937
|
-
const seen = /* @__PURE__ */ new Set();
|
|
7938
|
-
const out = [];
|
|
7939
|
-
for (const action of [...after, ...before]) {
|
|
7940
|
-
const step = action.stepId ?? NO_STEP;
|
|
7941
|
-
if (seen.has(step)) continue;
|
|
7942
|
-
seen.add(step);
|
|
7943
|
-
out.push(step);
|
|
7944
|
-
}
|
|
7945
|
-
return out;
|
|
8013
|
+
function playwrightTaskInstructions(suggestedPath) {
|
|
8014
|
+
return `You are rewriting a machine-generated Playwright test so it reuses this repository's existing test assets.
|
|
8015
|
+
|
|
8016
|
+
The mechanical draft below was compiled 1:1 from a recorded browser session — it is plain \`@playwright/test\` code with raw locators (e.g. \`page.getByRole("button", { name: "Submit" })\`). Rewrite it into the shape this repository's test suite actually uses: import and call the declared resources (page objects, step helpers, fixtures, shared constants) instead of inlining raw interactions, and follow the conventions.
|
|
8017
|
+
|
|
8018
|
+
Keep each step's assertions in the test body, under that step's own comment. A step's actions may move into a page object; what the step decides may not — it is the one thing a reviewer reads the file for, and an assertion inside a helper is invisible both to them and to the review that follows this pass.
|
|
8019
|
+
|
|
8020
|
+
Write the rewritten test to \`${suggestedPath}\` unless the conventions/examples clearly place tests elsewhere under the output directory. Locator preference when you do write raw locators: test id > accessible text > role > CSS.`;
|
|
7946
8021
|
}
|
|
7947
8022
|
/**
|
|
7948
|
-
*
|
|
7949
|
-
*
|
|
7950
|
-
*
|
|
8023
|
+
* runn generation pass: compile the spec into a runbook. Only the generic
|
|
8024
|
+
* runbook shape is prescribed here — concrete endpoints, payloads, and
|
|
8025
|
+
* response shapes must be verified against the backend sources, found via
|
|
8026
|
+
* Read/Grep/Glob.
|
|
7951
8027
|
*/
|
|
7952
|
-
function
|
|
7953
|
-
|
|
7954
|
-
|
|
7955
|
-
|
|
7956
|
-
|
|
7957
|
-
|
|
7958
|
-
|
|
7959
|
-
|
|
7960
|
-
|
|
7961
|
-
|
|
7962
|
-
|
|
7963
|
-
|
|
7964
|
-
kind: "added",
|
|
7965
|
-
step,
|
|
7966
|
-
after: describeAction(entry.after)
|
|
7967
|
-
});
|
|
7968
|
-
else if (entry.before) changes.push({
|
|
7969
|
-
kind: "removed",
|
|
7970
|
-
step,
|
|
7971
|
-
before: describeAction(entry.before)
|
|
7972
|
-
});
|
|
7973
|
-
return changes;
|
|
8028
|
+
function runnTaskInstructions(suggestedPath) {
|
|
8029
|
+
return `You are generating a runn runbook (YAML) that covers the test spec below against this repository's backend.
|
|
8030
|
+
|
|
8031
|
+
Runbook shape (generic runn structure):
|
|
8032
|
+
- \`desc:\` — a one-line description of the scenario.
|
|
8033
|
+
- \`runners:\` — the runners the steps use (typically an HTTP runner, e.g. \`req: \${API_ENDPOINT}\`; use env-var placeholders for endpoints, never hardcode hosts).
|
|
8034
|
+
- \`vars:\` — variable bindings for values reused across steps (test inputs, ids captured from responses).
|
|
8035
|
+
- \`steps:\` — one entry per spec step where possible: \`req\` steps perform the API calls, \`test\` steps assert on \`current.res\` (status, body fields). Bind values from earlier responses via \`steps[N]\` references when later steps need them.
|
|
8036
|
+
|
|
8037
|
+
Do NOT invent endpoints, request bodies, or response fields: use Read/Grep/Glob to find the backend sources (routing tables, handlers, schemas) and derive the concrete paths, methods, parameters, and response shapes from them. If a detail cannot be confirmed from the sources, say so in \`summary\` instead of guessing silently.
|
|
8038
|
+
|
|
8039
|
+
Write the runbook to \`${suggestedPath}\` unless the conventions/examples clearly use another layout under the output directory. If shared setup deserves its own helper runbook, output it as \`kind: "support"\`.`;
|
|
7974
8040
|
}
|
|
7975
|
-
|
|
7976
|
-
|
|
7977
|
-
|
|
7978
|
-
|
|
7979
|
-
|
|
7980
|
-
|
|
7981
|
-
|
|
7982
|
-
|
|
7983
|
-
|
|
7984
|
-
|
|
7985
|
-
|
|
7986
|
-
|
|
7987
|
-
|
|
7988
|
-
|
|
7989
|
-
|
|
7990
|
-
|
|
7991
|
-
|
|
7992
|
-
|
|
8041
|
+
/**
|
|
8042
|
+
* How many files the walk may collect. Depth alone does not bound it: one
|
|
8043
|
+
* barrel `index.ts` re-exporting a directory fans out to everything in it, and
|
|
8044
|
+
* the caller reads what the walk returns. Far more than an audit's budget can
|
|
8045
|
+
* hold, so the cap only stops the pathological case.
|
|
8046
|
+
*/
|
|
8047
|
+
const MAX_SUPPORT_FILES = 200;
|
|
8048
|
+
/** Extensions tried when a specifier names no file, TypeScript sources first. */
|
|
8049
|
+
const EXTENSIONS = [
|
|
8050
|
+
".ts",
|
|
8051
|
+
".tsx",
|
|
8052
|
+
".mts",
|
|
8053
|
+
".cts",
|
|
8054
|
+
".js",
|
|
8055
|
+
".jsx",
|
|
8056
|
+
".mjs",
|
|
8057
|
+
".cjs"
|
|
8058
|
+
];
|
|
8059
|
+
/**
|
|
8060
|
+
* Module specifiers, in the forms generated tests use: `from "x"`, a
|
|
8061
|
+
* side-effect `import "x"`, `import("x")`, and `require("x")`. Anchored on the
|
|
8062
|
+
* import syntax rather than on the string alone, so a specifier-looking string
|
|
8063
|
+
* in a locator or an asserted text is not one. A string this over-matches
|
|
8064
|
+
* (prose after the word "from" in a comment) resolves to no file and drops out.
|
|
8065
|
+
*/
|
|
8066
|
+
const IMPORT_SPECIFIER = /(?:\bfrom|\bimport\s*\(|\brequire\s*\(|\bimport)\s*(["'`])([^"'`]+)\1/g;
|
|
8067
|
+
/**
|
|
8068
|
+
* Every module specifier `source` loads, in source order. Shared with the gate
|
|
8069
|
+
* over what a generated test may import: which strings count as imports must
|
|
8070
|
+
* not differ between the walk that follows them and the check that refuses
|
|
8071
|
+
* some of them.
|
|
8072
|
+
*/
|
|
8073
|
+
function importedSpecifiers(source) {
|
|
8074
|
+
return [...source.matchAll(IMPORT_SPECIFIER)].map((m) => m[2]);
|
|
7993
8075
|
}
|
|
7994
|
-
/**
|
|
7995
|
-
|
|
7996
|
-
|
|
7997
|
-
|
|
7998
|
-
|
|
7999
|
-
|
|
8000
|
-
|
|
8001
|
-
|
|
8002
|
-
|
|
8003
|
-
|
|
8004
|
-
|
|
8005
|
-
|
|
8006
|
-
|
|
8007
|
-
|
|
8008
|
-
|
|
8009
|
-
|
|
8076
|
+
/**
|
|
8077
|
+
* Read `<cwd>/tsconfig.json`'s `baseUrl` / `paths`, following a relative
|
|
8078
|
+
* `extends` chain — a workspace commonly keeps its aliases in a base config,
|
|
8079
|
+
* and stopping at the leaf would drop exactly the page objects an audit needs.
|
|
8080
|
+
* A config published as a package is not followed: its aliases would resolve
|
|
8081
|
+
* inside `node_modules`, which this walker never reads anyway. Absent,
|
|
8082
|
+
* unreadable, or aliasless: null, and only relative imports are followed.
|
|
8083
|
+
*/
|
|
8084
|
+
async function loadTsconfigPaths(cwd) {
|
|
8085
|
+
let configPath = join(cwd, "tsconfig.json");
|
|
8086
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8087
|
+
while (configPath !== null && !seen.has(configPath)) {
|
|
8088
|
+
seen.add(configPath);
|
|
8089
|
+
const parsed = await readTsconfig(configPath);
|
|
8090
|
+
if (parsed === null) return null;
|
|
8091
|
+
const options = parsed.compilerOptions;
|
|
8092
|
+
if (options?.paths) return {
|
|
8093
|
+
baseUrlAbs: resolve(dirname(configPath), options.baseUrl ?? "."),
|
|
8094
|
+
paths: options.paths
|
|
8095
|
+
};
|
|
8096
|
+
configPath = typeof parsed.extends === "string" && parsed.extends.startsWith(".") ? withJsonExtension(resolve(dirname(configPath), parsed.extends)) : null;
|
|
8010
8097
|
}
|
|
8011
|
-
|
|
8012
|
-
|
|
8013
|
-
|
|
8014
|
-
|
|
8098
|
+
return null;
|
|
8099
|
+
}
|
|
8100
|
+
async function readTsconfig(path) {
|
|
8101
|
+
const raw = await readFile(path, "utf8").catch(() => null);
|
|
8102
|
+
if (raw === null) return null;
|
|
8103
|
+
try {
|
|
8104
|
+
return JSON.parse(stripJsonComments(raw));
|
|
8105
|
+
} catch {
|
|
8106
|
+
return null;
|
|
8015
8107
|
}
|
|
8016
|
-
if (unchangedSteps.length > 0) lines.push(`Unchanged: ${unchangedSteps.join(", ")}`, "");
|
|
8017
|
-
return lines.join("\n");
|
|
8018
8108
|
}
|
|
8019
|
-
|
|
8020
|
-
|
|
8021
|
-
|
|
8022
|
-
|
|
8023
|
-
|
|
8024
|
-
|
|
8025
|
-
|
|
8026
|
-
|
|
8027
|
-
|
|
8109
|
+
/** `extends` may name a config with the extension left off. */
|
|
8110
|
+
function withJsonExtension(path) {
|
|
8111
|
+
return path.endsWith(".json") ? path : `${path}.json`;
|
|
8112
|
+
}
|
|
8113
|
+
/** Strip line and block comments and trailing commas — tsconfig.json is JSONC. */
|
|
8114
|
+
function stripJsonComments(text) {
|
|
8115
|
+
return text.replace(/"(?:[^"\\]|\\.)*"|\/\*[\s\S]*?\*\/|\/\/[^\n]*/g, (match) => match.startsWith("\"") ? match : "").replace(/,(\s*[}\]])/g, "$1");
|
|
8116
|
+
}
|
|
8117
|
+
/**
|
|
8118
|
+
* Files reachable from `entryAbs` by following project-internal imports, in
|
|
8119
|
+
* breadth-first order and excluding the entry itself. Reading a file that
|
|
8120
|
+
* cannot be read yields no imports rather than an error: a missing import
|
|
8121
|
+
* target is the audit's business, not this walker's.
|
|
8122
|
+
*/
|
|
8123
|
+
async function collectSupportFiles(entryAbs, cwd, opts = {}) {
|
|
8124
|
+
return (await walkSupportFiles(entryAbs, cwd, opts)).files;
|
|
8125
|
+
}
|
|
8126
|
+
/** {@link collectSupportFiles}, saying whether a cap cut the walk short. */
|
|
8127
|
+
async function walkSupportFiles(entryAbs, cwd, opts = {}) {
|
|
8128
|
+
const maxDepth = opts.maxDepth ?? 3;
|
|
8129
|
+
const tsconfig = opts.tsconfig !== void 0 ? opts.tsconfig : await loadTsconfigPaths(cwd);
|
|
8130
|
+
const seen = new Set([entryAbs]);
|
|
8131
|
+
const found = [];
|
|
8132
|
+
let frontier = [{
|
|
8133
|
+
abs: entryAbs,
|
|
8134
|
+
source: await readSource(entryAbs)
|
|
8135
|
+
}];
|
|
8136
|
+
for (let depth = 0; depth < maxDepth && frontier.length > 0; depth++) {
|
|
8137
|
+
const levels = await Promise.all(frontier.map(async (file) => (await resolveImportsOf(file, cwd, tsconfig)).map((abs) => ({
|
|
8138
|
+
abs,
|
|
8139
|
+
from: file.abs
|
|
8140
|
+
}))));
|
|
8141
|
+
const next = [];
|
|
8142
|
+
let capped = false;
|
|
8143
|
+
for (const entry of levels.flat()) {
|
|
8144
|
+
if (seen.has(entry.abs)) continue;
|
|
8145
|
+
seen.add(entry.abs);
|
|
8146
|
+
next.push(entry);
|
|
8147
|
+
if (found.length + next.length >= MAX_SUPPORT_FILES) {
|
|
8148
|
+
capped = true;
|
|
8149
|
+
break;
|
|
8150
|
+
}
|
|
8151
|
+
}
|
|
8152
|
+
const sources = await Promise.all(next.map((entry) => readSource(entry.abs)));
|
|
8153
|
+
found.push(...next.map((entry, i) => ({
|
|
8154
|
+
...entry,
|
|
8155
|
+
source: sources[i]
|
|
8156
|
+
})));
|
|
8157
|
+
if (capped) return {
|
|
8158
|
+
files: found,
|
|
8159
|
+
truncated: true
|
|
8160
|
+
};
|
|
8161
|
+
frontier = next.map((entry, i) => ({
|
|
8162
|
+
abs: entry.abs,
|
|
8163
|
+
source: sources[i]
|
|
8164
|
+
}));
|
|
8028
8165
|
}
|
|
8029
|
-
return
|
|
8166
|
+
return {
|
|
8167
|
+
files: found,
|
|
8168
|
+
truncated: frontier.length > 0
|
|
8169
|
+
};
|
|
8170
|
+
}
|
|
8171
|
+
/** A file that cannot be read is empty here: it imports nothing and shows nothing. */
|
|
8172
|
+
function readSource(fileAbs) {
|
|
8173
|
+
return readFile(fileAbs, "utf8").catch(() => "");
|
|
8174
|
+
}
|
|
8175
|
+
/** Every project file one file imports, in source order. */
|
|
8176
|
+
async function resolveImportsOf(file, cwd, tsconfig) {
|
|
8177
|
+
return (await Promise.all(importedSpecifiers(file.source).map((s) => resolveSpecifier(s, dirname(file.abs), cwd, tsconfig)))).filter((p) => p !== null);
|
|
8030
8178
|
}
|
|
8031
|
-
//#endregion
|
|
8032
|
-
//#region src/claude/extract-json.ts
|
|
8033
8179
|
/**
|
|
8034
|
-
*
|
|
8035
|
-
*
|
|
8036
|
-
*
|
|
8180
|
+
* One specifier → an existing file inside the project, or null. Anything that
|
|
8181
|
+
* escapes the project root, or lands in `node_modules`, is dropped: it is not
|
|
8182
|
+
* the consumer's code to read.
|
|
8037
8183
|
*/
|
|
8038
|
-
function
|
|
8039
|
-
const
|
|
8040
|
-
|
|
8041
|
-
|
|
8042
|
-
|
|
8184
|
+
async function resolveSpecifier(specifier, fromDirAbs, cwd, tsconfig) {
|
|
8185
|
+
const candidates = specifier.startsWith(".") ? [resolve(fromDirAbs, specifier)] : isAbsolute(specifier) ? [] : aliasCandidates(specifier, tsconfig);
|
|
8186
|
+
for (const candidate of candidates) {
|
|
8187
|
+
const file = await resolveFile(candidate);
|
|
8188
|
+
if (file === null) continue;
|
|
8189
|
+
if (file.split(/[\\/]/).includes("node_modules")) return null;
|
|
8190
|
+
return file.startsWith(`${cwd}/`) ? file : null;
|
|
8191
|
+
}
|
|
8043
8192
|
return null;
|
|
8044
8193
|
}
|
|
8045
|
-
|
|
8046
|
-
|
|
8194
|
+
/** Every path a `paths` alias maps this specifier to, in config order. */
|
|
8195
|
+
function aliasCandidates(specifier, tsconfig) {
|
|
8196
|
+
if (!tsconfig) return [];
|
|
8197
|
+
const out = [];
|
|
8198
|
+
for (const [pattern, targets] of Object.entries(tsconfig.paths)) {
|
|
8199
|
+
const star = pattern.indexOf("*");
|
|
8200
|
+
if (star === -1) {
|
|
8201
|
+
if (pattern === specifier) out.push(...targets.map((t) => resolve(tsconfig.baseUrlAbs, t)));
|
|
8202
|
+
continue;
|
|
8203
|
+
}
|
|
8204
|
+
const prefix = pattern.slice(0, star);
|
|
8205
|
+
const suffix = pattern.slice(star + 1);
|
|
8206
|
+
if (!specifier.startsWith(prefix) || !specifier.endsWith(suffix)) continue;
|
|
8207
|
+
const matched = specifier.slice(prefix.length, specifier.length - suffix.length);
|
|
8208
|
+
out.push(...targets.map((t) => resolve(tsconfig.baseUrlAbs, t.replace("*", matched))));
|
|
8209
|
+
}
|
|
8210
|
+
return out;
|
|
8211
|
+
}
|
|
8212
|
+
/** A path → the file it names: itself, `<path><ext>`, or `<path>/index<ext>`. */
|
|
8213
|
+
async function resolveFile(pathAbs) {
|
|
8214
|
+
if (await isFile(pathAbs)) return pathAbs;
|
|
8215
|
+
const rewritten = pathAbs.replace(/\.(js|mjs|cjs)$/, "");
|
|
8216
|
+
if (rewritten !== pathAbs) {
|
|
8217
|
+
for (const ext of EXTENSIONS) if (await isFile(rewritten + ext)) return rewritten + ext;
|
|
8218
|
+
}
|
|
8219
|
+
for (const ext of EXTENSIONS) if (await isFile(pathAbs + ext)) return pathAbs + ext;
|
|
8220
|
+
for (const ext of EXTENSIONS) {
|
|
8221
|
+
const indexed = join(pathAbs, `index${ext}`);
|
|
8222
|
+
if (await isFile(indexed)) return indexed;
|
|
8223
|
+
}
|
|
8224
|
+
return null;
|
|
8225
|
+
}
|
|
8226
|
+
function isFile(pathAbs) {
|
|
8227
|
+
return stat(pathAbs).then((s) => s.isFile(), () => false);
|
|
8228
|
+
}
|
|
8229
|
+
/** `export const NAME` / `export function NAME` / `export class NAME`, and `export { A, B }`. */
|
|
8230
|
+
const EXPORTED = /export\s+(?:async\s+)?(?:const|let|function|class|enum)\s+([A-Za-z_$][\w$]*)/g;
|
|
8231
|
+
const EXPORT_LIST = /export\s*\{([^}]*)\}/g;
|
|
8232
|
+
const BLOCK_COMMENT = /\/\*[\s\S]*?\*\//g;
|
|
8233
|
+
/** A `//` that is not the tail of a `://` — the shape a URL in a string has. */
|
|
8234
|
+
const LINE_COMMENT = /(^|[^:])\/\/.*$/gm;
|
|
8047
8235
|
/**
|
|
8048
|
-
*
|
|
8049
|
-
*
|
|
8050
|
-
* The generation loop's only bar is "the test goes green", and a rewrite that
|
|
8051
|
-
* weakens an assertion clears that bar as easily as one that keeps it. Green
|
|
8052
|
-
* therefore does not mean checked, and nothing else looks. Observed cases: a
|
|
8053
|
-
* step whose expectation was "the linked page opens" asserting instead that
|
|
8054
|
-
* the *link* is still visible on the page it clicked from; another asserting
|
|
8055
|
-
* on a navigation element unrelated to the step.
|
|
8236
|
+
* The value names a file exports, as a reader would list them.
|
|
8056
8237
|
*
|
|
8057
|
-
*
|
|
8058
|
-
*
|
|
8059
|
-
*
|
|
8238
|
+
* Text, not a parser: this answers a warning, not a refusal, and a name missed
|
|
8239
|
+
* because it was written in a form this does not know costs nothing. A name
|
|
8240
|
+
* invented would cost a false warning, so the patterns only match declarations
|
|
8241
|
+
* that plainly are one, and comments are stripped first. Types are left out
|
|
8242
|
+
* for the same reason: a page object's own row type is referenced only inside
|
|
8243
|
+
* it, so listing it would report a definition that is used.
|
|
8060
8244
|
*/
|
|
8061
|
-
function
|
|
8062
|
-
|
|
8063
|
-
|
|
8064
|
-
|
|
8065
|
-
|
|
8066
|
-
|
|
8067
|
-
""
|
|
8068
|
-
|
|
8069
|
-
|
|
8070
|
-
" states (the expectation says a page opens and the code re-checks the",
|
|
8071
|
-
" element it clicked; the expectation names a path with an id in it and the",
|
|
8072
|
-
" pattern also matches the path without one);",
|
|
8073
|
-
"- what it asserts on is unrelated to what the step did (e.g. a navigation",
|
|
8074
|
-
" element that is present on every page);",
|
|
8075
|
-
"- it depends on something that varies between runs and is not part of the",
|
|
8076
|
-
" expectation (a count, an index, a position, wording that changes);",
|
|
8077
|
-
"- the expectation names WHICH screen the run arrives at, and the code",
|
|
8078
|
-
" checks only the address's shape. A pattern matches every screen shaped",
|
|
8079
|
-
" like that one — and screens that are made per category, per folder, per",
|
|
8080
|
-
" account are shaped alike by design, so arriving at the wrong one passes.",
|
|
8081
|
-
" Something the named screen shows has to be checked too;",
|
|
8082
|
-
"- it checks that something is ABSENT, and nothing in the test has waited",
|
|
8083
|
-
" for the screen that would show it to finish drawing. Absence is true of",
|
|
8084
|
-
" a page that has not rendered yet, so the check passes on an empty screen",
|
|
8085
|
-
" and passes again when the thing is really there but slow. Something the",
|
|
8086
|
-
" screen shows when it is ready has to be established first;",
|
|
8087
|
-
"- the expectation ties two things together — this row AND the count beside",
|
|
8088
|
-
" it, this screen AND what it shows — and the assertions check them apart.",
|
|
8089
|
-
" Read the expectation's own grammar: \"the row, with its count\" is one",
|
|
8090
|
-
" claim about one row, and two assertions that each search the whole page",
|
|
8091
|
-
" hold just as well when the count belongs to a different row. Two things",
|
|
8092
|
-
" the expectation merely lists (\"the heading and the description are",
|
|
8093
|
-
" shown\") are two claims, and checking them apart is right;",
|
|
8094
|
-
"- the expectation is about EACH of several things the case names, and the",
|
|
8095
|
-
" assertions reach only one of them. The others are not checked at all, and",
|
|
8096
|
-
" a comment saying they look alike is the fault being written down, not a",
|
|
8097
|
-
" reason — the case asked for each because any one of them can be the one",
|
|
8098
|
-
" that breaks;",
|
|
8099
|
-
"- it has no assertion at all.",
|
|
8100
|
-
"",
|
|
8101
|
-
"Do NOT report: style, naming, structure, missing coverage the spec never",
|
|
8102
|
-
"asked for, or an expectation you merely disagree with. A step that checks",
|
|
8103
|
-
"less than you would have written, but still fails when the expectation",
|
|
8104
|
-
"breaks, is fine — unless the expectation covers several things and the",
|
|
8105
|
-
"code can only break on one of them, which is the bullet above.",
|
|
8106
|
-
"",
|
|
8107
|
-
"## Steps",
|
|
8108
|
-
"",
|
|
8109
|
-
...input.steps.map(stepLine),
|
|
8110
|
-
"",
|
|
8111
|
-
...input.expectations && input.expectations.length > 0 ? [
|
|
8112
|
-
"## What the case expects",
|
|
8113
|
-
"",
|
|
8114
|
-
"Stated for the flow, not per step. Each belongs to the step that first",
|
|
8115
|
-
"makes it true; a step is unchecked when nothing decides the one that",
|
|
8116
|
-
"belongs to it.",
|
|
8117
|
-
"",
|
|
8118
|
-
...input.expectations.map((e) => `- ${e}`),
|
|
8119
|
-
""
|
|
8120
|
-
] : [],
|
|
8121
|
-
...input.cleanup && input.cleanup.length > 0 ? [
|
|
8122
|
-
"## Cleanup steps",
|
|
8123
|
-
"",
|
|
8124
|
-
...input.cleanup.map(stepLine),
|
|
8125
|
-
""
|
|
8126
|
-
] : [],
|
|
8127
|
-
"## Generated test",
|
|
8128
|
-
"",
|
|
8129
|
-
"```",
|
|
8130
|
-
input.source,
|
|
8131
|
-
"```",
|
|
8132
|
-
"",
|
|
8133
|
-
...input.support && input.support.length > 0 ? [
|
|
8134
|
-
"## What it leans on",
|
|
8135
|
-
"",
|
|
8136
|
-
"The locators the assertions above resolve through. An assertion is",
|
|
8137
|
-
"only as strong as its locator, so read them, and report the step when",
|
|
8138
|
-
"its locator makes the assertion unable to fail the way the",
|
|
8139
|
-
"expectation describes. Two shapes to look for:",
|
|
8140
|
-
"",
|
|
8141
|
-
"- it takes the first (or last) of several matching elements, and the",
|
|
8142
|
-
" step claims something about each of them — the others go unchecked;",
|
|
8143
|
-
"- it matches by a string that also appears elsewhere on the page (a",
|
|
8144
|
-
" nav item, a breadcrumb, a menu), so it is satisfied by that element",
|
|
8145
|
-
" even when the one the step names is absent;",
|
|
8146
|
-
"- it matches only the words, where the step is about a control — a",
|
|
8147
|
-
" button, a link, a field. Text with that wording is not the control,",
|
|
8148
|
-
" so the assertion holds on a page where the control is gone.",
|
|
8149
|
-
"",
|
|
8150
|
-
"Judge these only through the steps — a definition no step depends on",
|
|
8151
|
-
"is not a finding.",
|
|
8152
|
-
"",
|
|
8153
|
-
...input.support.flatMap((f) => [
|
|
8154
|
-
`### ${f.path}`,
|
|
8155
|
-
"",
|
|
8156
|
-
"```",
|
|
8157
|
-
f.source,
|
|
8158
|
-
"```",
|
|
8159
|
-
""
|
|
8160
|
-
])
|
|
8161
|
-
] : [],
|
|
8162
|
-
"Answer with one json block and nothing else:",
|
|
8163
|
-
"",
|
|
8164
|
-
"```json",
|
|
8165
|
-
"{ \"findings\": [ { \"stepId\": \"step-05\", \"problem\": \"…\" } ] }",
|
|
8166
|
-
"```",
|
|
8167
|
-
"",
|
|
8168
|
-
"`problem` is one sentence naming what the step claims and what the code",
|
|
8169
|
-
"checks instead. Empty `findings` means every step is decided.",
|
|
8170
|
-
languageDirective(input.language)
|
|
8171
|
-
].join("\n");
|
|
8245
|
+
function exportedNames(source) {
|
|
8246
|
+
const code = source.replace(BLOCK_COMMENT, "").replace(LINE_COMMENT, "$1");
|
|
8247
|
+
const names = /* @__PURE__ */ new Set();
|
|
8248
|
+
for (const m of code.matchAll(EXPORTED)) names.add(m[1]);
|
|
8249
|
+
for (const m of code.matchAll(EXPORT_LIST)) for (const part of m[1].split(",")) {
|
|
8250
|
+
const name = part.trim().split(/\s+as\s+/).pop()?.trim();
|
|
8251
|
+
if (name && /^[A-Za-z_$][\w$]*$/.test(name) && name !== "default") names.add(name);
|
|
8252
|
+
}
|
|
8253
|
+
return [...names];
|
|
8172
8254
|
}
|
|
8173
|
-
|
|
8174
|
-
|
|
8255
|
+
//#endregion
|
|
8256
|
+
//#region src/targets/emitted-review.ts
|
|
8257
|
+
function reviewEmittedFiles(input) {
|
|
8258
|
+
const locators = collectLocators(input.files);
|
|
8259
|
+
const said = input.caseText.join("\n");
|
|
8175
8260
|
return [
|
|
8176
|
-
|
|
8177
|
-
|
|
8178
|
-
|
|
8179
|
-
|
|
8261
|
+
...decidesNothing(input),
|
|
8262
|
+
...unassertedPath(input),
|
|
8263
|
+
...[...input.files].flatMap(([file, source]) => [
|
|
8264
|
+
...toolIdentifiers(file, source),
|
|
8265
|
+
...containerOfPageText(file, source),
|
|
8266
|
+
...unjustifiedFirst(file, source),
|
|
8267
|
+
...describeEcho(file, source),
|
|
8268
|
+
...unaskedAssertions(file, source, locators, said),
|
|
8269
|
+
...weakerTwin(file, source),
|
|
8270
|
+
...unreached(file, source, input)
|
|
8271
|
+
])
|
|
8272
|
+
];
|
|
8180
8273
|
}
|
|
8181
|
-
|
|
8182
|
-
|
|
8274
|
+
/**
|
|
8275
|
+
* A path with a placeholder segment in it — `/orders/{orderId}`. Writing one
|
|
8276
|
+
* in prose is a statement about the address, not about the page's contents:
|
|
8277
|
+
* there is no other reason to name the part that changes per run.
|
|
8278
|
+
*/
|
|
8279
|
+
const PLACEHOLDER_PATH = /\/[A-Za-z0-9_\-]+(?:\/[A-Za-z0-9_\-]+)*\/\{[A-Za-z0-9_]+\}/;
|
|
8280
|
+
/**
|
|
8281
|
+
* A case that says where the run must end up, and a test that never looks at
|
|
8282
|
+
* the address.
|
|
8283
|
+
*
|
|
8284
|
+
* The row being on screen is not the same claim as being on the screen that
|
|
8285
|
+
* shows it — a product that rendered the row without navigating passes, and
|
|
8286
|
+
* that is the failure the expectation was written to catch. Scoped to
|
|
8287
|
+
* placeholder paths because those are unambiguous; a path mentioned as scenery
|
|
8288
|
+
* ("on /settings, the button is shown") is not a claim about the address.
|
|
8289
|
+
*
|
|
8290
|
+
* Calibrated against a small set of hand-written case definitions: a few
|
|
8291
|
+
* name a placeholder path in their expectations, most of those already
|
|
8292
|
+
* assert on the URL, and the one that does not is the defect this was
|
|
8293
|
+
* written for.
|
|
8294
|
+
*/
|
|
8295
|
+
function unassertedPath(input) {
|
|
8296
|
+
const source = input.files.get(input.testPath);
|
|
8297
|
+
if (source === void 0) return [];
|
|
8298
|
+
const stated = input.caseText.find((text) => PLACEHOLDER_PATH.test(text));
|
|
8299
|
+
if (stated === void 0 || /\btoHaveURL\b/.test(source)) return [];
|
|
8300
|
+
return [{
|
|
8301
|
+
file: input.testPath,
|
|
8302
|
+
line: 1,
|
|
8303
|
+
rule: "unasserted-path",
|
|
8304
|
+
message: `the case says where the run ends up — "${PLACEHOLDER_PATH.exec(stated)?.[0]}" — and nothing here looks at the address. What the screen shows can be right while the screen is wrong. Assert the URL where the case says it changes`
|
|
8305
|
+
}];
|
|
8183
8306
|
}
|
|
8184
|
-
|
|
8185
|
-
|
|
8186
|
-
const FindingsSchema = z.object({ findings: z.array(z.object({
|
|
8187
|
-
stepId: z.string(),
|
|
8188
|
-
problem: z.string()
|
|
8189
|
-
})) });
|
|
8307
|
+
/** `readonly name =`, `name(...): Locator {`, and `export … name`. */
|
|
8308
|
+
const DECLARED = /(?:readonly\s+([A-Za-z0-9_]+)\s*=|\b([A-Za-z0-9_]+)\([^)]*\)\s*:\s*Locator\b|export\s+(?:async\s+)?(?:const|let|function|class)\s+([A-Za-z0-9_]+))/g;
|
|
8190
8309
|
/**
|
|
8191
|
-
*
|
|
8192
|
-
*
|
|
8193
|
-
*
|
|
8310
|
+
* A definition in a file this generation wrote that nothing reaches — not the
|
|
8311
|
+
* test it generated, and not the project's own test assets either.
|
|
8312
|
+
*
|
|
8313
|
+
* Both halves are needed. Without the first it is not this generation's
|
|
8314
|
+
* business; without the second every shared definition another case uses would
|
|
8315
|
+
* be reported. Together they say only what is true: nobody reaches this.
|
|
8194
8316
|
*/
|
|
8195
|
-
function
|
|
8196
|
-
const
|
|
8197
|
-
if (
|
|
8198
|
-
|
|
8199
|
-
|
|
8200
|
-
|
|
8201
|
-
|
|
8202
|
-
|
|
8317
|
+
function unreached(file, source, input) {
|
|
8318
|
+
const elsewhere = input.usedInProject?.get(file);
|
|
8319
|
+
if (elsewhere === void 0) return [];
|
|
8320
|
+
const corpus = [...input.files.values()].join("\n");
|
|
8321
|
+
return lines(source).flatMap((text, i) => {
|
|
8322
|
+
const declared = DECLARED.exec(text);
|
|
8323
|
+
DECLARED.lastIndex = 0;
|
|
8324
|
+
if (!declared) return [];
|
|
8325
|
+
const name = named(declared);
|
|
8326
|
+
if (!name || elsewhere.has(name)) return [];
|
|
8327
|
+
if (corpus.split(new RegExp(`\\b${name}\\b`)).length - 1 > 1) return [];
|
|
8328
|
+
return [{
|
|
8329
|
+
file,
|
|
8330
|
+
line: i + 1,
|
|
8331
|
+
rule: "unreached",
|
|
8332
|
+
message: `\`${name}\` is defined here and nothing reaches it — not this case's test, and nothing else in the project. Remove it`
|
|
8333
|
+
}];
|
|
8334
|
+
});
|
|
8203
8335
|
}
|
|
8336
|
+
const named = (m) => m[1] ?? m[2] ?? m[3] ?? "";
|
|
8204
8337
|
/**
|
|
8205
|
-
*
|
|
8206
|
-
*
|
|
8207
|
-
*
|
|
8208
|
-
*
|
|
8338
|
+
* Two names for one visible string, where one of them is strictly weaker: a
|
|
8339
|
+
* role and its accessible name next to a bare page-wide text match that then
|
|
8340
|
+
* picks a position out of the matches.
|
|
8341
|
+
*
|
|
8342
|
+
* The weak one adds no check — anything it could catch, the other catches
|
|
8343
|
+
* first — and it is satisfied by whichever element the page happens to order
|
|
8344
|
+
* first, which on a real page was the navigation entry rather than the heading
|
|
8345
|
+
* the case names. Both assertions then read as two checks and are one.
|
|
8209
8346
|
*
|
|
8210
|
-
*
|
|
8211
|
-
*
|
|
8212
|
-
* about which steps were supposed to decide something.
|
|
8347
|
+
* Calibrated against a hand-written corpus: the shape appears only in a file
|
|
8348
|
+
* a generation wrote. People do not write it.
|
|
8213
8349
|
*/
|
|
8214
|
-
function
|
|
8215
|
-
|
|
8350
|
+
function weakerTwin(file, source) {
|
|
8351
|
+
const byString = /* @__PURE__ */ new Map();
|
|
8352
|
+
lines(source).forEach((text, i) => {
|
|
8353
|
+
const declared = /(?:readonly|const)\s+([A-Za-z0-9_]+)\s*=\s*(.+?);\s*$/.exec(text);
|
|
8354
|
+
if (!declared) return;
|
|
8355
|
+
const [, name, expression] = declared;
|
|
8356
|
+
const weak = /\.(?:first|last|nth)\(/.test(expression) && expression.includes("getByText(");
|
|
8357
|
+
const strong = expression.includes("getByRole(") && !/\.(?:first|last|nth)\(/.test(expression);
|
|
8358
|
+
if (!weak && !strong) return;
|
|
8359
|
+
for (const m of expression.matchAll(VISIBLE_STRING)) {
|
|
8360
|
+
const value = m[1] ?? m[2];
|
|
8361
|
+
if (!value || value.includes("${")) continue;
|
|
8362
|
+
const seen = byString.get(value) ?? [];
|
|
8363
|
+
seen.push({
|
|
8364
|
+
name,
|
|
8365
|
+
line: i + 1,
|
|
8366
|
+
weak
|
|
8367
|
+
});
|
|
8368
|
+
byString.set(value, seen);
|
|
8369
|
+
}
|
|
8370
|
+
});
|
|
8371
|
+
const findings = [];
|
|
8372
|
+
for (const [value, defs] of byString) {
|
|
8373
|
+
const weak = defs.filter((d) => d.weak);
|
|
8374
|
+
if (weak.length === 0 || defs.every((d) => d.weak)) continue;
|
|
8375
|
+
const strong = defs.find((d) => !d.weak);
|
|
8376
|
+
for (const d of weak) findings.push({
|
|
8377
|
+
file,
|
|
8378
|
+
line: d.line,
|
|
8379
|
+
rule: "weaker-twin",
|
|
8380
|
+
message: `\`${d.name}\` and \`${strong.name}\` are both "${value}", and this one searches the whole page and takes a position out of the matches. It can only be satisfied by something \`${strong.name}\` already covers, or by the wrong element. Assert one of them`
|
|
8381
|
+
});
|
|
8382
|
+
}
|
|
8383
|
+
return findings;
|
|
8216
8384
|
}
|
|
8217
8385
|
/**
|
|
8218
|
-
*
|
|
8219
|
-
* told. Shared with the evidence table, which reaches the same verdict from
|
|
8220
|
-
* the same file — a reader must not meet two wordings for one fact.
|
|
8386
|
+
* A test with no assertion anywhere in it, for a case that states something.
|
|
8221
8387
|
*
|
|
8222
|
-
*
|
|
8223
|
-
*
|
|
8224
|
-
*
|
|
8225
|
-
|
|
8226
|
-
|
|
8227
|
-
|
|
8228
|
-
*
|
|
8229
|
-
*
|
|
8230
|
-
*
|
|
8388
|
+
* The per-step version of this question belongs to the reading, which knows
|
|
8389
|
+
* that "open the list" claims nothing and expects no assertion under it. This
|
|
8390
|
+
* one needs no judgement: whatever the case says, a file that decides nothing
|
|
8391
|
+
* at all does not check it. It is the floor under a case whose expectations
|
|
8392
|
+
* are stated for the flow rather than per step — the shape a markdown case has
|
|
8393
|
+
* — where the reading is otherwise the only thing looking.
|
|
8394
|
+
*
|
|
8395
|
+
* Calibrated like the rest: in the corpus this was calibrated on, not one
|
|
8396
|
+
* hand-written spec had zero assertions.
|
|
8231
8397
|
*/
|
|
8232
|
-
function
|
|
8233
|
-
const
|
|
8234
|
-
|
|
8235
|
-
|
|
8398
|
+
function decidesNothing(input) {
|
|
8399
|
+
const source = input.files.get(input.testPath);
|
|
8400
|
+
if (source === void 0 || input.caseText.join("").trim().length === 0) return [];
|
|
8401
|
+
if (/\bexpect\(|\bjudgeByLlm\b/.test(source)) return [];
|
|
8402
|
+
return [{
|
|
8403
|
+
file: input.testPath,
|
|
8404
|
+
line: 1,
|
|
8405
|
+
rule: "decides-nothing",
|
|
8406
|
+
message: "this test contains no assertion at all, so it passes whatever the product does. Decide what the case says must be true"
|
|
8407
|
+
}];
|
|
8236
8408
|
}
|
|
8237
|
-
|
|
8238
|
-
|
|
8239
|
-
|
|
8240
|
-
|
|
8241
|
-
|
|
8242
|
-
|
|
8243
|
-
|
|
8244
|
-
|
|
8245
|
-
|
|
8246
|
-
|
|
8247
|
-
|
|
8248
|
-
|
|
8249
|
-
complete: false,
|
|
8250
|
-
warnings: []
|
|
8251
|
-
};
|
|
8252
|
-
}
|
|
8253
|
-
const checked = assertionsByStep(source);
|
|
8254
|
-
const undecided = [...input.steps, ...input.cleanup ?? []].filter(isExpandedActionStep).filter(claimsAnOutcome).filter((step) => (checked.get(step.id) ?? []).length === 0).map((step) => ({
|
|
8255
|
-
stepId: step.id,
|
|
8256
|
-
problem: NOTHING_DECIDED
|
|
8257
|
-
}));
|
|
8258
|
-
const { result: answer, isError } = await (input.invoke ?? invokeClaudeStreaming)({
|
|
8259
|
-
prompt: verifiesSpecPrompt({
|
|
8260
|
-
steps: input.steps,
|
|
8261
|
-
source,
|
|
8262
|
-
language: input.language,
|
|
8263
|
-
...input.expectations && input.expectations.length > 0 ? { expectations: [...input.expectations] } : {},
|
|
8264
|
-
...input.cleanup && input.cleanup.length > 0 ? { cleanup: [...input.cleanup] } : {},
|
|
8265
|
-
...input.support && input.support.length > 0 ? { support: input.support } : {}
|
|
8266
|
-
}),
|
|
8267
|
-
allowedTools: [],
|
|
8268
|
-
maxTurns: 1,
|
|
8269
|
-
silenceBashLog: true,
|
|
8270
|
-
...input.model ? { model: input.model } : {},
|
|
8271
|
-
cwd: input.cwd
|
|
8272
|
-
}, () => {});
|
|
8273
|
-
const fromModel = isError ? null : parseVerifiesSpecFindings(answer);
|
|
8274
|
-
if (fromModel === null) warn(`could not check whether the generated test decides its spec (${isError ? "Claude returned an error" : "no usable answer"})`);
|
|
8275
|
-
const findings = mergeFindings(undecided, fromModel ?? []);
|
|
8276
|
-
return {
|
|
8277
|
-
findings,
|
|
8278
|
-
complete: fromModel !== null,
|
|
8279
|
-
warnings: findings.map((finding) => formatFinding(finding))
|
|
8280
|
-
};
|
|
8409
|
+
const lines = (source) => source.split("\n");
|
|
8410
|
+
/** ccqa's name has no business in a name the project now owns and maintains. */
|
|
8411
|
+
function toolIdentifiers(file, source) {
|
|
8412
|
+
return lines(source).flatMap((text, i) => {
|
|
8413
|
+
const m = /\b(?:const|let|var|readonly)\s+(ccqa[A-Za-z0-9_]*)/i.exec(text);
|
|
8414
|
+
return m ? [{
|
|
8415
|
+
file,
|
|
8416
|
+
line: i + 1,
|
|
8417
|
+
rule: "tool-identifier",
|
|
8418
|
+
message: `\`${m[1]}\` names the tool that wrote it. Name it for what it holds`
|
|
8419
|
+
}] : [];
|
|
8420
|
+
});
|
|
8281
8421
|
}
|
|
8282
8422
|
/**
|
|
8283
|
-
*
|
|
8284
|
-
*
|
|
8285
|
-
*
|
|
8423
|
+
* Searching the whole page for something a name calls a container: its text,
|
|
8424
|
+
* or a structural tag that says nothing about what the element is.
|
|
8425
|
+
*
|
|
8426
|
+
* Both forms, because the first version tested only `getByText` and the fix
|
|
8427
|
+
* pass answered it with `page.locator("div").filter({ hasText }).last()` —
|
|
8428
|
+
* the same page-wide search, now resting on DOM order, and invisible to a
|
|
8429
|
+
* rule that only knew one shape. A generation that escapes a finding by
|
|
8430
|
+
* changing shape is worse than one that never drew it.
|
|
8286
8431
|
*/
|
|
8287
|
-
|
|
8288
|
-
|
|
8289
|
-
return [...undecided, ...fromModel.filter((f) => !claimed.has(f.stepId))];
|
|
8290
|
-
}
|
|
8291
|
-
//#endregion
|
|
8292
|
-
//#region src/evidence/table.ts
|
|
8293
|
-
/** Assertion lines a reviewer can check without reading the whole file. */
|
|
8294
|
-
const ASSERTION = /^\s*(?:await\s+)?(?:expect|judgeByLlm)\b.*$/;
|
|
8432
|
+
const PAGE_ROOTED = /^this\.page\.(?:getByText\(|locator\(["'](?:div|span|p|li|ul|ol|section|article)["']\))/;
|
|
8433
|
+
const CONTAINER = /\b(?:readonly\s+)?([a-zA-Z0-9_]*(?:Row|Cell|Card|Item|List))\s*(?:=\s*|\([^)]*\)\s*:\s*Locator\s*\{\s*(?:return\s+)?)(.*)$/;
|
|
8295
8434
|
/**
|
|
8296
|
-
*
|
|
8297
|
-
* the
|
|
8298
|
-
*
|
|
8299
|
-
*
|
|
8435
|
+
* A name that says "one of the things in a list" built from a match against
|
|
8436
|
+
* the whole page. It finds the string anywhere — a heading, a toast, another
|
|
8437
|
+
* row — so what it proves is not what its name claims.
|
|
8438
|
+
*
|
|
8439
|
+
* The message names the shapes that fix it, because the first version said
|
|
8440
|
+
* only "scope it" and the fix pass answered with `page.locator("div")` filtered
|
|
8441
|
+
* by the same text and narrowed with `.last()` — page-wide still, and now
|
|
8442
|
+
* resting on which match the DOM happens to put last.
|
|
8300
8443
|
*/
|
|
8301
|
-
function
|
|
8302
|
-
const
|
|
8303
|
-
|
|
8304
|
-
|
|
8305
|
-
|
|
8306
|
-
|
|
8307
|
-
|
|
8308
|
-
|
|
8309
|
-
|
|
8310
|
-
|
|
8311
|
-
|
|
8312
|
-
|
|
8313
|
-
|
|
8444
|
+
function containerOfPageText(file, source) {
|
|
8445
|
+
const all = lines(source);
|
|
8446
|
+
return all.flatMap((text, i) => {
|
|
8447
|
+
const m = CONTAINER.exec(text);
|
|
8448
|
+
if (!m) return [];
|
|
8449
|
+
const body = m[2].trim() || (all[i + 1] ?? "").trim().replace(/^return\s+/, "");
|
|
8450
|
+
if (!PAGE_ROOTED.test(body)) return [];
|
|
8451
|
+
return [{
|
|
8452
|
+
file,
|
|
8453
|
+
line: i + 1,
|
|
8454
|
+
rule: "unscoped-container",
|
|
8455
|
+
message: `\`${m[1]}\` is named for one element among others, but matches that text anywhere on the page — a heading or a toast carrying it satisfies this too. Address the element itself: the role it has (\`getByRole("row"|"listitem"|"article"|...)\` narrowed with \`.filter({ hasText })\`), or its test id. Where the page offers neither, scope from a container this page object already addresses — searching the whole page for a bare \`div\` and taking \`.first()\`/\`.last()\` is not scoping, it only picks whichever the DOM happens to order that way.`
|
|
8456
|
+
}];
|
|
8457
|
+
});
|
|
8314
8458
|
}
|
|
8315
8459
|
/**
|
|
8316
|
-
*
|
|
8317
|
-
*
|
|
8318
|
-
*
|
|
8460
|
+
* Whether a written reason sits above this line.
|
|
8461
|
+
*
|
|
8462
|
+
* The comment belongs to the run of assertions it explains, not to each one,
|
|
8463
|
+
* so two consecutive checks under one note are one explanation. Both rules
|
|
8464
|
+
* that use this are the same rule underneath: an unusual choice is allowed,
|
|
8465
|
+
* and has to be said out loud — which is what a project's own guidelines ask
|
|
8466
|
+
* of the people writing these by hand.
|
|
8319
8467
|
*/
|
|
8320
|
-
function
|
|
8321
|
-
|
|
8322
|
-
|
|
8323
|
-
|
|
8324
|
-
const list = byStep.get(id);
|
|
8325
|
-
if (list) list.push(action);
|
|
8326
|
-
else byStep.set(id, [action]);
|
|
8327
|
-
}
|
|
8328
|
-
return byStep;
|
|
8329
|
-
}
|
|
8330
|
-
/** The same grouping, each action rendered for a reader. */
|
|
8331
|
-
function actionsByStep(actions) {
|
|
8332
|
-
return new Map([...groupByStep(actions)].map(([id, grouped]) => [id, grouped.map(describeAction)]));
|
|
8468
|
+
function explained(all, i) {
|
|
8469
|
+
let above = i - 1;
|
|
8470
|
+
while (above >= 0 && /\bexpect\(|^\s*$|^\s*\)/.test(all[above] ?? "")) above -= 1;
|
|
8471
|
+
return /^\s*\/\//.test(all[above] ?? "");
|
|
8333
8472
|
}
|
|
8334
8473
|
/**
|
|
8335
|
-
*
|
|
8336
|
-
*
|
|
8337
|
-
*
|
|
8338
|
-
* often a CSS/testing artifact as product copy, and would anchor to noise.
|
|
8474
|
+
* `.first()` written on an assertion narrows a locator that matched several
|
|
8475
|
+
* things, so the assertion holds for whichever came first. Sometimes that is
|
|
8476
|
+
* the point — "the newest row is at the top" — and then it is written down.
|
|
8339
8477
|
*
|
|
8340
|
-
*
|
|
8341
|
-
*
|
|
8342
|
-
*
|
|
8478
|
+
* Only where the assertion itself narrows. Following it into the definitions
|
|
8479
|
+
* fired throughout the hand-written corpus: a page object that resolves a
|
|
8480
|
+
* repeated element once, for every caller, is how that is normally written.
|
|
8343
8481
|
*/
|
|
8344
|
-
function
|
|
8345
|
-
const
|
|
8346
|
-
|
|
8347
|
-
|
|
8348
|
-
if (
|
|
8349
|
-
|
|
8350
|
-
|
|
8351
|
-
|
|
8352
|
-
|
|
8353
|
-
|
|
8354
|
-
|
|
8355
|
-
for (const action of actions) {
|
|
8356
|
-
for (const locator of [action.locator, action.target]) {
|
|
8357
|
-
if (!locator) continue;
|
|
8358
|
-
if (locator.by === "testid") add(locator.value, "testid");
|
|
8359
|
-
else if (locator.by === "placeholder" || locator.by === "label") add(locator.value);
|
|
8360
|
-
else if (locator.by === "role") add(locator.name);
|
|
8361
|
-
}
|
|
8362
|
-
if (action.assert === "text_visible" || action.assert === "text_not_visible") add(action.value);
|
|
8363
|
-
}
|
|
8364
|
-
return needles;
|
|
8365
|
-
}
|
|
8366
|
-
function buildEvidenceSteps(input) {
|
|
8367
|
-
const byStep = groupByStep([...input.recording.actions, ...input.recording.cleanup ?? []]);
|
|
8368
|
-
const assertions = assertionsByStep(input.test.source);
|
|
8369
|
-
return [...input.testCase.steps, ...input.testCase.cleanup].map((step) => {
|
|
8370
|
-
const actions = byStep.get(step.id) ?? [];
|
|
8371
|
-
return {
|
|
8372
|
-
id: step.id,
|
|
8373
|
-
instruction: "instruction" in step ? step.instruction : step.judgeByLlm,
|
|
8374
|
-
expected: "expected" in step ? step.expected : "",
|
|
8375
|
-
actions: actions.map(describeAction),
|
|
8376
|
-
assertions: assertions.get(step.id) ?? [],
|
|
8377
|
-
screenshots: input.screenshots.get(step.id) ?? [],
|
|
8378
|
-
needles: sourceNeedles(actions)
|
|
8379
|
-
};
|
|
8482
|
+
function unjustifiedFirst(file, source) {
|
|
8483
|
+
const all = lines(source);
|
|
8484
|
+
return all.flatMap((text, i) => {
|
|
8485
|
+
if (!/\bexpect\(/.test(text) || !/\.first\(\)/.test(text)) return [];
|
|
8486
|
+
if (explained(all, i)) return [];
|
|
8487
|
+
return [{
|
|
8488
|
+
file,
|
|
8489
|
+
line: i + 1,
|
|
8490
|
+
rule: "unjustified-first",
|
|
8491
|
+
message: "`.first()` in an assertion holds for whichever element came first. Say why that is the one, or address the element itself"
|
|
8492
|
+
}];
|
|
8380
8493
|
});
|
|
8381
8494
|
}
|
|
8382
|
-
/**
|
|
8383
|
-
|
|
8384
|
-
|
|
8385
|
-
|
|
8386
|
-
const
|
|
8387
|
-
|
|
8388
|
-
const
|
|
8389
|
-
|
|
8390
|
-
|
|
8391
|
-
|
|
8392
|
-
|
|
8393
|
-
|
|
8394
|
-
|
|
8395
|
-
|
|
8396
|
-
|
|
8397
|
-
];
|
|
8398
|
-
for (const [label, actions] of outsideTheCase(input, steps, labels)) lines.push(`${label}: ${fold(actions, labels)}`, "");
|
|
8399
|
-
lines.push(`| ${labels.step} | ${labels.instruction} | ${labels.recorded} | ${labels.decides}` + (anchors ? ` | ${labels.source}` : "") + ` | ${labels.screens} |`, "|---|---|---|---|" + (anchors ? "---|" : "") + "---|");
|
|
8400
|
-
for (const step of steps) {
|
|
8401
|
-
const cells = [
|
|
8402
|
-
step.id,
|
|
8403
|
-
cell$1(step.instruction),
|
|
8404
|
-
fold(step.actions, labels),
|
|
8405
|
-
step.assertions.length > 0 ? cell$1(step.assertions.join("<br>")) : `**${labels.nothing}**`,
|
|
8406
|
-
...anchors ? [sourceAnchorCells(step.needles, anchors, labels)] : [],
|
|
8407
|
-
step.screenshots.map((path) => ``).join(" ") || "—"
|
|
8408
|
-
];
|
|
8409
|
-
lines.push(`| ${cells.join(" | ")} |`);
|
|
8410
|
-
}
|
|
8411
|
-
lines.push("");
|
|
8412
|
-
if (input.testCase.expectations.length > 0) {
|
|
8413
|
-
lines.push(`## ${labels.expects}`, "");
|
|
8414
|
-
for (const expectation of input.testCase.expectations) lines.push(`- ${expectation}`);
|
|
8415
|
-
lines.push("");
|
|
8416
|
-
}
|
|
8417
|
-
if (input.cleanupUnchecked && input.cleanupUnchecked.length > 0) {
|
|
8418
|
-
lines.push(labels.cleanupUnchecked, "");
|
|
8419
|
-
for (const expectation of input.cleanupUnchecked) lines.push(`- ${expectation}`);
|
|
8420
|
-
lines.push("");
|
|
8421
|
-
}
|
|
8422
|
-
const findings = mergeFindings(steps.filter((step) => claimsAnOutcome(step) && step.assertions.length === 0).map((step) => ({
|
|
8423
|
-
stepId: step.id,
|
|
8424
|
-
problem: NOTHING_DECIDED
|
|
8425
|
-
})), input.review ?? []);
|
|
8426
|
-
lines.push(`## ${labels.review}`, "");
|
|
8427
|
-
if (findings.length === 0) lines.push(input.review === void 0 ? labels.reviewAbsent : labels.reviewClean);
|
|
8428
|
-
else {
|
|
8429
|
-
for (const finding of findings) lines.push(`- ${formatFinding(finding, labels)}`);
|
|
8430
|
-
if (input.review === void 0) lines.push("", labels.reviewPartial);
|
|
8431
|
-
}
|
|
8432
|
-
lines.push("");
|
|
8433
|
-
return lines.join("\n");
|
|
8495
|
+
/** A describe that repeats its only test reads as "X › X" in every report. */
|
|
8496
|
+
function describeEcho(file, source) {
|
|
8497
|
+
const describe = /test\.describe\(\s*["'](.+?)["']/.exec(source);
|
|
8498
|
+
if (!describe) return [];
|
|
8499
|
+
const titles = [...source.matchAll(/\n\s*test\(\s*["'](.+?)["']/g)].map((m) => m[1]);
|
|
8500
|
+
if (titles.length !== 1) return [];
|
|
8501
|
+
const bare = (s) => s.replace(/\s+@[\w-]+\s*$/, "").trim();
|
|
8502
|
+
if (bare(titles[0]) !== bare(describe[1])) return [];
|
|
8503
|
+
const line = lines(source).findIndex((t) => t.includes("test.describe(")) + 1;
|
|
8504
|
+
return [{
|
|
8505
|
+
file,
|
|
8506
|
+
line: Math.max(line, 1),
|
|
8507
|
+
rule: "describe-echo",
|
|
8508
|
+
message: "the describe repeats its only test's name. Drop it, or name the screen the test is about"
|
|
8509
|
+
}];
|
|
8434
8510
|
}
|
|
8435
|
-
/**
|
|
8436
|
-
function
|
|
8437
|
-
const
|
|
8438
|
-
|
|
8511
|
+
/** `name` → the expression it is defined as, across every file written. */
|
|
8512
|
+
function collectLocators(files) {
|
|
8513
|
+
const out = /* @__PURE__ */ new Map();
|
|
8514
|
+
for (const source of files.values()) {
|
|
8515
|
+
for (const m of source.matchAll(/\breadonly\s+([a-zA-Z0-9_]+)\s*=\s*([^;]+);/g)) out.set(m[1], m[2]);
|
|
8516
|
+
for (const m of source.matchAll(/\b([a-zA-Z0-9_]+)\([^)]*\)\s*:\s*Locator\s*\{\s*return\s+([^;]+);/g)) out.set(m[1], m[2]);
|
|
8517
|
+
}
|
|
8518
|
+
return out;
|
|
8439
8519
|
}
|
|
8440
8520
|
/**
|
|
8441
|
-
*
|
|
8442
|
-
*
|
|
8443
|
-
*
|
|
8521
|
+
* The forms that carry a string a person sees on screen. A whitelist, not a
|
|
8522
|
+
* blacklist: a test id, an ARIA role name and a CSS selector are all strings
|
|
8523
|
+
* too, and none of them is something a test case would ever mention — counted
|
|
8524
|
+
* as "what this assertion looks for", every `getByTestId` in a project that
|
|
8525
|
+
* prefers them would read as an assertion nobody asked for.
|
|
8444
8526
|
*/
|
|
8445
|
-
|
|
8446
|
-
|
|
8447
|
-
|
|
8448
|
-
|
|
8449
|
-
|
|
8450
|
-
|
|
8451
|
-
|
|
8452
|
-
|
|
8453
|
-
|
|
8454
|
-
|
|
8455
|
-
}
|
|
8527
|
+
const VISIBLE_STRING = /(?:getByText|getByLabel|getByPlaceholder|getByTitle|getByAltText|toHaveText|toContainText|toHaveValue)\(\s*["'`]([^"'`]+)|name:\s*["'`]([^"'`]+)/g;
|
|
8528
|
+
/**
|
|
8529
|
+
* The strings an assertion actually looks for: the ones on its own line plus
|
|
8530
|
+
* the ones in whatever page-object members it names.
|
|
8531
|
+
*/
|
|
8532
|
+
function expandLocators(text, locators) {
|
|
8533
|
+
let expanded = text;
|
|
8534
|
+
for (const m of text.matchAll(/\.([a-zA-Z0-9_]+)\b/g)) {
|
|
8535
|
+
const definition = locators.get(m[1]);
|
|
8536
|
+
if (definition) expanded += `\n${definition}`;
|
|
8456
8537
|
}
|
|
8457
|
-
|
|
8458
|
-
return cell$1([...confirmed, ...folded].join("<br>"));
|
|
8538
|
+
return expanded;
|
|
8459
8539
|
}
|
|
8460
|
-
|
|
8461
|
-
|
|
8462
|
-
if (entries.length <= FOLD_OVER) return cell$1(entries.join("<br>"));
|
|
8463
|
-
return `<details><summary>${entries.length} ${labels.operations}</summary>${cell$1(entries.join("<br>"))}</details>`;
|
|
8540
|
+
function assertedStrings(text, locators) {
|
|
8541
|
+
return [...expandLocators(text, locators).matchAll(VISIBLE_STRING)].map((m) => (m[1] ?? m[2] ?? "").trim()).filter((s) => s.length > 1 && !s.includes("${"));
|
|
8464
8542
|
}
|
|
8465
|
-
/**
|
|
8466
|
-
|
|
8467
|
-
|
|
8468
|
-
|
|
8469
|
-
|
|
8470
|
-
|
|
8471
|
-
|
|
8472
|
-
|
|
8473
|
-
|
|
8474
|
-
|
|
8475
|
-
|
|
8543
|
+
/**
|
|
8544
|
+
* An assertion about something the case never mentions.
|
|
8545
|
+
*
|
|
8546
|
+
* The case is the contract. Anything else seen along the way adds no
|
|
8547
|
+
* coverage, differs between recordings, and is the first thing to break —
|
|
8548
|
+
* and a reviewer reading the case cannot tell why the test checks it.
|
|
8549
|
+
*
|
|
8550
|
+
* Only fires when the assertion looks for a string at all: a URL pattern or
|
|
8551
|
+
* a value the test itself computed has nothing to compare against, and
|
|
8552
|
+
* silence is the honest answer there. And not when a reason sits above it —
|
|
8553
|
+
* checking that a precondition still holds, so the test fails where the
|
|
8554
|
+
* precondition broke rather than four steps later, is worth doing and is
|
|
8555
|
+
* written down when it is.
|
|
8556
|
+
*/
|
|
8557
|
+
function unaskedAssertions(file, source, locators, said) {
|
|
8558
|
+
const all = lines(source);
|
|
8559
|
+
return all.flatMap((text, i) => {
|
|
8560
|
+
if (!/\bexpect\(/.test(text)) return [];
|
|
8561
|
+
const strings = assertedStrings(text, locators);
|
|
8562
|
+
if (strings.length === 0 || strings.some((s) => said.includes(s))) return [];
|
|
8563
|
+
if (explained(all, i)) return [];
|
|
8564
|
+
return [{
|
|
8565
|
+
file,
|
|
8566
|
+
line: i + 1,
|
|
8567
|
+
rule: "unasked-assertion",
|
|
8568
|
+
message: `nothing in the case asks about ${strings.map((s) => `"${s}"`).join(", ")}. Assert what the case states, and drop the rest`
|
|
8569
|
+
}];
|
|
8570
|
+
});
|
|
8571
|
+
}
|
|
8572
|
+
/** The findings as the fix loop's other inputs look: a command and its output. */
|
|
8573
|
+
function formatEmittedReview(findings) {
|
|
8574
|
+
return findings.map((f) => `${f.file}:${f.line} [${f.rule}] ${f.message}`).join("\n");
|
|
8476
8575
|
}
|
|
8477
8576
|
//#endregion
|
|
8478
|
-
//#region src/
|
|
8577
|
+
//#region src/claude/extract-json.ts
|
|
8479
8578
|
/**
|
|
8480
|
-
*
|
|
8481
|
-
*
|
|
8482
|
-
*
|
|
8483
|
-
* supplied by the target, adds one more "do not drop this from the draft" rule
|
|
8484
|
-
* (e.g. Playwright's step-evidence calls).
|
|
8579
|
+
* Pulls a JSON object out of a Claude completion. Accepts either a fenced
|
|
8580
|
+
* ```json block or a bare `{...}` payload that constitutes the whole reply.
|
|
8581
|
+
* Returns null when neither shape is present.
|
|
8485
8582
|
*/
|
|
8486
|
-
function
|
|
8487
|
-
const
|
|
8488
|
-
|
|
8489
|
-
|
|
8490
|
-
|
|
8491
|
-
|
|
8492
|
-
if (hasDraft) {
|
|
8493
|
-
rules.push("4. **The draft is ground truth.** The mechanical draft below encodes the recorded route. Do not change the meaning of its operation sequence or its assertions — no reordering, dropping, or weakening. You may rewrite locators when the conventions or the sources you read justify a better one.");
|
|
8494
|
-
if (draftInvariant) rules.push(`5. ${draftInvariant}`);
|
|
8495
|
-
}
|
|
8496
|
-
return `## Reuse contract\n\n${rules.join("\n")}`;
|
|
8583
|
+
function extractJsonBlock(text) {
|
|
8584
|
+
const fenced = text.match(/```(?:json)?\s*\n([\s\S]*?)\n```/);
|
|
8585
|
+
if (fenced && fenced[1]) return fenced[1].trim();
|
|
8586
|
+
const trimmed = text.trim();
|
|
8587
|
+
if (trimmed.startsWith("{") && trimmed.endsWith("}")) return trimmed;
|
|
8588
|
+
return null;
|
|
8497
8589
|
}
|
|
8590
|
+
//#endregion
|
|
8591
|
+
//#region src/prompts/verifies-spec.ts
|
|
8498
8592
|
/**
|
|
8499
|
-
*
|
|
8500
|
-
*
|
|
8593
|
+
* Asks a reviewer who knows nothing about how the files came to exist whether
|
|
8594
|
+
* they would let them into this suite.
|
|
8595
|
+
*
|
|
8596
|
+
* The generation loop's only bar is "the test goes green", and a rewrite that
|
|
8597
|
+
* weakens an assertion clears that bar as easily as one that keeps it. Green
|
|
8598
|
+
* therefore does not mean checked, and nothing else looks. Observed cases: a
|
|
8599
|
+
* step whose expectation was "the linked page opens" asserting instead that
|
|
8600
|
+
* the *link* is still visible on the page it clicked from; another asserting
|
|
8601
|
+
* on a navigation element unrelated to the step.
|
|
8602
|
+
*
|
|
8603
|
+
* The prompt carries paths, not source: the reviewer opens the files itself,
|
|
8604
|
+
* which is also what lets it read the rest of the repository. That reading is
|
|
8605
|
+
* the point — "follow the existing implementation" is a rule no inline copy of
|
|
8606
|
+
* two files can be checked against, because the evidence for it is how many
|
|
8607
|
+
* other files do it that way.
|
|
8608
|
+
*
|
|
8609
|
+
* Nothing here may say, or let the reader infer, that the files were written
|
|
8610
|
+
* by a model. A reviewer told that judges the author instead of the code, and
|
|
8611
|
+
* the question this asks is what a colleague would say on the pull request.
|
|
8501
8612
|
*/
|
|
8502
|
-
function
|
|
8503
|
-
|
|
8504
|
-
|
|
8505
|
-
|
|
8506
|
-
|
|
8507
|
-
|
|
8508
|
-
|
|
8509
|
-
|
|
8510
|
-
|
|
8511
|
-
|
|
8512
|
-
|
|
8513
|
-
|
|
8514
|
-
|
|
8515
|
-
|
|
8516
|
-
|
|
8517
|
-
|
|
8518
|
-
|
|
8613
|
+
function verifiesSpecPrompt(input) {
|
|
8614
|
+
const rules = rulesReview(input.guides ?? []);
|
|
8615
|
+
return [
|
|
8616
|
+
"You are the QA lead who owns this end-to-end suite. A colleague has put the",
|
|
8617
|
+
"files below up for review. Open them, read whatever else in the repository you",
|
|
8618
|
+
"need in order to judge them, and answer as you would on their pull request.",
|
|
8619
|
+
"",
|
|
8620
|
+
"## Files under review",
|
|
8621
|
+
"",
|
|
8622
|
+
"Written by this change:",
|
|
8623
|
+
"",
|
|
8624
|
+
`- ${input.testPath} (the test)`,
|
|
8625
|
+
...(input.submitted ?? []).map((path) => `- ${path}`),
|
|
8626
|
+
"",
|
|
8627
|
+
...input.leansOn && input.leansOn.length > 0 ? [
|
|
8628
|
+
"Already in the repository, and imported by the test:",
|
|
8629
|
+
"",
|
|
8630
|
+
...input.leansOn.map((path) => `- ${path}`),
|
|
8631
|
+
""
|
|
8632
|
+
] : [],
|
|
8633
|
+
"A path listed here that you cannot open is itself a finding: report every step",
|
|
8634
|
+
"that depended on it and name the path, rather than passing over it as if it",
|
|
8635
|
+
"were clean.",
|
|
8636
|
+
"",
|
|
8637
|
+
"## The case this test must cover",
|
|
8638
|
+
"",
|
|
8639
|
+
"The only statement of what the test has to decide. Not its comments, not its",
|
|
8640
|
+
"name, not what the code looks like it is doing.",
|
|
8641
|
+
"",
|
|
8642
|
+
...input.steps.map(stepLine),
|
|
8643
|
+
"",
|
|
8644
|
+
...input.expectations && input.expectations.length > 0 ? [
|
|
8645
|
+
"### What the case expects",
|
|
8646
|
+
"",
|
|
8647
|
+
"Stated for the flow, not per step. Each belongs to the step that first",
|
|
8648
|
+
"makes it true; a step is unchecked when nothing decides the one that",
|
|
8649
|
+
"belongs to it.",
|
|
8650
|
+
"",
|
|
8651
|
+
...input.expectations.map((e) => `- ${e}`),
|
|
8652
|
+
""
|
|
8653
|
+
] : [],
|
|
8654
|
+
...input.cleanup && input.cleanup.length > 0 ? [
|
|
8655
|
+
"### Cleanup steps",
|
|
8656
|
+
"",
|
|
8657
|
+
...input.cleanup.map(stepLine),
|
|
8658
|
+
""
|
|
8659
|
+
] : [],
|
|
8660
|
+
"## Does the test decide the case",
|
|
8661
|
+
"",
|
|
8662
|
+
"For each step above, the test must contain assertions that could FAIL if the",
|
|
8663
|
+
"step's `expected` stopped holding. Report a step when:",
|
|
8664
|
+
"",
|
|
8665
|
+
"- its assertions cannot fail while the product is broken in the way the",
|
|
8666
|
+
" expectation describes — because they check something weaker than it",
|
|
8667
|
+
" states (the expectation says a page opens and the code re-checks the",
|
|
8668
|
+
" element it clicked; the expectation names a path with an id in it and the",
|
|
8669
|
+
" pattern also matches the path without one);",
|
|
8670
|
+
"- what it asserts on is unrelated to what the step did (e.g. a navigation",
|
|
8671
|
+
" element that is present on every page);",
|
|
8672
|
+
"- it depends on something that varies between runs and is not part of the",
|
|
8673
|
+
" expectation (a count, an index, a position, wording that changes);",
|
|
8674
|
+
"- the expectation names WHICH screen the run arrives at, and the code",
|
|
8675
|
+
" checks only the address's shape. A pattern matches every screen shaped",
|
|
8676
|
+
" like that one — and screens that are made per category, per folder, per",
|
|
8677
|
+
" account are shaped alike by design, so arriving at the wrong one passes.",
|
|
8678
|
+
" Something the named screen shows has to be checked too;",
|
|
8679
|
+
"- it checks that something is ABSENT, and nothing in the test has waited",
|
|
8680
|
+
" for the screen that would show it to finish drawing. Absence is true of",
|
|
8681
|
+
" a page that has not rendered yet, so the check passes on an empty screen",
|
|
8682
|
+
" and passes again when the thing is really there but slow. Something the",
|
|
8683
|
+
" screen shows when it is ready has to be established first;",
|
|
8684
|
+
"- the expectation ties two things together — this row AND the count beside",
|
|
8685
|
+
" it, this screen AND what it shows — and the assertions check them apart.",
|
|
8686
|
+
" Read the expectation's own grammar: \"the row, with its count\" is one",
|
|
8687
|
+
" claim about one row, and two assertions that each search the whole page",
|
|
8688
|
+
" hold just as well when the count belongs to a different row. Two things",
|
|
8689
|
+
" the expectation merely lists (\"the heading and the description are",
|
|
8690
|
+
" shown\") are two claims, and checking them apart is right;",
|
|
8691
|
+
"- the expectation is about EACH of several things the case names, and the",
|
|
8692
|
+
" assertions reach only one of them. The others are not checked at all, and",
|
|
8693
|
+
" a comment saying they look alike is the fault being written down, not a",
|
|
8694
|
+
" reason — the case asked for each because any one of them can be the one",
|
|
8695
|
+
" that breaks;",
|
|
8696
|
+
"- it has no assertion at all.",
|
|
8697
|
+
"",
|
|
8698
|
+
"An assertion is only as strong as the locator it names, and the locator is",
|
|
8699
|
+
"usually not in the test file. Open what the test resolves through, and report",
|
|
8700
|
+
"the step when its locator makes the assertion unable to fail the way the",
|
|
8701
|
+
"expectation describes. Three shapes to look for:",
|
|
8702
|
+
"",
|
|
8703
|
+
"- it takes the first (or last) of several matching elements, and the step",
|
|
8704
|
+
" claims something about each of them — the others go unchecked;",
|
|
8705
|
+
"- it matches by a string that also appears elsewhere on the page (a nav",
|
|
8706
|
+
" item, a breadcrumb, a menu), so it is satisfied by that element even when",
|
|
8707
|
+
" the one the step names is absent;",
|
|
8708
|
+
"- it matches only the words, where the step is about a control — a button,",
|
|
8709
|
+
" a link, a field. Text with that wording is not the control, so the",
|
|
8710
|
+
" assertion holds on a page where the control is gone.",
|
|
8711
|
+
"",
|
|
8712
|
+
"Judge a locator only through the steps — a definition no step depends on is",
|
|
8713
|
+
"not a finding.",
|
|
8714
|
+
"",
|
|
8715
|
+
"Do NOT report here: style, naming, structure, missing coverage the case never",
|
|
8716
|
+
"asked for, or an expectation you merely disagree with. A step that checks",
|
|
8717
|
+
"less than you would have written, but still fails when the expectation",
|
|
8718
|
+
"breaks, is fine — unless the expectation covers several things and the",
|
|
8719
|
+
"code can only break on one of them, which is the bullet above.",
|
|
8720
|
+
"",
|
|
8721
|
+
...rules.section,
|
|
8722
|
+
"When you have read enough to decide, answer with one json block and nothing",
|
|
8723
|
+
"else:",
|
|
8724
|
+
"",
|
|
8725
|
+
"```json",
|
|
8726
|
+
rules.answerShape,
|
|
8727
|
+
"```",
|
|
8728
|
+
"",
|
|
8729
|
+
"`problem` is one sentence naming what the step claims and what the code",
|
|
8730
|
+
"checks instead. Empty `findings` means every step is decided.",
|
|
8731
|
+
...rules.answerNote,
|
|
8732
|
+
languageDirective(input.language)
|
|
8733
|
+
].join("\n");
|
|
8519
8734
|
}
|
|
8520
|
-
|
|
8521
|
-
|
|
8522
|
-
return
|
|
8735
|
+
function stepLine(step) {
|
|
8736
|
+
if (!isExpandedActionStep(step)) return `- ${step.id}: judged by a model at run time — its claim is asserted by the injected call, so it needs no other assertion.`;
|
|
8737
|
+
return [
|
|
8738
|
+
`- ${step.id}`,
|
|
8739
|
+
` does: ${oneLine$3(step.instruction)}`,
|
|
8740
|
+
` expected: ${oneLine$3(step.expected)}`
|
|
8741
|
+
].join("\n");
|
|
8523
8742
|
}
|
|
8524
|
-
function
|
|
8525
|
-
|
|
8526
|
-
const source = step.from ? ` (read from \`${step.from}\`)` : "";
|
|
8527
|
-
return `- ${step.id}: judge by LLM${source}\n claim: ${body(step.judgeByLlm)}`;
|
|
8743
|
+
function oneLine$3(text) {
|
|
8744
|
+
return text.trim().split("\n").map((l) => l.trim()).join(" ");
|
|
8528
8745
|
}
|
|
8529
8746
|
/**
|
|
8530
|
-
*
|
|
8531
|
-
*
|
|
8747
|
+
* The second review: the same files, read against the rule documents the
|
|
8748
|
+
* project wrote and against what the rest of the suite already does.
|
|
8749
|
+
*
|
|
8750
|
+
* Every rule the generation loop could already check — a type error, a lint
|
|
8751
|
+
* rule, a gate of ccqa's own — is followed by the code it produces. A rule
|
|
8752
|
+
* that only exists as prose is not, and nothing looked: green, compiled, lint
|
|
8753
|
+
* clean and decides its case is the whole bar, and none of it reads the
|
|
8754
|
+
* project's own guides. Reused code is the worse half of it — a helper the
|
|
8755
|
+
* test imports was written under whatever rules existed when it was written,
|
|
8756
|
+
* and no later rule has ever been applied to it.
|
|
8757
|
+
*
|
|
8758
|
+
* The evidence is what keeps this honest in both directions. A model asked to
|
|
8759
|
+
* judge code against prose will find something to say about any code; asked to
|
|
8760
|
+
* quote the line it is judging by — or, where the convention is only in the
|
|
8761
|
+
* corpus, to say which files it read and how many of them agree — it can only
|
|
8762
|
+
* report what is there.
|
|
8763
|
+
*
|
|
8764
|
+
* Asked once for everything it decides: a project that wrote no rule document
|
|
8765
|
+
* gets no section, no second key in the answer, and nothing said about one.
|
|
8532
8766
|
*/
|
|
8533
|
-
function
|
|
8534
|
-
|
|
8535
|
-
|
|
8536
|
-
|
|
8537
|
-
|
|
8538
|
-
|
|
8539
|
-
|
|
8540
|
-
|
|
8541
|
-
|
|
8542
|
-
|
|
8543
|
-
|
|
8544
|
-
|
|
8545
|
-
|
|
8546
|
-
|
|
8547
|
-
|
|
8548
|
-
|
|
8549
|
-
|
|
8550
|
-
|
|
8551
|
-
|
|
8552
|
-
|
|
8553
|
-
|
|
8554
|
-
|
|
8555
|
-
|
|
8556
|
-
|
|
8767
|
+
function rulesReview(guides) {
|
|
8768
|
+
if (guides.length === 0) return {
|
|
8769
|
+
section: [],
|
|
8770
|
+
answerShape: "{ \"findings\": [ { \"stepId\": \"step-05\", \"problem\": \"…\" } ] }",
|
|
8771
|
+
answerNote: []
|
|
8772
|
+
};
|
|
8773
|
+
return {
|
|
8774
|
+
section: [
|
|
8775
|
+
"## Does the code belong in this suite",
|
|
8776
|
+
"",
|
|
8777
|
+
"A second review of the same files, separate from the one above.",
|
|
8778
|
+
"",
|
|
8779
|
+
"These are the rule documents this project wrote. Report a file when its code",
|
|
8780
|
+
"breaks a rule one of them states, and quote the line that states it. The",
|
|
8781
|
+
"quote is the check: a rule you cannot quote is a rule the project did not",
|
|
8782
|
+
"write, and reporting one of those is as wrong as missing one it did. A",
|
|
8783
|
+
"preference of your own is not a rule here, however reasonable.",
|
|
8784
|
+
"",
|
|
8785
|
+
"How the rest of the suite is written is the other half of this question, and",
|
|
8786
|
+
"it is not in the documents: the helper every other case navigates through,",
|
|
8787
|
+
"the fixture they all take, where page objects live and what they are called.",
|
|
8788
|
+
"Search the repository before you claim one — a convention is what the code",
|
|
8789
|
+
"does, not what you would have written — and report it with what you found:",
|
|
8790
|
+
"which files you read, and how many of them do it that way.",
|
|
8791
|
+
"",
|
|
8792
|
+
"\"Do NOT report here: style, naming, structure\" above governs the step",
|
|
8793
|
+
"findings. Those belong in this half instead, and only where a document",
|
|
8794
|
+
"states them or the corpus shows them.",
|
|
8795
|
+
"",
|
|
8796
|
+
"Read what the test leans on, not only what this change wrote. A helper it",
|
|
8797
|
+
"imports was written under whatever rules existed then, and nothing has read",
|
|
8798
|
+
"it against these since.",
|
|
8799
|
+
"",
|
|
8800
|
+
...guides.flatMap((g) => [
|
|
8801
|
+
`### ${g.path}`,
|
|
8802
|
+
"",
|
|
8803
|
+
fencedBlock(g.body),
|
|
8804
|
+
""
|
|
8805
|
+
])
|
|
8806
|
+
],
|
|
8807
|
+
answerShape: "{ \"findings\": [ { \"stepId\": \"step-05\", \"problem\": \"…\" } ], \"ruleViolations\": [ { \"file\": \"…\", \"guide\": \"…\", \"rule\": \"…\", \"code\": \"…\", \"severity\": \"blocking\" } ] }",
|
|
8808
|
+
answerNote: [
|
|
8809
|
+
"",
|
|
8810
|
+
"In `ruleViolations`: `file` is the file's path in this repository, and `code`",
|
|
8811
|
+
"the lines that break the rule. `guide` and `rule` are where the rule comes",
|
|
8812
|
+
"from and what it says — the document's path and its own words when you can",
|
|
8813
|
+
"quote one; otherwise the path of a file you read that shows the convention,",
|
|
8814
|
+
"and what you counted (\"N of M files under <dir> do X\"). `severity` is",
|
|
8815
|
+
"\"blocking\" when you would hold the change until it is fixed, and \"advisory\"",
|
|
8816
|
+
"when it is worth saying but you would approve anyway. An empty",
|
|
8817
|
+
"`ruleViolations` means every file follows every rule you could quote or",
|
|
8818
|
+
"count — write the key either way."
|
|
8819
|
+
]
|
|
8820
|
+
};
|
|
8557
8821
|
}
|
|
8822
|
+
//#endregion
|
|
8823
|
+
//#region src/ir/types.ts
|
|
8558
8824
|
/**
|
|
8559
|
-
*
|
|
8560
|
-
*
|
|
8825
|
+
* Tool-neutral intermediate representation (IR) for recorded browser
|
|
8826
|
+
* interactions. A trace produces `RecordedAction[]` (persisted as `ir.json`);
|
|
8827
|
+
* emitters translate it into target-specific test code and the replay
|
|
8828
|
+
* validator re-executes it. The IR replaces the old dual addressing
|
|
8829
|
+
* (raw selector strings vs `find_*` + findLocator fields) with a single
|
|
8830
|
+
* `Locator` model.
|
|
8561
8831
|
*/
|
|
8562
|
-
function buildLlmFixPrompt(input) {
|
|
8563
|
-
const files = input.files.map((f) => `### ${f.path} (${f.kind})\n\n\`\`\`\n${f.contents}\n\`\`\``).join("\n\n");
|
|
8564
|
-
return [
|
|
8565
|
-
`You generated ${input.targetId} test files for this project, but their verification run failed. Fix the files so the command passes. Keep the tested behaviour identical — fix locators, imports, syntax, and setup, never delete or weaken assertions to force a pass. You may Read/Grep/Glob the project to investigate.`,
|
|
8566
|
-
`## Failing command\n\n\`${input.command}\``,
|
|
8567
|
-
`## Command output (tail)\n\n\`\`\`\n${input.outputTail}\n\`\`\``,
|
|
8568
|
-
`## Current files\n\n${files}`,
|
|
8569
|
-
outputContract(input.testPath, input.writeRoots) + "\n- Output only the files that need changes; files you omit stay as they are.\n- If the failure is NOT caused by the generated files (an environment/setup issue) or the files are already correct, reply `{\"files\": [], \"summary\": \"<why no file change is needed>\"}` — verification is then simply re-run."
|
|
8570
|
-
].join("\n\n") + languageDirective(input.language);
|
|
8571
|
-
}
|
|
8572
8832
|
/**
|
|
8573
|
-
*
|
|
8574
|
-
*
|
|
8833
|
+
* The step id the recorder marks work that precedes the case's first step —
|
|
8834
|
+
* signing in, reaching the starting screen. Not a step of the case, and not
|
|
8835
|
+
* the same thing as an action it could not attribute at all.
|
|
8575
8836
|
*/
|
|
8576
|
-
|
|
8577
|
-
|
|
8578
|
-
|
|
8579
|
-
The mechanical draft below was compiled 1:1 from a recorded browser session — it is plain \`@playwright/test\` code with raw locators (e.g. \`page.getByRole("button", { name: "Submit" })\`). Rewrite it into the shape this repository's test suite actually uses: import and call the declared resources (page objects, step helpers, fixtures, shared constants) instead of inlining raw interactions, and follow the conventions.
|
|
8580
|
-
|
|
8581
|
-
Keep each step's assertions in the test body, under that step's own comment. A step's actions may move into a page object; what the step decides may not — it is the one thing a reviewer reads the file for, and an assertion inside a helper is invisible both to them and to the review that follows this pass.
|
|
8582
|
-
|
|
8583
|
-
Write the rewritten test to \`${suggestedPath}\` unless the conventions/examples clearly place tests elsewhere under the output directory. Locator preference when you do write raw locators: test id > accessible text > role > CSS.`;
|
|
8584
|
-
}
|
|
8837
|
+
const SETUP_STEP_ID = "setup";
|
|
8838
|
+
//#endregion
|
|
8839
|
+
//#region src/ir/route-diff.ts
|
|
8585
8840
|
/**
|
|
8586
|
-
*
|
|
8587
|
-
*
|
|
8588
|
-
*
|
|
8589
|
-
*
|
|
8841
|
+
* What changed between two recordings of the same spec.
|
|
8842
|
+
*
|
|
8843
|
+
* Re-recording replaces the route wholesale, and the interesting question
|
|
8844
|
+
* afterwards is never "what does the new route do" but "what moved". A locator
|
|
8845
|
+
* that changed shape says the screen was rebuilt; a check whose wording changed
|
|
8846
|
+
* says the copy moved; a step that gained operations says the flow grew. That
|
|
8847
|
+
* is the evidence a reviewer needs, and reading it out of two JSON files by eye
|
|
8848
|
+
* is exactly the work nobody does.
|
|
8849
|
+
*
|
|
8850
|
+
* The diff is per spec step, because a step is the unit the spec author wrote
|
|
8851
|
+
* and the unit a reviewer reasons in. Actions carry their `stepId` from the
|
|
8852
|
+
* trace, so the grouping is recorded, not guessed.
|
|
8590
8853
|
*/
|
|
8591
|
-
|
|
8592
|
-
|
|
8593
|
-
|
|
8594
|
-
|
|
8595
|
-
|
|
8596
|
-
|
|
8597
|
-
|
|
8598
|
-
|
|
8599
|
-
|
|
8600
|
-
|
|
8601
|
-
|
|
8602
|
-
Write the runbook to \`${suggestedPath}\` unless the conventions/examples clearly use another layout under the output directory. If shared setup deserves its own helper runbook, output it as \`kind: "support"\`.`;
|
|
8854
|
+
const NO_STEP = "(no step)";
|
|
8855
|
+
/** One action as a reviewer reads it: what it did, to what, with which value. */
|
|
8856
|
+
function describeAction(action) {
|
|
8857
|
+
const parts = [action.assert ? `assert ${action.assert}` : action.action];
|
|
8858
|
+
if (action.locator) parts.push(renderLocator(action.locator));
|
|
8859
|
+
if (action.index !== void 0) parts.push(`[${action.index}]`);
|
|
8860
|
+
if (action.value !== void 0 && action.value !== "") parts.push(JSON.stringify(action.value));
|
|
8861
|
+
if (action.target) parts.push(`→ ${renderLocator(action.target)}`);
|
|
8862
|
+
if (action.files?.length) parts.push(action.files.join(", "));
|
|
8863
|
+
return parts.join(" ");
|
|
8603
8864
|
}
|
|
8604
8865
|
/**
|
|
8605
|
-
*
|
|
8606
|
-
*
|
|
8607
|
-
*
|
|
8608
|
-
* hold, so the cap only stops the pathological case.
|
|
8609
|
-
*/
|
|
8610
|
-
const MAX_SUPPORT_FILES = 200;
|
|
8611
|
-
/** Extensions tried when a specifier names no file, TypeScript sources first. */
|
|
8612
|
-
const EXTENSIONS = [
|
|
8613
|
-
".ts",
|
|
8614
|
-
".tsx",
|
|
8615
|
-
".mts",
|
|
8616
|
-
".cts",
|
|
8617
|
-
".js",
|
|
8618
|
-
".jsx",
|
|
8619
|
-
".mjs",
|
|
8620
|
-
".cjs"
|
|
8621
|
-
];
|
|
8622
|
-
/**
|
|
8623
|
-
* Module specifiers, in the forms generated tests use: `from "x"`, a
|
|
8624
|
-
* side-effect `import "x"`, `import("x")`, and `require("x")`. A string this
|
|
8625
|
-
* over-matches (prose after the word "from" in a comment) resolves to no file
|
|
8626
|
-
* and drops out.
|
|
8866
|
+
* The same, prefixed by the step it belongs to. What a warning about one
|
|
8867
|
+
* action says, so a reader can find it in the case rather than counting
|
|
8868
|
+
* positions in a list.
|
|
8627
8869
|
*/
|
|
8628
|
-
|
|
8870
|
+
function describeStepAction(action) {
|
|
8871
|
+
return `${action.stepId ? `${action.stepId} ` : ""}${describeAction(action)}`;
|
|
8872
|
+
}
|
|
8629
8873
|
/**
|
|
8630
|
-
*
|
|
8631
|
-
*
|
|
8632
|
-
*
|
|
8633
|
-
* A config published as a package is not followed: its aliases would resolve
|
|
8634
|
-
* inside `node_modules`, which this walker never reads anyway. Absent,
|
|
8635
|
-
* unreadable, or aliasless: null, and only relative imports are followed.
|
|
8874
|
+
* A locator in full, unlike `describeLocator`'s log-line summary: two locators
|
|
8875
|
+
* that differ must render differently, or the diff reports a re-addressed
|
|
8876
|
+
* element as unchanged — the very thing it exists to show.
|
|
8636
8877
|
*/
|
|
8637
|
-
|
|
8638
|
-
|
|
8639
|
-
const
|
|
8640
|
-
|
|
8641
|
-
|
|
8642
|
-
|
|
8643
|
-
if (parsed === null) return null;
|
|
8644
|
-
const options = parsed.compilerOptions;
|
|
8645
|
-
if (options?.paths) return {
|
|
8646
|
-
baseUrlAbs: resolve(dirname(configPath), options.baseUrl ?? "."),
|
|
8647
|
-
paths: options.paths
|
|
8648
|
-
};
|
|
8649
|
-
configPath = typeof parsed.extends === "string" && parsed.extends.startsWith(".") ? withJsonExtension(resolve(dirname(configPath), parsed.extends)) : null;
|
|
8650
|
-
}
|
|
8651
|
-
return null;
|
|
8652
|
-
}
|
|
8653
|
-
async function readTsconfig(path) {
|
|
8654
|
-
const raw = await readFile(path, "utf8").catch(() => null);
|
|
8655
|
-
if (raw === null) return null;
|
|
8656
|
-
try {
|
|
8657
|
-
return JSON.parse(stripJsonComments(raw));
|
|
8658
|
-
} catch {
|
|
8659
|
-
return null;
|
|
8878
|
+
function renderLocator(locator) {
|
|
8879
|
+
if (locator.by === "css") return locator.value;
|
|
8880
|
+
const exact = locator.exact ? " exact" : "";
|
|
8881
|
+
if (locator.by === "role") {
|
|
8882
|
+
const name = locator.name === void 0 ? "" : `[name=${JSON.stringify(locator.name)}]`;
|
|
8883
|
+
return `role=${locator.value}${name}${exact}`;
|
|
8660
8884
|
}
|
|
8661
|
-
}
|
|
8662
|
-
/** `extends` may name a config with the extension left off. */
|
|
8663
|
-
function withJsonExtension(path) {
|
|
8664
|
-
return path.endsWith(".json") ? path : `${path}.json`;
|
|
8665
|
-
}
|
|
8666
|
-
/** Strip line and block comments and trailing commas — tsconfig.json is JSONC. */
|
|
8667
|
-
function stripJsonComments(text) {
|
|
8668
|
-
return text.replace(/"(?:[^"\\]|\\.)*"|\/\*[\s\S]*?\*\/|\/\/[^\n]*/g, (match) => match.startsWith("\"") ? match : "").replace(/,(\s*[}\]])/g, "$1");
|
|
8885
|
+
return `${locator.by}=${JSON.stringify(locator.value)}${exact}`;
|
|
8669
8886
|
}
|
|
8670
8887
|
/**
|
|
8671
|
-
*
|
|
8672
|
-
*
|
|
8673
|
-
*
|
|
8674
|
-
*
|
|
8888
|
+
* Identity of an action across recordings: what it did and to which named
|
|
8889
|
+
* thing. Deliberately blind to the locator and the exact value, so a button
|
|
8890
|
+
* addressed a new way still pairs with its old self and reads as "changed"
|
|
8891
|
+
* rather than as one removal plus one addition.
|
|
8675
8892
|
*/
|
|
8676
|
-
|
|
8677
|
-
return
|
|
8893
|
+
function identity(action) {
|
|
8894
|
+
return [
|
|
8895
|
+
action.action,
|
|
8896
|
+
action.assert ?? "",
|
|
8897
|
+
action.label ?? action.locator?.value ?? ""
|
|
8898
|
+
].join("|");
|
|
8678
8899
|
}
|
|
8679
|
-
|
|
8680
|
-
|
|
8681
|
-
const
|
|
8682
|
-
const
|
|
8683
|
-
const
|
|
8684
|
-
const
|
|
8685
|
-
|
|
8686
|
-
for (
|
|
8687
|
-
const
|
|
8688
|
-
|
|
8689
|
-
|
|
8690
|
-
|
|
8691
|
-
|
|
8692
|
-
|
|
8693
|
-
if (seen.has(entry.abs)) continue;
|
|
8694
|
-
seen.add(entry.abs);
|
|
8695
|
-
found.push(entry);
|
|
8696
|
-
next.push(entry.abs);
|
|
8697
|
-
if (found.length >= MAX_SUPPORT_FILES) return {
|
|
8698
|
-
files: found,
|
|
8699
|
-
truncated: true
|
|
8700
|
-
};
|
|
8701
|
-
}
|
|
8702
|
-
frontier = next;
|
|
8900
|
+
function diffRoutes(before, after) {
|
|
8901
|
+
const moved = findMoves(before, after);
|
|
8902
|
+
const movedForms = new Set(moved.map((m) => m.action));
|
|
8903
|
+
const stayed = (a) => !movedForms.has(describeAction(a));
|
|
8904
|
+
const inStep = (actions, step) => actions.filter((a) => (a.stepId ?? NO_STEP) === step && stayed(a));
|
|
8905
|
+
const changes = [];
|
|
8906
|
+
const unchangedSteps = [];
|
|
8907
|
+
for (const step of orderedSteps(before, after)) {
|
|
8908
|
+
const wasIn = inStep(before, step);
|
|
8909
|
+
const isIn = inStep(after, step);
|
|
8910
|
+
if (wasIn.length === 0 && isIn.length === 0) continue;
|
|
8911
|
+
const stepChanges = diffStep(step, wasIn, isIn);
|
|
8912
|
+
if (stepChanges.length === 0) unchangedSteps.push(step);
|
|
8913
|
+
else changes.push(...stepChanges);
|
|
8703
8914
|
}
|
|
8704
8915
|
return {
|
|
8705
|
-
|
|
8706
|
-
|
|
8916
|
+
changes,
|
|
8917
|
+
moved,
|
|
8918
|
+
unchangedSteps
|
|
8707
8919
|
};
|
|
8708
8920
|
}
|
|
8709
|
-
/** Every project file one file imports, in source order. */
|
|
8710
|
-
async function resolveImportsOf(fileAbs, cwd, tsconfig) {
|
|
8711
|
-
const source = await readFile(fileAbs, "utf8").catch(() => null);
|
|
8712
|
-
if (source === null) return [];
|
|
8713
|
-
const specifiers = [...source.matchAll(IMPORT_SPECIFIER)].map((m) => m[1]);
|
|
8714
|
-
return (await Promise.all(specifiers.map((s) => resolveSpecifier(s, dirname(fileAbs), cwd, tsconfig)))).filter((p) => p !== null);
|
|
8715
|
-
}
|
|
8716
8921
|
/**
|
|
8717
|
-
*
|
|
8718
|
-
*
|
|
8719
|
-
*
|
|
8922
|
+
* Operations both recordings hold under a different step id, matched on the
|
|
8923
|
+
* rendered form: same operation, same locator, same value. A form appearing
|
|
8924
|
+
* more than once in either recording is left to the per-step diff — which of
|
|
8925
|
+
* two identical operations moved is a question the recording cannot answer.
|
|
8720
8926
|
*/
|
|
8721
|
-
|
|
8722
|
-
const
|
|
8723
|
-
|
|
8724
|
-
const
|
|
8725
|
-
|
|
8726
|
-
|
|
8727
|
-
|
|
8927
|
+
function findMoves(before, after) {
|
|
8928
|
+
const index = (actions) => {
|
|
8929
|
+
const seen = /* @__PURE__ */ new Map();
|
|
8930
|
+
const repeated = /* @__PURE__ */ new Set();
|
|
8931
|
+
for (const action of actions) {
|
|
8932
|
+
const form = describeAction(action);
|
|
8933
|
+
if (seen.has(form)) repeated.add(form);
|
|
8934
|
+
seen.set(form, action);
|
|
8935
|
+
}
|
|
8936
|
+
for (const form of repeated) seen.delete(form);
|
|
8937
|
+
return seen;
|
|
8938
|
+
};
|
|
8939
|
+
const b = index(before);
|
|
8940
|
+
const moved = [];
|
|
8941
|
+
for (const [form, action] of index(after)) {
|
|
8942
|
+
const was = b.get(form);
|
|
8943
|
+
if (was === void 0) continue;
|
|
8944
|
+
const from = was.stepId ?? NO_STEP;
|
|
8945
|
+
const to = action.stepId ?? NO_STEP;
|
|
8946
|
+
if (from !== to) moved.push({
|
|
8947
|
+
action: form,
|
|
8948
|
+
from,
|
|
8949
|
+
to
|
|
8950
|
+
});
|
|
8728
8951
|
}
|
|
8729
|
-
return
|
|
8952
|
+
return moved;
|
|
8730
8953
|
}
|
|
8731
|
-
/** Every
|
|
8732
|
-
function
|
|
8733
|
-
|
|
8954
|
+
/** Every step id either recording mentions, in the new recording's order first. */
|
|
8955
|
+
function orderedSteps(before, after) {
|
|
8956
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8734
8957
|
const out = [];
|
|
8735
|
-
for (const [
|
|
8736
|
-
const
|
|
8737
|
-
if (
|
|
8738
|
-
|
|
8739
|
-
|
|
8740
|
-
}
|
|
8741
|
-
const prefix = pattern.slice(0, star);
|
|
8742
|
-
const suffix = pattern.slice(star + 1);
|
|
8743
|
-
if (!specifier.startsWith(prefix) || !specifier.endsWith(suffix)) continue;
|
|
8744
|
-
const matched = specifier.slice(prefix.length, specifier.length - suffix.length);
|
|
8745
|
-
out.push(...targets.map((t) => resolve(tsconfig.baseUrlAbs, t.replace("*", matched))));
|
|
8958
|
+
for (const action of [...after, ...before]) {
|
|
8959
|
+
const step = action.stepId ?? NO_STEP;
|
|
8960
|
+
if (seen.has(step)) continue;
|
|
8961
|
+
seen.add(step);
|
|
8962
|
+
out.push(step);
|
|
8746
8963
|
}
|
|
8747
8964
|
return out;
|
|
8748
8965
|
}
|
|
8749
|
-
/**
|
|
8750
|
-
|
|
8751
|
-
|
|
8752
|
-
|
|
8753
|
-
|
|
8754
|
-
|
|
8966
|
+
/**
|
|
8967
|
+
* Align one step's actions by identity (longest common subsequence), then
|
|
8968
|
+
* report the leftovers. An aligned pair whose rendered form differs is the
|
|
8969
|
+
* interesting case: same operation, new locator or new wording.
|
|
8970
|
+
*/
|
|
8971
|
+
function diffStep(step, before, after) {
|
|
8972
|
+
const changes = [];
|
|
8973
|
+
for (const entry of alignByIdentity(before, after)) if (entry.before && entry.after) {
|
|
8974
|
+
const b = describeAction(entry.before);
|
|
8975
|
+
const a = describeAction(entry.after);
|
|
8976
|
+
if (b !== a) changes.push({
|
|
8977
|
+
kind: "changed",
|
|
8978
|
+
step,
|
|
8979
|
+
before: b,
|
|
8980
|
+
after: a
|
|
8981
|
+
});
|
|
8982
|
+
} else if (entry.after) changes.push({
|
|
8983
|
+
kind: "added",
|
|
8984
|
+
step,
|
|
8985
|
+
after: describeAction(entry.after)
|
|
8986
|
+
});
|
|
8987
|
+
else if (entry.before) changes.push({
|
|
8988
|
+
kind: "removed",
|
|
8989
|
+
step,
|
|
8990
|
+
before: describeAction(entry.before)
|
|
8991
|
+
});
|
|
8992
|
+
return changes;
|
|
8993
|
+
}
|
|
8994
|
+
function alignByIdentity(before, after) {
|
|
8995
|
+
const lengths = Array.from({ length: before.length + 1 }, () => new Array(after.length + 1).fill(0));
|
|
8996
|
+
for (let i = before.length - 1; i >= 0; i--) for (let j = after.length - 1; j >= 0; j--) lengths[i][j] = identity(before[i]) === identity(after[j]) ? lengths[i + 1][j + 1] + 1 : Math.max(lengths[i + 1][j], lengths[i][j + 1]);
|
|
8997
|
+
const out = [];
|
|
8998
|
+
let i = 0;
|
|
8999
|
+
let j = 0;
|
|
9000
|
+
while (i < before.length && j < after.length) if (identity(before[i]) === identity(after[j])) {
|
|
9001
|
+
out.push({
|
|
9002
|
+
before: before[i],
|
|
9003
|
+
after: after[j]
|
|
9004
|
+
});
|
|
9005
|
+
i++;
|
|
9006
|
+
j++;
|
|
9007
|
+
} else if (lengths[i + 1][j] >= lengths[i][j + 1]) out.push({ before: before[i++] });
|
|
9008
|
+
else out.push({ after: after[j++] });
|
|
9009
|
+
while (i < before.length) out.push({ before: before[i++] });
|
|
9010
|
+
while (j < after.length) out.push({ after: after[j++] });
|
|
9011
|
+
return out;
|
|
9012
|
+
}
|
|
9013
|
+
/** The diff as markdown — what `ccqa record` saves beside the spec. */
|
|
9014
|
+
function renderRouteDiff(diff, report) {
|
|
9015
|
+
const { changes, moved, unchangedSteps } = diff;
|
|
9016
|
+
const lines = [`# Route diff — ${report.specKey}`, ""];
|
|
9017
|
+
lines.push(`Recorded ${report.before.recordedAt ?? "(unknown)"} → ${report.after.recordedAt ?? "(unknown)"}.`, "");
|
|
9018
|
+
if (report.before.origin !== report.after.origin) lines.push(`Origin: \`${report.before.origin ?? "(none)"}\` → \`${report.after.origin ?? "(none)"}\``, "");
|
|
9019
|
+
if (changes.length === 0 && moved.length === 0) {
|
|
9020
|
+
lines.push("The route is unchanged: same operations, locators and checks.", "");
|
|
9021
|
+
return lines.join("\n");
|
|
8755
9022
|
}
|
|
8756
|
-
|
|
8757
|
-
for (const
|
|
8758
|
-
const
|
|
8759
|
-
|
|
9023
|
+
if (changes.length === 0) lines.push("The route does the same things, in the same way. Only which step each was recorded under moved.", "");
|
|
9024
|
+
for (const step of groupByStep$1(changes)) {
|
|
9025
|
+
const title = report.stepTitles?.get(step.step);
|
|
9026
|
+
lines.push(`## ${step.step}${title ? ` — ${title}` : ""}`, "");
|
|
9027
|
+
for (const change of step.changes) lines.push(change.kind === "changed" ? `- changed: \`${change.before}\` → \`${change.after}\`` : change.kind === "added" ? `- added: \`${change.after}\`` : `- removed: \`${change.before}\``);
|
|
9028
|
+
lines.push("");
|
|
8760
9029
|
}
|
|
8761
|
-
|
|
8762
|
-
|
|
8763
|
-
|
|
8764
|
-
|
|
9030
|
+
if (moved.length > 0) {
|
|
9031
|
+
lines.push("## Recorded under a different step", "", "Same operation, same locator, same value — only its step attribution moved.", "");
|
|
9032
|
+
for (const move of moved) lines.push(`- \`${move.action}\`: ${move.from} → ${move.to}`);
|
|
9033
|
+
lines.push("");
|
|
9034
|
+
}
|
|
9035
|
+
if (unchangedSteps.length > 0) lines.push(`Unchanged: ${unchangedSteps.join(", ")}`, "");
|
|
9036
|
+
return lines.join("\n");
|
|
8765
9037
|
}
|
|
8766
|
-
|
|
8767
|
-
const
|
|
8768
|
-
const
|
|
8769
|
-
const
|
|
8770
|
-
|
|
8771
|
-
|
|
8772
|
-
|
|
8773
|
-
|
|
8774
|
-
|
|
8775
|
-
* Text, not a parser: this answers a warning, not a refusal, and a name missed
|
|
8776
|
-
* because it was written in a form this does not know costs nothing. A name
|
|
8777
|
-
* invented would cost a false warning, so the patterns only match declarations
|
|
8778
|
-
* that plainly are one, and comments are stripped first. Types are left out
|
|
8779
|
-
* for the same reason: a page object's own row type is referenced only inside
|
|
8780
|
-
* it, so listing it would report a definition that is used.
|
|
8781
|
-
*/
|
|
8782
|
-
function exportedNames(source) {
|
|
8783
|
-
const code = source.replace(BLOCK_COMMENT, "").replace(LINE_COMMENT, "$1");
|
|
8784
|
-
const names = /* @__PURE__ */ new Set();
|
|
8785
|
-
for (const m of code.matchAll(EXPORTED)) names.add(m[1]);
|
|
8786
|
-
for (const m of code.matchAll(EXPORT_LIST)) for (const part of m[1].split(",")) {
|
|
8787
|
-
const name = part.trim().split(/\s+as\s+/).pop()?.trim();
|
|
8788
|
-
if (name && /^[A-Za-z_$][\w$]*$/.test(name) && name !== "default") names.add(name);
|
|
9038
|
+
function groupByStep$1(changes) {
|
|
9039
|
+
const grouped = [];
|
|
9040
|
+
for (const change of changes) {
|
|
9041
|
+
const last = grouped.at(-1);
|
|
9042
|
+
if (last && last.step === change.step) last.changes.push(change);
|
|
9043
|
+
else grouped.push({
|
|
9044
|
+
step: change.step,
|
|
9045
|
+
changes: [change]
|
|
9046
|
+
});
|
|
8789
9047
|
}
|
|
8790
|
-
return
|
|
9048
|
+
return grouped;
|
|
8791
9049
|
}
|
|
8792
9050
|
//#endregion
|
|
8793
|
-
//#region src/
|
|
8794
|
-
|
|
8795
|
-
|
|
8796
|
-
|
|
8797
|
-
|
|
8798
|
-
|
|
8799
|
-
|
|
8800
|
-
|
|
8801
|
-
|
|
8802
|
-
...containerOfPageText(file, source),
|
|
8803
|
-
...unjustifiedFirst(file, source),
|
|
8804
|
-
...describeEcho(file, source),
|
|
8805
|
-
...unaskedAssertions(file, source, locators, said),
|
|
8806
|
-
...weakerTwin(file, source),
|
|
8807
|
-
...unreached(file, source, input)
|
|
8808
|
-
])
|
|
8809
|
-
];
|
|
8810
|
-
}
|
|
8811
|
-
/**
|
|
8812
|
-
* A path with a placeholder segment in it — `/orders/{orderId}`. Writing one
|
|
8813
|
-
* in prose is a statement about the address, not about the page's contents:
|
|
8814
|
-
* there is no other reason to name the part that changes per run.
|
|
9051
|
+
//#region src/evidence/table.ts
|
|
9052
|
+
/** Assertion lines a reviewer can check without reading the whole file. */
|
|
9053
|
+
const ASSERTION = /^\s*(?:await\s+)?(?:expect|judgeByLlm)\b.*$/;
|
|
9054
|
+
/**
|
|
9055
|
+
* Assertions grouped by the step they sit under, read from the boundary the
|
|
9056
|
+
* emitter leaves — a `test.step` title for a Playwright test, a comment for an
|
|
9057
|
+
* agent-browser one. A rewrite that moved an assertion into a page object
|
|
9058
|
+
* leaves nothing here, which is the honest answer: the reviewer cannot see it
|
|
9059
|
+
* either, and the row says so rather than implying coverage.
|
|
8815
9060
|
*/
|
|
8816
|
-
|
|
9061
|
+
function assertionsByStep(source) {
|
|
9062
|
+
const byStep = /* @__PURE__ */ new Map();
|
|
9063
|
+
let current = null;
|
|
9064
|
+
for (const line of stepLines(source)) {
|
|
9065
|
+
const boundary = parseStepComment(line);
|
|
9066
|
+
if (boundary) {
|
|
9067
|
+
current = boundary;
|
|
9068
|
+
if (!byStep.has(current)) byStep.set(current, []);
|
|
9069
|
+
continue;
|
|
9070
|
+
}
|
|
9071
|
+
if (current && ASSERTION.test(line)) byStep.get(current).push(line.trim());
|
|
9072
|
+
}
|
|
9073
|
+
return byStep;
|
|
9074
|
+
}
|
|
8817
9075
|
/**
|
|
8818
|
-
*
|
|
8819
|
-
* the
|
|
8820
|
-
*
|
|
8821
|
-
* The row being on screen is not the same claim as being on the screen that
|
|
8822
|
-
* shows it — a product that rendered the row without navigating passes, and
|
|
8823
|
-
* that is the failure the expectation was written to catch. Scoped to
|
|
8824
|
-
* placeholder paths because those are unambiguous; a path mentioned as scenery
|
|
8825
|
-
* ("on /settings, the button is shown") is not a claim about the address.
|
|
8826
|
-
*
|
|
8827
|
-
* Calibrated against a small set of hand-written case definitions: a few
|
|
8828
|
-
* name a placeholder path in their expectations, most of those already
|
|
8829
|
-
* assert on the URL, and the one that does not is the defect this was
|
|
8830
|
-
* written for.
|
|
9076
|
+
* Recorded actions grouped by the step that produced them. `"(no step)"` is
|
|
9077
|
+
* the bucket for an action the recorder could not attribute — kept rather than
|
|
9078
|
+
* dropped, since it still happened.
|
|
8831
9079
|
*/
|
|
8832
|
-
function
|
|
8833
|
-
const
|
|
8834
|
-
|
|
8835
|
-
|
|
8836
|
-
|
|
8837
|
-
|
|
8838
|
-
|
|
8839
|
-
|
|
8840
|
-
|
|
8841
|
-
|
|
8842
|
-
|
|
9080
|
+
function groupByStep(actions) {
|
|
9081
|
+
const byStep = /* @__PURE__ */ new Map();
|
|
9082
|
+
for (const action of actions) {
|
|
9083
|
+
const id = action.stepId ?? "(no step)";
|
|
9084
|
+
const list = byStep.get(id);
|
|
9085
|
+
if (list) list.push(action);
|
|
9086
|
+
else byStep.set(id, [action]);
|
|
9087
|
+
}
|
|
9088
|
+
return byStep;
|
|
9089
|
+
}
|
|
9090
|
+
/** The same grouping, each action rendered for a reader. */
|
|
9091
|
+
function actionsByStep(actions) {
|
|
9092
|
+
return new Map([...groupByStep(actions)].map(([id, grouped]) => [id, grouped.map(describeAction)]));
|
|
8843
9093
|
}
|
|
8844
|
-
/** `readonly name =`, `name(...): Locator {`, and `export … name`. */
|
|
8845
|
-
const DECLARED = /(?:readonly\s+([A-Za-z0-9_]+)\s*=|\b([A-Za-z0-9_]+)\([^)]*\)\s*:\s*Locator\b|export\s+(?:async\s+)?(?:const|let|function|class)\s+([A-Za-z0-9_]+))/g;
|
|
8846
9094
|
/**
|
|
8847
|
-
*
|
|
8848
|
-
*
|
|
9095
|
+
* The locator/assertion literals worth anchoring to the product's own source:
|
|
9096
|
+
* a testid, an accessible name, a placeholder, a label, an asserted text. Not
|
|
9097
|
+
* every locator strategy — `css`, `text` and `alt`/`title` locators are as
|
|
9098
|
+
* often a CSS/testing artifact as product copy, and would anchor to noise.
|
|
8849
9099
|
*
|
|
8850
|
-
*
|
|
8851
|
-
*
|
|
8852
|
-
*
|
|
9100
|
+
* A value containing `${...}` is a run-id-substituted value at replay time
|
|
9101
|
+
* and has no literal counterpart in the source, so it is dropped rather than
|
|
9102
|
+
* searched for and reported "not found".
|
|
8853
9103
|
*/
|
|
8854
|
-
function
|
|
8855
|
-
const
|
|
8856
|
-
|
|
8857
|
-
const
|
|
8858
|
-
|
|
8859
|
-
|
|
8860
|
-
|
|
8861
|
-
|
|
8862
|
-
|
|
8863
|
-
|
|
8864
|
-
|
|
8865
|
-
|
|
8866
|
-
|
|
8867
|
-
|
|
8868
|
-
|
|
8869
|
-
|
|
8870
|
-
|
|
9104
|
+
function sourceNeedles(actions) {
|
|
9105
|
+
const needles = [];
|
|
9106
|
+
const seen = /* @__PURE__ */ new Set();
|
|
9107
|
+
const add = (value, kind = "text") => {
|
|
9108
|
+
if (value === void 0 || value.includes("${") || seen.has(value)) return;
|
|
9109
|
+
seen.add(value);
|
|
9110
|
+
needles.push({
|
|
9111
|
+
value,
|
|
9112
|
+
kind
|
|
9113
|
+
});
|
|
9114
|
+
};
|
|
9115
|
+
for (const action of actions) {
|
|
9116
|
+
for (const locator of [action.locator, action.target]) {
|
|
9117
|
+
if (!locator) continue;
|
|
9118
|
+
if (locator.by === "testid") add(locator.value, "testid");
|
|
9119
|
+
else if (locator.by === "placeholder" || locator.by === "label") add(locator.value);
|
|
9120
|
+
else if (locator.by === "role") add(locator.name);
|
|
9121
|
+
}
|
|
9122
|
+
if (action.assert === "text_visible" || action.assert === "text_not_visible") add(action.value);
|
|
9123
|
+
}
|
|
9124
|
+
return needles;
|
|
9125
|
+
}
|
|
9126
|
+
function buildEvidenceSteps(input) {
|
|
9127
|
+
const byStep = groupByStep([...input.recording.actions, ...input.recording.cleanup ?? []]);
|
|
9128
|
+
const assertions = assertionsByStep(input.test.source);
|
|
9129
|
+
return [...input.testCase.steps, ...input.testCase.cleanup].map((step) => {
|
|
9130
|
+
const actions = byStep.get(step.id) ?? [];
|
|
9131
|
+
return {
|
|
9132
|
+
id: step.id,
|
|
9133
|
+
instruction: "instruction" in step ? step.instruction : step.judgeByLlm,
|
|
9134
|
+
expected: "expected" in step ? step.expected : "",
|
|
9135
|
+
actions: actions.map(describeAction),
|
|
9136
|
+
assertions: assertions.get(step.id) ?? [],
|
|
9137
|
+
screenshots: input.screenshots.get(step.id) ?? [],
|
|
9138
|
+
needles: sourceNeedles(actions)
|
|
9139
|
+
};
|
|
8871
9140
|
});
|
|
8872
9141
|
}
|
|
8873
|
-
|
|
9142
|
+
/** Entries past this many fold away: a cell taller than the eye stops being read. */
|
|
9143
|
+
const FOLD_OVER = 2;
|
|
9144
|
+
/** The evidence as markdown — a fragment, for whoever assembles the PR body. */
|
|
9145
|
+
function renderEvidence(input) {
|
|
9146
|
+
const labels = evidenceLabels(input.labels, input.language);
|
|
9147
|
+
const steps = buildEvidenceSteps(input);
|
|
9148
|
+
const anchors = input.anchors;
|
|
9149
|
+
const lines = [
|
|
9150
|
+
`# ${input.testCase.title}`,
|
|
9151
|
+
"",
|
|
9152
|
+
`${labels.case}: \`${input.testCase.ref.id}\``,
|
|
9153
|
+
`${labels.test}: \`${input.test.path}\``,
|
|
9154
|
+
`${labels.recordedAt}: ${input.recording.recordedAt ?? "(unknown)"}`,
|
|
9155
|
+
...input.recording.origin ? [`${labels.from}: \`${input.recording.origin}\``] : [],
|
|
9156
|
+
""
|
|
9157
|
+
];
|
|
9158
|
+
for (const [label, actions] of outsideTheCase(input, steps, labels)) lines.push(`${label}: ${fold(actions, labels)}`, "");
|
|
9159
|
+
lines.push(`| ${labels.step} | ${labels.instruction} | ${labels.recorded} | ${labels.decides}` + (anchors ? ` | ${labels.source}` : "") + ` | ${labels.screens} |`, "|---|---|---|---|" + (anchors ? "---|" : "") + "---|");
|
|
9160
|
+
for (const step of steps) {
|
|
9161
|
+
const cells = [
|
|
9162
|
+
step.id,
|
|
9163
|
+
cell$1(step.instruction),
|
|
9164
|
+
fold(step.actions, labels),
|
|
9165
|
+
step.assertions.length > 0 ? cell$1(step.assertions.join("<br>")) : `**${labels.nothing}**`,
|
|
9166
|
+
...anchors ? [sourceAnchorCells(step.needles, anchors, labels)] : [],
|
|
9167
|
+
step.screenshots.map((path) => ``).join(" ") || "—"
|
|
9168
|
+
];
|
|
9169
|
+
lines.push(`| ${cells.join(" | ")} |`);
|
|
9170
|
+
}
|
|
9171
|
+
lines.push("");
|
|
9172
|
+
if (input.testCase.expectations.length > 0) {
|
|
9173
|
+
lines.push(`## ${labels.expects}`, "");
|
|
9174
|
+
for (const expectation of input.testCase.expectations) lines.push(`- ${expectation}`);
|
|
9175
|
+
lines.push("");
|
|
9176
|
+
}
|
|
9177
|
+
if (input.cleanupUnchecked && input.cleanupUnchecked.length > 0) {
|
|
9178
|
+
lines.push(labels.cleanupUnchecked, "");
|
|
9179
|
+
for (const expectation of input.cleanupUnchecked) lines.push(`- ${expectation}`);
|
|
9180
|
+
lines.push("");
|
|
9181
|
+
}
|
|
9182
|
+
const findings = mergeFindings(steps.filter((step) => claimsAnOutcome(step) && step.assertions.length === 0).map((step) => ({
|
|
9183
|
+
stepId: step.id,
|
|
9184
|
+
problem: NOTHING_DECIDED
|
|
9185
|
+
})), input.review ?? []);
|
|
9186
|
+
lines.push(`## ${labels.review}`, "");
|
|
9187
|
+
if (findings.length === 0) lines.push(input.review === void 0 ? labels.reviewAbsent : labels.reviewClean);
|
|
9188
|
+
else {
|
|
9189
|
+
for (const finding of findings) lines.push(`- ${formatFinding(finding, labels)}`);
|
|
9190
|
+
if (input.review === void 0) lines.push("", labels.reviewPartial);
|
|
9191
|
+
}
|
|
9192
|
+
lines.push("");
|
|
9193
|
+
return lines.join("\n");
|
|
9194
|
+
}
|
|
9195
|
+
/** A table cell: pipes escaped, newlines folded, code voice for the machine parts. */
|
|
9196
|
+
function cell$1(text) {
|
|
9197
|
+
const clean = text.replaceAll("|", "\\|").replaceAll("\n", "<br>");
|
|
9198
|
+
return clean.length === 0 ? "—" : clean;
|
|
9199
|
+
}
|
|
8874
9200
|
/**
|
|
8875
|
-
*
|
|
8876
|
-
*
|
|
8877
|
-
*
|
|
8878
|
-
*
|
|
8879
|
-
* The weak one adds no check — anything it could catch, the other catches
|
|
8880
|
-
* first — and it is satisfied by whichever element the page happens to order
|
|
8881
|
-
* first, which on a real page was the navigation entry rather than the heading
|
|
8882
|
-
* the case names. Both assertions then read as two checks and are one.
|
|
8883
|
-
*
|
|
8884
|
-
* Calibrated against a hand-written corpus: the shape appears only in a file
|
|
8885
|
-
* a generation wrote. People do not write it.
|
|
9201
|
+
* One line per needle. "not found" is a claim about the product, so a needle
|
|
9202
|
+
* the scan never looked for says so instead — the reviewer's whole use of this
|
|
9203
|
+
* column is telling those two apart.
|
|
8886
9204
|
*/
|
|
8887
|
-
function
|
|
8888
|
-
const
|
|
8889
|
-
|
|
8890
|
-
|
|
8891
|
-
|
|
8892
|
-
|
|
8893
|
-
|
|
8894
|
-
|
|
8895
|
-
|
|
8896
|
-
|
|
8897
|
-
const value = m[1] ?? m[2];
|
|
8898
|
-
if (!value || value.includes("${")) continue;
|
|
8899
|
-
const seen = byString.get(value) ?? [];
|
|
8900
|
-
seen.push({
|
|
8901
|
-
name,
|
|
8902
|
-
line: i + 1,
|
|
8903
|
-
weak
|
|
8904
|
-
});
|
|
8905
|
-
byString.set(value, seen);
|
|
9205
|
+
function sourceAnchorCells(needles, anchors, labels) {
|
|
9206
|
+
const confirmed = [];
|
|
9207
|
+
const rest = [];
|
|
9208
|
+
for (const { value } of needles) {
|
|
9209
|
+
const anchor = anchors.found.get(value);
|
|
9210
|
+
if (!anchor) rest.push(`\`${value}\` — ${anchors.unsearched.has(value) ? labels.notSearched : labels.notFound}`);
|
|
9211
|
+
else if (anchor.places.length > 1) rest.push(`\`${value}\` — ${labels.ambiguous}: ${anchor.places.join(", ")}`);
|
|
9212
|
+
else {
|
|
9213
|
+
const partial = anchor.partial ? ` (${labels.partialMatch})` : "";
|
|
9214
|
+
confirmed.push(`\`${value}\` — ${anchor.places[0]}${partial}`);
|
|
8906
9215
|
}
|
|
8907
|
-
});
|
|
8908
|
-
const findings = [];
|
|
8909
|
-
for (const [value, defs] of byString) {
|
|
8910
|
-
const weak = defs.filter((d) => d.weak);
|
|
8911
|
-
if (weak.length === 0 || defs.every((d) => d.weak)) continue;
|
|
8912
|
-
const strong = defs.find((d) => !d.weak);
|
|
8913
|
-
for (const d of weak) findings.push({
|
|
8914
|
-
file,
|
|
8915
|
-
line: d.line,
|
|
8916
|
-
rule: "weaker-twin",
|
|
8917
|
-
message: `\`${d.name}\` and \`${strong.name}\` are both "${value}", and this one searches the whole page and takes a position out of the matches. It can only be satisfied by something \`${strong.name}\` already covers, or by the wrong element. Assert one of them`
|
|
8918
|
-
});
|
|
8919
9216
|
}
|
|
8920
|
-
|
|
9217
|
+
const folded = rest.length === 0 ? [] : [`<details><summary>${rest.length} ${labels.unconfirmed}</summary>${rest.join("<br>")}</details>`];
|
|
9218
|
+
return cell$1([...confirmed, ...folded].join("<br>"));
|
|
9219
|
+
}
|
|
9220
|
+
/** A cell that stays one line: past a couple of entries, the rest folds away. */
|
|
9221
|
+
function fold(entries, labels) {
|
|
9222
|
+
if (entries.length <= FOLD_OVER) return cell$1(entries.join("<br>"));
|
|
9223
|
+
return `<details><summary>${entries.length} ${labels.operations}</summary>${cell$1(entries.join("<br>"))}</details>`;
|
|
9224
|
+
}
|
|
9225
|
+
/** Recorded actions belonging to no step of the case, kept apart by why. */
|
|
9226
|
+
function outsideTheCase(input, steps, labels) {
|
|
9227
|
+
const ofTheCase = new Set(steps.map((step) => step.id));
|
|
9228
|
+
const grouped = actionsByStep([...input.recording.actions, ...input.recording.cleanup ?? []]);
|
|
9229
|
+
const setup = [];
|
|
9230
|
+
const rest = [];
|
|
9231
|
+
for (const [id, actions] of grouped) if (!ofTheCase.has(id)) (id === "setup" ? setup : rest).push(...actions);
|
|
9232
|
+
const rows = [];
|
|
9233
|
+
if (setup.length > 0) rows.push([labels.setup, setup]);
|
|
9234
|
+
if (rest.length > 0) rows.push([labels.unattributed, rest]);
|
|
9235
|
+
return rows;
|
|
8921
9236
|
}
|
|
9237
|
+
//#endregion
|
|
9238
|
+
//#region src/targets/verifies-spec.ts
|
|
9239
|
+
const FindingsSchema = z.object({ findings: z.array(z.object({
|
|
9240
|
+
stepId: z.string(),
|
|
9241
|
+
problem: z.string()
|
|
9242
|
+
})) });
|
|
9243
|
+
const ViolationSchema = z.object({
|
|
9244
|
+
file: z.string(),
|
|
9245
|
+
guide: z.string(),
|
|
9246
|
+
rule: z.string(),
|
|
9247
|
+
code: z.string(),
|
|
9248
|
+
severity: z.enum(["blocking", "advisory"]).catch("blocking")
|
|
9249
|
+
});
|
|
9250
|
+
const ViolationsSchema = z.object({ ruleViolations: z.array(z.unknown()) });
|
|
8922
9251
|
/**
|
|
8923
|
-
*
|
|
8924
|
-
*
|
|
8925
|
-
* The per-step version of this question belongs to the reading, which knows
|
|
8926
|
-
* that "open the list" claims nothing and expects no assertion under it. This
|
|
8927
|
-
* one needs no judgement: whatever the case says, a file that decides nothing
|
|
8928
|
-
* at all does not check it. It is the floor under a case whose expectations
|
|
8929
|
-
* are stated for the flow rather than per step — the shape a markdown case has
|
|
8930
|
-
* — where the reading is otherwise the only thing looking.
|
|
9252
|
+
* How much reading one review may do.
|
|
8931
9253
|
*
|
|
8932
|
-
*
|
|
8933
|
-
*
|
|
9254
|
+
* A runaway guard, not a working limit: the reviewer opens the files under
|
|
9255
|
+
* review, then searches the suite for each convention it wants to check, and a
|
|
9256
|
+
* review that reaches this is looping rather than working. Reaching it costs
|
|
9257
|
+
* the review, not the generate — the answer comes back unusable and the loop
|
|
9258
|
+
* treats it as one it could not obtain.
|
|
8934
9259
|
*/
|
|
8935
|
-
|
|
8936
|
-
|
|
8937
|
-
|
|
8938
|
-
|
|
8939
|
-
|
|
8940
|
-
|
|
8941
|
-
|
|
8942
|
-
|
|
8943
|
-
|
|
8944
|
-
|
|
8945
|
-
|
|
8946
|
-
|
|
8947
|
-
|
|
8948
|
-
|
|
8949
|
-
|
|
8950
|
-
|
|
8951
|
-
|
|
8952
|
-
file,
|
|
8953
|
-
line: i + 1,
|
|
8954
|
-
rule: "tool-identifier",
|
|
8955
|
-
message: `\`${m[1]}\` names the tool that wrote it. Name it for what it holds`
|
|
8956
|
-
}] : [];
|
|
8957
|
-
});
|
|
9260
|
+
const MAX_REVIEW_TURNS = 80;
|
|
9261
|
+
/** A wedged call, not a slow one: a review still silent after this is stuck. */
|
|
9262
|
+
const REVIEW_TIMEOUT_MS = 10 * 6e4;
|
|
9263
|
+
/** Read-only: a reviewer reads the repository and changes nothing in it. */
|
|
9264
|
+
const REVIEW_TOOLS = [
|
|
9265
|
+
"Read",
|
|
9266
|
+
"Grep",
|
|
9267
|
+
"Glob"
|
|
9268
|
+
];
|
|
9269
|
+
function parseAnswer(answer, schema) {
|
|
9270
|
+
const json = extractJsonBlock(answer);
|
|
9271
|
+
if (!json) return null;
|
|
9272
|
+
try {
|
|
9273
|
+
return schema.parse(JSON.parse(json));
|
|
9274
|
+
} catch {
|
|
9275
|
+
return null;
|
|
9276
|
+
}
|
|
8958
9277
|
}
|
|
8959
9278
|
/**
|
|
8960
|
-
*
|
|
8961
|
-
*
|
|
8962
|
-
*
|
|
8963
|
-
* Both forms, because the first version tested only `getByText` and the fix
|
|
8964
|
-
* pass answered it with `page.locator("div").filter({ hasText }).last()` —
|
|
8965
|
-
* the same page-wide search, now resting on DOM order, and invisible to a
|
|
8966
|
-
* rule that only knew one shape. A generation that escapes a finding by
|
|
8967
|
-
* changing shape is worse than one that never drew it.
|
|
9279
|
+
* Findings in the model's answer, or null when it did not answer in the
|
|
9280
|
+
* agreed shape. Null is not "no findings": the caller says so rather than
|
|
9281
|
+
* reporting a clean review it never got.
|
|
8968
9282
|
*/
|
|
8969
|
-
|
|
8970
|
-
|
|
9283
|
+
function parseVerifiesSpecFindings(answer) {
|
|
9284
|
+
return parseAnswer(answer, FindingsSchema)?.findings ?? null;
|
|
9285
|
+
}
|
|
8971
9286
|
/**
|
|
8972
|
-
*
|
|
8973
|
-
*
|
|
8974
|
-
*
|
|
8975
|
-
*
|
|
8976
|
-
* The message names the shapes that fix it, because the first version said
|
|
8977
|
-
* only "scope it" and the fix pass answered with `page.locator("div")` filtered
|
|
8978
|
-
* by the same text and narrowed with `.last()` — page-wide still, and now
|
|
8979
|
-
* resting on which match the DOM happens to put last.
|
|
9287
|
+
* Rule violations in the same answer, or null when that half did not come
|
|
9288
|
+
* back. Read apart from the findings on purpose: one answer carries two
|
|
9289
|
+
* reviews, and a malformed half must not discard the other.
|
|
8980
9290
|
*/
|
|
8981
|
-
function
|
|
8982
|
-
const
|
|
8983
|
-
|
|
8984
|
-
|
|
8985
|
-
|
|
8986
|
-
|
|
8987
|
-
if (!PAGE_ROOTED.test(body)) return [];
|
|
8988
|
-
return [{
|
|
8989
|
-
file,
|
|
8990
|
-
line: i + 1,
|
|
8991
|
-
rule: "unscoped-container",
|
|
8992
|
-
message: `\`${m[1]}\` is named for one element among others, but matches that text anywhere on the page — a heading or a toast carrying it satisfies this too. Address the element itself: the role it has (\`getByRole("row"|"listitem"|"article"|...)\` narrowed with \`.filter({ hasText })\`), or its test id. Where the page offers neither, scope from a container this page object already addresses — searching the whole page for a bare \`div\` and taking \`.first()\`/\`.last()\` is not scoping, it only picks whichever the DOM happens to order that way.`
|
|
8993
|
-
}];
|
|
9291
|
+
function parseGuideViolations(answer) {
|
|
9292
|
+
const entries = parseAnswer(answer, ViolationsSchema)?.ruleViolations;
|
|
9293
|
+
if (entries === void 0) return null;
|
|
9294
|
+
const violations = entries.flatMap((entry) => {
|
|
9295
|
+
const parsed = ViolationSchema.safeParse(entry);
|
|
9296
|
+
return parsed.success ? [parsed.data] : [];
|
|
8994
9297
|
});
|
|
9298
|
+
if (violations.length < entries.length) warn(`${entries.length - violations.length} rule violation(s) came back unreadable and were dropped`);
|
|
9299
|
+
return violations;
|
|
8995
9300
|
}
|
|
8996
9301
|
/**
|
|
8997
|
-
* Whether a
|
|
9302
|
+
* Whether a step claims an outcome of its own. "Open the list", "click Add",
|
|
9303
|
+
* "type a title" do not: nothing follows from them that a test could check,
|
|
9304
|
+
* and a hand-written test asserts nothing after them either. A markdown case
|
|
9305
|
+
* states its expectations for the flow, so none of its steps claim one.
|
|
8998
9306
|
*
|
|
8999
|
-
*
|
|
9000
|
-
*
|
|
9001
|
-
*
|
|
9002
|
-
* and has to be said out loud — which is what a project's own guidelines ask
|
|
9003
|
-
* of the people writing these by hand.
|
|
9307
|
+
* One predicate, two readers — the review below and the evidence table, which
|
|
9308
|
+
* reach the same verdict from the same file and must not be able to disagree
|
|
9309
|
+
* about which steps were supposed to decide something.
|
|
9004
9310
|
*/
|
|
9005
|
-
function
|
|
9006
|
-
|
|
9007
|
-
while (above >= 0 && /\bexpect\(|^\s*$|^\s*\)/.test(all[above] ?? "")) above -= 1;
|
|
9008
|
-
return /^\s*\/\//.test(all[above] ?? "");
|
|
9311
|
+
function claimsAnOutcome(step) {
|
|
9312
|
+
return (step.expected ?? "").trim().length > 0;
|
|
9009
9313
|
}
|
|
9010
9314
|
/**
|
|
9011
|
-
*
|
|
9012
|
-
*
|
|
9013
|
-
* the
|
|
9315
|
+
* What a step that states an outcome, and carries no assertion under it, is
|
|
9316
|
+
* told. Shared with the evidence table, which reaches the same verdict from
|
|
9317
|
+
* the same file — a reader must not meet two wordings for one fact.
|
|
9014
9318
|
*
|
|
9015
|
-
*
|
|
9016
|
-
*
|
|
9017
|
-
*
|
|
9319
|
+
* It reports what was seen, not what was concluded. An assertion a rewrite
|
|
9320
|
+
* moved into a page object is invisible here, and calling that "decides
|
|
9321
|
+
* nothing" would be ccqa claiming more than it looked at.
|
|
9018
9322
|
*/
|
|
9019
|
-
|
|
9020
|
-
const all = lines(source);
|
|
9021
|
-
return all.flatMap((text, i) => {
|
|
9022
|
-
if (!/\bexpect\(/.test(text) || !/\.first\(\)/.test(text)) return [];
|
|
9023
|
-
if (explained(all, i)) return [];
|
|
9024
|
-
return [{
|
|
9025
|
-
file,
|
|
9026
|
-
line: i + 1,
|
|
9027
|
-
rule: "unjustified-first",
|
|
9028
|
-
message: "`.first()` in an assertion holds for whichever element came first. Say why that is the one, or address the element itself"
|
|
9029
|
-
}];
|
|
9030
|
-
});
|
|
9031
|
-
}
|
|
9032
|
-
/** A describe that repeats its only test reads as "X › X" in every report. */
|
|
9033
|
-
function describeEcho(file, source) {
|
|
9034
|
-
const describe = /test\.describe\(\s*["'](.+?)["']/.exec(source);
|
|
9035
|
-
if (!describe) return [];
|
|
9036
|
-
const titles = [...source.matchAll(/\n\s*test\(\s*["'](.+?)["']/g)].map((m) => m[1]);
|
|
9037
|
-
if (titles.length !== 1) return [];
|
|
9038
|
-
const bare = (s) => s.replace(/\s+@[\w-]+\s*$/, "").trim();
|
|
9039
|
-
if (bare(titles[0]) !== bare(describe[1])) return [];
|
|
9040
|
-
const line = lines(source).findIndex((t) => t.includes("test.describe(")) + 1;
|
|
9041
|
-
return [{
|
|
9042
|
-
file,
|
|
9043
|
-
line: Math.max(line, 1),
|
|
9044
|
-
rule: "describe-echo",
|
|
9045
|
-
message: "the describe repeats its only test's name. Drop it, or name the screen the test is about"
|
|
9046
|
-
}];
|
|
9047
|
-
}
|
|
9048
|
-
/** `name` → the expression it is defined as, across every file written. */
|
|
9049
|
-
function collectLocators(files) {
|
|
9050
|
-
const out = /* @__PURE__ */ new Map();
|
|
9051
|
-
for (const source of files.values()) {
|
|
9052
|
-
for (const m of source.matchAll(/\breadonly\s+([a-zA-Z0-9_]+)\s*=\s*([^;]+);/g)) out.set(m[1], m[2]);
|
|
9053
|
-
for (const m of source.matchAll(/\b([a-zA-Z0-9_]+)\([^)]*\)\s*:\s*Locator\s*\{\s*return\s+([^;]+);/g)) out.set(m[1], m[2]);
|
|
9054
|
-
}
|
|
9055
|
-
return out;
|
|
9056
|
-
}
|
|
9323
|
+
const NOTHING_DECIDED = "no assertion is visible under this step in the generated test — one moved into a helper does not show here, and does not show to a reviewer reading the file either";
|
|
9057
9324
|
/**
|
|
9058
|
-
* The
|
|
9059
|
-
*
|
|
9060
|
-
*
|
|
9061
|
-
* as "what this assertion looks for", every `getByTestId` in a project that
|
|
9062
|
-
* prefers them would read as an assertion nobody asked for.
|
|
9325
|
+
* The warning a finding becomes, phrased so the reader knows the test is green
|
|
9326
|
+
* for nothing. A step deciding nothing is its own explanation, so only the
|
|
9327
|
+
* model's own words are worth appending.
|
|
9063
9328
|
*/
|
|
9064
|
-
|
|
9329
|
+
function formatFinding(finding, labels = evidenceLabels()) {
|
|
9330
|
+
const nothing = finding.problem === NOTHING_DECIDED;
|
|
9331
|
+
const claim = nothing ? labels.findingNothing : labels.findingUndecided;
|
|
9332
|
+
return `${finding.stepId}: ${claim}${nothing ? "" : ` — ${finding.problem}`}`;
|
|
9333
|
+
}
|
|
9065
9334
|
/**
|
|
9066
|
-
* The
|
|
9067
|
-
*
|
|
9335
|
+
* The warning a rule violation becomes: the file, the rule in the words it was
|
|
9336
|
+
* judged by, and where that came from. An advisory one says so — a reader must
|
|
9337
|
+
* be able to tell a line that was worth a fix round from one that was not. The
|
|
9338
|
+
* offending code is left out: it is what the fix pass needs, not what a reader
|
|
9339
|
+
* scanning a log does.
|
|
9068
9340
|
*/
|
|
9069
|
-
function
|
|
9070
|
-
|
|
9071
|
-
|
|
9072
|
-
const definition = locators.get(m[1]);
|
|
9073
|
-
if (definition) expanded += `\n${definition}`;
|
|
9074
|
-
}
|
|
9075
|
-
return expanded;
|
|
9076
|
-
}
|
|
9077
|
-
function assertedStrings(text, locators) {
|
|
9078
|
-
return [...expandLocators(text, locators).matchAll(VISIBLE_STRING)].map((m) => (m[1] ?? m[2] ?? "").trim()).filter((s) => s.length > 1 && !s.includes("${"));
|
|
9341
|
+
function formatViolation(violation) {
|
|
9342
|
+
const advisory = violation.severity === "advisory" ? ", advisory" : "";
|
|
9343
|
+
return `${violation.file}: ${violation.rule} (${violation.guide}${advisory})`;
|
|
9079
9344
|
}
|
|
9080
9345
|
/**
|
|
9081
|
-
*
|
|
9346
|
+
* Hand the written files to a reviewer who knows nothing about where they came
|
|
9347
|
+
* from, and ask what it would say on the pull request: whether each step is
|
|
9348
|
+
* decided (see `verifiesSpecPrompt`), and — where the project wrote rule
|
|
9349
|
+
* documents of its own — whether the code follows them and the suite it joins.
|
|
9082
9350
|
*
|
|
9083
|
-
*
|
|
9084
|
-
*
|
|
9085
|
-
*
|
|
9351
|
+
* It reads the files off disk with read-only tools rather than being shown
|
|
9352
|
+
* them, which is also what lets it read the rest of the repository: "follow
|
|
9353
|
+
* the existing implementation" cannot be checked against an inline copy of two
|
|
9354
|
+
* files, because the evidence for it is how many other files do it that way.
|
|
9086
9355
|
*
|
|
9087
|
-
*
|
|
9088
|
-
* a
|
|
9089
|
-
*
|
|
9090
|
-
*
|
|
9091
|
-
*
|
|
9092
|
-
*
|
|
9356
|
+
* Every invocation is a fresh session. The reviewer must not remember arguing
|
|
9357
|
+
* for a finding the previous round declined to act on, and what has already
|
|
9358
|
+
* been asked for is the loop's business, not the reviewer's.
|
|
9359
|
+
*
|
|
9360
|
+
* A review that could not be obtained answers `findings: null` and is logged —
|
|
9361
|
+
* it must not read as a pass, and it must also not fail the generate that
|
|
9362
|
+
* produced working files.
|
|
9093
9363
|
*/
|
|
9094
|
-
function
|
|
9095
|
-
const
|
|
9096
|
-
|
|
9097
|
-
|
|
9098
|
-
|
|
9099
|
-
|
|
9100
|
-
|
|
9101
|
-
|
|
9102
|
-
|
|
9103
|
-
|
|
9104
|
-
|
|
9105
|
-
|
|
9106
|
-
|
|
9107
|
-
|
|
9364
|
+
async function reviewGeneratedTest(input) {
|
|
9365
|
+
const tests = input.files.filter((f) => f.kind === "test");
|
|
9366
|
+
const source = tests.map((f) => f.contents).join("\n\n");
|
|
9367
|
+
if (source.trim().length === 0) {
|
|
9368
|
+
warn("could not check whether the generated test decides its spec (no test file to read)");
|
|
9369
|
+
return {
|
|
9370
|
+
findings: null,
|
|
9371
|
+
complete: false,
|
|
9372
|
+
warnings: []
|
|
9373
|
+
};
|
|
9374
|
+
}
|
|
9375
|
+
const checked = assertionsByStep(source);
|
|
9376
|
+
const undecided = [...input.steps, ...input.cleanup ?? []].filter(isExpandedActionStep).filter(claimsAnOutcome).filter((step) => (checked.get(step.id) ?? []).length === 0).map((step) => ({
|
|
9377
|
+
stepId: step.id,
|
|
9378
|
+
problem: NOTHING_DECIDED
|
|
9379
|
+
}));
|
|
9380
|
+
if (input.askModel === false) return {
|
|
9381
|
+
findings: undecided,
|
|
9382
|
+
complete: false,
|
|
9383
|
+
warnings: undecided.map((f) => formatFinding(f))
|
|
9384
|
+
};
|
|
9385
|
+
const invoke = input.invoke ?? invokeClaudeStreaming;
|
|
9386
|
+
const guides = input.guides ?? [];
|
|
9387
|
+
const { result: answer, isError } = await invoke({
|
|
9388
|
+
prompt: verifiesSpecPrompt({
|
|
9389
|
+
steps: input.steps,
|
|
9390
|
+
testPath: tests[0].path,
|
|
9391
|
+
submitted: input.files.filter((f) => f.kind !== "test").map((f) => f.path),
|
|
9392
|
+
language: input.language,
|
|
9393
|
+
...input.expectations && input.expectations.length > 0 ? { expectations: [...input.expectations] } : {},
|
|
9394
|
+
...input.cleanup && input.cleanup.length > 0 ? { cleanup: [...input.cleanup] } : {},
|
|
9395
|
+
...input.leansOn && input.leansOn.length > 0 ? { leansOn: input.leansOn } : {},
|
|
9396
|
+
guides
|
|
9397
|
+
}),
|
|
9398
|
+
allowedTools: REVIEW_TOOLS,
|
|
9399
|
+
maxTurns: MAX_REVIEW_TURNS,
|
|
9400
|
+
timeoutMs: REVIEW_TIMEOUT_MS,
|
|
9401
|
+
silenceBashLog: true,
|
|
9402
|
+
...input.model ? { model: input.model } : {},
|
|
9403
|
+
cwd: input.cwd
|
|
9404
|
+
}, () => {});
|
|
9405
|
+
const fromModel = isError ? null : parseVerifiesSpecFindings(answer);
|
|
9406
|
+
if (fromModel === null) warn(`could not check whether the generated test decides its spec (${isError ? "Claude returned an error" : "no usable answer"})`);
|
|
9407
|
+
const violations = guides.length === 0 || isError ? null : parseGuideViolations(answer);
|
|
9408
|
+
if (guides.length > 0 && !isError && violations === null) warn("could not check the generated code against the project's conventions (no usable answer)");
|
|
9409
|
+
const findings = mergeFindings(undecided, fromModel ?? []);
|
|
9410
|
+
return {
|
|
9411
|
+
findings,
|
|
9412
|
+
complete: fromModel !== null,
|
|
9413
|
+
...isError ? { reviewerFailed: true } : {},
|
|
9414
|
+
warnings: [...findings.map((finding) => formatFinding(finding)), ...(violations ?? []).map(formatViolation)],
|
|
9415
|
+
...violations ? { ruleViolations: violations } : {}
|
|
9416
|
+
};
|
|
9108
9417
|
}
|
|
9109
|
-
/**
|
|
9110
|
-
|
|
9111
|
-
|
|
9418
|
+
/**
|
|
9419
|
+
* The mechanical finding wins its step: it is a fact, not a reading. Shared
|
|
9420
|
+
* with the evidence table, which merges the same two sources — one of them
|
|
9421
|
+
* recomputed from the file as it is now — and must show each step once.
|
|
9422
|
+
*/
|
|
9423
|
+
function mergeFindings(undecided, fromModel) {
|
|
9424
|
+
const claimed = new Set(undecided.map((f) => f.stepId));
|
|
9425
|
+
return [...undecided, ...fromModel.filter((f) => !claimed.has(f.stepId))];
|
|
9112
9426
|
}
|
|
9113
9427
|
//#endregion
|
|
9114
9428
|
//#region src/runtime/evidence-constants.ts
|
|
@@ -9116,14 +9430,16 @@ function formatEmittedReview(findings) {
|
|
|
9116
9430
|
* Shared constants + helpers for step-boundary evidence: the env var that
|
|
9117
9431
|
* enables capture, the file-name sanitizer, and the reserved failure ids.
|
|
9118
9432
|
*
|
|
9119
|
-
*
|
|
9120
|
-
* / `captureFailureEvidence()` in `test-helpers.ts`
|
|
9121
|
-
*
|
|
9122
|
-
*
|
|
9123
|
-
* `
|
|
9124
|
-
*
|
|
9125
|
-
*
|
|
9126
|
-
*
|
|
9433
|
+
* Several producers write the `<id>.png` + `<id>.json` pairs —
|
|
9434
|
+
* `abStepEvidence()` / `captureFailureEvidence()` in `test-helpers.ts`
|
|
9435
|
+
* (agent-browser replays), `targets/playwright/trace-capture.ts` (frames read
|
|
9436
|
+
* out of a run's Playwright trace), and `ccqaStepBefore`/`ccqaStepAfter` in
|
|
9437
|
+
* `step-evidence.ts` for tests generated before the capture calls left the
|
|
9438
|
+
* committed file — and one consumer reads them back (`loadEvidenceForSpec` in
|
|
9439
|
+
* `report/evidence.ts`). They agree only on the contract here, so it is kept
|
|
9440
|
+
* under `runtime/` (free of CLI-side imports) so the generated-test modules —
|
|
9441
|
+
* imported via `ccqa/test-helpers` and `ccqa/step-evidence` — can share it
|
|
9442
|
+
* without dragging the CLI in.
|
|
9127
9443
|
*/
|
|
9128
9444
|
/**
|
|
9129
9445
|
* Env var naming the directory a spec's step evidence is written to. `ccqa
|
|
@@ -9132,6 +9448,15 @@ function formatEmittedReview(findings) {
|
|
|
9132
9448
|
* scattering screenshots.
|
|
9133
9449
|
*/
|
|
9134
9450
|
const EVIDENCE_DIR_ENV = "CCQA_EVIDENCE_DIR";
|
|
9451
|
+
/**
|
|
9452
|
+
* Make a step id safe to use as an evidence file-name stem. Both evidence
|
|
9453
|
+
* producers (agent-browser `abStepEvidence`, external `ccqa/step-evidence`)
|
|
9454
|
+
* name their PNG/JSON pair `<stem>.png` / `<stem>.json`, so they must agree on
|
|
9455
|
+
* this exact mapping.
|
|
9456
|
+
*/
|
|
9457
|
+
function sanitizeStepId(stepId) {
|
|
9458
|
+
return stepId.replace(/[^A-Za-z0-9_.-]/g, "_");
|
|
9459
|
+
}
|
|
9135
9460
|
//#endregion
|
|
9136
9461
|
//#region src/drift/affected.ts
|
|
9137
9462
|
const execFileP = promisify(execFile);
|
|
@@ -9404,37 +9729,51 @@ async function resolveResources(cwd, refs) {
|
|
|
9404
9729
|
const CONVENTIONS_MAX_BYTES = 64 * 1024;
|
|
9405
9730
|
/**
|
|
9406
9731
|
* Expand and read the conventions guides + examples, in declared order, into
|
|
9407
|
-
* prompt sections
|
|
9408
|
-
*
|
|
9409
|
-
*
|
|
9410
|
-
* truncated instead — an empty
|
|
9411
|
-
|
|
9412
|
-
|
|
9413
|
-
|
|
9414
|
-
|
|
9415
|
-
const
|
|
9416
|
-
|
|
9417
|
-
|
|
9418
|
-
|
|
9419
|
-
|
|
9732
|
+
* prompt sections, each saying which of the two it came from. Files are kept
|
|
9733
|
+
* whole while they fit the byte budget; a file that does not fit is dropped
|
|
9734
|
+
* with a warning naming it (never a silent truncation). Only when the very
|
|
9735
|
+
* first file alone exceeds the budget is it truncated instead — an empty
|
|
9736
|
+
* conventions injection would be worse.
|
|
9737
|
+
*/
|
|
9738
|
+
async function loadConventions(cwd, conventions, maxBytes = CONVENTIONS_MAX_BYTES) {
|
|
9739
|
+
const expand = async (patterns) => {
|
|
9740
|
+
const out = [];
|
|
9741
|
+
for (const pattern of patterns) {
|
|
9742
|
+
const matched = await expandPatternToFiles(cwd, pattern).catch((e) => {
|
|
9743
|
+
throw new Error(`conventions entry ${e.message}`);
|
|
9744
|
+
});
|
|
9745
|
+
for (const f of matched) if (!out.includes(f)) out.push(f);
|
|
9746
|
+
}
|
|
9747
|
+
return out;
|
|
9748
|
+
};
|
|
9749
|
+
const files = (await expand(conventions.guides)).map((path) => ({
|
|
9750
|
+
path,
|
|
9751
|
+
kind: "guide"
|
|
9752
|
+
}));
|
|
9753
|
+
for (const path of await expand(conventions.examples ?? [])) if (!files.some((f) => f.path === path)) files.push({
|
|
9754
|
+
path,
|
|
9755
|
+
kind: "example"
|
|
9756
|
+
});
|
|
9420
9757
|
const sections = [];
|
|
9421
9758
|
const warnings = [];
|
|
9422
9759
|
const dropped = [];
|
|
9423
9760
|
let used = 0;
|
|
9424
|
-
for (const file of files) {
|
|
9761
|
+
for (const { path: file, kind } of files) {
|
|
9425
9762
|
const body = await readFile(resolve(cwd, file), "utf8");
|
|
9426
9763
|
const size = Buffer.byteLength(body, "utf8");
|
|
9427
9764
|
if (used + size <= maxBytes) {
|
|
9428
9765
|
sections.push({
|
|
9429
9766
|
path: file,
|
|
9430
|
-
body
|
|
9767
|
+
body,
|
|
9768
|
+
kind
|
|
9431
9769
|
});
|
|
9432
9770
|
used += size;
|
|
9433
9771
|
} else if (sections.length === 0) {
|
|
9434
9772
|
const truncated = Buffer.from(body, "utf8").subarray(0, maxBytes).toString("utf8");
|
|
9435
9773
|
sections.push({
|
|
9436
9774
|
path: file,
|
|
9437
|
-
body: truncated
|
|
9775
|
+
body: truncated,
|
|
9776
|
+
kind
|
|
9438
9777
|
});
|
|
9439
9778
|
used = maxBytes;
|
|
9440
9779
|
warnings.push(`conventions file ${file} exceeds the ${maxBytes}-byte prompt budget alone — injected truncated`);
|
|
@@ -13465,11 +13804,12 @@ function emptySpecRow(args) {
|
|
|
13465
13804
|
* Step-boundary screenshot evidence: where it lives on disk and how a
|
|
13466
13805
|
* directory of `<id>.png` + `<id>.json` pairs becomes report rows.
|
|
13467
13806
|
*
|
|
13468
|
-
* Producer-agnostic on purpose.
|
|
13469
|
-
*
|
|
13470
|
-
*
|
|
13471
|
-
* have to agree on the file-pair convention documented here,
|
|
13472
|
-
* screenshot was taken.
|
|
13807
|
+
* Producer-agnostic on purpose. Very different writers fill these directories
|
|
13808
|
+
* — `abStepEvidence()` during an agent-browser replay, and, for a Playwright
|
|
13809
|
+
* target, frames read back out of the run's trace once the command has exited
|
|
13810
|
+
* — and they only have to agree on the file-pair convention documented here,
|
|
13811
|
+
* not on how or when the screenshot was taken. `pngFile` names the image
|
|
13812
|
+
* whatever its format: a trace's screencast frames are JPEG.
|
|
13473
13813
|
*/
|
|
13474
13814
|
/** `<reportDir>/evidence/<feature>/<spec>` — one directory per spec. */
|
|
13475
13815
|
function specEvidenceDir(reportDir, feature, spec) {
|
|
@@ -13555,6 +13895,296 @@ function toPosix(p) {
|
|
|
13555
13895
|
return p.split(/[\\/]/).join("/");
|
|
13556
13896
|
}
|
|
13557
13897
|
//#endregion
|
|
13898
|
+
//#region src/targets/playwright/trace-evidence.ts
|
|
13899
|
+
const EOCD_SIGNATURE = 101010256;
|
|
13900
|
+
const EOCD_SEARCH_WINDOW = 65557;
|
|
13901
|
+
/** A zip archive's directory, read up front. Stored (method 0) and deflate (method 8) only. */
|
|
13902
|
+
async function readZip(path) {
|
|
13903
|
+
const buf = await readFile(path);
|
|
13904
|
+
const eocd = findEndOfCentralDirectory(buf, path);
|
|
13905
|
+
const entryCount = buf.readUInt16LE(eocd + 10);
|
|
13906
|
+
const centralDirectoryOffset = buf.readUInt32LE(eocd + 16);
|
|
13907
|
+
if (entryCount === 65535 || centralDirectoryOffset === 4294967295) throw new Error(`${path} uses zip64, which this reader does not support`);
|
|
13908
|
+
const entries = readCentralDirectory(buf, centralDirectoryOffset, entryCount);
|
|
13909
|
+
return {
|
|
13910
|
+
names: [...entries.keys()],
|
|
13911
|
+
read(name) {
|
|
13912
|
+
const entry = entries.get(name);
|
|
13913
|
+
if (entry === void 0) return void 0;
|
|
13914
|
+
return readLocalEntryData(buf, entry.offset, entry.method, entry.compressedSize) ?? void 0;
|
|
13915
|
+
}
|
|
13916
|
+
};
|
|
13917
|
+
}
|
|
13918
|
+
function findEndOfCentralDirectory(buf, path) {
|
|
13919
|
+
const searchStart = Math.max(0, buf.length - EOCD_SEARCH_WINDOW);
|
|
13920
|
+
for (let i = buf.length - 22; i >= searchStart; i--) if (buf.readUInt32LE(i) === EOCD_SIGNATURE) return i;
|
|
13921
|
+
throw new Error(`${path}: not a zip archive (no end-of-central-directory record found)`);
|
|
13922
|
+
}
|
|
13923
|
+
function readCentralDirectory(buf, offset, entryCount) {
|
|
13924
|
+
const entries = /* @__PURE__ */ new Map();
|
|
13925
|
+
let pos = offset;
|
|
13926
|
+
for (let i = 0; i < entryCount; i++) {
|
|
13927
|
+
const method = buf.readUInt16LE(pos + 10);
|
|
13928
|
+
const compressedSize = buf.readUInt32LE(pos + 20);
|
|
13929
|
+
const nameLength = buf.readUInt16LE(pos + 28);
|
|
13930
|
+
const extraLength = buf.readUInt16LE(pos + 30);
|
|
13931
|
+
const commentLength = buf.readUInt16LE(pos + 32);
|
|
13932
|
+
const localHeaderOffset = buf.readUInt32LE(pos + 42);
|
|
13933
|
+
const name = buf.toString("utf8", pos + 46, pos + 46 + nameLength);
|
|
13934
|
+
entries.set(name, {
|
|
13935
|
+
offset: localHeaderOffset,
|
|
13936
|
+
method,
|
|
13937
|
+
compressedSize
|
|
13938
|
+
});
|
|
13939
|
+
pos += 46 + nameLength + extraLength + commentLength;
|
|
13940
|
+
}
|
|
13941
|
+
return entries;
|
|
13942
|
+
}
|
|
13943
|
+
function readLocalEntryData(buf, offset, method, compressedSize) {
|
|
13944
|
+
const nameLength = buf.readUInt16LE(offset + 26);
|
|
13945
|
+
const extraLength = buf.readUInt16LE(offset + 28);
|
|
13946
|
+
const dataStart = offset + 30 + nameLength + extraLength;
|
|
13947
|
+
const compressed = buf.subarray(dataStart, dataStart + compressedSize);
|
|
13948
|
+
if (method === 0) return Buffer.from(compressed);
|
|
13949
|
+
if (method === 8) return inflateRawSync(compressed);
|
|
13950
|
+
return null;
|
|
13951
|
+
}
|
|
13952
|
+
/**
|
|
13953
|
+
* The screencast frames and the `test.step` calls one trace file holds.
|
|
13954
|
+
*
|
|
13955
|
+
* A trace is JSON Lines, one event object per line; a corrupt or truncated
|
|
13956
|
+
* line must not sink the whole read, so parsing is best-effort per line.
|
|
13957
|
+
* Step events come in two shapes across Playwright versions — a `before`/
|
|
13958
|
+
* `after` pair sharing a `callId`, or a single merged `action` event — and
|
|
13959
|
+
* both are folded into the same `TraceStep` shape here so callers never see
|
|
13960
|
+
* the version skew.
|
|
13961
|
+
*/
|
|
13962
|
+
function parseTraceEvents(jsonl) {
|
|
13963
|
+
const frames = [];
|
|
13964
|
+
const steps = [];
|
|
13965
|
+
const pendingByCallId = /* @__PURE__ */ new Map();
|
|
13966
|
+
let maxTimestamp = -Infinity;
|
|
13967
|
+
for (const line of jsonl.split("\n")) {
|
|
13968
|
+
if (line.trim() === "") continue;
|
|
13969
|
+
let parsed;
|
|
13970
|
+
try {
|
|
13971
|
+
parsed = JSON.parse(line);
|
|
13972
|
+
} catch {
|
|
13973
|
+
continue;
|
|
13974
|
+
}
|
|
13975
|
+
if (typeof parsed !== "object" || parsed === null) continue;
|
|
13976
|
+
const event = parsed;
|
|
13977
|
+
for (const field of [
|
|
13978
|
+
"timestamp",
|
|
13979
|
+
"startTime",
|
|
13980
|
+
"endTime"
|
|
13981
|
+
]) {
|
|
13982
|
+
const value = event[field];
|
|
13983
|
+
if (typeof value === "number") maxTimestamp = Math.max(maxTimestamp, value);
|
|
13984
|
+
}
|
|
13985
|
+
const type = event.type;
|
|
13986
|
+
if (type === "screencast-frame") {
|
|
13987
|
+
if (typeof event.sha1 === "string" && typeof event.timestamp === "number") frames.push({
|
|
13988
|
+
sha1: event.sha1,
|
|
13989
|
+
timestamp: event.timestamp
|
|
13990
|
+
});
|
|
13991
|
+
} else if (type === "action") {
|
|
13992
|
+
if (typeof event.callId === "string" && typeof event.startTime === "number" && typeof event.endTime === "number") steps.push({
|
|
13993
|
+
title: titleOf(event),
|
|
13994
|
+
startTime: event.startTime,
|
|
13995
|
+
endTime: event.endTime,
|
|
13996
|
+
...errorOf(event)
|
|
13997
|
+
});
|
|
13998
|
+
} else if (type === "before") {
|
|
13999
|
+
if (typeof event.callId === "string" && typeof event.startTime === "number") pendingByCallId.set(event.callId, {
|
|
14000
|
+
title: titleOf(event),
|
|
14001
|
+
startTime: event.startTime
|
|
14002
|
+
});
|
|
14003
|
+
} else if (type === "after") {
|
|
14004
|
+
if (typeof event.callId === "string" && typeof event.endTime === "number") {
|
|
14005
|
+
const pending = pendingByCallId.get(event.callId);
|
|
14006
|
+
if (pending !== void 0) {
|
|
14007
|
+
pendingByCallId.delete(event.callId);
|
|
14008
|
+
steps.push({
|
|
14009
|
+
...pending,
|
|
14010
|
+
endTime: event.endTime,
|
|
14011
|
+
...errorOf(event)
|
|
14012
|
+
});
|
|
14013
|
+
}
|
|
14014
|
+
}
|
|
14015
|
+
}
|
|
14016
|
+
}
|
|
14017
|
+
for (const pending of pendingByCallId.values()) steps.push({
|
|
14018
|
+
...pending,
|
|
14019
|
+
endTime: maxTimestamp
|
|
14020
|
+
});
|
|
14021
|
+
frames.sort((a, b) => a.timestamp - b.timestamp);
|
|
14022
|
+
steps.sort((a, b) => a.startTime - b.startTime);
|
|
14023
|
+
return {
|
|
14024
|
+
frames,
|
|
14025
|
+
steps
|
|
14026
|
+
};
|
|
14027
|
+
}
|
|
14028
|
+
/** The step's failure message, in whichever shape the trace recorded it. */
|
|
14029
|
+
function errorOf(event) {
|
|
14030
|
+
const error = event.error;
|
|
14031
|
+
if (typeof error === "string") return { error };
|
|
14032
|
+
if (typeof error === "object" && error !== null) {
|
|
14033
|
+
const message = error.message;
|
|
14034
|
+
if (typeof message === "string") return { error: message };
|
|
14035
|
+
}
|
|
14036
|
+
return {};
|
|
14037
|
+
}
|
|
14038
|
+
function titleOf(event) {
|
|
14039
|
+
if (typeof event.apiName === "string") return event.apiName;
|
|
14040
|
+
if (typeof event.title === "string") return event.title;
|
|
14041
|
+
return "";
|
|
14042
|
+
}
|
|
14043
|
+
/**
|
|
14044
|
+
* The frame on screen at `time`: the last one captured at or before it, or —
|
|
14045
|
+
* when the step began before any frame was captured — the first frame after.
|
|
14046
|
+
* Null when the trace holds no frames at all. `frames` comes from
|
|
14047
|
+
* {@link parseTraceEvents}, which sorts them by timestamp.
|
|
14048
|
+
*/
|
|
14049
|
+
function frameAt(frames, time) {
|
|
14050
|
+
return frames.findLast((f) => f.timestamp <= time) ?? frames[0] ?? null;
|
|
14051
|
+
}
|
|
14052
|
+
//#endregion
|
|
14053
|
+
//#region src/targets/playwright/trace-capture.ts
|
|
14054
|
+
/**
|
|
14055
|
+
* Per-step screenshots for a Playwright test that contains nothing of ccqa's.
|
|
14056
|
+
*
|
|
14057
|
+
* The test a project commits is plain `@playwright/test`: its steps are
|
|
14058
|
+
* `test.step` blocks and there is no capture call to place a screenshot at
|
|
14059
|
+
* either boundary. So the screenshots are not taken during the run at all —
|
|
14060
|
+
* ccqa asks Playwright for a trace on the invocations it owns, and afterwards
|
|
14061
|
+
* reads the step boundaries and the screencast out of the archive. Nothing in
|
|
14062
|
+
* the consumer's repository changes, and nothing ccqa does here can turn a
|
|
14063
|
+
* passing test red: the whole path runs after the command has exited, and a
|
|
14064
|
+
* trace it cannot read costs the report its pictures and says so.
|
|
14065
|
+
*
|
|
14066
|
+
* The frames are the trace's own screencast, which Playwright records
|
|
14067
|
+
* downscaled — they are the filmstrip a reviewer already sees in the trace
|
|
14068
|
+
* viewer, not `page.screenshot()` output. That is the price of taking the
|
|
14069
|
+
* capture code out of the committed file.
|
|
14070
|
+
*/
|
|
14071
|
+
/** Shell characters that would hand an appended flag to a different program. */
|
|
14072
|
+
const SHELL_OPERATORS = /[|;&<>`$]/;
|
|
14073
|
+
/**
|
|
14074
|
+
* The command has to reach `playwright test` itself for an appended flag to
|
|
14075
|
+
* mean anything. A package script that wraps it is one indirection too many —
|
|
14076
|
+
* the flag may be swallowed by the script runner, and a mangled command is a
|
|
14077
|
+
* worse outcome than no screenshots.
|
|
14078
|
+
*/
|
|
14079
|
+
const PLAYWRIGHT_TEST = /(^|\s)playwright\s+test(\s|$)/;
|
|
14080
|
+
/**
|
|
14081
|
+
* The runCommand as ccqa must run it to get a trace back.
|
|
14082
|
+
*
|
|
14083
|
+
* Two shapes are refused rather than mangled: a command that reaches a second
|
|
14084
|
+
* program (the flag would land on the wrong one) and a command that already
|
|
14085
|
+
* directs its output somewhere ccqa does not read (the trace would be written
|
|
14086
|
+
* where nothing looks for it). A command that already asks for a trace keeps
|
|
14087
|
+
* its own setting — the project chose it, and `on` is what ccqa needs anyway
|
|
14088
|
+
* only when nothing else has spoken.
|
|
14089
|
+
*
|
|
14090
|
+
* A refusal answers `skip` with the reason and the command unchanged, so a
|
|
14091
|
+
* caller runs what the project configured either way and has one place to read
|
|
14092
|
+
* why the run will leave no trace.
|
|
14093
|
+
*/
|
|
14094
|
+
function amendForTrace(command, outputDir) {
|
|
14095
|
+
if (!PLAYWRIGHT_TEST.test(command)) return {
|
|
14096
|
+
command,
|
|
14097
|
+
skip: "ccqa reads a spec's step screenshots out of the run's Playwright trace, and can only ask for one when the target's runCommand invokes `playwright test` itself (e.g. `pnpm exec playwright test {files}`)"
|
|
14098
|
+
};
|
|
14099
|
+
if (SHELL_OPERATORS.test(command)) return {
|
|
14100
|
+
command,
|
|
14101
|
+
skip: "the target's runCommand uses shell operators, so ccqa cannot safely append the `--trace`/`--output` flags its step screenshots are read from"
|
|
14102
|
+
};
|
|
14103
|
+
const directsOutput = /(^|\s)--output[=\s]/.test(command);
|
|
14104
|
+
if (directsOutput && !command.includes(outputDir)) return {
|
|
14105
|
+
command,
|
|
14106
|
+
skip: "the target's runCommand passes its own --output, so ccqa cannot find the trace its step screenshots are read from — point that flag at `{artifactsDir}` or drop it"
|
|
14107
|
+
};
|
|
14108
|
+
const flags = [/(^|\s)--trace[=\s]/.test(command) ? "" : "--trace=on", directsOutput ? "" : `--output=${quoteForShell(outputDir)}`].filter(Boolean);
|
|
14109
|
+
return { command: flags.length > 0 ? `${command} ${flags.join(" ")}` : command };
|
|
14110
|
+
}
|
|
14111
|
+
/**
|
|
14112
|
+
* Make sure `evidenceDir` holds the spec's per-step screenshots, reading the
|
|
14113
|
+
* trace the run left in `artifactsDir` when nothing else wrote them. Returns
|
|
14114
|
+
* null when the directory has evidence in it, and otherwise the reason there
|
|
14115
|
+
* is nothing to show.
|
|
14116
|
+
*
|
|
14117
|
+
* A spec generated before the capture calls left the committed file writes its
|
|
14118
|
+
* own frames as it runs; those are the test's own and are left alone.
|
|
14119
|
+
*
|
|
14120
|
+
* Every step of every trace found is written, not just the first archive's: a
|
|
14121
|
+
* retried test leaves one archive per attempt, and the later one overwrites
|
|
14122
|
+
* the earlier, which is the attempt the row reports.
|
|
14123
|
+
*/
|
|
14124
|
+
async function captureStepEvidence(args) {
|
|
14125
|
+
if ((await readdir(args.evidenceDir).catch(() => [])).length > 0) return null;
|
|
14126
|
+
const archives = (await walkFiles$1(args.artifactsDir)).filter(isTraceArchive).sort().map((rel) => join(args.artifactsDir, ...rel.split("/")));
|
|
14127
|
+
if (archives.length === 0) return "the run left no Playwright trace, so there are no per-step screenshots to read — the generated test's `test.step` blocks are what ccqa reads them from";
|
|
14128
|
+
let written = 0;
|
|
14129
|
+
let frames = 0;
|
|
14130
|
+
const failures = [];
|
|
14131
|
+
for (const archive of archives) try {
|
|
14132
|
+
const archived = await writeArchiveEvidence(archive, args.evidenceDir);
|
|
14133
|
+
written += archived.written;
|
|
14134
|
+
frames += archived.frames;
|
|
14135
|
+
} catch (err) {
|
|
14136
|
+
failures.push(`${archive}: ${err instanceof Error ? err.message : String(err)}`);
|
|
14137
|
+
}
|
|
14138
|
+
if (written > 0) return null;
|
|
14139
|
+
if (failures.length > 0) return `ccqa could not read the run's Playwright trace (${failures[0]})`;
|
|
14140
|
+
if (frames === 0) return "the run's Playwright trace holds no screenshots — the step frames come from its screencast, which Playwright records only while the trace's `screenshots` option is on (`--trace=on` asks for it; a `trace` option in the project's config can turn it off)";
|
|
14141
|
+
return "the run's Playwright trace holds no step the case names — a rewrite pass may have reshaped the `test.step` titles the draft wrote; re-run `ccqa generate` for this spec";
|
|
14142
|
+
}
|
|
14143
|
+
/** One archive's steps, written as evidence, and the frames it had to write them from. */
|
|
14144
|
+
async function writeArchiveEvidence(archive, evidenceDir) {
|
|
14145
|
+
const entries = await readZip(archive);
|
|
14146
|
+
const { frames, steps } = parseTraceEvents(entries.names.filter((name) => name.endsWith(".trace")).map((name) => entries.read(name)?.toString("utf8") ?? "").join("\n"));
|
|
14147
|
+
if (frames.length === 0) return {
|
|
14148
|
+
written: 0,
|
|
14149
|
+
frames: 0
|
|
14150
|
+
};
|
|
14151
|
+
await mkdir(evidenceDir, { recursive: true });
|
|
14152
|
+
let written = 0;
|
|
14153
|
+
for (const step of steps) {
|
|
14154
|
+
const label = parseStepLabel(step.title);
|
|
14155
|
+
if (label === null) continue;
|
|
14156
|
+
const stem = sanitizeStepId(label.stepId);
|
|
14157
|
+
const after = await writeFrame(entries, frames, step.endTime, evidenceDir, stem);
|
|
14158
|
+
if (after === null) continue;
|
|
14159
|
+
const before = await writeFrame(entries, frames, step.startTime, evidenceDir, `${stem}.before`);
|
|
14160
|
+
await writeFile(join(evidenceDir, `${stem}.json`), `${JSON.stringify({
|
|
14161
|
+
stepId: label.stepId,
|
|
14162
|
+
source: label.source,
|
|
14163
|
+
pngFile: after,
|
|
14164
|
+
...before !== null ? { beforePngFile: before } : {},
|
|
14165
|
+
...step.error !== void 0 ? { failureSummary: step.error } : {},
|
|
14166
|
+
url: null,
|
|
14167
|
+
title: null,
|
|
14168
|
+
capturedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
14169
|
+
}, null, 2)}\n`, "utf8");
|
|
14170
|
+
written += 1;
|
|
14171
|
+
}
|
|
14172
|
+
return {
|
|
14173
|
+
written,
|
|
14174
|
+
frames: frames.length
|
|
14175
|
+
};
|
|
14176
|
+
}
|
|
14177
|
+
/** The frame on screen at `time`, written under `stem`. Null when there is none. */
|
|
14178
|
+
async function writeFrame(entries, frames, time, evidenceDir, stem) {
|
|
14179
|
+
const frame = frameAt(frames, time);
|
|
14180
|
+
if (frame === null) return null;
|
|
14181
|
+
const data = entries.read(`resources/${frame.sha1}`) ?? entries.read(`resources/${frame.sha1}.jpeg`);
|
|
14182
|
+
if (data === void 0) return null;
|
|
14183
|
+
const file = `${stem}.jpeg`;
|
|
14184
|
+
await writeFile(join(evidenceDir, file), data);
|
|
14185
|
+
return file;
|
|
14186
|
+
}
|
|
14187
|
+
//#endregion
|
|
13558
14188
|
//#region src/targets/run-command-runner.ts
|
|
13559
14189
|
/**
|
|
13560
14190
|
* Substitute `{files}` in a runCommand with the (shell-quoted) cwd-relative
|
|
@@ -13565,12 +14195,9 @@ function toPosix(p) {
|
|
|
13565
14195
|
*/
|
|
13566
14196
|
function substituteRunCommandFiles(runCommand, testFiles) {
|
|
13567
14197
|
if (!runCommand.includes("{files}")) return runCommand;
|
|
13568
|
-
const joined = testFiles.map(
|
|
14198
|
+
const joined = testFiles.map(quoteForShell).join(" ");
|
|
13569
14199
|
return runCommand.replaceAll("{files}", joined);
|
|
13570
14200
|
}
|
|
13571
|
-
function shellQuote$1(s) {
|
|
13572
|
-
return /^[A-Za-z0-9_./-]+$/.test(s) ? s : `'${s.replaceAll("'", `'\\''`)}'`;
|
|
13573
|
-
}
|
|
13574
14201
|
/**
|
|
13575
14202
|
* Shared `TestRunner` for external (runCommand) targets: for each spec it
|
|
13576
14203
|
* derives the test's path from the target's `testPath` and executes the
|
|
@@ -13653,13 +14280,13 @@ async function runOneSpec(ref, opts, blocks) {
|
|
|
13653
14280
|
force: true
|
|
13654
14281
|
});
|
|
13655
14282
|
await mkdir(artifactsDir, { recursive: true });
|
|
13656
|
-
const
|
|
13657
|
-
if (
|
|
13658
|
-
await rm(
|
|
14283
|
+
const evidence = opts.stepEvidence.supported ? { dir: specEvidenceDir(opts.reportDir, featureName, specName) } : { reason: opts.stepEvidence.reason };
|
|
14284
|
+
if ("dir" in evidence) {
|
|
14285
|
+
await rm(evidence.dir, {
|
|
13659
14286
|
recursive: true,
|
|
13660
14287
|
force: true
|
|
13661
14288
|
});
|
|
13662
|
-
await mkdir(
|
|
14289
|
+
await mkdir(evidence.dir, { recursive: true });
|
|
13663
14290
|
}
|
|
13664
14291
|
const measurement = opts.coverage !== void 0 && opts.browserCoverage.browser === "cdp" ? {
|
|
13665
14292
|
collector: opts.coverage,
|
|
@@ -13676,9 +14303,15 @@ async function runOneSpec(ref, opts, blocks) {
|
|
|
13676
14303
|
const childEnv = {
|
|
13677
14304
|
[ARTIFACTS_DIR_ENV]: artifactsDir,
|
|
13678
14305
|
CCQA_RUN_ID: buildRunId(),
|
|
13679
|
-
...
|
|
14306
|
+
..."dir" in evidence ? { [EVIDENCE_DIR_ENV]: evidence.dir } : {}
|
|
13680
14307
|
};
|
|
13681
14308
|
let command = substituteArtifactsDir(substituteRunCommandFiles(runCommand, testFiles), artifactsDir);
|
|
14309
|
+
let traceUnavailable;
|
|
14310
|
+
if ("dir" in evidence) {
|
|
14311
|
+
const amended = amendForTrace(command, artifactsDir);
|
|
14312
|
+
command = amended.command;
|
|
14313
|
+
traceUnavailable = amended.skip;
|
|
14314
|
+
}
|
|
13682
14315
|
let browserHandle;
|
|
13683
14316
|
let browserEngine;
|
|
13684
14317
|
let attachError;
|
|
@@ -13741,7 +14374,14 @@ async function runOneSpec(ref, opts, blocks) {
|
|
|
13741
14374
|
warn(`could not collect artifacts for ${featureName}/${specName}: ${err instanceof Error ? err.message : String(err)}`);
|
|
13742
14375
|
}
|
|
13743
14376
|
const artifactFields = artifacts && artifacts.length > 0 ? { artifacts } : {};
|
|
13744
|
-
const evidenceFields = await loadStepEvidence(
|
|
14377
|
+
const evidenceFields = await loadStepEvidence({
|
|
14378
|
+
opts,
|
|
14379
|
+
evidence,
|
|
14380
|
+
artifactsDir,
|
|
14381
|
+
spec: parsedSpec,
|
|
14382
|
+
blocks,
|
|
14383
|
+
...traceUnavailable !== void 0 ? { traceUnavailable } : {}
|
|
14384
|
+
});
|
|
13745
14385
|
if (outcome.exitCode === 0) return {
|
|
13746
14386
|
...emptySpecRow({
|
|
13747
14387
|
feature: featureName,
|
|
@@ -13780,22 +14420,28 @@ function coverageRowFields(opts, measured, attachError) {
|
|
|
13780
14420
|
}
|
|
13781
14421
|
/**
|
|
13782
14422
|
* The row's step screenshots, or — when there are none — the reason, so the
|
|
13783
|
-
* report never shows an empty evidence section without explanation.
|
|
13784
|
-
*
|
|
13785
|
-
*
|
|
13786
|
-
*
|
|
14423
|
+
* report never shows an empty evidence section without explanation.
|
|
14424
|
+
*
|
|
14425
|
+
* A command ccqa could not amend left no trace to read, and its reason is the
|
|
14426
|
+
* more useful one — so the capture is not attempted at all in that case.
|
|
13787
14427
|
*/
|
|
13788
|
-
async function loadStepEvidence(
|
|
13789
|
-
|
|
14428
|
+
async function loadStepEvidence(args) {
|
|
14429
|
+
const { opts } = args;
|
|
14430
|
+
if (!("dir" in args.evidence)) return {
|
|
13790
14431
|
evidence: null,
|
|
13791
|
-
evidenceUnavailable:
|
|
14432
|
+
evidenceUnavailable: args.evidence.reason
|
|
13792
14433
|
};
|
|
13793
|
-
const
|
|
14434
|
+
const evidenceDir = args.evidence.dir;
|
|
14435
|
+
const reason = args.traceUnavailable ?? await captureStepEvidence({
|
|
14436
|
+
artifactsDir: args.artifactsDir,
|
|
14437
|
+
evidenceDir
|
|
14438
|
+
});
|
|
14439
|
+
const descriptions = buildStepDescriptions(args.spec, args.blocks);
|
|
13794
14440
|
const evidence = await loadEvidenceForSpec(evidenceDir, opts.reportDir, descriptions);
|
|
13795
14441
|
if (evidence) return { evidence };
|
|
13796
14442
|
return {
|
|
13797
14443
|
evidence: null,
|
|
13798
|
-
evidenceUnavailable: "no step screenshots were captured
|
|
14444
|
+
evidenceUnavailable: reason ?? "no step screenshots were captured for this spec"
|
|
13799
14445
|
};
|
|
13800
14446
|
}
|
|
13801
14447
|
/**
|
|
@@ -14035,7 +14681,10 @@ function configuredWriteRoots(ctx, resources) {
|
|
|
14035
14681
|
async function generateWithLlmEngine(req) {
|
|
14036
14682
|
const { ctx } = req;
|
|
14037
14683
|
const resources = await resolveResources(ctx.cwd, ctx.resources);
|
|
14038
|
-
const conventions = await loadConventions(ctx.cwd,
|
|
14684
|
+
const conventions = await loadConventions(ctx.cwd, {
|
|
14685
|
+
guides: ctx.conventions.guides,
|
|
14686
|
+
examples: ctx.conventions.examples
|
|
14687
|
+
});
|
|
14039
14688
|
const warnings = [...conventions.warnings];
|
|
14040
14689
|
for (const w of conventions.warnings) warn(w);
|
|
14041
14690
|
meta("resources", resources.length);
|
|
@@ -14081,6 +14730,8 @@ async function generateWithLlmEngine(req) {
|
|
|
14081
14730
|
initialFiles: output.files,
|
|
14082
14731
|
summary: output.summary,
|
|
14083
14732
|
warnings,
|
|
14733
|
+
guides: conventions.sections.filter((s) => s.kind === "guide"),
|
|
14734
|
+
...req.stepEvidence ? { stepEvidence: req.stepEvidence } : {},
|
|
14084
14735
|
validateFile: req.validateFile,
|
|
14085
14736
|
invoke,
|
|
14086
14737
|
reading: req.reading
|
|
@@ -14107,6 +14758,8 @@ async function finalizePreparedFiles(req) {
|
|
|
14107
14758
|
initialFiles: req.files,
|
|
14108
14759
|
summary: req.summary,
|
|
14109
14760
|
warnings: req.warnings,
|
|
14761
|
+
guides: [],
|
|
14762
|
+
...req.stepEvidence ? { stepEvidence: req.stepEvidence } : {},
|
|
14110
14763
|
validateFile: req.validateFile,
|
|
14111
14764
|
invoke: req.invoke ?? invokeClaudeStreaming,
|
|
14112
14765
|
reading: req.reading
|
|
@@ -14130,10 +14783,14 @@ async function finalizeAndVerify(p) {
|
|
|
14130
14783
|
};
|
|
14131
14784
|
}
|
|
14132
14785
|
/**
|
|
14133
|
-
* The runCommand verification loop
|
|
14134
|
-
*
|
|
14135
|
-
*
|
|
14136
|
-
*
|
|
14786
|
+
* The runCommand verification loop. One round asks the project's own commands,
|
|
14787
|
+
* then the review, then the run — and a round is over when all three are
|
|
14788
|
+
* clean. On anything else the output goes to Claude for a corrected set and
|
|
14789
|
+
* the next round asks all three again: a fix written for a failing run can
|
|
14790
|
+
* break a rule the round before it cleared.
|
|
14791
|
+
*
|
|
14792
|
+
* `fix.mode` mirrors the agent-browser target's fix UX (which only ever
|
|
14793
|
+
* prompts inside its own fix loop, never for the first write):
|
|
14137
14794
|
*
|
|
14138
14795
|
* - `auto` — apply every fix rewrite automatically, up to `fix.maxRetries`.
|
|
14139
14796
|
* - `interactive` (default) — show each fix pass's per-file diff and ask
|
|
@@ -14147,82 +14804,63 @@ async function finalizeAndVerify(p) {
|
|
|
14147
14804
|
*/
|
|
14148
14805
|
async function runVerificationLoop(p, state) {
|
|
14149
14806
|
const runCommand = p.ctx.targetConfig.runCommand;
|
|
14807
|
+
let review;
|
|
14808
|
+
const finish = (passed) => {
|
|
14809
|
+
for (const w of review?.warnings ?? []) warn(w);
|
|
14810
|
+
return {
|
|
14811
|
+
passed,
|
|
14812
|
+
review
|
|
14813
|
+
};
|
|
14814
|
+
};
|
|
14150
14815
|
if (!runCommand) {
|
|
14151
14816
|
const checks = await runCheckCommands(p.ctx);
|
|
14152
14817
|
if (checks !== null) {
|
|
14153
14818
|
warn(`${checks.command} failed (exit ${checks.exitCode}) — generated files kept`);
|
|
14154
14819
|
return { passed: false };
|
|
14155
14820
|
}
|
|
14156
|
-
|
|
14157
|
-
|
|
14158
|
-
return
|
|
14159
|
-
passed: true,
|
|
14160
|
-
review
|
|
14161
|
-
};
|
|
14821
|
+
await refreshFromDisk(state);
|
|
14822
|
+
review = await readingOfEmitted(p, state, true);
|
|
14823
|
+
return finish(true);
|
|
14162
14824
|
}
|
|
14163
14825
|
const maxRetries = p.ctx.fix.mode === "non-interactive" ? 0 : p.ctx.fix.maxRetries;
|
|
14164
|
-
const captures = p.
|
|
14826
|
+
const captures = p.stepEvidence?.supported === true;
|
|
14165
14827
|
const outputScrub = buildProseEnvScrubMap([], []);
|
|
14166
14828
|
if (!p.ctx.fix.useSnapshot) warn(`--no-session-pin has no effect on the ${p.target} target — it captures no browser snapshot; the fix loop uses the command's output instead`);
|
|
14167
|
-
|
|
14829
|
+
const spent = /* @__PURE__ */ new Set();
|
|
14830
|
+
const askReviewer = maxRetries > 0;
|
|
14831
|
+
let reviewerGone = false;
|
|
14832
|
+
if (!askReviewer && p.reading) info("no fix round could act on a review — the reviewer is not asked; the mechanical read still runs");
|
|
14168
14833
|
for (let attempt = 0;; attempt++) {
|
|
14169
|
-
|
|
14170
|
-
|
|
14171
|
-
|
|
14172
|
-
|
|
14173
|
-
await
|
|
14174
|
-
|
|
14175
|
-
|
|
14176
|
-
|
|
14177
|
-
|
|
14178
|
-
|
|
14179
|
-
|
|
14180
|
-
|
|
14181
|
-
|
|
14182
|
-
[ARTIFACTS_DIR_ENV]: artifactsDir,
|
|
14183
|
-
CCQA_RUN_ID: buildRunId(),
|
|
14184
|
-
...evidenceDir ? { [EVIDENCE_DIR_ENV]: evidenceDir } : {}
|
|
14185
|
-
}), "run");
|
|
14186
|
-
} finally {
|
|
14187
|
-
await rm(artifactsDir, {
|
|
14188
|
-
recursive: true,
|
|
14189
|
-
force: true
|
|
14190
|
-
});
|
|
14191
|
-
}
|
|
14192
|
-
let failing = command;
|
|
14193
|
-
if (result.exitCode === 0) {
|
|
14194
|
-
const checks = await runCheckCommands(p.ctx);
|
|
14195
|
-
if (checks !== null) {
|
|
14196
|
-
result = checks;
|
|
14197
|
-
failing = checks.command;
|
|
14834
|
+
let failing = await runCheckCommands(p.ctx);
|
|
14835
|
+
let asked = [];
|
|
14836
|
+
if (failing === null) {
|
|
14837
|
+
await refreshFromDisk(state);
|
|
14838
|
+
const [reading, mechanical] = await Promise.all([readingOfEmitted(p, state, askReviewer && !reviewerGone), reviewOfEmitted(p.ctx, state, p.writeRoots)]);
|
|
14839
|
+
review = reading;
|
|
14840
|
+
if (review?.reviewerFailed) reviewerGone = true;
|
|
14841
|
+
const unchecked = uncheckedSteps(review, spent);
|
|
14842
|
+
const violations = guideViolations(review, spent);
|
|
14843
|
+
const reads = allReadings(mechanical, unchecked, violations);
|
|
14844
|
+
if (reads !== null && attempt < maxRetries) {
|
|
14845
|
+
asked = [...unchecked?.asked ?? [], ...violations?.asked ?? []];
|
|
14846
|
+
failing = reads;
|
|
14198
14847
|
} else {
|
|
14199
|
-
review
|
|
14200
|
-
const
|
|
14201
|
-
if (
|
|
14202
|
-
|
|
14203
|
-
if (reads !== null) warn("generated with review findings still open — the files are kept and the findings are recorded against the case, but nothing acted on them");
|
|
14204
|
-
return {
|
|
14205
|
-
passed: true,
|
|
14206
|
-
review
|
|
14207
|
-
};
|
|
14208
|
-
}
|
|
14209
|
-
result = reads;
|
|
14210
|
-
failing = reads.command;
|
|
14848
|
+
if (reads !== null) warn("generated with review findings still open — the files are kept and the findings are recorded against the case, but nothing acted on them");
|
|
14849
|
+
const run = await verifyRun(p, state, runCommand, attempt, captures);
|
|
14850
|
+
if (run.exitCode === 0) return finish(true);
|
|
14851
|
+
failing = run;
|
|
14211
14852
|
}
|
|
14212
14853
|
}
|
|
14213
14854
|
if (attempt >= maxRetries) {
|
|
14214
14855
|
warn(`verification still failing after ${maxRetries} fix attempt(s) — generated files kept`);
|
|
14215
14856
|
await clearCaseRun(p.ctx.ref);
|
|
14216
|
-
return
|
|
14217
|
-
passed: false,
|
|
14218
|
-
review
|
|
14219
|
-
};
|
|
14857
|
+
return finish(false);
|
|
14220
14858
|
}
|
|
14221
|
-
fix(`verification failed (exit ${
|
|
14859
|
+
fix(`verification failed (exit ${failing.exitCode}) — requesting a fix (${attempt + 1}/${maxRetries})`);
|
|
14222
14860
|
const fixPrompt = buildLlmFixPrompt({
|
|
14223
14861
|
targetId: p.target,
|
|
14224
|
-
command: failing,
|
|
14225
|
-
outputTail: scrubEnvValues(tail(
|
|
14862
|
+
command: failing.command,
|
|
14863
|
+
outputTail: scrubEnvValues(tail(failing.output), outputScrub),
|
|
14226
14864
|
files: [...state.entries()].map(([path, f]) => ({
|
|
14227
14865
|
path,
|
|
14228
14866
|
contents: f.contents,
|
|
@@ -14247,6 +14885,7 @@ async function runVerificationLoop(p, state) {
|
|
|
14247
14885
|
warn(`fix attempt ${attempt + 1} produced no usable output (${err instanceof Error ? err.message : String(err)}) — keeping current files`);
|
|
14248
14886
|
continue;
|
|
14249
14887
|
}
|
|
14888
|
+
for (const key of asked) spent.add(key);
|
|
14250
14889
|
if (output.files.length === 0) {
|
|
14251
14890
|
info(`fix pass reported no file changes — re-running verification (${output.summary || "no reason given"})`);
|
|
14252
14891
|
continue;
|
|
@@ -14254,20 +14893,64 @@ async function runVerificationLoop(p, state) {
|
|
|
14254
14893
|
if (p.ctx.fix.mode === "interactive" && !await confirmFixWrite(output.files, p.ctx.cwd)) {
|
|
14255
14894
|
info("fix not applied (declined) — keeping current files");
|
|
14256
14895
|
await clearCaseRun(p.ctx.ref);
|
|
14257
|
-
return
|
|
14258
|
-
passed: false,
|
|
14259
|
-
review
|
|
14260
|
-
};
|
|
14896
|
+
return finish(false);
|
|
14261
14897
|
}
|
|
14262
14898
|
await writeGeneratedFiles(p.ctx.cwd, output.files, state);
|
|
14263
14899
|
review = void 0;
|
|
14264
14900
|
}
|
|
14265
14901
|
}
|
|
14266
14902
|
/**
|
|
14267
|
-
*
|
|
14268
|
-
*
|
|
14269
|
-
* project
|
|
14270
|
-
|
|
14903
|
+
* One verification run of the files as they are now, in the shape the fix loop
|
|
14904
|
+
* consumes. The round's last question and its most expensive: it is asked only
|
|
14905
|
+
* of files the project's own commands and the review have already cleared.
|
|
14906
|
+
*/
|
|
14907
|
+
async function verifyRun(p, state, runCommand, attempt, captures) {
|
|
14908
|
+
const testFiles = [...state.entries()].filter(([, f]) => f.kind === "test").map(([rel]) => rel);
|
|
14909
|
+
const artifactsDir = await mkdtemp(join(tmpdir(), "ccqa-verify-artifacts-"));
|
|
14910
|
+
const evidenceDir = captures ? caseRunDir(p.ctx.ref) : null;
|
|
14911
|
+
if (evidenceDir) {
|
|
14912
|
+
await clearCaseRun(p.ctx.ref);
|
|
14913
|
+
await mkdir(evidenceDir, { recursive: true });
|
|
14914
|
+
}
|
|
14915
|
+
let command = substituteArtifactsDir(substituteRunCommandFiles(runCommand, testFiles), artifactsDir);
|
|
14916
|
+
let traceUnavailable;
|
|
14917
|
+
if (evidenceDir) {
|
|
14918
|
+
const amended = amendForTrace(command, artifactsDir);
|
|
14919
|
+
command = amended.command;
|
|
14920
|
+
traceUnavailable = amended.skip;
|
|
14921
|
+
}
|
|
14922
|
+
run(command);
|
|
14923
|
+
try {
|
|
14924
|
+
const result = await timedPhase(`verification run #${attempt + 1}`, () => runShellCommand(command, p.ctx.cwd, {
|
|
14925
|
+
...process.env,
|
|
14926
|
+
[ARTIFACTS_DIR_ENV]: artifactsDir,
|
|
14927
|
+
CCQA_RUN_ID: buildRunId(),
|
|
14928
|
+
...evidenceDir ? { [EVIDENCE_DIR_ENV]: evidenceDir } : {}
|
|
14929
|
+
}), "run");
|
|
14930
|
+
if (evidenceDir && result.exitCode === 0) {
|
|
14931
|
+
const unavailable = traceUnavailable ?? await captureStepEvidence({
|
|
14932
|
+
artifactsDir,
|
|
14933
|
+
evidenceDir
|
|
14934
|
+
});
|
|
14935
|
+
if (unavailable) warn(unavailable);
|
|
14936
|
+
}
|
|
14937
|
+
return {
|
|
14938
|
+
...result,
|
|
14939
|
+
command
|
|
14940
|
+
};
|
|
14941
|
+
} finally {
|
|
14942
|
+
await rm(artifactsDir, {
|
|
14943
|
+
recursive: true,
|
|
14944
|
+
force: true
|
|
14945
|
+
});
|
|
14946
|
+
}
|
|
14947
|
+
}
|
|
14948
|
+
/**
|
|
14949
|
+
* The project's own checks over the whole repository (type check, lint), asked
|
|
14950
|
+
* first in a round: they cost nothing and code that does not compile has a
|
|
14951
|
+
* more urgent problem than how it reads. Answers null when they all pass — or
|
|
14952
|
+
* when the project configured none — and the failing one's output otherwise,
|
|
14953
|
+
* in the shape the fix loop already consumes.
|
|
14271
14954
|
*/
|
|
14272
14955
|
async function runCheckCommands(ctx) {
|
|
14273
14956
|
for (const command of ctx.targetConfig.checkCommands) {
|
|
@@ -14307,14 +14990,39 @@ async function reviewOfEmitted(ctx, state, writeRoots) {
|
|
|
14307
14990
|
output: formatEmittedReview(findings)
|
|
14308
14991
|
};
|
|
14309
14992
|
}
|
|
14310
|
-
/**
|
|
14311
|
-
|
|
14312
|
-
|
|
14313
|
-
|
|
14993
|
+
/**
|
|
14994
|
+
* What this attempt wrote, as the target's reading opens it.
|
|
14995
|
+
*
|
|
14996
|
+
* Timed like the loop's other phases: the reading opens the repository and
|
|
14997
|
+
* reads it, which is minutes, and a wedged one waits out its whole timeout
|
|
14998
|
+
* with nothing on the terminal to say what is happening.
|
|
14999
|
+
*/
|
|
15000
|
+
async function readingOfEmitted(p, state, askModel) {
|
|
15001
|
+
const { reading } = p;
|
|
15002
|
+
if (!reading) return void 0;
|
|
15003
|
+
const files = [...state.entries()].map(([path, f]) => ({
|
|
14314
15004
|
path,
|
|
14315
15005
|
contents: f.contents,
|
|
14316
15006
|
kind: f.kind
|
|
14317
|
-
}))
|
|
15007
|
+
}));
|
|
15008
|
+
return timedPhase("review of the generated files", () => reading(files, p.guides, askModel), "fix");
|
|
15009
|
+
}
|
|
15010
|
+
/**
|
|
15011
|
+
* Re-read what is on disk into the engine's copy of it.
|
|
15012
|
+
*
|
|
15013
|
+
* A check command may rewrite what it checks — a formatter is the ordinary
|
|
15014
|
+
* case — and from here the files are read three ways: the review opens the
|
|
15015
|
+
* disk, while the fix prompt and the mechanical read are given what the engine
|
|
15016
|
+
* holds. They must not be told different things about one file.
|
|
15017
|
+
*/
|
|
15018
|
+
async function refreshFromDisk(state) {
|
|
15019
|
+
for (const [rel, f] of state) {
|
|
15020
|
+
const contents = await readFile(f.abs, "utf8").catch(() => null);
|
|
15021
|
+
if (contents !== null) state.set(rel, {
|
|
15022
|
+
...f,
|
|
15023
|
+
contents
|
|
15024
|
+
});
|
|
15025
|
+
}
|
|
14318
15026
|
}
|
|
14319
15027
|
/**
|
|
14320
15028
|
* The reading's findings, shaped like a failed check so the fix loop carries
|
|
@@ -14334,17 +15042,23 @@ async function readingOfEmitted(p, state) {
|
|
|
14334
15042
|
* (restoring what passed weakly) would hide a case the product does not meet.
|
|
14335
15043
|
*
|
|
14336
15044
|
* A review that could not be obtained is not acted on either: spending a round
|
|
14337
|
-
* answering a question nobody asked is worse than leaving it unanswered.
|
|
14338
|
-
|
|
14339
|
-
|
|
14340
|
-
|
|
15045
|
+
* answering a question nobody asked is worse than leaving it unanswered. Nor
|
|
15046
|
+
* is a finding a round has already asked for and got back unchanged: a step
|
|
15047
|
+
* the fix pass could not strengthen reports the same line every round, and
|
|
15048
|
+
* left alone it takes the whole budget and the run happens with none of it.
|
|
15049
|
+
*/
|
|
15050
|
+
function uncheckedSteps(review, alreadyAsked = /* @__PURE__ */ new Set()) {
|
|
15051
|
+
if (!review?.complete || review.findings === null) return null;
|
|
15052
|
+
const findings = review.findings.filter((f) => !alreadyAsked.has(findingKey(f)));
|
|
15053
|
+
if (findings.length === 0) return null;
|
|
14341
15054
|
return {
|
|
14342
15055
|
exitCode: 1,
|
|
14343
15056
|
command: "ccqa: reading of the generated test",
|
|
15057
|
+
asked: findings.map(findingKey),
|
|
14344
15058
|
output: [
|
|
14345
15059
|
"These steps pass without deciding what the case says they must:",
|
|
14346
15060
|
"",
|
|
14347
|
-
...
|
|
15061
|
+
...findings.map((f) => `- ${formatFinding(f)}`),
|
|
14348
15062
|
"",
|
|
14349
15063
|
"Strengthen the assertions (and the locators they resolve through) so each",
|
|
14350
15064
|
"step fails when its expectation stops holding. Do not weaken or delete a",
|
|
@@ -14353,6 +15067,72 @@ function uncheckedSteps(review) {
|
|
|
14353
15067
|
};
|
|
14354
15068
|
}
|
|
14355
15069
|
/**
|
|
15070
|
+
* The rule violations the reading found, shaped like a failed check so the fix
|
|
15071
|
+
* loop carries them the same way — or null when there is nothing to act on.
|
|
15072
|
+
*
|
|
15073
|
+
* Every rule the loop could already decide is followed by the code it
|
|
15074
|
+
* produces: a type error fails the check command, a lint rule fails the lint,
|
|
15075
|
+
* a convention ccqa itself knows fails the review above. A rule that lives
|
|
15076
|
+
* only in the project's prose is followed by nothing, because until here
|
|
15077
|
+
* nothing in the loop had read it. Reused code is the older half of the same
|
|
15078
|
+
* gap — a helper this generation imported rather than wrote was written under
|
|
15079
|
+
* whatever rules existed then, and no rule added since has been applied to it.
|
|
15080
|
+
*
|
|
15081
|
+
* Like the reading it rides with, it spends a fix round and never decides the
|
|
15082
|
+
* verdict: it is a model's judgement, and a judgement that could fail a
|
|
15083
|
+
* generate would make generation only as repeatable as the model behind it.
|
|
15084
|
+
* And like the reading, a violation a round has already asked for and got back
|
|
15085
|
+
* unchanged is not asked again — the model may decline a file it is not
|
|
15086
|
+
* allowed to write, and asking again spends a round that can only be declined
|
|
15087
|
+
* again.
|
|
15088
|
+
*
|
|
15089
|
+
* Read apart from the step findings, as it comes back: one answer carries two
|
|
15090
|
+
* reviews, and a half that arrived unreadable must not discard the half that
|
|
15091
|
+
* arrived. Absent violations are absent either way — there is nothing here to
|
|
15092
|
+
* act on when that half never came.
|
|
15093
|
+
*
|
|
15094
|
+
* Only what the reviewer would hold the change for. It is asked to sort its
|
|
15095
|
+
* own violations, because the alternative is a round spent on a line it would
|
|
15096
|
+
* have approved anyway. Every violation is still reported: `warnings` carries
|
|
15097
|
+
* the advisory ones too, marked as such.
|
|
15098
|
+
*/
|
|
15099
|
+
function guideViolations(review, alreadyAsked = /* @__PURE__ */ new Set()) {
|
|
15100
|
+
const violations = (review?.ruleViolations ?? []).filter((v) => v.severity === "blocking" && !alreadyAsked.has(violationKey(v)));
|
|
15101
|
+
if (violations.length === 0) return null;
|
|
15102
|
+
return {
|
|
15103
|
+
exitCode: 1,
|
|
15104
|
+
command: "ccqa: the project's own conventions",
|
|
15105
|
+
asked: violations.map(violationKey),
|
|
15106
|
+
output: [
|
|
15107
|
+
"These files break a rule this project's conventions state:",
|
|
15108
|
+
"",
|
|
15109
|
+
...violations.flatMap((v) => [
|
|
15110
|
+
`- ${formatViolation(v)}`,
|
|
15111
|
+
...v.code.trim().split("\n").map((line) => ` ${line}`),
|
|
15112
|
+
""
|
|
15113
|
+
]),
|
|
15114
|
+
"Rewrite each file so it follows the rule it breaks. A support file this",
|
|
15115
|
+
"generation did not write is still yours to correct while it is under a",
|
|
15116
|
+
"root you may write to; one outside them is not — say so in `summary`",
|
|
15117
|
+
"rather than rewriting it. Never weaken what the test checks to make a",
|
|
15118
|
+
"rule fit."
|
|
15119
|
+
].join("\n")
|
|
15120
|
+
};
|
|
15121
|
+
}
|
|
15122
|
+
/** One finding, as the loop recognises the same one coming back. */
|
|
15123
|
+
function findingKey(finding) {
|
|
15124
|
+
return [finding.stepId, finding.problem].join("\0");
|
|
15125
|
+
}
|
|
15126
|
+
/** One violation, as the loop recognises the same one coming back. */
|
|
15127
|
+
function violationKey(violation) {
|
|
15128
|
+
return [
|
|
15129
|
+
violation.file,
|
|
15130
|
+
violation.guide,
|
|
15131
|
+
violation.rule,
|
|
15132
|
+
violation.code.trim()
|
|
15133
|
+
].join("\0");
|
|
15134
|
+
}
|
|
15135
|
+
/**
|
|
14356
15136
|
* Every identifier the project's own test assets mention, outside the files
|
|
14357
15137
|
* this generation just wrote.
|
|
14358
15138
|
*
|
|
@@ -14399,10 +15179,10 @@ async function identifiersInProject(ctx, writeRoots, emitted) {
|
|
|
14399
15179
|
return found;
|
|
14400
15180
|
}
|
|
14401
15181
|
/**
|
|
14402
|
-
* The
|
|
14403
|
-
* answers
|
|
15182
|
+
* The readings of the finished files as one report, since one fix pass
|
|
15183
|
+
* answers all of them. Any may be absent; all absent is nothing to act on.
|
|
14404
15184
|
*/
|
|
14405
|
-
function
|
|
15185
|
+
function allReadings(...reports) {
|
|
14406
15186
|
const found = reports.filter((r) => r !== null);
|
|
14407
15187
|
if (found.length === 0) return null;
|
|
14408
15188
|
return {
|
|
@@ -14586,48 +15366,9 @@ function creationActionIndex(actions, markers) {
|
|
|
14586
15366
|
while (end > submitted && OBSERVING_ACTIONS.has(actions[end].action)) end -= 1;
|
|
14587
15367
|
return end;
|
|
14588
15368
|
}
|
|
14589
|
-
/** Module the emitted step-boundary capture calls import from. */
|
|
14590
|
-
const STEP_EVIDENCE_MODULE = "ccqa/step-evidence";
|
|
14591
15369
|
/** Module the emitted judge calls import from, and the call they make. */
|
|
14592
15370
|
const JUDGE_MODULE = "ccqa/judge";
|
|
14593
15371
|
const JUDGE_CALL = "judgeByLlm";
|
|
14594
|
-
/** Capture call emitted when a step is entered / closed. Exported for the generation gate. */
|
|
14595
|
-
const STEP_EVIDENCE_BEFORE = "ccqaStepBefore";
|
|
14596
|
-
const STEP_EVIDENCE_AFTER = "ccqaStepAfter";
|
|
14597
|
-
/** The exact boundary call for one step, as emitted and as the gate greps for it. */
|
|
14598
|
-
function stepEvidenceCall(fn, marker) {
|
|
14599
|
-
return injectedCall(fn, [j(marker.stepId), j(marker.source)]);
|
|
14600
|
-
}
|
|
14601
|
-
/**
|
|
14602
|
-
* A receiver expression: anything up to the argument's comma, allowing one
|
|
14603
|
-
* level of parentheses so `await ctx.newPage()` and `page.context().pages()[1]`
|
|
14604
|
-
* — how a rewrite names a tab a click opened — count as receivers.
|
|
14605
|
-
*/
|
|
14606
|
-
const RECEIVER = String.raw`(?:[^,()]|\([^()]*\))+`;
|
|
14607
|
-
/**
|
|
14608
|
-
* The call as emitted, plus the pattern that accepts it back on any page. The
|
|
14609
|
-
* receiver is the one part a rewrite is right to change: a click that opens a
|
|
14610
|
-
* new tab has to act on that tab. Everything else is required verbatim, the
|
|
14611
|
-
* `await` and `;` included — an unawaited capture races the end of the test,
|
|
14612
|
-
* and a mention in a comment is not a call.
|
|
14613
|
-
*/
|
|
14614
|
-
function injectedCall(name, args) {
|
|
14615
|
-
const tail = args.map(escapeRegExp).join(String.raw`\s*,\s*`);
|
|
14616
|
-
return {
|
|
14617
|
-
code: `await ${name}(page, ${args.join(", ")});`,
|
|
14618
|
-
pattern: new RegExp(String.raw`await\s+${name}\s*\(\s*${RECEIVER}\s*,\s*${tail}\s*,?\s*\)\s*;`)
|
|
14619
|
-
};
|
|
14620
|
-
}
|
|
14621
|
-
/**
|
|
14622
|
-
* The "preserve the step-evidence calls" rule the library-rewrite prompt must
|
|
14623
|
-
* carry, built from the same symbol/module constants the emitter injects and
|
|
14624
|
-
* the coverage gate greps for — so prompt, emitter, and gate share one truth.
|
|
14625
|
-
* A target that captures no step evidence simply doesn't pass this to the
|
|
14626
|
-
* engine, and the prompt then omits the rule entirely.
|
|
14627
|
-
*/
|
|
14628
|
-
function stepEvidencePreserveRule() {
|
|
14629
|
-
return `**Keep the \`${STEP_EVIDENCE_MODULE}\` calls.** The draft's \`await ${STEP_EVIDENCE_BEFORE}(page, ...)\` / \`await ${STEP_EVIDENCE_AFTER}(page, ...)\` lines are load-bearing: ccqa run reads the per-step screenshots they capture. Keep both calls for every step, in place around that step's actions, with their exact \`"<stepId>", "<source>"\` arguments — and keep the import. Pass a different page only when the step genuinely acts on one (a click that opens a new tab). If you move a step's actions into a page-object method, leave these two calls in the test body around the call to that method; do NOT move them inside the page object. Never wrap a step in a closure to hold them.`;
|
|
14630
|
-
}
|
|
14631
15372
|
/**
|
|
14632
15373
|
* Told to the rewrite because both are conventions the project greps: a
|
|
14633
15374
|
* missing tag drops the test out of whatever selection runs it, and a missing
|
|
@@ -14645,7 +15386,7 @@ function headerPreserveRule(header, titleSuffix) {
|
|
|
14645
15386
|
* rejection, which costs a whole round.
|
|
14646
15387
|
*/
|
|
14647
15388
|
function stepCommentPreserveRule() {
|
|
14648
|
-
return "**Keep
|
|
15389
|
+
return "**Keep every `test.step(...)` block, with its title exactly as the draft wrote it.** Those titles are how the evidence table, the run's per-step screenshots and the review of this test say which assertions belong to which step. A reshaped one parses as no step at all, and every step then reads as deciding nothing. Keep the wording, the numbering and the punctuation, and keep each step's actions inside its own block — a step's call to a page-object method belongs in the block, not outside it. Do not merge, split or reorder the blocks. A test already at this path may have been written by an older ccqa in a different shape: the draft is what this file must look like, not that one.";
|
|
14649
15390
|
}
|
|
14650
15391
|
/**
|
|
14651
15392
|
* Told to the rewrite, because the alternative failure is silent: a claim
|
|
@@ -14657,51 +15398,55 @@ function judgePreserveRule() {
|
|
|
14657
15398
|
}
|
|
14658
15399
|
function emitPlaywrightDraft(input) {
|
|
14659
15400
|
const { actions, testName, judgements = [], japanese = false } = input;
|
|
14660
|
-
const captures = input.stepEvidence !== false;
|
|
14661
15401
|
const stepMarkers = input.stepMarkers ?? [];
|
|
14662
15402
|
const markerByIndex = new Map(stepMarkers.map((m) => [m.actionIndex, m]));
|
|
14663
|
-
const
|
|
15403
|
+
const body = [];
|
|
14664
15404
|
let prevLine = null;
|
|
14665
|
-
let
|
|
15405
|
+
let open = null;
|
|
15406
|
+
const target = () => open ?? body;
|
|
15407
|
+
const openStep = (marker) => {
|
|
15408
|
+
if (!marker) return;
|
|
15409
|
+
open = openBlock(body, marker, japanese);
|
|
15410
|
+
};
|
|
14666
15411
|
const flushJudgements = (afterActionIndex) => {
|
|
14667
15412
|
for (const { step } of judgements.filter((j) => j.afterActionIndex === afterActionIndex)) {
|
|
14668
|
-
|
|
14669
|
-
if (captures) lines.push(stepEvidenceCall(STEP_EVIDENCE_AFTER, openMarker).code);
|
|
14670
|
-
openMarker = null;
|
|
14671
|
-
}
|
|
14672
|
-
if (lines.length > 0) lines.push("");
|
|
14673
|
-
lines.push(renderStepComment({
|
|
15413
|
+
openBlock(body, {
|
|
14674
15414
|
stepId: step.id,
|
|
14675
15415
|
source: step.source
|
|
14676
|
-
}, japanese));
|
|
14677
|
-
|
|
15416
|
+
}, japanese).push(judgeCall(step).code);
|
|
15417
|
+
open = null;
|
|
14678
15418
|
}
|
|
14679
15419
|
};
|
|
14680
|
-
const
|
|
14681
|
-
let
|
|
15420
|
+
const creationIndex = creationActionIndex(actions, stepMarkers);
|
|
15421
|
+
let createdAt = {
|
|
15422
|
+
block: body,
|
|
15423
|
+
index: 0
|
|
15424
|
+
};
|
|
14682
15425
|
flushJudgements(-1);
|
|
14683
15426
|
for (let i = 0; i < actions.length; i++) {
|
|
14684
|
-
|
|
15427
|
+
openStep(markerByIndex.get(i));
|
|
14685
15428
|
const action = actions[i];
|
|
14686
15429
|
if (overwrittenByNext(action, actions[i + 1])) continue;
|
|
14687
15430
|
const line = actionToLine(action, japanese);
|
|
14688
15431
|
if (line !== null && line !== prevLine) {
|
|
14689
|
-
if (action.replayUnstable)
|
|
14690
|
-
|
|
15432
|
+
if (action.replayUnstable) target().push(`// [warn] replay-unstable: ${action.replayReason ?? "(no reason recorded)"}`);
|
|
15433
|
+
target().push(...line.split("\n"));
|
|
14691
15434
|
prevLine = line;
|
|
14692
15435
|
}
|
|
14693
|
-
if (i ===
|
|
15436
|
+
if (i === creationIndex) createdAt = {
|
|
15437
|
+
block: target(),
|
|
15438
|
+
index: target().length
|
|
15439
|
+
};
|
|
14694
15440
|
flushJudgements(i);
|
|
14695
15441
|
}
|
|
14696
|
-
|
|
14697
|
-
const cleanupLines = emitCleanup(input.cleanup, captures, japanese, input.allowExpectInCleanup ?? true);
|
|
15442
|
+
const cleanupBody = emitCleanup(input.cleanup, japanese, input.allowExpectInCleanup ?? true);
|
|
14698
15443
|
const title = `${testName}${input.titleSuffix ?? ""}`;
|
|
14699
|
-
const runId = input.runId !== void 0 && [...
|
|
14700
|
-
const guarded = runId !== void 0 &&
|
|
14701
|
-
if (guarded)
|
|
14702
|
-
if (judgements.length > 0)
|
|
14703
|
-
const testLines = [...runId ? [`${RUN_ID_VAR} = ${runId.expression};`, ""] : [], ...
|
|
14704
|
-
const scoped =
|
|
15444
|
+
const runId = input.runId !== void 0 && [...codeLines(body), ...codeLines(cleanupBody)].some((line) => line.includes(RUN_ID_READ)) ? input.runId : void 0;
|
|
15445
|
+
const guarded = runId !== void 0 && cleanupBody.length > 0;
|
|
15446
|
+
if (guarded) createdAt.block.splice(createdAt.index, 0, `${CREATED_VAR} = true;`);
|
|
15447
|
+
if (judgements.length > 0) body.unshift("test.slow();", "");
|
|
15448
|
+
const testLines = [...runId ? [`${RUN_ID_VAR} = ${runId.expression};`, ""] : [], ...body];
|
|
15449
|
+
const scoped = cleanupBody.length > 0 || runId !== void 0;
|
|
14705
15450
|
const testParams = judgements.length > 0 ? "{ page }, testInfo" : "{ page }";
|
|
14706
15451
|
const declaration = [
|
|
14707
15452
|
...scoped ? [
|
|
@@ -14710,13 +15455,13 @@ function emitPlaywrightDraft(input) {
|
|
|
14710
15455
|
""
|
|
14711
15456
|
] : [],
|
|
14712
15457
|
`test(${j(title)}, async (${testParams}) => {`,
|
|
14713
|
-
|
|
15458
|
+
...renderBody(testLines, 2),
|
|
14714
15459
|
"});",
|
|
14715
|
-
...
|
|
15460
|
+
...cleanupBody.length > 0 ? [
|
|
14716
15461
|
"",
|
|
14717
15462
|
`test.afterEach(${j(cleanupTitle(input.cleanup?.stepMarkers, japanese))}, async ({ page }) => {`,
|
|
14718
15463
|
...guarded ? [` if (!${CREATED_VAR}) return;`] : [],
|
|
14719
|
-
|
|
15464
|
+
...renderBody(cleanupBody, 2),
|
|
14720
15465
|
"});"
|
|
14721
15466
|
] : []
|
|
14722
15467
|
];
|
|
@@ -14724,7 +15469,6 @@ function emitPlaywrightDraft(input) {
|
|
|
14724
15469
|
...input.header ? [input.header.trimEnd(), ""] : [],
|
|
14725
15470
|
`import { test, expect } from "@playwright/test";`,
|
|
14726
15471
|
...judgements.length > 0 ? [`import { ${JUDGE_CALL} } from ${j(JUDGE_MODULE)};`] : [],
|
|
14727
|
-
...stepMarkers.length > 0 && captures ? [`import { ${STEP_EVIDENCE_BEFORE}, ${STEP_EVIDENCE_AFTER} } from ${j(STEP_EVIDENCE_MODULE)};`] : [],
|
|
14728
15472
|
...runId ? [runId.import] : [],
|
|
14729
15473
|
"",
|
|
14730
15474
|
...declaration,
|
|
@@ -14732,49 +15476,61 @@ function emitPlaywrightDraft(input) {
|
|
|
14732
15476
|
].join("\n");
|
|
14733
15477
|
return runId ? source.replaceAll(RUN_ID_READ, RUN_ID_VAR) : source;
|
|
14734
15478
|
}
|
|
14735
|
-
|
|
15479
|
+
/**
|
|
15480
|
+
* Open a step's block at the end of `body` and return the array its lines go
|
|
15481
|
+
* in. Shared by the two places that walk an action list against markers, so a
|
|
15482
|
+
* change to how a boundary is written cannot apply to the steps and miss the
|
|
15483
|
+
* cleanup.
|
|
15484
|
+
*/
|
|
15485
|
+
function openBlock(body, marker, japanese) {
|
|
15486
|
+
if (body.length > 0) body.push("");
|
|
15487
|
+
const block = {
|
|
15488
|
+
title: renderStepLabel(marker, japanese),
|
|
15489
|
+
body: []
|
|
15490
|
+
};
|
|
15491
|
+
body.push(block);
|
|
15492
|
+
return block.body;
|
|
15493
|
+
}
|
|
15494
|
+
function renderBody(body, by) {
|
|
14736
15495
|
const pad = " ".repeat(by);
|
|
14737
|
-
|
|
15496
|
+
const out = [];
|
|
15497
|
+
for (const node of body) {
|
|
15498
|
+
if (typeof node === "string") {
|
|
15499
|
+
out.push(node === "" ? "" : `${pad}${node}`);
|
|
15500
|
+
continue;
|
|
15501
|
+
}
|
|
15502
|
+
out.push(`${pad}await test.step(${j(node.title)}, async () => {`);
|
|
15503
|
+
out.push(...renderBody(node.body, by + 2));
|
|
15504
|
+
out.push(`${pad}});`);
|
|
15505
|
+
}
|
|
15506
|
+
return out;
|
|
15507
|
+
}
|
|
15508
|
+
/** Every code line the body holds, step blocks included. */
|
|
15509
|
+
function codeLines(body) {
|
|
15510
|
+
return body.flatMap((node) => typeof node === "string" ? [node] : codeLines(node.body));
|
|
14738
15511
|
}
|
|
14739
15512
|
/**
|
|
14740
15513
|
* The recorded undo actions, with their step comments. Emitted from the
|
|
14741
15514
|
* recording like everything else: what the cleanup does was demonstrated in
|
|
14742
15515
|
* the browser, not guessed from the case's prose.
|
|
14743
15516
|
*/
|
|
14744
|
-
function emitCleanup(cleanup,
|
|
15517
|
+
function emitCleanup(cleanup, japanese, allowExpect) {
|
|
14745
15518
|
if (!cleanup || cleanup.actions.length === 0) return [];
|
|
14746
|
-
const
|
|
15519
|
+
const body = [];
|
|
14747
15520
|
const markerByIndex = new Map((cleanup.stepMarkers ?? []).map((m) => [m.actionIndex, m]));
|
|
14748
15521
|
let open = null;
|
|
15522
|
+
const target = () => open ?? body;
|
|
14749
15523
|
for (let i = 0; i < cleanup.actions.length; i++) {
|
|
14750
|
-
|
|
15524
|
+
const marker = markerByIndex.get(i);
|
|
15525
|
+
if (marker) open = openBlock(body, marker, japanese);
|
|
14751
15526
|
const action = cleanup.actions[i];
|
|
14752
15527
|
if (!allowExpect && action.action === "assert") continue;
|
|
14753
15528
|
if (overwrittenByNext(action, cleanup.actions[i + 1])) continue;
|
|
14754
|
-
if (action.replayUnstable)
|
|
15529
|
+
if (action.replayUnstable) target().push(`// [warn] replay-unstable: ${action.replayReason ?? "(no reason recorded)"}`);
|
|
14755
15530
|
const line = actionToLine(action, japanese);
|
|
14756
|
-
if (line !== null)
|
|
15531
|
+
if (line !== null) target().push(...line.split("\n"));
|
|
14757
15532
|
}
|
|
14758
|
-
|
|
14759
|
-
return lines;
|
|
14760
|
-
}
|
|
14761
|
-
/**
|
|
14762
|
-
* Enter the step a marker starts: close the one before it, comment the
|
|
14763
|
-
* boundary, and open the capture. Shared by the two places that walk an action
|
|
14764
|
-
* list against markers, so a change to how a boundary is written cannot apply
|
|
14765
|
-
* to the steps and miss the cleanup.
|
|
14766
|
-
*/
|
|
14767
|
-
function openStep(lines, marker, open, japanese, captures) {
|
|
14768
|
-
if (!marker) return open;
|
|
14769
|
-
if (open && captures) lines.push(stepEvidenceCall(STEP_EVIDENCE_AFTER, open).code);
|
|
14770
|
-
if (lines.length > 0) lines.push("");
|
|
14771
|
-
lines.push(renderStepComment(marker, japanese));
|
|
14772
|
-
if (captures) lines.push(stepEvidenceCall(STEP_EVIDENCE_BEFORE, marker).code);
|
|
14773
|
-
return marker;
|
|
14774
|
-
}
|
|
14775
|
-
/** Close the step still open at the end of an action list. */
|
|
14776
|
-
function closeStep(lines, open, captures) {
|
|
14777
|
-
if (open && captures) lines.push(stepEvidenceCall(STEP_EVIDENCE_AFTER, open).code);
|
|
15533
|
+
return body;
|
|
14778
15534
|
}
|
|
14779
15535
|
/**
|
|
14780
15536
|
* Render a locator (plus positional pick) as a Playwright locator expression.
|
|
@@ -14925,7 +15681,7 @@ function assertToLine(action, locator, japanese) {
|
|
|
14925
15681
|
break;
|
|
14926
15682
|
case void 0: break;
|
|
14927
15683
|
}
|
|
14928
|
-
if (comment && assertLine) return `${comment}\n
|
|
15684
|
+
if (comment && assertLine) return `${comment}\n${assertLine}`;
|
|
14929
15685
|
return assertLine ?? comment;
|
|
14930
15686
|
}
|
|
14931
15687
|
/**
|
|
@@ -14964,12 +15720,25 @@ const jExpr = (s) => envRefsToJsExpression(s);
|
|
|
14964
15720
|
* require it back. `testInfo` is always passed — the case's `test(...)`
|
|
14965
15721
|
* callback is emitted with that second parameter whenever it has a judgement
|
|
14966
15722
|
* — so the verdict lands on the report whether the claim held or not.
|
|
15723
|
+
*
|
|
15724
|
+
* The pattern accepts the call on any page: the receiver is the one part a
|
|
15725
|
+
* rewrite is right to change, because a click that opens a new tab has to be
|
|
15726
|
+
* judged on that tab. It allows one level of parentheses so `await
|
|
15727
|
+
* ctx.newPage()` and `page.context().pages()[1]` count as receivers.
|
|
15728
|
+
* Everything else is required verbatim, the `await` and `;` included — an
|
|
15729
|
+
* unawaited claim races the end of the test, and a mention in a comment is not
|
|
15730
|
+
* a call.
|
|
14967
15731
|
*/
|
|
14968
15732
|
function judgeCall(step) {
|
|
14969
15733
|
const args = [bracedRefsToJsExpression(step.judgeByLlm.trim())];
|
|
14970
15734
|
const optionsFields = step.from !== void 0 ? [`from: ${jExpr(step.from)}`, "testInfo"] : ["testInfo"];
|
|
14971
15735
|
args.push(`{ ${optionsFields.join(", ")} }`);
|
|
14972
|
-
|
|
15736
|
+
const receiver = String.raw`(?:[^,()]|\([^()]*\))+`;
|
|
15737
|
+
const tail = args.map(escapeRegExp).join(String.raw`\s*,\s*`);
|
|
15738
|
+
return {
|
|
15739
|
+
code: `await ${JUDGE_CALL}(page, ${args.join(", ")});`,
|
|
15740
|
+
pattern: new RegExp(String.raw`await\s+${JUDGE_CALL}\s*\(\s*${receiver}\s*,\s*${tail}\s*,?\s*\)\s*;`)
|
|
15741
|
+
};
|
|
14973
15742
|
}
|
|
14974
15743
|
//#endregion
|
|
14975
15744
|
//#region src/targets/playwright/browser-server.ts
|
|
@@ -15128,6 +15897,28 @@ async function waitForCdp(port) {
|
|
|
15128
15897
|
}
|
|
15129
15898
|
}
|
|
15130
15899
|
//#endregion
|
|
15900
|
+
//#region src/targets/types.ts
|
|
15901
|
+
/**
|
|
15902
|
+
* Whether a run of this target's tests yields per-step screenshots: the target
|
|
15903
|
+
* has to be able to produce them, and the project has to want them — asking
|
|
15904
|
+
* Playwright for a trace on every run is what producing them costs.
|
|
15905
|
+
*
|
|
15906
|
+
* Both halves are ANDed here, once, wherever a plugin and its config first
|
|
15907
|
+
* meet. A runner then reads a capability instead of deciding one again, and
|
|
15908
|
+
* the "no" a report row shows is written in a single place.
|
|
15909
|
+
*/
|
|
15910
|
+
function resolveStepEvidence(plugin, config) {
|
|
15911
|
+
const declared = plugin.stepEvidence ?? {
|
|
15912
|
+
supported: false,
|
|
15913
|
+
reason: `the "${plugin.id}" target does not capture step screenshots`
|
|
15914
|
+
};
|
|
15915
|
+
if (!declared.supported || config.hooks.stepEvidence) return declared;
|
|
15916
|
+
return {
|
|
15917
|
+
supported: false,
|
|
15918
|
+
reason: `the "${plugin.id}" target has step evidence turned off in .ccqa/config.yaml`
|
|
15919
|
+
};
|
|
15920
|
+
}
|
|
15921
|
+
//#endregion
|
|
15131
15922
|
//#region src/targets/playwright/index.ts
|
|
15132
15923
|
const PLAYWRIGHT_TARGET = "playwright";
|
|
15133
15924
|
/**
|
|
@@ -15159,7 +15950,7 @@ const playwrightTarget = {
|
|
|
15159
15950
|
async function generatePlaywrightTest(ctx) {
|
|
15160
15951
|
if (!ctx.recording) throw new Error(`the playwright target needs a recording — run \`ccqa record ${ctx.featureName}/${ctx.specName}\` first`);
|
|
15161
15952
|
if (ctx.targetConfig.testPath === void 0) warn(`no \`testPath\` configured for the playwright target — writing ${ctx.testPath}, the same path the agent-browser target uses for its vitest test. Set \`targets.playwright.testPath\` in .ccqa/config.yaml (e.g. \`e2e/specs/{feature}/{spec}.spec.ts\`) to keep them apart.`);
|
|
15162
|
-
return compileRecording(ctx, PLAYWRIGHT_TARGET);
|
|
15953
|
+
return compileRecording(ctx, PLAYWRIGHT_TARGET, resolveStepEvidence(playwrightTarget, ctx.targetConfig));
|
|
15163
15954
|
}
|
|
15164
15955
|
/**
|
|
15165
15956
|
* IR → a `@playwright/test` file, for every target whose tests are Playwright's.
|
|
@@ -15170,12 +15961,15 @@ async function generatePlaywrightTest(ctx) {
|
|
|
15170
15961
|
* the same gates over what the rewrite may drop. Sharing the pipeline is what
|
|
15171
15962
|
* keeps those gates from applying to one target and not the other — the way a
|
|
15172
15963
|
* second copy always eventually does.
|
|
15964
|
+
*
|
|
15965
|
+
* `stepEvidence` comes from the target that entered the pipeline, already
|
|
15966
|
+
* resolved against the project's config: the two targets differ in which
|
|
15967
|
+
* plugin answers, and only they know which one it is.
|
|
15173
15968
|
*/
|
|
15174
|
-
async function compileRecording(ctx, guidanceKind) {
|
|
15969
|
+
async function compileRecording(ctx, guidanceKind, stepEvidence) {
|
|
15175
15970
|
const actions = ctx.recording;
|
|
15176
15971
|
const expanded = ctx.steps;
|
|
15177
15972
|
const cleanup = ctx.cleanup.filter(isExpandedActionStep);
|
|
15178
|
-
const captures = ctx.targetConfig.hooks.stepEvidence;
|
|
15179
15973
|
const stepMarkers = buildStepMarkers(expanded.filter(isExpandedActionStep), actions);
|
|
15180
15974
|
const cleanupMarkers = buildStepMarkers(cleanup, ctx.cleanupRecording ?? []);
|
|
15181
15975
|
const unattributed = expanded.filter(isExpandedActionStep).map((s) => s.id).filter((id) => !stepMarkers.some((m) => m.stepId === id));
|
|
@@ -15196,7 +15990,6 @@ async function compileRecording(ctx, guidanceKind) {
|
|
|
15196
15990
|
stepMarkers: cleanupMarkers
|
|
15197
15991
|
} } : {},
|
|
15198
15992
|
...ctx.targetConfig.runId ? { runId: ctx.targetConfig.runId } : {},
|
|
15199
|
-
stepEvidence: captures,
|
|
15200
15993
|
allowExpectInCleanup: ctx.targetConfig.allowExpectInCleanup,
|
|
15201
15994
|
japanese: useJapanesePrompts(ctx.language)
|
|
15202
15995
|
});
|
|
@@ -15205,28 +15998,24 @@ async function compileRecording(ctx, guidanceKind) {
|
|
|
15205
15998
|
blank();
|
|
15206
15999
|
const invariants = [
|
|
15207
16000
|
stepMarkers.length > 0 || cleanupMarkers.length > 0 ? stepCommentPreserveRule() : "",
|
|
15208
|
-
stepMarkers.length > 0 && captures ? stepEvidencePreserveRule() : "",
|
|
15209
16001
|
judgements.length > 0 ? judgePreserveRule() : "",
|
|
15210
16002
|
header || titleSuffix ? headerPreserveRule(header, titleSuffix) : ""
|
|
15211
16003
|
].filter(Boolean).join("\n\n");
|
|
15212
16004
|
const injected = {
|
|
15213
16005
|
markers: [...stepMarkers, ...cleanupMarkers],
|
|
15214
|
-
stepEvidence: captures,
|
|
15215
16006
|
judgements,
|
|
15216
16007
|
header,
|
|
15217
16008
|
titleSuffix
|
|
15218
16009
|
};
|
|
15219
16010
|
const validateFile = (file) => {
|
|
15220
|
-
|
|
15221
|
-
const gaps = injectedCallGaps(file.contents, injected);
|
|
16011
|
+
const gaps = file.kind === "test" ? injectedCallGaps(file.contents, injected) : runtimeImportGaps(file.contents);
|
|
15222
16012
|
return gaps.length === 0 ? null : gaps.join("; ");
|
|
15223
16013
|
};
|
|
15224
|
-
const reading = (files) => reviewGeneratedTest({
|
|
15225
|
-
|
|
15226
|
-
|
|
15227
|
-
|
|
15228
|
-
|
|
15229
|
-
})),
|
|
16014
|
+
const reading = async (files, guides, askModel) => reviewGeneratedTest({
|
|
16015
|
+
files,
|
|
16016
|
+
askModel,
|
|
16017
|
+
leansOn: await leanedOn(files, ctx.cwd, ctx.testPath),
|
|
16018
|
+
guides,
|
|
15230
16019
|
steps: expanded,
|
|
15231
16020
|
...ctx.expectations.length > 0 ? { expectations: ctx.expectations } : {},
|
|
15232
16021
|
...ctx.cleanupExpectations.length > 0 && ctx.targetConfig.allowExpectInCleanup ? { cleanup } : {},
|
|
@@ -15244,6 +16033,7 @@ async function compileRecording(ctx, guidanceKind) {
|
|
|
15244
16033
|
contents: draft
|
|
15245
16034
|
},
|
|
15246
16035
|
...invariants ? { draftInvariant: invariants } : {},
|
|
16036
|
+
stepEvidence,
|
|
15247
16037
|
validateFile,
|
|
15248
16038
|
reading
|
|
15249
16039
|
}) : await finalizePreparedFiles({
|
|
@@ -15256,6 +16046,7 @@ async function compileRecording(ctx, guidanceKind) {
|
|
|
15256
16046
|
}],
|
|
15257
16047
|
summary: `test compiled from ${actions.length} recorded action(s)`,
|
|
15258
16048
|
warnings: [],
|
|
16049
|
+
stepEvidence,
|
|
15259
16050
|
validateFile,
|
|
15260
16051
|
reading
|
|
15261
16052
|
});
|
|
@@ -15313,6 +16104,21 @@ async function unusedSupportExports(result, written, cwd) {
|
|
|
15313
16104
|
return warnings;
|
|
15314
16105
|
}
|
|
15315
16106
|
/**
|
|
16107
|
+
* What the written test reaches by importing it, and this generation did not
|
|
16108
|
+
* write — the half nobody re-reads. A page object this case reuses was written
|
|
16109
|
+
* under whatever the project's rules were at the time, it is imported
|
|
16110
|
+
* verbatim, and the locators in it decide as much of this case as the ones in
|
|
16111
|
+
* the test file do. Followed from the imports, because that is how the code
|
|
16112
|
+
* itself finds them.
|
|
16113
|
+
*
|
|
16114
|
+
* Paths only: the review opens what it decides to open, so listing a file
|
|
16115
|
+
* costs nothing and there is no budget to spend.
|
|
16116
|
+
*/
|
|
16117
|
+
async function leanedOn(files, cwd, testPath) {
|
|
16118
|
+
const written = new Set(files.map((f) => resolve(cwd, f.path)));
|
|
16119
|
+
return (await collectSupportFiles(resolve(cwd, testPath), cwd)).filter((file) => !written.has(file.abs) && file.source !== "").map((file) => relative(cwd, file.abs));
|
|
16120
|
+
}
|
|
16121
|
+
/**
|
|
15316
16122
|
* The injected calls and stamped conventions `corpus` no longer has.
|
|
15317
16123
|
*
|
|
15318
16124
|
* Asked twice, of the same source: once of the reply, before it is written,
|
|
@@ -15321,21 +16127,16 @@ async function unusedSupportExports(result, written, cwd) {
|
|
|
15321
16127
|
* function, because a gate that answered differently in the two moments would
|
|
15322
16128
|
* be worse than either alone.
|
|
15323
16129
|
*/
|
|
15324
|
-
function injectedCallGaps(corpus, { markers,
|
|
15325
|
-
const warnings = [];
|
|
16130
|
+
function injectedCallGaps(corpus, { markers, judgements, header, titleSuffix }) {
|
|
16131
|
+
const warnings = [...runtimeImportGaps(corpus)];
|
|
15326
16132
|
const stamped = {
|
|
15327
16133
|
header,
|
|
15328
16134
|
titleSuffix
|
|
15329
16135
|
};
|
|
15330
|
-
const
|
|
16136
|
+
const titled = new Set(stepLines(corpus).map(parseStepBlock).filter((id) => id !== null));
|
|
15331
16137
|
for (const stepId of [...markers.map((m) => m.stepId), ...judgements.map((j) => j.step.id)]) {
|
|
15332
|
-
if (
|
|
15333
|
-
warnings.push(`step ${stepId}: the generated test no longer opens that step with the
|
|
15334
|
-
}
|
|
15335
|
-
if (stepEvidence) for (const m of markers) {
|
|
15336
|
-
const hasBefore = stepEvidenceCall(STEP_EVIDENCE_BEFORE, m).pattern.test(corpus);
|
|
15337
|
-
const hasAfter = stepEvidenceCall(STEP_EVIDENCE_AFTER, m).pattern.test(corpus);
|
|
15338
|
-
if (!hasBefore || !hasAfter) warnings.push(`step ${m.stepId}: generated test is missing its ${STEP_EVIDENCE_BEFORE}/${STEP_EVIDENCE_AFTER} call(s) — that step will have no report screenshots. A rewrite pass must not drop them.`);
|
|
16138
|
+
if (titled.has(stepId)) continue;
|
|
16139
|
+
warnings.push(`step ${stepId}: the generated test no longer opens that step with the \`test.step\` title the draft wrote. The evidence table, the run's screenshots and the spec review read it back to attribute assertions, so a reshaped one makes all three report the step as deciding nothing. Keep the title unchanged.`);
|
|
15339
16140
|
}
|
|
15340
16141
|
const firstHeaderLine = stamped.header.split("\n")[0]?.trim() ?? "";
|
|
15341
16142
|
if (firstHeaderLine && !corpus.includes(firstHeaderLine)) warnings.push("the generated test no longer opens with the configured header — a rewrite pass dropped it, so the file does not say which case it came from");
|
|
@@ -15347,6 +16148,29 @@ function injectedCallGaps(corpus, { markers, stepEvidence, judgements, header, t
|
|
|
15347
16148
|
return warnings;
|
|
15348
16149
|
}
|
|
15349
16150
|
/**
|
|
16151
|
+
* Anything of ccqa's the written test loads.
|
|
16152
|
+
*
|
|
16153
|
+
* A committed spec is the project's, and their CI type-checks it in a checkout
|
|
16154
|
+
* where ccqa is not installed — so an import of it is not a style question but
|
|
16155
|
+
* a broken build. The one exception is the judge: a claim is decided at run
|
|
16156
|
+
* time by a model, so a case that states one genuinely depends on it.
|
|
16157
|
+
*
|
|
16158
|
+
* Mechanical because nothing else catches it. The rewrite pass reads the
|
|
16159
|
+
* repository, and where an earlier ccqa already wrote a spec at this path it
|
|
16160
|
+
* imitates that spec's shape — which type-checks, runs, and passes every other
|
|
16161
|
+
* gate here.
|
|
16162
|
+
*/
|
|
16163
|
+
function runtimeImportGaps(corpus) {
|
|
16164
|
+
const loaded = /* @__PURE__ */ new Set();
|
|
16165
|
+
for (const specifier of importedSpecifiers(corpus)) {
|
|
16166
|
+
if (specifier !== "ccqa" && !specifier.startsWith("ccqa/")) continue;
|
|
16167
|
+
if (specifier === "ccqa/judge") continue;
|
|
16168
|
+
loaded.add(specifier);
|
|
16169
|
+
}
|
|
16170
|
+
if (loaded.size === 0) return [];
|
|
16171
|
+
return [`the generated test imports ${[...loaded].sort().join(", ")} — a spec ccqa writes is plain @playwright/test and must not load ccqa at run time, because the project type-checks and runs it where ccqa is not installed. Remove the import and every call into it, and write each of the case's steps as \`await test.step("<the draft's title>", async () => { ... })\` around that step's own actions. \`${JUDGE_MODULE}\` is the only ccqa import a spec may carry, and only a case with a \`judgeByLlm\` claim carries it.`];
|
|
16172
|
+
}
|
|
16173
|
+
/**
|
|
15350
16174
|
* Each claim paired with the action index it is asserted after: the last
|
|
15351
16175
|
* action of the nearest preceding step. A claim reads what the run has
|
|
15352
16176
|
* produced so far, so emitting it at the end of the test would judge a page
|
|
@@ -15464,16 +16288,13 @@ function validateRunnFile(file) {
|
|
|
15464
16288
|
* the config accepts, and any other is refused when the config loads.
|
|
15465
16289
|
*/
|
|
15466
16290
|
function createExternalTarget(id, config) {
|
|
15467
|
-
|
|
16291
|
+
const plugin = {
|
|
15468
16292
|
id,
|
|
15469
16293
|
input: "recording",
|
|
15470
|
-
generate: (ctx) => compileRecording(ctx, "playwright"),
|
|
16294
|
+
generate: (ctx) => compileRecording(ctx, "playwright", resolveStepEvidence(plugin, ctx.targetConfig)),
|
|
15471
16295
|
defaultTestPath: config.testPath ?? "",
|
|
15472
16296
|
runner: runCommandRunner,
|
|
15473
|
-
stepEvidence:
|
|
15474
|
-
supported: false,
|
|
15475
|
-
reason: `the "${id}" target has step evidence turned off in .ccqa/config.yaml`
|
|
15476
|
-
},
|
|
16297
|
+
stepEvidence: { supported: true },
|
|
15477
16298
|
judgeSteps: { supported: true },
|
|
15478
16299
|
browserCoverage: {
|
|
15479
16300
|
browser: "cdp",
|
|
@@ -15481,6 +16302,7 @@ function createExternalTarget(id, config) {
|
|
|
15481
16302
|
},
|
|
15482
16303
|
guidanceKind: "playwright"
|
|
15483
16304
|
};
|
|
16305
|
+
return plugin;
|
|
15484
16306
|
}
|
|
15485
16307
|
//#endregion
|
|
15486
16308
|
//#region src/targets/registry.ts
|
|
@@ -16992,11 +17814,11 @@ z.object({
|
|
|
16992
17814
|
/** Body of `DELETE /projects/:project/locks?profile=`. */
|
|
16993
17815
|
const ReleaseLocksRequestSchema = z.object({ holder: z.string().min(1) });
|
|
16994
17816
|
/**
|
|
16995
|
-
* A person's
|
|
16996
|
-
*
|
|
16997
|
-
* drift entry that parked the spec stays open, so the repair loop keeps
|
|
16998
|
-
* reason to fix the test, while the verdict stops asking a person for
|
|
16999
|
-
* person already
|
|
17817
|
+
* A person's judgement that an `ENVIRONMENT` failure's cause is gone, not a
|
|
17818
|
+
* hand re-test of the behaviour. It overrides the verdict, never the ledgers:
|
|
17819
|
+
* the drift entry that parked the spec stays open, so the repair loop keeps
|
|
17820
|
+
* its reason to fix the test, while the verdict stops asking a person for
|
|
17821
|
+
* what a person already answered.
|
|
17000
17822
|
*
|
|
17001
17823
|
* Anchored to the deploy head at the moment it was recorded, so it lapses on
|
|
17002
17824
|
* its own — a deploy reaching the spec, or the spec being edited, ends its
|
|
@@ -20166,7 +20988,7 @@ const pushCommand = new Command("push").description("Upload the report directory
|
|
|
20166
20988
|
meta("specs", `${run.specs.passed}/${run.specs.total} passed`);
|
|
20167
20989
|
info(`${resolveBaseUrl(opts)}/#/runs/${run.id}`);
|
|
20168
20990
|
}));
|
|
20169
|
-
const attestCommand = new Command("attest").argument("<feature/spec>", "Spec id, e.g. checkout/happy-path").description("Record
|
|
20991
|
+
const attestCommand = new Command("attest").argument("<feature/spec>", "Spec id, e.g. checkout/happy-path").description("Record a person's judgement that an ENVIRONMENT failure's cause is gone. The verdict answers manuallyVerified instead of asking a person for what a person already answered — the drift ledger is untouched, so the repair loop keeps its reason to fix the test. The attestation lapses on its own when a deploy reaches the spec or the spec is edited, and the spec rejoins the run cycle for the next run to settle.").requiredOption("--profile <name>", "Environment the attestation covers (e.g. 'stg'). It is anchored to that environment's current deploy head.").option("--by <name>", "Whose judgement this is. Required unless --revoke.").option("--note <text>", "What was broken, and what resolved it — the reader deciding whether to trust it sees this.").option("--revoke", "Withdraw the spec's attestation instead of recording one.").option(...hubUrlOption).option(...hubTokenOption).option("--project <name>", "Hub project. Defaults to the current directory's name.").option("--cwd <path>", "Directory the default --project name is resolved against.").action(withHubErrors(async (rawSpecId, opts) => {
|
|
20170
20992
|
const project = resolveProject(opts);
|
|
20171
20993
|
const hub = connect$2(opts);
|
|
20172
20994
|
const specId = requireSpecId(rawSpecId);
|
|
@@ -21049,7 +21871,7 @@ async function runLiveSpecs(cases, opts) {
|
|
|
21049
21871
|
meta("live-specs", cases.length);
|
|
21050
21872
|
const userPromptBundle = await loadPromptBundle(opts.hubContext ?? null, "live", cwd);
|
|
21051
21873
|
if (userPromptBundle !== null) meta("prompt", userPromptBundle.loaded.join(" + "));
|
|
21052
|
-
const conventions = await loadConventions(cwd, opts.conventions ?? []);
|
|
21874
|
+
const conventions = await loadConventions(cwd, { guides: opts.conventions ?? [] });
|
|
21053
21875
|
for (const w of conventions.warnings) warn(w);
|
|
21054
21876
|
const userPromptSuffix = [userPromptBundle?.text ?? "", ...conventions.sections.map((s) => `### ${s.path}\n\n${s.body}`)].filter((part) => part !== "").join("\n\n") || null;
|
|
21055
21877
|
const diffProvider = opts.diffProvider ?? null;
|
|
@@ -21576,10 +22398,7 @@ function externalGroupFor(plugin, targetConfig) {
|
|
|
21576
22398
|
runner: plugin.runner,
|
|
21577
22399
|
targetConfig,
|
|
21578
22400
|
defaultTestPath: plugin.defaultTestPath,
|
|
21579
|
-
stepEvidence: plugin
|
|
21580
|
-
supported: false,
|
|
21581
|
-
reason: `the "${plugin.id}" target does not capture step screenshots`
|
|
21582
|
-
},
|
|
22401
|
+
stepEvidence: resolveStepEvidence(plugin, targetConfig),
|
|
21583
22402
|
browserCoverage: plugin.browserCoverage
|
|
21584
22403
|
};
|
|
21585
22404
|
}
|
|
@@ -24865,10 +25684,7 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
|
|
|
24865
25684
|
if (!injected.ok) throw new RunUsageError(`could not restore ${opts.sessionState}: ${injected.error ?? "unknown error"} — the case expects to start signed in, so recording was not attempted`);
|
|
24866
25685
|
meta("session", opts.sessionState);
|
|
24867
25686
|
}
|
|
24868
|
-
const conventions =
|
|
24869
|
-
sections: [],
|
|
24870
|
-
warnings: []
|
|
24871
|
-
};
|
|
25687
|
+
const conventions = await loadConventions(opts.cwd ?? process.cwd(), { guides: opts.conventions ?? [] });
|
|
24872
25688
|
for (const w of conventions.warnings) warn(w);
|
|
24873
25689
|
const baseSystemPrompt = buildTraceSystemPrompt({
|
|
24874
25690
|
title: testCase.title,
|
|
@@ -30479,11 +31295,11 @@ function createGetAttestationsHandler(storage) {
|
|
|
30479
31295
|
};
|
|
30480
31296
|
}
|
|
30481
31297
|
/**
|
|
30482
|
-
* PUT /api/v1/projects/:project/attestations?profile= — record
|
|
30483
|
-
*
|
|
30484
|
-
* head: the anchor must be what the hub knows
|
|
30485
|
-
* not what the caller believes. Replaces any
|
|
30486
|
-
* spec.
|
|
31298
|
+
* PUT /api/v1/projects/:project/attestations?profile= — record a person's
|
|
31299
|
+
* judgement that a spec's environment failure is resolved. The hub stamps the
|
|
31300
|
+
* time and the profile's deploy head: the anchor must be what the hub knows
|
|
31301
|
+
* was deployed at this moment, not what the caller believes. Replaces any
|
|
31302
|
+
* previous attestation for the spec.
|
|
30487
31303
|
*/
|
|
30488
31304
|
function createPutAttestationHandler(storage) {
|
|
30489
31305
|
return async (ctx) => {
|