ccqa 1.55.0 → 1.56.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/ccqa.mjs +136 -29
- package/dist/hub-client/index.d.cts +2 -2
- package/dist/hub-client/index.d.mts +2 -2
- package/dist/package.json +1 -1
- package/package.json +1 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -16323,6 +16323,88 @@ function resolveTargetFrom(spec, config, registry) {
|
|
|
16323
16323
|
return plugin;
|
|
16324
16324
|
}
|
|
16325
16325
|
//#endregion
|
|
16326
|
+
//#region src/drift/renames.ts
|
|
16327
|
+
/**
|
|
16328
|
+
* What the audit says was renamed, and which of a case's files still say it.
|
|
16329
|
+
*
|
|
16330
|
+
* A rename is the one finding a machine can act on without reading prose, but
|
|
16331
|
+
* only once it is known *where* the old string survives: the case's document
|
|
16332
|
+
* decides whether an edit is needed, and the saved recording decides whether
|
|
16333
|
+
* recompiling that document is enough (ADR-0035). Both answers are string
|
|
16334
|
+
* comparisons against text ccqa already holds, which is what keeps them out of
|
|
16335
|
+
* the model's hands.
|
|
16336
|
+
*/
|
|
16337
|
+
/**
|
|
16338
|
+
* A string the test case still uses, and what the product renders in its place.
|
|
16339
|
+
*
|
|
16340
|
+
* Answered by the audit, never parsed back out of its prose: a quoted run is a
|
|
16341
|
+
* guess about one project's punctuation (ADR-0035).
|
|
16342
|
+
*/
|
|
16343
|
+
const RenameSchema = z.object({
|
|
16344
|
+
from: z.string(),
|
|
16345
|
+
to: z.string()
|
|
16346
|
+
});
|
|
16347
|
+
/**
|
|
16348
|
+
* The audit's renames, sanitized and checked against the document.
|
|
16349
|
+
*
|
|
16350
|
+
* A blank `from` matches everywhere and an identity pair edits nothing, so
|
|
16351
|
+
* neither is an instruction. Blank is tested after trimming and kept
|
|
16352
|
+
* untrimmed: a trimmed `from` is a string ccqa never checked the document for.
|
|
16353
|
+
* Two pairs naming the same `from` differently are the audit contradicting
|
|
16354
|
+
* itself, and both go — picking a winner would invent an answer nobody gave.
|
|
16355
|
+
*/
|
|
16356
|
+
function auditedRenames(document, renames) {
|
|
16357
|
+
const byFrom = /* @__PURE__ */ new Map();
|
|
16358
|
+
for (const r of renames) {
|
|
16359
|
+
if (r.from.trim() === "" || r.to.trim() === "" || r.from === r.to) continue;
|
|
16360
|
+
const kept = byFrom.get(r.from);
|
|
16361
|
+
if (kept === void 0) byFrom.set(r.from, r);
|
|
16362
|
+
else if (kept !== null && kept.to !== r.to) byFrom.set(r.from, null);
|
|
16363
|
+
}
|
|
16364
|
+
return [...byFrom.values()].filter((r) => r !== null).map((r) => ({
|
|
16365
|
+
...r,
|
|
16366
|
+
inDocument: document.includes(r.from)
|
|
16367
|
+
}));
|
|
16368
|
+
}
|
|
16369
|
+
/**
|
|
16370
|
+
* Whether the saved recording still names a string the audit says was renamed.
|
|
16371
|
+
*
|
|
16372
|
+
* It decides between the two machine repairs: a recording that names the old
|
|
16373
|
+
* string compiles it back in, so the case has to be recorded again rather than
|
|
16374
|
+
* regenerated. Missing one is not dangerous — the case routes to `regenerate`,
|
|
16375
|
+
* whose replay gate then refuses the dead recording — but it costs a browser
|
|
16376
|
+
* run to find that out.
|
|
16377
|
+
*
|
|
16378
|
+
* Walked value by value rather than over a serialization: `JSON.stringify`
|
|
16379
|
+
* escapes quotes and backslashes, so a `from` holding one would silently miss.
|
|
16380
|
+
*/
|
|
16381
|
+
function recordingNamesRenamed(recording, renames) {
|
|
16382
|
+
if (renames.length === 0) return false;
|
|
16383
|
+
return [...recording.actions, ...recording.cleanup ?? []].some((action) => actionStrings(action).some((text) => renames.some((r) => text.includes(r.from))));
|
|
16384
|
+
}
|
|
16385
|
+
/**
|
|
16386
|
+
* The strings a recorded action takes from the product: what it typed or
|
|
16387
|
+
* asserted, and how it addressed the element. What the recorder wrote about an
|
|
16388
|
+
* action — its observation, its step id, why it replayed badly — is ccqa's own
|
|
16389
|
+
* prose and would match on words the product never rendered.
|
|
16390
|
+
*/
|
|
16391
|
+
function actionStrings(action) {
|
|
16392
|
+
return [
|
|
16393
|
+
action.value,
|
|
16394
|
+
action.label,
|
|
16395
|
+
...locatorStrings(action.locator),
|
|
16396
|
+
...locatorStrings(action.target)
|
|
16397
|
+
].filter((s) => s !== void 0 && s !== "");
|
|
16398
|
+
}
|
|
16399
|
+
/**
|
|
16400
|
+
* A role locator is addressed by its accessible name; its `value` is the ARIA
|
|
16401
|
+
* role, which is that vocabulary rather than anything the product wrote.
|
|
16402
|
+
*/
|
|
16403
|
+
function locatorStrings(locator) {
|
|
16404
|
+
if (locator === void 0) return [];
|
|
16405
|
+
return locator.by === "role" ? [locator.name] : [locator.value];
|
|
16406
|
+
}
|
|
16407
|
+
//#endregion
|
|
16326
16408
|
//#region src/drift/types.ts
|
|
16327
16409
|
/** How everything downstream spells this case: its intent-source id, or `feature/spec`. */
|
|
16328
16410
|
function caseIdOf(target) {
|
|
@@ -16344,7 +16426,8 @@ const LocatorVerdictSchema = z.object({
|
|
|
16344
16426
|
/** The model's reply: a diagnosis, or `null` for "the spec still matches the code". */
|
|
16345
16427
|
const DriftReplySchema = z.object({
|
|
16346
16428
|
drift: DriftDiagnosisSchema.nullable(),
|
|
16347
|
-
locators: z.array(LocatorVerdictSchema).default([])
|
|
16429
|
+
locators: z.array(LocatorVerdictSchema).default([]),
|
|
16430
|
+
renames: z.array(RenameSchema.nullable().catch(null)).transform((rs) => rs.filter((r) => r !== null)).default([]).catch([])
|
|
16348
16431
|
});
|
|
16349
16432
|
/**
|
|
16350
16433
|
* How a label reads against `--exit-on`. The threshold asks "would a
|
|
@@ -27338,7 +27421,7 @@ These are the same definitions failure analysis uses on a case that actually fai
|
|
|
27338
27421
|
|
|
27339
27422
|
## What separates TEST_DRIFT from SPEC_CHANGE
|
|
27340
27423
|
|
|
27341
|
-
This is the distinction that matters, because the two lead to different actions: TEST_DRIFT gets the test
|
|
27424
|
+
This is the distinction that matters, because the two lead to different actions: TEST_DRIFT gets the stale string replaced wherever it is written and the test recompiled, SPEC_CHANGE gets a human to decide what the case should now say.
|
|
27342
27425
|
|
|
27343
27426
|
Ask whether the **intent** the step describes still exists in the product:
|
|
27344
27427
|
|
|
@@ -27433,12 +27516,17 @@ Drift found:
|
|
|
27433
27516
|
"evidence": [
|
|
27434
27517
|
{ "file": "<path:line>", "detail": "<what this proves>" }
|
|
27435
27518
|
]
|
|
27436
|
-
}
|
|
27519
|
+
},
|
|
27520
|
+
"renames": [
|
|
27521
|
+
{ "from": "<the string as the test case writes it>", "to": "<what the source renders in its place>" }
|
|
27522
|
+
]
|
|
27437
27523
|
}
|
|
27438
27524
|
\`\`\`
|
|
27439
27525
|
|
|
27440
27526
|
\`subDiagnosis\`: \`SELECTOR_DRIFT\` when a selector or string was renamed, \`OVER_ASSERTION\` when the case asserts something narrower than the product ever promised, \`NONE\` otherwise.
|
|
27441
27527
|
|
|
27528
|
+
\`renames\`: one entry per renamed string, when the label is \`TEST_DRIFT\` and \`subDiagnosis\` is \`SELECTOR_DRIFT\` — a finding that says a string was renamed and does not name the pair leaves the repair to a person. Omit the key otherwise. It sits beside \`drift\`, and beside \`locators\` where that was asked for. \`from\` is the string as the test case writes it — in the generated code, in the case's own document, or both — and \`to\` is what the source you cited renders in its place. Give each verbatim, with no quotes, ellipses or paraphrase of your own: each pair is looked up character for character in the file it claims to be about, and one that does not match is dropped. Naming a rename neither settles the label nor stands in for the citation the finding still needs.
|
|
27529
|
+
|
|
27442
27530
|
\`specChangeKind\`: set it only when the label is \`SPEC_CHANGE\`, and omit the field entirely otherwise. It says which repair the case needs — deleting it, or rewriting and re-recording it:
|
|
27443
27531
|
|
|
27444
27532
|
- \`FEATURE_REMOVED\` — the code no longer implements the behaviour at all: removed, moved elsewhere, or deliberately disabled. This is the stronger claim, so earn it: your evidence must point at where the implementation would be if it still existed.
|
|
@@ -28048,7 +28136,8 @@ async function checkSpec(target, opts) {
|
|
|
28048
28136
|
drift,
|
|
28049
28137
|
live: artifacts.live,
|
|
28050
28138
|
title: artifacts.title,
|
|
28051
|
-
documentPath: artifacts.intent.path
|
|
28139
|
+
documentPath: artifacts.intent.path,
|
|
28140
|
+
renames: auditedRenames(artifacts.intent.body, reply.renames)
|
|
28052
28141
|
};
|
|
28053
28142
|
} catch (e) {
|
|
28054
28143
|
lastError = `failed to parse drift reply: ${e.message}`;
|
|
@@ -28187,7 +28276,7 @@ function determineExitCode(results, threshold) {
|
|
|
28187
28276
|
//#endregion
|
|
28188
28277
|
//#region src/drift/to-report.ts
|
|
28189
28278
|
/** Tracks the drift prompt's own version — the two must never drift apart. */
|
|
28190
|
-
const DRIFT_REPORT_PROMPT_VERSION = "
|
|
28279
|
+
const DRIFT_REPORT_PROMPT_VERSION = "12";
|
|
28191
28280
|
/**
|
|
28192
28281
|
* Spec-level status under the given threshold, mirroring determineExitCode's
|
|
28193
28282
|
* per-spec logic (exit-code.ts) but scoped to a single SpecResult.
|
|
@@ -28268,6 +28357,10 @@ async function buildBrief(result, cwd, ctx) {
|
|
|
28268
28357
|
const drift = result.drift;
|
|
28269
28358
|
const id = caseIdOf(result.target);
|
|
28270
28359
|
const test = result.live ? null : await caseTestPath(result.target, ctx);
|
|
28360
|
+
const rewrite = drift.label === "TEST_DRIFT" ? (result.renames ?? []).filter((r) => r.inDocument).map(({ from, to }) => ({
|
|
28361
|
+
from,
|
|
28362
|
+
to
|
|
28363
|
+
})) : [];
|
|
28271
28364
|
return {
|
|
28272
28365
|
case: id,
|
|
28273
28366
|
kind: drift.label,
|
|
@@ -28278,39 +28371,53 @@ async function buildBrief(result, cwd, ctx) {
|
|
|
28278
28371
|
reasoning: drift.reasoning,
|
|
28279
28372
|
evidence: drift.evidence,
|
|
28280
28373
|
test,
|
|
28281
|
-
|
|
28374
|
+
document: documentOf(result, cwd),
|
|
28375
|
+
repair: await buildRepair(result, test, rewrite, cwd, ctx)
|
|
28282
28376
|
};
|
|
28283
28377
|
}
|
|
28284
|
-
|
|
28378
|
+
/**
|
|
28379
|
+
* The case's document as something that can be opened from anywhere. A case
|
|
28380
|
+
* filed outside the project relativizes to `../…`, which resolves against
|
|
28381
|
+
* whatever directory the fix job happens to be in rather than this one.
|
|
28382
|
+
*/
|
|
28383
|
+
function documentOf(result, cwd) {
|
|
28384
|
+
const rel = result.documentPath;
|
|
28385
|
+
if (rel === void 0) return null;
|
|
28386
|
+
return rel.startsWith("..") ? resolve(cwd, rel) : rel;
|
|
28387
|
+
}
|
|
28388
|
+
async function buildRepair(result, test, rewrite, cwd, ctx) {
|
|
28285
28389
|
const drift = result.drift;
|
|
28286
|
-
|
|
28390
|
+
const renames = result.renames ?? [];
|
|
28391
|
+
const external = (reason) => ({
|
|
28287
28392
|
route: "external",
|
|
28288
|
-
reason
|
|
28289
|
-
|
|
28290
|
-
|
|
28291
|
-
|
|
28292
|
-
|
|
28293
|
-
|
|
28294
|
-
|
|
28295
|
-
|
|
28296
|
-
reason: "the finding is not on the generated test — the document, or a file the test imports and a regeneration only reads. Either way, regenerating rewrites neither"
|
|
28393
|
+
reason,
|
|
28394
|
+
rewrite
|
|
28395
|
+
});
|
|
28396
|
+
if (drift.label !== "TEST_DRIFT") return external(`${drift.label} names no repair a machine can make: recompiling or re-recording reproduces the case as it stands`);
|
|
28397
|
+
if (result.live) return rewrite.length === 0 ? external("a live case is repaired by rewriting its document, and nothing the audit named is in it") : {
|
|
28398
|
+
route: "rewrite",
|
|
28399
|
+
reason: "a live case runs its document, so rewriting it is the whole repair — apply 'rewrite' to 'document', then run the case to verify.",
|
|
28400
|
+
rewrite
|
|
28297
28401
|
};
|
|
28402
|
+
if (test === null) return external("this case resolves to no generated test: there is nothing to regenerate");
|
|
28298
28403
|
const testAbs = resolve(cwd, test);
|
|
28299
|
-
const
|
|
28404
|
+
const recording = await getRecording(caseRefFor(result.target.caseId ?? {
|
|
28300
28405
|
featureName: result.target.featureName,
|
|
28301
28406
|
specName: result.target.specName
|
|
28302
|
-
}, cwd, await caseRecordingPath(result.target, ctx) ?? void 0)).catch(() => null)
|
|
28303
|
-
if (!
|
|
28304
|
-
|
|
28305
|
-
|
|
28306
|
-
|
|
28307
|
-
|
|
28308
|
-
|
|
28309
|
-
|
|
28310
|
-
};
|
|
28407
|
+
}, cwd, await caseRecordingPath(result.target, ctx) ?? void 0)).catch(() => null);
|
|
28408
|
+
if (!recording?.generated) return external("no generation stamp: ccqa did not write this test, or it predates the stamp");
|
|
28409
|
+
if (!await matchesGenerationStamp(recording.generated, testAbs)) return external("the test has been edited since it was generated");
|
|
28410
|
+
const apply = (why) => rewrite.length > 0 ? ` Apply 'rewrite' to 'document' first: ${why}.` : "";
|
|
28411
|
+
if (recordingNamesRenamed(recording, renames)) return {
|
|
28412
|
+
route: "rerecord",
|
|
28413
|
+
reason: `the saved recording names a renamed string, so a regeneration would compile it back in — re-record and verify.${apply("a re-recording drives the case as its document states it")}`,
|
|
28414
|
+
rewrite
|
|
28415
|
+
};
|
|
28416
|
+
if (drift.surface !== "generated" && rewrite.length === 0) return external("the finding is not on the generated test, and nothing it named is in the document — it is on a file the test imports, which a regeneration only reads, or no replacement was named");
|
|
28311
28417
|
return {
|
|
28312
28418
|
route: "regenerate",
|
|
28313
|
-
reason: `test drift
|
|
28419
|
+
reason: `test drift, and the saved recording names no renamed string — regenerate and verify.${apply("regenerating alone recompiles the same document")}`,
|
|
28420
|
+
rewrite
|
|
28314
28421
|
};
|
|
28315
28422
|
}
|
|
28316
28423
|
//#endregion
|
|
@@ -28530,7 +28637,7 @@ function selectSpecsNeedingAudit(targets, report, stillDrifted = /* @__PURE__ */
|
|
|
28530
28637
|
//#endregion
|
|
28531
28638
|
//#region src/cli/audit.ts
|
|
28532
28639
|
const DEFAULT_CONCURRENCY = 3;
|
|
28533
|
-
const auditCommand = addProfileOption(addLanguageOption(new Command("audit").argument("[feature/spec]", "Optional spec id. If omitted, every spec under .ccqa/features/ is checked.").description("Read each spec against the code it describes and report where the two have drifted. Static: no browser is run, so this is the cheap check to put in front of `ccqa run`.").optionsGroup("Which specs to audit:").option("--only-affected-by <ref>", "Only specs `ccqa select-specs` decides the diff against <ref> reaches (e.g. origin/main), by intersecting it with each spec's measured coverage from the hub. In pull_request CI, pass $GITHUB_BASE_REF. Specs without a measurement are audited rather than skipped.").option("--only-hub-audit-needed", "Only specs the hub says a deploy has landed on since the audit last read them. A spec that was never audited is always included, one the hub cannot answer for is audited rather than skipped, and one whose drift entry is still open is always re-audited — a merged fix changes only the spec tree, which no deploy answer covers. No git diff involved. Requires a hub connection and --hub-profile.").optionsGroup("How to run it:").option("--concurrency <n>", `Parallel spec checks (default: ${DEFAULT_CONCURRENCY})`).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").optionsGroup("What to do with the results:").option("--report-format <fmt>", "Output format: text | json | github", "text").option("--report-to-hub", "Push the result to a ccqa hub as a run (kind: drift), which is what updates the drift ledger. A spec it finds drifted answers `needsRepair` to `ccqa run --only-hub-rerun-needed`, and is not run until a person repairs it.").option("--exit-on <level>", "Exit non-zero on this severity or higher: warn | error", "error").option("--brief <dir>", "Also write one JSON file per finding under <dir>, named by case id: the verdict, its citations,
|
|
28640
|
+
const auditCommand = addProfileOption(addLanguageOption(new Command("audit").argument("[feature/spec]", "Optional spec id. If omitted, every spec under .ccqa/features/ is checked.").description("Read each spec against the code it describes and report where the two have drifted. Static: no browser is run, so this is the cheap check to put in front of `ccqa run`.").optionsGroup("Which specs to audit:").option("--only-affected-by <ref>", "Only specs `ccqa select-specs` decides the diff against <ref> reaches (e.g. origin/main), by intersecting it with each spec's measured coverage from the hub. In pull_request CI, pass $GITHUB_BASE_REF. Specs without a measurement are audited rather than skipped.").option("--only-hub-audit-needed", "Only specs the hub says a deploy has landed on since the audit last read them. A spec that was never audited is always included, one the hub cannot answer for is audited rather than skipped, and one whose drift entry is still open is always re-audited — a merged fix changes only the spec tree, which no deploy answer covers. No git diff involved. Requires a hub connection and --hub-profile.").optionsGroup("How to run it:").option("--concurrency <n>", `Parallel spec checks (default: ${DEFAULT_CONCURRENCY})`).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").optionsGroup("What to do with the results:").option("--report-format <fmt>", "Output format: text | json | github", "text").option("--report-to-hub", "Push the result to a ccqa hub as a run (kind: drift), which is what updates the drift ledger. A spec it finds drifted answers `needsRepair` to `ccqa run --only-hub-rerun-needed`, and is not run until a person repairs it.").option("--exit-on <level>", "Exit non-zero on this severity or higher: warn | error", "error").option("--brief <dir>", "Also write one JSON file per finding under <dir>, named by case id: the verdict, its citations, which repair the case needs (regenerate, re-record, rewrite its document, or a person), and any renamed strings to apply to that document first. What reads them is outside ccqa.").option("--dump-inputs <dir>", "Also write what each case's audit was given — its document, every file it was handed and the import that reached it, the source roots, and the prompt — as one markdown file per case under <dir>. For telling 'this never reached the audit' apart from 'it did, and nothing was reported'.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory used as both the .ccqa root and the codebase Claude reads. Useful for monorepos. Defaults to process.cwd().").option("--project <name>", "Logical project name for the pushed run. Defaults to the current directory's name.").option(...hubUrlOption).option(...hubTokenOption).option(...hubHeaderOption))).action(withUsageErrors(async (specPath, opts) => {
|
|
28534
28641
|
await withCostReporting("audit", () => runAudit(specPath, opts));
|
|
28535
28642
|
}));
|
|
28536
28643
|
async function runAudit(specPath, opts) {
|
|
@@ -652,8 +652,8 @@ declare const ReportSpecResultSchema: z.ZodObject<{
|
|
|
652
652
|
title: z.ZodNullable<z.ZodString>;
|
|
653
653
|
target: z.ZodOptional<z.ZodString>;
|
|
654
654
|
mode: z.ZodOptional<z.ZodEnum<{
|
|
655
|
-
live: "live";
|
|
656
655
|
deterministic: "deterministic";
|
|
656
|
+
live: "live";
|
|
657
657
|
}>>;
|
|
658
658
|
status: z.ZodEnum<{
|
|
659
659
|
passed: "passed";
|
|
@@ -874,8 +874,8 @@ declare const RunReportDataSchema: z.ZodObject<{
|
|
|
874
874
|
title: z.ZodNullable<z.ZodString>;
|
|
875
875
|
target: z.ZodOptional<z.ZodString>;
|
|
876
876
|
mode: z.ZodOptional<z.ZodEnum<{
|
|
877
|
-
live: "live";
|
|
878
877
|
deterministic: "deterministic";
|
|
878
|
+
live: "live";
|
|
879
879
|
}>>;
|
|
880
880
|
status: z.ZodEnum<{
|
|
881
881
|
passed: "passed";
|
|
@@ -652,8 +652,8 @@ declare const ReportSpecResultSchema: z.ZodObject<{
|
|
|
652
652
|
title: z.ZodNullable<z.ZodString>;
|
|
653
653
|
target: z.ZodOptional<z.ZodString>;
|
|
654
654
|
mode: z.ZodOptional<z.ZodEnum<{
|
|
655
|
-
live: "live";
|
|
656
655
|
deterministic: "deterministic";
|
|
656
|
+
live: "live";
|
|
657
657
|
}>>;
|
|
658
658
|
status: z.ZodEnum<{
|
|
659
659
|
passed: "passed";
|
|
@@ -874,8 +874,8 @@ declare const RunReportDataSchema: z.ZodObject<{
|
|
|
874
874
|
title: z.ZodNullable<z.ZodString>;
|
|
875
875
|
target: z.ZodOptional<z.ZodString>;
|
|
876
876
|
mode: z.ZodOptional<z.ZodEnum<{
|
|
877
|
-
live: "live";
|
|
878
877
|
deterministic: "deterministic";
|
|
878
|
+
live: "live";
|
|
879
879
|
}>>;
|
|
880
880
|
status: z.ZodEnum<{
|
|
881
881
|
passed: "passed";
|
package/dist/package.json
CHANGED