ccqa 1.55.0 → 1.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/ccqa.mjs +228 -88
- package/dist/hub-client/index.d.cts +2 -2
- package/dist/hub-client/index.d.mts +2 -2
- package/dist/package.json +1 -1
- package/package.json +1 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -16323,6 +16323,88 @@ function resolveTargetFrom(spec, config, registry) {
|
|
|
16323
16323
|
return plugin;
|
|
16324
16324
|
}
|
|
16325
16325
|
//#endregion
|
|
16326
|
+
//#region src/drift/renames.ts
|
|
16327
|
+
/**
|
|
16328
|
+
* What the audit says was renamed, and which of a case's files still say it.
|
|
16329
|
+
*
|
|
16330
|
+
* A rename is the one finding a machine can act on without reading prose, but
|
|
16331
|
+
* only once it is known *where* the old string survives: the case's document
|
|
16332
|
+
* decides whether an edit is needed, and the saved recording decides whether
|
|
16333
|
+
* recompiling that document is enough (ADR-0035). Both answers are string
|
|
16334
|
+
* comparisons against text ccqa already holds, which is what keeps them out of
|
|
16335
|
+
* the model's hands.
|
|
16336
|
+
*/
|
|
16337
|
+
/**
|
|
16338
|
+
* A string the test case still uses, and what the product renders in its place.
|
|
16339
|
+
*
|
|
16340
|
+
* Answered by the audit, never parsed back out of its prose: a quoted run is a
|
|
16341
|
+
* guess about one project's punctuation (ADR-0035).
|
|
16342
|
+
*/
|
|
16343
|
+
const RenameSchema = z.object({
|
|
16344
|
+
from: z.string(),
|
|
16345
|
+
to: z.string()
|
|
16346
|
+
});
|
|
16347
|
+
/**
|
|
16348
|
+
* The audit's renames, sanitized and checked against the document.
|
|
16349
|
+
*
|
|
16350
|
+
* A blank `from` matches everywhere and an identity pair edits nothing, so
|
|
16351
|
+
* neither is an instruction. Blank is tested after trimming and kept
|
|
16352
|
+
* untrimmed: a trimmed `from` is a string ccqa never checked the document for.
|
|
16353
|
+
* Two pairs naming the same `from` differently are the audit contradicting
|
|
16354
|
+
* itself, and both go — picking a winner would invent an answer nobody gave.
|
|
16355
|
+
*/
|
|
16356
|
+
function auditedRenames(document, renames) {
|
|
16357
|
+
const byFrom = /* @__PURE__ */ new Map();
|
|
16358
|
+
for (const r of renames) {
|
|
16359
|
+
if (r.from.trim() === "" || r.to.trim() === "" || r.from === r.to) continue;
|
|
16360
|
+
const kept = byFrom.get(r.from);
|
|
16361
|
+
if (kept === void 0) byFrom.set(r.from, r);
|
|
16362
|
+
else if (kept !== null && kept.to !== r.to) byFrom.set(r.from, null);
|
|
16363
|
+
}
|
|
16364
|
+
return [...byFrom.values()].filter((r) => r !== null).map((r) => ({
|
|
16365
|
+
...r,
|
|
16366
|
+
inDocument: document.includes(r.from)
|
|
16367
|
+
}));
|
|
16368
|
+
}
|
|
16369
|
+
/**
|
|
16370
|
+
* Whether the saved recording still names a string the audit says was renamed.
|
|
16371
|
+
*
|
|
16372
|
+
* It decides between the two machine repairs: a recording that names the old
|
|
16373
|
+
* string compiles it back in, so the case has to be recorded again rather than
|
|
16374
|
+
* regenerated. Missing one is not dangerous — the case routes to `regenerate`,
|
|
16375
|
+
* whose replay gate then refuses the dead recording — but it costs a browser
|
|
16376
|
+
* run to find that out.
|
|
16377
|
+
*
|
|
16378
|
+
* Walked value by value rather than over a serialization: `JSON.stringify`
|
|
16379
|
+
* escapes quotes and backslashes, so a `from` holding one would silently miss.
|
|
16380
|
+
*/
|
|
16381
|
+
function recordingNamesRenamed(recording, renames) {
|
|
16382
|
+
if (renames.length === 0) return false;
|
|
16383
|
+
return [...recording.actions, ...recording.cleanup ?? []].some((action) => actionStrings(action).some((text) => renames.some((r) => text.includes(r.from))));
|
|
16384
|
+
}
|
|
16385
|
+
/**
|
|
16386
|
+
* The strings a recorded action takes from the product: what it typed or
|
|
16387
|
+
* asserted, and how it addressed the element. What the recorder wrote about an
|
|
16388
|
+
* action — its observation, its step id, why it replayed badly — is ccqa's own
|
|
16389
|
+
* prose and would match on words the product never rendered.
|
|
16390
|
+
*/
|
|
16391
|
+
function actionStrings(action) {
|
|
16392
|
+
return [
|
|
16393
|
+
action.value,
|
|
16394
|
+
action.label,
|
|
16395
|
+
...locatorStrings(action.locator),
|
|
16396
|
+
...locatorStrings(action.target)
|
|
16397
|
+
].filter((s) => s !== void 0 && s !== "");
|
|
16398
|
+
}
|
|
16399
|
+
/**
|
|
16400
|
+
* A role locator is addressed by its accessible name; its `value` is the ARIA
|
|
16401
|
+
* role, which is that vocabulary rather than anything the product wrote.
|
|
16402
|
+
*/
|
|
16403
|
+
function locatorStrings(locator) {
|
|
16404
|
+
if (locator === void 0) return [];
|
|
16405
|
+
return locator.by === "role" ? [locator.name] : [locator.value];
|
|
16406
|
+
}
|
|
16407
|
+
//#endregion
|
|
16326
16408
|
//#region src/drift/types.ts
|
|
16327
16409
|
/** How everything downstream spells this case: its intent-source id, or `feature/spec`. */
|
|
16328
16410
|
function caseIdOf(target) {
|
|
@@ -16344,7 +16426,8 @@ const LocatorVerdictSchema = z.object({
|
|
|
16344
16426
|
/** The model's reply: a diagnosis, or `null` for "the spec still matches the code". */
|
|
16345
16427
|
const DriftReplySchema = z.object({
|
|
16346
16428
|
drift: DriftDiagnosisSchema.nullable(),
|
|
16347
|
-
locators: z.array(LocatorVerdictSchema).default([])
|
|
16429
|
+
locators: z.array(LocatorVerdictSchema).default([]),
|
|
16430
|
+
renames: z.array(RenameSchema.nullable().catch(null)).transform((rs) => rs.filter((r) => r !== null)).default([]).catch([])
|
|
16348
16431
|
});
|
|
16349
16432
|
/**
|
|
16350
16433
|
* How a label reads against `--exit-on`. The threshold asks "would a
|
|
@@ -27338,7 +27421,7 @@ These are the same definitions failure analysis uses on a case that actually fai
|
|
|
27338
27421
|
|
|
27339
27422
|
## What separates TEST_DRIFT from SPEC_CHANGE
|
|
27340
27423
|
|
|
27341
|
-
This is the distinction that matters, because the two lead to different actions: TEST_DRIFT gets the test
|
|
27424
|
+
This is the distinction that matters, because the two lead to different actions: TEST_DRIFT gets the stale string replaced wherever it is written and the test recompiled, SPEC_CHANGE gets a human to decide what the case should now say.
|
|
27342
27425
|
|
|
27343
27426
|
Ask whether the **intent** the step describes still exists in the product:
|
|
27344
27427
|
|
|
@@ -27433,12 +27516,17 @@ Drift found:
|
|
|
27433
27516
|
"evidence": [
|
|
27434
27517
|
{ "file": "<path:line>", "detail": "<what this proves>" }
|
|
27435
27518
|
]
|
|
27436
|
-
}
|
|
27519
|
+
},
|
|
27520
|
+
"renames": [
|
|
27521
|
+
{ "from": "<the string as the test case writes it>", "to": "<what the source renders in its place>" }
|
|
27522
|
+
]
|
|
27437
27523
|
}
|
|
27438
27524
|
\`\`\`
|
|
27439
27525
|
|
|
27440
27526
|
\`subDiagnosis\`: \`SELECTOR_DRIFT\` when a selector or string was renamed, \`OVER_ASSERTION\` when the case asserts something narrower than the product ever promised, \`NONE\` otherwise.
|
|
27441
27527
|
|
|
27528
|
+
\`renames\`: one entry per renamed string, when the label is \`TEST_DRIFT\` and \`subDiagnosis\` is \`SELECTOR_DRIFT\` — a finding that says a string was renamed and does not name the pair leaves the repair to a person. Omit the key otherwise. It sits beside \`drift\`, and beside \`locators\` where that was asked for. \`from\` is the string as the test case writes it — in the generated code, in the case's own document, or both — and \`to\` is what the source you cited renders in its place. Give each verbatim, with no quotes, ellipses or paraphrase of your own: each pair is looked up character for character in the file it claims to be about, and one that does not match is dropped. Naming a rename neither settles the label nor stands in for the citation the finding still needs.
|
|
27529
|
+
|
|
27442
27530
|
\`specChangeKind\`: set it only when the label is \`SPEC_CHANGE\`, and omit the field entirely otherwise. It says which repair the case needs — deleting it, or rewriting and re-recording it:
|
|
27443
27531
|
|
|
27444
27532
|
- \`FEATURE_REMOVED\` — the code no longer implements the behaviour at all: removed, moved elsewhere, or deliberately disabled. This is the stronger claim, so earn it: your evidence must point at where the implementation would be if it still existed.
|
|
@@ -27996,7 +28084,7 @@ async function checkSpec(target, opts) {
|
|
|
27996
28084
|
locators,
|
|
27997
28085
|
systemPrompt,
|
|
27998
28086
|
userPrompt
|
|
27999
|
-
}).
|
|
28087
|
+
}).catch((err) => warn(`${name}: could not write the audit inputs (${err.message})`));
|
|
28000
28088
|
const MAX_ATTEMPTS = 2;
|
|
28001
28089
|
let lastError = "";
|
|
28002
28090
|
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
|
@@ -28048,7 +28136,8 @@ async function checkSpec(target, opts) {
|
|
|
28048
28136
|
drift,
|
|
28049
28137
|
live: artifacts.live,
|
|
28050
28138
|
title: artifacts.title,
|
|
28051
|
-
documentPath: artifacts.intent.path
|
|
28139
|
+
documentPath: artifacts.intent.path,
|
|
28140
|
+
renames: auditedRenames(artifacts.intent.body, reply.renames)
|
|
28052
28141
|
};
|
|
28053
28142
|
} catch (e) {
|
|
28054
28143
|
lastError = `failed to parse drift reply: ${e.message}`;
|
|
@@ -28066,12 +28155,12 @@ async function checkSpec(target, opts) {
|
|
|
28066
28155
|
//#endregion
|
|
28067
28156
|
//#region src/drift/format.ts
|
|
28068
28157
|
/**
|
|
28069
|
-
* Render drift results
|
|
28070
|
-
*
|
|
28071
|
-
*
|
|
28158
|
+
* Render drift results for a terminal. `json` is not one of these: it prints
|
|
28159
|
+
* the same payload the report file holds, which is assembled asynchronously
|
|
28160
|
+
* and so is built once by the caller rather than rendered from results here
|
|
28161
|
+
* (`buildAuditReport`).
|
|
28072
28162
|
*/
|
|
28073
28163
|
function renderDrift(results, format, cwd) {
|
|
28074
|
-
if (format === "json") return renderJson(results);
|
|
28075
28164
|
if (format === "github") return renderGithub(results, cwd);
|
|
28076
28165
|
return renderText$1(results);
|
|
28077
28166
|
}
|
|
@@ -28106,16 +28195,6 @@ function renderText$1(results) {
|
|
|
28106
28195
|
out.push("");
|
|
28107
28196
|
return out.join("\n");
|
|
28108
28197
|
}
|
|
28109
|
-
function renderJson(results) {
|
|
28110
|
-
const payload = { specs: results.map((r) => ({
|
|
28111
|
-
feature: r.target.featureName,
|
|
28112
|
-
spec: r.target.specName,
|
|
28113
|
-
ok: r.ok,
|
|
28114
|
-
...r.error ? { error: r.error } : {},
|
|
28115
|
-
drift: r.drift
|
|
28116
|
-
})) };
|
|
28117
|
-
return `${JSON.stringify(payload, null, 2)}\n`;
|
|
28118
|
-
}
|
|
28119
28198
|
function renderGithub(results, cwd) {
|
|
28120
28199
|
const repoRoot = process.env["GITHUB_WORKSPACE"] ?? process.cwd();
|
|
28121
28200
|
const lines = [];
|
|
@@ -28187,7 +28266,7 @@ function determineExitCode(results, threshold) {
|
|
|
28187
28266
|
//#endregion
|
|
28188
28267
|
//#region src/drift/to-report.ts
|
|
28189
28268
|
/** Tracks the drift prompt's own version — the two must never drift apart. */
|
|
28190
|
-
const DRIFT_REPORT_PROMPT_VERSION = "
|
|
28269
|
+
const DRIFT_REPORT_PROMPT_VERSION = "12";
|
|
28191
28270
|
/**
|
|
28192
28271
|
* Spec-level status under the given threshold, mirroring determineExitCode's
|
|
28193
28272
|
* per-spec logic (exit-code.ts) but scoped to a single SpecResult.
|
|
@@ -28246,71 +28325,97 @@ function driftResultsToReport(results, meta) {
|
|
|
28246
28325
|
};
|
|
28247
28326
|
}
|
|
28248
28327
|
//#endregion
|
|
28249
|
-
//#region src/drift/
|
|
28328
|
+
//#region src/drift/audit-report.ts
|
|
28250
28329
|
/**
|
|
28251
|
-
*
|
|
28252
|
-
*
|
|
28330
|
+
* What `ccqa audit` leaves behind for a machine.
|
|
28331
|
+
*
|
|
28332
|
+
* The audit has three audiences: the person reading the terminal, the hub's
|
|
28333
|
+
* ledger, and whatever repairs the test. This is the third, and it is one
|
|
28334
|
+
* payload — written to `<report-dir>/audit.json` by every completed sweep and
|
|
28335
|
+
* printed verbatim by `--report-format json` (ADR-0036).
|
|
28253
28336
|
*/
|
|
28254
|
-
|
|
28255
|
-
|
|
28256
|
-
|
|
28257
|
-
|
|
28258
|
-
|
|
28259
|
-
|
|
28260
|
-
|
|
28261
|
-
|
|
28262
|
-
|
|
28263
|
-
|
|
28264
|
-
|
|
28265
|
-
})
|
|
28337
|
+
/** The file the report is written as, inside the report directory. */
|
|
28338
|
+
const AUDIT_REPORT_FILE = "audit.json";
|
|
28339
|
+
/**
|
|
28340
|
+
* The payload, for both the file and `--report-format json`. Each row's reads
|
|
28341
|
+
* are independent, so they resolve together rather than one case at a time.
|
|
28342
|
+
*/
|
|
28343
|
+
async function buildAuditReport(results, cwd, ctx) {
|
|
28344
|
+
return { specs: await Promise.all(results.map((r) => buildRow(r, cwd, ctx))) };
|
|
28345
|
+
}
|
|
28346
|
+
/** Write the report as `<dirAbs>/audit.json`, creating the directory. Returns its path. */
|
|
28347
|
+
async function writeAuditReport(report, dirAbs) {
|
|
28348
|
+
await mkdir(dirAbs, { recursive: true });
|
|
28349
|
+
const path = join(dirAbs, AUDIT_REPORT_FILE);
|
|
28350
|
+
await writeFile(path, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
|
28351
|
+
return path;
|
|
28266
28352
|
}
|
|
28267
|
-
async function
|
|
28353
|
+
async function buildRow(result, cwd, ctx) {
|
|
28268
28354
|
const drift = result.drift;
|
|
28269
|
-
const
|
|
28355
|
+
const row = {
|
|
28356
|
+
feature: result.target.featureName,
|
|
28357
|
+
spec: result.target.specName,
|
|
28358
|
+
case: caseIdOf(result.target),
|
|
28359
|
+
ok: result.ok,
|
|
28360
|
+
...result.error ? { error: result.error } : {},
|
|
28361
|
+
drift
|
|
28362
|
+
};
|
|
28363
|
+
if (drift === null) return row;
|
|
28270
28364
|
const test = result.live ? null : await caseTestPath(result.target, ctx);
|
|
28365
|
+
const rewrite = drift.label === "TEST_DRIFT" ? (result.renames ?? []).filter((r) => r.inDocument).map(({ from, to }) => ({
|
|
28366
|
+
from,
|
|
28367
|
+
to
|
|
28368
|
+
})) : [];
|
|
28271
28369
|
return {
|
|
28272
|
-
|
|
28273
|
-
kind: drift.label,
|
|
28274
|
-
surface: drift.surface,
|
|
28275
|
-
confidence: drift.confidence,
|
|
28276
|
-
headline: drift.headline,
|
|
28277
|
-
recommendation: drift.recommendation,
|
|
28278
|
-
reasoning: drift.reasoning,
|
|
28279
|
-
evidence: drift.evidence,
|
|
28370
|
+
...row,
|
|
28280
28371
|
test,
|
|
28281
|
-
|
|
28372
|
+
document: documentOf(result, cwd),
|
|
28373
|
+
repair: await buildRepair(result, test, rewrite, cwd, ctx)
|
|
28282
28374
|
};
|
|
28283
28375
|
}
|
|
28284
|
-
|
|
28376
|
+
/**
|
|
28377
|
+
* The case's document as something that can be opened from anywhere. A case
|
|
28378
|
+
* filed outside the project relativizes to `../…`, which resolves against
|
|
28379
|
+
* whatever directory the fix job happens to be in rather than this one.
|
|
28380
|
+
*/
|
|
28381
|
+
function documentOf(result, cwd) {
|
|
28382
|
+
const rel = result.documentPath;
|
|
28383
|
+
if (rel === void 0) return null;
|
|
28384
|
+
return rel.startsWith("..") ? resolve(cwd, rel) : rel;
|
|
28385
|
+
}
|
|
28386
|
+
async function buildRepair(result, test, rewrite, cwd, ctx) {
|
|
28285
28387
|
const drift = result.drift;
|
|
28286
|
-
|
|
28388
|
+
const renames = result.renames ?? [];
|
|
28389
|
+
const external = (reason) => ({
|
|
28287
28390
|
route: "external",
|
|
28288
|
-
reason
|
|
28289
|
-
|
|
28290
|
-
|
|
28291
|
-
|
|
28292
|
-
|
|
28293
|
-
|
|
28294
|
-
|
|
28295
|
-
|
|
28296
|
-
reason: "the finding is not on the generated test — the document, or a file the test imports and a regeneration only reads. Either way, regenerating rewrites neither"
|
|
28391
|
+
reason,
|
|
28392
|
+
rewrite
|
|
28393
|
+
});
|
|
28394
|
+
if (drift.label !== "TEST_DRIFT") return external(`${drift.label} names no repair a machine can make: recompiling or re-recording reproduces the case as it stands`);
|
|
28395
|
+
if (result.live) return rewrite.length === 0 ? external("a live case is repaired by rewriting its document, and nothing the audit named is in it") : {
|
|
28396
|
+
route: "rewrite",
|
|
28397
|
+
reason: "a live case runs its document, so rewriting it is the whole repair — apply 'rewrite' to 'document', then run the case to verify.",
|
|
28398
|
+
rewrite
|
|
28297
28399
|
};
|
|
28400
|
+
if (test === null) return external("this case resolves to no generated test: there is nothing to regenerate");
|
|
28298
28401
|
const testAbs = resolve(cwd, test);
|
|
28299
|
-
const
|
|
28402
|
+
const recording = await getRecording(caseRefFor(result.target.caseId ?? {
|
|
28300
28403
|
featureName: result.target.featureName,
|
|
28301
28404
|
specName: result.target.specName
|
|
28302
|
-
}, cwd, await caseRecordingPath(result.target, ctx) ?? void 0)).catch(() => null)
|
|
28303
|
-
if (!
|
|
28304
|
-
|
|
28305
|
-
|
|
28306
|
-
|
|
28307
|
-
|
|
28308
|
-
|
|
28309
|
-
|
|
28310
|
-
};
|
|
28405
|
+
}, cwd, await caseRecordingPath(result.target, ctx) ?? void 0)).catch(() => null);
|
|
28406
|
+
if (!recording?.generated) return external("no generation stamp: ccqa did not write this test, or it predates the stamp");
|
|
28407
|
+
if (!await matchesGenerationStamp(recording.generated, testAbs)) return external("the test has been edited since it was generated");
|
|
28408
|
+
const apply = (why) => rewrite.length > 0 ? ` Apply 'rewrite' to 'document' first: ${why}.` : "";
|
|
28409
|
+
if (recordingNamesRenamed(recording, renames)) return {
|
|
28410
|
+
route: "rerecord",
|
|
28411
|
+
reason: `the saved recording names a renamed string, so a regeneration would compile it back in — re-record and verify.${apply("a re-recording drives the case as its document states it")}`,
|
|
28412
|
+
rewrite
|
|
28413
|
+
};
|
|
28414
|
+
if (drift.surface !== "generated" && rewrite.length === 0) return external("the finding is not on the generated test, and nothing it named is in the document — it is on a file the test imports, which a regeneration only reads, or no replacement was named");
|
|
28311
28415
|
return {
|
|
28312
28416
|
route: "regenerate",
|
|
28313
|
-
reason: `test drift
|
|
28417
|
+
reason: `test drift, and the saved recording names no renamed string — regenerate and verify.${apply("regenerating alone recompiles the same document")}`,
|
|
28418
|
+
rewrite
|
|
28314
28419
|
};
|
|
28315
28420
|
}
|
|
28316
28421
|
//#endregion
|
|
@@ -28530,7 +28635,7 @@ function selectSpecsNeedingAudit(targets, report, stillDrifted = /* @__PURE__ */
|
|
|
28530
28635
|
//#endregion
|
|
28531
28636
|
//#region src/cli/audit.ts
|
|
28532
28637
|
const DEFAULT_CONCURRENCY = 3;
|
|
28533
|
-
const auditCommand = addProfileOption(addLanguageOption(new Command("audit").argument("[feature/spec]", "Optional spec id. If omitted, every spec under .ccqa/features/ is checked.").description("Read each spec against the code it describes and report where the two have drifted. Static: no browser is run, so this is the cheap check to put in front of `ccqa run`.").optionsGroup("Which specs to audit:").option("--only-affected-by <ref>", "Only specs `ccqa select-specs` decides the diff against <ref> reaches (e.g. origin/main), by intersecting it with each spec's measured coverage from the hub. In pull_request CI, pass $GITHUB_BASE_REF. Specs without a measurement are audited rather than skipped.").option("--only-hub-audit-needed", "Only specs the hub says a deploy has landed on since the audit last read them. A spec that was never audited is always included, one the hub cannot answer for is audited rather than skipped, and one whose drift entry is still open is always re-audited — a merged fix changes only the spec tree, which no deploy answer covers. No git diff involved. Requires a hub connection and --hub-profile.").optionsGroup("How to run it:").option("--concurrency <n>", `Parallel spec checks (default: ${DEFAULT_CONCURRENCY})`).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").optionsGroup("What to do with the results:").option("--report-format <fmt>", "
|
|
28638
|
+
const auditCommand = addProfileOption(addLanguageOption(new Command("audit").argument("[feature/spec]", "Optional spec id. If omitted, every spec under .ccqa/features/ is checked.").description("Read each spec against the code it describes and report where the two have drifted. Static: no browser is run, so this is the cheap check to put in front of `ccqa run`.").optionsGroup("Which specs to audit:").option("--only-affected-by <ref>", "Only specs `ccqa select-specs` decides the diff against <ref> reaches (e.g. origin/main), by intersecting it with each spec's measured coverage from the hub. In pull_request CI, pass $GITHUB_BASE_REF. Specs without a measurement are audited rather than skipped.").option("--only-hub-audit-needed", "Only specs the hub says a deploy has landed on since the audit last read them. A spec that was never audited is always included, one the hub cannot answer for is audited rather than skipped, and one whose drift entry is still open is always re-audited — a merged fix changes only the spec tree, which no deploy answer covers. No git diff involved. Requires a hub connection and --hub-profile.").optionsGroup("How to run it:").option("--concurrency <n>", `Parallel spec checks (default: ${DEFAULT_CONCURRENCY})`).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").optionsGroup("What to do with the results:").option("--report-dir <dir>", `Directory for the structured audit results (audit.json), which are always written. Default: ${DEFAULT_REPORT_DIR}/.`).option("--report-format <fmt>", "Terminal output format: text | json | github. 'json' prints the same payload audit.json holds, and nothing else.", "text").option("--report-to-hub", "Push the result to a ccqa hub as a run (kind: drift), which is what updates the drift ledger. A spec it finds drifted answers `needsRepair` to `ccqa run --only-hub-rerun-needed`, and is not run until a person repairs it.").option("--exit-on <level>", "Exit non-zero on this severity or higher: warn | error", "error").option("--dump-inputs <dir>", "Also write what each case's audit was given — its document, every file it was handed and the import that reached it, the source roots, and the prompt — as one markdown file per case under <dir>. For telling 'this never reached the audit' apart from 'it did, and nothing was reported'.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory used as both the .ccqa root and the codebase Claude reads. Useful for monorepos. Defaults to process.cwd().").option("--project <name>", "Logical project name for the pushed run. Defaults to the current directory's name.").option(...hubUrlOption).option(...hubTokenOption).option(...hubHeaderOption))).action(withUsageErrors(async (specPath, opts) => {
|
|
28534
28639
|
await withCostReporting("audit", () => runAudit(specPath, opts));
|
|
28535
28640
|
}));
|
|
28536
28641
|
async function runAudit(specPath, opts) {
|
|
@@ -28538,6 +28643,11 @@ async function runAudit(specPath, opts) {
|
|
|
28538
28643
|
const threshold = parseSeverity(opts.exitOn);
|
|
28539
28644
|
const concurrency = parseConcurrency(opts.concurrency);
|
|
28540
28645
|
const cwd = resolveCwd(opts.cwd);
|
|
28646
|
+
const reportDir = resolve(cwd, opts.reportDir ?? "ccqa-report");
|
|
28647
|
+
await rm(join(reportDir, AUDIT_REPORT_FILE), {
|
|
28648
|
+
force: true,
|
|
28649
|
+
recursive: true
|
|
28650
|
+
}).catch(() => {});
|
|
28541
28651
|
await ensureCcqaDir(cwd);
|
|
28542
28652
|
if (opts.onlyAffectedBy && specPath) {
|
|
28543
28653
|
error("--only-affected-by and an explicit spec id cannot be combined; it only applies to a full sweep");
|
|
@@ -28571,11 +28681,18 @@ async function runAudit(specPath, opts) {
|
|
|
28571
28681
|
const intent = artifactsContext.reader.target;
|
|
28572
28682
|
const sourceRoots = await resolveSourceRoots(cwd, config.sourceRoots);
|
|
28573
28683
|
let targets = await collectTargets(specPath, cwd, artifactsContext.reader);
|
|
28574
|
-
if (targets.length === 0)
|
|
28684
|
+
if (targets.length === 0) {
|
|
28685
|
+
const where = intent ? intent.module : ".ccqa/features/";
|
|
28686
|
+
await exitWithNoSpecs({
|
|
28687
|
+
format,
|
|
28688
|
+
reportDir
|
|
28689
|
+
}, "noSpecsFound", `no test cases found under ${where}`);
|
|
28690
|
+
}
|
|
28575
28691
|
if (format === "text") {
|
|
28576
28692
|
header("audit", specPath ?? `${targets.length} case${targets.length > 1 ? "s" : ""}`);
|
|
28577
28693
|
if (opts.cwd) meta("cwd", cwd);
|
|
28578
28694
|
for (const root of sourceRoots) meta("source", root.abs);
|
|
28695
|
+
if (opts.dumpInputs) meta("inputs", resolve(cwd, opts.dumpInputs));
|
|
28579
28696
|
}
|
|
28580
28697
|
if (opts.onlyHubAuditNeeded) {
|
|
28581
28698
|
const total = targets.length;
|
|
@@ -28590,12 +28707,18 @@ async function runAudit(specPath, opts) {
|
|
|
28590
28707
|
meta("hub", selection.summary);
|
|
28591
28708
|
meta("scoped", `${targets.length} of ${total} spec${total > 1 ? "s" : ""}`);
|
|
28592
28709
|
}
|
|
28593
|
-
if (targets.length === 0) exitWithNoSpecs(
|
|
28710
|
+
if (targets.length === 0) await exitWithNoSpecs({
|
|
28711
|
+
format,
|
|
28712
|
+
reportDir
|
|
28713
|
+
}, "allCurrent", "every spec has been audited since the last deploy that reached it");
|
|
28594
28714
|
holder = randomUUID();
|
|
28595
28715
|
const claimed = await claimSpecs(hub, hubProject, opts.hubProfile, targets, holder);
|
|
28596
28716
|
if (claimed.length < targets.length && format === "text") meta("held-elsewhere", `${targets.length - claimed.length} spec(s) another job is auditing`);
|
|
28597
28717
|
targets = claimed;
|
|
28598
|
-
if (targets.length === 0) exitWithNoSpecs(
|
|
28718
|
+
if (targets.length === 0) await exitWithNoSpecs({
|
|
28719
|
+
format,
|
|
28720
|
+
reportDir
|
|
28721
|
+
}, "allHeld", "every spec that needs auditing is already being audited by another job");
|
|
28599
28722
|
}
|
|
28600
28723
|
let baseRef = null;
|
|
28601
28724
|
if (opts.onlyAffectedBy) {
|
|
@@ -28609,7 +28732,10 @@ async function runAudit(specPath, opts) {
|
|
|
28609
28732
|
targets = selection.specs;
|
|
28610
28733
|
baseRef = selection.base.ref;
|
|
28611
28734
|
if (format === "text") meta("scoped", `${targets.length} of ${total} spec${total > 1 ? "s" : ""}`);
|
|
28612
|
-
if (targets.length === 0) exitWithNoSpecs(
|
|
28735
|
+
if (targets.length === 0) await exitWithNoSpecs({
|
|
28736
|
+
format,
|
|
28737
|
+
reportDir
|
|
28738
|
+
}, "noDiffIntersection", "no specs intersect the changed file set; nothing to check");
|
|
28613
28739
|
}
|
|
28614
28740
|
const blocks = await loadAvailableBlocks(cwd);
|
|
28615
28741
|
let pushConn = null;
|
|
@@ -28650,7 +28776,10 @@ async function runAudit(specPath, opts) {
|
|
|
28650
28776
|
} finally {
|
|
28651
28777
|
if (holder) await releaseSpecs(hub, hubProject, opts.hubProfile, holder);
|
|
28652
28778
|
}
|
|
28653
|
-
|
|
28779
|
+
const report = await buildAuditReport(results, cwd, artifactsContext);
|
|
28780
|
+
if (format === "json") process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
|
28781
|
+
else process.stdout.write(renderDrift(results, format, cwd));
|
|
28782
|
+
const written = await tryWriteReport(report, reportDir, format);
|
|
28654
28783
|
if (push) await sealDriftPush(push, {
|
|
28655
28784
|
results,
|
|
28656
28785
|
threshold,
|
|
@@ -28659,18 +28788,22 @@ async function runAudit(specPath, opts) {
|
|
|
28659
28788
|
baseRef,
|
|
28660
28789
|
promptCtx
|
|
28661
28790
|
});
|
|
28662
|
-
|
|
28663
|
-
|
|
28664
|
-
|
|
28665
|
-
|
|
28666
|
-
|
|
28667
|
-
|
|
28668
|
-
|
|
28669
|
-
|
|
28791
|
+
process.exit(written ? determineExitCode(results, threshold) : 2);
|
|
28792
|
+
}
|
|
28793
|
+
/**
|
|
28794
|
+
* Write the report, saying whether it landed. A sweep whose results never
|
|
28795
|
+
* landed has to fail: the next reader of `audit.json` would otherwise act on
|
|
28796
|
+
* the previous invocation's findings with nothing saying so.
|
|
28797
|
+
*/
|
|
28798
|
+
async function tryWriteReport(report, dirAbs, format) {
|
|
28799
|
+
try {
|
|
28800
|
+
const path = await writeAuditReport(report, dirAbs);
|
|
28801
|
+
if (format === "text") meta("report", path);
|
|
28802
|
+
return true;
|
|
28670
28803
|
} catch (e) {
|
|
28671
|
-
error(`could not write
|
|
28804
|
+
error(`could not write the audit report to ${dirAbs}: ${errMessage(e)}`);
|
|
28805
|
+
return false;
|
|
28672
28806
|
}
|
|
28673
|
-
process.exit(determineExitCode(results, threshold));
|
|
28674
28807
|
}
|
|
28675
28808
|
/** Longer than the audit's own timeout, so a sweep cannot outlive its claim. */
|
|
28676
28809
|
const AUDIT_LOCK_TTL_SECONDS = 5400;
|
|
@@ -28796,14 +28929,21 @@ async function resolveAuditPromptContext(opts, cwd, resolveCtx = resolveAuditHub
|
|
|
28796
28929
|
function asHubReadError(err) {
|
|
28797
28930
|
throw new RunUsageError(`could not read from the hub: ${errMessage(err)}`);
|
|
28798
28931
|
}
|
|
28799
|
-
|
|
28800
|
-
|
|
28932
|
+
/**
|
|
28933
|
+
* Nothing to audit — still a result, and still written. A reader that found
|
|
28934
|
+
* yesterday's `audit.json` where this sweep left none would act on findings
|
|
28935
|
+
* nothing had re-checked.
|
|
28936
|
+
*/
|
|
28937
|
+
async function exitWithNoSpecs(ctx, reason, message) {
|
|
28938
|
+
const { format, reportDir } = ctx;
|
|
28939
|
+
const report = {
|
|
28801
28940
|
specs: [],
|
|
28802
28941
|
skipped: reason
|
|
28803
|
-
}
|
|
28942
|
+
};
|
|
28943
|
+
if (format === "json") process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
|
28804
28944
|
else if (format === "text") info(message);
|
|
28805
28945
|
else if (format === "github" && reason === "noSpecsFound") process.stdout.write(`::warning::${message}\n`);
|
|
28806
|
-
process.exit(0);
|
|
28946
|
+
process.exit(await tryWriteReport(report, reportDir, format) ? 0 : 2);
|
|
28807
28947
|
}
|
|
28808
28948
|
/**
|
|
28809
28949
|
* The cases this sweep audits.
|
|
@@ -35,9 +35,9 @@ declare const RunSchema: z.ZodObject<{
|
|
|
35
35
|
running: "running";
|
|
36
36
|
}>;
|
|
37
37
|
kind: z.ZodDefault<z.ZodEnum<{
|
|
38
|
-
record: "record";
|
|
39
38
|
run: "run";
|
|
40
39
|
drift: "drift";
|
|
40
|
+
record: "record";
|
|
41
41
|
}>>;
|
|
42
42
|
drift: z.ZodDefault<z.ZodNullable<z.ZodObject<{
|
|
43
43
|
specs: z.ZodNumber;
|
|
@@ -831,9 +831,9 @@ type ReportSpecResult = z.infer<typeof ReportSpecResultSchema>;
|
|
|
831
831
|
declare const RunReportDataSchema: z.ZodObject<{
|
|
832
832
|
schemaVersion: z.ZodLiteral<1>;
|
|
833
833
|
kind: z.ZodDefault<z.ZodEnum<{
|
|
834
|
-
record: "record";
|
|
835
834
|
run: "run";
|
|
836
835
|
drift: "drift";
|
|
836
|
+
record: "record";
|
|
837
837
|
}>>;
|
|
838
838
|
createdAt: z.ZodString;
|
|
839
839
|
runId: z.ZodNullable<z.ZodString>;
|
|
@@ -35,9 +35,9 @@ declare const RunSchema: z.ZodObject<{
|
|
|
35
35
|
running: "running";
|
|
36
36
|
}>;
|
|
37
37
|
kind: z.ZodDefault<z.ZodEnum<{
|
|
38
|
-
record: "record";
|
|
39
38
|
run: "run";
|
|
40
39
|
drift: "drift";
|
|
40
|
+
record: "record";
|
|
41
41
|
}>>;
|
|
42
42
|
drift: z.ZodDefault<z.ZodNullable<z.ZodObject<{
|
|
43
43
|
specs: z.ZodNumber;
|
|
@@ -831,9 +831,9 @@ type ReportSpecResult = z.infer<typeof ReportSpecResultSchema>;
|
|
|
831
831
|
declare const RunReportDataSchema: z.ZodObject<{
|
|
832
832
|
schemaVersion: z.ZodLiteral<1>;
|
|
833
833
|
kind: z.ZodDefault<z.ZodEnum<{
|
|
834
|
-
record: "record";
|
|
835
834
|
run: "run";
|
|
836
835
|
drift: "drift";
|
|
836
|
+
record: "record";
|
|
837
837
|
}>>;
|
|
838
838
|
createdAt: z.ZodString;
|
|
839
839
|
runId: z.ZodNullable<z.ZodString>;
|
package/dist/package.json
CHANGED