ccqa 1.31.0 → 1.31.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/ccqa.mjs +96 -28
- package/dist/package.json +1 -1
- package/package.json +1 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -15523,7 +15523,7 @@ const VALIDATION_MODES = ["lenient", "strict"];
|
|
|
15523
15523
|
const recordCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("record").argument("<feature/spec>", "Spec id in '<feature>/<spec>' form (resolves to .ccqa/features/<feature>/test-cases/<spec>/)").description("Record a test from a spec: run agent-browser to collect actions (trace), then compile them into runnable code via the spec's target (generate) — a vitest test.spec.ts for agent-browser, a @playwright/test spec for the playwright target. Recording-backed targets only; spec-input targets like runn have no trace step (use `ccqa generate`), and agent-browser live specs need no recording.").optionsGroup("How to record:").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--instruction <text>", "Extra guidance for the recording agent — e.g. the drift audit's finding when re-recording a drifted spec.").option("--trace-validation <mode>", "What to do with actions that fail post-trace validation: 'lenient' (default) tags them; 'strict' drops them.", (raw) => {
|
|
15524
15524
|
if (VALIDATION_MODES.includes(raw)) return raw;
|
|
15525
15525
|
throw new Error(`--trace-validation must be one of ${VALIDATION_MODES.join(" | ")}`);
|
|
15526
|
-
}, "lenient").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--trace-only", "Stop after the trace step; do not generate test code").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--overwrite", "Replace an existing test.spec.ts without warning").option("--report-to-hub", "Leave a run (kind: record) on the hub saying this spec was recorded and what the recording spent on Claude, so a budget summed over the hub's runs sees it. It advances no ledger: a recording verifies nothing.").optionsGroup("Learning:").option("--learn-hub-trace-prompt", "After the trace finishes, ask Claude to refresh the \"record.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
15526
|
+
}, "lenient").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--timeout <seconds>", "Abort the recording after this many seconds, wherever it is (trace, generate, auto-fix): reap the browser session, seal the open hub run (--report-to-hub) with a 'timed out' note, and exit 124. Prefer this over wrapping the command in an external `timeout`, whose SIGTERM may never reach this process.", parseTimeoutSeconds).option("--trace-only", "Stop after the trace step; do not generate test code").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--overwrite", "Replace an existing test.spec.ts without warning").option("--report-to-hub", "Leave a run (kind: record) on the hub saying this spec was recorded and what the recording spent on Claude, so a budget summed over the hub's runs sees it. It advances no ledger: a recording verifies nothing.").optionsGroup("Learning:").option("--learn-hub-trace-prompt", "After the trace finishes, ask Claude to refresh the \"record.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
15527
15527
|
await withCostReporting("record", () => runRecord(specPath, opts));
|
|
15528
15528
|
}));
|
|
15529
15529
|
async function runRecord(specPath, opts) {
|
|
@@ -15578,21 +15578,29 @@ async function runRecord(specPath, opts) {
|
|
|
15578
15578
|
let recorded = false;
|
|
15579
15579
|
let sealed = true;
|
|
15580
15580
|
let tracingStep;
|
|
15581
|
-
let
|
|
15581
|
+
let abortCause;
|
|
15582
15582
|
const teardown = createRunTeardown();
|
|
15583
15583
|
teardown.onFinalize(async () => {
|
|
15584
15584
|
if (!push) return;
|
|
15585
|
-
|
|
15586
|
-
sealed = await sealRecordPush(push, featureName, specName, recorded, note);
|
|
15585
|
+
sealed = await sealRecordPush(push, featureName, specName, recorded, abortCause !== void 0 ? abortNote(abortCause, tracingStep) : void 0);
|
|
15587
15586
|
});
|
|
15588
15587
|
const disposeSignalHandlers = installTeardownSignalHandlers(teardown, (sig) => {
|
|
15589
|
-
|
|
15588
|
+
abortCause = `terminated by signal (${sig})`;
|
|
15590
15589
|
});
|
|
15590
|
+
let deadline;
|
|
15591
|
+
if (opts.timeout !== void 0) {
|
|
15592
|
+
const seconds = opts.timeout;
|
|
15593
|
+
deadline = setTimeout(() => {
|
|
15594
|
+
abortCause = `timed out after ${seconds}s`;
|
|
15595
|
+
error(`--timeout: ${abortNote(abortCause, tracingStep)}`);
|
|
15596
|
+
teardown.run().finally(() => process.exit(124));
|
|
15597
|
+
}, seconds * 1e3);
|
|
15598
|
+
deadline.unref();
|
|
15599
|
+
}
|
|
15591
15600
|
try {
|
|
15592
|
-
let traceResult = null;
|
|
15593
15601
|
let generated = true;
|
|
15594
15602
|
try {
|
|
15595
|
-
traceResult = await runTrace(featureName, specName, opts.model, opts.traceValidation ?? "lenient", language, {
|
|
15603
|
+
const traceResult = await runTrace(featureName, specName, opts.model, opts.traceValidation ?? "lenient", language, {
|
|
15596
15604
|
cwd: cwdForProfile,
|
|
15597
15605
|
hubContext,
|
|
15598
15606
|
...opts.instruction ? { instruction: opts.instruction } : {},
|
|
@@ -15602,6 +15610,15 @@ async function runRecord(specPath, opts) {
|
|
|
15602
15610
|
});
|
|
15603
15611
|
tracingStep = void 0;
|
|
15604
15612
|
blank();
|
|
15613
|
+
await learnFromTrace({
|
|
15614
|
+
enabled: opts.learnHubTracePrompt === true,
|
|
15615
|
+
featureName,
|
|
15616
|
+
specName,
|
|
15617
|
+
traceResult,
|
|
15618
|
+
hubContext,
|
|
15619
|
+
...opts.model ? { model: opts.model } : {},
|
|
15620
|
+
...language ? { language } : {}
|
|
15621
|
+
});
|
|
15605
15622
|
if (!opts.traceOnly) generated = (await runGenerate(featureName, specName, {
|
|
15606
15623
|
maxRetries: parseInt(opts.autoFixMaxRetries ?? "3", 10),
|
|
15607
15624
|
fixMode: toFixMode(opts.autoFix ?? "interactive"),
|
|
@@ -15616,25 +15633,52 @@ async function runRecord(specPath, opts) {
|
|
|
15616
15633
|
} finally {
|
|
15617
15634
|
await releaseLock();
|
|
15618
15635
|
}
|
|
15619
|
-
if (opts.learnHubTracePrompt && traceResult !== null) {
|
|
15620
|
-
blank();
|
|
15621
|
-
await updateAgentPrompt({
|
|
15622
|
-
kind: "record",
|
|
15623
|
-
flag: "--learn-hub-trace-prompt",
|
|
15624
|
-
runSummary: buildRecordRunSummary(featureName, specName, traceResult),
|
|
15625
|
-
hubContext,
|
|
15626
|
-
...opts.model ? { model: opts.model } : {},
|
|
15627
|
-
...language ? { language } : {}
|
|
15628
|
-
});
|
|
15629
|
-
}
|
|
15630
15636
|
recorded = generated;
|
|
15631
15637
|
} finally {
|
|
15638
|
+
if (deadline !== void 0) clearTimeout(deadline);
|
|
15632
15639
|
await teardown.run();
|
|
15633
15640
|
disposeSignalHandlers();
|
|
15634
15641
|
}
|
|
15635
15642
|
if (!sealed) process.exit(2);
|
|
15636
15643
|
if (!recorded) process.exit(1);
|
|
15637
15644
|
}
|
|
15645
|
+
/** `--timeout <seconds>`: a positive whole number of seconds. */
|
|
15646
|
+
function parseTimeoutSeconds(raw) {
|
|
15647
|
+
const n = Number(raw);
|
|
15648
|
+
if (!Number.isFinite(n) || n <= 0 || Math.floor(n) !== n) throw new Error(`--timeout must be a positive integer number of seconds, got "${raw}"`);
|
|
15649
|
+
return n;
|
|
15650
|
+
}
|
|
15651
|
+
/**
|
|
15652
|
+
* The one line an aborted recording leaves on its hub row: what ended it
|
|
15653
|
+
* ("terminated by signal (SIGTERM)", "timed out after 900s") plus the spec
|
|
15654
|
+
* step that was tracing, when one was in flight. The signal handlers and the
|
|
15655
|
+
* --timeout deadline both seal through this, so every abort dies with the
|
|
15656
|
+
* same shape of reason.
|
|
15657
|
+
*/
|
|
15658
|
+
function abortNote(cause, tracingStep) {
|
|
15659
|
+
return `${cause}${tracingStep !== void 0 ? ` during ${tracingStep}` : ""}`;
|
|
15660
|
+
}
|
|
15661
|
+
/**
|
|
15662
|
+
* The rule for `--learn-hub-trace-prompt`: a browser trace ran → learn from
|
|
15663
|
+
* it; no trace → stay silent. `runRecord` calls this immediately after the
|
|
15664
|
+
* trace, before generate — sequencing it after the generate/auto-fix half
|
|
15665
|
+
* (as record once did) let any death there discard a completed trace's
|
|
15666
|
+
* learnings. Returns whether the refresh fired, and takes the updater as a
|
|
15667
|
+
* seam so the rule is testable without a browser.
|
|
15668
|
+
*/
|
|
15669
|
+
async function learnFromTrace(args, update = updateAgentPrompt) {
|
|
15670
|
+
if (!args.enabled || args.traceResult === null) return false;
|
|
15671
|
+
blank();
|
|
15672
|
+
await update({
|
|
15673
|
+
kind: "record",
|
|
15674
|
+
flag: "--learn-hub-trace-prompt",
|
|
15675
|
+
runSummary: buildRecordRunSummary(args.featureName, args.specName, args.traceResult),
|
|
15676
|
+
hubContext: args.hubContext,
|
|
15677
|
+
...args.model !== void 0 ? { model: args.model } : {},
|
|
15678
|
+
...args.language !== void 0 ? { language: args.language } : {}
|
|
15679
|
+
});
|
|
15680
|
+
return true;
|
|
15681
|
+
}
|
|
15638
15682
|
/**
|
|
15639
15683
|
* Close the record run with the one row this command produced, answering
|
|
15640
15684
|
* whether it closed. One spec is recorded per invocation, so one row is the
|
|
@@ -16443,19 +16487,42 @@ async function fetchAuditNeed(ctx, profile) {
|
|
|
16443
16487
|
throw new RunUsageError(`${FLAG}: could not ask the hub which specs need auditing: ${errMessage(err)}`);
|
|
16444
16488
|
}
|
|
16445
16489
|
}
|
|
16490
|
+
/**
|
|
16491
|
+
* Specs whose drift-ledger entry is still open. The hub's audit-need answer is
|
|
16492
|
+
* deploy-based, and a merged fix changes only the spec tree — no deploy lands
|
|
16493
|
+
* on the spec, so the hub would never call it due again and the entry would
|
|
16494
|
+
* stay open forever. A drifted spec is due until the audit itself clears it.
|
|
16495
|
+
* Unreadable ledger degrades to the deploy-based answer alone: the sweep must
|
|
16496
|
+
* not die over the supplementary question.
|
|
16497
|
+
*/
|
|
16498
|
+
async function fetchStillDrifted(ctx) {
|
|
16499
|
+
try {
|
|
16500
|
+
const ledger = await ctx.hub.getDriftLedger(ctx.project);
|
|
16501
|
+
return new Set(Object.entries(ledger.specs ?? {}).filter(([, entry]) => entry.label != null).map(([key]) => key));
|
|
16502
|
+
} catch {
|
|
16503
|
+
return /* @__PURE__ */ new Set();
|
|
16504
|
+
}
|
|
16505
|
+
}
|
|
16446
16506
|
/** Worst-known-first, so the line leads with what has never been looked at. */
|
|
16447
16507
|
const SUMMARY_ORDER = rankedOrder({
|
|
16448
16508
|
neverAudited: 0,
|
|
16449
|
-
|
|
16450
|
-
|
|
16451
|
-
|
|
16452
|
-
|
|
16509
|
+
stillDrifted: 1,
|
|
16510
|
+
cannotTell: 2,
|
|
16511
|
+
deployReached: 3,
|
|
16512
|
+
held: 4,
|
|
16513
|
+
current: 5
|
|
16453
16514
|
});
|
|
16454
|
-
function selectSpecsNeedingAudit(targets, report) {
|
|
16515
|
+
function selectSpecsNeedingAudit(targets, report, stillDrifted = /* @__PURE__ */ new Set()) {
|
|
16455
16516
|
const counts = /* @__PURE__ */ new Map();
|
|
16456
16517
|
const selected = [];
|
|
16457
16518
|
for (const target of targets) {
|
|
16458
|
-
const
|
|
16519
|
+
const key = specKey(target);
|
|
16520
|
+
if (stillDrifted.has(key)) {
|
|
16521
|
+
counts.set("stillDrifted", (counts.get("stillDrifted") ?? 0) + 1);
|
|
16522
|
+
selected.push(target);
|
|
16523
|
+
continue;
|
|
16524
|
+
}
|
|
16525
|
+
const need = report.specs[key] ?? { because: "neverAudited" };
|
|
16459
16526
|
counts.set(need.because, (counts.get(need.because) ?? 0) + 1);
|
|
16460
16527
|
if (needsAudit(need)) selected.push(target);
|
|
16461
16528
|
}
|
|
@@ -16467,7 +16534,7 @@ function selectSpecsNeedingAudit(targets, report) {
|
|
|
16467
16534
|
//#endregion
|
|
16468
16535
|
//#region src/cli/audit.ts
|
|
16469
16536
|
const DEFAULT_CONCURRENCY = 3;
|
|
16470
|
-
const auditCommand = addProfileOption(addLanguageOption(new Command("audit").argument("[feature/spec]", "Optional spec id. If omitted, every spec under .ccqa/features/ is checked.").description("Read each spec against the code it describes and report where the two have drifted. Static: no browser is run, so this is the cheap check to put in front of `ccqa run`.").optionsGroup("Which specs to audit:").option("--only-affected-by <ref>", "Only specs `ccqa select-specs` judges reached by the diff against <ref> (e.g. origin/main). In pull_request CI, pass $GITHUB_BASE_REF. Costs one model call; specs it cannot decide are audited rather than skipped.").option("--only-hub-audit-needed", "Only specs the hub says a deploy has landed on since the audit last read them. A spec that was never audited is always included,
|
|
16537
|
+
const auditCommand = addProfileOption(addLanguageOption(new Command("audit").argument("[feature/spec]", "Optional spec id. If omitted, every spec under .ccqa/features/ is checked.").description("Read each spec against the code it describes and report where the two have drifted. Static: no browser is run, so this is the cheap check to put in front of `ccqa run`.").optionsGroup("Which specs to audit:").option("--only-affected-by <ref>", "Only specs `ccqa select-specs` judges reached by the diff against <ref> (e.g. origin/main). In pull_request CI, pass $GITHUB_BASE_REF. Costs one model call; specs it cannot decide are audited rather than skipped.").option("--only-hub-audit-needed", "Only specs the hub says a deploy has landed on since the audit last read them. A spec that was never audited is always included, one the hub cannot answer for is audited rather than skipped, and one whose drift entry is still open is always re-audited — a merged fix changes only the spec tree, which no deploy answer covers. No git diff involved. Requires a hub connection and --hub-profile.").optionsGroup("How to run it:").option("--concurrency <n>", `Parallel spec checks (default: ${DEFAULT_CONCURRENCY})`).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").optionsGroup("What to do with the results:").option("--report-format <fmt>", "Output format: text | json | github", "text").option("--report-to-hub", "Push the result to a ccqa hub as a run (kind: drift), which is what updates the drift ledger. A spec it finds drifted answers `needsRepair` to `ccqa run --only-hub-rerun-needed`, and is not run until a person repairs it.").option("--exit-on <level>", "Exit non-zero on this severity or higher: warn | error", "error").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory used as both the .ccqa root and the codebase Claude reads. Useful for monorepos. Defaults to process.cwd().").option("--project <name>", "Logical project name for the pushed run. Defaults to the current directory's name.").option(...hubUrlOption).option(...hubTokenOption).option(...hubHeaderOption))).action(withUsageErrors(async (specPath, opts) => {
|
|
16471
16538
|
await withCostReporting("audit", () => runAudit(specPath, opts));
|
|
16472
16539
|
}));
|
|
16473
16540
|
async function runAudit(specPath, opts) {
|
|
@@ -16511,11 +16578,12 @@ async function runAudit(specPath, opts) {
|
|
|
16511
16578
|
}
|
|
16512
16579
|
if (opts.onlyHubAuditNeeded) {
|
|
16513
16580
|
const total = targets.length;
|
|
16514
|
-
const
|
|
16581
|
+
const ctx = {
|
|
16515
16582
|
hub,
|
|
16516
16583
|
project: hubProject
|
|
16517
|
-
}
|
|
16518
|
-
const
|
|
16584
|
+
};
|
|
16585
|
+
const [report, stillDrifted] = await Promise.all([fetchAuditNeed(ctx, opts.hubProfile), fetchStillDrifted(ctx)]);
|
|
16586
|
+
const selection = selectSpecsNeedingAudit(targets, report, stillDrifted);
|
|
16519
16587
|
targets = selection.selected;
|
|
16520
16588
|
if (format === "text") {
|
|
16521
16589
|
meta("hub", selection.summary);
|
package/dist/package.json
CHANGED