ccqa 1.13.0 → 1.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -234
- package/dist/bin/ccqa.mjs +185 -189
- package/dist/hub-client/index.d.mts +8 -3
- package/dist/package.json +1 -1
- package/package.json +1 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -477,7 +477,7 @@ function getBlocksDir(cwd) {
|
|
|
477
477
|
/**
|
|
478
478
|
* Inverse of `getBlockDir`. Given a file path that appears in a git diff,
|
|
479
479
|
* return the block name if the path points at the block's spec.yaml, else
|
|
480
|
-
* null. Used by `
|
|
480
|
+
* null. Used by `audit --only-affected-by` to invalidate specs whose included blocks
|
|
481
481
|
* were edited. (v0.4 inlines blocks into every spec's own trace, so the
|
|
482
482
|
* block directory holds only spec.yaml — no per-block recording lives
|
|
483
483
|
* here anymore.)
|
|
@@ -524,24 +524,22 @@ const USER_PROMPT_MAX_BYTES = 32768;
|
|
|
524
524
|
* Load the prompt bundle from the hub for one guidance kind ("record" /
|
|
525
525
|
* "live" / an LLM-generation target such as "playwright" or "runn").
|
|
526
526
|
* Best-effort: no hub client, a fetch failure, or both prompts absent all
|
|
527
|
-
*
|
|
527
|
+
* A prompt that was never stored resolves to null. A hub that cannot be
|
|
528
|
+
* reached throws: running with silently different guidance than the project
|
|
529
|
+
* configured is worse than stopping.
|
|
528
530
|
*/
|
|
529
531
|
async function loadPromptBundleFromHub(ctx, kind) {
|
|
530
532
|
if (!ctx) return null;
|
|
531
533
|
const userName = `${kind}.user`;
|
|
532
534
|
const agentName = `${kind}.agent`;
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
});
|
|
542
|
-
} catch {
|
|
543
|
-
return null;
|
|
544
|
-
}
|
|
535
|
+
const [userText, agentText] = await Promise.all([ctx.hub.getPrompt(ctx.project, userName).then(normalizePromptText), ctx.hub.getPrompt(ctx.project, agentName).then(normalizePromptText)]);
|
|
536
|
+
return assemblePromptBundle({
|
|
537
|
+
text: userText,
|
|
538
|
+
label: userName
|
|
539
|
+
}, {
|
|
540
|
+
text: agentText,
|
|
541
|
+
label: agentName
|
|
542
|
+
});
|
|
545
543
|
}
|
|
546
544
|
/**
|
|
547
545
|
* Shared concatenation logic behind `loadPromptBundleFromHub`: section
|
|
@@ -557,7 +555,7 @@ function assemblePromptBundle(user, agent) {
|
|
|
557
555
|
loaded.push(user.label);
|
|
558
556
|
}
|
|
559
557
|
if (agent.text !== null) {
|
|
560
|
-
sections.push(`### Agent learnings (auto-updated by ccqa --
|
|
558
|
+
sections.push(`### Agent learnings (auto-updated by ccqa's --learn-*-prompt flags)\n\n${agent.text}`);
|
|
561
559
|
loaded.push(agent.label);
|
|
562
560
|
}
|
|
563
561
|
let text = sections.join("\n\n");
|
|
@@ -2475,7 +2473,7 @@ const FailureAnalysisSchema = z.object({
|
|
|
2475
2473
|
});
|
|
2476
2474
|
/**
|
|
2477
2475
|
* What a drift audit may conclude, in the same vocabulary `ccqa run
|
|
2478
|
-
* --
|
|
2476
|
+
* --on-fail-explain` uses for a failure. One question, one answer, the same
|
|
2479
2477
|
* words whether it was reached by running the spec or by reading the code — so
|
|
2480
2478
|
* a reader never translates between two taxonomies, and the hub renders,
|
|
2481
2479
|
* grades and learns from both through one path.
|
|
@@ -3312,7 +3310,7 @@ function relativeToCwd(path, cwd) {
|
|
|
3312
3310
|
/** The model's reply: a diagnosis, or `null` for "the spec still matches the code". */
|
|
3313
3311
|
const DriftReplySchema = z.object({ drift: DriftDiagnosisSchema.nullable() });
|
|
3314
3312
|
/**
|
|
3315
|
-
* How a label reads against `--
|
|
3313
|
+
* How a label reads against `--exit-on`. The threshold asks "would a
|
|
3316
3314
|
* deterministic replay fail today", which is what the label already answers:
|
|
3317
3315
|
* both findings mean the spec no longer describes the code, while `UNKNOWN`
|
|
3318
3316
|
* means the audit could not tell and should not fail a build on its own.
|
|
@@ -3971,14 +3969,10 @@ ${customPrompt.guidance}
|
|
|
3971
3969
|
*/
|
|
3972
3970
|
async function fetchCustomPrompt(ctx) {
|
|
3973
3971
|
if (!ctx) return null;
|
|
3974
|
-
|
|
3975
|
-
|
|
3976
|
-
|
|
3977
|
-
|
|
3978
|
-
return parsed.success ? parsed.data : null;
|
|
3979
|
-
} catch {
|
|
3980
|
-
return null;
|
|
3981
|
-
}
|
|
3972
|
+
const raw = await ctx.hub.getPrompt(ctx.project, "analysis-custom-prompt");
|
|
3973
|
+
if (raw === null) return null;
|
|
3974
|
+
const parsed = AnalysisCustomPromptSchema.safeParse(JSON.parse(raw));
|
|
3975
|
+
return parsed.success ? parsed.data : null;
|
|
3982
3976
|
}
|
|
3983
3977
|
/**
|
|
3984
3978
|
* Render the human-maintained `triage.user` guidance as a prompt section, or
|
|
@@ -4006,12 +4000,8 @@ ${trimmed}
|
|
|
4006
4000
|
*/
|
|
4007
4001
|
async function fetchTriageUserPrompt(ctx) {
|
|
4008
4002
|
if (!ctx) return null;
|
|
4009
|
-
|
|
4010
|
-
|
|
4011
|
-
return trimmed ? trimmed : null;
|
|
4012
|
-
} catch {
|
|
4013
|
-
return null;
|
|
4014
|
-
}
|
|
4003
|
+
const trimmed = (await ctx.hub.getPrompt(ctx.project, "triage.user"))?.trim();
|
|
4004
|
+
return trimmed ? trimmed : null;
|
|
4015
4005
|
}
|
|
4016
4006
|
/**
|
|
4017
4007
|
* Short, stable content hash for a `triage.user` prompt. The Markdown body
|
|
@@ -4812,12 +4802,12 @@ async function readDotenv(path) {
|
|
|
4812
4802
|
}
|
|
4813
4803
|
return parseDotenv(content);
|
|
4814
4804
|
}
|
|
4815
|
-
/** Absolute path of the default `.env` ccqa loads when `--profile` is absent. */
|
|
4805
|
+
/** Absolute path of the default `.env` ccqa loads when `--hub-profile` is absent. */
|
|
4816
4806
|
function defaultEnvPath(cwd) {
|
|
4817
4807
|
return join(cwd, ".env");
|
|
4818
4808
|
}
|
|
4819
4809
|
/**
|
|
4820
|
-
* Load `<cwd>/.env`, the default when no `--profile` is given. A missing `.env`
|
|
4810
|
+
* Load `<cwd>/.env`, the default when no `--hub-profile` is given. A missing `.env`
|
|
4821
4811
|
* is fine (returns `null`) — the run falls back to the existing `process.env`.
|
|
4822
4812
|
*/
|
|
4823
4813
|
async function loadDefaultEnv(cwd) {
|
|
@@ -5023,7 +5013,7 @@ function addLanguageOption(command) {
|
|
|
5023
5013
|
* `record`), registered identically so help text and behaviour don't drift.
|
|
5024
5014
|
*/
|
|
5025
5015
|
function addProfileOption(command) {
|
|
5026
|
-
return command.option("--profile <name>", "Load this profile's variables from the hub into the environment before resolving spec ${VAR} references (URLs, credentials), so one spec can
|
|
5016
|
+
return command.option("--hub-profile <name>", "Load this profile's variables from the hub into the environment before resolving spec ${VAR} references (URLs, credentials), so one spec can run as a different tenant, account or role without a copy per set of values. A profile is a value set, not an environment — ccqa tracks one verification environment. Profile values override the inherited environment. Requires --hub-url/--hub-token (or CCQA_HUB_URL/CCQA_HUB_TOKEN).");
|
|
5027
5017
|
}
|
|
5028
5018
|
/**
|
|
5029
5019
|
* Shared `--hub-url` / `--hub-token` flags for commands that optionally talk
|
|
@@ -5901,7 +5891,7 @@ async function runVerificationLoop(p, ref, state) {
|
|
|
5901
5891
|
const runCommand = p.ctx.targetConfig.runCommand;
|
|
5902
5892
|
if (!runCommand) return true;
|
|
5903
5893
|
const maxRetries = p.ctx.fix.mode === "non-interactive" ? 0 : p.ctx.fix.maxRetries;
|
|
5904
|
-
if (!p.ctx.fix.useSnapshot) warn(`--no-
|
|
5894
|
+
if (!p.ctx.fix.useSnapshot) warn(`--no-session-pin has no effect on the ${p.target} target — it captures no browser snapshot; the fix loop uses the command's output instead`);
|
|
5905
5895
|
for (let attempt = 0;; attempt++) {
|
|
5906
5896
|
const testFiles = [...state.entries()].filter(([, f]) => f.kind === "test").map(([rel]) => rel);
|
|
5907
5897
|
const artifactsDir = await mkdtemp(join(tmpdir(), "ccqa-verify-artifacts-"));
|
|
@@ -6089,8 +6079,8 @@ const C$1 = {
|
|
|
6089
6079
|
* script, so it keeps its own caller in `cli/run-live.ts`; only the
|
|
6090
6080
|
* `ANALYSIS_DISABLED` string is shared with it.
|
|
6091
6081
|
*/
|
|
6092
|
-
/** `analysisSkipped` for a failed row when `--
|
|
6093
|
-
const ANALYSIS_DISABLED = "skipped: --
|
|
6082
|
+
/** `analysisSkipped` for a failed row when `--on-fail-explain` was not requested. */
|
|
6083
|
+
const ANALYSIS_DISABLED = "skipped: --on-fail-explain not enabled";
|
|
6094
6084
|
/**
|
|
6095
6085
|
* Create one analysis pass. The returned object is stateful on purpose: the
|
|
6096
6086
|
* "source diff unavailable" notice and the summary block's header are printed
|
|
@@ -6610,7 +6600,7 @@ async function detectDefaultBranch(cwd) {
|
|
|
6610
6600
|
/**
|
|
6611
6601
|
* Fetch the last-green ledger for this run — one hub round trip, logged as
|
|
6612
6602
|
* the run's analysis-base meta line. Fails fast (RunUsageError) when the hub
|
|
6613
|
-
* can't serve it: `--
|
|
6603
|
+
* can't serve it: `--on-fail-explain` explicitly opted into
|
|
6614
6604
|
* hub-backed baselines, so a broken hub connection is a usage error, never a
|
|
6615
6605
|
* silent no-baseline run.
|
|
6616
6606
|
*/
|
|
@@ -6633,7 +6623,7 @@ async function fetchLastGreenLedger(hubCtx, profile, cwd) {
|
|
|
6633
6623
|
return entries;
|
|
6634
6624
|
}
|
|
6635
6625
|
/**
|
|
6636
|
-
* Per-spec baseline resolver for `--
|
|
6626
|
+
* Per-spec baseline resolver for `--on-fail-explain` without an explicit base. A spec
|
|
6637
6627
|
* missing from the ledger (never green on a pushed run yet) or whose
|
|
6638
6628
|
* baseline commit isn't in this checkout resolves to a skip — the run
|
|
6639
6629
|
* continues; only that spec's classification is withheld, with the reason
|
|
@@ -6687,7 +6677,7 @@ async function deployHeadSha(hub, project, profile) {
|
|
|
6687
6677
|
*
|
|
6688
6678
|
* Captured before any spec executes and asserted on both push paths
|
|
6689
6679
|
* (`?deployedSha=` on `POST /runs` via `ccqa hub push`, and on `POST
|
|
6690
|
-
* /runs/open` for `--
|
|
6680
|
+
* /runs/open` for `--report-to-hub`). Left to itself the hub reads its own
|
|
6691
6681
|
* deploy-log head when the call lands — after the whole run for a single-shot
|
|
6692
6682
|
* push, after the deterministic phase for an incremental one — so a deploy
|
|
6693
6683
|
* landing in that window would be recorded as the run's baseline and
|
|
@@ -6712,7 +6702,7 @@ async function tryDeployHeadSha(hubCtx, profile) {
|
|
|
6712
6702
|
*
|
|
6713
6703
|
* This exists because a selection can be wrong in a way that costs money.
|
|
6714
6704
|
* `ccqa select-specs`'s model judgment is not infallible, and both
|
|
6715
|
-
* `--
|
|
6705
|
+
* `--only-affected-by` and `--only-hub-rerun-needed` decide from it, so a human has to
|
|
6716
6706
|
* be able to read the selection back before a live spec spends a Claude
|
|
6717
6707
|
* budget on it.
|
|
6718
6708
|
*
|
|
@@ -6744,53 +6734,9 @@ function formatDryRunLines(agentBrowser, routed) {
|
|
|
6744
6734
|
return tagged.map((t) => ` ${t.key.padEnd(width)} ${t.tag}`);
|
|
6745
6735
|
}
|
|
6746
6736
|
//#endregion
|
|
6747
|
-
//#region src/run/audited-clean.ts
|
|
6748
|
-
/**
|
|
6749
|
-
* Fetch the drift ledger and reduce it to the specs that are safe to run.
|
|
6750
|
-
*
|
|
6751
|
-
* A spec qualifies only when the ledger holds an entry for it *and* that entry
|
|
6752
|
-
* found no drift. A spec that has never been audited does not qualify: the
|
|
6753
|
-
* point of the flag is to spend a run only where a cheap audit already said
|
|
6754
|
-
* the spec still describes the code, and "never looked" is not that.
|
|
6755
|
-
*/
|
|
6756
|
-
async function fetchAuditedLedger(hubCtx) {
|
|
6757
|
-
let ledger;
|
|
6758
|
-
try {
|
|
6759
|
-
ledger = await hubCtx.hub.getDriftLedger(hubCtx.project);
|
|
6760
|
-
} catch (err) {
|
|
6761
|
-
throw new RunUsageError(`--only-audited-clean: could not fetch the drift ledger from the hub: ${errMessage(err)}`);
|
|
6762
|
-
}
|
|
6763
|
-
const clean = /* @__PURE__ */ new Set();
|
|
6764
|
-
const audited = /* @__PURE__ */ new Set();
|
|
6765
|
-
for (const [key, entry] of Object.entries(ledger.specs)) {
|
|
6766
|
-
audited.add(key);
|
|
6767
|
-
if (entry.label === null) clean.add(key);
|
|
6768
|
-
}
|
|
6769
|
-
return {
|
|
6770
|
-
clean,
|
|
6771
|
-
audited
|
|
6772
|
-
};
|
|
6773
|
-
}
|
|
6774
|
-
function selectAuditedClean(specs, ledger) {
|
|
6775
|
-
const selected = [];
|
|
6776
|
-
let unaudited = 0;
|
|
6777
|
-
let drifted = 0;
|
|
6778
|
-
for (const spec of specs) {
|
|
6779
|
-
const key = specKey(spec);
|
|
6780
|
-
if (ledger.clean.has(key)) selected.push(spec);
|
|
6781
|
-
else if (ledger.audited.has(key)) drifted++;
|
|
6782
|
-
else unaudited++;
|
|
6783
|
-
}
|
|
6784
|
-
return {
|
|
6785
|
-
selected,
|
|
6786
|
-
unaudited,
|
|
6787
|
-
drifted
|
|
6788
|
-
};
|
|
6789
|
-
}
|
|
6790
|
-
//#endregion
|
|
6791
6737
|
//#region src/run/rerun-selection.ts
|
|
6792
6738
|
/**
|
|
6793
|
-
* `ccqa run --
|
|
6739
|
+
* `ccqa run --only-hub-rerun-needed`: select specs from the hub's re-run verdicts
|
|
6794
6740
|
* instead of from a git diff (ADR-0010). The baseline is not a ref at all —
|
|
6795
6741
|
* it is each spec's own last run, positioned against the deploy log the
|
|
6796
6742
|
* consuming deploy job feeds the hub — so this path does no git work.
|
|
@@ -6802,12 +6748,12 @@ function selectAuditedClean(specs, ledger) {
|
|
|
6802
6748
|
/** First ccqa release whose hub serves `GET /projects/:project/rerun`. */
|
|
6803
6749
|
const RERUN_MIN_HUB_VERSION = "1.9";
|
|
6804
6750
|
/**
|
|
6805
|
-
* The profile `--
|
|
6751
|
+
* The profile `--only-hub-rerun-needed` asks about. Mandatory: two environments sit
|
|
6806
6752
|
* at different commits and the deploy log is per-profile, so "needs re-run"
|
|
6807
6753
|
* has no profile-free answer.
|
|
6808
6754
|
*/
|
|
6809
6755
|
function requireRerunProfile(profile) {
|
|
6810
|
-
if (profile === void 0) throw new RunUsageError("--only-
|
|
6756
|
+
if (profile === void 0) throw new RunUsageError("--only-hub-rerun-needed requires --hub-profile <name>: the deploy log it reads is per-profile, so which specs need a re-run has no answer without one");
|
|
6811
6757
|
return profile;
|
|
6812
6758
|
}
|
|
6813
6759
|
/**
|
|
@@ -6820,9 +6766,9 @@ async function fetchRerunReport(hubCtx, profile) {
|
|
|
6820
6766
|
report = await hubCtx.hub.getRerun(hubCtx.project, { profile });
|
|
6821
6767
|
} catch (err) {
|
|
6822
6768
|
if (err instanceof HubApiError && err.status === 404) throw new RunUsageError(explainNotFound(hubCtx, err));
|
|
6823
|
-
throw new RunUsageError(`--only-
|
|
6769
|
+
throw new RunUsageError(`--only-hub-rerun-needed: could not ask the hub which specs need a re-run: ${errMessage(err)}`);
|
|
6824
6770
|
}
|
|
6825
|
-
if (report.deployHead === null) throw new RunUsageError(`--only-
|
|
6771
|
+
if (report.deployHead === null) throw new RunUsageError(`--only-hub-rerun-needed: no deploy has been recorded for profile "${profile}" of project "${hubCtx.project}", so nothing can be compared against. Wire \`ccqa hub deploy record\` into the deploy job, or select with --only-affected-by <ref> instead.`);
|
|
6826
6772
|
return {
|
|
6827
6773
|
...report,
|
|
6828
6774
|
deployHead: report.deployHead
|
|
@@ -6834,11 +6780,12 @@ async function fetchRerunReport(hubCtx, profile) {
|
|
|
6834
6780
|
* means the hub does not serve this route at all.
|
|
6835
6781
|
*/
|
|
6836
6782
|
function explainNotFound(hubCtx, err) {
|
|
6837
|
-
if (err.code === "no_perspectives") return `--only-
|
|
6838
|
-
return `--only-
|
|
6783
|
+
if (err.code === "no_perspectives") return `--only-hub-rerun-needed: project "${hubCtx.project}" has no perspectives document on the hub, so no spec is registered to compare against a deploy. Run \`ccqa perspectives\` first.`;
|
|
6784
|
+
return `--only-hub-rerun-needed: this hub does not serve re-run verdicts — it needs ccqa ${RERUN_MIN_HUB_VERSION} or newer. Upgrade the hub, or select with --only-affected-by <ref> instead.`;
|
|
6839
6785
|
}
|
|
6840
6786
|
/** States the summary line reports, worst-known-first. */
|
|
6841
6787
|
const SUMMARY_ORDER = [
|
|
6788
|
+
"blocked",
|
|
6842
6789
|
"needed",
|
|
6843
6790
|
"unknown",
|
|
6844
6791
|
"neverRun",
|
|
@@ -6856,7 +6803,7 @@ const UNANSWERABLE = new Set([
|
|
|
6856
6803
|
*
|
|
6857
6804
|
* `needed` is always selected. `unknown` and `neverRun` are "the question
|
|
6858
6805
|
* cannot be answered", so they are excluded by default and opted into with
|
|
6859
|
-
* `--
|
|
6806
|
+
* `--only-hub-rerun-needed-with-unknown` — fail-open on request, never silently. `notNeeded` and
|
|
6860
6807
|
* `notEvaluated` are never selected.
|
|
6861
6808
|
*/
|
|
6862
6809
|
function selectSpecsNeedingRerun(specs, report, opts) {
|
|
@@ -7634,7 +7581,7 @@ function safeOriginPath(href) {
|
|
|
7634
7581
|
*
|
|
7635
7582
|
* Two kinds share one namespace:
|
|
7636
7583
|
* - "guidance": the record/live prompt bundle — `.user.md` (human-maintained)
|
|
7637
|
-
* and `.agent.md` (auto-rewritten by `ccqa run --
|
|
7584
|
+
* and `.agent.md` (auto-rewritten by `ccqa run --learn-hub-live-prompt`) —
|
|
7638
7585
|
* plus `triage.user`, the human-maintained guidance injected into the
|
|
7639
7586
|
* failure-analysis (triage) prompt.
|
|
7640
7587
|
* - "custom-prompt": `analysis-custom-prompt` — Claude-written calibration guidance
|
|
@@ -8039,11 +7986,19 @@ z.record(z.string(), SpecTouchSchema);
|
|
|
8039
7986
|
const RerunStateSchema = z.enum([
|
|
8040
7987
|
"needed",
|
|
8041
7988
|
"notNeeded",
|
|
7989
|
+
"blocked",
|
|
8042
7990
|
"unknown",
|
|
8043
7991
|
"neverRun",
|
|
8044
7992
|
"notEvaluated"
|
|
8045
7993
|
]);
|
|
8046
7994
|
/**
|
|
7995
|
+
* Why a spec is `blocked`. Always carried: the two answers differ in who
|
|
7996
|
+
* repairs them and how long that takes — a stale recording is re-recorded
|
|
7997
|
+
* automatically within minutes, a changed spec waits for a human — so a view
|
|
7998
|
+
* that showed only "blocked" would hide the distinction that matters most.
|
|
7999
|
+
*/
|
|
8000
|
+
const RerunBlockedReasonSchema = z.enum(["testDrift", "specChange"]);
|
|
8001
|
+
/**
|
|
8047
8002
|
* Why a spec is `unknown`. Always carried, so the view can name the missing
|
|
8048
8003
|
* input ("no deploy log for this profile") instead of shrugging. `unknown` is
|
|
8049
8004
|
* never rendered as "not needed".
|
|
@@ -8072,6 +8027,7 @@ const DeployRefSchema = z.object({
|
|
|
8072
8027
|
const SpecRerunSchema = z.object({
|
|
8073
8028
|
state: RerunStateSchema,
|
|
8074
8029
|
reason: RerunUnknownReasonSchema.optional(),
|
|
8030
|
+
blockedReason: RerunBlockedReasonSchema.optional(),
|
|
8075
8031
|
lastRun: SpecLedgerEntrySchema.nullable(),
|
|
8076
8032
|
lastGreen: SpecLedgerEntrySchema.nullable(),
|
|
8077
8033
|
lastRed: SpecLedgerEntrySchema.nullable(),
|
|
@@ -8103,7 +8059,7 @@ z.object({
|
|
|
8103
8059
|
lastRed: z.record(z.string(), SpecLedgerEntrySchema).default({})
|
|
8104
8060
|
});
|
|
8105
8061
|
/**
|
|
8106
|
-
* One spec's last drift audit, as recorded by `ccqa
|
|
8062
|
+
* One spec's last drift audit, as recorded by `ccqa audit --report-to-hub`. Unlike the
|
|
8107
8063
|
* spec ledger above, this carries no profile: drift asks whether a spec still
|
|
8108
8064
|
* describes the code, which has nothing to do with which environment is
|
|
8109
8065
|
* running it (ADR-0010 draws the same line for "needs re-run").
|
|
@@ -8313,7 +8269,7 @@ function isCcqaPath(path) {
|
|
|
8313
8269
|
}
|
|
8314
8270
|
/**
|
|
8315
8271
|
* A malformed reply costs the whole selection, so it is worth one more call
|
|
8316
|
-
* before giving up. `ccqa
|
|
8272
|
+
* before giving up. `ccqa audit` retries per spec for the same reason; this
|
|
8317
8273
|
* call carries every undecided spec at once, so the blast radius is larger,
|
|
8318
8274
|
* not smaller. Observed in practice: three runs over one commit produced a
|
|
8319
8275
|
* parse failure, a clean answer, and a different clean answer.
|
|
@@ -8510,7 +8466,7 @@ const sessionCaptureCommand = new Command("capture").description("Open a headed
|
|
|
8510
8466
|
error(err.message);
|
|
8511
8467
|
process.exit(2);
|
|
8512
8468
|
}
|
|
8513
|
-
header("session
|
|
8469
|
+
header("session capture", name);
|
|
8514
8470
|
meta("project", project);
|
|
8515
8471
|
meta("profile", opts.profile ?? "default");
|
|
8516
8472
|
blank();
|
|
@@ -8631,7 +8587,7 @@ const sessionPush = new Command("push").description("Upload a locally-saved brow
|
|
|
8631
8587
|
state = await loadStorageState(path);
|
|
8632
8588
|
} catch (err) {
|
|
8633
8589
|
error(`could not read session "${name}" at ${path}: ${err instanceof Error ? err.message : String(err)}`);
|
|
8634
|
-
hint(`create it first with: ccqa session
|
|
8590
|
+
hint(`create it first with: ccqa hub session capture ${name}${opts.profile ? ` --profile ${opts.profile}` : ""}`);
|
|
8635
8591
|
process.exit(2);
|
|
8636
8592
|
}
|
|
8637
8593
|
await connect(opts).putSession(project, profile, name, state);
|
|
@@ -8718,12 +8674,12 @@ const promptPush = new Command("push").description("Upload a locally-generated p
|
|
|
8718
8674
|
body = await readFile(path, "utf8");
|
|
8719
8675
|
} catch (err) {
|
|
8720
8676
|
error(`could not read prompt "${name}" at ${path}: ${err instanceof Error ? err.message : String(err)}`);
|
|
8721
|
-
hint("nothing to push; generate it first (e.g. ccqa run --
|
|
8677
|
+
hint("nothing to push; generate it first (e.g. ccqa run --learn-hub-live-prompt)");
|
|
8722
8678
|
process.exit(2);
|
|
8723
8679
|
}
|
|
8724
8680
|
if (body.trim().length === 0) {
|
|
8725
8681
|
error(`prompt "${name}" at ${path} is empty`);
|
|
8726
|
-
hint("nothing to push; generate it first (e.g. ccqa run --
|
|
8682
|
+
hint("nothing to push; generate it first (e.g. ccqa run --learn-hub-live-prompt)");
|
|
8727
8683
|
process.exit(2);
|
|
8728
8684
|
}
|
|
8729
8685
|
await connect(opts).putPrompt(project, name, body);
|
|
@@ -8749,7 +8705,7 @@ const promptRm = new Command("rm").description("Delete a prompt from the hub.").
|
|
|
8749
8705
|
info(`deleted prompt "${name}" from the hub`);
|
|
8750
8706
|
}));
|
|
8751
8707
|
const promptCommand = new Command("prompt").description("Manage prompt assets (per-flow user/agent guidance, triage user guidance, analysis custom prompt) stored on the hub (fetched automatically by `ccqa run` at run time).").addCommand(promptPush).addCommand(promptLs).addCommand(promptRm);
|
|
8752
|
-
const deployRecord = new Command("record").description("Tell the hub what a deploy shipped, so it can answer which specs need a re-run (`ccqa run --
|
|
8708
|
+
const deployRecord = new Command("record").description("Tell the hub what a deploy shipped, so it can answer which specs need a re-run (`ccqa run --only-hub-rerun-needed`). Run this from the deploy job, after the deploy succeeds. The changed paths are computed locally with a two-dot `git diff <previous> <sha>`; a job that has only curl and git can POST the same body directly (see docs/hub.md).").requiredOption("--profile <name>", "Environment this deploy landed in (e.g. 'stg'). Required: dev and stg sit at different commits, so the deploy log is per-profile.").requiredOption("--sha <sha>", "Commit that was deployed.").option("--previous <sha>", "Commit this deploy replaced. Defaults to the profile's current deploy-log head on the hub. With neither, there's nothing to diff against: changedPaths is unset and --select is skipped.").option("--ref <ref>", "Ref that was deployed (branch or tag). Recorded for display only.").option("--select", "Also decide which specs this deploy reaches (`ccqa select-specs`) and send the verdict with it. Without it the deploy is a hole in the range: specs behind it report 'unknown' rather than 'not needed'.").option("-m, --model <name>", "Model for --select. Claude alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option(...hubUrlOption).option(...hubTokenOption).option("--project <name>", "Project whose deploy log this entry joins. Defaults to the current directory's name.").option("--cwd <path>", "Directory the git diff and the default --project name are resolved against.").action(withHubErrors(async (opts) => {
|
|
8753
8709
|
const cwd = resolveCwd(opts.cwd);
|
|
8754
8710
|
const project = resolveProject(opts);
|
|
8755
8711
|
const hub = connect(opts);
|
|
@@ -8828,7 +8784,7 @@ function describeSelection(selection, diffAvailable) {
|
|
|
8828
8784
|
const values = Object.values(selection);
|
|
8829
8785
|
return `${values.filter((s) => s.verdict === "needed").length} needed / ${values.filter((s) => s.verdict === "unknown").length} unknown / ${values.length} specs`;
|
|
8830
8786
|
}
|
|
8831
|
-
const deployCommand = new Command("deploy").description("Report deploys to the hub, the input behind `ccqa run --
|
|
8787
|
+
const deployCommand = new Command("deploy").description("Report deploys to the hub, the input behind `ccqa run --only-hub-rerun-needed`.").addCommand(deployRecord);
|
|
8832
8788
|
const pushCommand = new Command("push").description("Upload the report directory of a finished `ccqa run --report` to the hub as a run. Run this after `ccqa run` (use `if: always()` in CI so failing runs are pushed too).").option("--report-dir <dir>", `Report directory to push. Default: ${DEFAULT_REPORT_DIR}/`).option("--project <name>", "Logical project name for the run. Defaults to the current directory's name.").option("--branch <name>", "Branch label. Defaults to $GITHUB_HEAD_REF / $GITHUB_REF_NAME / current git branch.").option("--profile <name>", "Profile (environment) the run executed against. Recorded for display; runs are not scoped by profile.").option(...hubUrlOption).option(...hubTokenOption).option("--cwd <path>", "Directory the report dir is resolved against (defaults to the current directory).").action(withHubErrors(async (opts) => {
|
|
8833
8789
|
const cwd = resolveCwd(opts.cwd);
|
|
8834
8790
|
const reportDir = join(cwd, opts.reportDir ?? "ccqa-report");
|
|
@@ -8971,7 +8927,7 @@ function generateLiveSessionName() {
|
|
|
8971
8927
|
* Project-specific guidance ("the admin tenant is foo.example", "session
|
|
8972
8928
|
* times out at X minutes", …) is appended from
|
|
8973
8929
|
* `.ccqa/prompts/live.user.md` (human-maintained) and
|
|
8974
|
-
* `.ccqa/prompts/live.agent.md` (updated by `ccqa run --
|
|
8930
|
+
* `.ccqa/prompts/live.agent.md` (updated by `ccqa run --learn-hub-live-prompt`)
|
|
8975
8931
|
* by the caller, so ccqa stays clean of downstream-product context.
|
|
8976
8932
|
*
|
|
8977
8933
|
* Constraint posture: `ccqa record` (trace) enforces a strict selector
|
|
@@ -9726,7 +9682,7 @@ const verifiedSessions = /* @__PURE__ */ new Set();
|
|
|
9726
9682
|
* each named session from the hub (`.ccqa/sessions/*.json` is no longer
|
|
9727
9683
|
* read here). Every name must load as a valid agent-browser state (the spec
|
|
9728
9684
|
* assumes it starts signed-in); a missing/malformed session fails with a
|
|
9729
|
-
* `ccqa session
|
|
9685
|
+
* `ccqa hub session capture` hint instead of running unauthenticated.
|
|
9730
9686
|
*
|
|
9731
9687
|
* If a session carries an embedded verify URL (bootstrap saved it), the
|
|
9732
9688
|
* restore is health-checked before the run starts, so an expired/unusable
|
|
@@ -9769,7 +9725,7 @@ async function resolveSessionState(names, hubCtx, profile, verify = verifySessio
|
|
|
9769
9725
|
if (!check.restored) return {
|
|
9770
9726
|
ok: false,
|
|
9771
9727
|
error: `session '${name}' did not restore to a signed-in page — ${check.reason}`,
|
|
9772
|
-
hint: `re-bootstrap it: ccqa session
|
|
9728
|
+
hint: `re-bootstrap it: ccqa hub session capture ${name}${profileFlag}`
|
|
9773
9729
|
};
|
|
9774
9730
|
verifiedSessions.add(memoKey);
|
|
9775
9731
|
}
|
|
@@ -9779,7 +9735,7 @@ async function resolveSessionState(names, hubCtx, profile, verify = verifySessio
|
|
|
9779
9735
|
if (broken.length > 0) return {
|
|
9780
9736
|
ok: false,
|
|
9781
9737
|
error: `session not usable on the hub: ${broken.join(", ")}`,
|
|
9782
|
-
hint: `create it with: ${broken.map((name) => `ccqa session
|
|
9738
|
+
hint: `create it with: ${broken.map((name) => `ccqa hub session capture ${name}${profileFlag}`).join(" · ")}`
|
|
9783
9739
|
};
|
|
9784
9740
|
const statePath = await writeMergedTempState(mergeStorageStates(loaded));
|
|
9785
9741
|
return {
|
|
@@ -11652,7 +11608,7 @@ async function groupSpecsByTarget(specs, config, cwd, resolve = resolveTarget) {
|
|
|
11652
11608
|
* then each external target group through its runner. Every row is upserted
|
|
11653
11609
|
* into the incremental report the moment it exists — the runner reports each
|
|
11654
11610
|
* spec through `onSpecComplete` as it finishes — so an interrupt keeps what
|
|
11655
|
-
* already ran and `--
|
|
11611
|
+
* already ran and `--report-to-hub` streams spec by spec. Rows are also
|
|
11656
11612
|
* returned for the tail phase (failure analysis) and the final batch write. A
|
|
11657
11613
|
* crashing runner marks its own specs failed instead of aborting the run.
|
|
11658
11614
|
*/
|
|
@@ -11788,7 +11744,7 @@ function createIncrementalReport(reportDir, envelope, sink) {
|
|
|
11788
11744
|
//#endregion
|
|
11789
11745
|
//#region src/prompts/agent-update.ts
|
|
11790
11746
|
/**
|
|
11791
|
-
* Build the prompts used by `--
|
|
11747
|
+
* Build the prompts used by the `--learn-*-prompt` flags to refresh
|
|
11792
11748
|
* `.ccqa/prompts/<kind>.agent.md` after a run:
|
|
11793
11749
|
* - `ccqa run` (live) → `live.agent`
|
|
11794
11750
|
* - `ccqa record` (trace) → `record.agent`
|
|
@@ -12044,10 +12000,7 @@ async function updateAgentPrompt(args) {
|
|
|
12044
12000
|
warn(`${flag} skipped (${auth.reason})`);
|
|
12045
12001
|
return;
|
|
12046
12002
|
}
|
|
12047
|
-
if (!hubContext) {
|
|
12048
|
-
warn(`${flag} skipped (hub connection required; pass --hub-url/--hub-token or set CCQA_HUB_URL/CCQA_HUB_TOKEN)`);
|
|
12049
|
-
return;
|
|
12050
|
-
}
|
|
12003
|
+
if (!hubContext) throw new Error(`${flag} requires a hub connection (--hub-url/--hub-token or CCQA_HUB_URL/CCQA_HUB_TOKEN)`);
|
|
12051
12004
|
const { hub, project } = hubContext;
|
|
12052
12005
|
const promptName = `${kind}.agent`;
|
|
12053
12006
|
try {
|
|
@@ -12169,6 +12122,19 @@ async function resolveVitestConfig(cwd) {
|
|
|
12169
12122
|
function resolveReportDir(reportDir, cwd) {
|
|
12170
12123
|
return resolve(cwd, reportDir ?? "ccqa-report");
|
|
12171
12124
|
}
|
|
12125
|
+
/**
|
|
12126
|
+
* Turn a hub transport failure into a usage error, as a `.catch` so the
|
|
12127
|
+
* tuple types of the `Promise.all` it guards survive.
|
|
12128
|
+
*
|
|
12129
|
+
* Without it the raw `fetch failed` escapes as an unhandled rejection: a stack
|
|
12130
|
+
* trace and exit 1, where the user needs "the hub is unreachable" and exit 2.
|
|
12131
|
+
* Errors the callers already shaped pass through — they say more than this
|
|
12132
|
+
* wrapper could.
|
|
12133
|
+
*/
|
|
12134
|
+
function asHubReadError(err) {
|
|
12135
|
+
if (err instanceof RunUsageError) throw err;
|
|
12136
|
+
throw new RunUsageError(`could not read from the hub: ${errMessage(err)}`);
|
|
12137
|
+
}
|
|
12172
12138
|
/** De-dupe by `featureName/specName`, keeping first-seen order. */
|
|
12173
12139
|
function dedupeSpecs(specs) {
|
|
12174
12140
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -12189,10 +12155,10 @@ function dedupeSpecs(specs) {
|
|
|
12189
12155
|
* maps it to `process.exit(2)`).
|
|
12190
12156
|
*/
|
|
12191
12157
|
async function executeRun(targets, opts) {
|
|
12192
|
-
const filtering = Boolean(opts.onlyAffectedBy || opts.
|
|
12158
|
+
const filtering = Boolean(opts.onlyAffectedBy || opts.onlyHubRerunNeeded);
|
|
12193
12159
|
if (filtering && targets.length > 0) throw new RunUsageError("a --only-* filter and an explicit spec target cannot be combined");
|
|
12194
|
-
const rerunProfile = opts.
|
|
12195
|
-
if (opts.
|
|
12160
|
+
const rerunProfile = opts.onlyHubRerunNeeded === true ? requireRerunProfile(opts.hubProfile) : null;
|
|
12161
|
+
if (opts.onlyHubRerunNeededWithUnknown && rerunProfile === null) warn("--only-hub-rerun-needed-with-unknown is ignored: it only applies to --only-hub-rerun-needed");
|
|
12196
12162
|
const forExecution = opts.dryRun !== true;
|
|
12197
12163
|
const cwd = opts.cwd ?? process.cwd();
|
|
12198
12164
|
const wantsLastGreen = opts.onFailExplain === true && opts.onFailExplainBase === void 0;
|
|
@@ -12217,8 +12183,8 @@ async function executeRun(targets, opts) {
|
|
|
12217
12183
|
meta("analysis-base", `${fixedBase.ref} (${fixedBase.sha.slice(0, 12)}, ${fixedBase.source})`);
|
|
12218
12184
|
}
|
|
12219
12185
|
if (forExecution) try {
|
|
12220
|
-
if (opts.
|
|
12221
|
-
profile: opts.
|
|
12186
|
+
if (opts.hubProfile !== void 0) await resolveProfileEnv({
|
|
12187
|
+
profile: opts.hubProfile,
|
|
12222
12188
|
project: resolveProjectOrThrow(opts.project, cwd),
|
|
12223
12189
|
cwd,
|
|
12224
12190
|
hubUrl: opts.hubUrl,
|
|
@@ -12234,7 +12200,7 @@ async function executeRun(targets, opts) {
|
|
|
12234
12200
|
if (err instanceof RunUsageError) throw err;
|
|
12235
12201
|
if (err instanceof ProjectNameError) throw new RunUsageError(err.message);
|
|
12236
12202
|
if (err instanceof HubConnectionError || err instanceof HubApiError) throw new RunUsageError(err.message);
|
|
12237
|
-
throw new RunUsageError(`failed to load profile "${opts.
|
|
12203
|
+
throw new RunUsageError(`failed to load profile "${opts.hubProfile}": ${errMessage(err)}`);
|
|
12238
12204
|
}
|
|
12239
12205
|
let hubCtx = null;
|
|
12240
12206
|
try {
|
|
@@ -12250,16 +12216,16 @@ async function executeRun(targets, opts) {
|
|
|
12250
12216
|
}
|
|
12251
12217
|
if (wantsLastGreen && hubCtx == null) throw new RunUsageError("--on-fail-explain needs a hub connection for the per-spec last-green baselines (--hub-url/--hub-token or CCQA_HUB_URL/CCQA_HUB_TOKEN), or an explicit --on-fail-explain-base <ref>");
|
|
12252
12218
|
const ledgerHub = wantsLastGreen ? hubCtx : null;
|
|
12253
|
-
if (rerunProfile !== null && hubCtx == null) throw new RunUsageError("--only-
|
|
12254
|
-
if (opts.
|
|
12255
|
-
|
|
12219
|
+
if (rerunProfile !== null && hubCtx == null) throw new RunUsageError("--only-hub-rerun-needed requires a hub connection (--hub-url/--hub-token or CCQA_HUB_URL/CCQA_HUB_TOKEN)");
|
|
12220
|
+
if (opts.reportToHub && hubCtx == null) throw new RunUsageError("--report-to-hub requires a hub connection (--hub-url/--hub-token or CCQA_HUB_URL/CCQA_HUB_TOKEN)");
|
|
12221
|
+
if (opts.learnHubLivePrompt && hubCtx == null) throw new RunUsageError("--learn-hub-live-prompt requires a hub connection (--hub-url/--hub-token or CCQA_HUB_URL/CCQA_HUB_TOKEN)");
|
|
12222
|
+
const [customPrompt, triageUserPrompt, ledgerEntries, rerunReport, fetchedDeployHead] = await Promise.all([
|
|
12256
12223
|
forExecution ? fetchCustomPrompt(hubCtx) : null,
|
|
12257
12224
|
forExecution ? fetchTriageUserPrompt(hubCtx) : null,
|
|
12258
|
-
forExecution && ledgerHub ? fetchLastGreenLedger(ledgerHub, opts.
|
|
12225
|
+
forExecution && ledgerHub ? fetchLastGreenLedger(ledgerHub, opts.hubProfile, cwd) : null,
|
|
12259
12226
|
rerunProfile !== null && hubCtx ? fetchRerunReport(hubCtx, rerunProfile) : null,
|
|
12260
|
-
forExecution && hubCtx && opts.
|
|
12261
|
-
|
|
12262
|
-
]);
|
|
12227
|
+
forExecution && hubCtx && opts.hubProfile && rerunProfile === null ? tryDeployHeadSha(hubCtx, opts.hubProfile) : null
|
|
12228
|
+
]).catch(asHubReadError);
|
|
12263
12229
|
const deployedSha = rerunReport?.deployHead.sha ?? fetchedDeployHead;
|
|
12264
12230
|
if (ledgerEntries) diffProvider = createDiffProvider({
|
|
12265
12231
|
resolveBase: createLastGreenResolver(ledgerEntries, cwd),
|
|
@@ -12286,24 +12252,19 @@ async function executeRun(targets, opts) {
|
|
|
12286
12252
|
const before = specs.length;
|
|
12287
12253
|
let unanswerable = 0;
|
|
12288
12254
|
if (rerunReport) {
|
|
12289
|
-
const selection = selectSpecsNeedingRerun(specs, rerunReport, { includeUnknown: opts.
|
|
12255
|
+
const selection = selectSpecsNeedingRerun(specs, rerunReport, { includeUnknown: opts.onlyHubRerunNeededWithUnknown === true });
|
|
12290
12256
|
specs = selection.selected;
|
|
12291
12257
|
unanswerable = selection.excludedUnanswerable;
|
|
12292
12258
|
meta("stale-base", `deploy ${rerunReport.deployHead.sha.slice(0, 12)} (profile ${rerunReport.profile})`);
|
|
12293
12259
|
meta("stale-states", selection.summary);
|
|
12294
12260
|
}
|
|
12295
|
-
if (auditedLedger) {
|
|
12296
|
-
const picked = selectAuditedClean(specs, auditedLedger);
|
|
12297
|
-
specs = picked.selected;
|
|
12298
|
-
meta("audit-states", `${picked.selected.length} clean, ${picked.drifted} drifted, ${picked.unaudited} never audited`);
|
|
12299
|
-
}
|
|
12300
12261
|
if (opts.onlyAffectedBy) specs = (await collectChangedSpecs(specs, {
|
|
12301
12262
|
cwd,
|
|
12302
12263
|
base: opts.onlyAffectedBy,
|
|
12303
12264
|
...opts.model ? { model: opts.model } : {}
|
|
12304
12265
|
})).specs;
|
|
12305
12266
|
meta("selected", `${specs.length} of ${before} spec${before === 1 ? "" : "s"}`);
|
|
12306
|
-
if (specs.length === 0 && unanswerable > 0) hint(`${unanswerable} spec(s) were excluded because the hub could not tell whether they need a re-run; pass --only-
|
|
12267
|
+
if (specs.length === 0 && unanswerable > 0) hint(`${unanswerable} spec(s) were excluded because the hub could not tell whether they need a re-run; pass --only-hub-rerun-needed-with-unknown to run them anyway`);
|
|
12307
12268
|
}
|
|
12308
12269
|
if (specs.length === 0) {
|
|
12309
12270
|
warn("no specs to run");
|
|
@@ -12328,7 +12289,7 @@ async function executeRun(targets, opts) {
|
|
|
12328
12289
|
const why = "it only applies to agent-browser 'mode: live' specs, and this run has none";
|
|
12329
12290
|
if (typeof opts.liveStepRetry === "number" && opts.liveStepRetry > 0) warn(`--live-step-retry is ignored: ${why}`);
|
|
12330
12291
|
if (opts.liveArtifactsDir) warn(`--live-artifacts-dir is ignored: ${why}`);
|
|
12331
|
-
if (opts.
|
|
12292
|
+
if (opts.learnHubLivePrompt) warn(`--learn-live-prompt is ignored: ${why}`);
|
|
12332
12293
|
} else if (opts.liveArtifactsDir && liveSpecs.length > 1) warn("--out is ignored when running multiple live specs");
|
|
12333
12294
|
if (detSpecs.length === 0 && opts.replaySkipEvidence === true) warn("--no-evidence is ignored: it only applies to agent-browser 'mode: deterministic' specs, and this run has none");
|
|
12334
12295
|
blank();
|
|
@@ -12343,7 +12304,6 @@ async function executeRun(targets, opts) {
|
|
|
12343
12304
|
};
|
|
12344
12305
|
}
|
|
12345
12306
|
const det = await runDeterministicSpecs(detSpecs, opts, cwd, reportDir);
|
|
12346
|
-
if (opts.reportToHub && hubCtx == null) warn("--push-report requires --hub-url/--hub-token (or CCQA_HUB_URL/CCQA_HUB_TOKEN); skipping push");
|
|
12347
12307
|
let hubRunId = null;
|
|
12348
12308
|
let hubSink;
|
|
12349
12309
|
if (hubCtx != null && opts.reportToHub) try {
|
|
@@ -12353,7 +12313,7 @@ async function executeRun(targets, opts) {
|
|
|
12353
12313
|
const opened = await hubCtx.hub.openRun({
|
|
12354
12314
|
project: hubCtx.project,
|
|
12355
12315
|
...branch ? { branch } : {},
|
|
12356
|
-
...opts.
|
|
12316
|
+
...opts.hubProfile ? { profile: opts.hubProfile } : {},
|
|
12357
12317
|
...git.head ? { gitHead: git.head } : {},
|
|
12358
12318
|
...deployedSha ? { deployedSha } : {},
|
|
12359
12319
|
...ciRunId ? { ciRunId } : {},
|
|
@@ -12375,7 +12335,7 @@ async function executeRun(targets, opts) {
|
|
|
12375
12335
|
}
|
|
12376
12336
|
} };
|
|
12377
12337
|
} catch (err) {
|
|
12378
|
-
|
|
12338
|
+
throw new RunUsageError(`--report-to-hub: could not open a run on the hub (${errMessage(err)})`);
|
|
12379
12339
|
}
|
|
12380
12340
|
const incrementalReport = createIncrementalReport(reportDir, buildReportEnvelope({
|
|
12381
12341
|
git,
|
|
@@ -12414,7 +12374,7 @@ async function executeRun(targets, opts) {
|
|
|
12414
12374
|
reportDir,
|
|
12415
12375
|
...typeof opts.liveStepRetry === "number" ? { retry: opts.liveStepRetry } : {},
|
|
12416
12376
|
concurrency: opts.concurrency ?? 1,
|
|
12417
|
-
...opts.
|
|
12377
|
+
...opts.hubProfile ? { profile: opts.hubProfile } : {},
|
|
12418
12378
|
diffProvider,
|
|
12419
12379
|
hubContext: hubCtx,
|
|
12420
12380
|
customPrompt,
|
|
@@ -12476,7 +12436,7 @@ async function executeRun(targets, opts) {
|
|
|
12476
12436
|
}
|
|
12477
12437
|
}
|
|
12478
12438
|
}
|
|
12479
|
-
if (opts.
|
|
12439
|
+
if (opts.learnHubLivePrompt && liveSpecs.length > 0) {
|
|
12480
12440
|
blank();
|
|
12481
12441
|
await updateAgentPrompt({
|
|
12482
12442
|
kind: "live",
|
|
@@ -12789,7 +12749,7 @@ const PATCH_FILES_RAW_BUDGET = 20 * 1024 * 1024;
|
|
|
12789
12749
|
/**
|
|
12790
12750
|
* Add one row's file assets to `acc` as `{ reportDir-relative posix path →
|
|
12791
12751
|
* base64 }`. Every kind of screenshot a row can carry has to be collected here,
|
|
12792
|
-
* or `--
|
|
12752
|
+
* or `--report-to-hub` — the way CI publishes — silently ships a report whose
|
|
12793
12753
|
* images 404 on the hub: a live row's per-step PNGs
|
|
12794
12754
|
* (`liveRun.steps[].beforePng/afterPng`), a script-driven row's step evidence
|
|
12795
12755
|
* (`evidence[].pngPath` / `beforePngPath`, written by agent-browser replays and
|
|
@@ -13018,14 +12978,14 @@ function installTeardownSignalHandlers(teardown) {
|
|
|
13018
12978
|
}
|
|
13019
12979
|
//#endregion
|
|
13020
12980
|
//#region src/cli/run.ts
|
|
13021
|
-
const runCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("run").argument("[targets...]", "Specs to run, space-separated: each '<feature>/<spec>', '<feature>', or omit for all. Duplicates are de-duped.").description("Run specs, on any target. Agent-browser specs replay the recorded test.spec.ts under vitest (default), or, with spec.yaml `mode: live`, have Claude drive agent-browser live per step. External-target specs (playwright, runn) run through the target's configured `runCommand`. A structured report (report.json + evidence) is always written; use --report-to-hub to also stream it to a hub.").optionsGroup("Which specs to run:").option("--only-affected-by <ref>", "Only specs `ccqa select-specs` judges reached by the diff against <ref> (e.g. origin/main). In pull_request CI, pass $GITHUB_BASE_REF. Cannot be combined with an explicit spec id.").option("--only-
|
|
12981
|
+
const runCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("run").argument("[targets...]", "Specs to run, space-separated: each '<feature>/<spec>', '<feature>', or omit for all. Duplicates are de-duped.").description("Run specs, on any target. Agent-browser specs replay the recorded test.spec.ts under vitest (default), or, with spec.yaml `mode: live`, have Claude drive agent-browser live per step. External-target specs (playwright, runn) run through the target's configured `runCommand`. A structured report (report.json + evidence) is always written; use --report-to-hub to also stream it to a hub.").optionsGroup("Which specs to run:").option("--only-affected-by <ref>", "Only specs `ccqa select-specs` judges reached by the diff against <ref> (e.g. origin/main). In pull_request CI, pass $GITHUB_BASE_REF. Cannot be combined with an explicit spec id.").option("--only-hub-rerun-needed", "Only specs the hub answers `needed` for: their last result no longer covers what is deployed. Specs the audit rejected answer `blocked` and are never taken — a run cannot repair a spec. No git diff involved. Requires a hub connection and --hub-profile.").option("--only-hub-rerun-needed-with-unknown", "With --only-hub-rerun-needed: also take specs whose re-run need the hub cannot answer ('unknown') and specs that never ran ('neverRun'). Off by default: an unanswerable question is reported, not guessed.").option("--dry-run", "Print the specs this invocation would run, then exit 0 without executing anything and without writing a report. Works with every selection flag.").optionsGroup("How to run them:").option("--concurrency <n>", "Run up to N specs in parallel within each phase (deterministic / external-target / live), never across phases. Default 1 (sequential). Live specs each get an isolated agent-browser session; high values spawn many headed Chrome instances.", parseConcurrency$1, 1).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--live-step-retry <n>", "(live only) Retry each failed step up to N more times before recording failure. This retries a step, not the whole spec — see --on-fail-explain-rerun for that.", (raw) => {
|
|
13022
12982
|
const n = Number(raw);
|
|
13023
12983
|
if (!Number.isFinite(n) || n < 0 || Math.floor(n) !== n) throw new Error(`--live-step-retry must be a non-negative integer, got "${raw}"`);
|
|
13024
12984
|
return n;
|
|
13025
12985
|
}, 0).option("--live-artifacts-dir <dir>", "(live only) Override the per-spec artifact directory. Default: <specDir>/runs/<runId>. Ignored when running multiple specs.").option("--replay-skip-evidence", `(deterministic replay only) Skip step-boundary evidence capture (PNG + meta JSON written to ${DEFAULT_REPORT_DIR}/${EVIDENCE_SUBDIR}/ by default).`).optionsGroup("What to do about failures:").option("--on-fail-explain", "Classify each failure against the source diff since the commit where that spec last passed (per-spec baselines from the hub). Off by default — no Claude calls without it.").option("--on-fail-explain-base <ref>", "With --on-fail-explain: diff against <ref> instead of each spec's last green. Use when there is no hub to hold the baselines.").optionsGroup("What to do with the results:").option("--report-dir <dir>", `Directory for the structured run results (report.json + evidence PNGs), which are always written. Default: ${DEFAULT_REPORT_DIR}/.`).option("--report-format <fmt>", "Additional output format alongside HTML: 'text' (default), 'json' (writes report.json), 'github' (GitHub Actions annotations on stdout).", (raw) => {
|
|
13026
12986
|
if (REPORT_FORMATS.includes(raw)) return raw;
|
|
13027
12987
|
throw new Error(`--report-format must be one of ${REPORT_FORMATS.join(" | ")}`);
|
|
13028
|
-
}, "text").option("--report-to-hub", "Incrementally push the run report to the hub as the run progresses (open → patch per spec → finalize). Requires --hub-url/--hub-token (or CCQA_HUB_URL/CCQA_HUB_TOKEN). Without it, hub credentials are used only to fetch variables/sessions/prompts, not to push.").optionsGroup("Learning:").option("--learn-live-prompt", "(live only) After the run finishes, ask Claude to refresh the \"live.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(async (targets, opts) => {
|
|
12988
|
+
}, "text").option("--report-to-hub", "Incrementally push the run report to the hub as the run progresses (open → patch per spec → finalize). Requires --hub-url/--hub-token (or CCQA_HUB_URL/CCQA_HUB_TOKEN). Without it, hub credentials are used only to fetch variables/sessions/prompts, not to push.").optionsGroup("Learning:").option("--learn-hub-live-prompt", "(live only) After the run finishes, ask Claude to refresh the \"live.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(async (targets, opts) => {
|
|
13029
12989
|
await runCliAction(targets, opts);
|
|
13030
12990
|
});
|
|
13031
12991
|
/** Parse --concurrency: a positive integer. Rejects 0, negatives, non-integers. */
|
|
@@ -13041,11 +13001,7 @@ function parseConcurrency$1(raw) {
|
|
|
13041
13001
|
function headerTarget(targets, opts) {
|
|
13042
13002
|
if (targets.length === 1) return targets[0];
|
|
13043
13003
|
if (targets.length > 1) return `${targets.length} targets`;
|
|
13044
|
-
const filters = [
|
|
13045
|
-
opts.onlyAffectedBy ? "affected" : null,
|
|
13046
|
-
opts.onlyStale ? "stale" : null,
|
|
13047
|
-
opts.onlyAuditedClean ? "audited clean" : null
|
|
13048
|
-
].filter((s) => s !== null);
|
|
13004
|
+
const filters = [opts.onlyAffectedBy ? "affected" : null, opts.onlyHubRerunNeeded ? "needs re-run" : null].filter((s) => s !== null);
|
|
13049
13005
|
return filters.length === 0 ? "(all specs)" : `(${filters.join(" + ")})`;
|
|
13050
13006
|
}
|
|
13051
13007
|
/**
|
|
@@ -15142,21 +15098,21 @@ async function runGenerateLocked(featureName, specName, opts, cwd) {
|
|
|
15142
15098
|
hint(`run 'ccqa run ${featureName}/${specName}' to execute the test`);
|
|
15143
15099
|
}
|
|
15144
15100
|
/**
|
|
15145
|
-
* `ccqa generate --
|
|
15101
|
+
* `ccqa generate --learn-hub-codegen-prompt`: refresh the target's learned
|
|
15146
15102
|
* `<target>.agent` playbook from this generation. Only targets that declare a
|
|
15147
15103
|
* `guidanceKind` (the LLM-generating ones: playwright, runn) have such a
|
|
15148
15104
|
* prompt — agent-browser's codegen is mechanical, so point at `ccqa record
|
|
15149
|
-
* --
|
|
15105
|
+
* --learn-hub-trace-prompt` for its tracer instead.
|
|
15150
15106
|
*/
|
|
15151
15107
|
async function runGenerateAgentPromptUpdate(target, featureName, specName, result, opts, cwd) {
|
|
15152
15108
|
if (target.guidanceKind === void 0) {
|
|
15153
|
-
warn(`--
|
|
15109
|
+
warn(`--learn-hub-codegen-prompt has no effect on the "${target.id}" target — it has no learned generation prompt (only LLM-generating targets like playwright/runn do)`);
|
|
15154
15110
|
return;
|
|
15155
15111
|
}
|
|
15156
15112
|
blank();
|
|
15157
15113
|
await updateAgentPrompt({
|
|
15158
15114
|
kind: target.guidanceKind,
|
|
15159
|
-
flag: "--learn-codegen-prompt",
|
|
15115
|
+
flag: "--learn-hub-codegen-prompt",
|
|
15160
15116
|
runSummary: buildGenerateRunSummary(target.id, featureName, specName, result, cwd),
|
|
15161
15117
|
hubContext: opts.hubContext ?? null,
|
|
15162
15118
|
...opts.model ? { model: opts.model } : {},
|
|
@@ -15182,7 +15138,7 @@ async function confirmOverwrite(path) {
|
|
|
15182
15138
|
rl.close();
|
|
15183
15139
|
}
|
|
15184
15140
|
}
|
|
15185
|
-
const generateCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("generate").argument("<feature/spec>", "Spec id in '<feature>/<spec>' form (resolves to .ccqa/features/<feature>/test-cases/<spec>/)").description("Generate test code from a spec via its target plugin. Recording-backed targets compile the existing ir.json (run `ccqa record` first); spec-input targets generate directly from the spec.").optionsGroup("How to generate:").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--target <id>", "Generate through this target instead of the spec's own — e.g. emit a Playwright spec from an agent-browser recording. The spec's `target:` stays the default for `ccqa run`.").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--overwrite", "Replace previously generated test code without warning").optionsGroup("Learning:").option("--learn-codegen-prompt", "After generation, ask Claude to refresh the target's \"<target>.agent\" learning prompt on the hub from a summary of the run. LLM-generating targets (playwright, runn) only; requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
15141
|
+
const generateCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("generate").argument("<feature/spec>", "Spec id in '<feature>/<spec>' form (resolves to .ccqa/features/<feature>/test-cases/<spec>/)").description("Generate test code from a spec via its target plugin. Recording-backed targets compile the existing ir.json (run `ccqa record` first); spec-input targets generate directly from the spec.").optionsGroup("How to generate:").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--target <id>", "Generate through this target instead of the spec's own — e.g. emit a Playwright spec from an agent-browser recording. The spec's `target:` stays the default for `ccqa run`.").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--overwrite", "Replace previously generated test code without warning").optionsGroup("Learning:").option("--learn-hub-codegen-prompt", "After generation, ask Claude to refresh the target's \"<target>.agent\" learning prompt on the hub from a summary of the run. LLM-generating targets (playwright, runn) only; requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
15186
15142
|
const { featureName, specName } = parseSpecPath(specPath);
|
|
15187
15143
|
const language = opts.language ?? "auto";
|
|
15188
15144
|
const cwd = resolveCwd(opts.cwd);
|
|
@@ -15191,9 +15147,9 @@ const generateCommand = addHubOptions(addProfileOption(addLanguageOption(new Com
|
|
|
15191
15147
|
hubToken: opts.hubToken,
|
|
15192
15148
|
hubHeader: opts.hubHeader
|
|
15193
15149
|
});
|
|
15194
|
-
const project = opts.
|
|
15195
|
-
if (opts.
|
|
15196
|
-
profile: opts.
|
|
15150
|
+
const project = opts.hubProfile !== void 0 || hubClient !== null ? resolveProject(opts) : void 0;
|
|
15151
|
+
if (opts.hubProfile !== void 0) await applyProfileFromOption({
|
|
15152
|
+
profile: opts.hubProfile,
|
|
15197
15153
|
project,
|
|
15198
15154
|
cwd,
|
|
15199
15155
|
hubUrl: opts.hubUrl,
|
|
@@ -15205,6 +15161,10 @@ const generateCommand = addHubOptions(addProfileOption(addLanguageOption(new Com
|
|
|
15205
15161
|
project: "",
|
|
15206
15162
|
cwd
|
|
15207
15163
|
});
|
|
15164
|
+
if (opts.learnHubCodegenPrompt && hubClient === null) {
|
|
15165
|
+
error("--learn-hub-codegen-prompt requires a hub connection (--hub-url/--hub-token or CCQA_HUB_URL/CCQA_HUB_TOKEN)");
|
|
15166
|
+
process.exit(2);
|
|
15167
|
+
}
|
|
15208
15168
|
const hubContext = hubClient && project ? {
|
|
15209
15169
|
hub: hubClient,
|
|
15210
15170
|
project
|
|
@@ -15220,7 +15180,7 @@ const generateCommand = addHubOptions(addProfileOption(addLanguageOption(new Com
|
|
|
15220
15180
|
targetOverride: opts.target,
|
|
15221
15181
|
cwd,
|
|
15222
15182
|
hubContext,
|
|
15223
|
-
updateAgentPrompt: opts.
|
|
15183
|
+
updateAgentPrompt: opts.learnHubCodegenPrompt ?? false
|
|
15224
15184
|
});
|
|
15225
15185
|
} catch (e) {
|
|
15226
15186
|
if (e instanceof SpecLockedError) {
|
|
@@ -15259,7 +15219,7 @@ const VALIDATION_MODES = ["lenient", "strict"];
|
|
|
15259
15219
|
const recordCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("record").argument("<feature/spec>", "Spec id in '<feature>/<spec>' form (resolves to .ccqa/features/<feature>/test-cases/<spec>/)").description("Record a test from a spec: run agent-browser to collect actions (trace), then compile them into runnable code via the spec's target (generate) — a vitest test.spec.ts for agent-browser, a @playwright/test spec for the playwright target. Recording-backed targets only; spec-input targets like runn have no trace step (use `ccqa generate`), and agent-browser live specs need no recording.").optionsGroup("How to record:").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--trace-validation <mode>", "What to do with actions that fail post-trace validation: 'lenient' (default) tags them; 'strict' drops them.", (raw) => {
|
|
15260
15220
|
if (VALIDATION_MODES.includes(raw)) return raw;
|
|
15261
15221
|
throw new Error(`--trace-validation must be one of ${VALIDATION_MODES.join(" | ")}`);
|
|
15262
|
-
}, "lenient").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--trace-only", "Stop after the trace step; do not generate test code").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--overwrite", "Replace an existing test.spec.ts without warning").optionsGroup("Learning:").option("--learn-trace-prompt", "After the trace finishes, ask Claude to refresh the \"record.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
15222
|
+
}, "lenient").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--trace-only", "Stop after the trace step; do not generate test code").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--overwrite", "Replace an existing test.spec.ts without warning").optionsGroup("Learning:").option("--learn-hub-trace-prompt", "After the trace finishes, ask Claude to refresh the \"record.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
15263
15223
|
await withCostTally(async () => {
|
|
15264
15224
|
try {
|
|
15265
15225
|
await runRecord(specPath, opts);
|
|
@@ -15279,9 +15239,9 @@ async function runRecord(specPath, opts) {
|
|
|
15279
15239
|
error(`target "${target.id}" does not use a browser recording — run 'ccqa generate ${featureName}/${specName}' instead`);
|
|
15280
15240
|
process.exit(2);
|
|
15281
15241
|
}
|
|
15282
|
-
const project = opts.
|
|
15283
|
-
if (opts.
|
|
15284
|
-
profile: opts.
|
|
15242
|
+
const project = opts.hubProfile !== void 0 ? resolveProject(opts) : void 0;
|
|
15243
|
+
if (opts.hubProfile !== void 0) await applyProfileFromOption({
|
|
15244
|
+
profile: opts.hubProfile,
|
|
15285
15245
|
project,
|
|
15286
15246
|
cwd: cwdForProfile,
|
|
15287
15247
|
hubUrl: opts.hubUrl,
|
|
@@ -15303,6 +15263,10 @@ async function runRecord(specPath, opts) {
|
|
|
15303
15263
|
hub: hubClientForTrace,
|
|
15304
15264
|
project: hubProject
|
|
15305
15265
|
} : null;
|
|
15266
|
+
if (opts.learnHubTracePrompt && hubContext === null) {
|
|
15267
|
+
error("--learn-hub-trace-prompt requires a hub connection (--hub-url/--hub-token or CCQA_HUB_URL/CCQA_HUB_TOKEN)");
|
|
15268
|
+
process.exit(2);
|
|
15269
|
+
}
|
|
15306
15270
|
const releaseLock = await acquireSpecLock(featureName, specName, "record", cwdForProfile).catch((e) => {
|
|
15307
15271
|
if (e instanceof SpecLockedError) {
|
|
15308
15272
|
error(e.message);
|
|
@@ -15330,11 +15294,11 @@ async function runRecord(specPath, opts) {
|
|
|
15330
15294
|
} finally {
|
|
15331
15295
|
await releaseLock();
|
|
15332
15296
|
}
|
|
15333
|
-
if (opts.
|
|
15297
|
+
if (opts.learnHubTracePrompt && traceResult !== null) {
|
|
15334
15298
|
blank();
|
|
15335
15299
|
await updateAgentPrompt({
|
|
15336
15300
|
kind: "record",
|
|
15337
|
-
flag: "--learn-trace-prompt",
|
|
15301
|
+
flag: "--learn-hub-trace-prompt",
|
|
15338
15302
|
runSummary: buildRecordRunSummary(featureName, specName, traceResult),
|
|
15339
15303
|
hubContext,
|
|
15340
15304
|
...opts.model ? { model: opts.model } : {},
|
|
@@ -15615,7 +15579,7 @@ function specStatus(result, threshold) {
|
|
|
15615
15579
|
return "passed";
|
|
15616
15580
|
}
|
|
15617
15581
|
/**
|
|
15618
|
-
* Adapts `ccqa
|
|
15582
|
+
* Adapts `ccqa audit` results into the shared RunReportData shape so they can
|
|
15619
15583
|
* be pushed to the hub (`ccqa audit --report-to-hub`) and rendered by the same report
|
|
15620
15584
|
* UI as `ccqa run`/`ccqa live`. Browser-execution fields (testCounts,
|
|
15621
15585
|
* evidence, liveRun, ...) don't apply to a drift audit and are always null —
|
|
@@ -15664,7 +15628,7 @@ function driftResultsToReport(results, meta) {
|
|
|
15664
15628
|
//#endregion
|
|
15665
15629
|
//#region src/cli/audit.ts
|
|
15666
15630
|
const DEFAULT_CONCURRENCY = 3;
|
|
15667
|
-
const auditCommand = addLanguageOption(new Command("audit").argument("[feature/spec]", "Optional spec id. If omitted, every spec under .ccqa/features/ is checked.").description("Read each spec against the code it describes and report where the two have drifted. Static: no browser is run, so this is the cheap check to put in front of `ccqa run`.").optionsGroup("Which specs to audit:").option("--only-affected-by <ref>", "Only specs `ccqa select-specs` judges reached by the diff against <ref> (e.g. origin/main). In pull_request CI, pass $GITHUB_BASE_REF. Costs one model call; specs it cannot decide are audited rather than skipped.").optionsGroup("How to run it:").option("--concurrency <n>", `Parallel spec checks (default: ${DEFAULT_CONCURRENCY})`).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").optionsGroup("What to do with the results:").option("--report-format <fmt>", "Output format: text | json | github", "text").option("--report-to-hub", "Push the result to a ccqa hub as a run (kind: drift), which is what updates the drift ledger `ccqa run --only-
|
|
15631
|
+
const auditCommand = addLanguageOption(new Command("audit").argument("[feature/spec]", "Optional spec id. If omitted, every spec under .ccqa/features/ is checked.").description("Read each spec against the code it describes and report where the two have drifted. Static: no browser is run, so this is the cheap check to put in front of `ccqa run`.").optionsGroup("Which specs to audit:").option("--only-affected-by <ref>", "Only specs `ccqa select-specs` judges reached by the diff against <ref> (e.g. origin/main). In pull_request CI, pass $GITHUB_BASE_REF. Costs one model call; specs it cannot decide are audited rather than skipped.").optionsGroup("How to run it:").option("--concurrency <n>", `Parallel spec checks (default: ${DEFAULT_CONCURRENCY})`).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").optionsGroup("What to do with the results:").option("--report-format <fmt>", "Output format: text | json | github", "text").option("--report-to-hub", "Push the result to a ccqa hub as a run (kind: drift), which is what updates the drift ledger. A spec it finds drifted answers `blocked` to `ccqa run --only-hub-rerun-needed`, and is not run until the drift clears.").option("--exit-on <level>", "Exit non-zero on this severity or higher: warn | error", "error").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory used as both the .ccqa root and the codebase Claude reads. Useful for monorepos. Defaults to process.cwd().").option("--project <name>", "Logical project name for the pushed run. Defaults to the current directory's name.").option(...hubUrlOption).option(...hubTokenOption).option(...hubHeaderOption)).action(withUsageErrors(async (specPath, opts) => {
|
|
15668
15632
|
await withCostTally(() => runAudit(specPath, opts));
|
|
15669
15633
|
}));
|
|
15670
15634
|
async function runAudit(specPath, opts) {
|
|
@@ -15723,9 +15687,9 @@ async function runAudit(specPath, opts) {
|
|
|
15723
15687
|
}
|
|
15724
15688
|
/**
|
|
15725
15689
|
* Push a finished drift audit to a ccqa hub as a `kind: "drift"` run, so it
|
|
15726
|
-
* shows up alongside `ccqa run` runs in the hub UI.
|
|
15727
|
-
*
|
|
15728
|
-
*
|
|
15690
|
+
* shows up alongside `ccqa run` runs in the hub UI. A missing hub connection
|
|
15691
|
+
* is a usage error, not a silent skip — a CI job that asked to publish and
|
|
15692
|
+
* did not must say so.
|
|
15729
15693
|
*
|
|
15730
15694
|
* `resolveHub` is injectable so tests can supply a fake `HubClient` without
|
|
15731
15695
|
* a real hub connection; it defaults to the real flag/env resolution.
|
|
@@ -15734,8 +15698,8 @@ async function pushDriftResults(args, resolveHub = resolveHubClient) {
|
|
|
15734
15698
|
const { results, threshold, cwd, opts, format, baseRef } = args;
|
|
15735
15699
|
const hub = resolveHub(opts);
|
|
15736
15700
|
if (!hub) {
|
|
15737
|
-
|
|
15738
|
-
|
|
15701
|
+
error("--report-to-hub requires a hub connection (--hub-url/--hub-token or CCQA_HUB_URL/CCQA_HUB_TOKEN)");
|
|
15702
|
+
process.exit(2);
|
|
15739
15703
|
}
|
|
15740
15704
|
try {
|
|
15741
15705
|
const project = resolveProject({
|
|
@@ -15812,7 +15776,7 @@ function parseFormat$1(raw) {
|
|
|
15812
15776
|
function parseSeverity(raw) {
|
|
15813
15777
|
const v = raw ?? "error";
|
|
15814
15778
|
if (v === "warn" || v === "error") return v;
|
|
15815
|
-
error(`invalid --
|
|
15779
|
+
error(`invalid --exit-on: ${v} (expected warn|error)`);
|
|
15816
15780
|
process.exit(2);
|
|
15817
15781
|
}
|
|
15818
15782
|
function parseConcurrency(raw) {
|
|
@@ -16282,7 +16246,7 @@ function countSpecs(results) {
|
|
|
16282
16246
|
* or `red` (the last outcome), which are orthogonal axes.
|
|
16283
16247
|
*
|
|
16284
16248
|
* Best-effort — a ledger failure must not fail the push; the ledger is an
|
|
16285
|
-
* accelerator for `--
|
|
16249
|
+
* accelerator for `--on-fail-explain` baselines and re-run selection, not
|
|
16286
16250
|
* part of the run record. Runs without a branch or gitHead can't be placed in
|
|
16287
16251
|
* the ledger and are skipped.
|
|
16288
16252
|
*
|
|
@@ -16926,7 +16890,7 @@ function createGetLastGreenHandler(storage) {
|
|
|
16926
16890
|
/**
|
|
16927
16891
|
* GET /api/v1/projects/:project/drift
|
|
16928
16892
|
*
|
|
16929
|
-
* Every spec's last `ccqa
|
|
16893
|
+
* Every spec's last `ccqa audit --report-to-hub` result, keyed by "feature/spec". No
|
|
16930
16894
|
* `?profile=` — drift asks whether a spec still describes the code, which
|
|
16931
16895
|
* has nothing to do with which environment is running it, unlike the
|
|
16932
16896
|
* `/rerun` and `/last-green` endpoints. Merged across every branch (newest
|
|
@@ -17110,7 +17074,7 @@ async function loadSpecTargets(perspectives, project) {
|
|
|
17110
17074
|
//#endregion
|
|
17111
17075
|
//#region src/hub/core/rerun.ts
|
|
17112
17076
|
function computeRerun(input) {
|
|
17113
|
-
const { specs, ledger, log, touchIndex } = input;
|
|
17077
|
+
const { specs, ledger, log, touchIndex, drift } = input;
|
|
17114
17078
|
const notEvaluated = log.entries.length === 0 && Object.keys(ledger.run).length === 0 && Object.keys(ledger.green).length === 0;
|
|
17115
17079
|
const positionBySha = /* @__PURE__ */ new Map();
|
|
17116
17080
|
log.entries.forEach((entry, i) => {
|
|
@@ -17135,7 +17099,12 @@ function computeRerun(input) {
|
|
|
17135
17099
|
lastGreen: ledger.green[spec.key] ?? null,
|
|
17136
17100
|
lastRed: ledger.red[spec.key] ?? null
|
|
17137
17101
|
};
|
|
17138
|
-
|
|
17102
|
+
const blocked = blockedBy(drift, spec.key);
|
|
17103
|
+
out[spec.key] = blocked ? {
|
|
17104
|
+
state: "blocked",
|
|
17105
|
+
blockedReason: blocked,
|
|
17106
|
+
...coords
|
|
17107
|
+
} : notEvaluated ? {
|
|
17139
17108
|
state: "notEvaluated",
|
|
17140
17109
|
...coords
|
|
17141
17110
|
} : {
|
|
@@ -17145,6 +17114,21 @@ function computeRerun(input) {
|
|
|
17145
17114
|
}
|
|
17146
17115
|
return out;
|
|
17147
17116
|
}
|
|
17117
|
+
/**
|
|
17118
|
+
* Why the audit rejects this spec, or null when it does not.
|
|
17119
|
+
*
|
|
17120
|
+
* Only a finding blocks. A spec with no ledger entry was never audited, and a
|
|
17121
|
+
* `UNKNOWN` entry is the audit saying it could not tell — neither is a reason
|
|
17122
|
+
* to withhold a run, and treating them as one would stop every newly written
|
|
17123
|
+
* spec from ever executing.
|
|
17124
|
+
*/
|
|
17125
|
+
function blockedBy(drift, key) {
|
|
17126
|
+
switch (drift.specs[key]?.label) {
|
|
17127
|
+
case "TEST_DRIFT": return "testDrift";
|
|
17128
|
+
case "SPEC_CHANGE": return "specChange";
|
|
17129
|
+
default: return null;
|
|
17130
|
+
}
|
|
17131
|
+
}
|
|
17148
17132
|
function verdict(spec, lastRun, log, positionBySha, touchIndex, range) {
|
|
17149
17133
|
if (!lastRun) return { state: "neverRun" };
|
|
17150
17134
|
if (log.entries.length === 0) return unknown("noDeployLog");
|
|
@@ -17187,21 +17171,23 @@ function unknown(reason) {
|
|
|
17187
17171
|
/**
|
|
17188
17172
|
* GET /api/v1/projects/:project/rerun?profile=
|
|
17189
17173
|
*
|
|
17190
|
-
* Per spec: is
|
|
17174
|
+
* Per spec: is it worth running, and if not, why? Set arithmetic over the spec
|
|
17191
17175
|
* ledger, the profile's deploy log and each deploy's per-spec touch verdicts
|
|
17192
|
-
* recorded by `ccqa select-specs` (ADR-0010, ADR-0011)
|
|
17193
|
-
*
|
|
17194
|
-
*
|
|
17176
|
+
* recorded by `ccqa select-specs` (ADR-0010, ADR-0011), plus the drift ledger —
|
|
17177
|
+
* a spec the audit rejected answers `blocked`, because re-running it cannot
|
|
17178
|
+
* clear what is wrong with it. The spec ledger is read across every branch: a
|
|
17179
|
+
* run exercises the deployed environment whatever branch its code came from.
|
|
17195
17180
|
*/
|
|
17196
17181
|
function createGetRerunHandler(storage) {
|
|
17197
17182
|
return async (ctx) => {
|
|
17198
17183
|
const project = requireSafeSegment(ctx.params.project, "project");
|
|
17199
17184
|
const profile = requireProfileParam(ctx.url);
|
|
17200
|
-
const [specs, ledger, log, touchIndex] = await Promise.all([
|
|
17185
|
+
const [specs, ledger, log, touchIndex, drift] = await Promise.all([
|
|
17201
17186
|
loadSpecTargets(storage.perspectives, project),
|
|
17202
17187
|
storage.ledger.getMerged(project, profile),
|
|
17203
17188
|
storage.deploys.getLog(project, profile),
|
|
17204
|
-
storage.deploys.getTouchIndex(project, profile)
|
|
17189
|
+
storage.deploys.getTouchIndex(project, profile),
|
|
17190
|
+
storage.driftLedger.getMerged(project)
|
|
17205
17191
|
]);
|
|
17206
17192
|
if (specs === null) throw new HttpError(404, "no_perspectives", `no perspectives stored for project "${project}" — push one with \`ccqa perspectives\` before asking which specs need a re-run`);
|
|
17207
17193
|
const head = log.entries[log.entries.length - 1];
|
|
@@ -17217,7 +17203,8 @@ function createGetRerunHandler(storage) {
|
|
|
17217
17203
|
specs,
|
|
17218
17204
|
ledger,
|
|
17219
17205
|
log,
|
|
17220
|
-
touchIndex
|
|
17206
|
+
touchIndex,
|
|
17207
|
+
drift
|
|
17221
17208
|
})
|
|
17222
17209
|
});
|
|
17223
17210
|
};
|
|
@@ -18028,7 +18015,7 @@ const HTML_BODY = `
|
|
|
18028
18015
|
<div style="font-weight:600" data-i18n="session.help.title">How to get this JSON</div>
|
|
18029
18016
|
<ol class="help-steps">
|
|
18030
18017
|
<li><span class="step-n">1</span><div class="step-b"><span data-i18n="session.help.step1">Run this in your terminal and log in by hand when the browser opens:</span>
|
|
18031
|
-
<div class="cmd"><code id="session-help-cmd">ccqa session
|
|
18018
|
+
<div class="cmd"><code id="session-help-cmd">ccqa hub session capture <name></code><button type="button" class="copy" id="session-help-copy"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><rect x="9" y="9" width="11" height="11" rx="2"/><path d="M5 15V5a2 2 0 0 1 2-2h10"/></svg><span data-i18n="common.copy">Copy</span></button></div>
|
|
18032
18019
|
</div></li>
|
|
18033
18020
|
<li><span class="step-n">2</span><div class="step-b"><span data-i18n="session.help.step2">Open the saved file and paste its contents below:</span>
|
|
18034
18021
|
<div style="margin-top:5px"><span class="path">.ccqa/sessions/<profile>/<name>.json</span></div>
|
|
@@ -18580,6 +18567,7 @@ const CSS = `
|
|
|
18580
18567
|
eats the next rule, and a backtick ends the template literal this CSS
|
|
18581
18568
|
lives in. */
|
|
18582
18569
|
.sg-drift-found, .sg-needed { background: var(--amber-fill); }
|
|
18570
|
+
.sg-blocked { background: var(--fail); }
|
|
18583
18571
|
.sg-unknown, .sg-drift-unknown { background: var(--info); }
|
|
18584
18572
|
.sg-drift-clean, .sg-notneeded { background: var(--pass); }
|
|
18585
18573
|
.sg-drift-none, .sg-neverrun { background: var(--muted-2); }
|
|
@@ -18788,6 +18776,7 @@ const CLIENT_JS = `
|
|
|
18788
18776
|
"perspectives.result.ci": "CI",
|
|
18789
18777
|
"perspectives.rerun.state.needed": "Re-run needed",
|
|
18790
18778
|
"perspectives.rerun.state.notNeeded": "Not needed",
|
|
18779
|
+
"perspectives.rerun.state.blocked": "Blocked by the audit",
|
|
18791
18780
|
"perspectives.rerun.state.unknown": "Can't tell",
|
|
18792
18781
|
"perspectives.rerun.state.neverRun": "Never run",
|
|
18793
18782
|
"perspectives.rerun.state.notEvaluated": "Not evaluated",
|
|
@@ -18800,6 +18789,8 @@ const CLIENT_JS = `
|
|
|
18800
18789
|
"perspectives.rerun.touchedUnknown": "a deploy since the last run matched this case",
|
|
18801
18790
|
"perspectives.rerun.neverRunHint": "no result recorded for this profile yet",
|
|
18802
18791
|
"perspectives.rerun.notEvaluatedHint": "no run and no deploy has ever been recorded for this profile",
|
|
18792
|
+
"perspectives.rerun.blocked.testDrift": "the generated test no longer matches the code — re-record it",
|
|
18793
|
+
"perspectives.rerun.blocked.specChange": "the spec describes something the code no longer does — a human decides",
|
|
18803
18794
|
"perspectives.rerun.why.noSelectionInRange": "a deploy in range was recorded without a spec selection",
|
|
18804
18795
|
"perspectives.rerun.why.selectionUnknown": "the selector could not tell whether this case was affected",
|
|
18805
18796
|
"perspectives.rerun.why.noDeployLog": "no deploy log for this profile",
|
|
@@ -18937,6 +18928,7 @@ const CLIENT_JS = `
|
|
|
18937
18928
|
"perspectives.result.ci": "CI",
|
|
18938
18929
|
"perspectives.rerun.state.needed": "要再実行",
|
|
18939
18930
|
"perspectives.rerun.state.notNeeded": "不要",
|
|
18931
|
+
"perspectives.rerun.state.blocked": "監査で保留",
|
|
18940
18932
|
"perspectives.rerun.state.unknown": "判定できない",
|
|
18941
18933
|
"perspectives.rerun.state.neverRun": "未実行",
|
|
18942
18934
|
"perspectives.rerun.state.notEvaluated": "未評価",
|
|
@@ -18949,6 +18941,8 @@ const CLIENT_JS = `
|
|
|
18949
18941
|
"perspectives.rerun.touchedUnknown": "前回実行以降のデプロイがこのケースに一致する変更を行っています",
|
|
18950
18942
|
"perspectives.rerun.neverRunHint": "このプロファイルでの実行記録がまだありません",
|
|
18951
18943
|
"perspectives.rerun.notEvaluatedHint": "このプロファイルには実行もデプロイも記録がありません",
|
|
18944
|
+
"perspectives.rerun.blocked.testDrift": "生成されたテストが古くなっています。録り直してください",
|
|
18945
|
+
"perspectives.rerun.blocked.specChange": "spec がコードのやめた動作を書いています。人が判断します",
|
|
18952
18946
|
"perspectives.rerun.why.noSelectionInRange": "対象範囲に判定を伴わないデプロイがあります",
|
|
18953
18947
|
"perspectives.rerun.why.selectionUnknown": "影響の有無を判定できませんでした",
|
|
18954
18948
|
"perspectives.rerun.why.noDeployLog": "このプロファイルのデプロイ記録がありません",
|
|
@@ -20154,7 +20148,7 @@ const CLIENT_JS = `
|
|
|
20154
20148
|
}
|
|
20155
20149
|
|
|
20156
20150
|
// A drift-kind row's diagnosis lives in analysis regardless of status
|
|
20157
|
-
// (an UNKNOWN-labelled finding below the --
|
|
20151
|
+
// (an UNKNOWN-labelled finding below the --exit-on threshold still
|
|
20158
20152
|
// "passes" but has something to show); a normal run only ever classifies
|
|
20159
20153
|
// a failed spec.
|
|
20160
20154
|
var hasAnalysis = isDrift ? !!r.analysis : r.status === "failed" && r.analysis;
|
|
@@ -20989,6 +20983,7 @@ const CLIENT_JS = `
|
|
|
20989
20983
|
if (!head) return t("perspectives.rerun.noDeployHead");
|
|
20990
20984
|
return t("perspectives.rerun.vsDeploy") + " " + shortSha(head.sha) + " · " + relTime(head.at);
|
|
20991
20985
|
}
|
|
20986
|
+
if (rr.state === "blocked") return rerunReasonText("perspectives.rerun.blocked.", rr.blockedReason || "");
|
|
20992
20987
|
if (rr.state === "unknown") return rerunReasonText("perspectives.rerun.why.", rr.reason || "");
|
|
20993
20988
|
return rerunCannotJudge(rr);
|
|
20994
20989
|
}
|
|
@@ -21001,12 +20996,13 @@ const CLIENT_JS = `
|
|
|
21001
20996
|
// browser to click through.
|
|
21002
20997
|
|
|
21003
20998
|
// Bar segments in drawing order: what to act on first, then what needs no
|
|
21004
|
-
// action, then what was never measured. "
|
|
21005
|
-
//
|
|
21006
|
-
//
|
|
21007
|
-
|
|
20999
|
+
// action, then what was never measured. "blocked" leads because it is the
|
|
21000
|
+
// only state a run cannot clear — someone has to repair the spec. "unknown"
|
|
21001
|
+
// keeps its own place and its own colour: folding it into "notNeeded" would
|
|
21002
|
+
// turn "we cannot say" into "all clear", which is what ADR-0010 forbids.
|
|
21003
|
+
var RERUN_ORDER = ["blocked", "needed", "unknown", "notNeeded", "neverRun", "notEvaluated"];
|
|
21008
21004
|
var RERUN_SEG_CLASS = {
|
|
21009
|
-
needed: "sg-needed", unknown: "sg-unknown", notNeeded: "sg-notneeded",
|
|
21005
|
+
blocked: "sg-blocked", needed: "sg-needed", unknown: "sg-unknown", notNeeded: "sg-notneeded",
|
|
21010
21006
|
neverRun: "sg-neverrun", notEvaluated: "sg-noteval"
|
|
21011
21007
|
};
|
|
21012
21008
|
|
|
@@ -21016,7 +21012,7 @@ const CLIENT_JS = `
|
|
|
21016
21012
|
// state this UI does not know reads as unknown for the same reason: an
|
|
21017
21013
|
// answer we cannot interpret is not evidence that nothing is needed.
|
|
21018
21014
|
function rerunComposition(verdicts) {
|
|
21019
|
-
var counts = { needed: 0, unknown: 0, notNeeded: 0, neverRun: 0, notEvaluated: 0 };
|
|
21015
|
+
var counts = { blocked: 0, needed: 0, unknown: 0, notNeeded: 0, neverRun: 0, notEvaluated: 0 };
|
|
21020
21016
|
verdicts.forEach(function (rr) {
|
|
21021
21017
|
if (!rr || !rr.state) { counts.notEvaluated += 1; return; }
|
|
21022
21018
|
var known = Object.prototype.hasOwnProperty.call(counts, rr.state);
|
|
@@ -21147,7 +21143,7 @@ const CLIENT_JS = `
|
|
|
21147
21143
|
//
|
|
21148
21144
|
// "unknown" keeps its own state rather than folding into the last result:
|
|
21149
21145
|
// it means the hub cannot say whether that result still holds, and
|
|
21150
|
-
// --
|
|
21146
|
+
// --only-hub-rerun-needed does not re-run it without --only-hub-rerun-needed-with-unknown. Showing
|
|
21151
21147
|
// it as passed or failed would claim a confidence nothing supports.
|
|
21152
21148
|
function perspRunState(rr) {
|
|
21153
21149
|
if (!rr) return null;
|