ccqa 1.26.3 → 1.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/ccqa.mjs +444 -111
- package/dist/hub-client/index.d.mts +14 -2
- package/dist/package.json +1 -1
- package/package.json +1 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -3356,6 +3356,15 @@ const LiveReportRunSchema = z.object({
|
|
|
3356
3356
|
steps: z.array(LiveReportStepSchema),
|
|
3357
3357
|
cost: ReportCostSchema
|
|
3358
3358
|
});
|
|
3359
|
+
/**
|
|
3360
|
+
* What a second attempt at a failed spec showed (`--on-fail-explain-rerun`).
|
|
3361
|
+
* "passed" means the failure did not reproduce, which is the evidence a single
|
|
3362
|
+
* run cannot hold; "failed" means it did.
|
|
3363
|
+
*
|
|
3364
|
+
* The row's `status` never moves with it. The spec failed, and a passing second
|
|
3365
|
+
* attempt explains that failure rather than undoing it.
|
|
3366
|
+
*/
|
|
3367
|
+
const ReportRerunSchema = z.object({ outcome: z.enum(["passed", "failed"]) });
|
|
3359
3368
|
const ReportSpecResultSchema = z.object({
|
|
3360
3369
|
feature: z.string(),
|
|
3361
3370
|
spec: z.string(),
|
|
@@ -3377,6 +3386,7 @@ const ReportSpecResultSchema = z.object({
|
|
|
3377
3386
|
assertions: z.array(ReportAssertionSchema).nullable(),
|
|
3378
3387
|
analysis: FailureAnalysisSchema.nullable(),
|
|
3379
3388
|
analysisSkipped: z.string().nullable(),
|
|
3389
|
+
rerun: ReportRerunSchema.optional(),
|
|
3380
3390
|
customPromptVersion: z.string().optional(),
|
|
3381
3391
|
analysisBase: z.object({
|
|
3382
3392
|
ref: z.string(),
|
|
@@ -4819,30 +4829,31 @@ async function readCostFileTotal(path) {
|
|
|
4819
4829
|
//#endregion
|
|
4820
4830
|
//#region src/cli/draft.ts
|
|
4821
4831
|
const CATEGORY_LABEL = DRAFT_CATEGORY_LABEL;
|
|
4822
|
-
const draftCommand = addLanguageOption(new Command("draft").argument("[feature/spec]", "Optional spec path (e.g. tasks/create-and-complete). If omitted, Claude proposes one from your intent.").description("Interactively draft and refine a spec.yaml with Claude Code").option("--instruction <text>", "Non-interactive single-shot instruction (skips the interactive loop)").option("-y, --yes", "Apply each generated patch without asking [y/N]", false)).action(withUsageErrors(async (specPath, opts) => {
|
|
4832
|
+
const draftCommand = addLanguageOption(new Command("draft").argument("[feature/spec]", "Optional spec path (e.g. tasks/create-and-complete). If omitted, Claude proposes one from your intent.").description("Interactively draft and refine a spec.yaml with Claude Code").option("--instruction <text>", "Non-interactive single-shot instruction (skips the interactive loop)").option("-y, --yes", "Apply each generated patch without asking [y/N]", false).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.")).action(withUsageErrors(async (specPath, opts) => {
|
|
4823
4833
|
await withCostReporting("draft", () => runDraftCli(specPath, opts));
|
|
4824
4834
|
}));
|
|
4825
4835
|
async function runDraftCli(specPath, opts) {
|
|
4826
|
-
|
|
4836
|
+
const cwd = resolveCwd(opts.cwd);
|
|
4837
|
+
await ensureCcqaDir(cwd);
|
|
4827
4838
|
let featureName;
|
|
4828
4839
|
let specName;
|
|
4829
4840
|
let prefilledIntent = null;
|
|
4830
4841
|
if (specPath) ({featureName, specName} = parseSpecPath(specPath));
|
|
4831
4842
|
else {
|
|
4832
|
-
const { naming, intent } = await proposeNaming(opts);
|
|
4843
|
+
const { naming, intent } = await proposeNaming(opts, cwd);
|
|
4833
4844
|
featureName = naming.featureName;
|
|
4834
4845
|
specName = naming.specName;
|
|
4835
4846
|
prefilledIntent = intent;
|
|
4836
4847
|
}
|
|
4837
|
-
await runDraft(featureName, specName, opts, prefilledIntent);
|
|
4848
|
+
await runDraft(featureName, specName, opts, cwd, prefilledIntent);
|
|
4838
4849
|
}
|
|
4839
|
-
async function runDraft(featureName, specName, opts, prefilledIntent) {
|
|
4850
|
+
async function runDraft(featureName, specName, opts, cwd, prefilledIntent) {
|
|
4840
4851
|
header("draft", `${featureName}/${specName}`);
|
|
4841
4852
|
const ja = useJapanesePrompts(opts.language);
|
|
4842
4853
|
const oneShot = opts.instruction !== void 0;
|
|
4843
4854
|
let useIntentOnce = prefilledIntent !== null && !oneShot;
|
|
4844
4855
|
while (true) {
|
|
4845
|
-
const existing = await tryReadSpecFile(featureName, specName);
|
|
4856
|
+
const existing = await tryReadSpecFile(featureName, specName, cwd);
|
|
4846
4857
|
const isFirstRun = existing === null;
|
|
4847
4858
|
let userInput;
|
|
4848
4859
|
if (oneShot) userInput = opts.instruction ?? "";
|
|
@@ -4860,7 +4871,9 @@ async function runDraft(featureName, specName, opts, prefilledIntent) {
|
|
|
4860
4871
|
existing,
|
|
4861
4872
|
userInput: userInput.trim(),
|
|
4862
4873
|
autoApply: opts.yes === true,
|
|
4863
|
-
language: opts.language
|
|
4874
|
+
language: opts.language,
|
|
4875
|
+
model: opts.model,
|
|
4876
|
+
cwd
|
|
4864
4877
|
});
|
|
4865
4878
|
if (oneShot) process.exit(turnResult.hasError && !turnResult.applied ? 1 : 0);
|
|
4866
4879
|
blank();
|
|
@@ -4872,9 +4885,9 @@ async function runDraft(featureName, specName, opts, prefilledIntent) {
|
|
|
4872
4885
|
}
|
|
4873
4886
|
}
|
|
4874
4887
|
async function runOneTurn(input) {
|
|
4875
|
-
const { featureName, specName, existing, userInput, autoApply, language } = input;
|
|
4888
|
+
const { featureName, specName, existing, userInput, autoApply, language, model, cwd } = input;
|
|
4876
4889
|
const isFirstRun = existing === null;
|
|
4877
|
-
const systemPrompt = buildDraftSystemPrompt(await loadAvailableBlocks()) + languageDirective(language);
|
|
4890
|
+
const systemPrompt = buildDraftSystemPrompt(await loadAvailableBlocks(cwd)) + languageDirective(language);
|
|
4878
4891
|
const userPrompt = buildDraftPrompt({
|
|
4879
4892
|
mode: isFirstRun ? "create" : "refine",
|
|
4880
4893
|
existing: existing ?? "",
|
|
@@ -4891,7 +4904,9 @@ async function runOneTurn(input) {
|
|
|
4891
4904
|
"Grep",
|
|
4892
4905
|
"Glob"
|
|
4893
4906
|
],
|
|
4894
|
-
silenceBashLog: true
|
|
4907
|
+
silenceBashLog: true,
|
|
4908
|
+
...model ? { model } : {},
|
|
4909
|
+
cwd
|
|
4895
4910
|
}, (msg) => {
|
|
4896
4911
|
if (msg.type !== "assistant") return;
|
|
4897
4912
|
for (const block of msg.message.content ?? []) if (block.type === "tool_use") toolCounts[block.name] = (toolCounts[block.name] ?? 0) + 1;
|
|
@@ -4953,7 +4968,7 @@ async function runOneTurn(input) {
|
|
|
4953
4968
|
applied: false
|
|
4954
4969
|
};
|
|
4955
4970
|
}
|
|
4956
|
-
meta("saved", await saveSpecFile(featureName, specName, report.patch));
|
|
4971
|
+
meta("saved", await saveSpecFile(featureName, specName, report.patch, cwd));
|
|
4957
4972
|
return {
|
|
4958
4973
|
hasError,
|
|
4959
4974
|
applied: true
|
|
@@ -5028,7 +5043,7 @@ function writeFinding(issue) {
|
|
|
5028
5043
|
process.stdout.write(` ${issue.message}\n`);
|
|
5029
5044
|
if (issue.detail) process.stdout.write(` └ ${issue.detail.replace(/\n/g, "\n ")}\n`);
|
|
5030
5045
|
}
|
|
5031
|
-
async function proposeNaming(opts) {
|
|
5046
|
+
async function proposeNaming(opts, cwd) {
|
|
5032
5047
|
const ja = useJapanesePrompts(opts.language);
|
|
5033
5048
|
const oneShot = opts.instruction !== void 0;
|
|
5034
5049
|
const intent = oneShot ? opts.instruction ?? "" : await prompt(ja ? "何をテストしたいですか? > " : "What do you want to test? > ");
|
|
@@ -5036,7 +5051,7 @@ async function proposeNaming(opts) {
|
|
|
5036
5051
|
error("intent required to propose a feature/spec name");
|
|
5037
5052
|
process.exit(1);
|
|
5038
5053
|
}
|
|
5039
|
-
const tree = await listFeatureTree();
|
|
5054
|
+
const tree = await listFeatureTree(cwd);
|
|
5040
5055
|
const treeForPrompt = tree.map((f) => ({
|
|
5041
5056
|
featureName: f.featureName,
|
|
5042
5057
|
specs: f.specs.map((s) => ({ specName: s.specName }))
|
|
@@ -5050,7 +5065,9 @@ async function proposeNaming(opts) {
|
|
|
5050
5065
|
"Read",
|
|
5051
5066
|
"Grep",
|
|
5052
5067
|
"Glob"
|
|
5053
|
-
]
|
|
5068
|
+
],
|
|
5069
|
+
...opts.model ? { model: opts.model } : {},
|
|
5070
|
+
cwd
|
|
5054
5071
|
}, () => {});
|
|
5055
5072
|
if (isError) {
|
|
5056
5073
|
error("Claude failed during naming");
|
|
@@ -6346,6 +6363,131 @@ async function readGeneratedTestSources(ref, cwd) {
|
|
|
6346
6363
|
}
|
|
6347
6364
|
return parts.join("\n\n");
|
|
6348
6365
|
}
|
|
6366
|
+
//#endregion
|
|
6367
|
+
//#region src/run/explain-rerun.ts
|
|
6368
|
+
/**
|
|
6369
|
+
* `ccqa run --on-fail-explain-rerun`: run a failed spec a second time and let
|
|
6370
|
+
* the result settle what one run cannot.
|
|
6371
|
+
*
|
|
6372
|
+
* `ENVIRONMENT` is the only cause with no artifact to read — a service that is
|
|
6373
|
+
* down, an expired credential, a timing race. When the log names it the
|
|
6374
|
+
* classifier can call it, and when it does not the honest answer is `UNKNOWN`
|
|
6375
|
+
* (ADR-0016). The evidence that would settle either is whether a second
|
|
6376
|
+
* attempt at the same commit passes, and this phase is what collects it.
|
|
6377
|
+
*
|
|
6378
|
+
* It runs after the classification, on the rows it produced, because `auto`
|
|
6379
|
+
* keys off the label. The second attempt is discarded except for its verdict:
|
|
6380
|
+
* it is not a row of this run, it is why one of the rows is red.
|
|
6381
|
+
*/
|
|
6382
|
+
const EXPLAIN_RERUN_MODES = [
|
|
6383
|
+
"auto",
|
|
6384
|
+
"always",
|
|
6385
|
+
"never"
|
|
6386
|
+
];
|
|
6387
|
+
/**
|
|
6388
|
+
* The labels a second attempt can settle. `UNKNOWN` is the refusal the feature
|
|
6389
|
+
* exists to turn into an answer; `ENVIRONMENT` is rerun to confirm, since a
|
|
6390
|
+
* failure that reproduces is not the timing race that reading alone cannot
|
|
6391
|
+
* rule out.
|
|
6392
|
+
*/
|
|
6393
|
+
const RERUNNABLE_LABELS = ["UNKNOWN", "ENVIRONMENT"];
|
|
6394
|
+
/** Evidence sentences the rerun writes onto the row, in the classifier's own currency. */
|
|
6395
|
+
const DID_NOT_REPRODUCE = "a second attempt at the same commit passed: the failure is not reproducible";
|
|
6396
|
+
const REPRODUCED = "a second attempt at the same commit failed too: the failure is reproducible";
|
|
6397
|
+
/**
|
|
6398
|
+
* Confidence for a label the rerun settled. High, because the observation is
|
|
6399
|
+
* direct rather than read out of a diff — but short of certainty, since "did
|
|
6400
|
+
* not reproduce" is still an inference about the first attempt.
|
|
6401
|
+
*/
|
|
6402
|
+
const RERUN_SETTLED_CONFIDENCE = .95;
|
|
6403
|
+
/** Whether this row is one `mode` asks for a second attempt at. */
|
|
6404
|
+
function wantsRerun(row, mode) {
|
|
6405
|
+
if (mode === "never" || row.status !== "failed" || row.analysis === null) return false;
|
|
6406
|
+
return mode === "always" || RERUNNABLE_LABELS.includes(row.analysis.label);
|
|
6407
|
+
}
|
|
6408
|
+
/**
|
|
6409
|
+
* Rerun the failures `mode` selects and fold each verdict into its row.
|
|
6410
|
+
* Returns every row in the order given; the ones not rerun pass through
|
|
6411
|
+
* untouched.
|
|
6412
|
+
*
|
|
6413
|
+
* A rerun that throws leaves its row as the classifier left it, named in a
|
|
6414
|
+
* warning: a second attempt that never ran has established nothing, and
|
|
6415
|
+
* pretending otherwise is the one thing this phase must not do.
|
|
6416
|
+
*/
|
|
6417
|
+
async function rerunExplainedFailures(rows, opts) {
|
|
6418
|
+
const eligible = rows.filter((row) => wantsRerun(row, opts.mode));
|
|
6419
|
+
if (eligible.length === 0) return [...rows];
|
|
6420
|
+
const budget = opts.maxSpecs ?? eligible.length;
|
|
6421
|
+
const skipped = eligible.slice(budget);
|
|
6422
|
+
emitRaw(`\n${C$1.cyan}${C$1.bold}──────── failure rerun ────────${C$1.reset}\n\n`);
|
|
6423
|
+
const applied = /* @__PURE__ */ new Map();
|
|
6424
|
+
for (const row of eligible.slice(0, budget)) {
|
|
6425
|
+
const key = `${row.feature}/${row.spec}`;
|
|
6426
|
+
info(`rerun: ${key}`);
|
|
6427
|
+
let outcome;
|
|
6428
|
+
try {
|
|
6429
|
+
outcome = await opts.execute({
|
|
6430
|
+
featureName: row.feature,
|
|
6431
|
+
specName: row.spec
|
|
6432
|
+
});
|
|
6433
|
+
} catch (err) {
|
|
6434
|
+
warn(`rerun failed to execute ${key} (${errMessage(err)}); its label stands as first classified`);
|
|
6435
|
+
continue;
|
|
6436
|
+
}
|
|
6437
|
+
const next = applyRerun(row, outcome);
|
|
6438
|
+
applied.set(key, next);
|
|
6439
|
+
printRerun(key, outcome, next);
|
|
6440
|
+
}
|
|
6441
|
+
if (skipped.length > 0) warn(`--on-fail-explain-rerun-max-specs ${budget} reached: not rerun, so their labels stand as first classified — ` + skipped.map((row) => `${row.feature}/${row.spec}`).join(", "));
|
|
6442
|
+
return rows.map((row) => applied.get(`${row.feature}/${row.spec}`) ?? row);
|
|
6443
|
+
}
|
|
6444
|
+
/**
|
|
6445
|
+
* Fold one verdict into its row. The row stays failed either way — the spec
|
|
6446
|
+
* failed, and what the rerun changes is why.
|
|
6447
|
+
*
|
|
6448
|
+
* A failure that did not reproduce is environmental, so the label says so.
|
|
6449
|
+
* One that did reproduce names no artifact it did not name before, so the
|
|
6450
|
+
* label stands: ADR-0016 asks a label to be earned, and "not a flake" earns
|
|
6451
|
+
* none of the three that point at something in the repository. What was
|
|
6452
|
+
* learned lands in the evidence instead, where a human triaging the row reads
|
|
6453
|
+
* it.
|
|
6454
|
+
*/
|
|
6455
|
+
function applyRerun(row, outcome) {
|
|
6456
|
+
const analysis = row.analysis;
|
|
6457
|
+
if (analysis === null) return {
|
|
6458
|
+
...row,
|
|
6459
|
+
rerun: { outcome }
|
|
6460
|
+
};
|
|
6461
|
+
const evidence = [...analysis.evidence, { detail: outcome === "passed" ? DID_NOT_REPRODUCE : REPRODUCED }];
|
|
6462
|
+
if (outcome === "failed" || !RERUNNABLE_LABELS.includes(analysis.label)) return {
|
|
6463
|
+
...row,
|
|
6464
|
+
analysis: {
|
|
6465
|
+
...analysis,
|
|
6466
|
+
evidence
|
|
6467
|
+
},
|
|
6468
|
+
rerun: { outcome }
|
|
6469
|
+
};
|
|
6470
|
+
return {
|
|
6471
|
+
...row,
|
|
6472
|
+
analysis: {
|
|
6473
|
+
...analysis,
|
|
6474
|
+
label: "ENVIRONMENT",
|
|
6475
|
+
confidence: RERUN_SETTLED_CONFIDENCE,
|
|
6476
|
+
headline: "the failure did not reproduce on a second attempt",
|
|
6477
|
+
recommendation: "treat the first attempt as environmental (a transient service, credential, seed-data or timing problem); nothing in the repository is implicated.",
|
|
6478
|
+
reasoning: `${analysis.reasoning}\n\n${DID_NOT_REPRODUCE}`.trim(),
|
|
6479
|
+
evidence
|
|
6480
|
+
},
|
|
6481
|
+
rerun: { outcome }
|
|
6482
|
+
};
|
|
6483
|
+
}
|
|
6484
|
+
/** One rerun spec's line in the rerun block: what happened, and where it left the label. */
|
|
6485
|
+
function printRerun(key, outcome, row) {
|
|
6486
|
+
const icon = outcome === "passed" ? `${C$1.green}✔${C$1.reset}` : `${C$1.red}✖${C$1.reset}`;
|
|
6487
|
+
const what = outcome === "passed" ? "did not reproduce" : "reproduced";
|
|
6488
|
+
const label = row.analysis?.label;
|
|
6489
|
+
emitRaw(`${icon} ${C$1.bold}${key}${C$1.reset} → ${what}${label ? ` ${C$1.dim}(${label})${C$1.reset}` : ""}\n`);
|
|
6490
|
+
}
|
|
6349
6491
|
/**
|
|
6350
6492
|
* Capture the PR diff used as context for failure analysis. `--relative`
|
|
6351
6493
|
* re-roots paths to `cwd` and drops changes outside it, so a monorepo
|
|
@@ -9184,6 +9326,93 @@ function isStorageStateShape(state) {
|
|
|
9184
9326
|
return typeof state === "object" && state !== null && Array.isArray(state.cookies) && Array.isArray(state.origins);
|
|
9185
9327
|
}
|
|
9186
9328
|
//#endregion
|
|
9329
|
+
//#region src/runtime/env-scrub.ts
|
|
9330
|
+
/**
|
|
9331
|
+
* Build a list of `[envValue, "${VAR}"]` pairs for every `${VAR}` reference
|
|
9332
|
+
* mentioned in the spec OR in any of its expanded (block-inlined) steps.
|
|
9333
|
+
* Used at trace time to scrub recorded Claude-text outputs so a value the
|
|
9334
|
+
* spec author intentionally threaded through `process.env` is preserved as
|
|
9335
|
+
* `${VAR}` in `ir.json` rather than baked in as the concrete
|
|
9336
|
+
* trace-time value.
|
|
9337
|
+
*
|
|
9338
|
+
* Why we walk `spec.steps` AND `expanded`:
|
|
9339
|
+
* - `spec.steps` carries the spec's own `instruction` / `expected` + each
|
|
9340
|
+
* include's raw `params` (which may themselves be `${ENV}` refs).
|
|
9341
|
+
* - `expanded` carries the inlined block-internal steps, whose
|
|
9342
|
+
* `instruction` / `expected` may *also* contain `${ENV}` refs that
|
|
9343
|
+
* don't go through include params.
|
|
9344
|
+
*
|
|
9345
|
+
* Only refs whose env value is currently non-empty land in the map —
|
|
9346
|
+
* scrubbing against an empty string would corrupt unrelated empty strings
|
|
9347
|
+
* in the action stream. Names whose env is unset are returned via
|
|
9348
|
+
* `unresolved` so the caller can warn the user.
|
|
9349
|
+
*
|
|
9350
|
+
* Longer values sort first so a `${SHORT}` whose value is a substring of a
|
|
9351
|
+
* `${LONG}` value doesn't clobber the longer one.
|
|
9352
|
+
*
|
|
9353
|
+
* `title` is deliberately NOT scanned — it never reaches the recorded action
|
|
9354
|
+
* stream.
|
|
9355
|
+
*/
|
|
9356
|
+
function buildSpecEnvScrub(spec, expanded) {
|
|
9357
|
+
const refNames = /* @__PURE__ */ new Set();
|
|
9358
|
+
for (const step of spec.steps) if (isIncludeStep(step)) for (const v of Object.values(step.params ?? {})) collect(v, refNames);
|
|
9359
|
+
else {
|
|
9360
|
+
collect(step.instruction, refNames);
|
|
9361
|
+
collect(step.expected, refNames);
|
|
9362
|
+
}
|
|
9363
|
+
for (const step of expanded) {
|
|
9364
|
+
collect(step.instruction, refNames);
|
|
9365
|
+
collect(step.expected, refNames);
|
|
9366
|
+
}
|
|
9367
|
+
const map = [];
|
|
9368
|
+
const unresolved = [];
|
|
9369
|
+
for (const name of refNames) {
|
|
9370
|
+
const value = process.env[name];
|
|
9371
|
+
if (typeof value === "string" && value.length > 0) map.push([value, "${" + name + "}"]);
|
|
9372
|
+
else unresolved.push(name);
|
|
9373
|
+
}
|
|
9374
|
+
map.sort((a, b) => b[0].length - a[0].length);
|
|
9375
|
+
return {
|
|
9376
|
+
map,
|
|
9377
|
+
unresolved
|
|
9378
|
+
};
|
|
9379
|
+
}
|
|
9380
|
+
function collect(value, into) {
|
|
9381
|
+
for (const name of iterEnvRefNames(value)) into.add(name);
|
|
9382
|
+
}
|
|
9383
|
+
/** Shorter than this, a value is no secret and matches inside ordinary words. */
|
|
9384
|
+
const MIN_PROSE_SCRUB_LENGTH = 4;
|
|
9385
|
+
/** Long enough to clear the length bar, still ordinary prose / JSON. */
|
|
9386
|
+
const COMMON_PROSE_VALUES = new Set([
|
|
9387
|
+
"true",
|
|
9388
|
+
"false",
|
|
9389
|
+
"null",
|
|
9390
|
+
"none",
|
|
9391
|
+
"undefined"
|
|
9392
|
+
]);
|
|
9393
|
+
/**
|
|
9394
|
+
* Scrub map for a live run, built like {@link buildSpecEnvScrub} but without
|
|
9395
|
+
* the values that read as ordinary text (`"1"`, `"true"`). A live step records
|
|
9396
|
+
* paragraphs of model prose, so replacing every occurrence of such a value
|
|
9397
|
+
* would cost more meaning than it protects; record scrubs single command
|
|
9398
|
+
* lines, where the same trade favours keeping them.
|
|
9399
|
+
*/
|
|
9400
|
+
function buildLiveEnvScrubMap(spec, expanded) {
|
|
9401
|
+
return buildSpecEnvScrub(spec, expanded).map.filter(([value]) => value.length >= MIN_PROSE_SCRUB_LENGTH && !COMMON_PROSE_VALUES.has(value.toLowerCase()));
|
|
9402
|
+
}
|
|
9403
|
+
/**
|
|
9404
|
+
* Replace every occurrence of an env value with its `${VAR}` placeholder in
|
|
9405
|
+
* `text`. **Caller invariant**: the map must be sorted longest-value-first
|
|
9406
|
+
* so a shorter value doesn't shadow a longer one that contains it as a
|
|
9407
|
+
* substring. `buildSpecEnvScrub` upholds this; hand-built maps should too.
|
|
9408
|
+
*/
|
|
9409
|
+
function scrubEnvValues(text, scrubMap) {
|
|
9410
|
+
if (scrubMap.length === 0) return text;
|
|
9411
|
+
let out = text;
|
|
9412
|
+
for (const [value, placeholder] of scrubMap) if (out.includes(value)) out = out.replaceAll(value, placeholder);
|
|
9413
|
+
return out;
|
|
9414
|
+
}
|
|
9415
|
+
//#endregion
|
|
9187
9416
|
//#region src/claude/agent-browser-invoke.ts
|
|
9188
9417
|
function agentBrowserInvokeBase(input) {
|
|
9189
9418
|
return {
|
|
@@ -9446,7 +9675,7 @@ async function runLiveExecutor(input) {
|
|
|
9446
9675
|
info(` retry ${attempt}/${retries} for ${step$1.id}`);
|
|
9447
9676
|
}
|
|
9448
9677
|
const outcome = lastOutcome;
|
|
9449
|
-
|
|
9678
|
+
const recorded = scrubLiveStepText({
|
|
9450
9679
|
stepId: step$1.id,
|
|
9451
9680
|
source: step$1.source,
|
|
9452
9681
|
instruction: step$1.instruction,
|
|
@@ -9459,10 +9688,11 @@ async function runLiveExecutor(input) {
|
|
|
9459
9688
|
durationMs: Date.now() - stepStartedAt,
|
|
9460
9689
|
cost: outcome.cost,
|
|
9461
9690
|
commands: outcome.commands
|
|
9462
|
-
});
|
|
9463
|
-
|
|
9691
|
+
}, input.envScrubMap);
|
|
9692
|
+
stepResults.push(recorded);
|
|
9693
|
+
if (outcome.status === "passed") step("STEP_DONE", step$1.id, recorded.reasoning);
|
|
9464
9694
|
else {
|
|
9465
|
-
step("ASSERTION_FAILED", step$1.id,
|
|
9695
|
+
step("ASSERTION_FAILED", step$1.id, recorded.reasoning);
|
|
9466
9696
|
overallFailed = true;
|
|
9467
9697
|
}
|
|
9468
9698
|
}
|
|
@@ -9502,7 +9732,8 @@ async function runLiveExecutor(input) {
|
|
|
9502
9732
|
const transcript = transcriptParts.join("\n");
|
|
9503
9733
|
const after = takeScreenshot(input.sessionName, paths.afterPng, { fullPage: true });
|
|
9504
9734
|
if (!after.ok) warn(`screenshot (after, ${step.id}) failed: ${after.error}`);
|
|
9505
|
-
|
|
9735
|
+
const scrubbed = scrubEnvValues(transcript, input.envScrubMap);
|
|
9736
|
+
await writeFile(paths.logTxt, scrubbed || "(no assistant text captured)", "utf-8");
|
|
9506
9737
|
const { status, reasoning } = judgeStepOutcome({
|
|
9507
9738
|
step,
|
|
9508
9739
|
isError,
|
|
@@ -9604,6 +9835,18 @@ function judgeStepOutcome({ step, isError, errorDetail, judged }) {
|
|
|
9604
9835
|
reasoning: judged.stepId === step.id ? baseReason : `(stepId mismatch: model wrote ${judged.stepId}) ${baseReason}`
|
|
9605
9836
|
};
|
|
9606
9837
|
}
|
|
9838
|
+
/**
|
|
9839
|
+
* Scrub the strings a step carries that the model authored: its verdict prose
|
|
9840
|
+
* and the commands it issued. `instruction` / `expected` are copied from the
|
|
9841
|
+
* spec, which keeps `${VAR}` symbolic, so they need nothing.
|
|
9842
|
+
*/
|
|
9843
|
+
function scrubLiveStepText(step, scrubMap) {
|
|
9844
|
+
return {
|
|
9845
|
+
...step,
|
|
9846
|
+
reasoning: scrubEnvValues(step.reasoning, scrubMap),
|
|
9847
|
+
commands: step.commands.map((c) => scrubEnvValues(c, scrubMap))
|
|
9848
|
+
};
|
|
9849
|
+
}
|
|
9607
9850
|
function buildSkippedStep(step, reason) {
|
|
9608
9851
|
return {
|
|
9609
9852
|
stepId: step.id,
|
|
@@ -10015,6 +10258,7 @@ async function runOneSpec(args) {
|
|
|
10015
10258
|
}
|
|
10016
10259
|
const spec = parseTestSpec(specContent);
|
|
10017
10260
|
const expanded = expandSpec(spec, { blocks: await loadAllBlocks(cwd) });
|
|
10261
|
+
const envScrubMap = buildLiveEnvScrubMap(spec, expanded);
|
|
10018
10262
|
meta("spec", spec.title);
|
|
10019
10263
|
meta("steps", expanded.length);
|
|
10020
10264
|
const includes = collectIncludedBlockNames(spec);
|
|
@@ -10053,6 +10297,7 @@ async function runOneSpec(args) {
|
|
|
10053
10297
|
runId,
|
|
10054
10298
|
runDir,
|
|
10055
10299
|
sessionName,
|
|
10300
|
+
envScrubMap,
|
|
10056
10301
|
statePath,
|
|
10057
10302
|
verifyUrl,
|
|
10058
10303
|
systemPromptSuffix: userPromptSuffix,
|
|
@@ -12565,6 +12810,7 @@ async function executeRun(targets, opts) {
|
|
|
12565
12810
|
const forExecution = opts.dryRun !== true;
|
|
12566
12811
|
const cwd = opts.cwd ?? process.cwd();
|
|
12567
12812
|
const wantsLastGreen = opts.onFailExplain === true && opts.onFailExplainBase === void 0;
|
|
12813
|
+
const rerunMode = opts.onFailExplain === true ? opts.onFailExplainRerun ?? "never" : "never";
|
|
12568
12814
|
const [head, fixedBase] = await Promise.all([getGitHead(cwd), forExecution && opts.onFailExplain && opts.onFailExplainBase !== void 0 ? resolveAnalysisBase(opts.onFailExplainBase, "--on-fail-explain-base", cwd) : null]);
|
|
12569
12815
|
const git = {
|
|
12570
12816
|
head,
|
|
@@ -12736,6 +12982,7 @@ async function executeRun(targets, opts) {
|
|
|
12736
12982
|
if (opts.liveArtifactsDir) warn(`--live-artifacts-dir is ignored: ${why}`);
|
|
12737
12983
|
if (opts.learnHubLivePrompt) warn(`--learn-live-prompt is ignored: ${why}`);
|
|
12738
12984
|
} else if (opts.liveArtifactsDir && liveSpecs.length > 1) warn("--out is ignored when running multiple live specs");
|
|
12985
|
+
if (opts.onFailExplainRerun !== void 0 && rerunMode === "never" && opts.onFailExplainRerun !== "never") warn("--on-fail-explain-rerun is ignored: without --on-fail-explain nothing is classified, so a second attempt would settle nothing");
|
|
12739
12986
|
if (detSpecs.length === 0 && opts.replaySkipEvidence === true) warn("--no-evidence is ignored: it only applies to agent-browser 'mode: deterministic' specs, and this run has none");
|
|
12740
12987
|
blank();
|
|
12741
12988
|
if (opts.dryRun) {
|
|
@@ -12817,7 +13064,7 @@ async function executeRun(targets, opts) {
|
|
|
12817
13064
|
...opts.language ? { language: opts.language } : {},
|
|
12818
13065
|
report: incrementalReport
|
|
12819
13066
|
});
|
|
12820
|
-
const
|
|
13067
|
+
const liveOpts = {
|
|
12821
13068
|
...opts.model ? { model: opts.model } : {},
|
|
12822
13069
|
...opts.language ? { language: opts.language } : {},
|
|
12823
13070
|
...opts.liveArtifactsDir && liveSpecs.length === 1 ? { out: opts.liveArtifactsDir } : {},
|
|
@@ -12833,7 +13080,8 @@ async function executeRun(targets, opts) {
|
|
|
12833
13080
|
triageUserPrompt,
|
|
12834
13081
|
...opts.teardown ? { teardown: opts.teardown } : {},
|
|
12835
13082
|
report: incrementalReport
|
|
12836
|
-
}
|
|
13083
|
+
};
|
|
13084
|
+
const live = await runLiveSpecs(liveSpecs, liveOpts);
|
|
12837
13085
|
let overallExitCode = det.exitCode !== 0 ? 1 : 0;
|
|
12838
13086
|
if (live.failedCount > 0) overallExitCode = 1;
|
|
12839
13087
|
if (externalRows.some((r) => r.status === "failed")) overallExitCode = 1;
|
|
@@ -12851,11 +13099,23 @@ async function executeRun(targets, opts) {
|
|
|
12851
13099
|
const analyzedExternalRows = await analyzeExternalRows(externalRows, analysisRun);
|
|
12852
13100
|
report = await writeUnifiedReport({
|
|
12853
13101
|
reportDir,
|
|
12854
|
-
results: [
|
|
13102
|
+
results: await rerunExplainedFailures([
|
|
12855
13103
|
...detResults,
|
|
12856
13104
|
...analyzedExternalRows,
|
|
12857
13105
|
...live.reportResults
|
|
12858
|
-
],
|
|
13106
|
+
], {
|
|
13107
|
+
mode: rerunMode,
|
|
13108
|
+
maxSpecs: opts.onFailExplainRerunMaxSpecs ?? null,
|
|
13109
|
+
execute: createRerunExecutor({
|
|
13110
|
+
detSpecs,
|
|
13111
|
+
liveSpecs,
|
|
13112
|
+
dispatch,
|
|
13113
|
+
liveOpts,
|
|
13114
|
+
opts,
|
|
13115
|
+
cwd,
|
|
13116
|
+
resources
|
|
13117
|
+
})
|
|
13118
|
+
}),
|
|
12859
13119
|
git,
|
|
12860
13120
|
customPromptVersion,
|
|
12861
13121
|
triageUserPromptHash,
|
|
@@ -13135,6 +13395,68 @@ async function analyzeDeterministicSummaries(summaries, cwd, reportDir, { pass }
|
|
|
13135
13395
|
return results;
|
|
13136
13396
|
}
|
|
13137
13397
|
/**
|
|
13398
|
+
* How `--on-fail-explain-rerun` re-executes one spec: on the same path that
|
|
13399
|
+
* just ran it, with the same inputs, so the second attempt answers the same
|
|
13400
|
+
* question the first did.
|
|
13401
|
+
*
|
|
13402
|
+
* Everything it writes goes to a throwaway report directory. The failed run's
|
|
13403
|
+
* screenshots and artifacts are the record of what happened, and a rerun that
|
|
13404
|
+
* shares their paths would overwrite them with a different attempt's — the
|
|
13405
|
+
* external runner and the live adapter both recreate a spec's directories.
|
|
13406
|
+
* Nothing here upserts a row or classifies: the attempt is evidence about the
|
|
13407
|
+
* run's row, not a row of its own.
|
|
13408
|
+
*/
|
|
13409
|
+
function createRerunExecutor(ctx) {
|
|
13410
|
+
const detKeys = new Set(ctx.detSpecs.map(specKey));
|
|
13411
|
+
const liveKeys = new Set(ctx.liveSpecs.map(specKey));
|
|
13412
|
+
return async (ref) => {
|
|
13413
|
+
const key = specKey(ref);
|
|
13414
|
+
const scratch = await mkdtemp(join(tmpdir(), "ccqa-rerun-"));
|
|
13415
|
+
try {
|
|
13416
|
+
if (detKeys.has(key)) {
|
|
13417
|
+
const summary = await runOneDeterministicSpec(ref, 0, {
|
|
13418
|
+
cwd: ctx.cwd,
|
|
13419
|
+
tmpDir: scratch,
|
|
13420
|
+
vitestConfig: await resolveVitestConfig(ctx.cwd),
|
|
13421
|
+
captureOutput: false,
|
|
13422
|
+
reportDir: scratch,
|
|
13423
|
+
captureEvidence: false
|
|
13424
|
+
});
|
|
13425
|
+
return summary !== null && !failedSpec(summary) ? "passed" : "failed";
|
|
13426
|
+
}
|
|
13427
|
+
if (liveKeys.has(key)) {
|
|
13428
|
+
const { report: _streamed, ...liveOpts } = ctx.liveOpts;
|
|
13429
|
+
return (await runLiveSpecs([ref], {
|
|
13430
|
+
...liveOpts,
|
|
13431
|
+
reportDir: scratch,
|
|
13432
|
+
diffProvider: null,
|
|
13433
|
+
concurrency: 1
|
|
13434
|
+
})).failedCount > 0 ? "failed" : "passed";
|
|
13435
|
+
}
|
|
13436
|
+
const group = ctx.dispatch.external.find((g) => g.specs.some((s) => specKey(s) === key));
|
|
13437
|
+
if (group === void 0) throw new Error(`${key} has no execution path to re-run`);
|
|
13438
|
+
const [row] = await group.runner.run([ref], {
|
|
13439
|
+
cwd: ctx.cwd,
|
|
13440
|
+
reportDir: scratch,
|
|
13441
|
+
concurrency: 1,
|
|
13442
|
+
resources: ctx.resources,
|
|
13443
|
+
...ctx.opts.model ? { model: ctx.opts.model } : {},
|
|
13444
|
+
...ctx.opts.language ? { language: ctx.opts.language } : {},
|
|
13445
|
+
targetId: group.targetId,
|
|
13446
|
+
targetConfig: group.targetConfig,
|
|
13447
|
+
stepEvidence: group.stepEvidence,
|
|
13448
|
+
onSpecComplete: async () => {}
|
|
13449
|
+
});
|
|
13450
|
+
return row?.status === "passed" ? "passed" : "failed";
|
|
13451
|
+
} finally {
|
|
13452
|
+
await rm(scratch, {
|
|
13453
|
+
recursive: true,
|
|
13454
|
+
force: true
|
|
13455
|
+
});
|
|
13456
|
+
}
|
|
13457
|
+
};
|
|
13458
|
+
}
|
|
13459
|
+
/**
|
|
13138
13460
|
* Build the report envelope — every `RunReportData` field except `results`.
|
|
13139
13461
|
* Extracted so the incremental writer (which flushes report.json after each
|
|
13140
13462
|
* spec) and the final batch write share one source of truth for these fields.
|
|
@@ -13434,7 +13756,10 @@ const runCommand = addHubOptions(addProfileOption(addLanguageOption(new Command(
|
|
|
13434
13756
|
const n = Number(raw);
|
|
13435
13757
|
if (!Number.isFinite(n) || n < 0 || Math.floor(n) !== n) throw new Error(`--live-step-retry must be a non-negative integer, got "${raw}"`);
|
|
13436
13758
|
return n;
|
|
13437
|
-
}, 0).option("--live-artifacts-dir <dir>", "(live only) Override the per-spec artifact directory. Default: <specDir>/runs/<runId>. Ignored when running multiple specs.").option("--replay-skip-evidence", `(deterministic replay only) Skip step-boundary evidence capture (PNG + meta JSON written to ${DEFAULT_REPORT_DIR}/${EVIDENCE_SUBDIR}/ by default).`).optionsGroup("What to do about failures:").option("--on-fail-explain", "Classify each failure against the source diff since the commit where that spec last passed (per-spec baselines from the hub). Off by default — no Claude calls without it.").option("--on-fail-explain-base <ref>", "With --on-fail-explain: diff against <ref> instead of each spec's last green. Use when there is no hub to hold the baselines.").
|
|
13759
|
+
}, 0).option("--live-artifacts-dir <dir>", "(live only) Override the per-spec artifact directory. Default: <specDir>/runs/<runId>. Ignored when running multiple specs.").option("--replay-skip-evidence", `(deterministic replay only) Skip step-boundary evidence capture (PNG + meta JSON written to ${DEFAULT_REPORT_DIR}/${EVIDENCE_SUBDIR}/ by default).`).optionsGroup("What to do about failures:").option("--on-fail-explain", "Classify each failure against the source diff since the commit where that spec last passed (per-spec baselines from the hub). Off by default — no Claude calls without it.").option("--on-fail-explain-base <ref>", "With --on-fail-explain: diff against <ref> instead of each spec's last green. Use when there is no hub to hold the baselines.").option("--on-fail-explain-rerun <when>", "With --on-fail-explain: run a failed spec a second time so the classifier can tell a flake from a real failure. 'auto' reruns the failures whose label turns on it (UNKNOWN, ENVIRONMENT), 'always' every classified failure, 'never' (default) none. A second attempt that passes labels the row ENVIRONMENT; the spec still counts as failed. Costs a full spec execution each — live specs included.", (raw) => {
|
|
13760
|
+
if (EXPLAIN_RERUN_MODES.includes(raw)) return raw;
|
|
13761
|
+
throw new Error(`--on-fail-explain-rerun must be one of ${EXPLAIN_RERUN_MODES.join(" | ")}`);
|
|
13762
|
+
}, "never").option("--on-fail-explain-rerun-max-specs <n>", "Rerun at most N specs, in report order; the rest are named in the run summary and keep the label they were first given. Default: no cap. The knob for an environment having a bad day, where the alternative is turning the reruns off entirely.", parseRerunMaxSpecs).optionsGroup("What to do with the results:").option("--report-dir <dir>", `Directory for the structured run results (report.json + evidence PNGs), which are always written. Default: ${DEFAULT_REPORT_DIR}/.`).option("--report-format <fmt>", "Additional output format alongside HTML: 'text' (default), 'json' (writes report.json), 'github' (GitHub Actions annotations on stdout).", (raw) => {
|
|
13438
13763
|
if (REPORT_FORMATS.includes(raw)) return raw;
|
|
13439
13764
|
throw new Error(`--report-format must be one of ${REPORT_FORMATS.join(" | ")}`);
|
|
13440
13765
|
}, "text").option("--report-to-hub", "Incrementally push the run report to the hub as the run progresses (open → patch per spec → finalize). Requires --hub-url/--hub-token (or CCQA_HUB_URL/CCQA_HUB_TOKEN). Without it, hub credentials are used only to fetch variables/sessions/prompts, not to push.").optionsGroup("Learning:").option("--learn-hub-live-prompt", "(live only) After the run finishes, ask Claude to refresh the \"live.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(async (targets, opts) => {
|
|
@@ -13449,6 +13774,12 @@ function parseConcurrency$1(raw) {
|
|
|
13449
13774
|
}
|
|
13450
13775
|
return n;
|
|
13451
13776
|
}
|
|
13777
|
+
/** Parse --on-fail-explain-rerun-max-specs: a positive integer. Zero is `--on-fail-explain-rerun never`. */
|
|
13778
|
+
function parseRerunMaxSpecs(raw) {
|
|
13779
|
+
const n = Number(raw);
|
|
13780
|
+
if (!Number.isInteger(n) || n < 1) throw new Error(`--on-fail-explain-rerun-max-specs must be a positive integer, got "${raw}" (for none, pass --on-fail-explain-rerun never)`);
|
|
13781
|
+
return n;
|
|
13782
|
+
}
|
|
13452
13783
|
/** Header label shown after `ccqa run`: the lone target, a count, or how they were selected. */
|
|
13453
13784
|
function headerTarget(targets, opts) {
|
|
13454
13785
|
if (targets.length === 1) return targets[0];
|
|
@@ -14270,73 +14601,6 @@ function isPassiveAction(action) {
|
|
|
14270
14601
|
return action === "snapshot" || action === "wait" || action === "assert";
|
|
14271
14602
|
}
|
|
14272
14603
|
//#endregion
|
|
14273
|
-
//#region src/runtime/env-scrub.ts
|
|
14274
|
-
/**
|
|
14275
|
-
* Build a list of `[envValue, "${VAR}"]` pairs for every `${VAR}` reference
|
|
14276
|
-
* mentioned in the spec OR in any of its expanded (block-inlined) steps.
|
|
14277
|
-
* Used at trace time to scrub recorded Claude-text outputs so a value the
|
|
14278
|
-
* spec author intentionally threaded through `process.env` is preserved as
|
|
14279
|
-
* `${VAR}` in `ir.json` rather than baked in as the concrete
|
|
14280
|
-
* trace-time value.
|
|
14281
|
-
*
|
|
14282
|
-
* Why we walk `spec.steps` AND `expanded`:
|
|
14283
|
-
* - `spec.steps` carries the spec's own `instruction` / `expected` + each
|
|
14284
|
-
* include's raw `params` (which may themselves be `${ENV}` refs).
|
|
14285
|
-
* - `expanded` carries the inlined block-internal steps, whose
|
|
14286
|
-
* `instruction` / `expected` may *also* contain `${ENV}` refs that
|
|
14287
|
-
* don't go through include params.
|
|
14288
|
-
*
|
|
14289
|
-
* Only refs whose env value is currently non-empty land in the map —
|
|
14290
|
-
* scrubbing against an empty string would corrupt unrelated empty strings
|
|
14291
|
-
* in the action stream. Names whose env is unset are returned via
|
|
14292
|
-
* `unresolved` so the caller can warn the user.
|
|
14293
|
-
*
|
|
14294
|
-
* Longer values sort first so a `${SHORT}` whose value is a substring of a
|
|
14295
|
-
* `${LONG}` value doesn't clobber the longer one.
|
|
14296
|
-
*
|
|
14297
|
-
* `title` is deliberately NOT scanned — it never reaches the recorded action
|
|
14298
|
-
* stream.
|
|
14299
|
-
*/
|
|
14300
|
-
function buildSpecEnvScrub(spec, expanded) {
|
|
14301
|
-
const refNames = /* @__PURE__ */ new Set();
|
|
14302
|
-
for (const step of spec.steps) if (isIncludeStep(step)) for (const v of Object.values(step.params ?? {})) collect(v, refNames);
|
|
14303
|
-
else {
|
|
14304
|
-
collect(step.instruction, refNames);
|
|
14305
|
-
collect(step.expected, refNames);
|
|
14306
|
-
}
|
|
14307
|
-
for (const step of expanded) {
|
|
14308
|
-
collect(step.instruction, refNames);
|
|
14309
|
-
collect(step.expected, refNames);
|
|
14310
|
-
}
|
|
14311
|
-
const map = [];
|
|
14312
|
-
const unresolved = [];
|
|
14313
|
-
for (const name of refNames) {
|
|
14314
|
-
const value = process.env[name];
|
|
14315
|
-
if (typeof value === "string" && value.length > 0) map.push([value, "${" + name + "}"]);
|
|
14316
|
-
else unresolved.push(name);
|
|
14317
|
-
}
|
|
14318
|
-
map.sort((a, b) => b[0].length - a[0].length);
|
|
14319
|
-
return {
|
|
14320
|
-
map,
|
|
14321
|
-
unresolved
|
|
14322
|
-
};
|
|
14323
|
-
}
|
|
14324
|
-
function collect(value, into) {
|
|
14325
|
-
for (const name of iterEnvRefNames(value)) into.add(name);
|
|
14326
|
-
}
|
|
14327
|
-
/**
|
|
14328
|
-
* Replace every occurrence of an env value with its `${VAR}` placeholder in
|
|
14329
|
-
* `text`. **Caller invariant**: the map must be sorted longest-value-first
|
|
14330
|
-
* so a shorter value doesn't shadow a longer one that contains it as a
|
|
14331
|
-
* substring. `buildSpecEnvScrub` upholds this; hand-built maps should too.
|
|
14332
|
-
*/
|
|
14333
|
-
function scrubEnvValues(text, scrubMap) {
|
|
14334
|
-
if (scrubMap.length === 0) return text;
|
|
14335
|
-
let out = text;
|
|
14336
|
-
for (const [value, placeholder] of scrubMap) if (out.includes(value)) out = out.replaceAll(value, placeholder);
|
|
14337
|
-
return out;
|
|
14338
|
-
}
|
|
14339
|
-
//#endregion
|
|
14340
14604
|
//#region src/runtime/literal-scrub.ts
|
|
14341
14605
|
/**
|
|
14342
14606
|
* Patterns are listed in roughly descending confidence — a hit on `clock-hms`
|
|
@@ -17132,6 +17396,32 @@ function mergeBucket(into, from) {
|
|
|
17132
17396
|
}
|
|
17133
17397
|
}
|
|
17134
17398
|
//#endregion
|
|
17399
|
+
//#region src/hub/core/retention.ts
|
|
17400
|
+
/**
|
|
17401
|
+
* Drop everything past the newest `maxRuns` of the (project, branch) that
|
|
17402
|
+
* `run` belongs to, taking each evicted run's artifacts and triage records
|
|
17403
|
+
* with it.
|
|
17404
|
+
*
|
|
17405
|
+
* Runs at a terminal state trigger this, rather than a sweep at startup: the
|
|
17406
|
+
* hub whose disk grows is the one written to constantly and never restarted,
|
|
17407
|
+
* which is exactly the one a startup sweep never fires on.
|
|
17408
|
+
*
|
|
17409
|
+
* Best-effort like the ledger updates it runs beside — a lost sweep costs
|
|
17410
|
+
* disk, while failing the push it hangs off would cost the run.
|
|
17411
|
+
*/
|
|
17412
|
+
async function sweepRunRetention(storage, run, maxRuns) {
|
|
17413
|
+
try {
|
|
17414
|
+
const group = (await storage.runs.list({ project: run.project })).filter((r) => r.branch === run.branch && r.status !== "running");
|
|
17415
|
+
for (const evicted of group.slice(maxRuns)) {
|
|
17416
|
+
await storage.runs.delete(evicted.id);
|
|
17417
|
+
await storage.artifacts.delete(evicted.id);
|
|
17418
|
+
await storage.triage.deleteAll(evicted.id);
|
|
17419
|
+
}
|
|
17420
|
+
} catch (err) {
|
|
17421
|
+
console.error(`hub: run retention sweep failed for "${run.project}": ${err instanceof Error ? err.message : String(err)}`);
|
|
17422
|
+
}
|
|
17423
|
+
}
|
|
17424
|
+
//#endregion
|
|
17135
17425
|
//#region src/hub/api/validate.ts
|
|
17136
17426
|
/**
|
|
17137
17427
|
* Validators for the request parameters handlers read straight off the wire:
|
|
@@ -17195,6 +17485,7 @@ const DEFAULT_MAX_PUSH_BYTES = 32 * 1024 * 1024;
|
|
|
17195
17485
|
*/
|
|
17196
17486
|
function createPushRunHandler(config) {
|
|
17197
17487
|
const maxPushBytes = config.maxPushBytes ?? DEFAULT_MAX_PUSH_BYTES;
|
|
17488
|
+
const maxRunsPerBranch = config.maxRunsPerBranch ?? 200;
|
|
17198
17489
|
return async (ctx) => {
|
|
17199
17490
|
const { project, branch, profile, kind, deployedSha } = parseRunScope(ctx);
|
|
17200
17491
|
const body = await readBody(ctx.req, maxPushBytes);
|
|
@@ -17246,6 +17537,7 @@ function createPushRunHandler(config) {
|
|
|
17246
17537
|
await config.storage.runs.create(run);
|
|
17247
17538
|
await updateSpecLedger(config.storage, run, report.results);
|
|
17248
17539
|
await updateDriftLedger(config.storage, run, report.results);
|
|
17540
|
+
await sweepRunRetention(config.storage, run, maxRunsPerBranch);
|
|
17249
17541
|
sendJson(ctx.res, 201, run);
|
|
17250
17542
|
} finally {
|
|
17251
17543
|
await rm(dir, {
|
|
@@ -17485,6 +17777,7 @@ async function deployHeadMovedDuringRun(storage, run) {
|
|
|
17485
17777
|
*/
|
|
17486
17778
|
function createPatchRunHandler(config) {
|
|
17487
17779
|
const maxPushBytes = config.maxPushBytes ?? DEFAULT_MAX_PUSH_BYTES;
|
|
17780
|
+
const maxRunsPerBranch = config.maxRunsPerBranch ?? 200;
|
|
17488
17781
|
return async (ctx) => {
|
|
17489
17782
|
const id = ctx.params.id;
|
|
17490
17783
|
const run = await getRunOr404(config.storage, id);
|
|
@@ -17548,7 +17841,10 @@ function createPatchRunHandler(config) {
|
|
|
17548
17841
|
};
|
|
17549
17842
|
const updated = await config.storage.runs.update(id, patch);
|
|
17550
17843
|
await updateDriftLedger(config.storage, updated, mergedResults);
|
|
17551
|
-
if (done)
|
|
17844
|
+
if (done) {
|
|
17845
|
+
await updateSpecLedger(config.storage, updated, mergedResults);
|
|
17846
|
+
await sweepRunRetention(config.storage, updated, maxRunsPerBranch);
|
|
17847
|
+
}
|
|
17552
17848
|
sendJson(ctx.res, 200, updated);
|
|
17553
17849
|
};
|
|
17554
17850
|
}
|
|
@@ -20186,6 +20482,7 @@ const CLIENT_JS = `
|
|
|
20186
20482
|
"detail.back": "Runs", "detail.specs": "Specs",
|
|
20187
20483
|
"detail.download": "Download artifacts",
|
|
20188
20484
|
"detail.triage": "Triage",
|
|
20485
|
+
"detail.notKept": "This run is no longer kept — the hub keeps only the most recent runs of each branch.",
|
|
20189
20486
|
"meta.branch": "Branch", "meta.specs": "Specs", "meta.cost": "Cost",
|
|
20190
20487
|
"meta.created": "Created", "meta.passed": "passed", "meta.profile": "Profile",
|
|
20191
20488
|
"meta.drift": "Drift",
|
|
@@ -20349,6 +20646,7 @@ const CLIENT_JS = `
|
|
|
20349
20646
|
"detail.back": "実行", "detail.specs": "スペック",
|
|
20350
20647
|
"detail.download": "アーティファクトをダウンロード",
|
|
20351
20648
|
"detail.triage": "トリアージ",
|
|
20649
|
+
"detail.notKept": "この実行はもう保持されていません — ハブは各ブランチの直近の実行だけを保持します。",
|
|
20352
20650
|
"meta.branch": "ブランチ", "meta.specs": "スペック", "meta.cost": "コスト",
|
|
20353
20651
|
"meta.created": "作成", "meta.passed": "合格", "meta.profile": "プロファイル",
|
|
20354
20652
|
"meta.drift": "ドリフト",
|
|
@@ -20639,7 +20937,9 @@ const CLIENT_JS = `
|
|
|
20639
20937
|
// A reverse proxy can answer with non-JSON (an HTML 502 page) — fall
|
|
20640
20938
|
// back to the status line instead of a JSON-parse error message.
|
|
20641
20939
|
return res.json().catch(function () { return null; }).then(function (b) {
|
|
20642
|
-
|
|
20940
|
+
var err = new Error((b && b.error && b.error.message) || (res.status + " " + res.statusText));
|
|
20941
|
+
err.status = res.status;
|
|
20942
|
+
throw err;
|
|
20643
20943
|
});
|
|
20644
20944
|
}
|
|
20645
20945
|
return res.status === 204 ? null : res.json();
|
|
@@ -22036,12 +22336,10 @@ const CLIENT_JS = `
|
|
|
22036
22336
|
|
|
22037
22337
|
// ── run detail: orchestration ───────────────────────────────────────
|
|
22038
22338
|
|
|
22039
|
-
function detailError(
|
|
22040
|
-
|
|
22041
|
-
|
|
22042
|
-
|
|
22043
|
-
e.textContent = what + ": " + err.message;
|
|
22044
|
-
};
|
|
22339
|
+
function detailError(msg) {
|
|
22340
|
+
var e = document.getElementById("detail-error");
|
|
22341
|
+
e.hidden = false;
|
|
22342
|
+
e.textContent = msg;
|
|
22045
22343
|
}
|
|
22046
22344
|
|
|
22047
22345
|
function openRunDetail(runId) {
|
|
@@ -22057,9 +22355,17 @@ const CLIENT_JS = `
|
|
|
22057
22355
|
document.getElementById("detail-spec-count").textContent = "";
|
|
22058
22356
|
document.getElementById("triage-summary").textContent = "";
|
|
22059
22357
|
|
|
22358
|
+
// Retention drops a run but never the ledger entries pointing at it, so a
|
|
22359
|
+
// Perspectives link can outlive its target. That 404 is the whole story of
|
|
22360
|
+
// the page, so it speaks for the report's failure too.
|
|
22361
|
+
var runGone = false;
|
|
22362
|
+
|
|
22060
22363
|
apiFetch("/api/v1/runs/" + encodeURIComponent(runId)).then(function (run) {
|
|
22061
22364
|
renderRunHead(run);
|
|
22062
|
-
}).catch(
|
|
22365
|
+
}).catch(function (err) {
|
|
22366
|
+
runGone = err.status === 404;
|
|
22367
|
+
detailError(runGone ? t("detail.notKept") : "Error loading run: " + err.message);
|
|
22368
|
+
});
|
|
22063
22369
|
|
|
22064
22370
|
apiFetch("/api/v1/runs/" + encodeURIComponent(runId) + "/report").then(function (report) {
|
|
22065
22371
|
// Draw the spec cards first from the report alone, then re-draw once
|
|
@@ -22075,7 +22381,9 @@ const CLIENT_JS = `
|
|
|
22075
22381
|
loadTriage(runId, isDrift, function (loaded) {
|
|
22076
22382
|
renderSpecCards(runId, report.results, loaded, isDrift);
|
|
22077
22383
|
});
|
|
22078
|
-
}).catch(
|
|
22384
|
+
}).catch(function (err) {
|
|
22385
|
+
if (!runGone) detailError("Error loading report: " + err.message);
|
|
22386
|
+
});
|
|
22079
22387
|
}
|
|
22080
22388
|
|
|
22081
22389
|
// ── learning jobs ────────────────────────────────────────────────────
|
|
@@ -24477,14 +24785,17 @@ function registerRoutes(router, config, queue) {
|
|
|
24477
24785
|
ctx.res.setHeader("Cache-Control", "no-cache");
|
|
24478
24786
|
ctx.res.end(renderHubUi());
|
|
24479
24787
|
});
|
|
24788
|
+
const retention = config.maxRunsPerBranch != null ? { maxRunsPerBranch: config.maxRunsPerBranch } : {};
|
|
24480
24789
|
router.post("/api/v1/runs", createPushRunHandler({
|
|
24481
24790
|
storage,
|
|
24482
|
-
...config.maxPushBytes ? { maxPushBytes: config.maxPushBytes } : {}
|
|
24791
|
+
...config.maxPushBytes ? { maxPushBytes: config.maxPushBytes } : {},
|
|
24792
|
+
...retention
|
|
24483
24793
|
}));
|
|
24484
24794
|
router.post("/api/v1/runs/open", createOpenRunHandler({ storage }));
|
|
24485
24795
|
router.patch("/api/v1/runs/:id", createPatchRunHandler({
|
|
24486
24796
|
storage,
|
|
24487
|
-
...config.maxPushBytes != null ? { maxPushBytes: config.maxPushBytes } : {}
|
|
24797
|
+
...config.maxPushBytes != null ? { maxPushBytes: config.maxPushBytes } : {},
|
|
24798
|
+
...retention
|
|
24488
24799
|
}));
|
|
24489
24800
|
router.get("/api/v1/runs", createListRunsHandler(storage));
|
|
24490
24801
|
router.get("/api/v1/runs/:id", createGetRunHandler(storage));
|
|
@@ -24598,12 +24909,14 @@ async function writeJson(path, value) {
|
|
|
24598
24909
|
* update concurrently.
|
|
24599
24910
|
*/
|
|
24600
24911
|
const updateChains = /* @__PURE__ */ new Map();
|
|
24601
|
-
|
|
24602
|
-
|
|
24603
|
-
|
|
24604
|
-
|
|
24605
|
-
|
|
24606
|
-
|
|
24912
|
+
/**
|
|
24913
|
+
* Queue `work` behind whatever is already in flight for `path`. A delete takes
|
|
24914
|
+
* the same chain as the updates it removes: `writeJson` recreates the parent
|
|
24915
|
+
* directory, so an unordered delete would be silently undone by an update that
|
|
24916
|
+
* was already queued.
|
|
24917
|
+
*/
|
|
24918
|
+
async function serialize(path, work) {
|
|
24919
|
+
const next = (updateChains.get(path) ?? Promise.resolve()).catch(() => {}).then(work);
|
|
24607
24920
|
updateChains.set(path, next);
|
|
24608
24921
|
try {
|
|
24609
24922
|
return await next;
|
|
@@ -24611,6 +24924,13 @@ async function updateJson(path, mutate) {
|
|
|
24611
24924
|
if (updateChains.get(path) === next) updateChains.delete(path);
|
|
24612
24925
|
}
|
|
24613
24926
|
}
|
|
24927
|
+
async function updateJson(path, mutate) {
|
|
24928
|
+
return await serialize(path, async () => {
|
|
24929
|
+
const updated = mutate(await readJson(path));
|
|
24930
|
+
await writeJson(path, updated);
|
|
24931
|
+
return updated;
|
|
24932
|
+
});
|
|
24933
|
+
}
|
|
24614
24934
|
/** Read a raw file, returning `null` when it doesn't exist. */
|
|
24615
24935
|
async function readBytesOrNull(path) {
|
|
24616
24936
|
try {
|
|
@@ -24861,6 +25181,10 @@ function createFileArtifactStore(root) {
|
|
|
24861
25181
|
async updateJsonFile(runId, relPath, mutate) {
|
|
24862
25182
|
assertSafeRelPath(relPath);
|
|
24863
25183
|
await updateJson(join(artifactsRunDir(root, runId), relPath), mutate);
|
|
25184
|
+
},
|
|
25185
|
+
async delete(runId) {
|
|
25186
|
+
const dir = artifactsRunDir(root, runId);
|
|
25187
|
+
await serialize(join(dir, "report.json"), () => removePath(dir));
|
|
24864
25188
|
}
|
|
24865
25189
|
};
|
|
24866
25190
|
}
|
|
@@ -25121,6 +25445,9 @@ function createFileRunStore(root) {
|
|
|
25121
25445
|
};
|
|
25122
25446
|
});
|
|
25123
25447
|
},
|
|
25448
|
+
async delete(id) {
|
|
25449
|
+
await serialize(runMetaPath(root, id), () => removePath(runDir(root, id)));
|
|
25450
|
+
},
|
|
25124
25451
|
async list({ project, branch, status, kinds, since, until, limit }) {
|
|
25125
25452
|
const ids = await listSubdirsOrEmpty(runsDir(root));
|
|
25126
25453
|
const inWindow = windowFilter({
|
|
@@ -25275,6 +25602,10 @@ function createFileTriageStore(root) {
|
|
|
25275
25602
|
return (current ?? []).filter((r) => !(r.feature === feature && r.spec === spec));
|
|
25276
25603
|
});
|
|
25277
25604
|
},
|
|
25605
|
+
async deleteAll(runId) {
|
|
25606
|
+
const path = triagePath(root, runId);
|
|
25607
|
+
await serialize(path, () => removePath(path));
|
|
25608
|
+
},
|
|
25278
25609
|
async list(runId) {
|
|
25279
25610
|
return await readJson(triagePath(root, runId)) ?? [];
|
|
25280
25611
|
}
|
|
@@ -25316,7 +25647,7 @@ function createHubStorage(config) {
|
|
|
25316
25647
|
}
|
|
25317
25648
|
//#endregion
|
|
25318
25649
|
//#region src/cli/serve.ts
|
|
25319
|
-
const serveCommand = new Command("serve").description("Start the ccqa hub: a small control-plane HTTP server that aggregates CI run results, sessions, variables, and triage records. It does not execute tests — CI/local `ccqa run` produces reports and `ccqa hub push` uploads them here. Any HTTP client (docs/hub-api.md) can talk to it.").option("--port <n>", "TCP port to listen on.", "8787").option("--data-dir <path>", "Directory to store runs, sessions, and variables in.", "./ccqa-hub-data").option("--allow-origin <origin>", "CORS-allowed origin for browser clients (repeatable). Omit for no cross-origin access.", (val, prev) => [...prev, val], []).option("--max-push-mb <n>", "Reject pushed report bundles larger than this (MB). Default 32.", parsePositiveInt).action(async (opts) => {
|
|
25650
|
+
const serveCommand = new Command("serve").description("Start the ccqa hub: a small control-plane HTTP server that aggregates CI run results, sessions, variables, and triage records. It does not execute tests — CI/local `ccqa run` produces reports and `ccqa hub push` uploads them here. Any HTTP client (docs/hub-api.md) can talk to it.").option("--port <n>", "TCP port to listen on.", "8787").option("--data-dir <path>", "Directory to store runs, sessions, and variables in.", "./ccqa-hub-data").option("--allow-origin <origin>", "CORS-allowed origin for browser clients (repeatable). Omit for no cross-origin access.", (val, prev) => [...prev, val], []).option("--max-push-mb <n>", "Reject pushed report bundles larger than this (MB). Default 32.", parsePositiveInt).option("--max-runs-per-branch <n>", "Keep this many runs per project and branch; older ones are deleted with their artifacts and grades. Default 200.", parsePositiveInt).action(async (opts) => {
|
|
25320
25651
|
await runServe(opts);
|
|
25321
25652
|
});
|
|
25322
25653
|
function parsePositiveInt(raw) {
|
|
@@ -25351,7 +25682,8 @@ async function runServe(opts) {
|
|
|
25351
25682
|
token,
|
|
25352
25683
|
encryptionKey,
|
|
25353
25684
|
allowedOrigins: opts.allowOrigin ?? [],
|
|
25354
|
-
...opts.maxPushMb ? { maxPushBytes: opts.maxPushMb * 1024 * 1024 } : {}
|
|
25685
|
+
...opts.maxPushMb ? { maxPushBytes: opts.maxPushMb * 1024 * 1024 } : {},
|
|
25686
|
+
...opts.maxRunsPerBranch ? { maxRunsPerBranch: opts.maxRunsPerBranch } : {}
|
|
25355
25687
|
});
|
|
25356
25688
|
const requestedPort = Number(opts.port);
|
|
25357
25689
|
if (!Number.isInteger(requestedPort) || requestedPort < 0 || requestedPort > 65535) {
|
|
@@ -25368,6 +25700,7 @@ async function runServe(opts) {
|
|
|
25368
25700
|
header("serve", `port ${boundPort}`);
|
|
25369
25701
|
meta("data-dir", dataDir);
|
|
25370
25702
|
meta("encryption", encryptionKey ? "enabled" : "disabled (no CCQA_HUB_ENCRYPTION_KEY)");
|
|
25703
|
+
meta("run retention", `${opts.maxRunsPerBranch ?? 200} per project/branch`);
|
|
25371
25704
|
const auth = driftAuthAvailable();
|
|
25372
25705
|
meta("triage learning", auth.ok ? "available" : `unavailable (${auth.reason} — learning jobs will fail)`);
|
|
25373
25706
|
if (opts.allowOrigin && opts.allowOrigin.length > 0) meta("cors", opts.allowOrigin.join(", "));
|
|
@@ -35,9 +35,9 @@ declare const RunSchema: z.ZodObject<{
|
|
|
35
35
|
running: "running";
|
|
36
36
|
}>;
|
|
37
37
|
kind: z.ZodDefault<z.ZodEnum<{
|
|
38
|
+
record: "record";
|
|
38
39
|
run: "run";
|
|
39
40
|
drift: "drift";
|
|
40
|
-
record: "record";
|
|
41
41
|
}>>;
|
|
42
42
|
drift: z.ZodDefault<z.ZodNullable<z.ZodObject<{
|
|
43
43
|
specs: z.ZodNumber;
|
|
@@ -533,6 +533,12 @@ declare const ReportSpecResultSchema: z.ZodObject<{
|
|
|
533
533
|
}>>;
|
|
534
534
|
}, z.core.$strip>>;
|
|
535
535
|
analysisSkipped: z.ZodNullable<z.ZodString>;
|
|
536
|
+
rerun: z.ZodOptional<z.ZodObject<{
|
|
537
|
+
outcome: z.ZodEnum<{
|
|
538
|
+
passed: "passed";
|
|
539
|
+
failed: "failed";
|
|
540
|
+
}>;
|
|
541
|
+
}, z.core.$strip>>;
|
|
536
542
|
customPromptVersion: z.ZodOptional<z.ZodString>;
|
|
537
543
|
analysisBase: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
538
544
|
ref: z.ZodString;
|
|
@@ -615,9 +621,9 @@ type ReportSpecResult = z.infer<typeof ReportSpecResultSchema>;
|
|
|
615
621
|
declare const RunReportDataSchema: z.ZodObject<{
|
|
616
622
|
schemaVersion: z.ZodLiteral<1>;
|
|
617
623
|
kind: z.ZodDefault<z.ZodEnum<{
|
|
624
|
+
record: "record";
|
|
618
625
|
run: "run";
|
|
619
626
|
drift: "drift";
|
|
620
|
-
record: "record";
|
|
621
627
|
}>>;
|
|
622
628
|
createdAt: z.ZodString;
|
|
623
629
|
runId: z.ZodNullable<z.ZodString>;
|
|
@@ -711,6 +717,12 @@ declare const RunReportDataSchema: z.ZodObject<{
|
|
|
711
717
|
}>>;
|
|
712
718
|
}, z.core.$strip>>;
|
|
713
719
|
analysisSkipped: z.ZodNullable<z.ZodString>;
|
|
720
|
+
rerun: z.ZodOptional<z.ZodObject<{
|
|
721
|
+
outcome: z.ZodEnum<{
|
|
722
|
+
passed: "passed";
|
|
723
|
+
failed: "failed";
|
|
724
|
+
}>;
|
|
725
|
+
}, z.core.$strip>>;
|
|
714
726
|
customPromptVersion: z.ZodOptional<z.ZodString>;
|
|
715
727
|
analysisBase: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
716
728
|
ref: z.ZodString;
|
package/dist/package.json
CHANGED