ccqa 1.22.0 → 1.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/ccqa.mjs +125 -88
- package/dist/package.json +1 -1
- package/package.json +1 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -15315,16 +15315,18 @@ const DEFAULT_CONCURRENCY$1 = 3;
|
|
|
15315
15315
|
* `cli/run` calls this with just the failing specs after vitest.
|
|
15316
15316
|
*/
|
|
15317
15317
|
async function analyzeDrift(input) {
|
|
15318
|
-
const { targets, cwd, blocks, concurrency = DEFAULT_CONCURRENCY$1, model, language, guidance, onSpecStart } = input;
|
|
15318
|
+
const { targets, cwd, blocks, concurrency = DEFAULT_CONCURRENCY$1, model, language, guidance, onSpecStart, onSpecDone } = input;
|
|
15319
15319
|
return runPool(targets, concurrency, async (target) => {
|
|
15320
15320
|
onSpecStart?.(target);
|
|
15321
|
-
|
|
15321
|
+
const result = await checkSpec(target, {
|
|
15322
15322
|
cwd,
|
|
15323
15323
|
blocks,
|
|
15324
15324
|
model,
|
|
15325
15325
|
language,
|
|
15326
15326
|
guidance
|
|
15327
15327
|
});
|
|
15328
|
+
await onSpecDone?.(result);
|
|
15329
|
+
return result;
|
|
15328
15330
|
});
|
|
15329
15331
|
}
|
|
15330
15332
|
async function checkSpec(target, opts) {
|
|
@@ -15513,37 +15515,35 @@ function specStatus(result, threshold) {
|
|
|
15513
15515
|
return "passed";
|
|
15514
15516
|
}
|
|
15515
15517
|
/**
|
|
15516
|
-
*
|
|
15517
|
-
*
|
|
15518
|
-
*
|
|
15519
|
-
*
|
|
15520
|
-
*
|
|
15521
|
-
*
|
|
15518
|
+
* One audited spec as a report row. Browser-execution fields don't apply to an
|
|
15519
|
+
* audit and stay empty — which is why `mode` is carried separately: nothing
|
|
15520
|
+
* ran, but which surfaces were audited is still a fact about the row. The
|
|
15521
|
+
* diagnosis goes into `analysis` because for a `kind: "drift"` report the
|
|
15522
|
+
* diagnosis IS the row's verdict, so it renders through the same card a failed
|
|
15523
|
+
* `ccqa run` spec does.
|
|
15522
15524
|
*
|
|
15523
|
-
*
|
|
15524
|
-
*
|
|
15525
|
-
* card a failed `ccqa run` spec does. `reasoning` has no drift-audit
|
|
15526
|
-
* equivalent (the audit gives one headline, not a deliberation) so it is
|
|
15527
|
-
* filled with an empty string to satisfy `FailureAnalysisSchema`.
|
|
15525
|
+
* Shared by the incremental push and the final report: the hub upserts by
|
|
15526
|
+
* feature/spec, so two mappings would let the closing patch rewrite history.
|
|
15528
15527
|
*/
|
|
15529
|
-
function
|
|
15530
|
-
|
|
15531
|
-
|
|
15532
|
-
|
|
15533
|
-
|
|
15528
|
+
function driftResultToRow(result, threshold) {
|
|
15529
|
+
return {
|
|
15530
|
+
...emptySpecRow({
|
|
15531
|
+
feature: result.target.featureName,
|
|
15532
|
+
spec: result.target.specName,
|
|
15533
|
+
title: result.title ?? null,
|
|
15534
|
+
status: specStatus(result, threshold)
|
|
15535
|
+
}),
|
|
15534
15536
|
...result.live === void 0 ? {} : { mode: result.live ? "live" : "deterministic" },
|
|
15535
|
-
|
|
15536
|
-
|
|
15537
|
-
|
|
15538
|
-
|
|
15539
|
-
|
|
15540
|
-
|
|
15541
|
-
|
|
15542
|
-
|
|
15543
|
-
|
|
15544
|
-
|
|
15545
|
-
liveRun: null
|
|
15546
|
-
}));
|
|
15537
|
+
analysis: result.drift ?? null
|
|
15538
|
+
};
|
|
15539
|
+
}
|
|
15540
|
+
/**
|
|
15541
|
+
* Adapts `ccqa audit` results into the shared RunReportData shape so they can
|
|
15542
|
+
* be pushed to the hub (`ccqa audit --report-to-hub`) and rendered by the same
|
|
15543
|
+
* report UI as `ccqa run`/`ccqa live`.
|
|
15544
|
+
*/
|
|
15545
|
+
function driftResultsToReport(results, meta) {
|
|
15546
|
+
const specResults = results.map((result) => driftResultToRow(result, meta.threshold));
|
|
15547
15547
|
return {
|
|
15548
15548
|
schemaVersion: 1,
|
|
15549
15549
|
kind: "drift",
|
|
@@ -15823,8 +15823,13 @@ async function runAudit(specPath, opts) {
|
|
|
15823
15823
|
requireReportToHubConnection(opts);
|
|
15824
15824
|
let results;
|
|
15825
15825
|
let promptCtx;
|
|
15826
|
+
let push = null;
|
|
15826
15827
|
try {
|
|
15827
15828
|
promptCtx = await resolveAuditPromptContext(opts, cwd);
|
|
15829
|
+
if (opts.reportToHub) {
|
|
15830
|
+
push = await openDriftPush(opts, cwd);
|
|
15831
|
+
if (format === "text") info(`hub: incremental drift run opened (${push.runId})`);
|
|
15832
|
+
}
|
|
15828
15833
|
results = await analyzeDrift({
|
|
15829
15834
|
targets,
|
|
15830
15835
|
cwd,
|
|
@@ -15835,16 +15840,18 @@ async function runAudit(specPath, opts) {
|
|
|
15835
15840
|
guidance: promptCtx.guidance,
|
|
15836
15841
|
onSpecStart: (t) => {
|
|
15837
15842
|
if (format === "text") info(`checking ${t.featureName}/${t.specName}`);
|
|
15843
|
+
},
|
|
15844
|
+
onSpecDone: async (r) => {
|
|
15845
|
+
if (push) await sendDriftRow(push, r, threshold);
|
|
15838
15846
|
}
|
|
15839
15847
|
});
|
|
15840
15848
|
} finally {
|
|
15841
15849
|
if (holder) await releaseSpecs(hub, hubProject, opts.hubProfile, holder);
|
|
15842
15850
|
}
|
|
15843
15851
|
process.stdout.write(renderDrift(results, format, cwd));
|
|
15844
|
-
if (
|
|
15852
|
+
if (push) await sealDriftPush(push, {
|
|
15845
15853
|
results,
|
|
15846
15854
|
threshold,
|
|
15847
|
-
cwd,
|
|
15848
15855
|
opts,
|
|
15849
15856
|
format,
|
|
15850
15857
|
baseRef,
|
|
@@ -15893,59 +15900,92 @@ function requireReportToHubConnection(opts) {
|
|
|
15893
15900
|
process.exit(2);
|
|
15894
15901
|
}
|
|
15895
15902
|
/**
|
|
15896
|
-
*
|
|
15897
|
-
*
|
|
15898
|
-
*
|
|
15899
|
-
*
|
|
15903
|
+
* Open the drift run this sweep patches into. A failure here is fatal, as it
|
|
15904
|
+
* is for `ccqa run`: a job that asked to publish and cannot reach the hub has
|
|
15905
|
+
* not done what it was told, and the audit has no local artifact to fall back
|
|
15906
|
+
* on — the hub is its only output. Raised before any spec is checked, so
|
|
15907
|
+
* nothing is wasted. Not retried: a dropped response after the hub committed
|
|
15908
|
+
* would leave a second orphan running run.
|
|
15900
15909
|
*
|
|
15901
|
-
* `resolveHub` is injectable
|
|
15902
|
-
* a real hub connection; it defaults to the real flag/env resolution.
|
|
15910
|
+
* `resolveHub` is injectable for tests.
|
|
15903
15911
|
*/
|
|
15904
|
-
async function
|
|
15905
|
-
const { results, threshold, cwd, opts, format, baseRef, promptCtx } = args;
|
|
15912
|
+
async function openDriftPush(opts, cwd, resolveHub = resolveHubClient) {
|
|
15906
15913
|
const hub = resolveHub(opts);
|
|
15907
|
-
if (!hub)
|
|
15908
|
-
|
|
15909
|
-
|
|
15914
|
+
if (!hub) throw new RunUsageError("--report-to-hub requires a hub connection (--hub-url/--hub-token or CCQA_HUB_URL/CCQA_HUB_TOKEN)");
|
|
15915
|
+
const project = resolveProject({
|
|
15916
|
+
project: opts.project,
|
|
15917
|
+
cwd
|
|
15918
|
+
});
|
|
15919
|
+
const [branch, gitHead] = await Promise.all([detectBranch(cwd), getGitHead(cwd)]);
|
|
15920
|
+
const ciRunId = githubRunId();
|
|
15921
|
+
const runUrl = githubRunUrl();
|
|
15922
|
+
try {
|
|
15923
|
+
return {
|
|
15924
|
+
hub,
|
|
15925
|
+
runId: (await hub.openRun({
|
|
15926
|
+
project,
|
|
15927
|
+
kind: "drift",
|
|
15928
|
+
...branch ? { branch } : {},
|
|
15929
|
+
...opts.hubProfile ? { profile: opts.hubProfile } : {},
|
|
15930
|
+
...gitHead ? { gitHead } : {},
|
|
15931
|
+
...ciRunId ? { ciRunId } : {},
|
|
15932
|
+
...runUrl ? { runUrl } : {}
|
|
15933
|
+
})).id,
|
|
15934
|
+
gitHead
|
|
15935
|
+
};
|
|
15936
|
+
} catch (err) {
|
|
15937
|
+
throw new RunUsageError(`--report-to-hub: could not open a run on the hub (${errMessage(err)})`);
|
|
15910
15938
|
}
|
|
15939
|
+
}
|
|
15940
|
+
/**
|
|
15941
|
+
* Send one finished spec. A failure is warned and swallowed rather than
|
|
15942
|
+
* thrown: the seal resends every row, so a dropped patch costs freshness for
|
|
15943
|
+
* the rest of the sweep, not the record.
|
|
15944
|
+
*/
|
|
15945
|
+
async function sendDriftRow(push, result, threshold) {
|
|
15911
15946
|
try {
|
|
15912
|
-
|
|
15913
|
-
|
|
15914
|
-
|
|
15947
|
+
await push.hub.patchRun(push.runId, {
|
|
15948
|
+
rows: [driftResultToRow(result, threshold)],
|
|
15949
|
+
reportMeta: { cost: currentReportCost() }
|
|
15915
15950
|
});
|
|
15916
|
-
|
|
15917
|
-
|
|
15918
|
-
|
|
15919
|
-
|
|
15920
|
-
|
|
15921
|
-
|
|
15922
|
-
|
|
15923
|
-
|
|
15924
|
-
|
|
15951
|
+
} catch (err) {
|
|
15952
|
+
warn(`hub: could not push ${result.target.featureName}/${result.target.specName}: ${errMessage(err)}`);
|
|
15953
|
+
}
|
|
15954
|
+
}
|
|
15955
|
+
/**
|
|
15956
|
+
* Close the run with every row and the envelope metadata. Resends all rows
|
|
15957
|
+
* rather than only the unsent ones, so this one call also repairs any row
|
|
15958
|
+
* whose mid-sweep patch failed. Status is left to the hub, which derives it
|
|
15959
|
+
* from the rows.
|
|
15960
|
+
*/
|
|
15961
|
+
async function sealDriftPush(push, args) {
|
|
15962
|
+
const { results, threshold, opts, format, baseRef, promptCtx } = args;
|
|
15963
|
+
const report = driftResultsToReport(results, {
|
|
15964
|
+
threshold,
|
|
15965
|
+
git: {
|
|
15966
|
+
head: push.gitHead,
|
|
15967
|
+
base: baseRef ?? null
|
|
15968
|
+
},
|
|
15969
|
+
customPromptVersion: promptCtx?.customPromptVersion ?? null,
|
|
15970
|
+
triageUserPromptHash: promptCtx?.triageUserPromptHash ?? null
|
|
15971
|
+
});
|
|
15972
|
+
try {
|
|
15973
|
+
await push.hub.patchRun(push.runId, {
|
|
15974
|
+
rows: report.results,
|
|
15975
|
+
done: true,
|
|
15976
|
+
reportMeta: {
|
|
15977
|
+
git: report.git,
|
|
15978
|
+
promptVersion: report.promptVersion,
|
|
15979
|
+
customPromptVersion: report.customPromptVersion,
|
|
15980
|
+
...report.triageUserPromptHash ? { triageUserPromptHash: report.triageUserPromptHash } : {},
|
|
15981
|
+
cost: report.cost
|
|
15982
|
+
}
|
|
15925
15983
|
});
|
|
15926
|
-
const dir = await mkdtemp(join(tmpdir(), "ccqa-drift-push-"));
|
|
15927
|
-
try {
|
|
15928
|
-
await writeFile(join(dir, "report.json"), JSON.stringify(report, null, 2), "utf8");
|
|
15929
|
-
const archive = await packDirToTarGz(dir);
|
|
15930
|
-
const run = await hub.pushRun(archive, {
|
|
15931
|
-
project,
|
|
15932
|
-
...branch ? { branch } : {},
|
|
15933
|
-
kind: "drift"
|
|
15934
|
-
});
|
|
15935
|
-
if (format === "text") info(`pushed drift result to hub: ${(opts.hubUrl ?? process.env.CCQA_HUB_URL ?? "").replace(/\/+$/, "")}/#/runs/${run.id}`);
|
|
15936
|
-
} finally {
|
|
15937
|
-
await rm(dir, {
|
|
15938
|
-
recursive: true,
|
|
15939
|
-
force: true
|
|
15940
|
-
});
|
|
15941
|
-
}
|
|
15942
15984
|
} catch (err) {
|
|
15943
|
-
|
|
15944
|
-
|
|
15945
|
-
process.exit(2);
|
|
15946
|
-
}
|
|
15947
|
-
throw err;
|
|
15985
|
+
error(`hub: could not close the drift run ${push.runId}: ${errMessage(err)}`);
|
|
15986
|
+
process.exit(2);
|
|
15948
15987
|
}
|
|
15988
|
+
if (format === "text") info(`pushed drift result to hub: ${(opts.hubUrl ?? process.env.CCQA_HUB_URL ?? "").replace(/\/+$/, "")}/#/runs/${push.runId}`);
|
|
15949
15989
|
}
|
|
15950
15990
|
/**
|
|
15951
15991
|
* Adapts `AuditOptions` to `resolveHubContext`'s flat option shape. Unlike
|
|
@@ -17157,23 +17197,20 @@ function createPatchRunHandler(config) {
|
|
|
17157
17197
|
};
|
|
17158
17198
|
});
|
|
17159
17199
|
if (evidence) for (const [relPath, b64] of Object.entries(evidence)) await config.storage.artifacts.putFile(id, relPath, Buffer.from(b64, "base64"));
|
|
17160
|
-
const patch =
|
|
17161
|
-
status: finalStatus ?? (specs.failed > 0 ? "failed" : "passed"),
|
|
17200
|
+
const patch = {
|
|
17162
17201
|
specs,
|
|
17163
17202
|
costUsd,
|
|
17164
17203
|
...run.kind === "drift" ? { drift: summarizeDrift(mergedResults) } : {},
|
|
17165
|
-
...
|
|
17166
|
-
|
|
17167
|
-
|
|
17168
|
-
|
|
17169
|
-
|
|
17170
|
-
|
|
17204
|
+
...done ? {
|
|
17205
|
+
status: finalStatus ?? (specs.failed > 0 ? "failed" : "passed"),
|
|
17206
|
+
...reportMeta?.git?.head ? { gitHead: reportMeta.git.head } : {},
|
|
17207
|
+
...reportMeta?.promptVersion ? { promptVersion: reportMeta.promptVersion } : {},
|
|
17208
|
+
...await deployHeadMovedDuringRun(config.storage, run) ? { deployedShaAmbiguous: true } : {}
|
|
17209
|
+
} : {}
|
|
17171
17210
|
};
|
|
17172
17211
|
const updated = await config.storage.runs.update(id, patch);
|
|
17173
|
-
|
|
17174
|
-
|
|
17175
|
-
await updateDriftLedger(config.storage, updated, mergedResults);
|
|
17176
|
-
}
|
|
17212
|
+
await updateDriftLedger(config.storage, updated, mergedResults);
|
|
17213
|
+
if (done) await updateSpecLedger(config.storage, updated, mergedResults);
|
|
17177
17214
|
sendJson(ctx.res, 200, updated);
|
|
17178
17215
|
};
|
|
17179
17216
|
}
|
package/dist/package.json
CHANGED