ccqa 1.20.1 → 1.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/ccqa.mjs +312 -173
- package/dist/hub-client/index.d.mts +11 -0
- package/dist/package.json +1 -1
- package/package.json +1 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -4,7 +4,7 @@ import { t as EVIDENCE_DIR_ENV } from "../evidence-constants-C425F7ZG.mjs";
|
|
|
4
4
|
import { a as formatAgentBrowserUnavailableMessage, i as assertAgentBrowserAvailable, n as spawnAB, o as pathWithAgentBrowserShim, r as AgentBrowserUnavailableError, s as resolveAgentBrowserBin$1, t as sleepSync } from "../spawn-ab-CRIVfWpw.mjs";
|
|
5
5
|
import { createRequire } from "node:module";
|
|
6
6
|
import { Command } from "commander";
|
|
7
|
-
import { accessSync, createWriteStream, existsSync, readFileSync, rmSync } from "node:fs";
|
|
7
|
+
import { accessSync, appendFileSync, createWriteStream, existsSync, readFileSync, rmSync } from "node:fs";
|
|
8
8
|
import { fileURLToPath } from "node:url";
|
|
9
9
|
import { createCipheriv, createDecipheriv, createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
|
|
10
10
|
import { access, chmod, cp, mkdir, mkdtemp, open, readFile, readdir, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
|
|
@@ -771,13 +771,21 @@ function waitExit(child) {
|
|
|
771
771
|
//#endregion
|
|
772
772
|
//#region src/runtime/live-artifacts.ts
|
|
773
773
|
/**
|
|
774
|
-
* Build a sortable run id
|
|
775
|
-
*
|
|
776
|
-
*
|
|
777
|
-
*
|
|
774
|
+
* Build a sortable, unique run id. ISO8601 with `:` / `.` replaced so it's
|
|
775
|
+
* filename-safe, timestamp first so run directories still sort by time, and a
|
|
776
|
+
* random suffix because the timestamp alone does not separate two specs.
|
|
777
|
+
*
|
|
778
|
+
* The pool launches specs back-to-back, so at `--concurrency > 1` two of them
|
|
779
|
+
* land in the same millisecond. A spec that puts `${CCQA_RUN_ID}` in the name
|
|
780
|
+
* of something it creates would then share that name with its neighbour, and
|
|
781
|
+
* each would find — and delete — the other's row. Nothing fails; the
|
|
782
|
+
* assertions just read the wrong state.
|
|
783
|
+
*
|
|
784
|
+
* Caller is expected to mkdir the directory once and pass
|
|
785
|
+
* `runDir = <baseDir>/<runId>` to the path helpers below.
|
|
778
786
|
*/
|
|
779
787
|
function buildRunId() {
|
|
780
|
-
return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")
|
|
788
|
+
return `${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${randomUUID().slice(0, 8)}`;
|
|
781
789
|
}
|
|
782
790
|
/**
|
|
783
791
|
* Per-step artifact paths under a run directory. `<runDir>/steps/<stepId>.*`.
|
|
@@ -1361,8 +1369,8 @@ async function withCostTally(fn) {
|
|
|
1361
1369
|
* The active scope's total so far, or null outside one.
|
|
1362
1370
|
*
|
|
1363
1371
|
* Read rather than pushed at the caller because commands end in
|
|
1364
|
-
* `process.exit`, which never reaches a `finally
|
|
1365
|
-
* total
|
|
1372
|
+
* `process.exit`, which never reaches a `finally`; whoever opened the scope
|
|
1373
|
+
* reads the total on the way out (see `withCostReporting`).
|
|
1366
1374
|
*
|
|
1367
1375
|
* Fields stay `null` when no invocation reported them, so a caller can tell
|
|
1368
1376
|
* "nothing was billed" from "the SDK didn't say" (mock runs, SDK errors).
|
|
@@ -1425,6 +1433,17 @@ function resolveEndpointEnv() {
|
|
|
1425
1433
|
}
|
|
1426
1434
|
return endpointEnv;
|
|
1427
1435
|
}
|
|
1436
|
+
/**
|
|
1437
|
+
* Drop endpoint variables that are present but empty, so an empty value never
|
|
1438
|
+
* reaches the Claude Code process as an override. "Set to nothing" is how a
|
|
1439
|
+
* caller that cannot omit the key says "use the default" — a CI job wiring
|
|
1440
|
+
* `ANTHROPIC_BASE_URL` from an unset repository variable, most of all.
|
|
1441
|
+
*/
|
|
1442
|
+
function withoutEmptyEndpointVars(env) {
|
|
1443
|
+
const out = { ...env };
|
|
1444
|
+
for (const key of ENDPOINT_ENV_KEYS) if (out[key] === "") delete out[key];
|
|
1445
|
+
return out;
|
|
1446
|
+
}
|
|
1428
1447
|
let nativeBinaryWarned = false;
|
|
1429
1448
|
/**
|
|
1430
1449
|
* Warn once per process when the SDK's per-platform native binary is missing:
|
|
@@ -1441,10 +1460,10 @@ async function invokeClaudeStreaming(options, onEvent) {
|
|
|
1441
1460
|
const { prompt, systemPrompt, allowedTools, disableBuiltinTools = false, disableThinking = false, mcpServers, maxTurns, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, relaxAbConstraints = false } = options;
|
|
1442
1461
|
const resolvedModel = resolveModel(model);
|
|
1443
1462
|
const hasEndpointEnv = Object.keys(resolveEndpointEnv()).length > 0;
|
|
1444
|
-
const mergedEnv = env || hasEndpointEnv ? {
|
|
1463
|
+
const mergedEnv = env || hasEndpointEnv ? withoutEmptyEndpointVars({
|
|
1445
1464
|
...process.env,
|
|
1446
1465
|
...env
|
|
1447
|
-
} : void 0;
|
|
1466
|
+
}) : void 0;
|
|
1448
1467
|
let lastAbToolUseId = null;
|
|
1449
1468
|
const claimAbToolUse = (toolUseId) => {
|
|
1450
1469
|
if (toolUseId !== lastAbToolUseId) return false;
|
|
@@ -3227,20 +3246,6 @@ const ReportArtifactSchema = z.object({
|
|
|
3227
3246
|
sizeBytes: z.number()
|
|
3228
3247
|
});
|
|
3229
3248
|
/**
|
|
3230
|
-
* Per-step row for a live-mode run (spec.yaml `mode: live`). Mirrors the
|
|
3231
|
-
* structure produced by `src/runtime/live-executor.ts:LiveStepResult` but
|
|
3232
|
-
* encoded against the report schema so the HTML renderer can carry both
|
|
3233
|
-
* deterministic (`evidence`) and live (`liveRun`) sources of step-boundary
|
|
3234
|
-
* screenshots.
|
|
3235
|
-
*
|
|
3236
|
-
* `beforePng` / `afterPng` are RELATIVE to the report directory, same
|
|
3237
|
-
* convention as `ReportEvidenceSchema.pngPath` above. The caller copies the
|
|
3238
|
-
* PNG files into `<reportDir>/evidence/<feature>/<spec>/` and computes the
|
|
3239
|
-
* relative path with `node:path`'s `relative()`, so the report directory is
|
|
3240
|
-
* self-contained: it can be archived and shipped on its own (e.g. a hub
|
|
3241
|
-
* push) without also bundling the `.ccqa` runs dir.
|
|
3242
|
-
*/
|
|
3243
|
-
/**
|
|
3244
3249
|
* Per-step / per-run cost+usage record, pulled from the SDK's `result` message.
|
|
3245
3250
|
* Every numeric field is nullable so the report can carry partial telemetry
|
|
3246
3251
|
* (e.g. when the SDK omits a field, or when a step was skipped).
|
|
@@ -3248,7 +3253,7 @@ const ReportArtifactSchema = z.object({
|
|
|
3248
3253
|
* `models` is the union of model ids the SDK reported using; usually a
|
|
3249
3254
|
* single element, but the SDK can fan out across models in some modes.
|
|
3250
3255
|
*/
|
|
3251
|
-
const
|
|
3256
|
+
const ReportCostSchema = z.object({
|
|
3252
3257
|
totalCostUsd: z.number().nullable(),
|
|
3253
3258
|
durationApiMs: z.number().nullable(),
|
|
3254
3259
|
numTurns: z.number().nullable(),
|
|
@@ -3258,6 +3263,20 @@ const LiveReportCostSchema = z.object({
|
|
|
3258
3263
|
outputTokens: z.number().nullable(),
|
|
3259
3264
|
models: z.array(z.string())
|
|
3260
3265
|
});
|
|
3266
|
+
/**
|
|
3267
|
+
* Per-step row for a live-mode run (spec.yaml `mode: live`). Mirrors the
|
|
3268
|
+
* structure produced by `src/runtime/live-executor.ts:LiveStepResult` but
|
|
3269
|
+
* encoded against the report schema so the HTML renderer can carry both
|
|
3270
|
+
* deterministic (`evidence`) and live (`liveRun`) sources of step-boundary
|
|
3271
|
+
* screenshots.
|
|
3272
|
+
*
|
|
3273
|
+
* `beforePng` / `afterPng` are RELATIVE to the report directory, same
|
|
3274
|
+
* convention as `ReportEvidenceSchema.pngPath` above. The caller copies the
|
|
3275
|
+
* PNG files into `<reportDir>/evidence/<feature>/<spec>/` and computes the
|
|
3276
|
+
* relative path with `node:path`'s `relative()`, so the report directory is
|
|
3277
|
+
* self-contained: it can be archived and shipped on its own (e.g. a hub
|
|
3278
|
+
* push) without also bundling the `.ccqa` runs dir.
|
|
3279
|
+
*/
|
|
3261
3280
|
const LiveReportStepSchema = z.object({
|
|
3262
3281
|
stepId: z.string(),
|
|
3263
3282
|
source: z.string(),
|
|
@@ -3272,7 +3291,7 @@ const LiveReportStepSchema = z.object({
|
|
|
3272
3291
|
beforePng: z.string().nullable(),
|
|
3273
3292
|
afterPng: z.string().nullable(),
|
|
3274
3293
|
durationMs: z.number(),
|
|
3275
|
-
cost:
|
|
3294
|
+
cost: ReportCostSchema,
|
|
3276
3295
|
commands: z.array(z.string()).optional()
|
|
3277
3296
|
});
|
|
3278
3297
|
const LiveReportRunSchema = z.object({
|
|
@@ -3281,7 +3300,7 @@ const LiveReportRunSchema = z.object({
|
|
|
3281
3300
|
startedAt: z.string(),
|
|
3282
3301
|
durationMs: z.number(),
|
|
3283
3302
|
steps: z.array(LiveReportStepSchema),
|
|
3284
|
-
cost:
|
|
3303
|
+
cost: ReportCostSchema
|
|
3285
3304
|
});
|
|
3286
3305
|
const ReportSpecResultSchema = z.object({
|
|
3287
3306
|
feature: z.string(),
|
|
@@ -3351,6 +3370,7 @@ const RunReportDataSchema = z.object({
|
|
|
3351
3370
|
customPromptVersion: z.string().nullable().default(null),
|
|
3352
3371
|
triageUserPromptHash: z.string().optional(),
|
|
3353
3372
|
deployedSha: z.string().optional(),
|
|
3373
|
+
cost: ReportCostSchema.nullable().default(null),
|
|
3354
3374
|
results: z.array(ReportSpecResultSchema)
|
|
3355
3375
|
});
|
|
3356
3376
|
/** Shape of the "export labels" download produced by the report's client-side JS. */
|
|
@@ -4581,9 +4601,137 @@ const DraftNamingSchema = z.object({
|
|
|
4581
4601
|
reason: z.string().optional()
|
|
4582
4602
|
});
|
|
4583
4603
|
//#endregion
|
|
4604
|
+
//#region src/runtime/live-cost-format.ts
|
|
4605
|
+
/**
|
|
4606
|
+
* Compact one-line cost summary. Format:
|
|
4607
|
+
* "$0.1234 · 4 turns · 42 in / 6,511 out · 2.0M cached · sonnet"
|
|
4608
|
+
* Returns null only when the invocation reported nothing at all (a mock run,
|
|
4609
|
+
* an SDK error, or a command that never called a model).
|
|
4610
|
+
*
|
|
4611
|
+
* The price is one segment among several, not a precondition. An endpoint the
|
|
4612
|
+
* SDK has no pricing table for — any Anthropic-compatible gateway in front of
|
|
4613
|
+
* a third-party model — reports usage but no `total_cost_usd`, and dropping
|
|
4614
|
+
* the whole line there would hide real consumption behind silence. Tokens come
|
|
4615
|
+
* from the API response rather than a price list, so they survive that case
|
|
4616
|
+
* and become the signal to read.
|
|
4617
|
+
*
|
|
4618
|
+
* `compact: false` (default for CLI logs) keeps raw numbers and adds a
|
|
4619
|
+
* `model=...` segment. `compact: true` (HTML chip) thousand-separates fresh
|
|
4620
|
+
* tokens, abbreviates cache-read with K/M, drops the `model=` prefix.
|
|
4621
|
+
*/
|
|
4622
|
+
function formatLiveCost(cost, options) {
|
|
4623
|
+
const compact = options.compact;
|
|
4624
|
+
const sep = compact ? " · " : " / ";
|
|
4625
|
+
const parts = [];
|
|
4626
|
+
if (cost.totalCostUsd !== null) parts.push(`$${cost.totalCostUsd.toFixed(4)}`);
|
|
4627
|
+
if (cost.numTurns !== null) parts.push(`${cost.numTurns} turns`);
|
|
4628
|
+
if (cost.inputTokens !== null || cost.outputTokens !== null) {
|
|
4629
|
+
const i = cost.inputTokens ?? 0;
|
|
4630
|
+
const o = cost.outputTokens ?? 0;
|
|
4631
|
+
parts.push(compact ? `${formatNumber(i)} in / ${formatNumber(o)} out` : `${i}+${o} tokens`);
|
|
4632
|
+
}
|
|
4633
|
+
if (cost.cacheReadInputTokens !== null && cost.cacheReadInputTokens > 0) parts.push(compact ? `${formatTokenK(cost.cacheReadInputTokens)} cached` : `${cost.cacheReadInputTokens} cache-read`);
|
|
4634
|
+
if (!compact && cost.models.length > 0) parts.push(`model=${cost.models.join(",")}`);
|
|
4635
|
+
return parts.length > 0 ? parts.join(sep) : null;
|
|
4636
|
+
}
|
|
4637
|
+
/** Thousand-separated count for token figures. */
|
|
4638
|
+
function formatNumber(n) {
|
|
4639
|
+
return n.toLocaleString("en-US");
|
|
4640
|
+
}
|
|
4641
|
+
/** Compact token count: 9,043,456 → "9.0M", 12000 → "12K", small → plain. */
|
|
4642
|
+
function formatTokenK(n) {
|
|
4643
|
+
if (n >= 1e6) return `${(n / 1e6).toFixed(1)}M`;
|
|
4644
|
+
if (n >= 1e4) return `${Math.round(n / 1e3)}K`;
|
|
4645
|
+
if (n >= 1e3) return `${(n / 1e3).toFixed(1)}K`;
|
|
4646
|
+
return n.toLocaleString("en-US");
|
|
4647
|
+
}
|
|
4648
|
+
//#endregion
|
|
4649
|
+
//#region src/cli/cost-line.ts
|
|
4650
|
+
/**
|
|
4651
|
+
* Report what this command spent on Claude: a human line on stderr, and — when
|
|
4652
|
+
* `CCQA_COST_FILE` is set — one JSON line appended to that file.
|
|
4653
|
+
*
|
|
4654
|
+
* stderr rather than stdout because `audit --report-format json` and
|
|
4655
|
+
* `select-specs` put machine-readable output on stdout, and a cost line mixed
|
|
4656
|
+
* into it breaks the consumer. Cost is diagnostics about the command, not its
|
|
4657
|
+
* output.
|
|
4658
|
+
*
|
|
4659
|
+
* No-op outside a `withCostTally` scope: a command that never opened one has no
|
|
4660
|
+
* number to report, which is not the same as having spent nothing.
|
|
4661
|
+
*
|
|
4662
|
+
* Reached through `withCostReporting`; exported for its own test.
|
|
4663
|
+
*/
|
|
4664
|
+
function reportCost(command) {
|
|
4665
|
+
const cost = readCostTally();
|
|
4666
|
+
if (cost === null) return;
|
|
4667
|
+
const summary = formatLiveCost(cost, { compact: false });
|
|
4668
|
+
if (summary) process.stderr.write(`[cost] ${summary}\n`);
|
|
4669
|
+
appendCostRecord(command, cost);
|
|
4670
|
+
}
|
|
4671
|
+
/**
|
|
4672
|
+
* Run a command's action inside a cost tally and report the total once, however
|
|
4673
|
+
* the command ends.
|
|
4674
|
+
*
|
|
4675
|
+
* Neither half covers the other. Commands that end deep inside themselves with
|
|
4676
|
+
* `process.exit` never unwind, so only the `exit` listener sees them — and both
|
|
4677
|
+
* it and `reportCost` are synchronous, so the report still lands. Commands that
|
|
4678
|
+
* return normally are past `withCostTally`'s scope by the time `exit` fires, so
|
|
4679
|
+
* only the `finally` can still read the tally.
|
|
4680
|
+
*
|
|
4681
|
+
* Exactly one of the two runs: the `finally` detaches the listener before
|
|
4682
|
+
* reporting, and a `process.exit` terminates before the `finally` is reached.
|
|
4683
|
+
*
|
|
4684
|
+
* A signal handler only reports if it was REGISTERED inside this scope. Node
|
|
4685
|
+
* creates the SIGINT/SIGTERM handle when its first listener is added and binds
|
|
4686
|
+
* the async context then, so a handler installed at module load would read an
|
|
4687
|
+
* empty tally and drop the whole report on interrupt. `run.ts` installs its
|
|
4688
|
+
* teardown handlers inside the callback for that reason.
|
|
4689
|
+
*/
|
|
4690
|
+
async function withCostReporting(command, fn) {
|
|
4691
|
+
return withCostTally(async () => {
|
|
4692
|
+
const report = () => reportCost(command);
|
|
4693
|
+
process.once("exit", report);
|
|
4694
|
+
try {
|
|
4695
|
+
return await fn();
|
|
4696
|
+
} finally {
|
|
4697
|
+
process.off("exit", report);
|
|
4698
|
+
report();
|
|
4699
|
+
}
|
|
4700
|
+
});
|
|
4701
|
+
}
|
|
4702
|
+
/**
|
|
4703
|
+
* Append one JSONL record to `$CCQA_COST_FILE`. Appended, never truncated: a CI
|
|
4704
|
+
* job invokes ccqa several times (select-specs, audit, run) and the point of the
|
|
4705
|
+
* file is that one `jq -s 'map(.totalCostUsd)|add'` covers the whole job.
|
|
4706
|
+
*
|
|
4707
|
+
* Written even when `totalCostUsd` is null — that the command ran and was not
|
|
4708
|
+
* billed is itself the answer. Synchronous because every caller is about to
|
|
4709
|
+
* `process.exit`, which would drop a pending async write.
|
|
4710
|
+
*/
|
|
4711
|
+
function appendCostRecord(command, cost) {
|
|
4712
|
+
const path = process.env.CCQA_COST_FILE;
|
|
4713
|
+
if (!path) return;
|
|
4714
|
+
const record = {
|
|
4715
|
+
command,
|
|
4716
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4717
|
+
totalCostUsd: cost.totalCostUsd,
|
|
4718
|
+
numTurns: cost.numTurns,
|
|
4719
|
+
inputTokens: cost.inputTokens,
|
|
4720
|
+
outputTokens: cost.outputTokens,
|
|
4721
|
+
cacheReadInputTokens: cost.cacheReadInputTokens,
|
|
4722
|
+
models: cost.models
|
|
4723
|
+
};
|
|
4724
|
+
try {
|
|
4725
|
+
appendFileSync(path, `${JSON.stringify(record)}\n`, "utf8");
|
|
4726
|
+
} catch {}
|
|
4727
|
+
}
|
|
4728
|
+
//#endregion
|
|
4584
4729
|
//#region src/cli/draft.ts
|
|
4585
4730
|
const CATEGORY_LABEL = DRAFT_CATEGORY_LABEL;
|
|
4586
4731
|
const draftCommand = addLanguageOption(new Command("draft").argument("[feature/spec]", "Optional spec path (e.g. tasks/create-and-complete). If omitted, Claude proposes one from your intent.").description("Interactively draft and refine a spec.yaml with Claude Code").option("--instruction <text>", "Non-interactive single-shot instruction (skips the interactive loop)").option("-y, --yes", "Apply each generated patch without asking [y/N]", false)).action(withUsageErrors(async (specPath, opts) => {
|
|
4732
|
+
await withCostReporting("draft", () => runDraftCli(specPath, opts));
|
|
4733
|
+
}));
|
|
4734
|
+
async function runDraftCli(specPath, opts) {
|
|
4587
4735
|
await ensureCcqaDir();
|
|
4588
4736
|
let featureName;
|
|
4589
4737
|
let specName;
|
|
@@ -4596,7 +4744,7 @@ const draftCommand = addLanguageOption(new Command("draft").argument("[feature/s
|
|
|
4596
4744
|
prefilledIntent = intent;
|
|
4597
4745
|
}
|
|
4598
4746
|
await runDraft(featureName, specName, opts, prefilledIntent);
|
|
4599
|
-
}
|
|
4747
|
+
}
|
|
4600
4748
|
async function runDraft(featureName, specName, opts, prefilledIntent) {
|
|
4601
4749
|
header("draft", `${featureName}/${specName}`);
|
|
4602
4750
|
const ja = useJapanesePrompts(opts.language);
|
|
@@ -6611,6 +6759,7 @@ z.object({
|
|
|
6611
6759
|
}),
|
|
6612
6760
|
gitHead: z.string().nullable(),
|
|
6613
6761
|
promptVersion: z.string(),
|
|
6762
|
+
costUsd: z.number().nullable().optional(),
|
|
6614
6763
|
ciRunId: z.string().nullable(),
|
|
6615
6764
|
runUrl: z.string().nullable().optional(),
|
|
6616
6765
|
reportCreatedAt: z.string(),
|
|
@@ -7136,6 +7285,35 @@ function emitGithubAnnotations(data) {
|
|
|
7136
7285
|
return lines;
|
|
7137
7286
|
}
|
|
7138
7287
|
//#endregion
|
|
7288
|
+
//#region src/claude/to-report-cost.ts
|
|
7289
|
+
/**
|
|
7290
|
+
* Narrow an invocation's cost to the shape the report carries.
|
|
7291
|
+
*
|
|
7292
|
+
* The two differ by exactly one field — the report keeps only the API-time
|
|
7293
|
+
* figure, not `durationMs` — so this is a rest-spread rather than a
|
|
7294
|
+
* field-by-field copy: a field added to `ClaudeInvocationCost` and to
|
|
7295
|
+
* `ReportCost` then flows through without an edit here, and one added to only
|
|
7296
|
+
* the former fails to compile. Written out by hand, the same growth would
|
|
7297
|
+
* silently drop the new field, and per-step and run-level cost would stop
|
|
7298
|
+
* describing the same thing.
|
|
7299
|
+
*/
|
|
7300
|
+
function toReportCost(cost) {
|
|
7301
|
+
const { durationMs: _durationMs, ...rest } = cost;
|
|
7302
|
+
return rest;
|
|
7303
|
+
}
|
|
7304
|
+
//#endregion
|
|
7305
|
+
//#region src/report/run-cost.ts
|
|
7306
|
+
/**
|
|
7307
|
+
* What the running command has spent on Claude so far, in the report's shape.
|
|
7308
|
+
*
|
|
7309
|
+
* Null outside a `withCostTally` scope — a library caller of `executeRun`, or a
|
|
7310
|
+
* unit test — which is not the same as "spent nothing".
|
|
7311
|
+
*/
|
|
7312
|
+
function currentReportCost() {
|
|
7313
|
+
const cost = readCostTally();
|
|
7314
|
+
return cost === null ? null : toReportCost(cost);
|
|
7315
|
+
}
|
|
7316
|
+
//#endregion
|
|
7139
7317
|
//#region src/run/spec-catalog.ts
|
|
7140
7318
|
async function readSpecs(refs, cwd) {
|
|
7141
7319
|
const entries = await Promise.all(refs.map(async (ref) => {
|
|
@@ -8681,6 +8859,9 @@ const promptRm = new Command("rm").description("Delete a prompt from the hub.").
|
|
|
8681
8859
|
}));
|
|
8682
8860
|
const promptCommand = new Command("prompt").description("Manage prompt assets (per-flow user/agent guidance, triage/audit user guidance, learned calibration prompts) stored on the hub (fetched automatically by `ccqa run` / `ccqa audit` at run time).").addCommand(promptPush).addCommand(promptLs).addCommand(promptRm);
|
|
8683
8861
|
const deployRecord = new Command("record").description("Tell the hub what a deploy shipped, so it can answer which specs need a re-run (`ccqa run --only-hub-rerun-needed`). Run this from the deploy job, after the deploy succeeds. The changed paths are computed locally with a two-dot `git diff <previous> <sha>`; a job that has only curl and git can POST the same body directly (see docs/hub.md).").requiredOption("--profile <name>", "Environment this deploy landed in (e.g. 'stg'). Required: dev and stg sit at different commits, so the deploy log is per-profile.").requiredOption("--sha <sha>", "Commit that was deployed.").option("--previous <sha>", "Commit this deploy replaced. Omit it and the hub's current log head is used — the normal case, recording no discontinuity. Pass a sha that differs from the head and the hub records one (gapBefore) in the chain: use this for a first record with a real baseline, or to re-anchor a head that no longer matches reality. With no head and nothing passed, there's nothing to diff against: changedPaths is unset and the spec selection is skipped.").option("--ref <ref>", "Ref that was deployed (branch or tag). Recorded for display only.").option("--no-select-specs", "Record the deploy without deciding which specs it reaches. The entry then becomes a hole in the range — every spec behind it is assumed reached rather than being cleared, and nothing can fill it in later, since the hub has no checkout to diff. Only pass this when no Claude credential is available; it costs one model call to leave the range clearable.").option("-m, --model <name>", "Model for the spec selection. Claude alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option(...hubUrlOption).option(...hubTokenOption).option("--project <name>", "Project whose deploy log this entry joins. Defaults to the current directory's name.").option("--cwd <path>", "Directory the git diff and the default --project name are resolved against.").action(withHubErrors(async (opts) => {
|
|
8862
|
+
await withCostReporting("hub deploy record", () => runDeployRecord(opts));
|
|
8863
|
+
}));
|
|
8864
|
+
async function runDeployRecord(opts) {
|
|
8684
8865
|
const cwd = resolveCwd(opts.cwd);
|
|
8685
8866
|
const project = resolveProject(opts);
|
|
8686
8867
|
const hub = connect(opts);
|
|
@@ -8709,7 +8890,7 @@ const deployRecord = new Command("record").description("Tell the hub what a depl
|
|
|
8709
8890
|
process.exit(1);
|
|
8710
8891
|
}
|
|
8711
8892
|
info(`recorded deploy #${entry.index}`);
|
|
8712
|
-
}
|
|
8893
|
+
}
|
|
8713
8894
|
/**
|
|
8714
8895
|
* Decide which specs this deploy reaches, in the shape the hub stores.
|
|
8715
8896
|
*
|
|
@@ -8804,67 +8985,6 @@ function isStorageStateShape(state) {
|
|
|
8804
8985
|
return typeof state === "object" && state !== null && Array.isArray(state.cookies) && Array.isArray(state.origins);
|
|
8805
8986
|
}
|
|
8806
8987
|
//#endregion
|
|
8807
|
-
//#region src/runtime/live-cost-format.ts
|
|
8808
|
-
/**
|
|
8809
|
-
* Compact one-line cost summary. Format:
|
|
8810
|
-
* "$0.1234 · 4 turns · 42 in / 6,511 out · 2.0M cached · sonnet"
|
|
8811
|
-
* Returns null when no cost data is available (mock runs / SDK errors).
|
|
8812
|
-
*
|
|
8813
|
-
* `compact: false` (default for CLI logs) keeps raw numbers and adds a
|
|
8814
|
-
* `model=...` segment. `compact: true` (HTML chip) thousand-separates fresh
|
|
8815
|
-
* tokens, abbreviates cache-read with K/M, drops the `model=` prefix.
|
|
8816
|
-
*/
|
|
8817
|
-
function formatLiveCost(cost, options) {
|
|
8818
|
-
if (cost.totalCostUsd === null) return null;
|
|
8819
|
-
const compact = options.compact;
|
|
8820
|
-
const sep = compact ? " · " : " / ";
|
|
8821
|
-
const parts = [`$${cost.totalCostUsd.toFixed(4)}`];
|
|
8822
|
-
if (cost.numTurns !== null) parts.push(`${cost.numTurns} turns`);
|
|
8823
|
-
if (cost.inputTokens !== null || cost.outputTokens !== null) {
|
|
8824
|
-
const i = cost.inputTokens ?? 0;
|
|
8825
|
-
const o = cost.outputTokens ?? 0;
|
|
8826
|
-
parts.push(compact ? `${formatNumber(i)} in / ${formatNumber(o)} out` : `${i}+${o} tokens`);
|
|
8827
|
-
}
|
|
8828
|
-
if (cost.cacheReadInputTokens !== null && cost.cacheReadInputTokens > 0) parts.push(compact ? `${formatTokenK(cost.cacheReadInputTokens)} cached` : `${cost.cacheReadInputTokens} cache-read`);
|
|
8829
|
-
if (!compact && cost.models.length > 0) parts.push(`model=${cost.models.join(",")}`);
|
|
8830
|
-
return parts.join(sep);
|
|
8831
|
-
}
|
|
8832
|
-
/**
|
|
8833
|
-
* Sum of per-spec costs for a batch. Used only by the CLI batch summary.
|
|
8834
|
-
* Returns null when no spec has cost data.
|
|
8835
|
-
*/
|
|
8836
|
-
function formatLiveBatchCost(costs) {
|
|
8837
|
-
let totalUsd = 0;
|
|
8838
|
-
let seen = false;
|
|
8839
|
-
let totalIn = 0;
|
|
8840
|
-
let totalOut = 0;
|
|
8841
|
-
let totalCacheRead = 0;
|
|
8842
|
-
for (const c of costs) {
|
|
8843
|
-
if (c.totalCostUsd !== null) {
|
|
8844
|
-
totalUsd += c.totalCostUsd;
|
|
8845
|
-
seen = true;
|
|
8846
|
-
}
|
|
8847
|
-
totalIn += c.inputTokens ?? 0;
|
|
8848
|
-
totalOut += c.outputTokens ?? 0;
|
|
8849
|
-
totalCacheRead += c.cacheReadInputTokens ?? 0;
|
|
8850
|
-
}
|
|
8851
|
-
if (!seen) return null;
|
|
8852
|
-
const parts = [`$${totalUsd.toFixed(4)}`, `${totalIn}+${totalOut} tokens`];
|
|
8853
|
-
if (totalCacheRead > 0) parts.push(`${totalCacheRead} cache-read`);
|
|
8854
|
-
return parts.join(" / ");
|
|
8855
|
-
}
|
|
8856
|
-
/** Thousand-separated count for token figures. */
|
|
8857
|
-
function formatNumber(n) {
|
|
8858
|
-
return n.toLocaleString("en-US");
|
|
8859
|
-
}
|
|
8860
|
-
/** Compact token count: 9,043,456 → "9.0M", 12000 → "12K", small → plain. */
|
|
8861
|
-
function formatTokenK(n) {
|
|
8862
|
-
if (n >= 1e6) return `${(n / 1e6).toFixed(1)}M`;
|
|
8863
|
-
if (n >= 1e4) return `${Math.round(n / 1e3)}K`;
|
|
8864
|
-
if (n >= 1e3) return `${(n / 1e3).toFixed(1)}K`;
|
|
8865
|
-
return n.toLocaleString("en-US");
|
|
8866
|
-
}
|
|
8867
|
-
//#endregion
|
|
8868
8988
|
//#region src/claude/agent-browser-invoke.ts
|
|
8869
8989
|
function agentBrowserInvokeBase(input) {
|
|
8870
8990
|
return {
|
|
@@ -8883,14 +9003,9 @@ function agentBrowserInvokeBase(input) {
|
|
|
8883
9003
|
}
|
|
8884
9004
|
//#endregion
|
|
8885
9005
|
//#region src/prompts/live.ts
|
|
8886
|
-
/**
|
|
8887
|
-
* Unique agent-browser session name. The runId is millisecond-precision wall
|
|
8888
|
-
* clock, so under `--concurrency > 1` two specs can start in the same
|
|
8889
|
-
* millisecond and collide; a random suffix guarantees each spec gets its own
|
|
8890
|
-
* Chrome session and state never bleeds across parallel runs.
|
|
8891
|
-
*/
|
|
9006
|
+
/** Unique agent-browser session name, so parallel specs never share a Chrome. */
|
|
8892
9007
|
function generateLiveSessionName() {
|
|
8893
|
-
return `ccqa-live-${buildRunId()}
|
|
9008
|
+
return `ccqa-live-${buildRunId()}`;
|
|
8894
9009
|
}
|
|
8895
9010
|
/**
|
|
8896
9011
|
* Static prefix of the `ccqa run` (live spec) system prompt. Built once per
|
|
@@ -9179,16 +9294,7 @@ async function runLiveExecutor(input) {
|
|
|
9179
9294
|
});
|
|
9180
9295
|
isError = result.isError;
|
|
9181
9296
|
errorDetail = result.errorDetail;
|
|
9182
|
-
cost =
|
|
9183
|
-
totalCostUsd: result.cost.totalCostUsd,
|
|
9184
|
-
durationApiMs: result.cost.durationApiMs,
|
|
9185
|
-
numTurns: result.cost.numTurns,
|
|
9186
|
-
inputTokens: result.cost.inputTokens,
|
|
9187
|
-
cacheCreationInputTokens: result.cost.cacheCreationInputTokens,
|
|
9188
|
-
cacheReadInputTokens: result.cost.cacheReadInputTokens,
|
|
9189
|
-
outputTokens: result.cost.outputTokens,
|
|
9190
|
-
models: result.cost.models
|
|
9191
|
-
};
|
|
9297
|
+
cost = toReportCost(result.cost);
|
|
9192
9298
|
} catch (err) {
|
|
9193
9299
|
isError = true;
|
|
9194
9300
|
errorDetail = err instanceof Error ? err.message : String(err);
|
|
@@ -9576,7 +9682,6 @@ async function runLiveSpecs(specs, opts) {
|
|
|
9576
9682
|
const failedCount = runs.filter((r) => r.kind === "error" || r.kind === "run" && r.result.status === "failed").length;
|
|
9577
9683
|
blank();
|
|
9578
9684
|
meta("live-summary", `${runs.length - failedCount} passed / ${failedCount} failed`);
|
|
9579
|
-
logBatchCost(runs);
|
|
9580
9685
|
return {
|
|
9581
9686
|
failedCount,
|
|
9582
9687
|
reportResults: built.flatMap((b) => b.row ? [b.row] : [])
|
|
@@ -9779,10 +9884,6 @@ async function runOneSpec(args) {
|
|
|
9779
9884
|
await closeSession(sessionName);
|
|
9780
9885
|
}
|
|
9781
9886
|
}
|
|
9782
|
-
function logBatchCost(runs) {
|
|
9783
|
-
const line = formatLiveBatchCost(runs.flatMap((r) => r.kind === "run" ? [r.result.cost] : []));
|
|
9784
|
-
if (line) meta("total-cost", line);
|
|
9785
|
-
}
|
|
9786
9887
|
/**
|
|
9787
9888
|
* Classify one failed live run via `analyzeFailure` — same prompt as the
|
|
9788
9889
|
* deterministic path (Issue #47), fed the live transcript instead of the
|
|
@@ -11660,13 +11761,14 @@ async function runExternalSpecs(dispatch, ctx) {
|
|
|
11660
11761
|
}
|
|
11661
11762
|
//#endregion
|
|
11662
11763
|
//#region src/run/incremental-report.ts
|
|
11663
|
-
function createIncrementalReport(reportDir, envelope, sink) {
|
|
11764
|
+
function createIncrementalReport(reportDir, envelope, sink, costNow) {
|
|
11664
11765
|
const byKey = /* @__PURE__ */ new Map();
|
|
11665
11766
|
const reportPath = join(reportDir, "report.json");
|
|
11666
11767
|
let queue = Promise.resolve();
|
|
11667
11768
|
const key = (r) => `${r.feature}/${r.spec}`;
|
|
11668
11769
|
const buildData = () => ({
|
|
11669
11770
|
...envelope,
|
|
11771
|
+
...costNow ? { cost: costNow() } : {},
|
|
11670
11772
|
results: [...byKey.values()]
|
|
11671
11773
|
});
|
|
11672
11774
|
const doFlush = async () => {
|
|
@@ -12414,7 +12516,8 @@ async function executeRun(targets, opts) {
|
|
|
12414
12516
|
const evidence = await readRowFilesBase64(row, reportDir);
|
|
12415
12517
|
await hubCtx.hub.patchRun(runId, {
|
|
12416
12518
|
rows: [row],
|
|
12417
|
-
evidence
|
|
12519
|
+
evidence,
|
|
12520
|
+
reportMeta: { cost: currentReportCost() }
|
|
12418
12521
|
});
|
|
12419
12522
|
hubPatchEverSucceeded = true;
|
|
12420
12523
|
} catch (err) {
|
|
@@ -12433,7 +12536,7 @@ async function executeRun(targets, opts) {
|
|
|
12433
12536
|
triageUserPromptHash,
|
|
12434
12537
|
deployedSha,
|
|
12435
12538
|
opts
|
|
12436
|
-
}), hubSink);
|
|
12539
|
+
}), hubSink, currentReportCost);
|
|
12437
12540
|
let completedNormally = false;
|
|
12438
12541
|
opts.teardown?.onFinalize(async () => {
|
|
12439
12542
|
if (completedNormally) return;
|
|
@@ -12442,7 +12545,8 @@ async function executeRun(targets, opts) {
|
|
|
12442
12545
|
await hubCtx.hub.patchRun(hubRunId, {
|
|
12443
12546
|
rows: incrementalReport.rows(),
|
|
12444
12547
|
done: true,
|
|
12445
|
-
finalStatus: "failed"
|
|
12548
|
+
finalStatus: "failed",
|
|
12549
|
+
reportMeta: { cost: currentReportCost() }
|
|
12446
12550
|
});
|
|
12447
12551
|
} catch (err) {
|
|
12448
12552
|
warn(`hub: could not finalize interrupted run ${hubRunId}: ${errMessage(err)}`);
|
|
@@ -12803,7 +12907,8 @@ function buildReportEnvelope(args) {
|
|
|
12803
12907
|
promptVersion: "13",
|
|
12804
12908
|
customPromptVersion,
|
|
12805
12909
|
...triageUserPromptHash !== null ? { triageUserPromptHash } : {},
|
|
12806
|
-
...deployedSha !== null ? { deployedSha } : {}
|
|
12910
|
+
...deployedSha !== null ? { deployedSha } : {},
|
|
12911
|
+
cost: currentReportCost()
|
|
12807
12912
|
};
|
|
12808
12913
|
}
|
|
12809
12914
|
/** Write the unified JSON (+ optional GitHub-annotation) report for one run. Returns the report data. */
|
|
@@ -13100,23 +13205,24 @@ function headerTarget(targets, opts) {
|
|
|
13100
13205
|
async function runCliAction(targets, opts) {
|
|
13101
13206
|
header("run", headerTarget(targets, opts));
|
|
13102
13207
|
const cwd = resolveCwd(opts.cwd);
|
|
13103
|
-
const
|
|
13104
|
-
|
|
13105
|
-
|
|
13106
|
-
|
|
13107
|
-
|
|
13108
|
-
|
|
13109
|
-
|
|
13110
|
-
|
|
13111
|
-
|
|
13112
|
-
|
|
13113
|
-
|
|
13114
|
-
|
|
13115
|
-
|
|
13116
|
-
|
|
13117
|
-
|
|
13118
|
-
|
|
13119
|
-
|
|
13208
|
+
const exitCode = await withCostReporting("run", async () => {
|
|
13209
|
+
const teardown = createRunTeardown();
|
|
13210
|
+
const disposeSignalHandlers = installTeardownSignalHandlers(teardown);
|
|
13211
|
+
try {
|
|
13212
|
+
return (await executeRun(targets, {
|
|
13213
|
+
...opts,
|
|
13214
|
+
cwd,
|
|
13215
|
+
teardown
|
|
13216
|
+
})).exitCode;
|
|
13217
|
+
} catch (err) {
|
|
13218
|
+
if (!(err instanceof RunUsageError)) throw err;
|
|
13219
|
+
error(err.message);
|
|
13220
|
+
return err.exitCode;
|
|
13221
|
+
} finally {
|
|
13222
|
+
await teardown.run();
|
|
13223
|
+
disposeSignalHandlers();
|
|
13224
|
+
}
|
|
13225
|
+
});
|
|
13120
13226
|
process.exit(exitCode);
|
|
13121
13227
|
}
|
|
13122
13228
|
//#endregion
|
|
@@ -14688,6 +14794,9 @@ async function confirmOverwrite(path) {
|
|
|
14688
14794
|
}
|
|
14689
14795
|
}
|
|
14690
14796
|
const generateCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("generate").argument("<feature/spec>", "Spec id in '<feature>/<spec>' form (resolves to .ccqa/features/<feature>/test-cases/<spec>/)").description("Generate test code from a spec via its target plugin. Recording-backed targets compile the existing ir.json (run `ccqa record` first); spec-input targets generate directly from the spec.").optionsGroup("How to generate:").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--target <id>", "Generate through this target instead of the spec's own — e.g. emit a Playwright spec from an agent-browser recording. The spec's `target:` stays the default for `ccqa run`.").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--overwrite", "Replace previously generated test code without warning").optionsGroup("Learning:").option("--learn-hub-codegen-prompt", "After generation, ask Claude to refresh the target's \"<target>.agent\" learning prompt on the hub from a summary of the run. LLM-generating targets (playwright, runn) only; requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
14797
|
+
await withCostReporting("generate", () => runGenerateCli(specPath, opts));
|
|
14798
|
+
}));
|
|
14799
|
+
async function runGenerateCli(specPath, opts) {
|
|
14691
14800
|
const { featureName, specName } = parseSpecPath(specPath);
|
|
14692
14801
|
const language = opts.language ?? "auto";
|
|
14693
14802
|
const cwd = resolveCwd(opts.cwd);
|
|
@@ -14738,21 +14847,6 @@ const generateCommand = addHubOptions(addProfileOption(addLanguageOption(new Com
|
|
|
14738
14847
|
}
|
|
14739
14848
|
throw e;
|
|
14740
14849
|
}
|
|
14741
|
-
}));
|
|
14742
|
-
//#endregion
|
|
14743
|
-
//#region src/cli/cost-line.ts
|
|
14744
|
-
/**
|
|
14745
|
-
* Write what this command spent on Claude to stderr.
|
|
14746
|
-
*
|
|
14747
|
-
* stderr rather than stdout because `drift --format json` and `select-specs`
|
|
14748
|
-
* put machine-readable output on stdout, and a cost line mixed into it breaks
|
|
14749
|
-
* the consumer. Cost is diagnostics about the command, not its output.
|
|
14750
|
-
*/
|
|
14751
|
-
function reportCost() {
|
|
14752
|
-
const cost = readCostTally();
|
|
14753
|
-
if (cost === null) return;
|
|
14754
|
-
const summary = formatLiveCost(cost, { compact: false });
|
|
14755
|
-
if (summary) process.stderr.write(`[cost] ${summary}\n`);
|
|
14756
14850
|
}
|
|
14757
14851
|
//#endregion
|
|
14758
14852
|
//#region src/cli/record.ts
|
|
@@ -14761,13 +14855,7 @@ const recordCommand = addHubOptions(addProfileOption(addLanguageOption(new Comma
|
|
|
14761
14855
|
if (VALIDATION_MODES.includes(raw)) return raw;
|
|
14762
14856
|
throw new Error(`--trace-validation must be one of ${VALIDATION_MODES.join(" | ")}`);
|
|
14763
14857
|
}, "lenient").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--trace-only", "Stop after the trace step; do not generate test code").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--overwrite", "Replace an existing test.spec.ts without warning").optionsGroup("Learning:").option("--learn-hub-trace-prompt", "After the trace finishes, ask Claude to refresh the \"record.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
14764
|
-
await
|
|
14765
|
-
try {
|
|
14766
|
-
await runRecord(specPath, opts);
|
|
14767
|
-
} finally {
|
|
14768
|
-
reportCost();
|
|
14769
|
-
}
|
|
14770
|
-
});
|
|
14858
|
+
await withCostReporting("record", () => runRecord(specPath, opts));
|
|
14771
14859
|
}));
|
|
14772
14860
|
async function runRecord(specPath, opts) {
|
|
14773
14861
|
const { featureName, specName } = parseSpecPath(specPath);
|
|
@@ -15451,6 +15539,7 @@ function driftResultsToReport(results, meta) {
|
|
|
15451
15539
|
promptVersion: meta.promptVersion ?? DRIFT_REPORT_PROMPT_VERSION,
|
|
15452
15540
|
customPromptVersion: meta.customPromptVersion ?? null,
|
|
15453
15541
|
...meta.triageUserPromptHash ? { triageUserPromptHash: meta.triageUserPromptHash } : {},
|
|
15542
|
+
cost: currentReportCost(),
|
|
15454
15543
|
results: specResults
|
|
15455
15544
|
};
|
|
15456
15545
|
}
|
|
@@ -15640,7 +15729,7 @@ function selectSpecsNeedingAudit(targets, report) {
|
|
|
15640
15729
|
//#region src/cli/audit.ts
|
|
15641
15730
|
const DEFAULT_CONCURRENCY = 3;
|
|
15642
15731
|
const auditCommand = addProfileOption(addLanguageOption(new Command("audit").argument("[feature/spec]", "Optional spec id. If omitted, every spec under .ccqa/features/ is checked.").description("Read each spec against the code it describes and report where the two have drifted. Static: no browser is run, so this is the cheap check to put in front of `ccqa run`.").optionsGroup("Which specs to audit:").option("--only-affected-by <ref>", "Only specs `ccqa select-specs` judges reached by the diff against <ref> (e.g. origin/main). In pull_request CI, pass $GITHUB_BASE_REF. Costs one model call; specs it cannot decide are audited rather than skipped.").option("--only-hub-audit-needed", "Only specs the hub says a deploy has landed on since the audit last read them. A spec that was never audited is always included, and one the hub cannot answer for is audited rather than skipped. No git diff involved. Requires a hub connection and --hub-profile.").optionsGroup("How to run it:").option("--concurrency <n>", `Parallel spec checks (default: ${DEFAULT_CONCURRENCY})`).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").optionsGroup("What to do with the results:").option("--report-format <fmt>", "Output format: text | json | github", "text").option("--report-to-hub", "Push the result to a ccqa hub as a run (kind: drift), which is what updates the drift ledger. A spec it finds drifted answers `needsRepair` to `ccqa run --only-hub-rerun-needed`, and is not run until a person repairs it.").option("--exit-on <level>", "Exit non-zero on this severity or higher: warn | error", "error").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory used as both the .ccqa root and the codebase Claude reads. Useful for monorepos. Defaults to process.cwd().").option("--project <name>", "Logical project name for the pushed run. Defaults to the current directory's name.").option(...hubUrlOption).option(...hubTokenOption).option(...hubHeaderOption))).action(withUsageErrors(async (specPath, opts) => {
|
|
15643
|
-
await
|
|
15732
|
+
await withCostReporting("audit", () => runAudit(specPath, opts));
|
|
15644
15733
|
}));
|
|
15645
15734
|
async function runAudit(specPath, opts) {
|
|
15646
15735
|
const format = parseFormat$1(opts.reportFormat);
|
|
@@ -15745,7 +15834,6 @@ async function runAudit(specPath, opts) {
|
|
|
15745
15834
|
baseRef,
|
|
15746
15835
|
promptCtx
|
|
15747
15836
|
});
|
|
15748
|
-
reportCost();
|
|
15749
15837
|
process.exit(determineExitCode(results, threshold));
|
|
15750
15838
|
}
|
|
15751
15839
|
/** Longer than the audit's own timeout, so a sweep cannot outlive its claim. */
|
|
@@ -15896,7 +15984,6 @@ function asHubReadError(err) {
|
|
|
15896
15984
|
throw new RunUsageError(`could not read from the hub: ${errMessage(err)}`);
|
|
15897
15985
|
}
|
|
15898
15986
|
function exitWithNoSpecs(format, reason, message) {
|
|
15899
|
-
reportCost();
|
|
15900
15987
|
if (format === "json") process.stdout.write(`${JSON.stringify({
|
|
15901
15988
|
specs: [],
|
|
15902
15989
|
skipped: reason
|
|
@@ -16039,8 +16126,7 @@ For each test case above, write a 1–2 sentence factual \`summary\` of what it
|
|
|
16039
16126
|
//#endregion
|
|
16040
16127
|
//#region src/cli/perspectives.ts
|
|
16041
16128
|
const perspectivesCommand = addHubOptions(addLanguageOption(new Command("perspectives").description("Generate/update the project's perspectives document on the hub — a factual inventory of existing test coverage (no severity, no gap analysis)").option("--instruction <text>", "Hint to steer how summaries are written").option("-y, --yes", "Apply without asking [y/N]", false).option("--verify", "Check the hub document against the local specs (mechanical fields only) and exit 1 when it is stale. No Claude calls — cheap enough for CI.", false).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID").option("--project <name>", "Hub project to store the document under (default: cwd directory name)"))).action(withHubErrors(async (opts) => {
|
|
16042
|
-
|
|
16043
|
-
else await runPerspectives(opts);
|
|
16129
|
+
await withCostReporting("perspectives", () => opts.verify ? runPerspectivesCheck(opts) : runPerspectives(opts));
|
|
16044
16130
|
}));
|
|
16045
16131
|
/**
|
|
16046
16132
|
* `--check`: compare the hub document against a freshly-built local skeleton
|
|
@@ -16415,7 +16501,7 @@ function parseSummaries(json) {
|
|
|
16415
16501
|
//#region src/cli/select-specs.ts
|
|
16416
16502
|
const DEFAULT_HEAD = "HEAD";
|
|
16417
16503
|
const selectSpecsCommand = new Command("select-specs").description("Decide which specs a range of commits reaches. Reads the diff and the spec inventory and returns one verdict per spec: needed | notNeeded | unknown.").requiredOption("--base <ref>", "Commit the range starts at — typically what is currently deployed, or the previous commit on the branch.").option("--head <ref>", `Commit the range ends at (default: ${DEFAULT_HEAD})`).option("--cwd <path>", "Working directory used as both the .ccqa root and the codebase Claude reads. Changes outside it are reported but never attributed to a spec. Defaults to process.cwd().").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--format <fmt>", "Output format: text | json", "text").action(async (opts) => {
|
|
16418
|
-
await
|
|
16504
|
+
await withCostReporting("select-specs", () => runSelectSpecs(opts));
|
|
16419
16505
|
});
|
|
16420
16506
|
async function runSelectSpecs(opts) {
|
|
16421
16507
|
const format = parseFormat(opts.format);
|
|
@@ -16463,7 +16549,6 @@ async function runSelectSpecs(opts) {
|
|
|
16463
16549
|
...opts.model ? { model: opts.model } : {}
|
|
16464
16550
|
});
|
|
16465
16551
|
process.stdout.write(format === "json" ? `${JSON.stringify(report, null, 2)}\n` : renderText(report));
|
|
16466
|
-
reportCost();
|
|
16467
16552
|
process.exit(0);
|
|
16468
16553
|
}
|
|
16469
16554
|
const VERDICT_ORDER = [
|
|
@@ -16755,6 +16840,7 @@ function createPushRunHandler(config) {
|
|
|
16755
16840
|
},
|
|
16756
16841
|
gitHead: report.git.head,
|
|
16757
16842
|
promptVersion: report.promptVersion,
|
|
16843
|
+
costUsd: report.cost?.totalCostUsd ?? null,
|
|
16758
16844
|
ciRunId: report.runId,
|
|
16759
16845
|
runUrl: report.runUrl ?? null,
|
|
16760
16846
|
reportCreatedAt: report.createdAt,
|
|
@@ -16805,6 +16891,7 @@ function createOpenRunHandler(config) {
|
|
|
16805
16891
|
},
|
|
16806
16892
|
gitHead: gitHead || null,
|
|
16807
16893
|
promptVersion: "",
|
|
16894
|
+
costUsd: null,
|
|
16808
16895
|
ciRunId: ciRunId || null,
|
|
16809
16896
|
runUrl: runUrl || null,
|
|
16810
16897
|
reportCreatedAt: now,
|
|
@@ -16828,7 +16915,8 @@ const PatchRunRequestSchema = z.object({
|
|
|
16828
16915
|
promptVersion: z.string().optional(),
|
|
16829
16916
|
customPromptVersion: z.string().nullable().optional(),
|
|
16830
16917
|
runUrl: z.string().nullable().optional(),
|
|
16831
|
-
triageUserPromptHash: z.string().optional()
|
|
16918
|
+
triageUserPromptHash: z.string().optional(),
|
|
16919
|
+
cost: ReportCostSchema.nullable().optional()
|
|
16832
16920
|
}).partial().optional()
|
|
16833
16921
|
});
|
|
16834
16922
|
/** Insert or replace `rows` into `results`, upserting by feature/spec identity. */
|
|
@@ -16988,6 +17076,7 @@ function createPatchRunHandler(config) {
|
|
|
16988
17076
|
const { rows, evidence, done, finalStatus, reportMeta } = await readJsonBody(ctx.req, maxPushBytes, PatchRunRequestSchema, "request body");
|
|
16989
17077
|
let specs = run.specs;
|
|
16990
17078
|
let mergedResults = [];
|
|
17079
|
+
let costUsd = run.costUsd ?? null;
|
|
16991
17080
|
await config.storage.artifacts.updateJsonFile(id, "report.json", (current) => {
|
|
16992
17081
|
const base = current ?? {
|
|
16993
17082
|
schemaVersion: 1,
|
|
@@ -17001,7 +17090,8 @@ function createPatchRunHandler(config) {
|
|
|
17001
17090
|
model: null,
|
|
17002
17091
|
language: null,
|
|
17003
17092
|
promptVersion: "",
|
|
17004
|
-
customPromptVersion: null
|
|
17093
|
+
customPromptVersion: null,
|
|
17094
|
+
cost: null
|
|
17005
17095
|
};
|
|
17006
17096
|
const envelope = {
|
|
17007
17097
|
...base,
|
|
@@ -17016,11 +17106,13 @@ function createPatchRunHandler(config) {
|
|
|
17016
17106
|
...reportMeta?.promptVersion !== void 0 ? { promptVersion: reportMeta.promptVersion } : {},
|
|
17017
17107
|
...reportMeta?.customPromptVersion !== void 0 ? { customPromptVersion: reportMeta.customPromptVersion } : {},
|
|
17018
17108
|
...reportMeta?.runUrl !== void 0 ? { runUrl: reportMeta.runUrl } : {},
|
|
17019
|
-
...reportMeta?.triageUserPromptHash !== void 0 ? { triageUserPromptHash: reportMeta.triageUserPromptHash } : {}
|
|
17109
|
+
...reportMeta?.triageUserPromptHash !== void 0 ? { triageUserPromptHash: reportMeta.triageUserPromptHash } : {},
|
|
17110
|
+
...reportMeta?.cost !== void 0 ? { cost: reportMeta.cost } : {}
|
|
17020
17111
|
};
|
|
17021
17112
|
const merged = mergeResults(current?.results ?? [], rows);
|
|
17022
17113
|
specs = countSpecs(merged);
|
|
17023
17114
|
mergedResults = merged;
|
|
17115
|
+
costUsd = envelope.cost?.totalCostUsd ?? null;
|
|
17024
17116
|
return {
|
|
17025
17117
|
...envelope,
|
|
17026
17118
|
results: merged
|
|
@@ -17030,11 +17122,15 @@ function createPatchRunHandler(config) {
|
|
|
17030
17122
|
const patch = done ? {
|
|
17031
17123
|
status: finalStatus ?? (specs.failed > 0 ? "failed" : "passed"),
|
|
17032
17124
|
specs,
|
|
17125
|
+
costUsd,
|
|
17033
17126
|
...run.kind === "drift" ? { drift: summarizeDrift(mergedResults) } : {},
|
|
17034
17127
|
...reportMeta?.git?.head ? { gitHead: reportMeta.git.head } : {},
|
|
17035
17128
|
...reportMeta?.promptVersion ? { promptVersion: reportMeta.promptVersion } : {},
|
|
17036
17129
|
...await deployHeadMovedDuringRun(config.storage, run) ? { deployedShaAmbiguous: true } : {}
|
|
17037
|
-
} : {
|
|
17130
|
+
} : {
|
|
17131
|
+
specs,
|
|
17132
|
+
costUsd
|
|
17133
|
+
};
|
|
17038
17134
|
const updated = await config.storage.runs.update(id, patch);
|
|
17039
17135
|
if (done) {
|
|
17040
17136
|
await updateSpecLedger(config.storage, updated, mergedResults);
|
|
@@ -18585,6 +18681,7 @@ const HTML_BODY = `
|
|
|
18585
18681
|
<section id="view-runs">
|
|
18586
18682
|
<div class="page-bar">
|
|
18587
18683
|
<h1 data-i18n="runs.title">Runs</h1>
|
|
18684
|
+
<span class="total" id="runs-total-cost" hidden></span>
|
|
18588
18685
|
<div class="spacer"></div>
|
|
18589
18686
|
${refreshButton("runs-refresh")}
|
|
18590
18687
|
</div>
|
|
@@ -18592,7 +18689,7 @@ const HTML_BODY = `
|
|
|
18592
18689
|
<div class="card" id="runs-card">
|
|
18593
18690
|
<div class="table-wrap">
|
|
18594
18691
|
<table>
|
|
18595
|
-
<thead><tr><th data-i18n="runs.col.run">Run</th><th data-i18n="runs.col.branch">Branch</th><th data-i18n="meta.profile">Profile</th><th data-i18n="runs.col.status">Status</th><th data-i18n="runs.col.specs">Specs</th><th data-i18n="runs.col.created">Created</th></tr></thead>
|
|
18692
|
+
<thead><tr><th data-i18n="runs.col.run">Run</th><th data-i18n="runs.col.branch">Branch</th><th data-i18n="meta.profile">Profile</th><th data-i18n="runs.col.status">Status</th><th data-i18n="runs.col.specs">Specs</th><th data-i18n="runs.col.cost">Cost</th><th data-i18n="runs.col.created">Created</th></tr></thead>
|
|
18596
18693
|
<tbody id="runs-tbody"></tbody>
|
|
18597
18694
|
</table>
|
|
18598
18695
|
</div>
|
|
@@ -18948,6 +19045,7 @@ const CSS = `
|
|
|
18948
19045
|
.page-bar .back:hover { color: var(--fg); }
|
|
18949
19046
|
.page-bar .back svg { width: 15px; height: 15px; }
|
|
18950
19047
|
.page-bar .filters { display: flex; gap: 8px; margin-left: 8px; }
|
|
19048
|
+
.page-bar .total { font-size: 12px; color: var(--muted); font-variant-numeric: tabular-nums; white-space: nowrap; }
|
|
18951
19049
|
.page-bar .spacer { flex: 1; }
|
|
18952
19050
|
.content { padding: 18px 24px 48px; }
|
|
18953
19051
|
|
|
@@ -19489,11 +19587,12 @@ const CLIENT_JS = `
|
|
|
19489
19587
|
"runs.title": "Runs", "runs.empty": "Select a project to see its runs.",
|
|
19490
19588
|
"runs.none": "No runs yet for this project.", "projects.none": "No projects yet. Create one to get started.", "projects.noneShort": "No projects yet",
|
|
19491
19589
|
"runs.col.run": "Run", "runs.col.branch": "Branch", "runs.col.status": "Status",
|
|
19492
|
-
"runs.col.specs": "Specs", "runs.col.created": "Created",
|
|
19590
|
+
"runs.col.specs": "Specs", "runs.col.cost": "Cost", "runs.col.created": "Created",
|
|
19591
|
+
"runs.totalCost": "Cost of these {n}:",
|
|
19493
19592
|
"detail.back": "Runs", "detail.specs": "Specs",
|
|
19494
19593
|
"detail.download": "Download artifacts",
|
|
19495
19594
|
"detail.triage": "Triage",
|
|
19496
|
-
"meta.branch": "Branch", "meta.specs": "Specs",
|
|
19595
|
+
"meta.branch": "Branch", "meta.specs": "Specs", "meta.cost": "Cost",
|
|
19497
19596
|
"meta.created": "Created", "meta.passed": "passed", "meta.profile": "Profile",
|
|
19498
19597
|
"meta.drift": "Drift",
|
|
19499
19598
|
"diag.cause": "Cause", "diag.fix": "Fix",
|
|
@@ -19646,11 +19745,12 @@ const CLIENT_JS = `
|
|
|
19646
19745
|
"runs.title": "実行", "runs.empty": "プロジェクトを選択すると実行一覧が表示されます。",
|
|
19647
19746
|
"runs.none": "このプロジェクトにはまだ実行がありません。", "projects.none": "まだプロジェクトがありません。作成して始めましょう。", "projects.noneShort": "プロジェクトなし",
|
|
19648
19747
|
"runs.col.run": "実行", "runs.col.branch": "ブランチ", "runs.col.status": "ステータス",
|
|
19649
|
-
"runs.col.specs": "スペック", "runs.col.created": "作成",
|
|
19748
|
+
"runs.col.specs": "スペック", "runs.col.cost": "コスト", "runs.col.created": "作成",
|
|
19749
|
+
"runs.totalCost": "この {n} 件のコスト:",
|
|
19650
19750
|
"detail.back": "実行", "detail.specs": "スペック",
|
|
19651
19751
|
"detail.download": "アーティファクトをダウンロード",
|
|
19652
19752
|
"detail.triage": "トリアージ",
|
|
19653
|
-
"meta.branch": "ブランチ", "meta.specs": "スペック",
|
|
19753
|
+
"meta.branch": "ブランチ", "meta.specs": "スペック", "meta.cost": "コスト",
|
|
19654
19754
|
"meta.created": "作成", "meta.passed": "合格", "meta.profile": "プロファイル",
|
|
19655
19755
|
"meta.drift": "ドリフト",
|
|
19656
19756
|
"diag.cause": "原因", "diag.fix": "対処",
|
|
@@ -20042,6 +20142,11 @@ const CLIENT_JS = `
|
|
|
20042
20142
|
return run.kind === "drift" ? driftFoundBadge(driftRunState(run), "drift.run.") : statusBadge(run.status);
|
|
20043
20143
|
}
|
|
20044
20144
|
|
|
20145
|
+
// A run's Claude spend, in the same $x.xxxx form as the per-step badge.
|
|
20146
|
+
// A run that billed nothing, and one stored before costs were recorded, both
|
|
20147
|
+
// arrive as a non-number — printing $0.0000 would claim a measured zero.
|
|
20148
|
+
function costText(usd) { return typeof usd === "number" ? "$" + usd.toFixed(4) : "—"; }
|
|
20149
|
+
|
|
20045
20150
|
// One chip per non-zero drift label, worded via labelText — the same
|
|
20046
20151
|
// vocabulary the diagnosis card uses — so a run's summary never disagrees
|
|
20047
20152
|
// with its own spec cards. A zero-count label is omitted, same reasoning as
|
|
@@ -20260,9 +20365,38 @@ const CLIENT_JS = `
|
|
|
20260
20365
|
|
|
20261
20366
|
// ── runs list ────────────────────────────────────────────────────────
|
|
20262
20367
|
|
|
20368
|
+
// What the listed runs cost together — the accumulating number an operator
|
|
20369
|
+
// reads to decide how often CI should run, so it follows whatever filter
|
|
20370
|
+
// produced the list. Hidden when no listed run carries a cost at all, since
|
|
20371
|
+
// a "$0.0000" total would read as "CI is free" rather than "nothing measured".
|
|
20372
|
+
//
|
|
20373
|
+
// The label names the run count on purpose. The list is capped (limit=50), so
|
|
20374
|
+
// an unqualified "total" would quietly under-report a project's spend the
|
|
20375
|
+
// moment it has more runs than that — the one number this feature exists to
|
|
20376
|
+
// get right.
|
|
20377
|
+
function renderRunsTotalCost(runs) {
|
|
20378
|
+
var span = document.getElementById("runs-total-cost");
|
|
20379
|
+
// null until the first measured run, which is what distinguishes "nothing
|
|
20380
|
+
// was measured" from "the measured total happens to be zero".
|
|
20381
|
+
var total = null;
|
|
20382
|
+
// The runs that actually contributed, not the runs on screen. Runs stored
|
|
20383
|
+
// before costs were recorded carry nothing, so counting the list would
|
|
20384
|
+
// credit the sum to rows that gave it nothing — the exact misreading the
|
|
20385
|
+
// count is here to prevent.
|
|
20386
|
+
var measured = 0;
|
|
20387
|
+
runs.forEach(function (r) {
|
|
20388
|
+
if (typeof r.costUsd === "number") { total = (total || 0) + r.costUsd; measured++; }
|
|
20389
|
+
});
|
|
20390
|
+
span.hidden = total === null;
|
|
20391
|
+
if (total !== null) {
|
|
20392
|
+
span.textContent = t("runs.totalCost").replace("{n}", measured) + " " + costText(total);
|
|
20393
|
+
}
|
|
20394
|
+
}
|
|
20395
|
+
|
|
20263
20396
|
function renderRunsList(runs) {
|
|
20264
20397
|
var tbody = document.getElementById("runs-tbody");
|
|
20265
20398
|
clear(tbody);
|
|
20399
|
+
renderRunsTotalCost(runs);
|
|
20266
20400
|
var empty = document.getElementById("runs-empty");
|
|
20267
20401
|
if (runs.length === 0) {
|
|
20268
20402
|
empty.hidden = false;
|
|
@@ -20325,6 +20459,7 @@ const CLIENT_JS = `
|
|
|
20325
20459
|
specsCell.appendChild(specsWrap);
|
|
20326
20460
|
tr.appendChild(specsCell);
|
|
20327
20461
|
|
|
20462
|
+
tr.appendChild(el("td", "muted num", costText(r.costUsd)));
|
|
20328
20463
|
tr.appendChild(el("td", "muted num", relTime(r.createdAt)));
|
|
20329
20464
|
tbody.appendChild(tr);
|
|
20330
20465
|
});
|
|
@@ -20337,6 +20472,7 @@ const CLIENT_JS = `
|
|
|
20337
20472
|
.then(function (data) { renderRunsList(data.runs); })
|
|
20338
20473
|
.catch(function (err) {
|
|
20339
20474
|
clear(document.getElementById("runs-tbody"));
|
|
20475
|
+
renderRunsTotalCost([]);
|
|
20340
20476
|
empty.hidden = false;
|
|
20341
20477
|
empty.textContent = "Error loading runs: " + err.message;
|
|
20342
20478
|
});
|
|
@@ -20396,6 +20532,9 @@ const CLIENT_JS = `
|
|
|
20396
20532
|
} else {
|
|
20397
20533
|
metaItem(t("meta.specs"), run.specs.passed + " / " + run.specs.total + " " + t("meta.passed"));
|
|
20398
20534
|
}
|
|
20535
|
+
// Everything this run spent on Claude — live browsing, triage, the audit a
|
|
20536
|
+
// failure triggers, spec selection — not the sum of the per-step badges.
|
|
20537
|
+
metaItem(t("meta.cost"), costText(run.costUsd));
|
|
20399
20538
|
metaItem(t("meta.created"), relTime(run.createdAt));
|
|
20400
20539
|
head.appendChild(meta);
|
|
20401
20540
|
|
|
@@ -20679,7 +20818,7 @@ const CLIENT_JS = `
|
|
|
20679
20818
|
steps.forEach(function (s, i) {
|
|
20680
20819
|
var built = stepCard(s.status, "#" + (i + 1), s.instruction, s.expected, s.reasoning);
|
|
20681
20820
|
if (s.cost && s.cost.totalCostUsd != null) {
|
|
20682
|
-
built.head.appendChild(el("span", "cost",
|
|
20821
|
+
built.head.appendChild(el("span", "cost", costText(s.cost.totalCostUsd)));
|
|
20683
20822
|
}
|
|
20684
20823
|
if (s.beforePng || s.afterPng) {
|
|
20685
20824
|
var frames = el("div", "step-frames");
|
|
@@ -59,6 +59,7 @@ declare const RunSchema: z.ZodObject<{
|
|
|
59
59
|
}, z.core.$strip>;
|
|
60
60
|
gitHead: z.ZodNullable<z.ZodString>;
|
|
61
61
|
promptVersion: z.ZodString;
|
|
62
|
+
costUsd: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
|
|
62
63
|
ciRunId: z.ZodNullable<z.ZodString>;
|
|
63
64
|
runUrl: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
64
65
|
reportCreatedAt: z.ZodString;
|
|
@@ -575,6 +576,16 @@ declare const RunReportDataSchema: z.ZodObject<{
|
|
|
575
576
|
customPromptVersion: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
576
577
|
triageUserPromptHash: z.ZodOptional<z.ZodString>;
|
|
577
578
|
deployedSha: z.ZodOptional<z.ZodString>;
|
|
579
|
+
cost: z.ZodDefault<z.ZodNullable<z.ZodObject<{
|
|
580
|
+
totalCostUsd: z.ZodNullable<z.ZodNumber>;
|
|
581
|
+
durationApiMs: z.ZodNullable<z.ZodNumber>;
|
|
582
|
+
numTurns: z.ZodNullable<z.ZodNumber>;
|
|
583
|
+
inputTokens: z.ZodNullable<z.ZodNumber>;
|
|
584
|
+
cacheCreationInputTokens: z.ZodNullable<z.ZodNumber>;
|
|
585
|
+
cacheReadInputTokens: z.ZodNullable<z.ZodNumber>;
|
|
586
|
+
outputTokens: z.ZodNullable<z.ZodNumber>;
|
|
587
|
+
models: z.ZodArray<z.ZodString>;
|
|
588
|
+
}, z.core.$strip>>>;
|
|
578
589
|
results: z.ZodArray<z.ZodObject<{
|
|
579
590
|
feature: z.ZodString;
|
|
580
591
|
spec: z.ZodString;
|
package/dist/package.json
CHANGED