ccqa 1.20.1 → 1.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/ccqa.mjs +385 -182
- package/dist/hub-client/index.d.mts +19 -0
- package/dist/package.json +1 -1
- package/package.json +1 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -4,7 +4,7 @@ import { t as EVIDENCE_DIR_ENV } from "../evidence-constants-C425F7ZG.mjs";
|
|
|
4
4
|
import { a as formatAgentBrowserUnavailableMessage, i as assertAgentBrowserAvailable, n as spawnAB, o as pathWithAgentBrowserShim, r as AgentBrowserUnavailableError, s as resolveAgentBrowserBin$1, t as sleepSync } from "../spawn-ab-CRIVfWpw.mjs";
|
|
5
5
|
import { createRequire } from "node:module";
|
|
6
6
|
import { Command } from "commander";
|
|
7
|
-
import { accessSync, createWriteStream, existsSync, readFileSync, rmSync } from "node:fs";
|
|
7
|
+
import { accessSync, appendFileSync, createWriteStream, existsSync, readFileSync, rmSync } from "node:fs";
|
|
8
8
|
import { fileURLToPath } from "node:url";
|
|
9
9
|
import { createCipheriv, createDecipheriv, createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
|
|
10
10
|
import { access, chmod, cp, mkdir, mkdtemp, open, readFile, readdir, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
|
|
@@ -771,13 +771,21 @@ function waitExit(child) {
|
|
|
771
771
|
//#endregion
|
|
772
772
|
//#region src/runtime/live-artifacts.ts
|
|
773
773
|
/**
|
|
774
|
-
* Build a sortable run id
|
|
775
|
-
*
|
|
776
|
-
*
|
|
777
|
-
*
|
|
774
|
+
* Build a sortable, unique run id. ISO8601 with `:` / `.` replaced so it's
|
|
775
|
+
* filename-safe, timestamp first so run directories still sort by time, and a
|
|
776
|
+
* random suffix because the timestamp alone does not separate two specs.
|
|
777
|
+
*
|
|
778
|
+
* The pool launches specs back-to-back, so at `--concurrency > 1` two of them
|
|
779
|
+
* land in the same millisecond. A spec that puts `${CCQA_RUN_ID}` in the name
|
|
780
|
+
* of something it creates would then share that name with its neighbour, and
|
|
781
|
+
* each would find — and delete — the other's row. Nothing fails; the
|
|
782
|
+
* assertions just read the wrong state.
|
|
783
|
+
*
|
|
784
|
+
* Caller is expected to mkdir the directory once and pass
|
|
785
|
+
* `runDir = <baseDir>/<runId>` to the path helpers below.
|
|
778
786
|
*/
|
|
779
787
|
function buildRunId() {
|
|
780
|
-
return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")
|
|
788
|
+
return `${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${randomUUID().slice(0, 8)}`;
|
|
781
789
|
}
|
|
782
790
|
/**
|
|
783
791
|
* Per-step artifact paths under a run directory. `<runDir>/steps/<stepId>.*`.
|
|
@@ -1361,8 +1369,8 @@ async function withCostTally(fn) {
|
|
|
1361
1369
|
* The active scope's total so far, or null outside one.
|
|
1362
1370
|
*
|
|
1363
1371
|
* Read rather than pushed at the caller because commands end in
|
|
1364
|
-
* `process.exit`, which never reaches a `finally
|
|
1365
|
-
* total
|
|
1372
|
+
* `process.exit`, which never reaches a `finally`; whoever opened the scope
|
|
1373
|
+
* reads the total on the way out (see `withCostReporting`).
|
|
1366
1374
|
*
|
|
1367
1375
|
* Fields stay `null` when no invocation reported them, so a caller can tell
|
|
1368
1376
|
* "nothing was billed" from "the SDK didn't say" (mock runs, SDK errors).
|
|
@@ -1425,6 +1433,17 @@ function resolveEndpointEnv() {
|
|
|
1425
1433
|
}
|
|
1426
1434
|
return endpointEnv;
|
|
1427
1435
|
}
|
|
1436
|
+
/**
|
|
1437
|
+
* Drop endpoint variables that are present but empty, so an empty value never
|
|
1438
|
+
* reaches the Claude Code process as an override. "Set to nothing" is how a
|
|
1439
|
+
* caller that cannot omit the key says "use the default" — a CI job wiring
|
|
1440
|
+
* `ANTHROPIC_BASE_URL` from an unset repository variable, most of all.
|
|
1441
|
+
*/
|
|
1442
|
+
function withoutEmptyEndpointVars(env) {
|
|
1443
|
+
const out = { ...env };
|
|
1444
|
+
for (const key of ENDPOINT_ENV_KEYS) if (out[key] === "") delete out[key];
|
|
1445
|
+
return out;
|
|
1446
|
+
}
|
|
1428
1447
|
let nativeBinaryWarned = false;
|
|
1429
1448
|
/**
|
|
1430
1449
|
* Warn once per process when the SDK's per-platform native binary is missing:
|
|
@@ -1441,10 +1460,10 @@ async function invokeClaudeStreaming(options, onEvent) {
|
|
|
1441
1460
|
const { prompt, systemPrompt, allowedTools, disableBuiltinTools = false, disableThinking = false, mcpServers, maxTurns, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, relaxAbConstraints = false } = options;
|
|
1442
1461
|
const resolvedModel = resolveModel(model);
|
|
1443
1462
|
const hasEndpointEnv = Object.keys(resolveEndpointEnv()).length > 0;
|
|
1444
|
-
const mergedEnv = env || hasEndpointEnv ? {
|
|
1463
|
+
const mergedEnv = env || hasEndpointEnv ? withoutEmptyEndpointVars({
|
|
1445
1464
|
...process.env,
|
|
1446
1465
|
...env
|
|
1447
|
-
} : void 0;
|
|
1466
|
+
}) : void 0;
|
|
1448
1467
|
let lastAbToolUseId = null;
|
|
1449
1468
|
const claimAbToolUse = (toolUseId) => {
|
|
1450
1469
|
if (toolUseId !== lastAbToolUseId) return false;
|
|
@@ -3227,20 +3246,6 @@ const ReportArtifactSchema = z.object({
|
|
|
3227
3246
|
sizeBytes: z.number()
|
|
3228
3247
|
});
|
|
3229
3248
|
/**
|
|
3230
|
-
* Per-step row for a live-mode run (spec.yaml `mode: live`). Mirrors the
|
|
3231
|
-
* structure produced by `src/runtime/live-executor.ts:LiveStepResult` but
|
|
3232
|
-
* encoded against the report schema so the HTML renderer can carry both
|
|
3233
|
-
* deterministic (`evidence`) and live (`liveRun`) sources of step-boundary
|
|
3234
|
-
* screenshots.
|
|
3235
|
-
*
|
|
3236
|
-
* `beforePng` / `afterPng` are RELATIVE to the report directory, same
|
|
3237
|
-
* convention as `ReportEvidenceSchema.pngPath` above. The caller copies the
|
|
3238
|
-
* PNG files into `<reportDir>/evidence/<feature>/<spec>/` and computes the
|
|
3239
|
-
* relative path with `node:path`'s `relative()`, so the report directory is
|
|
3240
|
-
* self-contained: it can be archived and shipped on its own (e.g. a hub
|
|
3241
|
-
* push) without also bundling the `.ccqa` runs dir.
|
|
3242
|
-
*/
|
|
3243
|
-
/**
|
|
3244
3249
|
* Per-step / per-run cost+usage record, pulled from the SDK's `result` message.
|
|
3245
3250
|
* Every numeric field is nullable so the report can carry partial telemetry
|
|
3246
3251
|
* (e.g. when the SDK omits a field, or when a step was skipped).
|
|
@@ -3248,7 +3253,7 @@ const ReportArtifactSchema = z.object({
|
|
|
3248
3253
|
* `models` is the union of model ids the SDK reported using; usually a
|
|
3249
3254
|
* single element, but the SDK can fan out across models in some modes.
|
|
3250
3255
|
*/
|
|
3251
|
-
const
|
|
3256
|
+
const ReportCostSchema = z.object({
|
|
3252
3257
|
totalCostUsd: z.number().nullable(),
|
|
3253
3258
|
durationApiMs: z.number().nullable(),
|
|
3254
3259
|
numTurns: z.number().nullable(),
|
|
@@ -3258,6 +3263,20 @@ const LiveReportCostSchema = z.object({
|
|
|
3258
3263
|
outputTokens: z.number().nullable(),
|
|
3259
3264
|
models: z.array(z.string())
|
|
3260
3265
|
});
|
|
3266
|
+
/**
|
|
3267
|
+
* Per-step row for a live-mode run (spec.yaml `mode: live`). Mirrors the
|
|
3268
|
+
* structure produced by `src/runtime/live-executor.ts:LiveStepResult` but
|
|
3269
|
+
* encoded against the report schema so the HTML renderer can carry both
|
|
3270
|
+
* deterministic (`evidence`) and live (`liveRun`) sources of step-boundary
|
|
3271
|
+
* screenshots.
|
|
3272
|
+
*
|
|
3273
|
+
* `beforePng` / `afterPng` are RELATIVE to the report directory, same
|
|
3274
|
+
* convention as `ReportEvidenceSchema.pngPath` above. The caller copies the
|
|
3275
|
+
* PNG files into `<reportDir>/evidence/<feature>/<spec>/` and computes the
|
|
3276
|
+
* relative path with `node:path`'s `relative()`, so the report directory is
|
|
3277
|
+
* self-contained: it can be archived and shipped on its own (e.g. a hub
|
|
3278
|
+
* push) without also bundling the `.ccqa` runs dir.
|
|
3279
|
+
*/
|
|
3261
3280
|
const LiveReportStepSchema = z.object({
|
|
3262
3281
|
stepId: z.string(),
|
|
3263
3282
|
source: z.string(),
|
|
@@ -3272,7 +3291,7 @@ const LiveReportStepSchema = z.object({
|
|
|
3272
3291
|
beforePng: z.string().nullable(),
|
|
3273
3292
|
afterPng: z.string().nullable(),
|
|
3274
3293
|
durationMs: z.number(),
|
|
3275
|
-
cost:
|
|
3294
|
+
cost: ReportCostSchema,
|
|
3276
3295
|
commands: z.array(z.string()).optional()
|
|
3277
3296
|
});
|
|
3278
3297
|
const LiveReportRunSchema = z.object({
|
|
@@ -3281,7 +3300,7 @@ const LiveReportRunSchema = z.object({
|
|
|
3281
3300
|
startedAt: z.string(),
|
|
3282
3301
|
durationMs: z.number(),
|
|
3283
3302
|
steps: z.array(LiveReportStepSchema),
|
|
3284
|
-
cost:
|
|
3303
|
+
cost: ReportCostSchema
|
|
3285
3304
|
});
|
|
3286
3305
|
const ReportSpecResultSchema = z.object({
|
|
3287
3306
|
feature: z.string(),
|
|
@@ -3351,6 +3370,7 @@ const RunReportDataSchema = z.object({
|
|
|
3351
3370
|
customPromptVersion: z.string().nullable().default(null),
|
|
3352
3371
|
triageUserPromptHash: z.string().optional(),
|
|
3353
3372
|
deployedSha: z.string().optional(),
|
|
3373
|
+
cost: ReportCostSchema.nullable().default(null),
|
|
3354
3374
|
results: z.array(ReportSpecResultSchema)
|
|
3355
3375
|
});
|
|
3356
3376
|
/** Shape of the "export labels" download produced by the report's client-side JS. */
|
|
@@ -4581,9 +4601,137 @@ const DraftNamingSchema = z.object({
|
|
|
4581
4601
|
reason: z.string().optional()
|
|
4582
4602
|
});
|
|
4583
4603
|
//#endregion
|
|
4604
|
+
//#region src/runtime/live-cost-format.ts
|
|
4605
|
+
/**
|
|
4606
|
+
* Compact one-line cost summary. Format:
|
|
4607
|
+
* "$0.1234 · 4 turns · 42 in / 6,511 out · 2.0M cached · sonnet"
|
|
4608
|
+
* Returns null only when the invocation reported nothing at all (a mock run,
|
|
4609
|
+
* an SDK error, or a command that never called a model).
|
|
4610
|
+
*
|
|
4611
|
+
* The price is one segment among several, not a precondition. An endpoint the
|
|
4612
|
+
* SDK has no pricing table for — any Anthropic-compatible gateway in front of
|
|
4613
|
+
* a third-party model — reports usage but no `total_cost_usd`, and dropping
|
|
4614
|
+
* the whole line there would hide real consumption behind silence. Tokens come
|
|
4615
|
+
* from the API response rather than a price list, so they survive that case
|
|
4616
|
+
* and become the signal to read.
|
|
4617
|
+
*
|
|
4618
|
+
* `compact: false` (default for CLI logs) keeps raw numbers and adds a
|
|
4619
|
+
* `model=...` segment. `compact: true` (HTML chip) thousand-separates fresh
|
|
4620
|
+
* tokens, abbreviates cache-read with K/M, drops the `model=` prefix.
|
|
4621
|
+
*/
|
|
4622
|
+
function formatLiveCost(cost, options) {
|
|
4623
|
+
const compact = options.compact;
|
|
4624
|
+
const sep = compact ? " · " : " / ";
|
|
4625
|
+
const parts = [];
|
|
4626
|
+
if (cost.totalCostUsd !== null) parts.push(`$${cost.totalCostUsd.toFixed(4)}`);
|
|
4627
|
+
if (cost.numTurns !== null) parts.push(`${cost.numTurns} turns`);
|
|
4628
|
+
if (cost.inputTokens !== null || cost.outputTokens !== null) {
|
|
4629
|
+
const i = cost.inputTokens ?? 0;
|
|
4630
|
+
const o = cost.outputTokens ?? 0;
|
|
4631
|
+
parts.push(compact ? `${formatNumber(i)} in / ${formatNumber(o)} out` : `${i}+${o} tokens`);
|
|
4632
|
+
}
|
|
4633
|
+
if (cost.cacheReadInputTokens !== null && cost.cacheReadInputTokens > 0) parts.push(compact ? `${formatTokenK(cost.cacheReadInputTokens)} cached` : `${cost.cacheReadInputTokens} cache-read`);
|
|
4634
|
+
if (!compact && cost.models.length > 0) parts.push(`model=${cost.models.join(",")}`);
|
|
4635
|
+
return parts.length > 0 ? parts.join(sep) : null;
|
|
4636
|
+
}
|
|
4637
|
+
/** Thousand-separated count for token figures. */
|
|
4638
|
+
function formatNumber(n) {
|
|
4639
|
+
return n.toLocaleString("en-US");
|
|
4640
|
+
}
|
|
4641
|
+
/** Compact token count: 9,043,456 → "9.0M", 12000 → "12K", small → plain. */
|
|
4642
|
+
function formatTokenK(n) {
|
|
4643
|
+
if (n >= 1e6) return `${(n / 1e6).toFixed(1)}M`;
|
|
4644
|
+
if (n >= 1e4) return `${Math.round(n / 1e3)}K`;
|
|
4645
|
+
if (n >= 1e3) return `${(n / 1e3).toFixed(1)}K`;
|
|
4646
|
+
return n.toLocaleString("en-US");
|
|
4647
|
+
}
|
|
4648
|
+
//#endregion
|
|
4649
|
+
//#region src/cli/cost-line.ts
|
|
4650
|
+
/**
|
|
4651
|
+
* Report what this command spent on Claude: a human line on stderr, and — when
|
|
4652
|
+
* `CCQA_COST_FILE` is set — one JSON line appended to that file.
|
|
4653
|
+
*
|
|
4654
|
+
* stderr rather than stdout because `audit --report-format json` and
|
|
4655
|
+
* `select-specs` put machine-readable output on stdout, and a cost line mixed
|
|
4656
|
+
* into it breaks the consumer. Cost is diagnostics about the command, not its
|
|
4657
|
+
* output.
|
|
4658
|
+
*
|
|
4659
|
+
* No-op outside a `withCostTally` scope: a command that never opened one has no
|
|
4660
|
+
* number to report, which is not the same as having spent nothing.
|
|
4661
|
+
*
|
|
4662
|
+
* Reached through `withCostReporting`; exported for its own test.
|
|
4663
|
+
*/
|
|
4664
|
+
function reportCost(command) {
|
|
4665
|
+
const cost = readCostTally();
|
|
4666
|
+
if (cost === null) return;
|
|
4667
|
+
const summary = formatLiveCost(cost, { compact: false });
|
|
4668
|
+
if (summary) process.stderr.write(`[cost] ${summary}\n`);
|
|
4669
|
+
appendCostRecord(command, cost);
|
|
4670
|
+
}
|
|
4671
|
+
/**
|
|
4672
|
+
* Run a command's action inside a cost tally and report the total once, however
|
|
4673
|
+
* the command ends.
|
|
4674
|
+
*
|
|
4675
|
+
* Neither half covers the other. Commands that end deep inside themselves with
|
|
4676
|
+
* `process.exit` never unwind, so only the `exit` listener sees them — and both
|
|
4677
|
+
* it and `reportCost` are synchronous, so the report still lands. Commands that
|
|
4678
|
+
* return normally are past `withCostTally`'s scope by the time `exit` fires, so
|
|
4679
|
+
* only the `finally` can still read the tally.
|
|
4680
|
+
*
|
|
4681
|
+
* Exactly one of the two runs: the `finally` detaches the listener before
|
|
4682
|
+
* reporting, and a `process.exit` terminates before the `finally` is reached.
|
|
4683
|
+
*
|
|
4684
|
+
* A signal handler only reports if it was REGISTERED inside this scope. Node
|
|
4685
|
+
* creates the SIGINT/SIGTERM handle when its first listener is added and binds
|
|
4686
|
+
* the async context then, so a handler installed at module load would read an
|
|
4687
|
+
* empty tally and drop the whole report on interrupt. `run.ts` installs its
|
|
4688
|
+
* teardown handlers inside the callback for that reason.
|
|
4689
|
+
*/
|
|
4690
|
+
async function withCostReporting(command, fn) {
|
|
4691
|
+
return withCostTally(async () => {
|
|
4692
|
+
const report = () => reportCost(command);
|
|
4693
|
+
process.once("exit", report);
|
|
4694
|
+
try {
|
|
4695
|
+
return await fn();
|
|
4696
|
+
} finally {
|
|
4697
|
+
process.off("exit", report);
|
|
4698
|
+
report();
|
|
4699
|
+
}
|
|
4700
|
+
});
|
|
4701
|
+
}
|
|
4702
|
+
/**
|
|
4703
|
+
* Append one JSONL record to `$CCQA_COST_FILE`. Appended, never truncated: a CI
|
|
4704
|
+
* job invokes ccqa several times (select-specs, audit, run) and the point of the
|
|
4705
|
+
* file is that one `jq -s 'map(.totalCostUsd)|add'` covers the whole job.
|
|
4706
|
+
*
|
|
4707
|
+
* Written even when `totalCostUsd` is null — that the command ran and was not
|
|
4708
|
+
* billed is itself the answer. Synchronous because every caller is about to
|
|
4709
|
+
* `process.exit`, which would drop a pending async write.
|
|
4710
|
+
*/
|
|
4711
|
+
function appendCostRecord(command, cost) {
|
|
4712
|
+
const path = process.env.CCQA_COST_FILE;
|
|
4713
|
+
if (!path) return;
|
|
4714
|
+
const record = {
|
|
4715
|
+
command,
|
|
4716
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4717
|
+
totalCostUsd: cost.totalCostUsd,
|
|
4718
|
+
numTurns: cost.numTurns,
|
|
4719
|
+
inputTokens: cost.inputTokens,
|
|
4720
|
+
outputTokens: cost.outputTokens,
|
|
4721
|
+
cacheReadInputTokens: cost.cacheReadInputTokens,
|
|
4722
|
+
models: cost.models
|
|
4723
|
+
};
|
|
4724
|
+
try {
|
|
4725
|
+
appendFileSync(path, `${JSON.stringify(record)}\n`, "utf8");
|
|
4726
|
+
} catch {}
|
|
4727
|
+
}
|
|
4728
|
+
//#endregion
|
|
4584
4729
|
//#region src/cli/draft.ts
|
|
4585
4730
|
const CATEGORY_LABEL = DRAFT_CATEGORY_LABEL;
|
|
4586
4731
|
const draftCommand = addLanguageOption(new Command("draft").argument("[feature/spec]", "Optional spec path (e.g. tasks/create-and-complete). If omitted, Claude proposes one from your intent.").description("Interactively draft and refine a spec.yaml with Claude Code").option("--instruction <text>", "Non-interactive single-shot instruction (skips the interactive loop)").option("-y, --yes", "Apply each generated patch without asking [y/N]", false)).action(withUsageErrors(async (specPath, opts) => {
|
|
4732
|
+
await withCostReporting("draft", () => runDraftCli(specPath, opts));
|
|
4733
|
+
}));
|
|
4734
|
+
async function runDraftCli(specPath, opts) {
|
|
4587
4735
|
await ensureCcqaDir();
|
|
4588
4736
|
let featureName;
|
|
4589
4737
|
let specName;
|
|
@@ -4596,7 +4744,7 @@ const draftCommand = addLanguageOption(new Command("draft").argument("[feature/s
|
|
|
4596
4744
|
prefilledIntent = intent;
|
|
4597
4745
|
}
|
|
4598
4746
|
await runDraft(featureName, specName, opts, prefilledIntent);
|
|
4599
|
-
}
|
|
4747
|
+
}
|
|
4600
4748
|
async function runDraft(featureName, specName, opts, prefilledIntent) {
|
|
4601
4749
|
header("draft", `${featureName}/${specName}`);
|
|
4602
4750
|
const ja = useJapanesePrompts(opts.language);
|
|
@@ -6611,6 +6759,7 @@ z.object({
|
|
|
6611
6759
|
}),
|
|
6612
6760
|
gitHead: z.string().nullable(),
|
|
6613
6761
|
promptVersion: z.string(),
|
|
6762
|
+
costUsd: z.number().nullable().optional(),
|
|
6614
6763
|
ciRunId: z.string().nullable(),
|
|
6615
6764
|
runUrl: z.string().nullable().optional(),
|
|
6616
6765
|
reportCreatedAt: z.string(),
|
|
@@ -6693,10 +6842,26 @@ const SpecLedgerEntrySchema = z.object({
|
|
|
6693
6842
|
deployedSha: z.string().nullable().optional(),
|
|
6694
6843
|
deployedShaAmbiguous: z.boolean().optional()
|
|
6695
6844
|
});
|
|
6845
|
+
/**
|
|
6846
|
+
* A red-bucket entry: where the failure happened, plus what it was. The cause
|
|
6847
|
+
* is copied off the run report's `analysis` so a reader learns why a spec is
|
|
6848
|
+
* red without fetching the report of every red spec.
|
|
6849
|
+
*
|
|
6850
|
+
* Only the red bucket carries it. A pass has no cause, so putting these on
|
|
6851
|
+
* `SpecLedgerEntry` would invite writing them where they cannot exist.
|
|
6852
|
+
*
|
|
6853
|
+
* Both fields are optional and mean the same thing by their absence — nothing
|
|
6854
|
+
* is on record. Failure analysis is opt-in (`--on-fail-explain`), and entries
|
|
6855
|
+
* written before this release have neither field.
|
|
6856
|
+
*/
|
|
6857
|
+
const SpecRedLedgerEntrySchema = SpecLedgerEntrySchema.extend({
|
|
6858
|
+
label: PredictedLabelSchema.optional(),
|
|
6859
|
+
headline: z.string().optional()
|
|
6860
|
+
});
|
|
6696
6861
|
z.object({
|
|
6697
6862
|
green: z.record(z.string(), SpecLedgerEntrySchema).default({}),
|
|
6698
6863
|
run: z.record(z.string(), SpecLedgerEntrySchema).default({}),
|
|
6699
|
-
red: z.record(z.string(),
|
|
6864
|
+
red: z.record(z.string(), SpecRedLedgerEntrySchema).default({})
|
|
6700
6865
|
});
|
|
6701
6866
|
/** One deploy, as the consumer's deploy job reported it (ADR-0010). */
|
|
6702
6867
|
const DeployEntrySchema = z.object({
|
|
@@ -6876,7 +7041,7 @@ const SpecRerunSchema = z.object({
|
|
|
6876
7041
|
heldBy: SpecLockSchema.nullable(),
|
|
6877
7042
|
lastRun: SpecLedgerEntrySchema.nullable(),
|
|
6878
7043
|
lastGreen: SpecLedgerEntrySchema.nullable(),
|
|
6879
|
-
lastRed:
|
|
7044
|
+
lastRed: SpecRedLedgerEntrySchema.nullable(),
|
|
6880
7045
|
touchedBy: z.array(z.string()).optional(),
|
|
6881
7046
|
touchedByDeploy: DeployRefSchema.nullable().optional()
|
|
6882
7047
|
});
|
|
@@ -6926,7 +7091,7 @@ z.object({
|
|
|
6926
7091
|
z.object({
|
|
6927
7092
|
entries: z.record(z.string(), SpecLedgerEntrySchema),
|
|
6928
7093
|
lastRun: z.record(z.string(), SpecLedgerEntrySchema).default({}),
|
|
6929
|
-
lastRed: z.record(z.string(),
|
|
7094
|
+
lastRed: z.record(z.string(), SpecRedLedgerEntrySchema).default({})
|
|
6930
7095
|
});
|
|
6931
7096
|
/**
|
|
6932
7097
|
* One spec's last drift audit, as recorded by `ccqa audit --report-to-hub`. Unlike the
|
|
@@ -7136,6 +7301,35 @@ function emitGithubAnnotations(data) {
|
|
|
7136
7301
|
return lines;
|
|
7137
7302
|
}
|
|
7138
7303
|
//#endregion
|
|
7304
|
+
//#region src/claude/to-report-cost.ts
|
|
7305
|
+
/**
|
|
7306
|
+
* Narrow an invocation's cost to the shape the report carries.
|
|
7307
|
+
*
|
|
7308
|
+
* The two differ by exactly one field — the report keeps only the API-time
|
|
7309
|
+
* figure, not `durationMs` — so this is a rest-spread rather than a
|
|
7310
|
+
* field-by-field copy: a field added to `ClaudeInvocationCost` and to
|
|
7311
|
+
* `ReportCost` then flows through without an edit here, and one added to only
|
|
7312
|
+
* the former fails to compile. Written out by hand, the same growth would
|
|
7313
|
+
* silently drop the new field, and per-step and run-level cost would stop
|
|
7314
|
+
* describing the same thing.
|
|
7315
|
+
*/
|
|
7316
|
+
function toReportCost(cost) {
|
|
7317
|
+
const { durationMs: _durationMs, ...rest } = cost;
|
|
7318
|
+
return rest;
|
|
7319
|
+
}
|
|
7320
|
+
//#endregion
|
|
7321
|
+
//#region src/report/run-cost.ts
|
|
7322
|
+
/**
|
|
7323
|
+
* What the running command has spent on Claude so far, in the report's shape.
|
|
7324
|
+
*
|
|
7325
|
+
* Null outside a `withCostTally` scope — a library caller of `executeRun`, or a
|
|
7326
|
+
* unit test — which is not the same as "spent nothing".
|
|
7327
|
+
*/
|
|
7328
|
+
function currentReportCost() {
|
|
7329
|
+
const cost = readCostTally();
|
|
7330
|
+
return cost === null ? null : toReportCost(cost);
|
|
7331
|
+
}
|
|
7332
|
+
//#endregion
|
|
7139
7333
|
//#region src/run/spec-catalog.ts
|
|
7140
7334
|
async function readSpecs(refs, cwd) {
|
|
7141
7335
|
const entries = await Promise.all(refs.map(async (ref) => {
|
|
@@ -8681,6 +8875,9 @@ const promptRm = new Command("rm").description("Delete a prompt from the hub.").
|
|
|
8681
8875
|
}));
|
|
8682
8876
|
const promptCommand = new Command("prompt").description("Manage prompt assets (per-flow user/agent guidance, triage/audit user guidance, learned calibration prompts) stored on the hub (fetched automatically by `ccqa run` / `ccqa audit` at run time).").addCommand(promptPush).addCommand(promptLs).addCommand(promptRm);
|
|
8683
8877
|
const deployRecord = new Command("record").description("Tell the hub what a deploy shipped, so it can answer which specs need a re-run (`ccqa run --only-hub-rerun-needed`). Run this from the deploy job, after the deploy succeeds. The changed paths are computed locally with a two-dot `git diff <previous> <sha>`; a job that has only curl and git can POST the same body directly (see docs/hub.md).").requiredOption("--profile <name>", "Environment this deploy landed in (e.g. 'stg'). Required: dev and stg sit at different commits, so the deploy log is per-profile.").requiredOption("--sha <sha>", "Commit that was deployed.").option("--previous <sha>", "Commit this deploy replaced. Omit it and the hub's current log head is used — the normal case, recording no discontinuity. Pass a sha that differs from the head and the hub records one (gapBefore) in the chain: use this for a first record with a real baseline, or to re-anchor a head that no longer matches reality. With no head and nothing passed, there's nothing to diff against: changedPaths is unset and the spec selection is skipped.").option("--ref <ref>", "Ref that was deployed (branch or tag). Recorded for display only.").option("--no-select-specs", "Record the deploy without deciding which specs it reaches. The entry then becomes a hole in the range — every spec behind it is assumed reached rather than being cleared, and nothing can fill it in later, since the hub has no checkout to diff. Only pass this when no Claude credential is available; it costs one model call to leave the range clearable.").option("-m, --model <name>", "Model for the spec selection. Claude alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option(...hubUrlOption).option(...hubTokenOption).option("--project <name>", "Project whose deploy log this entry joins. Defaults to the current directory's name.").option("--cwd <path>", "Directory the git diff and the default --project name are resolved against.").action(withHubErrors(async (opts) => {
|
|
8878
|
+
await withCostReporting("hub deploy record", () => runDeployRecord(opts));
|
|
8879
|
+
}));
|
|
8880
|
+
async function runDeployRecord(opts) {
|
|
8684
8881
|
const cwd = resolveCwd(opts.cwd);
|
|
8685
8882
|
const project = resolveProject(opts);
|
|
8686
8883
|
const hub = connect(opts);
|
|
@@ -8709,7 +8906,7 @@ const deployRecord = new Command("record").description("Tell the hub what a depl
|
|
|
8709
8906
|
process.exit(1);
|
|
8710
8907
|
}
|
|
8711
8908
|
info(`recorded deploy #${entry.index}`);
|
|
8712
|
-
}
|
|
8909
|
+
}
|
|
8713
8910
|
/**
|
|
8714
8911
|
* Decide which specs this deploy reaches, in the shape the hub stores.
|
|
8715
8912
|
*
|
|
@@ -8804,67 +9001,6 @@ function isStorageStateShape(state) {
|
|
|
8804
9001
|
return typeof state === "object" && state !== null && Array.isArray(state.cookies) && Array.isArray(state.origins);
|
|
8805
9002
|
}
|
|
8806
9003
|
//#endregion
|
|
8807
|
-
//#region src/runtime/live-cost-format.ts
|
|
8808
|
-
/**
|
|
8809
|
-
* Compact one-line cost summary. Format:
|
|
8810
|
-
* "$0.1234 · 4 turns · 42 in / 6,511 out · 2.0M cached · sonnet"
|
|
8811
|
-
* Returns null when no cost data is available (mock runs / SDK errors).
|
|
8812
|
-
*
|
|
8813
|
-
* `compact: false` (default for CLI logs) keeps raw numbers and adds a
|
|
8814
|
-
* `model=...` segment. `compact: true` (HTML chip) thousand-separates fresh
|
|
8815
|
-
* tokens, abbreviates cache-read with K/M, drops the `model=` prefix.
|
|
8816
|
-
*/
|
|
8817
|
-
function formatLiveCost(cost, options) {
|
|
8818
|
-
if (cost.totalCostUsd === null) return null;
|
|
8819
|
-
const compact = options.compact;
|
|
8820
|
-
const sep = compact ? " · " : " / ";
|
|
8821
|
-
const parts = [`$${cost.totalCostUsd.toFixed(4)}`];
|
|
8822
|
-
if (cost.numTurns !== null) parts.push(`${cost.numTurns} turns`);
|
|
8823
|
-
if (cost.inputTokens !== null || cost.outputTokens !== null) {
|
|
8824
|
-
const i = cost.inputTokens ?? 0;
|
|
8825
|
-
const o = cost.outputTokens ?? 0;
|
|
8826
|
-
parts.push(compact ? `${formatNumber(i)} in / ${formatNumber(o)} out` : `${i}+${o} tokens`);
|
|
8827
|
-
}
|
|
8828
|
-
if (cost.cacheReadInputTokens !== null && cost.cacheReadInputTokens > 0) parts.push(compact ? `${formatTokenK(cost.cacheReadInputTokens)} cached` : `${cost.cacheReadInputTokens} cache-read`);
|
|
8829
|
-
if (!compact && cost.models.length > 0) parts.push(`model=${cost.models.join(",")}`);
|
|
8830
|
-
return parts.join(sep);
|
|
8831
|
-
}
|
|
8832
|
-
/**
|
|
8833
|
-
* Sum of per-spec costs for a batch. Used only by the CLI batch summary.
|
|
8834
|
-
* Returns null when no spec has cost data.
|
|
8835
|
-
*/
|
|
8836
|
-
function formatLiveBatchCost(costs) {
|
|
8837
|
-
let totalUsd = 0;
|
|
8838
|
-
let seen = false;
|
|
8839
|
-
let totalIn = 0;
|
|
8840
|
-
let totalOut = 0;
|
|
8841
|
-
let totalCacheRead = 0;
|
|
8842
|
-
for (const c of costs) {
|
|
8843
|
-
if (c.totalCostUsd !== null) {
|
|
8844
|
-
totalUsd += c.totalCostUsd;
|
|
8845
|
-
seen = true;
|
|
8846
|
-
}
|
|
8847
|
-
totalIn += c.inputTokens ?? 0;
|
|
8848
|
-
totalOut += c.outputTokens ?? 0;
|
|
8849
|
-
totalCacheRead += c.cacheReadInputTokens ?? 0;
|
|
8850
|
-
}
|
|
8851
|
-
if (!seen) return null;
|
|
8852
|
-
const parts = [`$${totalUsd.toFixed(4)}`, `${totalIn}+${totalOut} tokens`];
|
|
8853
|
-
if (totalCacheRead > 0) parts.push(`${totalCacheRead} cache-read`);
|
|
8854
|
-
return parts.join(" / ");
|
|
8855
|
-
}
|
|
8856
|
-
/** Thousand-separated count for token figures. */
|
|
8857
|
-
function formatNumber(n) {
|
|
8858
|
-
return n.toLocaleString("en-US");
|
|
8859
|
-
}
|
|
8860
|
-
/** Compact token count: 9,043,456 → "9.0M", 12000 → "12K", small → plain. */
|
|
8861
|
-
function formatTokenK(n) {
|
|
8862
|
-
if (n >= 1e6) return `${(n / 1e6).toFixed(1)}M`;
|
|
8863
|
-
if (n >= 1e4) return `${Math.round(n / 1e3)}K`;
|
|
8864
|
-
if (n >= 1e3) return `${(n / 1e3).toFixed(1)}K`;
|
|
8865
|
-
return n.toLocaleString("en-US");
|
|
8866
|
-
}
|
|
8867
|
-
//#endregion
|
|
8868
9004
|
//#region src/claude/agent-browser-invoke.ts
|
|
8869
9005
|
function agentBrowserInvokeBase(input) {
|
|
8870
9006
|
return {
|
|
@@ -8883,14 +9019,9 @@ function agentBrowserInvokeBase(input) {
|
|
|
8883
9019
|
}
|
|
8884
9020
|
//#endregion
|
|
8885
9021
|
//#region src/prompts/live.ts
|
|
8886
|
-
/**
|
|
8887
|
-
* Unique agent-browser session name. The runId is millisecond-precision wall
|
|
8888
|
-
* clock, so under `--concurrency > 1` two specs can start in the same
|
|
8889
|
-
* millisecond and collide; a random suffix guarantees each spec gets its own
|
|
8890
|
-
* Chrome session and state never bleeds across parallel runs.
|
|
8891
|
-
*/
|
|
9022
|
+
/** Unique agent-browser session name, so parallel specs never share a Chrome. */
|
|
8892
9023
|
function generateLiveSessionName() {
|
|
8893
|
-
return `ccqa-live-${buildRunId()}
|
|
9024
|
+
return `ccqa-live-${buildRunId()}`;
|
|
8894
9025
|
}
|
|
8895
9026
|
/**
|
|
8896
9027
|
* Static prefix of the `ccqa run` (live spec) system prompt. Built once per
|
|
@@ -9179,16 +9310,7 @@ async function runLiveExecutor(input) {
|
|
|
9179
9310
|
});
|
|
9180
9311
|
isError = result.isError;
|
|
9181
9312
|
errorDetail = result.errorDetail;
|
|
9182
|
-
cost =
|
|
9183
|
-
totalCostUsd: result.cost.totalCostUsd,
|
|
9184
|
-
durationApiMs: result.cost.durationApiMs,
|
|
9185
|
-
numTurns: result.cost.numTurns,
|
|
9186
|
-
inputTokens: result.cost.inputTokens,
|
|
9187
|
-
cacheCreationInputTokens: result.cost.cacheCreationInputTokens,
|
|
9188
|
-
cacheReadInputTokens: result.cost.cacheReadInputTokens,
|
|
9189
|
-
outputTokens: result.cost.outputTokens,
|
|
9190
|
-
models: result.cost.models
|
|
9191
|
-
};
|
|
9313
|
+
cost = toReportCost(result.cost);
|
|
9192
9314
|
} catch (err) {
|
|
9193
9315
|
isError = true;
|
|
9194
9316
|
errorDetail = err instanceof Error ? err.message : String(err);
|
|
@@ -9576,7 +9698,6 @@ async function runLiveSpecs(specs, opts) {
|
|
|
9576
9698
|
const failedCount = runs.filter((r) => r.kind === "error" || r.kind === "run" && r.result.status === "failed").length;
|
|
9577
9699
|
blank();
|
|
9578
9700
|
meta("live-summary", `${runs.length - failedCount} passed / ${failedCount} failed`);
|
|
9579
|
-
logBatchCost(runs);
|
|
9580
9701
|
return {
|
|
9581
9702
|
failedCount,
|
|
9582
9703
|
reportResults: built.flatMap((b) => b.row ? [b.row] : [])
|
|
@@ -9779,10 +9900,6 @@ async function runOneSpec(args) {
|
|
|
9779
9900
|
await closeSession(sessionName);
|
|
9780
9901
|
}
|
|
9781
9902
|
}
|
|
9782
|
-
function logBatchCost(runs) {
|
|
9783
|
-
const line = formatLiveBatchCost(runs.flatMap((r) => r.kind === "run" ? [r.result.cost] : []));
|
|
9784
|
-
if (line) meta("total-cost", line);
|
|
9785
|
-
}
|
|
9786
9903
|
/**
|
|
9787
9904
|
* Classify one failed live run via `analyzeFailure` — same prompt as the
|
|
9788
9905
|
* deterministic path (Issue #47), fed the live transcript instead of the
|
|
@@ -11660,13 +11777,14 @@ async function runExternalSpecs(dispatch, ctx) {
|
|
|
11660
11777
|
}
|
|
11661
11778
|
//#endregion
|
|
11662
11779
|
//#region src/run/incremental-report.ts
|
|
11663
|
-
function createIncrementalReport(reportDir, envelope, sink) {
|
|
11780
|
+
function createIncrementalReport(reportDir, envelope, sink, costNow) {
|
|
11664
11781
|
const byKey = /* @__PURE__ */ new Map();
|
|
11665
11782
|
const reportPath = join(reportDir, "report.json");
|
|
11666
11783
|
let queue = Promise.resolve();
|
|
11667
11784
|
const key = (r) => `${r.feature}/${r.spec}`;
|
|
11668
11785
|
const buildData = () => ({
|
|
11669
11786
|
...envelope,
|
|
11787
|
+
...costNow ? { cost: costNow() } : {},
|
|
11670
11788
|
results: [...byKey.values()]
|
|
11671
11789
|
});
|
|
11672
11790
|
const doFlush = async () => {
|
|
@@ -12414,7 +12532,8 @@ async function executeRun(targets, opts) {
|
|
|
12414
12532
|
const evidence = await readRowFilesBase64(row, reportDir);
|
|
12415
12533
|
await hubCtx.hub.patchRun(runId, {
|
|
12416
12534
|
rows: [row],
|
|
12417
|
-
evidence
|
|
12535
|
+
evidence,
|
|
12536
|
+
reportMeta: { cost: currentReportCost() }
|
|
12418
12537
|
});
|
|
12419
12538
|
hubPatchEverSucceeded = true;
|
|
12420
12539
|
} catch (err) {
|
|
@@ -12433,7 +12552,7 @@ async function executeRun(targets, opts) {
|
|
|
12433
12552
|
triageUserPromptHash,
|
|
12434
12553
|
deployedSha,
|
|
12435
12554
|
opts
|
|
12436
|
-
}), hubSink);
|
|
12555
|
+
}), hubSink, currentReportCost);
|
|
12437
12556
|
let completedNormally = false;
|
|
12438
12557
|
opts.teardown?.onFinalize(async () => {
|
|
12439
12558
|
if (completedNormally) return;
|
|
@@ -12442,7 +12561,8 @@ async function executeRun(targets, opts) {
|
|
|
12442
12561
|
await hubCtx.hub.patchRun(hubRunId, {
|
|
12443
12562
|
rows: incrementalReport.rows(),
|
|
12444
12563
|
done: true,
|
|
12445
|
-
finalStatus: "failed"
|
|
12564
|
+
finalStatus: "failed",
|
|
12565
|
+
reportMeta: { cost: currentReportCost() }
|
|
12446
12566
|
});
|
|
12447
12567
|
} catch (err) {
|
|
12448
12568
|
warn(`hub: could not finalize interrupted run ${hubRunId}: ${errMessage(err)}`);
|
|
@@ -12803,7 +12923,8 @@ function buildReportEnvelope(args) {
|
|
|
12803
12923
|
promptVersion: "13",
|
|
12804
12924
|
customPromptVersion,
|
|
12805
12925
|
...triageUserPromptHash !== null ? { triageUserPromptHash } : {},
|
|
12806
|
-
...deployedSha !== null ? { deployedSha } : {}
|
|
12926
|
+
...deployedSha !== null ? { deployedSha } : {},
|
|
12927
|
+
cost: currentReportCost()
|
|
12807
12928
|
};
|
|
12808
12929
|
}
|
|
12809
12930
|
/** Write the unified JSON (+ optional GitHub-annotation) report for one run. Returns the report data. */
|
|
@@ -13100,23 +13221,24 @@ function headerTarget(targets, opts) {
|
|
|
13100
13221
|
async function runCliAction(targets, opts) {
|
|
13101
13222
|
header("run", headerTarget(targets, opts));
|
|
13102
13223
|
const cwd = resolveCwd(opts.cwd);
|
|
13103
|
-
const
|
|
13104
|
-
|
|
13105
|
-
|
|
13106
|
-
|
|
13107
|
-
|
|
13108
|
-
|
|
13109
|
-
|
|
13110
|
-
|
|
13111
|
-
|
|
13112
|
-
|
|
13113
|
-
|
|
13114
|
-
|
|
13115
|
-
|
|
13116
|
-
|
|
13117
|
-
|
|
13118
|
-
|
|
13119
|
-
|
|
13224
|
+
const exitCode = await withCostReporting("run", async () => {
|
|
13225
|
+
const teardown = createRunTeardown();
|
|
13226
|
+
const disposeSignalHandlers = installTeardownSignalHandlers(teardown);
|
|
13227
|
+
try {
|
|
13228
|
+
return (await executeRun(targets, {
|
|
13229
|
+
...opts,
|
|
13230
|
+
cwd,
|
|
13231
|
+
teardown
|
|
13232
|
+
})).exitCode;
|
|
13233
|
+
} catch (err) {
|
|
13234
|
+
if (!(err instanceof RunUsageError)) throw err;
|
|
13235
|
+
error(err.message);
|
|
13236
|
+
return err.exitCode;
|
|
13237
|
+
} finally {
|
|
13238
|
+
await teardown.run();
|
|
13239
|
+
disposeSignalHandlers();
|
|
13240
|
+
}
|
|
13241
|
+
});
|
|
13120
13242
|
process.exit(exitCode);
|
|
13121
13243
|
}
|
|
13122
13244
|
//#endregion
|
|
@@ -14688,6 +14810,9 @@ async function confirmOverwrite(path) {
|
|
|
14688
14810
|
}
|
|
14689
14811
|
}
|
|
14690
14812
|
const generateCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("generate").argument("<feature/spec>", "Spec id in '<feature>/<spec>' form (resolves to .ccqa/features/<feature>/test-cases/<spec>/)").description("Generate test code from a spec via its target plugin. Recording-backed targets compile the existing ir.json (run `ccqa record` first); spec-input targets generate directly from the spec.").optionsGroup("How to generate:").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--target <id>", "Generate through this target instead of the spec's own — e.g. emit a Playwright spec from an agent-browser recording. The spec's `target:` stays the default for `ccqa run`.").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--overwrite", "Replace previously generated test code without warning").optionsGroup("Learning:").option("--learn-hub-codegen-prompt", "After generation, ask Claude to refresh the target's \"<target>.agent\" learning prompt on the hub from a summary of the run. LLM-generating targets (playwright, runn) only; requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
14813
|
+
await withCostReporting("generate", () => runGenerateCli(specPath, opts));
|
|
14814
|
+
}));
|
|
14815
|
+
async function runGenerateCli(specPath, opts) {
|
|
14691
14816
|
const { featureName, specName } = parseSpecPath(specPath);
|
|
14692
14817
|
const language = opts.language ?? "auto";
|
|
14693
14818
|
const cwd = resolveCwd(opts.cwd);
|
|
@@ -14738,21 +14863,6 @@ const generateCommand = addHubOptions(addProfileOption(addLanguageOption(new Com
|
|
|
14738
14863
|
}
|
|
14739
14864
|
throw e;
|
|
14740
14865
|
}
|
|
14741
|
-
}));
|
|
14742
|
-
//#endregion
|
|
14743
|
-
//#region src/cli/cost-line.ts
|
|
14744
|
-
/**
|
|
14745
|
-
* Write what this command spent on Claude to stderr.
|
|
14746
|
-
*
|
|
14747
|
-
* stderr rather than stdout because `drift --format json` and `select-specs`
|
|
14748
|
-
* put machine-readable output on stdout, and a cost line mixed into it breaks
|
|
14749
|
-
* the consumer. Cost is diagnostics about the command, not its output.
|
|
14750
|
-
*/
|
|
14751
|
-
function reportCost() {
|
|
14752
|
-
const cost = readCostTally();
|
|
14753
|
-
if (cost === null) return;
|
|
14754
|
-
const summary = formatLiveCost(cost, { compact: false });
|
|
14755
|
-
if (summary) process.stderr.write(`[cost] ${summary}\n`);
|
|
14756
14866
|
}
|
|
14757
14867
|
//#endregion
|
|
14758
14868
|
//#region src/cli/record.ts
|
|
@@ -14761,13 +14871,7 @@ const recordCommand = addHubOptions(addProfileOption(addLanguageOption(new Comma
|
|
|
14761
14871
|
if (VALIDATION_MODES.includes(raw)) return raw;
|
|
14762
14872
|
throw new Error(`--trace-validation must be one of ${VALIDATION_MODES.join(" | ")}`);
|
|
14763
14873
|
}, "lenient").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--trace-only", "Stop after the trace step; do not generate test code").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--overwrite", "Replace an existing test.spec.ts without warning").optionsGroup("Learning:").option("--learn-hub-trace-prompt", "After the trace finishes, ask Claude to refresh the \"record.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
14764
|
-
await
|
|
14765
|
-
try {
|
|
14766
|
-
await runRecord(specPath, opts);
|
|
14767
|
-
} finally {
|
|
14768
|
-
reportCost();
|
|
14769
|
-
}
|
|
14770
|
-
});
|
|
14874
|
+
await withCostReporting("record", () => runRecord(specPath, opts));
|
|
14771
14875
|
}));
|
|
14772
14876
|
async function runRecord(specPath, opts) {
|
|
14773
14877
|
const { featureName, specName } = parseSpecPath(specPath);
|
|
@@ -15451,6 +15555,7 @@ function driftResultsToReport(results, meta) {
|
|
|
15451
15555
|
promptVersion: meta.promptVersion ?? DRIFT_REPORT_PROMPT_VERSION,
|
|
15452
15556
|
customPromptVersion: meta.customPromptVersion ?? null,
|
|
15453
15557
|
...meta.triageUserPromptHash ? { triageUserPromptHash: meta.triageUserPromptHash } : {},
|
|
15558
|
+
cost: currentReportCost(),
|
|
15454
15559
|
results: specResults
|
|
15455
15560
|
};
|
|
15456
15561
|
}
|
|
@@ -15640,7 +15745,7 @@ function selectSpecsNeedingAudit(targets, report) {
|
|
|
15640
15745
|
//#region src/cli/audit.ts
|
|
15641
15746
|
const DEFAULT_CONCURRENCY = 3;
|
|
15642
15747
|
const auditCommand = addProfileOption(addLanguageOption(new Command("audit").argument("[feature/spec]", "Optional spec id. If omitted, every spec under .ccqa/features/ is checked.").description("Read each spec against the code it describes and report where the two have drifted. Static: no browser is run, so this is the cheap check to put in front of `ccqa run`.").optionsGroup("Which specs to audit:").option("--only-affected-by <ref>", "Only specs `ccqa select-specs` judges reached by the diff against <ref> (e.g. origin/main). In pull_request CI, pass $GITHUB_BASE_REF. Costs one model call; specs it cannot decide are audited rather than skipped.").option("--only-hub-audit-needed", "Only specs the hub says a deploy has landed on since the audit last read them. A spec that was never audited is always included, and one the hub cannot answer for is audited rather than skipped. No git diff involved. Requires a hub connection and --hub-profile.").optionsGroup("How to run it:").option("--concurrency <n>", `Parallel spec checks (default: ${DEFAULT_CONCURRENCY})`).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").optionsGroup("What to do with the results:").option("--report-format <fmt>", "Output format: text | json | github", "text").option("--report-to-hub", "Push the result to a ccqa hub as a run (kind: drift), which is what updates the drift ledger. A spec it finds drifted answers `needsRepair` to `ccqa run --only-hub-rerun-needed`, and is not run until a person repairs it.").option("--exit-on <level>", "Exit non-zero on this severity or higher: warn | error", "error").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory used as both the .ccqa root and the codebase Claude reads. Useful for monorepos. Defaults to process.cwd().").option("--project <name>", "Logical project name for the pushed run. Defaults to the current directory's name.").option(...hubUrlOption).option(...hubTokenOption).option(...hubHeaderOption))).action(withUsageErrors(async (specPath, opts) => {
|
|
15643
|
-
await
|
|
15748
|
+
await withCostReporting("audit", () => runAudit(specPath, opts));
|
|
15644
15749
|
}));
|
|
15645
15750
|
async function runAudit(specPath, opts) {
|
|
15646
15751
|
const format = parseFormat$1(opts.reportFormat);
|
|
@@ -15745,7 +15850,6 @@ async function runAudit(specPath, opts) {
|
|
|
15745
15850
|
baseRef,
|
|
15746
15851
|
promptCtx
|
|
15747
15852
|
});
|
|
15748
|
-
reportCost();
|
|
15749
15853
|
process.exit(determineExitCode(results, threshold));
|
|
15750
15854
|
}
|
|
15751
15855
|
/** Longer than the audit's own timeout, so a sweep cannot outlive its claim. */
|
|
@@ -15896,7 +16000,6 @@ function asHubReadError(err) {
|
|
|
15896
16000
|
throw new RunUsageError(`could not read from the hub: ${errMessage(err)}`);
|
|
15897
16001
|
}
|
|
15898
16002
|
function exitWithNoSpecs(format, reason, message) {
|
|
15899
|
-
reportCost();
|
|
15900
16003
|
if (format === "json") process.stdout.write(`${JSON.stringify({
|
|
15901
16004
|
specs: [],
|
|
15902
16005
|
skipped: reason
|
|
@@ -16039,8 +16142,7 @@ For each test case above, write a 1–2 sentence factual \`summary\` of what it
|
|
|
16039
16142
|
//#endregion
|
|
16040
16143
|
//#region src/cli/perspectives.ts
|
|
16041
16144
|
const perspectivesCommand = addHubOptions(addLanguageOption(new Command("perspectives").description("Generate/update the project's perspectives document on the hub — a factual inventory of existing test coverage (no severity, no gap analysis)").option("--instruction <text>", "Hint to steer how summaries are written").option("-y, --yes", "Apply without asking [y/N]", false).option("--verify", "Check the hub document against the local specs (mechanical fields only) and exit 1 when it is stale. No Claude calls — cheap enough for CI.", false).option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID").option("--project <name>", "Hub project to store the document under (default: cwd directory name)"))).action(withHubErrors(async (opts) => {
|
|
16042
|
-
|
|
16043
|
-
else await runPerspectives(opts);
|
|
16145
|
+
await withCostReporting("perspectives", () => opts.verify ? runPerspectivesCheck(opts) : runPerspectives(opts));
|
|
16044
16146
|
}));
|
|
16045
16147
|
/**
|
|
16046
16148
|
* `--check`: compare the hub document against a freshly-built local skeleton
|
|
@@ -16415,7 +16517,7 @@ function parseSummaries(json) {
|
|
|
16415
16517
|
//#region src/cli/select-specs.ts
|
|
16416
16518
|
const DEFAULT_HEAD = "HEAD";
|
|
16417
16519
|
const selectSpecsCommand = new Command("select-specs").description("Decide which specs a range of commits reaches. Reads the diff and the spec inventory and returns one verdict per spec: needed | notNeeded | unknown.").requiredOption("--base <ref>", "Commit the range starts at — typically what is currently deployed, or the previous commit on the branch.").option("--head <ref>", `Commit the range ends at (default: ${DEFAULT_HEAD})`).option("--cwd <path>", "Working directory used as both the .ccqa root and the codebase Claude reads. Changes outside it are reported but never attributed to a spec. Defaults to process.cwd().").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--format <fmt>", "Output format: text | json", "text").action(async (opts) => {
|
|
16418
|
-
await
|
|
16520
|
+
await withCostReporting("select-specs", () => runSelectSpecs(opts));
|
|
16419
16521
|
});
|
|
16420
16522
|
async function runSelectSpecs(opts) {
|
|
16421
16523
|
const format = parseFormat(opts.format);
|
|
@@ -16463,7 +16565,6 @@ async function runSelectSpecs(opts) {
|
|
|
16463
16565
|
...opts.model ? { model: opts.model } : {}
|
|
16464
16566
|
});
|
|
16465
16567
|
process.stdout.write(format === "json" ? `${JSON.stringify(report, null, 2)}\n` : renderText(report));
|
|
16466
|
-
reportCost();
|
|
16467
16568
|
process.exit(0);
|
|
16468
16569
|
}
|
|
16469
16570
|
const VERDICT_ORDER = [
|
|
@@ -16670,12 +16771,17 @@ function toLedger(raw) {
|
|
|
16670
16771
|
}
|
|
16671
16772
|
/** Fold `from` into `into` in place, per bucket and key, newest `at` winning. */
|
|
16672
16773
|
function mergeLedgerInto(into, from) {
|
|
16673
|
-
|
|
16674
|
-
|
|
16675
|
-
|
|
16676
|
-
}
|
|
16774
|
+
mergeBucket(into.green, from.green);
|
|
16775
|
+
mergeBucket(into.run, from.run);
|
|
16776
|
+
mergeBucket(into.red, from.red);
|
|
16677
16777
|
return into;
|
|
16678
16778
|
}
|
|
16779
|
+
function mergeBucket(into, from) {
|
|
16780
|
+
for (const [key, entry] of Object.entries(from)) {
|
|
16781
|
+
const prev = into[key];
|
|
16782
|
+
if (!prev || prev.at <= entry.at) into[key] = entry;
|
|
16783
|
+
}
|
|
16784
|
+
}
|
|
16679
16785
|
//#endregion
|
|
16680
16786
|
//#region src/hub/api/validate.ts
|
|
16681
16787
|
/**
|
|
@@ -16755,6 +16861,7 @@ function createPushRunHandler(config) {
|
|
|
16755
16861
|
},
|
|
16756
16862
|
gitHead: report.git.head,
|
|
16757
16863
|
promptVersion: report.promptVersion,
|
|
16864
|
+
costUsd: report.cost?.totalCostUsd ?? null,
|
|
16758
16865
|
ciRunId: report.runId,
|
|
16759
16866
|
runUrl: report.runUrl ?? null,
|
|
16760
16867
|
reportCreatedAt: report.createdAt,
|
|
@@ -16805,6 +16912,7 @@ function createOpenRunHandler(config) {
|
|
|
16805
16912
|
},
|
|
16806
16913
|
gitHead: gitHead || null,
|
|
16807
16914
|
promptVersion: "",
|
|
16915
|
+
costUsd: null,
|
|
16808
16916
|
ciRunId: ciRunId || null,
|
|
16809
16917
|
runUrl: runUrl || null,
|
|
16810
16918
|
reportCreatedAt: now,
|
|
@@ -16828,7 +16936,8 @@ const PatchRunRequestSchema = z.object({
|
|
|
16828
16936
|
promptVersion: z.string().optional(),
|
|
16829
16937
|
customPromptVersion: z.string().nullable().optional(),
|
|
16830
16938
|
runUrl: z.string().nullable().optional(),
|
|
16831
|
-
triageUserPromptHash: z.string().optional()
|
|
16939
|
+
triageUserPromptHash: z.string().optional(),
|
|
16940
|
+
cost: ReportCostSchema.nullable().optional()
|
|
16832
16941
|
}).partial().optional()
|
|
16833
16942
|
});
|
|
16834
16943
|
/** Insert or replace `rows` into `results`, upserting by feature/spec identity. */
|
|
@@ -16883,7 +16992,7 @@ async function updateSpecLedger(storage, run, results) {
|
|
|
16883
16992
|
const key = `${row.feature}/${row.spec}`;
|
|
16884
16993
|
ledger.run[key] = entry;
|
|
16885
16994
|
if (row.status === "passed") ledger.green[key] = entry;
|
|
16886
|
-
else ledger.red[key] = entry;
|
|
16995
|
+
else ledger.red[key] = redEntry(entry, row);
|
|
16887
16996
|
}
|
|
16888
16997
|
if (Object.keys(ledger.run).length === 0) return;
|
|
16889
16998
|
try {
|
|
@@ -16893,6 +17002,23 @@ async function updateSpecLedger(storage, run, results) {
|
|
|
16893
17002
|
}
|
|
16894
17003
|
}
|
|
16895
17004
|
/**
|
|
17005
|
+
* The red bucket's entry: the coordinate every bucket carries, plus what the
|
|
17006
|
+
* failure analysis concluded, so a reader of the ledger learns why a spec is
|
|
17007
|
+
* red without fetching its report.
|
|
17008
|
+
*
|
|
17009
|
+
* A field with nothing behind it is left out rather than written empty:
|
|
17010
|
+
* `analysis` is null when the run did not ask for one (`--on-fail-explain` is
|
|
17011
|
+
* opt-in), and a model that answered with no headline leaves `headline` "".
|
|
17012
|
+
*/
|
|
17013
|
+
function redEntry(entry, row) {
|
|
17014
|
+
if (!row.analysis) return entry;
|
|
17015
|
+
return {
|
|
17016
|
+
...entry,
|
|
17017
|
+
label: row.analysis.label,
|
|
17018
|
+
...row.analysis.headline ? { headline: row.analysis.headline } : {}
|
|
17019
|
+
};
|
|
17020
|
+
}
|
|
17021
|
+
/**
|
|
16896
17022
|
* Advance the drift ledger from a terminal `kind: "drift"` run: each row's
|
|
16897
17023
|
* `analysis` (the diagnosis `driftResultsToReport` put there) becomes that
|
|
16898
17024
|
* spec's newest audit entry. No profile — drift asks whether the spec still
|
|
@@ -16988,6 +17114,7 @@ function createPatchRunHandler(config) {
|
|
|
16988
17114
|
const { rows, evidence, done, finalStatus, reportMeta } = await readJsonBody(ctx.req, maxPushBytes, PatchRunRequestSchema, "request body");
|
|
16989
17115
|
let specs = run.specs;
|
|
16990
17116
|
let mergedResults = [];
|
|
17117
|
+
let costUsd = run.costUsd ?? null;
|
|
16991
17118
|
await config.storage.artifacts.updateJsonFile(id, "report.json", (current) => {
|
|
16992
17119
|
const base = current ?? {
|
|
16993
17120
|
schemaVersion: 1,
|
|
@@ -17001,7 +17128,8 @@ function createPatchRunHandler(config) {
|
|
|
17001
17128
|
model: null,
|
|
17002
17129
|
language: null,
|
|
17003
17130
|
promptVersion: "",
|
|
17004
|
-
customPromptVersion: null
|
|
17131
|
+
customPromptVersion: null,
|
|
17132
|
+
cost: null
|
|
17005
17133
|
};
|
|
17006
17134
|
const envelope = {
|
|
17007
17135
|
...base,
|
|
@@ -17016,11 +17144,13 @@ function createPatchRunHandler(config) {
|
|
|
17016
17144
|
...reportMeta?.promptVersion !== void 0 ? { promptVersion: reportMeta.promptVersion } : {},
|
|
17017
17145
|
...reportMeta?.customPromptVersion !== void 0 ? { customPromptVersion: reportMeta.customPromptVersion } : {},
|
|
17018
17146
|
...reportMeta?.runUrl !== void 0 ? { runUrl: reportMeta.runUrl } : {},
|
|
17019
|
-
...reportMeta?.triageUserPromptHash !== void 0 ? { triageUserPromptHash: reportMeta.triageUserPromptHash } : {}
|
|
17147
|
+
...reportMeta?.triageUserPromptHash !== void 0 ? { triageUserPromptHash: reportMeta.triageUserPromptHash } : {},
|
|
17148
|
+
...reportMeta?.cost !== void 0 ? { cost: reportMeta.cost } : {}
|
|
17020
17149
|
};
|
|
17021
17150
|
const merged = mergeResults(current?.results ?? [], rows);
|
|
17022
17151
|
specs = countSpecs(merged);
|
|
17023
17152
|
mergedResults = merged;
|
|
17153
|
+
costUsd = envelope.cost?.totalCostUsd ?? null;
|
|
17024
17154
|
return {
|
|
17025
17155
|
...envelope,
|
|
17026
17156
|
results: merged
|
|
@@ -17030,11 +17160,15 @@ function createPatchRunHandler(config) {
|
|
|
17030
17160
|
const patch = done ? {
|
|
17031
17161
|
status: finalStatus ?? (specs.failed > 0 ? "failed" : "passed"),
|
|
17032
17162
|
specs,
|
|
17163
|
+
costUsd,
|
|
17033
17164
|
...run.kind === "drift" ? { drift: summarizeDrift(mergedResults) } : {},
|
|
17034
17165
|
...reportMeta?.git?.head ? { gitHead: reportMeta.git.head } : {},
|
|
17035
17166
|
...reportMeta?.promptVersion ? { promptVersion: reportMeta.promptVersion } : {},
|
|
17036
17167
|
...await deployHeadMovedDuringRun(config.storage, run) ? { deployedShaAmbiguous: true } : {}
|
|
17037
|
-
} : {
|
|
17168
|
+
} : {
|
|
17169
|
+
specs,
|
|
17170
|
+
costUsd
|
|
17171
|
+
};
|
|
17038
17172
|
const updated = await config.storage.runs.update(id, patch);
|
|
17039
17173
|
if (done) {
|
|
17040
17174
|
await updateSpecLedger(config.storage, updated, mergedResults);
|
|
@@ -18585,6 +18719,7 @@ const HTML_BODY = `
|
|
|
18585
18719
|
<section id="view-runs">
|
|
18586
18720
|
<div class="page-bar">
|
|
18587
18721
|
<h1 data-i18n="runs.title">Runs</h1>
|
|
18722
|
+
<span class="total" id="runs-total-cost" hidden></span>
|
|
18588
18723
|
<div class="spacer"></div>
|
|
18589
18724
|
${refreshButton("runs-refresh")}
|
|
18590
18725
|
</div>
|
|
@@ -18592,7 +18727,7 @@ const HTML_BODY = `
|
|
|
18592
18727
|
<div class="card" id="runs-card">
|
|
18593
18728
|
<div class="table-wrap">
|
|
18594
18729
|
<table>
|
|
18595
|
-
<thead><tr><th data-i18n="runs.col.run">Run</th><th data-i18n="runs.col.branch">Branch</th><th data-i18n="meta.profile">Profile</th><th data-i18n="runs.col.status">Status</th><th data-i18n="runs.col.specs">Specs</th><th data-i18n="runs.col.created">Created</th></tr></thead>
|
|
18730
|
+
<thead><tr><th data-i18n="runs.col.run">Run</th><th data-i18n="runs.col.branch">Branch</th><th data-i18n="meta.profile">Profile</th><th data-i18n="runs.col.status">Status</th><th data-i18n="runs.col.specs">Specs</th><th data-i18n="runs.col.cost">Cost</th><th data-i18n="runs.col.created">Created</th></tr></thead>
|
|
18596
18731
|
<tbody id="runs-tbody"></tbody>
|
|
18597
18732
|
</table>
|
|
18598
18733
|
</div>
|
|
@@ -18948,6 +19083,7 @@ const CSS = `
|
|
|
18948
19083
|
.page-bar .back:hover { color: var(--fg); }
|
|
18949
19084
|
.page-bar .back svg { width: 15px; height: 15px; }
|
|
18950
19085
|
.page-bar .filters { display: flex; gap: 8px; margin-left: 8px; }
|
|
19086
|
+
.page-bar .total { font-size: 12px; color: var(--muted); font-variant-numeric: tabular-nums; white-space: nowrap; }
|
|
18951
19087
|
.page-bar .spacer { flex: 1; }
|
|
18952
19088
|
.content { padding: 18px 24px 48px; }
|
|
18953
19089
|
|
|
@@ -19489,11 +19625,12 @@ const CLIENT_JS = `
|
|
|
19489
19625
|
"runs.title": "Runs", "runs.empty": "Select a project to see its runs.",
|
|
19490
19626
|
"runs.none": "No runs yet for this project.", "projects.none": "No projects yet. Create one to get started.", "projects.noneShort": "No projects yet",
|
|
19491
19627
|
"runs.col.run": "Run", "runs.col.branch": "Branch", "runs.col.status": "Status",
|
|
19492
|
-
"runs.col.specs": "Specs", "runs.col.created": "Created",
|
|
19628
|
+
"runs.col.specs": "Specs", "runs.col.cost": "Cost", "runs.col.created": "Created",
|
|
19629
|
+
"runs.totalCost": "Cost of these {n}:",
|
|
19493
19630
|
"detail.back": "Runs", "detail.specs": "Specs",
|
|
19494
19631
|
"detail.download": "Download artifacts",
|
|
19495
19632
|
"detail.triage": "Triage",
|
|
19496
|
-
"meta.branch": "Branch", "meta.specs": "Specs",
|
|
19633
|
+
"meta.branch": "Branch", "meta.specs": "Specs", "meta.cost": "Cost",
|
|
19497
19634
|
"meta.created": "Created", "meta.passed": "passed", "meta.profile": "Profile",
|
|
19498
19635
|
"meta.drift": "Drift",
|
|
19499
19636
|
"diag.cause": "Cause", "diag.fix": "Fix",
|
|
@@ -19646,11 +19783,12 @@ const CLIENT_JS = `
|
|
|
19646
19783
|
"runs.title": "実行", "runs.empty": "プロジェクトを選択すると実行一覧が表示されます。",
|
|
19647
19784
|
"runs.none": "このプロジェクトにはまだ実行がありません。", "projects.none": "まだプロジェクトがありません。作成して始めましょう。", "projects.noneShort": "プロジェクトなし",
|
|
19648
19785
|
"runs.col.run": "実行", "runs.col.branch": "ブランチ", "runs.col.status": "ステータス",
|
|
19649
|
-
"runs.col.specs": "スペック", "runs.col.created": "作成",
|
|
19786
|
+
"runs.col.specs": "スペック", "runs.col.cost": "コスト", "runs.col.created": "作成",
|
|
19787
|
+
"runs.totalCost": "この {n} 件のコスト:",
|
|
19650
19788
|
"detail.back": "実行", "detail.specs": "スペック",
|
|
19651
19789
|
"detail.download": "アーティファクトをダウンロード",
|
|
19652
19790
|
"detail.triage": "トリアージ",
|
|
19653
|
-
"meta.branch": "ブランチ", "meta.specs": "スペック",
|
|
19791
|
+
"meta.branch": "ブランチ", "meta.specs": "スペック", "meta.cost": "コスト",
|
|
19654
19792
|
"meta.created": "作成", "meta.passed": "合格", "meta.profile": "プロファイル",
|
|
19655
19793
|
"meta.drift": "ドリフト",
|
|
19656
19794
|
"diag.cause": "原因", "diag.fix": "対処",
|
|
@@ -20042,6 +20180,11 @@ const CLIENT_JS = `
|
|
|
20042
20180
|
return run.kind === "drift" ? driftFoundBadge(driftRunState(run), "drift.run.") : statusBadge(run.status);
|
|
20043
20181
|
}
|
|
20044
20182
|
|
|
20183
|
+
// A run's Claude spend, in the same $x.xxxx form as the per-step badge.
|
|
20184
|
+
// A run that billed nothing, and one stored before costs were recorded, both
|
|
20185
|
+
// arrive as a non-number — printing $0.0000 would claim a measured zero.
|
|
20186
|
+
function costText(usd) { return typeof usd === "number" ? "$" + usd.toFixed(4) : "—"; }
|
|
20187
|
+
|
|
20045
20188
|
// One chip per non-zero drift label, worded via labelText — the same
|
|
20046
20189
|
// vocabulary the diagnosis card uses — so a run's summary never disagrees
|
|
20047
20190
|
// with its own spec cards. A zero-count label is omitted, same reasoning as
|
|
@@ -20260,9 +20403,38 @@ const CLIENT_JS = `
|
|
|
20260
20403
|
|
|
20261
20404
|
// ── runs list ────────────────────────────────────────────────────────
|
|
20262
20405
|
|
|
20406
|
+
// What the listed runs cost together — the accumulating number an operator
|
|
20407
|
+
// reads to decide how often CI should run, so it follows whatever filter
|
|
20408
|
+
// produced the list. Hidden when no listed run carries a cost at all, since
|
|
20409
|
+
// a "$0.0000" total would read as "CI is free" rather than "nothing measured".
|
|
20410
|
+
//
|
|
20411
|
+
// The label names the run count on purpose. The list is capped (limit=50), so
|
|
20412
|
+
// an unqualified "total" would quietly under-report a project's spend the
|
|
20413
|
+
// moment it has more runs than that — the one number this feature exists to
|
|
20414
|
+
// get right.
|
|
20415
|
+
function renderRunsTotalCost(runs) {
|
|
20416
|
+
var span = document.getElementById("runs-total-cost");
|
|
20417
|
+
// null until the first measured run, which is what distinguishes "nothing
|
|
20418
|
+
// was measured" from "the measured total happens to be zero".
|
|
20419
|
+
var total = null;
|
|
20420
|
+
// The runs that actually contributed, not the runs on screen. Runs stored
|
|
20421
|
+
// before costs were recorded carry nothing, so counting the list would
|
|
20422
|
+
// credit the sum to rows that gave it nothing — the exact misreading the
|
|
20423
|
+
// count is here to prevent.
|
|
20424
|
+
var measured = 0;
|
|
20425
|
+
runs.forEach(function (r) {
|
|
20426
|
+
if (typeof r.costUsd === "number") { total = (total || 0) + r.costUsd; measured++; }
|
|
20427
|
+
});
|
|
20428
|
+
span.hidden = total === null;
|
|
20429
|
+
if (total !== null) {
|
|
20430
|
+
span.textContent = t("runs.totalCost").replace("{n}", measured) + " " + costText(total);
|
|
20431
|
+
}
|
|
20432
|
+
}
|
|
20433
|
+
|
|
20263
20434
|
function renderRunsList(runs) {
|
|
20264
20435
|
var tbody = document.getElementById("runs-tbody");
|
|
20265
20436
|
clear(tbody);
|
|
20437
|
+
renderRunsTotalCost(runs);
|
|
20266
20438
|
var empty = document.getElementById("runs-empty");
|
|
20267
20439
|
if (runs.length === 0) {
|
|
20268
20440
|
empty.hidden = false;
|
|
@@ -20325,6 +20497,7 @@ const CLIENT_JS = `
|
|
|
20325
20497
|
specsCell.appendChild(specsWrap);
|
|
20326
20498
|
tr.appendChild(specsCell);
|
|
20327
20499
|
|
|
20500
|
+
tr.appendChild(el("td", "muted num", costText(r.costUsd)));
|
|
20328
20501
|
tr.appendChild(el("td", "muted num", relTime(r.createdAt)));
|
|
20329
20502
|
tbody.appendChild(tr);
|
|
20330
20503
|
});
|
|
@@ -20337,6 +20510,7 @@ const CLIENT_JS = `
|
|
|
20337
20510
|
.then(function (data) { renderRunsList(data.runs); })
|
|
20338
20511
|
.catch(function (err) {
|
|
20339
20512
|
clear(document.getElementById("runs-tbody"));
|
|
20513
|
+
renderRunsTotalCost([]);
|
|
20340
20514
|
empty.hidden = false;
|
|
20341
20515
|
empty.textContent = "Error loading runs: " + err.message;
|
|
20342
20516
|
});
|
|
@@ -20396,6 +20570,9 @@ const CLIENT_JS = `
|
|
|
20396
20570
|
} else {
|
|
20397
20571
|
metaItem(t("meta.specs"), run.specs.passed + " / " + run.specs.total + " " + t("meta.passed"));
|
|
20398
20572
|
}
|
|
20573
|
+
// Everything this run spent on Claude — live browsing, triage, the audit a
|
|
20574
|
+
// failure triggers, spec selection — not the sum of the per-step badges.
|
|
20575
|
+
metaItem(t("meta.cost"), costText(run.costUsd));
|
|
20399
20576
|
metaItem(t("meta.created"), relTime(run.createdAt));
|
|
20400
20577
|
head.appendChild(meta);
|
|
20401
20578
|
|
|
@@ -20679,7 +20856,7 @@ const CLIENT_JS = `
|
|
|
20679
20856
|
steps.forEach(function (s, i) {
|
|
20680
20857
|
var built = stepCard(s.status, "#" + (i + 1), s.instruction, s.expected, s.reasoning);
|
|
20681
20858
|
if (s.cost && s.cost.totalCostUsd != null) {
|
|
20682
|
-
built.head.appendChild(el("span", "cost",
|
|
20859
|
+
built.head.appendChild(el("span", "cost", costText(s.cost.totalCostUsd)));
|
|
20683
20860
|
}
|
|
20684
20861
|
if (s.beforePng || s.afterPng) {
|
|
20685
20862
|
var frames = el("div", "step-frames");
|
|
@@ -22066,6 +22243,17 @@ const CLIENT_JS = `
|
|
|
22066
22243
|
badge.appendChild(document.createTextNode(" " + t("perspectives.run.state." + runState)));
|
|
22067
22244
|
td.appendChild(badge);
|
|
22068
22245
|
|
|
22246
|
+
// What the failure was, as the run's analysis called it — the same place
|
|
22247
|
+
// the audit column names its drift label. Absent on a red entry the run
|
|
22248
|
+
// never analyzed, and on entries written before the ledger carried it.
|
|
22249
|
+
if (runState === "failed" && rr.lastRed && rr.lastRed.label) {
|
|
22250
|
+
var cause = el("span", "cellsub", labelText(rr.lastRed.label));
|
|
22251
|
+
// The one-line conclusion, for a pointer. The detail panel shows it in
|
|
22252
|
+
// full, so nothing here depends on finding it.
|
|
22253
|
+
if (rr.lastRed.headline) cause.title = rr.lastRed.headline;
|
|
22254
|
+
td.appendChild(cause);
|
|
22255
|
+
}
|
|
22256
|
+
|
|
22069
22257
|
// The sub-line is the coordinate of the result being reported — the same
|
|
22070
22258
|
// "when is this from" evidence the audit column carries above, so the two
|
|
22071
22259
|
// axes read as siblings. Why the currency matters belongs to the verdict
|
|
@@ -22284,6 +22472,21 @@ const CLIENT_JS = `
|
|
|
22284
22472
|
return wrap;
|
|
22285
22473
|
}
|
|
22286
22474
|
|
|
22475
|
+
// The failure: which run it was, then what the analysis concluded. The
|
|
22476
|
+
// headline is model output, already localized server-side, so it is shown as
|
|
22477
|
+
// written. A run made without failure analysis carries neither field, and the
|
|
22478
|
+
// row is then the coordinate alone — what it has always been.
|
|
22479
|
+
function rerunFailureValue(entry) {
|
|
22480
|
+
var wrap = el("div");
|
|
22481
|
+
wrap.appendChild(ledgerLine(entry));
|
|
22482
|
+
if (entry.label) {
|
|
22483
|
+
var line = el("div", "d-prose", labelText(entry.label));
|
|
22484
|
+
if (entry.headline) line.appendChild(document.createTextNode(" · " + entry.headline));
|
|
22485
|
+
wrap.appendChild(line);
|
|
22486
|
+
}
|
|
22487
|
+
return wrap;
|
|
22488
|
+
}
|
|
22489
|
+
|
|
22287
22490
|
// Detail row: a definition list of the case's fields plus the note editor.
|
|
22288
22491
|
// Built with createElement/textContent throughout — every field here is
|
|
22289
22492
|
// API-derived, so none of it may go through innerHTML.
|
|
@@ -22317,7 +22520,7 @@ const CLIENT_JS = `
|
|
|
22317
22520
|
var rr = ledgerEntryFor(perspState.rerun, feature, spec);
|
|
22318
22521
|
if (rr) {
|
|
22319
22522
|
row(rerunEvidenceLabelKey(rr), rerunEvidenceValue(rr));
|
|
22320
|
-
if (rerunHasFailure(rr)) row("perspectives.d.lastRed",
|
|
22523
|
+
if (rerunHasFailure(rr)) row("perspectives.d.lastRed", rerunFailureValue(rr.lastRed));
|
|
22321
22524
|
}
|
|
22322
22525
|
frag.appendChild(dl);
|
|
22323
22526
|
|
|
@@ -59,6 +59,7 @@ declare const RunSchema: z.ZodObject<{
|
|
|
59
59
|
}, z.core.$strip>;
|
|
60
60
|
gitHead: z.ZodNullable<z.ZodString>;
|
|
61
61
|
promptVersion: z.ZodString;
|
|
62
|
+
costUsd: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
|
|
62
63
|
ciRunId: z.ZodNullable<z.ZodString>;
|
|
63
64
|
runUrl: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
64
65
|
reportCreatedAt: z.ZodString;
|
|
@@ -313,6 +314,14 @@ declare const RerunReportSchema: z.ZodObject<{
|
|
|
313
314
|
at: z.ZodString;
|
|
314
315
|
deployedSha: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
315
316
|
deployedShaAmbiguous: z.ZodOptional<z.ZodBoolean>;
|
|
317
|
+
label: z.ZodOptional<z.ZodEnum<{
|
|
318
|
+
TEST_DRIFT: "TEST_DRIFT";
|
|
319
|
+
SPEC_CHANGE: "SPEC_CHANGE";
|
|
320
|
+
PRODUCT_BUG: "PRODUCT_BUG";
|
|
321
|
+
ENVIRONMENT: "ENVIRONMENT";
|
|
322
|
+
UNKNOWN: "UNKNOWN";
|
|
323
|
+
}>>;
|
|
324
|
+
headline: z.ZodOptional<z.ZodString>;
|
|
316
325
|
}, z.core.$strip>>;
|
|
317
326
|
touchedBy: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
318
327
|
touchedByDeploy: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
@@ -575,6 +584,16 @@ declare const RunReportDataSchema: z.ZodObject<{
|
|
|
575
584
|
customPromptVersion: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
576
585
|
triageUserPromptHash: z.ZodOptional<z.ZodString>;
|
|
577
586
|
deployedSha: z.ZodOptional<z.ZodString>;
|
|
587
|
+
cost: z.ZodDefault<z.ZodNullable<z.ZodObject<{
|
|
588
|
+
totalCostUsd: z.ZodNullable<z.ZodNumber>;
|
|
589
|
+
durationApiMs: z.ZodNullable<z.ZodNumber>;
|
|
590
|
+
numTurns: z.ZodNullable<z.ZodNumber>;
|
|
591
|
+
inputTokens: z.ZodNullable<z.ZodNumber>;
|
|
592
|
+
cacheCreationInputTokens: z.ZodNullable<z.ZodNumber>;
|
|
593
|
+
cacheReadInputTokens: z.ZodNullable<z.ZodNumber>;
|
|
594
|
+
outputTokens: z.ZodNullable<z.ZodNumber>;
|
|
595
|
+
models: z.ZodArray<z.ZodString>;
|
|
596
|
+
}, z.core.$strip>>>;
|
|
578
597
|
results: z.ZodArray<z.ZodObject<{
|
|
579
598
|
feature: z.ZodString;
|
|
580
599
|
spec: z.ZodString;
|
package/dist/package.json
CHANGED