ccqa 1.27.0 → 1.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/ccqa.mjs +221 -119
- package/dist/package.json +1 -1
- package/package.json +2 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -237,17 +237,11 @@ const DEFAULT_SPEC_MODE = "deterministic";
|
|
|
237
237
|
* `required: false` makes it optional. `secret: true` flags the value as
|
|
238
238
|
* sensitive — codegen renders such values as `process.env.<NAME> ?? ""`
|
|
239
239
|
* template literals so the secret never ends up baked into test.spec.ts.
|
|
240
|
-
* `dummy` is a placeholder value surfaced by the draft / drift prompts
|
|
241
|
-
* (which see the block in isolation, before any include site exists);
|
|
242
|
-
* `description` is the param's semantic role, also consumed by those
|
|
243
|
-
* prompts and by spec authors browsing the block.
|
|
244
240
|
*/
|
|
245
241
|
const BlockParamSchema = z.object({
|
|
246
242
|
name: z.string().min(1),
|
|
247
243
|
required: z.boolean().optional(),
|
|
248
|
-
secret: z.boolean().optional()
|
|
249
|
-
dummy: z.string().optional(),
|
|
250
|
-
description: z.string().optional()
|
|
244
|
+
secret: z.boolean().optional()
|
|
251
245
|
}).strict();
|
|
252
246
|
/**
|
|
253
247
|
* Block schema. Block steps are restricted to ActionStep — nested blocks are
|
|
@@ -270,15 +264,25 @@ function isParamRequired(param) {
|
|
|
270
264
|
}
|
|
271
265
|
//#endregion
|
|
272
266
|
//#region src/spec/parser.ts
|
|
273
|
-
/**
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
267
|
+
/** The spec/block root (an `unrecognized_keys` issue there has an empty path). */
|
|
268
|
+
const atRoot = (path) => path.length === 0;
|
|
269
|
+
/** A block param entry — the issue path is `params.<index>`. */
|
|
270
|
+
const atBlockParam = (path) => path.length === 2 && path[0] === "params" && typeof path[1] === "number";
|
|
271
|
+
const UNREAD_PARAM_FIELD = "nothing reads it (a block param reaches the prompts as its name, required and secret only). Delete the line.";
|
|
272
|
+
const REMOVED_FIELDS = {
|
|
273
|
+
relatedPaths: {
|
|
274
|
+
at: atRoot,
|
|
275
|
+
message: "which specs a change affects is now decided by `ccqa select-specs`, which reads the diff instead of a declared path list. Delete the field."
|
|
276
|
+
},
|
|
277
|
+
dummy: {
|
|
278
|
+
at: atBlockParam,
|
|
279
|
+
message: UNREAD_PARAM_FIELD
|
|
280
|
+
},
|
|
281
|
+
description: {
|
|
282
|
+
at: atBlockParam,
|
|
283
|
+
message: UNREAD_PARAM_FIELD
|
|
284
|
+
}
|
|
285
|
+
};
|
|
282
286
|
/** Parse a spec.yaml. Schema rejections are rewritten with actionable messages. */
|
|
283
287
|
function parseTestSpec(content, source = "spec.yaml") {
|
|
284
288
|
const raw = parseYamlOrThrow(content, source);
|
|
@@ -336,9 +340,9 @@ function humanizeIssue(issue, isBlock) {
|
|
|
336
340
|
if (issue.code === "unrecognized_keys") {
|
|
337
341
|
const keys = Array.isArray(issue.keys) ? issue.keys : [];
|
|
338
342
|
if (isBlock && keys.includes("include")) return `Nested blocks are not supported — flatten by inlining the included block's steps into this block.`;
|
|
339
|
-
const removed = keys.filter((k) => k
|
|
340
|
-
const stillUnknown = keys.filter((k) => !(
|
|
341
|
-
const parts = removed.map((k) => `\`${k}\` is no longer part of the spec schema — ${REMOVED_FIELDS[k]}`);
|
|
343
|
+
const removed = keys.filter((k) => REMOVED_FIELDS[k]?.at(issue.path));
|
|
344
|
+
const stillUnknown = keys.filter((k) => !REMOVED_FIELDS[k]?.at(issue.path));
|
|
345
|
+
const parts = removed.map((k) => `\`${k}\` is no longer part of the spec schema — ${REMOVED_FIELDS[k].message}`);
|
|
342
346
|
if (stillUnknown.length > 0) parts.push(`Unknown keys: ${stillUnknown.join(", ")}`);
|
|
343
347
|
return parts.join(" ");
|
|
344
348
|
}
|
|
@@ -512,8 +516,8 @@ async function loadAllBlocks(cwd) {
|
|
|
512
516
|
* Co-located with `loadAllBlocks` so callers don't have to remember the
|
|
513
517
|
* isParamRequired / secret-default mapping.
|
|
514
518
|
*/
|
|
515
|
-
|
|
516
|
-
return [...
|
|
519
|
+
function projectAvailableBlocks(blocks) {
|
|
520
|
+
return [...blocks.entries()].map(([name, block]) => ({
|
|
517
521
|
name,
|
|
518
522
|
title: block.title,
|
|
519
523
|
params: (block.params ?? []).map((p) => ({
|
|
@@ -523,6 +527,10 @@ async function loadAvailableBlocks(cwd) {
|
|
|
523
527
|
}))
|
|
524
528
|
}));
|
|
525
529
|
}
|
|
530
|
+
/** `loadAllBlocks` + `projectAvailableBlocks`, for callers that need only the projection. */
|
|
531
|
+
async function loadAvailableBlocks(cwd) {
|
|
532
|
+
return projectAvailableBlocks(await loadAllBlocks(cwd));
|
|
533
|
+
}
|
|
526
534
|
const USER_PROMPT_MAX_BYTES = 32768;
|
|
527
535
|
/**
|
|
528
536
|
* Load the prompt bundle from the hub for one guidance kind ("record" /
|
|
@@ -1298,6 +1306,95 @@ function opt(key, value) {
|
|
|
1298
1306
|
return value ? { [key]: value } : {};
|
|
1299
1307
|
}
|
|
1300
1308
|
//#endregion
|
|
1309
|
+
//#region src/runtime/env-scrub.ts
|
|
1310
|
+
/**
|
|
1311
|
+
* Build a list of `[envValue, "${VAR}"]` pairs for every `${VAR}` reference
|
|
1312
|
+
* mentioned in the spec OR in any of its expanded (block-inlined) steps.
|
|
1313
|
+
* Used at trace time to scrub recorded Claude-text outputs so a value the
|
|
1314
|
+
* spec author intentionally threaded through `process.env` is preserved as
|
|
1315
|
+
* `${VAR}` in `ir.json` rather than baked in as the concrete
|
|
1316
|
+
* trace-time value.
|
|
1317
|
+
*
|
|
1318
|
+
* Why we walk `spec.steps` AND `expanded`:
|
|
1319
|
+
* - `spec.steps` carries the spec's own `instruction` / `expected` + each
|
|
1320
|
+
* include's raw `params` (which may themselves be `${ENV}` refs).
|
|
1321
|
+
* - `expanded` carries the inlined block-internal steps, whose
|
|
1322
|
+
* `instruction` / `expected` may *also* contain `${ENV}` refs that
|
|
1323
|
+
* don't go through include params.
|
|
1324
|
+
*
|
|
1325
|
+
* Only refs whose env value is currently non-empty land in the map —
|
|
1326
|
+
* scrubbing against an empty string would corrupt unrelated empty strings
|
|
1327
|
+
* in the action stream. Names whose env is unset are returned via
|
|
1328
|
+
* `unresolved` so the caller can warn the user.
|
|
1329
|
+
*
|
|
1330
|
+
* Longer values sort first so a `${SHORT}` whose value is a substring of a
|
|
1331
|
+
* `${LONG}` value doesn't clobber the longer one.
|
|
1332
|
+
*
|
|
1333
|
+
* `title` is deliberately NOT scanned — it never reaches the recorded action
|
|
1334
|
+
* stream.
|
|
1335
|
+
*/
|
|
1336
|
+
function buildSpecEnvScrub(spec, expanded) {
|
|
1337
|
+
const refNames = /* @__PURE__ */ new Set();
|
|
1338
|
+
for (const step of spec.steps) if (isIncludeStep(step)) for (const v of Object.values(step.params ?? {})) collect(v, refNames);
|
|
1339
|
+
else {
|
|
1340
|
+
collect(step.instruction, refNames);
|
|
1341
|
+
collect(step.expected, refNames);
|
|
1342
|
+
}
|
|
1343
|
+
for (const step of expanded) {
|
|
1344
|
+
collect(step.instruction, refNames);
|
|
1345
|
+
collect(step.expected, refNames);
|
|
1346
|
+
}
|
|
1347
|
+
const map = [];
|
|
1348
|
+
const unresolved = [];
|
|
1349
|
+
for (const name of refNames) {
|
|
1350
|
+
const value = process.env[name];
|
|
1351
|
+
if (typeof value === "string" && value.length > 0) map.push([value, "${" + name + "}"]);
|
|
1352
|
+
else unresolved.push(name);
|
|
1353
|
+
}
|
|
1354
|
+
map.sort((a, b) => b[0].length - a[0].length);
|
|
1355
|
+
return {
|
|
1356
|
+
map,
|
|
1357
|
+
unresolved
|
|
1358
|
+
};
|
|
1359
|
+
}
|
|
1360
|
+
function collect(value, into) {
|
|
1361
|
+
for (const name of iterEnvRefNames(value)) into.add(name);
|
|
1362
|
+
}
|
|
1363
|
+
/** Shorter than this, a value is no secret and matches inside ordinary words. */
|
|
1364
|
+
const MIN_PROSE_SCRUB_LENGTH = 4;
|
|
1365
|
+
/** Long enough to clear the length bar, still ordinary prose / JSON. */
|
|
1366
|
+
const COMMON_PROSE_VALUES = new Set([
|
|
1367
|
+
"true",
|
|
1368
|
+
"false",
|
|
1369
|
+
"null",
|
|
1370
|
+
"none",
|
|
1371
|
+
"undefined"
|
|
1372
|
+
]);
|
|
1373
|
+
/**
|
|
1374
|
+
* Scrub map for model output, built like {@link buildSpecEnvScrub} but
|
|
1375
|
+
* without the values that read as ordinary text (`"1"`, `"true"`): prose
|
|
1376
|
+
* runs to paragraphs, where replacing every occurrence of such a value
|
|
1377
|
+
* costs more meaning than it protects. Record's own scrub keeps them for
|
|
1378
|
+
* its single command lines; the live path reuses this one map for its Bash
|
|
1379
|
+
* command log too, trading that short-value coverage for not building a
|
|
1380
|
+
* second map.
|
|
1381
|
+
*/
|
|
1382
|
+
function buildProseEnvScrubMap(spec, expanded) {
|
|
1383
|
+
return buildSpecEnvScrub(spec, expanded).map.filter(([value]) => value.length >= MIN_PROSE_SCRUB_LENGTH && !COMMON_PROSE_VALUES.has(value.toLowerCase()));
|
|
1384
|
+
}
|
|
1385
|
+
/**
|
|
1386
|
+
* Replace every occurrence of an env value with its `${VAR}` placeholder in
|
|
1387
|
+
* `text`. **Caller invariant**: the map must be sorted longest-value-first
|
|
1388
|
+
* so a shorter value doesn't shadow a longer one that contains it as a
|
|
1389
|
+
* substring. `buildSpecEnvScrub` upholds this; hand-built maps should too.
|
|
1390
|
+
*/
|
|
1391
|
+
function scrubEnvValues(text, scrubMap) {
|
|
1392
|
+
if (scrubMap.length === 0) return text;
|
|
1393
|
+
let out = text;
|
|
1394
|
+
for (const [value, placeholder] of scrubMap) if (out.includes(value)) out = out.replaceAll(value, placeholder);
|
|
1395
|
+
return out;
|
|
1396
|
+
}
|
|
1397
|
+
//#endregion
|
|
1301
1398
|
//#region src/claude/native-binary.ts
|
|
1302
1399
|
const require$1 = createRequire(import.meta.url);
|
|
1303
1400
|
/**
|
|
@@ -1457,7 +1554,7 @@ function warnOnceIfNativeBinaryMissing() {
|
|
|
1457
1554
|
if (missing) warn(missingNativeBinaryMessage(missing));
|
|
1458
1555
|
}
|
|
1459
1556
|
async function invokeClaudeStreaming(options, onEvent) {
|
|
1460
|
-
const { prompt, systemPrompt, allowedTools, disableBuiltinTools = false, disableThinking = false, mcpServers, maxTurns, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, relaxAbConstraints = false } = options;
|
|
1557
|
+
const { prompt, systemPrompt, allowedTools, disableBuiltinTools = false, disableThinking = false, mcpServers, maxTurns, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, envScrubMap = [], relaxAbConstraints = false } = options;
|
|
1461
1558
|
const resolvedModel = resolveModel(model);
|
|
1462
1559
|
const hasEndpointEnv = Object.keys(resolveEndpointEnv()).length > 0;
|
|
1463
1560
|
const mergedEnv = env || hasEndpointEnv ? withoutEmptyEndpointVars({
|
|
@@ -1561,7 +1658,7 @@ async function invokeClaudeStreaming(options, onEvent) {
|
|
|
1561
1658
|
if (msg.type === "assistant" && !silenceBashLog) {
|
|
1562
1659
|
for (const block of msg.message.content ?? []) if (block.type === "tool_use" && block.name === "Bash") {
|
|
1563
1660
|
const cmd = block.input?.["command"];
|
|
1564
|
-
if (typeof cmd === "string") bash(cmd);
|
|
1661
|
+
if (typeof cmd === "string") bash(scrubEnvValues(cmd, envScrubMap));
|
|
1565
1662
|
}
|
|
1566
1663
|
}
|
|
1567
1664
|
if (msg.type === "result") {
|
|
@@ -3486,6 +3583,36 @@ function buildDiffMcpServer(getFileDiff) {
|
|
|
3486
3583
|
* with confidence 0 rather than throwing — the report must always render.
|
|
3487
3584
|
*/
|
|
3488
3585
|
async function analyzeFailure(input, options) {
|
|
3586
|
+
return scrubOutcome(await classifyFailure(input, options), options.envScrubMap ?? []);
|
|
3587
|
+
}
|
|
3588
|
+
/**
|
|
3589
|
+
* Mask the profile values the classifier may have quoted: its Read/Grep reach
|
|
3590
|
+
* the repository, so a local `.env` is in reach of its prose even when the
|
|
3591
|
+
* evidence it was handed is clean. A literal match — a value the model
|
|
3592
|
+
* paraphrases still gets through. Only what the classifier authored is
|
|
3593
|
+
* covered: the row's other evidence (`failureLogExcerpt`, `diffExcerpt`)
|
|
3594
|
+
* passes to the report and the hub as its producer wrote it.
|
|
3595
|
+
*/
|
|
3596
|
+
function scrubOutcome(outcome, scrubMap) {
|
|
3597
|
+
if (scrubMap.length === 0) return outcome;
|
|
3598
|
+
const scrub = (text) => scrubEnvValues(text, scrubMap);
|
|
3599
|
+
const { analysis } = outcome;
|
|
3600
|
+
return {
|
|
3601
|
+
...outcome,
|
|
3602
|
+
raw: scrub(outcome.raw),
|
|
3603
|
+
analysis: {
|
|
3604
|
+
...analysis,
|
|
3605
|
+
headline: scrub(analysis.headline),
|
|
3606
|
+
recommendation: scrub(analysis.recommendation),
|
|
3607
|
+
reasoning: scrub(analysis.reasoning),
|
|
3608
|
+
evidence: analysis.evidence.map((item) => ({
|
|
3609
|
+
...item,
|
|
3610
|
+
detail: scrub(item.detail)
|
|
3611
|
+
}))
|
|
3612
|
+
}
|
|
3613
|
+
};
|
|
3614
|
+
}
|
|
3615
|
+
async function classifyFailure(input, options) {
|
|
3489
3616
|
const { result: raw, isError } = await invokeClaudeStreaming({
|
|
3490
3617
|
prompt: buildFailureAnalysisPrompt(input),
|
|
3491
3618
|
allowedTools: [
|
|
@@ -4460,6 +4587,44 @@ function withHubErrors(fn) {
|
|
|
4460
4587
|
};
|
|
4461
4588
|
}
|
|
4462
4589
|
//#endregion
|
|
4590
|
+
//#region src/cli/repo-local-profiles.ts
|
|
4591
|
+
/** Where a named profile's variables lived before profiles moved to the hub. */
|
|
4592
|
+
const PROFILES_DIR = ".ccqa/profiles";
|
|
4593
|
+
/**
|
|
4594
|
+
* Flag `.ccqa/profiles/<name>.env` files the move to hub-stored profiles left
|
|
4595
|
+
* behind. Warns, never fails: the run resolved its variables from the right
|
|
4596
|
+
* place, so the file's existence is the only thing wrong. A tracked one is
|
|
4597
|
+
* called out separately — that is a committed credential, not just dead weight.
|
|
4598
|
+
*/
|
|
4599
|
+
async function warnRepoLocalProfiles(cwd) {
|
|
4600
|
+
const paths = (await readdir(join(cwd, PROFILES_DIR)).catch(() => [])).filter((name) => name.endsWith(".env")).sort().map((name) => `${PROFILES_DIR}/${name}`);
|
|
4601
|
+
if (paths.length === 0) return;
|
|
4602
|
+
warn(`ccqa does not read repo-local profile files — the values in ${paths.join(", ")} are not in effect for this run. Profile variables come from the hub: register them with \`ccqa hub var set --profile <name>\`, then delete the ${noun(paths.length)}.`);
|
|
4603
|
+
const tracked = await trackedPaths(paths, cwd);
|
|
4604
|
+
if (tracked.length === 0) return;
|
|
4605
|
+
warn(`tracked by git: ${tracked.join(", ")} — a profile file holds credentials, so whatever is in there is committed. Rotate those values; deleting the ${noun(tracked.length)} now does not un-commit them.`);
|
|
4606
|
+
}
|
|
4607
|
+
function noun(n) {
|
|
4608
|
+
return n === 1 ? "file" : "files";
|
|
4609
|
+
}
|
|
4610
|
+
/**
|
|
4611
|
+
* The subset git reports as tracked. Outside a repository the question has no
|
|
4612
|
+
* answer, so stay silent rather than accuse or reassure on a guess.
|
|
4613
|
+
*/
|
|
4614
|
+
async function trackedPaths(paths, cwd) {
|
|
4615
|
+
try {
|
|
4616
|
+
const { stdout } = await execFileP("git", [
|
|
4617
|
+
"ls-files",
|
|
4618
|
+
"-z",
|
|
4619
|
+
"--",
|
|
4620
|
+
...paths
|
|
4621
|
+
], { cwd });
|
|
4622
|
+
return stdout.split("\0").filter((p) => p !== "");
|
|
4623
|
+
} catch {
|
|
4624
|
+
return [];
|
|
4625
|
+
}
|
|
4626
|
+
}
|
|
4627
|
+
//#endregion
|
|
4463
4628
|
//#region src/cli/options.ts
|
|
4464
4629
|
/**
|
|
4465
4630
|
* Shared `--language` flag. Every Claude-driven command writes some
|
|
@@ -4511,6 +4676,7 @@ async function applyProfileFromOption(opts) {
|
|
|
4511
4676
|
* rather than skipping it.
|
|
4512
4677
|
*/
|
|
4513
4678
|
async function resolveProfileEnv(opts) {
|
|
4679
|
+
await warnRepoLocalProfiles(opts.cwd);
|
|
4514
4680
|
if (opts.profile !== void 0) await applyNamedProfile(opts.profile, opts.project, opts.cwd, opts);
|
|
4515
4681
|
else await applyDefaultEnv(opts.cwd);
|
|
4516
4682
|
}
|
|
@@ -6225,6 +6391,7 @@ function createFailureAnalysisPass(deps) {
|
|
|
6225
6391
|
};
|
|
6226
6392
|
const baselineMissing = specDiffResult.ok ? null : specDiffResult.skip;
|
|
6227
6393
|
info(`failure analysis: ${featureName}/${specName}${baselineMissing ? " (no baseline — classifying from current source)" : ""}`);
|
|
6394
|
+
const envScrubMap = specEnvScrubMap(input.parsedSpec, deps.parsedBlocks);
|
|
6228
6395
|
const outcome = await analyzeFailure({
|
|
6229
6396
|
script: await input.readScript(),
|
|
6230
6397
|
hasGeneratedSurface: true,
|
|
@@ -6244,7 +6411,8 @@ function createFailureAnalysisPass(deps) {
|
|
|
6244
6411
|
}, {
|
|
6245
6412
|
...deps.model ? { model: deps.model } : {},
|
|
6246
6413
|
cwd: deps.cwd,
|
|
6247
|
-
getFileDiff: specDiff?.fileDiff ?? (() => null)
|
|
6414
|
+
getFileDiff: specDiff?.fileDiff ?? (() => null),
|
|
6415
|
+
envScrubMap
|
|
6248
6416
|
});
|
|
6249
6417
|
if (!printedHeader) {
|
|
6250
6418
|
printedHeader = true;
|
|
@@ -6258,6 +6426,20 @@ function createFailureAnalysisPass(deps) {
|
|
|
6258
6426
|
};
|
|
6259
6427
|
} };
|
|
6260
6428
|
}
|
|
6429
|
+
/**
|
|
6430
|
+
* The `${VAR}` values this spec resolved, for masking them out of the
|
|
6431
|
+
* classifier's prose. Read from `process.env` here rather than at run start
|
|
6432
|
+
* (where the live path builds its own): a profile is applied once per
|
|
6433
|
+
* invocation, so this is still what the spec ran against.
|
|
6434
|
+
*/
|
|
6435
|
+
function specEnvScrubMap(spec, blocks) {
|
|
6436
|
+
if (spec === null) return [];
|
|
6437
|
+
try {
|
|
6438
|
+
return buildProseEnvScrubMap(spec, expandSpec(spec, { blocks }));
|
|
6439
|
+
} catch {
|
|
6440
|
+
return buildProseEnvScrubMap(spec, []);
|
|
6441
|
+
}
|
|
6442
|
+
}
|
|
6261
6443
|
/** One classified spec's line in the failure-analysis block. */
|
|
6262
6444
|
function printAnalysis(featureName, specName, analysis) {
|
|
6263
6445
|
const pct = Math.round(analysis.confidence * 100);
|
|
@@ -6310,6 +6492,7 @@ async function analyzeExternalRows(rows, run) {
|
|
|
6310
6492
|
readScript: () => readGeneratedTestSources(ref, deps.cwd),
|
|
6311
6493
|
failureLog: row.failureLogExcerpt ?? "",
|
|
6312
6494
|
specYaml: row.specYaml,
|
|
6495
|
+
parsedSpec: tryParseTestSpec(row.specYaml),
|
|
6313
6496
|
target: row.target ?? "agent-browser",
|
|
6314
6497
|
artifactsDir: readableArtifactsDir(ref, deps)
|
|
6315
6498
|
});
|
|
@@ -9326,93 +9509,6 @@ function isStorageStateShape(state) {
|
|
|
9326
9509
|
return typeof state === "object" && state !== null && Array.isArray(state.cookies) && Array.isArray(state.origins);
|
|
9327
9510
|
}
|
|
9328
9511
|
//#endregion
|
|
9329
|
-
//#region src/runtime/env-scrub.ts
|
|
9330
|
-
/**
|
|
9331
|
-
* Build a list of `[envValue, "${VAR}"]` pairs for every `${VAR}` reference
|
|
9332
|
-
* mentioned in the spec OR in any of its expanded (block-inlined) steps.
|
|
9333
|
-
* Used at trace time to scrub recorded Claude-text outputs so a value the
|
|
9334
|
-
* spec author intentionally threaded through `process.env` is preserved as
|
|
9335
|
-
* `${VAR}` in `ir.json` rather than baked in as the concrete
|
|
9336
|
-
* trace-time value.
|
|
9337
|
-
*
|
|
9338
|
-
* Why we walk `spec.steps` AND `expanded`:
|
|
9339
|
-
* - `spec.steps` carries the spec's own `instruction` / `expected` + each
|
|
9340
|
-
* include's raw `params` (which may themselves be `${ENV}` refs).
|
|
9341
|
-
* - `expanded` carries the inlined block-internal steps, whose
|
|
9342
|
-
* `instruction` / `expected` may *also* contain `${ENV}` refs that
|
|
9343
|
-
* don't go through include params.
|
|
9344
|
-
*
|
|
9345
|
-
* Only refs whose env value is currently non-empty land in the map —
|
|
9346
|
-
* scrubbing against an empty string would corrupt unrelated empty strings
|
|
9347
|
-
* in the action stream. Names whose env is unset are returned via
|
|
9348
|
-
* `unresolved` so the caller can warn the user.
|
|
9349
|
-
*
|
|
9350
|
-
* Longer values sort first so a `${SHORT}` whose value is a substring of a
|
|
9351
|
-
* `${LONG}` value doesn't clobber the longer one.
|
|
9352
|
-
*
|
|
9353
|
-
* `title` is deliberately NOT scanned — it never reaches the recorded action
|
|
9354
|
-
* stream.
|
|
9355
|
-
*/
|
|
9356
|
-
function buildSpecEnvScrub(spec, expanded) {
|
|
9357
|
-
const refNames = /* @__PURE__ */ new Set();
|
|
9358
|
-
for (const step of spec.steps) if (isIncludeStep(step)) for (const v of Object.values(step.params ?? {})) collect(v, refNames);
|
|
9359
|
-
else {
|
|
9360
|
-
collect(step.instruction, refNames);
|
|
9361
|
-
collect(step.expected, refNames);
|
|
9362
|
-
}
|
|
9363
|
-
for (const step of expanded) {
|
|
9364
|
-
collect(step.instruction, refNames);
|
|
9365
|
-
collect(step.expected, refNames);
|
|
9366
|
-
}
|
|
9367
|
-
const map = [];
|
|
9368
|
-
const unresolved = [];
|
|
9369
|
-
for (const name of refNames) {
|
|
9370
|
-
const value = process.env[name];
|
|
9371
|
-
if (typeof value === "string" && value.length > 0) map.push([value, "${" + name + "}"]);
|
|
9372
|
-
else unresolved.push(name);
|
|
9373
|
-
}
|
|
9374
|
-
map.sort((a, b) => b[0].length - a[0].length);
|
|
9375
|
-
return {
|
|
9376
|
-
map,
|
|
9377
|
-
unresolved
|
|
9378
|
-
};
|
|
9379
|
-
}
|
|
9380
|
-
function collect(value, into) {
|
|
9381
|
-
for (const name of iterEnvRefNames(value)) into.add(name);
|
|
9382
|
-
}
|
|
9383
|
-
/** Shorter than this, a value is no secret and matches inside ordinary words. */
|
|
9384
|
-
const MIN_PROSE_SCRUB_LENGTH = 4;
|
|
9385
|
-
/** Long enough to clear the length bar, still ordinary prose / JSON. */
|
|
9386
|
-
const COMMON_PROSE_VALUES = new Set([
|
|
9387
|
-
"true",
|
|
9388
|
-
"false",
|
|
9389
|
-
"null",
|
|
9390
|
-
"none",
|
|
9391
|
-
"undefined"
|
|
9392
|
-
]);
|
|
9393
|
-
/**
|
|
9394
|
-
* Scrub map for a live run, built like {@link buildSpecEnvScrub} but without
|
|
9395
|
-
* the values that read as ordinary text (`"1"`, `"true"`). A live step records
|
|
9396
|
-
* paragraphs of model prose, so replacing every occurrence of such a value
|
|
9397
|
-
* would cost more meaning than it protects; record scrubs single command
|
|
9398
|
-
* lines, where the same trade favours keeping them.
|
|
9399
|
-
*/
|
|
9400
|
-
function buildLiveEnvScrubMap(spec, expanded) {
|
|
9401
|
-
return buildSpecEnvScrub(spec, expanded).map.filter(([value]) => value.length >= MIN_PROSE_SCRUB_LENGTH && !COMMON_PROSE_VALUES.has(value.toLowerCase()));
|
|
9402
|
-
}
|
|
9403
|
-
/**
|
|
9404
|
-
* Replace every occurrence of an env value with its `${VAR}` placeholder in
|
|
9405
|
-
* `text`. **Caller invariant**: the map must be sorted longest-value-first
|
|
9406
|
-
* so a shorter value doesn't shadow a longer one that contains it as a
|
|
9407
|
-
* substring. `buildSpecEnvScrub` upholds this; hand-built maps should too.
|
|
9408
|
-
*/
|
|
9409
|
-
function scrubEnvValues(text, scrubMap) {
|
|
9410
|
-
if (scrubMap.length === 0) return text;
|
|
9411
|
-
let out = text;
|
|
9412
|
-
for (const [value, placeholder] of scrubMap) if (out.includes(value)) out = out.replaceAll(value, placeholder);
|
|
9413
|
-
return out;
|
|
9414
|
-
}
|
|
9415
|
-
//#endregion
|
|
9416
9512
|
//#region src/claude/agent-browser-invoke.ts
|
|
9417
9513
|
function agentBrowserInvokeBase(input) {
|
|
9418
9514
|
return {
|
|
@@ -9710,6 +9806,7 @@ async function runLiveExecutor(input) {
|
|
|
9710
9806
|
prompt: userPrompt,
|
|
9711
9807
|
systemPrompt,
|
|
9712
9808
|
model: input.model,
|
|
9809
|
+
envScrubMap: input.envScrubMap,
|
|
9713
9810
|
relaxAbConstraints: true
|
|
9714
9811
|
}, (msg) => {
|
|
9715
9812
|
if (msg.type !== "assistant") return;
|
|
@@ -10258,7 +10355,7 @@ async function runOneSpec(args) {
|
|
|
10258
10355
|
}
|
|
10259
10356
|
const spec = parseTestSpec(specContent);
|
|
10260
10357
|
const expanded = expandSpec(spec, { blocks: await loadAllBlocks(cwd) });
|
|
10261
|
-
const envScrubMap =
|
|
10358
|
+
const envScrubMap = buildProseEnvScrubMap(spec, expanded);
|
|
10262
10359
|
meta("spec", spec.title);
|
|
10263
10360
|
meta("steps", expanded.length);
|
|
10264
10361
|
const includes = collectIncludedBlockNames(spec);
|
|
@@ -10320,6 +10417,7 @@ async function runOneSpec(args) {
|
|
|
10320
10417
|
specName,
|
|
10321
10418
|
runDir,
|
|
10322
10419
|
specYaml: specContent,
|
|
10420
|
+
envScrubMap,
|
|
10323
10421
|
result
|
|
10324
10422
|
};
|
|
10325
10423
|
} finally {
|
|
@@ -10378,7 +10476,8 @@ async function analyzeOneLiveFailure(r, diffProvider, auth, blocks, opts, cwd) {
|
|
|
10378
10476
|
}, {
|
|
10379
10477
|
...opts.model ? { model: opts.model } : {},
|
|
10380
10478
|
cwd,
|
|
10381
|
-
getFileDiff: specDiff?.fileDiff ?? (() => null)
|
|
10479
|
+
getFileDiff: specDiff?.fileDiff ?? (() => null),
|
|
10480
|
+
envScrubMap: r.envScrubMap
|
|
10382
10481
|
});
|
|
10383
10482
|
const pct = Math.round(outcome.analysis.confidence * 100);
|
|
10384
10483
|
const headline = outcome.analysis.headline.trim() || (outcome.analysis.reasoning.split("\n")[0] ?? "").trim();
|
|
@@ -12877,6 +12976,7 @@ async function executeRun(targets, opts) {
|
|
|
12877
12976
|
});
|
|
12878
12977
|
const triageUserPromptHash = triageUserPrompt ? hashTriageUserPrompt(triageUserPrompt) : null;
|
|
12879
12978
|
const reportDir = resolveReportDir(opts.reportDir, cwd);
|
|
12979
|
+
const parsedBlocks = await loadAllBlocks(cwd);
|
|
12880
12980
|
const analysisDeps = {
|
|
12881
12981
|
diffProvider,
|
|
12882
12982
|
auth: diffProvider ? driftAuthAvailable() : {
|
|
@@ -12885,7 +12985,8 @@ async function executeRun(targets, opts) {
|
|
|
12885
12985
|
},
|
|
12886
12986
|
cwd,
|
|
12887
12987
|
reportDir,
|
|
12888
|
-
blocks:
|
|
12988
|
+
blocks: projectAvailableBlocks(parsedBlocks),
|
|
12989
|
+
parsedBlocks,
|
|
12889
12990
|
...opts.model ? { model: opts.model } : {},
|
|
12890
12991
|
...opts.language ? { language: opts.language } : {},
|
|
12891
12992
|
customPrompt,
|
|
@@ -13334,14 +13435,13 @@ function failedSpec(s) {
|
|
|
13334
13435
|
* specs share). Degrades — never throws — when Claude auth or the git diff
|
|
13335
13436
|
* aren't available. Caller writes report.json.
|
|
13336
13437
|
*/
|
|
13337
|
-
async function analyzeDeterministicSummaries(summaries, cwd, reportDir, { pass }) {
|
|
13338
|
-
const allBlocks = await loadAllBlocks(cwd);
|
|
13438
|
+
async function analyzeDeterministicSummaries(summaries, cwd, reportDir, { pass, deps }) {
|
|
13339
13439
|
const results = [];
|
|
13340
13440
|
for (const s of summaries) {
|
|
13341
13441
|
const assertions = collectAssertions(s);
|
|
13342
13442
|
const specYaml = await tryReadSpecFile(s.featureName, s.specName, cwd);
|
|
13343
13443
|
const parsedSpec = tryParseTestSpec(specYaml);
|
|
13344
|
-
const stepDescriptions = buildStepDescriptions(parsedSpec,
|
|
13444
|
+
const stepDescriptions = buildStepDescriptions(parsedSpec, deps.parsedBlocks);
|
|
13345
13445
|
const evidence = await loadEvidenceForSpec(s.evidenceDir, reportDir, stepDescriptions);
|
|
13346
13446
|
const base = {
|
|
13347
13447
|
feature: s.featureName,
|
|
@@ -13377,6 +13477,7 @@ async function analyzeDeterministicSummaries(summaries, cwd, reportDir, { pass }
|
|
|
13377
13477
|
readScript: () => readScriptSafe(s.scriptFile),
|
|
13378
13478
|
failureLog,
|
|
13379
13479
|
specYaml,
|
|
13480
|
+
parsedSpec,
|
|
13380
13481
|
target: AGENT_BROWSER_TARGET
|
|
13381
13482
|
});
|
|
13382
13483
|
results.push({
|
|
@@ -14794,6 +14895,7 @@ async function runTrace(featureName, specName, model, validationMode = "lenient"
|
|
|
14794
14895
|
runId: sessionName
|
|
14795
14896
|
}),
|
|
14796
14897
|
model,
|
|
14898
|
+
envScrubMap,
|
|
14797
14899
|
onAbAction: ({ abAction, stepId, assertMarker }) => {
|
|
14798
14900
|
const stepForCommand = stepTracker.fromCommand(stepId);
|
|
14799
14901
|
const line = abAction === void 0 ? null : scrubEnvValues(abAction, envScrubMap);
|
|
@@ -24086,8 +24188,8 @@ const CLIENT_JS = `
|
|
|
24086
24188
|
}
|
|
24087
24189
|
|
|
24088
24190
|
// ── profile switching (per-tab dropdowns) ──────────────────────────────
|
|
24089
|
-
// Profiles scope variables + sessions (a profile is a set of env vars,
|
|
24090
|
-
//
|
|
24191
|
+
// Profiles scope variables + sessions (a profile is a set of env vars) and,
|
|
24192
|
+
// since ADR-0010, the needs-re-run verdict:
|
|
24091
24193
|
// two environments sit at different commits, so that question has no
|
|
24092
24194
|
// profile-free answer. Prompts are project-wide and runs are cross-profile,
|
|
24093
24195
|
// so there is still no header-level selector — Secrets and Perspectives each
|
package/dist/package.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ccqa",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.29.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Browser test recorder powered by Claude Code and agent-browser",
|
|
6
6
|
"repository": {
|
|
@@ -73,6 +73,7 @@
|
|
|
73
73
|
"test": "vitest run",
|
|
74
74
|
"test:unit": "vitest run src/",
|
|
75
75
|
"test:e2e": "vitest run tests/e2e",
|
|
76
|
+
"release:check": "node --experimental-strip-types src/release/check.ts",
|
|
76
77
|
"prepublishOnly": "pnpm typecheck && pnpm test && pnpm build"
|
|
77
78
|
},
|
|
78
79
|
"engines": {
|