ccqa 1.28.0 → 1.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/ccqa.mjs +201 -124
- package/dist/package.json +1 -1
- package/package.json +1 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -237,17 +237,11 @@ const DEFAULT_SPEC_MODE = "deterministic";
|
|
|
237
237
|
* `required: false` makes it optional. `secret: true` flags the value as
|
|
238
238
|
* sensitive — codegen renders such values as `process.env.<NAME> ?? ""`
|
|
239
239
|
* template literals so the secret never ends up baked into test.spec.ts.
|
|
240
|
-
* `dummy` is a placeholder value surfaced by the draft / drift prompts
|
|
241
|
-
* (which see the block in isolation, before any include site exists);
|
|
242
|
-
* `description` is the param's semantic role, also consumed by those
|
|
243
|
-
* prompts and by spec authors browsing the block.
|
|
244
240
|
*/
|
|
245
241
|
const BlockParamSchema = z.object({
|
|
246
242
|
name: z.string().min(1),
|
|
247
243
|
required: z.boolean().optional(),
|
|
248
|
-
secret: z.boolean().optional()
|
|
249
|
-
dummy: z.string().optional(),
|
|
250
|
-
description: z.string().optional()
|
|
244
|
+
secret: z.boolean().optional()
|
|
251
245
|
}).strict();
|
|
252
246
|
/**
|
|
253
247
|
* Block schema. Block steps are restricted to ActionStep — nested blocks are
|
|
@@ -270,15 +264,25 @@ function isParamRequired(param) {
|
|
|
270
264
|
}
|
|
271
265
|
//#endregion
|
|
272
266
|
//#region src/spec/parser.ts
|
|
273
|
-
/**
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
267
|
+
/** The spec/block root (an `unrecognized_keys` issue there has an empty path). */
|
|
268
|
+
const atRoot = (path) => path.length === 0;
|
|
269
|
+
/** A block param entry — the issue path is `params.<index>`. */
|
|
270
|
+
const atBlockParam = (path) => path.length === 2 && path[0] === "params" && typeof path[1] === "number";
|
|
271
|
+
const UNREAD_PARAM_FIELD = "nothing reads it (a block param reaches the prompts as its name, required and secret only). Delete the line.";
|
|
272
|
+
const REMOVED_FIELDS = {
|
|
273
|
+
relatedPaths: {
|
|
274
|
+
at: atRoot,
|
|
275
|
+
message: "which specs a change affects is now decided by `ccqa select-specs`, which reads the diff instead of a declared path list. Delete the field."
|
|
276
|
+
},
|
|
277
|
+
dummy: {
|
|
278
|
+
at: atBlockParam,
|
|
279
|
+
message: UNREAD_PARAM_FIELD
|
|
280
|
+
},
|
|
281
|
+
description: {
|
|
282
|
+
at: atBlockParam,
|
|
283
|
+
message: UNREAD_PARAM_FIELD
|
|
284
|
+
}
|
|
285
|
+
};
|
|
282
286
|
/** Parse a spec.yaml. Schema rejections are rewritten with actionable messages. */
|
|
283
287
|
function parseTestSpec(content, source = "spec.yaml") {
|
|
284
288
|
const raw = parseYamlOrThrow(content, source);
|
|
@@ -336,9 +340,9 @@ function humanizeIssue(issue, isBlock) {
|
|
|
336
340
|
if (issue.code === "unrecognized_keys") {
|
|
337
341
|
const keys = Array.isArray(issue.keys) ? issue.keys : [];
|
|
338
342
|
if (isBlock && keys.includes("include")) return `Nested blocks are not supported — flatten by inlining the included block's steps into this block.`;
|
|
339
|
-
const removed = keys.filter((k) => k
|
|
340
|
-
const stillUnknown = keys.filter((k) => !(
|
|
341
|
-
const parts = removed.map((k) => `\`${k}\` is no longer part of the spec schema — ${REMOVED_FIELDS[k]}`);
|
|
343
|
+
const removed = keys.filter((k) => REMOVED_FIELDS[k]?.at(issue.path));
|
|
344
|
+
const stillUnknown = keys.filter((k) => !REMOVED_FIELDS[k]?.at(issue.path));
|
|
345
|
+
const parts = removed.map((k) => `\`${k}\` is no longer part of the spec schema — ${REMOVED_FIELDS[k].message}`);
|
|
342
346
|
if (stillUnknown.length > 0) parts.push(`Unknown keys: ${stillUnknown.join(", ")}`);
|
|
343
347
|
return parts.join(" ");
|
|
344
348
|
}
|
|
@@ -512,8 +516,8 @@ async function loadAllBlocks(cwd) {
|
|
|
512
516
|
* Co-located with `loadAllBlocks` so callers don't have to remember the
|
|
513
517
|
* isParamRequired / secret-default mapping.
|
|
514
518
|
*/
|
|
515
|
-
|
|
516
|
-
return [...
|
|
519
|
+
function projectAvailableBlocks(blocks) {
|
|
520
|
+
return [...blocks.entries()].map(([name, block]) => ({
|
|
517
521
|
name,
|
|
518
522
|
title: block.title,
|
|
519
523
|
params: (block.params ?? []).map((p) => ({
|
|
@@ -523,6 +527,10 @@ async function loadAvailableBlocks(cwd) {
|
|
|
523
527
|
}))
|
|
524
528
|
}));
|
|
525
529
|
}
|
|
530
|
+
/** `loadAllBlocks` + `projectAvailableBlocks`, for callers that need only the projection. */
|
|
531
|
+
async function loadAvailableBlocks(cwd) {
|
|
532
|
+
return projectAvailableBlocks(await loadAllBlocks(cwd));
|
|
533
|
+
}
|
|
526
534
|
const USER_PROMPT_MAX_BYTES = 32768;
|
|
527
535
|
/**
|
|
528
536
|
* Load the prompt bundle from the hub for one guidance kind ("record" /
|
|
@@ -863,10 +871,12 @@ function cloudProviderEnabled() {
|
|
|
863
871
|
/**
|
|
864
872
|
* Probe whether the host has any credential the Anthropic SDK can pick up:
|
|
865
873
|
* 1. ANTHROPIC_API_KEY env var (CI / scripted use)
|
|
866
|
-
* 2.
|
|
874
|
+
* 2. CLAUDE_CODE_OAUTH_TOKEN env var (a long-lived subscription token from
|
|
875
|
+
* `claude setup-token`, the headless-CI counterpart of a login)
|
|
876
|
+
* 3. CLAUDE_CODE_USE_BEDROCK / CLAUDE_CODE_USE_VERTEX (cloud-provider
|
|
867
877
|
* endpoints authenticated by the cloud SDK's credential chain)
|
|
868
|
-
*
|
|
869
|
-
*
|
|
878
|
+
* 4. ~/.claude/.credentials.json (Claude Code login, file-based platforms)
|
|
879
|
+
* 5. macOS Keychain item "Claude Code-credentials" (Claude Code login on
|
|
870
880
|
* darwin stores the OAuth credentials in the Keychain, not on disk)
|
|
871
881
|
*
|
|
872
882
|
* Claude-driven hooks are opt-in, so the caller only consults this after the
|
|
@@ -874,14 +884,16 @@ function cloudProviderEnabled() {
|
|
|
874
884
|
* that surfaces as "analysis skipped".
|
|
875
885
|
*/
|
|
876
886
|
function driftAuthAvailable() {
|
|
877
|
-
const key
|
|
878
|
-
|
|
887
|
+
for (const key of ["ANTHROPIC_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN"]) {
|
|
888
|
+
const value = process.env[key];
|
|
889
|
+
if (typeof value === "string" && value.length > 0) return { ok: true };
|
|
890
|
+
}
|
|
879
891
|
if (cloudProviderEnabled()) return { ok: true };
|
|
880
892
|
if (existsSync(join(homedir(), ".claude", ".credentials.json"))) return { ok: true };
|
|
881
893
|
if (process.platform === "darwin" && keychainHasClaudeCredentials()) return { ok: true };
|
|
882
894
|
return {
|
|
883
895
|
ok: false,
|
|
884
|
-
reason: "no ANTHROPIC_API_KEY / Bedrock or Vertex env / claude login"
|
|
896
|
+
reason: "no ANTHROPIC_API_KEY / CLAUDE_CODE_OAUTH_TOKEN / Bedrock or Vertex env / claude login"
|
|
885
897
|
};
|
|
886
898
|
}
|
|
887
899
|
/**
|
|
@@ -1298,6 +1310,95 @@ function opt(key, value) {
|
|
|
1298
1310
|
return value ? { [key]: value } : {};
|
|
1299
1311
|
}
|
|
1300
1312
|
//#endregion
|
|
1313
|
+
//#region src/runtime/env-scrub.ts
|
|
1314
|
+
/**
|
|
1315
|
+
* Build a list of `[envValue, "${VAR}"]` pairs for every `${VAR}` reference
|
|
1316
|
+
* mentioned in the spec OR in any of its expanded (block-inlined) steps.
|
|
1317
|
+
* Used at trace time to scrub recorded Claude-text outputs so a value the
|
|
1318
|
+
* spec author intentionally threaded through `process.env` is preserved as
|
|
1319
|
+
* `${VAR}` in `ir.json` rather than baked in as the concrete
|
|
1320
|
+
* trace-time value.
|
|
1321
|
+
*
|
|
1322
|
+
* Why we walk `spec.steps` AND `expanded`:
|
|
1323
|
+
* - `spec.steps` carries the spec's own `instruction` / `expected` + each
|
|
1324
|
+
* include's raw `params` (which may themselves be `${ENV}` refs).
|
|
1325
|
+
* - `expanded` carries the inlined block-internal steps, whose
|
|
1326
|
+
* `instruction` / `expected` may *also* contain `${ENV}` refs that
|
|
1327
|
+
* don't go through include params.
|
|
1328
|
+
*
|
|
1329
|
+
* Only refs whose env value is currently non-empty land in the map —
|
|
1330
|
+
* scrubbing against an empty string would corrupt unrelated empty strings
|
|
1331
|
+
* in the action stream. Names whose env is unset are returned via
|
|
1332
|
+
* `unresolved` so the caller can warn the user.
|
|
1333
|
+
*
|
|
1334
|
+
* Longer values sort first so a `${SHORT}` whose value is a substring of a
|
|
1335
|
+
* `${LONG}` value doesn't clobber the longer one.
|
|
1336
|
+
*
|
|
1337
|
+
* `title` is deliberately NOT scanned — it never reaches the recorded action
|
|
1338
|
+
* stream.
|
|
1339
|
+
*/
|
|
1340
|
+
function buildSpecEnvScrub(spec, expanded) {
|
|
1341
|
+
const refNames = /* @__PURE__ */ new Set();
|
|
1342
|
+
for (const step of spec.steps) if (isIncludeStep(step)) for (const v of Object.values(step.params ?? {})) collect(v, refNames);
|
|
1343
|
+
else {
|
|
1344
|
+
collect(step.instruction, refNames);
|
|
1345
|
+
collect(step.expected, refNames);
|
|
1346
|
+
}
|
|
1347
|
+
for (const step of expanded) {
|
|
1348
|
+
collect(step.instruction, refNames);
|
|
1349
|
+
collect(step.expected, refNames);
|
|
1350
|
+
}
|
|
1351
|
+
const map = [];
|
|
1352
|
+
const unresolved = [];
|
|
1353
|
+
for (const name of refNames) {
|
|
1354
|
+
const value = process.env[name];
|
|
1355
|
+
if (typeof value === "string" && value.length > 0) map.push([value, "${" + name + "}"]);
|
|
1356
|
+
else unresolved.push(name);
|
|
1357
|
+
}
|
|
1358
|
+
map.sort((a, b) => b[0].length - a[0].length);
|
|
1359
|
+
return {
|
|
1360
|
+
map,
|
|
1361
|
+
unresolved
|
|
1362
|
+
};
|
|
1363
|
+
}
|
|
1364
|
+
function collect(value, into) {
|
|
1365
|
+
for (const name of iterEnvRefNames(value)) into.add(name);
|
|
1366
|
+
}
|
|
1367
|
+
/** Shorter than this, a value is no secret and matches inside ordinary words. */
|
|
1368
|
+
const MIN_PROSE_SCRUB_LENGTH = 4;
|
|
1369
|
+
/** Long enough to clear the length bar, still ordinary prose / JSON. */
|
|
1370
|
+
const COMMON_PROSE_VALUES = new Set([
|
|
1371
|
+
"true",
|
|
1372
|
+
"false",
|
|
1373
|
+
"null",
|
|
1374
|
+
"none",
|
|
1375
|
+
"undefined"
|
|
1376
|
+
]);
|
|
1377
|
+
/**
|
|
1378
|
+
* Scrub map for model output, built like {@link buildSpecEnvScrub} but
|
|
1379
|
+
* without the values that read as ordinary text (`"1"`, `"true"`): prose
|
|
1380
|
+
* runs to paragraphs, where replacing every occurrence of such a value
|
|
1381
|
+
* costs more meaning than it protects. Record's own scrub keeps them for
|
|
1382
|
+
* its single command lines; the live path reuses this one map for its Bash
|
|
1383
|
+
* command log too, trading that short-value coverage for not building a
|
|
1384
|
+
* second map.
|
|
1385
|
+
*/
|
|
1386
|
+
function buildProseEnvScrubMap(spec, expanded) {
|
|
1387
|
+
return buildSpecEnvScrub(spec, expanded).map.filter(([value]) => value.length >= MIN_PROSE_SCRUB_LENGTH && !COMMON_PROSE_VALUES.has(value.toLowerCase()));
|
|
1388
|
+
}
|
|
1389
|
+
/**
|
|
1390
|
+
* Replace every occurrence of an env value with its `${VAR}` placeholder in
|
|
1391
|
+
* `text`. **Caller invariant**: the map must be sorted longest-value-first
|
|
1392
|
+
* so a shorter value doesn't shadow a longer one that contains it as a
|
|
1393
|
+
* substring. `buildSpecEnvScrub` upholds this; hand-built maps should too.
|
|
1394
|
+
*/
|
|
1395
|
+
function scrubEnvValues(text, scrubMap) {
|
|
1396
|
+
if (scrubMap.length === 0) return text;
|
|
1397
|
+
let out = text;
|
|
1398
|
+
for (const [value, placeholder] of scrubMap) if (out.includes(value)) out = out.replaceAll(value, placeholder);
|
|
1399
|
+
return out;
|
|
1400
|
+
}
|
|
1401
|
+
//#endregion
|
|
1301
1402
|
//#region src/claude/native-binary.ts
|
|
1302
1403
|
const require$1 = createRequire(import.meta.url);
|
|
1303
1404
|
/**
|
|
@@ -1412,18 +1513,27 @@ function resolveModel(explicit) {
|
|
|
1412
1513
|
* - `ANTHROPIC_AUTH_TOKEN` — sent as `Authorization: Bearer <token>`.
|
|
1413
1514
|
* - `ANTHROPIC_API_KEY` — API key, when used instead of a token.
|
|
1414
1515
|
* - `ANTHROPIC_CUSTOM_HEADERS` — extra request headers.
|
|
1516
|
+
* - `CLAUDE_CODE_OAUTH_TOKEN` — long-lived subscription token from
|
|
1517
|
+
* `claude setup-token`, the headless-CI counterpart of a login.
|
|
1415
1518
|
*/
|
|
1416
1519
|
const ENDPOINT_ENV_KEYS = [
|
|
1417
1520
|
"ANTHROPIC_BASE_URL",
|
|
1418
1521
|
"ANTHROPIC_AUTH_TOKEN",
|
|
1419
1522
|
"ANTHROPIC_API_KEY",
|
|
1420
|
-
"ANTHROPIC_CUSTOM_HEADERS"
|
|
1523
|
+
"ANTHROPIC_CUSTOM_HEADERS",
|
|
1524
|
+
"CLAUDE_CODE_OAUTH_TOKEN"
|
|
1421
1525
|
];
|
|
1422
1526
|
/**
|
|
1423
1527
|
* Collects the endpoint/auth variables set in the current process environment
|
|
1424
1528
|
* so they can be forwarded, verbatim, to every Claude Code invocation. Returns
|
|
1425
1529
|
* only the keys that are actually set (non-empty), so unset variables never
|
|
1426
1530
|
* override the SDK's own defaults.
|
|
1531
|
+
*
|
|
1532
|
+
* When both credentials are present the OAuth token wins and the API key is
|
|
1533
|
+
* not forwarded. Left to the CLI the API key would win, which makes "switch a
|
|
1534
|
+
* CI job to the subscription token" require unwiring the key everywhere; with
|
|
1535
|
+
* the precedence here, adding the one variable is the whole switch, and
|
|
1536
|
+
* removing it is the whole rollback.
|
|
1427
1537
|
*/
|
|
1428
1538
|
function resolveEndpointEnv() {
|
|
1429
1539
|
const endpointEnv = {};
|
|
@@ -1431,6 +1541,7 @@ function resolveEndpointEnv() {
|
|
|
1431
1541
|
const value = process.env[key];
|
|
1432
1542
|
if (value && value.length > 0) endpointEnv[key] = value;
|
|
1433
1543
|
}
|
|
1544
|
+
if (endpointEnv["CLAUDE_CODE_OAUTH_TOKEN"]) delete endpointEnv["ANTHROPIC_API_KEY"];
|
|
1434
1545
|
return endpointEnv;
|
|
1435
1546
|
}
|
|
1436
1547
|
/**
|
|
@@ -1457,7 +1568,7 @@ function warnOnceIfNativeBinaryMissing() {
|
|
|
1457
1568
|
if (missing) warn(missingNativeBinaryMessage(missing));
|
|
1458
1569
|
}
|
|
1459
1570
|
async function invokeClaudeStreaming(options, onEvent) {
|
|
1460
|
-
const { prompt, systemPrompt, allowedTools, disableBuiltinTools = false, disableThinking = false, mcpServers, maxTurns, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, relaxAbConstraints = false } = options;
|
|
1571
|
+
const { prompt, systemPrompt, allowedTools, disableBuiltinTools = false, disableThinking = false, mcpServers, maxTurns, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, envScrubMap = [], relaxAbConstraints = false } = options;
|
|
1461
1572
|
const resolvedModel = resolveModel(model);
|
|
1462
1573
|
const hasEndpointEnv = Object.keys(resolveEndpointEnv()).length > 0;
|
|
1463
1574
|
const mergedEnv = env || hasEndpointEnv ? withoutEmptyEndpointVars({
|
|
@@ -1561,7 +1672,7 @@ async function invokeClaudeStreaming(options, onEvent) {
|
|
|
1561
1672
|
if (msg.type === "assistant" && !silenceBashLog) {
|
|
1562
1673
|
for (const block of msg.message.content ?? []) if (block.type === "tool_use" && block.name === "Bash") {
|
|
1563
1674
|
const cmd = block.input?.["command"];
|
|
1564
|
-
if (typeof cmd === "string") bash(cmd);
|
|
1675
|
+
if (typeof cmd === "string") bash(scrubEnvValues(cmd, envScrubMap));
|
|
1565
1676
|
}
|
|
1566
1677
|
}
|
|
1567
1678
|
if (msg.type === "result") {
|
|
@@ -3486,6 +3597,36 @@ function buildDiffMcpServer(getFileDiff) {
|
|
|
3486
3597
|
* with confidence 0 rather than throwing — the report must always render.
|
|
3487
3598
|
*/
|
|
3488
3599
|
async function analyzeFailure(input, options) {
|
|
3600
|
+
return scrubOutcome(await classifyFailure(input, options), options.envScrubMap ?? []);
|
|
3601
|
+
}
|
|
3602
|
+
/**
|
|
3603
|
+
* Mask the profile values the classifier may have quoted: its Read/Grep reach
|
|
3604
|
+
* the repository, so a local `.env` is in reach of its prose even when the
|
|
3605
|
+
* evidence it was handed is clean. A literal match — a value the model
|
|
3606
|
+
* paraphrases still gets through. Only what the classifier authored is
|
|
3607
|
+
* covered: the row's other evidence (`failureLogExcerpt`, `diffExcerpt`)
|
|
3608
|
+
* passes to the report and the hub as its producer wrote it.
|
|
3609
|
+
*/
|
|
3610
|
+
function scrubOutcome(outcome, scrubMap) {
|
|
3611
|
+
if (scrubMap.length === 0) return outcome;
|
|
3612
|
+
const scrub = (text) => scrubEnvValues(text, scrubMap);
|
|
3613
|
+
const { analysis } = outcome;
|
|
3614
|
+
return {
|
|
3615
|
+
...outcome,
|
|
3616
|
+
raw: scrub(outcome.raw),
|
|
3617
|
+
analysis: {
|
|
3618
|
+
...analysis,
|
|
3619
|
+
headline: scrub(analysis.headline),
|
|
3620
|
+
recommendation: scrub(analysis.recommendation),
|
|
3621
|
+
reasoning: scrub(analysis.reasoning),
|
|
3622
|
+
evidence: analysis.evidence.map((item) => ({
|
|
3623
|
+
...item,
|
|
3624
|
+
detail: scrub(item.detail)
|
|
3625
|
+
}))
|
|
3626
|
+
}
|
|
3627
|
+
};
|
|
3628
|
+
}
|
|
3629
|
+
async function classifyFailure(input, options) {
|
|
3489
3630
|
const { result: raw, isError } = await invokeClaudeStreaming({
|
|
3490
3631
|
prompt: buildFailureAnalysisPrompt(input),
|
|
3491
3632
|
allowedTools: [
|
|
@@ -6264,6 +6405,7 @@ function createFailureAnalysisPass(deps) {
|
|
|
6264
6405
|
};
|
|
6265
6406
|
const baselineMissing = specDiffResult.ok ? null : specDiffResult.skip;
|
|
6266
6407
|
info(`failure analysis: ${featureName}/${specName}${baselineMissing ? " (no baseline — classifying from current source)" : ""}`);
|
|
6408
|
+
const envScrubMap = specEnvScrubMap(input.parsedSpec, deps.parsedBlocks);
|
|
6267
6409
|
const outcome = await analyzeFailure({
|
|
6268
6410
|
script: await input.readScript(),
|
|
6269
6411
|
hasGeneratedSurface: true,
|
|
@@ -6283,7 +6425,8 @@ function createFailureAnalysisPass(deps) {
|
|
|
6283
6425
|
}, {
|
|
6284
6426
|
...deps.model ? { model: deps.model } : {},
|
|
6285
6427
|
cwd: deps.cwd,
|
|
6286
|
-
getFileDiff: specDiff?.fileDiff ?? (() => null)
|
|
6428
|
+
getFileDiff: specDiff?.fileDiff ?? (() => null),
|
|
6429
|
+
envScrubMap
|
|
6287
6430
|
});
|
|
6288
6431
|
if (!printedHeader) {
|
|
6289
6432
|
printedHeader = true;
|
|
@@ -6297,6 +6440,20 @@ function createFailureAnalysisPass(deps) {
|
|
|
6297
6440
|
};
|
|
6298
6441
|
} };
|
|
6299
6442
|
}
|
|
6443
|
+
/**
|
|
6444
|
+
* The `${VAR}` values this spec resolved, for masking them out of the
|
|
6445
|
+
* classifier's prose. Read from `process.env` here rather than at run start
|
|
6446
|
+
* (where the live path builds its own): a profile is applied once per
|
|
6447
|
+
* invocation, so this is still what the spec ran against.
|
|
6448
|
+
*/
|
|
6449
|
+
function specEnvScrubMap(spec, blocks) {
|
|
6450
|
+
if (spec === null) return [];
|
|
6451
|
+
try {
|
|
6452
|
+
return buildProseEnvScrubMap(spec, expandSpec(spec, { blocks }));
|
|
6453
|
+
} catch {
|
|
6454
|
+
return buildProseEnvScrubMap(spec, []);
|
|
6455
|
+
}
|
|
6456
|
+
}
|
|
6300
6457
|
/** One classified spec's line in the failure-analysis block. */
|
|
6301
6458
|
function printAnalysis(featureName, specName, analysis) {
|
|
6302
6459
|
const pct = Math.round(analysis.confidence * 100);
|
|
@@ -6349,6 +6506,7 @@ async function analyzeExternalRows(rows, run) {
|
|
|
6349
6506
|
readScript: () => readGeneratedTestSources(ref, deps.cwd),
|
|
6350
6507
|
failureLog: row.failureLogExcerpt ?? "",
|
|
6351
6508
|
specYaml: row.specYaml,
|
|
6509
|
+
parsedSpec: tryParseTestSpec(row.specYaml),
|
|
6352
6510
|
target: row.target ?? "agent-browser",
|
|
6353
6511
|
artifactsDir: readableArtifactsDir(ref, deps)
|
|
6354
6512
|
});
|
|
@@ -9365,93 +9523,6 @@ function isStorageStateShape(state) {
|
|
|
9365
9523
|
return typeof state === "object" && state !== null && Array.isArray(state.cookies) && Array.isArray(state.origins);
|
|
9366
9524
|
}
|
|
9367
9525
|
//#endregion
|
|
9368
|
-
//#region src/runtime/env-scrub.ts
|
|
9369
|
-
/**
|
|
9370
|
-
* Build a list of `[envValue, "${VAR}"]` pairs for every `${VAR}` reference
|
|
9371
|
-
* mentioned in the spec OR in any of its expanded (block-inlined) steps.
|
|
9372
|
-
* Used at trace time to scrub recorded Claude-text outputs so a value the
|
|
9373
|
-
* spec author intentionally threaded through `process.env` is preserved as
|
|
9374
|
-
* `${VAR}` in `ir.json` rather than baked in as the concrete
|
|
9375
|
-
* trace-time value.
|
|
9376
|
-
*
|
|
9377
|
-
* Why we walk `spec.steps` AND `expanded`:
|
|
9378
|
-
* - `spec.steps` carries the spec's own `instruction` / `expected` + each
|
|
9379
|
-
* include's raw `params` (which may themselves be `${ENV}` refs).
|
|
9380
|
-
* - `expanded` carries the inlined block-internal steps, whose
|
|
9381
|
-
* `instruction` / `expected` may *also* contain `${ENV}` refs that
|
|
9382
|
-
* don't go through include params.
|
|
9383
|
-
*
|
|
9384
|
-
* Only refs whose env value is currently non-empty land in the map —
|
|
9385
|
-
* scrubbing against an empty string would corrupt unrelated empty strings
|
|
9386
|
-
* in the action stream. Names whose env is unset are returned via
|
|
9387
|
-
* `unresolved` so the caller can warn the user.
|
|
9388
|
-
*
|
|
9389
|
-
* Longer values sort first so a `${SHORT}` whose value is a substring of a
|
|
9390
|
-
* `${LONG}` value doesn't clobber the longer one.
|
|
9391
|
-
*
|
|
9392
|
-
* `title` is deliberately NOT scanned — it never reaches the recorded action
|
|
9393
|
-
* stream.
|
|
9394
|
-
*/
|
|
9395
|
-
function buildSpecEnvScrub(spec, expanded) {
|
|
9396
|
-
const refNames = /* @__PURE__ */ new Set();
|
|
9397
|
-
for (const step of spec.steps) if (isIncludeStep(step)) for (const v of Object.values(step.params ?? {})) collect(v, refNames);
|
|
9398
|
-
else {
|
|
9399
|
-
collect(step.instruction, refNames);
|
|
9400
|
-
collect(step.expected, refNames);
|
|
9401
|
-
}
|
|
9402
|
-
for (const step of expanded) {
|
|
9403
|
-
collect(step.instruction, refNames);
|
|
9404
|
-
collect(step.expected, refNames);
|
|
9405
|
-
}
|
|
9406
|
-
const map = [];
|
|
9407
|
-
const unresolved = [];
|
|
9408
|
-
for (const name of refNames) {
|
|
9409
|
-
const value = process.env[name];
|
|
9410
|
-
if (typeof value === "string" && value.length > 0) map.push([value, "${" + name + "}"]);
|
|
9411
|
-
else unresolved.push(name);
|
|
9412
|
-
}
|
|
9413
|
-
map.sort((a, b) => b[0].length - a[0].length);
|
|
9414
|
-
return {
|
|
9415
|
-
map,
|
|
9416
|
-
unresolved
|
|
9417
|
-
};
|
|
9418
|
-
}
|
|
9419
|
-
function collect(value, into) {
|
|
9420
|
-
for (const name of iterEnvRefNames(value)) into.add(name);
|
|
9421
|
-
}
|
|
9422
|
-
/** Shorter than this, a value is no secret and matches inside ordinary words. */
|
|
9423
|
-
const MIN_PROSE_SCRUB_LENGTH = 4;
|
|
9424
|
-
/** Long enough to clear the length bar, still ordinary prose / JSON. */
|
|
9425
|
-
const COMMON_PROSE_VALUES = new Set([
|
|
9426
|
-
"true",
|
|
9427
|
-
"false",
|
|
9428
|
-
"null",
|
|
9429
|
-
"none",
|
|
9430
|
-
"undefined"
|
|
9431
|
-
]);
|
|
9432
|
-
/**
|
|
9433
|
-
* Scrub map for a live run, built like {@link buildSpecEnvScrub} but without
|
|
9434
|
-
* the values that read as ordinary text (`"1"`, `"true"`). A live step records
|
|
9435
|
-
* paragraphs of model prose, so replacing every occurrence of such a value
|
|
9436
|
-
* would cost more meaning than it protects; record scrubs single command
|
|
9437
|
-
* lines, where the same trade favours keeping them.
|
|
9438
|
-
*/
|
|
9439
|
-
function buildLiveEnvScrubMap(spec, expanded) {
|
|
9440
|
-
return buildSpecEnvScrub(spec, expanded).map.filter(([value]) => value.length >= MIN_PROSE_SCRUB_LENGTH && !COMMON_PROSE_VALUES.has(value.toLowerCase()));
|
|
9441
|
-
}
|
|
9442
|
-
/**
|
|
9443
|
-
* Replace every occurrence of an env value with its `${VAR}` placeholder in
|
|
9444
|
-
* `text`. **Caller invariant**: the map must be sorted longest-value-first
|
|
9445
|
-
* so a shorter value doesn't shadow a longer one that contains it as a
|
|
9446
|
-
* substring. `buildSpecEnvScrub` upholds this; hand-built maps should too.
|
|
9447
|
-
*/
|
|
9448
|
-
function scrubEnvValues(text, scrubMap) {
|
|
9449
|
-
if (scrubMap.length === 0) return text;
|
|
9450
|
-
let out = text;
|
|
9451
|
-
for (const [value, placeholder] of scrubMap) if (out.includes(value)) out = out.replaceAll(value, placeholder);
|
|
9452
|
-
return out;
|
|
9453
|
-
}
|
|
9454
|
-
//#endregion
|
|
9455
9526
|
//#region src/claude/agent-browser-invoke.ts
|
|
9456
9527
|
function agentBrowserInvokeBase(input) {
|
|
9457
9528
|
return {
|
|
@@ -9749,6 +9820,7 @@ async function runLiveExecutor(input) {
|
|
|
9749
9820
|
prompt: userPrompt,
|
|
9750
9821
|
systemPrompt,
|
|
9751
9822
|
model: input.model,
|
|
9823
|
+
envScrubMap: input.envScrubMap,
|
|
9752
9824
|
relaxAbConstraints: true
|
|
9753
9825
|
}, (msg) => {
|
|
9754
9826
|
if (msg.type !== "assistant") return;
|
|
@@ -10297,7 +10369,7 @@ async function runOneSpec(args) {
|
|
|
10297
10369
|
}
|
|
10298
10370
|
const spec = parseTestSpec(specContent);
|
|
10299
10371
|
const expanded = expandSpec(spec, { blocks: await loadAllBlocks(cwd) });
|
|
10300
|
-
const envScrubMap =
|
|
10372
|
+
const envScrubMap = buildProseEnvScrubMap(spec, expanded);
|
|
10301
10373
|
meta("spec", spec.title);
|
|
10302
10374
|
meta("steps", expanded.length);
|
|
10303
10375
|
const includes = collectIncludedBlockNames(spec);
|
|
@@ -10359,6 +10431,7 @@ async function runOneSpec(args) {
|
|
|
10359
10431
|
specName,
|
|
10360
10432
|
runDir,
|
|
10361
10433
|
specYaml: specContent,
|
|
10434
|
+
envScrubMap,
|
|
10362
10435
|
result
|
|
10363
10436
|
};
|
|
10364
10437
|
} finally {
|
|
@@ -10417,7 +10490,8 @@ async function analyzeOneLiveFailure(r, diffProvider, auth, blocks, opts, cwd) {
|
|
|
10417
10490
|
}, {
|
|
10418
10491
|
...opts.model ? { model: opts.model } : {},
|
|
10419
10492
|
cwd,
|
|
10420
|
-
getFileDiff: specDiff?.fileDiff ?? (() => null)
|
|
10493
|
+
getFileDiff: specDiff?.fileDiff ?? (() => null),
|
|
10494
|
+
envScrubMap: r.envScrubMap
|
|
10421
10495
|
});
|
|
10422
10496
|
const pct = Math.round(outcome.analysis.confidence * 100);
|
|
10423
10497
|
const headline = outcome.analysis.headline.trim() || (outcome.analysis.reasoning.split("\n")[0] ?? "").trim();
|
|
@@ -12916,6 +12990,7 @@ async function executeRun(targets, opts) {
|
|
|
12916
12990
|
});
|
|
12917
12991
|
const triageUserPromptHash = triageUserPrompt ? hashTriageUserPrompt(triageUserPrompt) : null;
|
|
12918
12992
|
const reportDir = resolveReportDir(opts.reportDir, cwd);
|
|
12993
|
+
const parsedBlocks = await loadAllBlocks(cwd);
|
|
12919
12994
|
const analysisDeps = {
|
|
12920
12995
|
diffProvider,
|
|
12921
12996
|
auth: diffProvider ? driftAuthAvailable() : {
|
|
@@ -12924,7 +12999,8 @@ async function executeRun(targets, opts) {
|
|
|
12924
12999
|
},
|
|
12925
13000
|
cwd,
|
|
12926
13001
|
reportDir,
|
|
12927
|
-
blocks:
|
|
13002
|
+
blocks: projectAvailableBlocks(parsedBlocks),
|
|
13003
|
+
parsedBlocks,
|
|
12928
13004
|
...opts.model ? { model: opts.model } : {},
|
|
12929
13005
|
...opts.language ? { language: opts.language } : {},
|
|
12930
13006
|
customPrompt,
|
|
@@ -13373,14 +13449,13 @@ function failedSpec(s) {
|
|
|
13373
13449
|
* specs share). Degrades — never throws — when Claude auth or the git diff
|
|
13374
13450
|
* aren't available. Caller writes report.json.
|
|
13375
13451
|
*/
|
|
13376
|
-
async function analyzeDeterministicSummaries(summaries, cwd, reportDir, { pass }) {
|
|
13377
|
-
const allBlocks = await loadAllBlocks(cwd);
|
|
13452
|
+
async function analyzeDeterministicSummaries(summaries, cwd, reportDir, { pass, deps }) {
|
|
13378
13453
|
const results = [];
|
|
13379
13454
|
for (const s of summaries) {
|
|
13380
13455
|
const assertions = collectAssertions(s);
|
|
13381
13456
|
const specYaml = await tryReadSpecFile(s.featureName, s.specName, cwd);
|
|
13382
13457
|
const parsedSpec = tryParseTestSpec(specYaml);
|
|
13383
|
-
const stepDescriptions = buildStepDescriptions(parsedSpec,
|
|
13458
|
+
const stepDescriptions = buildStepDescriptions(parsedSpec, deps.parsedBlocks);
|
|
13384
13459
|
const evidence = await loadEvidenceForSpec(s.evidenceDir, reportDir, stepDescriptions);
|
|
13385
13460
|
const base = {
|
|
13386
13461
|
feature: s.featureName,
|
|
@@ -13416,6 +13491,7 @@ async function analyzeDeterministicSummaries(summaries, cwd, reportDir, { pass }
|
|
|
13416
13491
|
readScript: () => readScriptSafe(s.scriptFile),
|
|
13417
13492
|
failureLog,
|
|
13418
13493
|
specYaml,
|
|
13494
|
+
parsedSpec,
|
|
13419
13495
|
target: AGENT_BROWSER_TARGET
|
|
13420
13496
|
});
|
|
13421
13497
|
results.push({
|
|
@@ -14833,6 +14909,7 @@ async function runTrace(featureName, specName, model, validationMode = "lenient"
|
|
|
14833
14909
|
runId: sessionName
|
|
14834
14910
|
}),
|
|
14835
14911
|
model,
|
|
14912
|
+
envScrubMap,
|
|
14836
14913
|
onAbAction: ({ abAction, stepId, assertMarker }) => {
|
|
14837
14914
|
const stepForCommand = stepTracker.fromCommand(stepId);
|
|
14838
14915
|
const line = abAction === void 0 ? null : scrubEnvValues(abAction, envScrubMap);
|
package/dist/package.json
CHANGED