ccqa 1.48.1 → 1.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { $ as
|
|
2
|
+
import { $ as isIncludeStep, A as meta, B as expandActionSteps, C as blank, D as header, E as fix, F as timedPhase, G as parseBlockSpec, H as isExpandedActionStep, I as warn, J as AGENT_BROWSER_TARGET, K as parseTestSpec, L as withBuffer, M as progressEnd, N as run, O as hint, P as step, Q as TargetIdSchema, R as CREDENTIAL_ENV_KEYS, S as promoteMarkedAssert, T as error, U as isExpandedJudgeByLlmStep, V as expandSpec, W as isJudgeBody, X as SessionNameSchema, Y as DEFAULT_SPEC_MODE, Z as SpecModeSchema, _ as scrubUnstableActions, a as truncate$2, b as scrubEnvValues, c as toAgentBrowserArgs, d as surfaceAxisAside, et as isParamRequired, f as surfaceDefinitionBlock, g as formatUnstableDrop, h as withCostTally, i as isObject, it as resolveEnvRefs, j as progress, k as info, l as numberLines, m as readCostTally, n as diagnose, nt as envRefsToJsExpression, o as describeLocator, p as invokeClaudeStreaming, q as tryParseTestSpec, r as extractJsonCandidates, rt as iterEnvRefNames, s as locatorToSelector, t as clamp, tt as bracedRefsToJsExpression, u as outputLanguageBlock, v as buildProseEnvScrubMap, w as emitRaw, x as parseAbActionLine, y as buildSpecEnvScrub, z as collectIncludedBlockNames } from "../diagnose-CZms9Cer.mjs";
|
|
3
3
|
import { HubApiError, createHubClient, hubRequest } from "../hub-client/index.mjs";
|
|
4
4
|
import { t as EVIDENCE_DIR_ENV } from "../evidence-constants-Cm_S_5od.mjs";
|
|
5
5
|
import { a as formatAgentBrowserUnavailableMessage, i as assertAgentBrowserAvailable, n as spawnAB, o as pathWithAgentBrowserShim, r as AgentBrowserUnavailableError, s as resolveAgentBrowserBin$1, t as sleepSync } from "../spawn-ab-CR_Sr7wh.mjs";
|
|
@@ -23,6 +23,7 @@ import { connect as connect$1 } from "node:tls";
|
|
|
23
23
|
import { gunzipSync, gzipSync } from "node:zlib";
|
|
24
24
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
25
25
|
import { createInterface as createInterface$1 } from "node:readline";
|
|
26
|
+
import { pipeline } from "node:stream/promises";
|
|
26
27
|
//#region src/run/report-constants.ts
|
|
27
28
|
/**
|
|
28
29
|
* Pure report/run constants with no runtime dependencies. Kept separate from
|
|
@@ -17062,149 +17063,6 @@ function isPassiveAction(action) {
|
|
|
17062
17063
|
return action === "snapshot" || action === "wait" || action === "assert";
|
|
17063
17064
|
}
|
|
17064
17065
|
//#endregion
|
|
17065
|
-
//#region src/runtime/literal-scrub.ts
|
|
17066
|
-
/**
|
|
17067
|
-
* Patterns are listed in roughly descending confidence — a hit on `clock-hms`
|
|
17068
|
-
* is almost certainly bad; a hit on `unix-epoch-sec` (`1[0-9]{9}`) gates on
|
|
17069
|
-
* the value starting with `1`, which empirically rules out most SKU / order-id
|
|
17070
|
-
* false positives while still catching epoch seconds in the 2001-2033 window.
|
|
17071
|
-
*
|
|
17072
|
-
* Relative-time labels ("just now", "N minutes ago", "N分前") are the same
|
|
17073
|
-
* class of problem as wall-clock literals: the page shows them, Claude
|
|
17074
|
-
* captures them, and they're stale before the test ever replays. We only
|
|
17075
|
-
* catch the unambiguous variants — bare "now" or "minute" would false-fire
|
|
17076
|
-
* on routine UI copy.
|
|
17077
|
-
*/
|
|
17078
|
-
const UNSTABLE_PATTERNS = [
|
|
17079
|
-
{
|
|
17080
|
-
id: "clock-hms",
|
|
17081
|
-
pattern: /\b\d{2}:\d{2}:\d{2}\b/,
|
|
17082
|
-
label: "clock time HH:MM:SS"
|
|
17083
|
-
},
|
|
17084
|
-
{
|
|
17085
|
-
id: "iso-datetime",
|
|
17086
|
-
pattern: /\b\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/,
|
|
17087
|
-
label: "ISO datetime"
|
|
17088
|
-
},
|
|
17089
|
-
{
|
|
17090
|
-
id: "iso-date",
|
|
17091
|
-
pattern: /\b\d{4}-\d{2}-\d{2}\b/,
|
|
17092
|
-
label: "ISO date YYYY-MM-DD"
|
|
17093
|
-
},
|
|
17094
|
-
{
|
|
17095
|
-
id: "unix-epoch-sec",
|
|
17096
|
-
pattern: /\b1[0-9]{9}\b/,
|
|
17097
|
-
label: "Unix epoch seconds"
|
|
17098
|
-
},
|
|
17099
|
-
{
|
|
17100
|
-
id: "unix-epoch-ms",
|
|
17101
|
-
pattern: /\b1[0-9]{12}\b/,
|
|
17102
|
-
label: "Unix epoch milliseconds"
|
|
17103
|
-
},
|
|
17104
|
-
{
|
|
17105
|
-
id: "relative-time-en",
|
|
17106
|
-
pattern: /\b\d+\s+(second|minute|hour|day|week|month|year)s?\s+ago\b/i,
|
|
17107
|
-
label: "English relative time (`N <unit> ago`)"
|
|
17108
|
-
},
|
|
17109
|
-
{
|
|
17110
|
-
id: "relative-now-en",
|
|
17111
|
-
pattern: /\bjust\s+now\b/i,
|
|
17112
|
-
label: "English `just now`"
|
|
17113
|
-
},
|
|
17114
|
-
{
|
|
17115
|
-
id: "relative-time-ja",
|
|
17116
|
-
pattern: /\d+\s*(秒|分|時間|日|週間|か月|ヶ月|年)前/,
|
|
17117
|
-
label: "Japanese relative time (`N<unit>前`)"
|
|
17118
|
-
},
|
|
17119
|
-
{
|
|
17120
|
-
id: "relative-now-ja",
|
|
17121
|
-
pattern: /たった今/,
|
|
17122
|
-
label: "Japanese `たった今`"
|
|
17123
|
-
},
|
|
17124
|
-
{
|
|
17125
|
-
id: "ja-date-full",
|
|
17126
|
-
pattern: /\d{4}年\d{1,2}月\d{1,2}日/,
|
|
17127
|
-
label: "Japanese date YYYY年M月D日"
|
|
17128
|
-
},
|
|
17129
|
-
{
|
|
17130
|
-
id: "ja-date-md",
|
|
17131
|
-
pattern: /(?<!年)(?<!\d)\d{1,2}月\d{1,2}日(?![間目])/,
|
|
17132
|
-
label: "Japanese date M月D日"
|
|
17133
|
-
}
|
|
17134
|
-
];
|
|
17135
|
-
/**
|
|
17136
|
-
* Inspect a single action and return every (field, pattern) pair that
|
|
17137
|
-
* fired. An empty array means the action is safe to keep.
|
|
17138
|
-
*/
|
|
17139
|
-
function detectUnstableLiterals(action) {
|
|
17140
|
-
const fields = [
|
|
17141
|
-
["locator.value", action.locator?.value],
|
|
17142
|
-
["locator.name", action.locator?.by === "role" ? action.locator.name : void 0],
|
|
17143
|
-
["target.value", action.target?.value],
|
|
17144
|
-
["value", action.value],
|
|
17145
|
-
["label", action.label],
|
|
17146
|
-
["observation", action.observation]
|
|
17147
|
-
];
|
|
17148
|
-
const hits = [];
|
|
17149
|
-
for (const [field, raw] of fields) {
|
|
17150
|
-
if (typeof raw !== "string" || raw.length === 0) continue;
|
|
17151
|
-
for (const p of UNSTABLE_PATTERNS) {
|
|
17152
|
-
const m = raw.match(p.pattern);
|
|
17153
|
-
if (m) hits.push({
|
|
17154
|
-
field,
|
|
17155
|
-
patternId: p.id,
|
|
17156
|
-
match: m[0]
|
|
17157
|
-
});
|
|
17158
|
-
}
|
|
17159
|
-
}
|
|
17160
|
-
return hits;
|
|
17161
|
-
}
|
|
17162
|
-
/**
|
|
17163
|
-
* Walk every recorded action and split it into kept / dropped buckets. A
|
|
17164
|
-
* `snapshot` action is treated specially: its `observation` field is just a
|
|
17165
|
-
* comment in the generated script, so we keep the action even if its
|
|
17166
|
-
* `observation` carries an unstable literal — the comment will be wrong but
|
|
17167
|
-
* the script will still run. All other actions get dropped on any hit
|
|
17168
|
-
* because their locator / `value` would otherwise drive an unreproducible
|
|
17169
|
-
* interaction.
|
|
17170
|
-
*/
|
|
17171
|
-
function scrubUnstableActions(actions) {
|
|
17172
|
-
const kept = [];
|
|
17173
|
-
const dropped = [];
|
|
17174
|
-
for (let i = 0; i < actions.length; i++) {
|
|
17175
|
-
const action = actions[i];
|
|
17176
|
-
const hits = detectUnstableLiterals(action);
|
|
17177
|
-
if (hits.length === 0) {
|
|
17178
|
-
kept.push(action);
|
|
17179
|
-
continue;
|
|
17180
|
-
}
|
|
17181
|
-
if (action.action === "snapshot" && hits.every((h) => h.field === "observation")) {
|
|
17182
|
-
kept.push(action);
|
|
17183
|
-
continue;
|
|
17184
|
-
}
|
|
17185
|
-
dropped.push({
|
|
17186
|
-
index: i,
|
|
17187
|
-
action,
|
|
17188
|
-
hits
|
|
17189
|
-
});
|
|
17190
|
-
}
|
|
17191
|
-
return {
|
|
17192
|
-
kept,
|
|
17193
|
-
dropped
|
|
17194
|
-
};
|
|
17195
|
-
}
|
|
17196
|
-
/**
|
|
17197
|
-
* Human-readable summary of one drop, suitable for `log.warn`. The format
|
|
17198
|
-
* mirrors `replay-validate`'s drop line so both sources of warnings look
|
|
17199
|
-
* uniform in the trace output.
|
|
17200
|
-
*/
|
|
17201
|
-
function formatUnstableDrop(drop) {
|
|
17202
|
-
const { action, hits } = drop;
|
|
17203
|
-
const ids = [...new Set(hits.map((h) => h.patternId))].join(", ");
|
|
17204
|
-
const samples = hits.map((h) => `${h.field}="${h.match}"`).join(", ");
|
|
17205
|
-
return `${`${action.action}${action.assert ? " " + action.assert : ""}`}: contains unstable literal (${ids}) — ${samples}`;
|
|
17206
|
-
}
|
|
17207
|
-
//#endregion
|
|
17208
17066
|
//#region src/cli/trace.ts
|
|
17209
17067
|
/**
|
|
17210
17068
|
* Step ids (in spec order) whose kept actions include no assertion. A step
|
|
@@ -29546,7 +29404,7 @@ const PRUNE_AGE_SLACK_MS = 3600 * 1e3;
|
|
|
29546
29404
|
* event per line, appended in place (not atomic-rewritten — an append must
|
|
29547
29405
|
* not cost the whole stream). A reader can therefore observe a partial final
|
|
29548
29406
|
* line mid-append; the read side counts such lines as skipped rather than
|
|
29549
|
-
* failing, and the prune's full rewrite goes through
|
|
29407
|
+
* failing, and the prune's full rewrite goes through a temp file + rename.
|
|
29550
29408
|
*/
|
|
29551
29409
|
function createFileCoverageEventStore(root, caps) {
|
|
29552
29410
|
const maxEvents = caps?.maxEvents ?? 2e5;
|
|
@@ -29558,15 +29416,15 @@ function createFileCoverageEventStore(root, caps) {
|
|
|
29558
29416
|
async function loadState(project, path) {
|
|
29559
29417
|
const cached = states.get(project);
|
|
29560
29418
|
if (cached) return cached;
|
|
29561
|
-
const
|
|
29419
|
+
const tail = await statTail(path);
|
|
29562
29420
|
const state = {
|
|
29563
29421
|
nextSeq: 1,
|
|
29564
29422
|
count: 0,
|
|
29565
|
-
bytes:
|
|
29423
|
+
bytes: tail?.size ?? 0,
|
|
29566
29424
|
oldestAt: null,
|
|
29567
|
-
endsWithNewline:
|
|
29425
|
+
endsWithNewline: tail?.endsWithNewline ?? true
|
|
29568
29426
|
};
|
|
29569
|
-
for (const rawLine of
|
|
29427
|
+
for await (const rawLine of streamLines(path)) {
|
|
29570
29428
|
const line = parseLine(rawLine);
|
|
29571
29429
|
if (line === null) continue;
|
|
29572
29430
|
if (line.seq >= state.nextSeq) state.nextSeq = line.seq + 1;
|
|
@@ -29581,18 +29439,39 @@ function createFileCoverageEventStore(root, caps) {
|
|
|
29581
29439
|
const overBytes = state.bytes > maxBytes;
|
|
29582
29440
|
const overAge = state.oldestAt !== null && state.oldestAt < now - retentionMs - PRUNE_AGE_SLACK_MS;
|
|
29583
29441
|
if (!overCount && !overBytes && !overAge) return;
|
|
29584
|
-
const lines = await readLines(path);
|
|
29585
29442
|
const cutoff = now - retentionMs;
|
|
29586
|
-
const
|
|
29443
|
+
const freshSizes = [];
|
|
29444
|
+
for await (const line of streamFreshLines(path, cutoff)) freshSizes.push(Buffer.byteLength(JSON.stringify(line)) + 1);
|
|
29587
29445
|
const keep = overCount ? Math.max(0, maxEvents - pruneBatch) : maxEvents;
|
|
29588
|
-
let
|
|
29589
|
-
if (overBytes)
|
|
29590
|
-
|
|
29591
|
-
|
|
29592
|
-
|
|
29593
|
-
|
|
29594
|
-
|
|
29595
|
-
|
|
29446
|
+
let firstKept = freshSizes.length > keep ? freshSizes.length - keep : 0;
|
|
29447
|
+
if (overBytes) firstKept = firstWithinBytes(freshSizes, firstKept, pruneBytesTarget);
|
|
29448
|
+
await mkdir(dirname(path), { recursive: true });
|
|
29449
|
+
const tmp = `${path}.${randomUUID()}.tmp`;
|
|
29450
|
+
let keptCount = 0;
|
|
29451
|
+
let keptBytes = 0;
|
|
29452
|
+
let oldestKeptAt = null;
|
|
29453
|
+
try {
|
|
29454
|
+
await pipeline(async function* () {
|
|
29455
|
+
let freshIdx = 0;
|
|
29456
|
+
for await (const line of streamFreshLines(path, cutoff)) {
|
|
29457
|
+
freshIdx += 1;
|
|
29458
|
+
if (freshIdx <= firstKept) continue;
|
|
29459
|
+
const text = JSON.stringify(line) + "\n";
|
|
29460
|
+
keptCount += 1;
|
|
29461
|
+
keptBytes += Buffer.byteLength(text);
|
|
29462
|
+
if (oldestKeptAt === null) oldestKeptAt = line.at;
|
|
29463
|
+
yield text;
|
|
29464
|
+
}
|
|
29465
|
+
}, createWriteStream(tmp, { encoding: "utf8" }));
|
|
29466
|
+
} catch (err) {
|
|
29467
|
+
await rm(tmp, { force: true });
|
|
29468
|
+
throw err;
|
|
29469
|
+
}
|
|
29470
|
+
await rename(tmp, path);
|
|
29471
|
+
const dropped = state.count - keptCount;
|
|
29472
|
+
state.count = keptCount;
|
|
29473
|
+
state.bytes = keptBytes;
|
|
29474
|
+
state.oldestAt = oldestKeptAt;
|
|
29596
29475
|
state.endsWithNewline = true;
|
|
29597
29476
|
if (dropped > 0) console.warn(`hub: coverage inbox for "${project}": dropped ${dropped} events past retention (${maxEvents} events / ${Math.round(maxBytes / 1048576)} MiB / ${Math.round(retentionMs / 864e5)} days)`);
|
|
29598
29477
|
}
|
|
@@ -29661,22 +29540,46 @@ function createFileCoverageEventStore(root, caps) {
|
|
|
29661
29540
|
}
|
|
29662
29541
|
};
|
|
29663
29542
|
}
|
|
29664
|
-
/**
|
|
29665
|
-
function
|
|
29666
|
-
let
|
|
29667
|
-
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
|
29668
|
-
total += Buffer.byteLength(JSON.stringify(lines[i])) + 1;
|
|
29669
|
-
if (total > budget) return lines.slice(i + 1);
|
|
29670
|
-
}
|
|
29671
|
-
return lines;
|
|
29672
|
-
}
|
|
29673
|
-
async function readRaw(path) {
|
|
29543
|
+
/** Size and trailing-newline state of the stream file, or null when it doesn't exist. */
|
|
29544
|
+
async function statTail(path) {
|
|
29545
|
+
let fh;
|
|
29674
29546
|
try {
|
|
29675
|
-
|
|
29547
|
+
fh = await open(path, "r");
|
|
29676
29548
|
} catch (err) {
|
|
29677
|
-
if (err
|
|
29549
|
+
if (isNotFound(err)) return null;
|
|
29678
29550
|
throw err;
|
|
29679
29551
|
}
|
|
29552
|
+
try {
|
|
29553
|
+
const { size } = await fh.stat();
|
|
29554
|
+
if (size === 0) return {
|
|
29555
|
+
size,
|
|
29556
|
+
endsWithNewline: true
|
|
29557
|
+
};
|
|
29558
|
+
const tail = Buffer.alloc(1);
|
|
29559
|
+
await fh.read(tail, 0, 1, size - 1);
|
|
29560
|
+
return {
|
|
29561
|
+
size,
|
|
29562
|
+
endsWithNewline: tail[0] === 10
|
|
29563
|
+
};
|
|
29564
|
+
} finally {
|
|
29565
|
+
await fh.close();
|
|
29566
|
+
}
|
|
29567
|
+
}
|
|
29568
|
+
/** Retention-window lines only, in file order (= append order = seq order). */
|
|
29569
|
+
async function* streamFreshLines(path, cutoff) {
|
|
29570
|
+
for await (const rawLine of streamLines(path)) {
|
|
29571
|
+
const line = parseLine(rawLine);
|
|
29572
|
+
if (line !== null && line.at >= cutoff) yield line;
|
|
29573
|
+
}
|
|
29574
|
+
}
|
|
29575
|
+
/** Start of the longest suffix of `sizes` that fits `budget` (never below `lower`). */
|
|
29576
|
+
function firstWithinBytes(sizes, lower, budget) {
|
|
29577
|
+
let total = 0;
|
|
29578
|
+
for (let i = sizes.length - 1; i >= lower; i -= 1) {
|
|
29579
|
+
total += sizes[i];
|
|
29580
|
+
if (total > budget) return i + 1;
|
|
29581
|
+
}
|
|
29582
|
+
return lower;
|
|
29680
29583
|
}
|
|
29681
29584
|
/**
|
|
29682
29585
|
* The stream's non-empty lines, one at a time. Streamed rather than read as one
|
|
@@ -29693,21 +29596,12 @@ async function* streamLines(path) {
|
|
|
29693
29596
|
try {
|
|
29694
29597
|
for await (const line of lines) if (line !== "") yield line;
|
|
29695
29598
|
} catch (err) {
|
|
29696
|
-
if (!(err
|
|
29599
|
+
if (!isNotFound(err)) throw err;
|
|
29697
29600
|
} finally {
|
|
29698
29601
|
lines.close();
|
|
29699
29602
|
input.destroy();
|
|
29700
29603
|
}
|
|
29701
29604
|
}
|
|
29702
|
-
/** Every parseable line of the log; partial or corrupt lines are silently omitted (the read side counts them). */
|
|
29703
|
-
async function readLines(path) {
|
|
29704
|
-
const lines = [];
|
|
29705
|
-
for await (const rawLine of streamLines(path)) {
|
|
29706
|
-
const line = parseLine(rawLine);
|
|
29707
|
-
if (line !== null) lines.push(line);
|
|
29708
|
-
}
|
|
29709
|
-
return lines;
|
|
29710
|
-
}
|
|
29711
29605
|
function parseLine(rawLine) {
|
|
29712
29606
|
let value;
|
|
29713
29607
|
try {
|
|
@@ -966,6 +966,189 @@ function scrubEnvValues(text, scrubMap) {
|
|
|
966
966
|
return out;
|
|
967
967
|
}
|
|
968
968
|
//#endregion
|
|
969
|
+
//#region src/runtime/literal-scrub.ts
|
|
970
|
+
/**
|
|
971
|
+
* Patterns are listed in roughly descending confidence — a hit on `clock-hms`
|
|
972
|
+
* is almost certainly bad; a hit on `unix-epoch-sec` (`1[0-9]{9}`) gates on
|
|
973
|
+
* the value starting with `1`, which empirically rules out most SKU / order-id
|
|
974
|
+
* false positives while still catching epoch seconds in the 2001-2033 window.
|
|
975
|
+
*
|
|
976
|
+
* Relative-time labels ("just now", "N minutes ago", "N分前") are the same
|
|
977
|
+
* class of problem as wall-clock literals: the page shows them, Claude
|
|
978
|
+
* captures them, and they're stale before the test ever replays. We only
|
|
979
|
+
* catch the unambiguous variants — bare "now" or "minute" would false-fire
|
|
980
|
+
* on routine UI copy.
|
|
981
|
+
*/
|
|
982
|
+
const UNSTABLE_PATTERNS = [
|
|
983
|
+
{
|
|
984
|
+
id: "clock-hms",
|
|
985
|
+
pattern: /\b\d{2}:\d{2}:\d{2}\b/,
|
|
986
|
+
label: "clock time HH:MM:SS"
|
|
987
|
+
},
|
|
988
|
+
{
|
|
989
|
+
id: "iso-datetime",
|
|
990
|
+
pattern: /\b\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/,
|
|
991
|
+
label: "ISO datetime"
|
|
992
|
+
},
|
|
993
|
+
{
|
|
994
|
+
id: "iso-date",
|
|
995
|
+
pattern: /\b\d{4}-\d{2}-\d{2}\b/,
|
|
996
|
+
label: "ISO date YYYY-MM-DD"
|
|
997
|
+
},
|
|
998
|
+
{
|
|
999
|
+
id: "unix-epoch-sec",
|
|
1000
|
+
pattern: /\b1[0-9]{9}\b/,
|
|
1001
|
+
label: "Unix epoch seconds"
|
|
1002
|
+
},
|
|
1003
|
+
{
|
|
1004
|
+
id: "unix-epoch-ms",
|
|
1005
|
+
pattern: /\b1[0-9]{12}\b/,
|
|
1006
|
+
label: "Unix epoch milliseconds"
|
|
1007
|
+
},
|
|
1008
|
+
{
|
|
1009
|
+
id: "relative-time-en",
|
|
1010
|
+
pattern: /\b\d+\s+(second|minute|hour|day|week|month|year)s?\s+ago\b/i,
|
|
1011
|
+
label: "English relative time (`N <unit> ago`)"
|
|
1012
|
+
},
|
|
1013
|
+
{
|
|
1014
|
+
id: "relative-now-en",
|
|
1015
|
+
pattern: /\bjust\s+now\b/i,
|
|
1016
|
+
label: "English `just now`"
|
|
1017
|
+
},
|
|
1018
|
+
{
|
|
1019
|
+
id: "relative-time-ja",
|
|
1020
|
+
pattern: /\d+\s*(秒|分|時間|日|週間|か月|ヶ月|年)前/,
|
|
1021
|
+
label: "Japanese relative time (`N<unit>前`)"
|
|
1022
|
+
},
|
|
1023
|
+
{
|
|
1024
|
+
id: "relative-now-ja",
|
|
1025
|
+
pattern: /たった今/,
|
|
1026
|
+
label: "Japanese `たった今`"
|
|
1027
|
+
},
|
|
1028
|
+
{
|
|
1029
|
+
id: "ja-date-full",
|
|
1030
|
+
pattern: /\d{4}年\d{1,2}月\d{1,2}日/,
|
|
1031
|
+
label: "Japanese date YYYY年M月D日"
|
|
1032
|
+
},
|
|
1033
|
+
{
|
|
1034
|
+
id: "ja-date-md",
|
|
1035
|
+
pattern: /(?<!年)(?<!\d)\d{1,2}月\d{1,2}日(?![間目])/,
|
|
1036
|
+
label: "Japanese date M月D日"
|
|
1037
|
+
}
|
|
1038
|
+
];
|
|
1039
|
+
/**
|
|
1040
|
+
* Opaque machine-generated id shapes. Only the unambiguous forms are listed:
|
|
1041
|
+
* long digit or hex runs also appear in addresses a step legitimately names
|
|
1042
|
+
* (a date path, a numeric tenant id, a content hash), and a false hit here
|
|
1043
|
+
* dead-ends that step — the guard in `claude/invoke.ts` blocks the `open`
|
|
1044
|
+
* and the navigate check below drops the action. A missed id still fails
|
|
1045
|
+
* verification at record time, so the trade is deliberate.
|
|
1046
|
+
*
|
|
1047
|
+
* Boundaries are alphanumeric lookarounds, not `\b`, so `item_01H8…`
|
|
1048
|
+
* (word-char `_`) is still caught; `i` accepts lowercase ULIDs (a 26-letter
|
|
1049
|
+
* lowercase run inside a URL is rare enough to risk). Ids that come from
|
|
1050
|
+
* `${ENV_VAR}` values are symbolised before either check runs, never match.
|
|
1051
|
+
*/
|
|
1052
|
+
const OPAQUE_ID_PATTERNS = [{
|
|
1053
|
+
id: "ulid",
|
|
1054
|
+
pattern: /(?<![0-9A-Za-z])[0-9A-HJKMNP-TV-Z]{26}(?![0-9A-Za-z])/i,
|
|
1055
|
+
label: "ULID"
|
|
1056
|
+
}, {
|
|
1057
|
+
id: "uuid",
|
|
1058
|
+
pattern: /(?<![0-9A-Za-z])[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?![0-9A-Za-z])/i,
|
|
1059
|
+
label: "UUID"
|
|
1060
|
+
}];
|
|
1061
|
+
/** First opaque-id hit in `text`, or null. Also used by the `open` guard in `claude/invoke.ts`. */
|
|
1062
|
+
function findOpaqueIdSegment(text) {
|
|
1063
|
+
for (const p of OPAQUE_ID_PATTERNS) {
|
|
1064
|
+
const m = text.match(p.pattern);
|
|
1065
|
+
if (m) return {
|
|
1066
|
+
patternId: p.id,
|
|
1067
|
+
match: m[0]
|
|
1068
|
+
};
|
|
1069
|
+
}
|
|
1070
|
+
return null;
|
|
1071
|
+
}
|
|
1072
|
+
/**
|
|
1073
|
+
* Inspect a single action and return every (field, pattern) pair that
|
|
1074
|
+
* fired. An empty array means the action is safe to keep.
|
|
1075
|
+
*/
|
|
1076
|
+
function detectUnstableLiterals(action) {
|
|
1077
|
+
const fields = [
|
|
1078
|
+
["locator.value", action.locator?.value],
|
|
1079
|
+
["locator.name", action.locator?.by === "role" ? action.locator.name : void 0],
|
|
1080
|
+
["target.value", action.target?.value],
|
|
1081
|
+
["value", action.value],
|
|
1082
|
+
["label", action.label],
|
|
1083
|
+
["observation", action.observation]
|
|
1084
|
+
];
|
|
1085
|
+
const hits = [];
|
|
1086
|
+
if (action.action === "navigate" && typeof action.value === "string") {
|
|
1087
|
+
const opaque = findOpaqueIdSegment(action.value);
|
|
1088
|
+
if (opaque) hits.push({
|
|
1089
|
+
field: "value",
|
|
1090
|
+
...opaque
|
|
1091
|
+
});
|
|
1092
|
+
}
|
|
1093
|
+
for (const [field, raw] of fields) {
|
|
1094
|
+
if (typeof raw !== "string" || raw.length === 0) continue;
|
|
1095
|
+
for (const p of UNSTABLE_PATTERNS) {
|
|
1096
|
+
const m = raw.match(p.pattern);
|
|
1097
|
+
if (m) hits.push({
|
|
1098
|
+
field,
|
|
1099
|
+
patternId: p.id,
|
|
1100
|
+
match: m[0]
|
|
1101
|
+
});
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
return hits;
|
|
1105
|
+
}
|
|
1106
|
+
/**
|
|
1107
|
+
* Walk every recorded action and split it into kept / dropped buckets. A
|
|
1108
|
+
* `snapshot` action is treated specially: its `observation` field is just a
|
|
1109
|
+
* comment in the generated script, so we keep the action even if its
|
|
1110
|
+
* `observation` carries an unstable literal — the comment will be wrong but
|
|
1111
|
+
* the script will still run. All other actions get dropped on any hit
|
|
1112
|
+
* because their locator / `value` would otherwise drive an unreproducible
|
|
1113
|
+
* interaction.
|
|
1114
|
+
*/
|
|
1115
|
+
function scrubUnstableActions(actions) {
|
|
1116
|
+
const kept = [];
|
|
1117
|
+
const dropped = [];
|
|
1118
|
+
for (let i = 0; i < actions.length; i++) {
|
|
1119
|
+
const action = actions[i];
|
|
1120
|
+
const hits = detectUnstableLiterals(action);
|
|
1121
|
+
if (hits.length === 0) {
|
|
1122
|
+
kept.push(action);
|
|
1123
|
+
continue;
|
|
1124
|
+
}
|
|
1125
|
+
if (action.action === "snapshot" && hits.every((h) => h.field === "observation")) {
|
|
1126
|
+
kept.push(action);
|
|
1127
|
+
continue;
|
|
1128
|
+
}
|
|
1129
|
+
dropped.push({
|
|
1130
|
+
index: i,
|
|
1131
|
+
action,
|
|
1132
|
+
hits
|
|
1133
|
+
});
|
|
1134
|
+
}
|
|
1135
|
+
return {
|
|
1136
|
+
kept,
|
|
1137
|
+
dropped
|
|
1138
|
+
};
|
|
1139
|
+
}
|
|
1140
|
+
/**
|
|
1141
|
+
* Human-readable summary of one drop, suitable for `log.warn`. The format
|
|
1142
|
+
* mirrors `replay-validate`'s drop line so both sources of warnings look
|
|
1143
|
+
* uniform in the trace output.
|
|
1144
|
+
*/
|
|
1145
|
+
function formatUnstableDrop(drop) {
|
|
1146
|
+
const { action, hits } = drop;
|
|
1147
|
+
const ids = [...new Set(hits.map((h) => h.patternId))].join(", ");
|
|
1148
|
+
const samples = hits.map((h) => `${h.field}="${h.match}"`).join(", ");
|
|
1149
|
+
return `${`${action.action}${action.assert ? " " + action.assert : ""}`}: contains unstable literal (${ids}) — ${samples}`;
|
|
1150
|
+
}
|
|
1151
|
+
//#endregion
|
|
969
1152
|
//#region src/claude/native-binary.ts
|
|
970
1153
|
const require = createRequire(import.meta.url);
|
|
971
1154
|
/**
|
|
@@ -1190,6 +1373,11 @@ async function invokeClaudeStreaming(options, onEvent) {
|
|
|
1190
1373
|
decision: "block",
|
|
1191
1374
|
reason: "Do not suppress errors on `agent-browser` commands. Remove `|| true`, `|| :`, `2>/dev/null`, `; true`, and similar redirects so ccqa can detect failures and roll back unsuccessful attempts. Run the command standalone and let it surface its exit code."
|
|
1192
1375
|
};
|
|
1376
|
+
const runProducedUrl = findRunProducedOpenUrl(cmd, envScrubMap);
|
|
1377
|
+
if (runProducedUrl !== null) return {
|
|
1378
|
+
decision: "block",
|
|
1379
|
+
reason: `Do not open ${runProducedUrl} — that address was produced by this run (the id in it belongs to a record this run created), so the generated test would open a record later runs do not have. Reach the page the way a person does: click through from where the run already is. If the spec's instruction really names this exact address, put the id in a profile variable so it survives replay.`
|
|
1380
|
+
};
|
|
1193
1381
|
}
|
|
1194
1382
|
const assertMarker = relaxAbConstraints ? null : extractCcqaAssertFromBashCommand(cmd);
|
|
1195
1383
|
const ab = relaxAbConstraints ? null : extractAbActionFromBashCommand(cmd) ?? (assertMarker !== null ? extractObservationAbAction(cmd) : null);
|
|
@@ -1338,14 +1526,22 @@ function shellTokenize(s) {
|
|
|
1338
1526
|
if (cur) tokens.push(cur);
|
|
1339
1527
|
return tokens;
|
|
1340
1528
|
}
|
|
1341
|
-
/**
|
|
1342
|
-
|
|
1529
|
+
/**
|
|
1530
|
+
* Positional tokens of an `agent-browser [flags] <subcommand> [args...]`
|
|
1531
|
+
* command string, value-taking flags dropped. Empty when `cmd` is not an
|
|
1532
|
+
* agent-browser call.
|
|
1533
|
+
*/
|
|
1534
|
+
function abPositionalTokens(cmd) {
|
|
1343
1535
|
const abIdx = cmd.indexOf("agent-browser");
|
|
1344
|
-
if (abIdx === -1) return
|
|
1536
|
+
if (abIdx === -1) return [];
|
|
1345
1537
|
const parts = shellTokenize(cmd.slice(abIdx + 13).trim());
|
|
1346
1538
|
let i = 0;
|
|
1347
1539
|
while (i < parts.length && parts[i].startsWith("-")) i += 2;
|
|
1348
|
-
return parts
|
|
1540
|
+
return parts.slice(i);
|
|
1541
|
+
}
|
|
1542
|
+
/** Extracts the subcommand from an `agent-browser [flags] <subcommand> [args...]` command string. */
|
|
1543
|
+
function extractAbSubcommand(cmd) {
|
|
1544
|
+
return abPositionalTokens(cmd)[0] ?? null;
|
|
1349
1545
|
}
|
|
1350
1546
|
/** Returns true if the agent-browser subcommand is blocked (eval/js/find/etc). */
|
|
1351
1547
|
function isBlockedAbSubcommand(cmd) {
|
|
@@ -1411,6 +1607,20 @@ function hasRefSelector(cmd) {
|
|
|
1411
1607
|
return false;
|
|
1412
1608
|
}
|
|
1413
1609
|
/**
|
|
1610
|
+
* Detect `agent-browser open <url>` whose address carries an opaque
|
|
1611
|
+
* machine-generated id (ULID / UUID) that no `${ENV_VAR}` value accounts
|
|
1612
|
+
* for. Such an address was produced by the run itself — this is the
|
|
1613
|
+
* mechanical form of the trace prompt's "Open only a URL the step names"
|
|
1614
|
+
* rule, which alone does not stop every model. Returns the env-scrubbed URL
|
|
1615
|
+
* for the block message, or null when the command is fine.
|
|
1616
|
+
*/
|
|
1617
|
+
function findRunProducedOpenUrl(cmd, envScrubMap) {
|
|
1618
|
+
const [sub, url] = abPositionalTokens(cmd);
|
|
1619
|
+
if (sub !== "open" || !url) return null;
|
|
1620
|
+
const scrubbed = scrubEnvValues(url, envScrubMap);
|
|
1621
|
+
return findOpaqueIdSegment(scrubbed) !== null ? scrubbed : null;
|
|
1622
|
+
}
|
|
1623
|
+
/**
|
|
1414
1624
|
* Split `cmd` into shell statements at unquoted separators (`;`, `|`, `&`,
|
|
1415
1625
|
* newline; consecutive separator chars like `&&` count once). String
|
|
1416
1626
|
* literals are honoured so `fill "a;b"` stays a single statement. This is a
|
|
@@ -2219,4 +2429,4 @@ function clamp(n, lo, hi) {
|
|
|
2219
2429
|
return n;
|
|
2220
2430
|
}
|
|
2221
2431
|
//#endregion
|
|
2222
|
-
export {
|
|
2432
|
+
export { isIncludeStep as $, meta as A, expandActionSteps as B, blank as C, header as D, fix as E, timedPhase as F, parseBlockSpec as G, isExpandedActionStep as H, warn as I, AGENT_BROWSER_TARGET as J, parseTestSpec as K, withBuffer as L, progressEnd as M, run as N, hint as O, step as P, TargetIdSchema as Q, CREDENTIAL_ENV_KEYS as R, promoteMarkedAssert as S, error as T, isExpandedJudgeByLlmStep as U, expandSpec as V, isJudgeBody as W, SessionNameSchema as X, DEFAULT_SPEC_MODE as Y, SpecModeSchema as Z, scrubUnstableActions as _, truncate as a, scrubEnvValues as b, toAgentBrowserArgs as c, surfaceAxisAside as d, isParamRequired as et, surfaceDefinitionBlock as f, formatUnstableDrop as g, withCostTally as h, isObject as i, resolveEnvRefs as it, progress as j, info as k, numberLines as l, readCostTally as m, diagnose as n, envRefsToJsExpression as nt, describeLocator as o, invokeClaudeStreaming as p, tryParseTestSpec as q, extractJsonCandidates as r, iterEnvRefNames as rt, locatorToSelector as s, clamp as t, bracedRefsToJsExpression as tt, outputLanguageBlock as u, buildProseEnvScrubMap as v, emitRaw as w, parseAbActionLine as x, buildSpecEnvScrub as y, collectIncludedBlockNames as z };
|
|
@@ -35,9 +35,9 @@ declare const RunSchema: z.ZodObject<{
|
|
|
35
35
|
running: "running";
|
|
36
36
|
}>;
|
|
37
37
|
kind: z.ZodDefault<z.ZodEnum<{
|
|
38
|
+
record: "record";
|
|
38
39
|
run: "run";
|
|
39
40
|
drift: "drift";
|
|
40
|
-
record: "record";
|
|
41
41
|
}>>;
|
|
42
42
|
drift: z.ZodDefault<z.ZodNullable<z.ZodObject<{
|
|
43
43
|
specs: z.ZodNumber;
|
|
@@ -640,8 +640,8 @@ declare const ReportSpecResultSchema: z.ZodObject<{
|
|
|
640
640
|
title: z.ZodNullable<z.ZodString>;
|
|
641
641
|
target: z.ZodOptional<z.ZodString>;
|
|
642
642
|
mode: z.ZodOptional<z.ZodEnum<{
|
|
643
|
-
live: "live";
|
|
644
643
|
deterministic: "deterministic";
|
|
644
|
+
live: "live";
|
|
645
645
|
}>>;
|
|
646
646
|
status: z.ZodEnum<{
|
|
647
647
|
passed: "passed";
|
|
@@ -811,9 +811,9 @@ type ReportSpecResult = z.infer<typeof ReportSpecResultSchema>;
|
|
|
811
811
|
declare const RunReportDataSchema: z.ZodObject<{
|
|
812
812
|
schemaVersion: z.ZodLiteral<1>;
|
|
813
813
|
kind: z.ZodDefault<z.ZodEnum<{
|
|
814
|
+
record: "record";
|
|
814
815
|
run: "run";
|
|
815
816
|
drift: "drift";
|
|
816
|
-
record: "record";
|
|
817
817
|
}>>;
|
|
818
818
|
createdAt: z.ZodString;
|
|
819
819
|
runId: z.ZodNullable<z.ZodString>;
|
|
@@ -854,8 +854,8 @@ declare const RunReportDataSchema: z.ZodObject<{
|
|
|
854
854
|
title: z.ZodNullable<z.ZodString>;
|
|
855
855
|
target: z.ZodOptional<z.ZodString>;
|
|
856
856
|
mode: z.ZodOptional<z.ZodEnum<{
|
|
857
|
-
live: "live";
|
|
858
857
|
deterministic: "deterministic";
|
|
858
|
+
live: "live";
|
|
859
859
|
}>>;
|
|
860
860
|
status: z.ZodEnum<{
|
|
861
861
|
passed: "passed";
|
package/dist/package.json
CHANGED
package/dist/runtime/judge.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as truncate, p as invokeClaudeStreaming, r as extractJsonCandidates } from "../diagnose-
|
|
1
|
+
import { a as truncate, p as invokeClaudeStreaming, r as extractJsonCandidates } from "../diagnose-CZms9Cer.mjs";
|
|
2
2
|
//#region src/runtime/judge.ts
|
|
3
3
|
const SYSTEM_PROMPT = [
|
|
4
4
|
"You decide whether a claim holds for a piece of text taken from a web page under test.",
|