ccqa 1.3.1 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/bin/ccqa.mjs +825 -350
- package/dist/hub-client/index.d.mts +41 -0
- package/dist/hub-client/index.mjs +10 -0
- package/dist/package.json +1 -1
- package/package.json +1 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -11,10 +11,10 @@ import { basename, dirname, isAbsolute, join, normalize, posix, relative, resolv
|
|
|
11
11
|
import { parse, stringify } from "yaml";
|
|
12
12
|
import { ZodError, z } from "zod";
|
|
13
13
|
import { execFile, spawn, spawnSync } from "node:child_process";
|
|
14
|
-
import { query } from "@anthropic-ai/claude-agent-sdk";
|
|
14
|
+
import { createSdkMcpServer, query, tool } from "@anthropic-ai/claude-agent-sdk";
|
|
15
15
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
16
|
-
import { promisify } from "node:util";
|
|
17
16
|
import { createCipheriv, createDecipheriv, createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
|
|
17
|
+
import { promisify } from "node:util";
|
|
18
18
|
import { gunzipSync, gzipSync } from "node:zlib";
|
|
19
19
|
import { createInterface } from "node:readline";
|
|
20
20
|
import { createInterface as createInterface$1 } from "node:readline/promises";
|
|
@@ -1287,7 +1287,7 @@ function warnOnceIfNativeBinaryMissing() {
|
|
|
1287
1287
|
if (missing) warn(missingNativeBinaryMessage(missing));
|
|
1288
1288
|
}
|
|
1289
1289
|
async function invokeClaudeStreaming(options, onEvent) {
|
|
1290
|
-
const { prompt, systemPrompt, allowedTools, disableBuiltinTools = false, disableThinking = false, maxTurns, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, relaxAbConstraints = false } = options;
|
|
1290
|
+
const { prompt, systemPrompt, allowedTools, disableBuiltinTools = false, disableThinking = false, mcpServers, maxTurns, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, relaxAbConstraints = false } = options;
|
|
1291
1291
|
const resolvedModel = resolveModel(model);
|
|
1292
1292
|
const hasEndpointEnv = Object.keys(resolveEndpointEnv()).length > 0;
|
|
1293
1293
|
const mergedEnv = env || hasEndpointEnv ? {
|
|
@@ -1309,6 +1309,7 @@ async function invokeClaudeStreaming(options, onEvent) {
|
|
|
1309
1309
|
...resolvedModel ? { model: resolvedModel } : {},
|
|
1310
1310
|
...cwd ? { cwd } : {},
|
|
1311
1311
|
...mergedEnv ? { env: mergedEnv } : {},
|
|
1312
|
+
...mcpServers ? { mcpServers } : {},
|
|
1312
1313
|
...disableBuiltinTools ? { tools: [] } : {},
|
|
1313
1314
|
...disableThinking ? { thinking: { type: "disabled" } } : {},
|
|
1314
1315
|
hooks: onAbAction || onAbActionFailed ? {
|
|
@@ -2284,171 +2285,6 @@ async function checkSpec(target, opts) {
|
|
|
2284
2285
|
};
|
|
2285
2286
|
}
|
|
2286
2287
|
//#endregion
|
|
2287
|
-
//#region src/drift/affected.ts
|
|
2288
|
-
const execFileP = promisify(execFile);
|
|
2289
|
-
/**
|
|
2290
|
-
* Resolve the base ref to diff against for `ccqa drift --changed`.
|
|
2291
|
-
* Precedence: explicit override > GITHUB_BASE_REF > origin/main.
|
|
2292
|
-
*/
|
|
2293
|
-
function resolveBaseRef(explicit) {
|
|
2294
|
-
if (explicit && explicit.length > 0) return explicit;
|
|
2295
|
-
const ghBase = process.env["GITHUB_BASE_REF"];
|
|
2296
|
-
if (ghBase && ghBase.length > 0) return ghBase.startsWith("origin/") ? ghBase : `origin/${ghBase}`;
|
|
2297
|
-
return "origin/main";
|
|
2298
|
-
}
|
|
2299
|
-
/**
|
|
2300
|
-
* Run `git diff --name-status base...HEAD` from `cwd` and return one entry per
|
|
2301
|
-
* changed file. Renames are reported under their NEW path with status
|
|
2302
|
-
* "renamed" — the OLD path is dropped because the spec mapping is against the
|
|
2303
|
-
* post-rename layout.
|
|
2304
|
-
*
|
|
2305
|
-
* Paths are re-rooted to be relative to `cwd`, not the git repo root. In a
|
|
2306
|
-
* monorepo where `cwd` is a sub-package (e.g. `apps/foo`), git emits paths
|
|
2307
|
-
* relative to the repo root, but specs declare relatedPaths relative to
|
|
2308
|
-
* their own package. Changes outside `cwd` are kept under their repo-root
|
|
2309
|
-
* path and flagged `outsideCwd` — they only scope a spec in when the spec
|
|
2310
|
-
* explicitly declares a repo-root-relative glob, so an unrelated PR can
|
|
2311
|
-
* never accidentally match the app-relative globs.
|
|
2312
|
-
*/
|
|
2313
|
-
async function getChangedFiles(base, cwd) {
|
|
2314
|
-
const [{ stdout: rootOut }, { stdout: diffOut }] = await Promise.all([execFileP("git", ["rev-parse", "--show-toplevel"], { cwd }), execFileP("git", [
|
|
2315
|
-
"diff",
|
|
2316
|
-
"--name-status",
|
|
2317
|
-
"-M",
|
|
2318
|
-
`${base}...HEAD`
|
|
2319
|
-
], {
|
|
2320
|
-
cwd,
|
|
2321
|
-
maxBuffer: 32 * 1024 * 1024
|
|
2322
|
-
})]);
|
|
2323
|
-
return rerootChangedFiles(parseGitDiffOutput(diffOut), rootOut.trim(), cwd);
|
|
2324
|
-
}
|
|
2325
|
-
/**
|
|
2326
|
-
* Convert paths in `entries` from git-repo-root relative to `cwd` relative.
|
|
2327
|
-
* Entries outside `cwd` keep their repo-root path and are flagged
|
|
2328
|
-
* `outsideCwd`. Exported for unit tests.
|
|
2329
|
-
*/
|
|
2330
|
-
function rerootChangedFiles(entries, repoRoot, cwd) {
|
|
2331
|
-
const prefix = relative(repoRoot, cwd);
|
|
2332
|
-
if (!prefix) return entries;
|
|
2333
|
-
const out = [];
|
|
2334
|
-
for (const e of entries) {
|
|
2335
|
-
const rel = relative(prefix, e.path);
|
|
2336
|
-
if (rel.startsWith("..") || rel === "") out.push({
|
|
2337
|
-
...e,
|
|
2338
|
-
outsideCwd: true
|
|
2339
|
-
});
|
|
2340
|
-
else out.push({
|
|
2341
|
-
...e,
|
|
2342
|
-
path: rel
|
|
2343
|
-
});
|
|
2344
|
-
}
|
|
2345
|
-
return out;
|
|
2346
|
-
}
|
|
2347
|
-
function parseGitDiffOutput(stdout) {
|
|
2348
|
-
const out = [];
|
|
2349
|
-
for (const line of stdout.split("\n")) {
|
|
2350
|
-
if (!line.trim()) continue;
|
|
2351
|
-
const parts = line.split(" ");
|
|
2352
|
-
const code = parts[0];
|
|
2353
|
-
if (!code) continue;
|
|
2354
|
-
if (code.startsWith("R")) {
|
|
2355
|
-
const newPath = parts[2];
|
|
2356
|
-
if (newPath) out.push({
|
|
2357
|
-
path: newPath,
|
|
2358
|
-
status: "renamed"
|
|
2359
|
-
});
|
|
2360
|
-
continue;
|
|
2361
|
-
}
|
|
2362
|
-
if (code.startsWith("C")) {
|
|
2363
|
-
const newPath = parts[2];
|
|
2364
|
-
if (newPath) out.push({
|
|
2365
|
-
path: newPath,
|
|
2366
|
-
status: "added"
|
|
2367
|
-
});
|
|
2368
|
-
continue;
|
|
2369
|
-
}
|
|
2370
|
-
const path = parts[1];
|
|
2371
|
-
if (!path) continue;
|
|
2372
|
-
switch (code[0]) {
|
|
2373
|
-
case "A":
|
|
2374
|
-
out.push({
|
|
2375
|
-
path,
|
|
2376
|
-
status: "added"
|
|
2377
|
-
});
|
|
2378
|
-
break;
|
|
2379
|
-
case "M":
|
|
2380
|
-
case "T":
|
|
2381
|
-
out.push({
|
|
2382
|
-
path,
|
|
2383
|
-
status: "modified"
|
|
2384
|
-
});
|
|
2385
|
-
break;
|
|
2386
|
-
case "D":
|
|
2387
|
-
out.push({
|
|
2388
|
-
path,
|
|
2389
|
-
status: "deleted"
|
|
2390
|
-
});
|
|
2391
|
-
break;
|
|
2392
|
-
default: out.push({
|
|
2393
|
-
path,
|
|
2394
|
-
status: "modified"
|
|
2395
|
-
});
|
|
2396
|
-
}
|
|
2397
|
-
}
|
|
2398
|
-
return out;
|
|
2399
|
-
}
|
|
2400
|
-
function stripLeadingDotSlash(s) {
|
|
2401
|
-
return s.startsWith("./") ? s.slice(2) : s;
|
|
2402
|
-
}
|
|
2403
|
-
const REGEX_CACHE = /* @__PURE__ */ new Map();
|
|
2404
|
-
/** Compiles `pattern` to a RegExp, memoized so repeated `--changed` matches don't re-build. */
|
|
2405
|
-
function compileGlob(pattern) {
|
|
2406
|
-
const cached = REGEX_CACHE.get(pattern);
|
|
2407
|
-
if (cached) return cached;
|
|
2408
|
-
const compiled = globToRegExp(stripLeadingDotSlash(pattern));
|
|
2409
|
-
REGEX_CACHE.set(pattern, compiled);
|
|
2410
|
-
return compiled;
|
|
2411
|
-
}
|
|
2412
|
-
function globToRegExp(pattern) {
|
|
2413
|
-
let re = "^";
|
|
2414
|
-
let i = 0;
|
|
2415
|
-
while (i < pattern.length) {
|
|
2416
|
-
const ch = pattern[i];
|
|
2417
|
-
if (ch === "?") {
|
|
2418
|
-
re += "[^/]";
|
|
2419
|
-
i++;
|
|
2420
|
-
continue;
|
|
2421
|
-
}
|
|
2422
|
-
if (ch !== "*") {
|
|
2423
|
-
re += /[.+^${}()|[\]\\]/.test(ch) ? "\\" + ch : ch;
|
|
2424
|
-
i++;
|
|
2425
|
-
continue;
|
|
2426
|
-
}
|
|
2427
|
-
if (pattern[i + 1] !== "*") {
|
|
2428
|
-
re += "[^/]*";
|
|
2429
|
-
i++;
|
|
2430
|
-
continue;
|
|
2431
|
-
}
|
|
2432
|
-
const hasLeadingSlash = re.endsWith("/");
|
|
2433
|
-
const hasTrailingSlash = pattern[i + 2] === "/";
|
|
2434
|
-
if (hasLeadingSlash) re = re.slice(0, -1);
|
|
2435
|
-
if (hasLeadingSlash || hasTrailingSlash) re += "(?:/?.*)?";
|
|
2436
|
-
else re += ".*";
|
|
2437
|
-
i += hasTrailingSlash ? 3 : 2;
|
|
2438
|
-
}
|
|
2439
|
-
return new RegExp(re + "$");
|
|
2440
|
-
}
|
|
2441
|
-
/**
|
|
2442
|
-
* Returns true if `changedPath` is covered by any of `relatedPaths`. An empty
|
|
2443
|
-
* `relatedPaths` returns false — callers handle the "unscoped spec" case
|
|
2444
|
-
* separately (treat the spec as always-affected) before calling this.
|
|
2445
|
-
*/
|
|
2446
|
-
function isPathAffectedBy(changedPath, relatedPaths) {
|
|
2447
|
-
const stripped = stripLeadingDotSlash(changedPath);
|
|
2448
|
-
for (const pattern of relatedPaths) if (compileGlob(pattern).test(stripped)) return true;
|
|
2449
|
-
return false;
|
|
2450
|
-
}
|
|
2451
|
-
//#endregion
|
|
2452
2288
|
//#region src/drift/auth.ts
|
|
2453
2289
|
/**
|
|
2454
2290
|
* Probe whether the host has any credential the Anthropic SDK can pick up:
|
|
@@ -3153,6 +2989,10 @@ const ReportSpecResultSchema = z.object({
|
|
|
3153
2989
|
assertions: z.array(ReportAssertionSchema).nullable(),
|
|
3154
2990
|
analysis: FailureAnalysisSchema.nullable(),
|
|
3155
2991
|
analysisSkipped: z.string().nullable(),
|
|
2992
|
+
analysisBase: z.object({
|
|
2993
|
+
ref: z.string(),
|
|
2994
|
+
sha: z.string()
|
|
2995
|
+
}).nullable().optional(),
|
|
3156
2996
|
driftIssues: z.array(DraftIssueSchema).nullable(),
|
|
3157
2997
|
failureLogExcerpt: z.string().nullable(),
|
|
3158
2998
|
diffExcerpt: z.string().nullable(),
|
|
@@ -3161,15 +3001,33 @@ const ReportSpecResultSchema = z.object({
|
|
|
3161
3001
|
artifacts: z.array(ReportArtifactSchema).optional(),
|
|
3162
3002
|
liveRun: LiveReportRunSchema.nullable()
|
|
3163
3003
|
});
|
|
3004
|
+
/**
|
|
3005
|
+
* Which rule produced the analysis baseline: "explicit" (a value was
|
|
3006
|
+
* passed), "github-base-ref" (derived from a pull_request event), or
|
|
3007
|
+
* "last-green" (per-spec baselines from the hub ledger — `baseSha` is then
|
|
3008
|
+
* null and each analyzed row carries its own `analysisBase`). Lets accuracy
|
|
3009
|
+
* numbers be stratified by baseline provenance. Single source of truth —
|
|
3010
|
+
* `src/run/git-context.ts`'s `BaseSource` type and the hub's PATCH schema
|
|
3011
|
+
* both derive from this.
|
|
3012
|
+
*/
|
|
3013
|
+
const BaseSourceSchema = z.enum([
|
|
3014
|
+
"explicit",
|
|
3015
|
+
"github-base-ref",
|
|
3016
|
+
"last-green"
|
|
3017
|
+
]);
|
|
3018
|
+
/** The report envelope's git block; also referenced by the hub's PATCH reportMeta schema. */
|
|
3019
|
+
const GitEnvelopeSchema = z.object({
|
|
3020
|
+
head: z.string().nullable(),
|
|
3021
|
+
base: z.string().nullable(),
|
|
3022
|
+
baseSha: z.string().nullable().optional(),
|
|
3023
|
+
baseSource: BaseSourceSchema.nullable().optional()
|
|
3024
|
+
});
|
|
3164
3025
|
const RunReportDataSchema = z.object({
|
|
3165
3026
|
schemaVersion: z.literal(1),
|
|
3166
3027
|
kind: z.enum(["run", "drift"]).default("run"),
|
|
3167
3028
|
createdAt: z.string(),
|
|
3168
3029
|
runId: z.string().nullable(),
|
|
3169
|
-
git:
|
|
3170
|
-
head: z.string().nullable(),
|
|
3171
|
-
base: z.string().nullable()
|
|
3172
|
-
}),
|
|
3030
|
+
git: GitEnvelopeSchema,
|
|
3173
3031
|
model: z.string().nullable(),
|
|
3174
3032
|
language: z.string().nullable().default(null),
|
|
3175
3033
|
promptVersion: z.string(),
|
|
@@ -3294,13 +3152,54 @@ function hashTriageUserPrompt(text) {
|
|
|
3294
3152
|
}
|
|
3295
3153
|
//#endregion
|
|
3296
3154
|
//#region src/report/prompt.ts
|
|
3155
|
+
/**
|
|
3156
|
+
* Fully-qualified name of the on-demand file-diff tool, as the model calls
|
|
3157
|
+
* it. Lives here (not analyze.ts) because the prompt text below references
|
|
3158
|
+
* it — one source of truth. The name is the SDK's `mcp__<server>__<tool>`
|
|
3159
|
+
* composition of the server ("diff") and tool ("changed_file_diff") that
|
|
3160
|
+
* analyze.ts registers; changing either side must keep the two in sync.
|
|
3161
|
+
*/
|
|
3162
|
+
const CHANGED_FILE_DIFF_TOOL = "mcp__diff__changed_file_diff";
|
|
3297
3163
|
function buildFailureAnalysisPrompt(input) {
|
|
3298
|
-
const { script, specYaml, failureLog, liveTranscriptExcerpt, diffPatch, changedFiles, baseRef, driftIssues, outputLanguage = "auto", triageUserPrompt, customPrompt } = input;
|
|
3164
|
+
const { script, specYaml, failureLog, liveTranscriptExcerpt, diffPatch, changedFiles, baseRef, baseSource = null, range = null, driftIssues, outputLanguage = "auto", triageUserPrompt, customPrompt } = input;
|
|
3165
|
+
const lastGreen = baseSource === "last-green";
|
|
3299
3166
|
const triageUserPromptBlock = buildTriageUserPromptBlock(triageUserPrompt);
|
|
3300
3167
|
const customPromptBlock = buildCustomPromptBlock(customPrompt);
|
|
3301
|
-
|
|
3168
|
+
const languageBlock = outputLanguageBlock(outputLanguage, "`reasoning`, `detail`", "label names (TEST_DRIFT, etc.)");
|
|
3169
|
+
const executionBlock = buildExecutionEvidenceBlock(script, failureLog, liveTranscriptExcerpt);
|
|
3170
|
+
const baseLabel = lastGreen ? `this spec's last passing commit${baseRef && baseRef !== "last-green" ? ` (${baseRef})` : ""}` : baseRef ?? "base";
|
|
3171
|
+
const rangeNote = range ? ` — spans ${range.commitCount} commit${range.commitCount === 1 ? "" : "s"} over ${range.days} day${range.days === 1 ? "" : "s"}` : "";
|
|
3172
|
+
let diffBlock;
|
|
3173
|
+
if (diffPatch === null) diffBlock = `## Source changes
|
|
3174
|
+
|
|
3175
|
+
No diff context is available (the base ref could not be resolved, or there are no changes). Classify from the failure log, the spec, and what you can read in the repository — and be correspondingly more conservative: prefer UNKNOWN over a confident SPEC_CHANGE/PRODUCT_BUG call without diff evidence.
|
|
3176
|
+
`;
|
|
3177
|
+
else if (diffPatch.length === 0) diffBlock = `## Source changes since ${baseLabel}${rangeNote}
|
|
3178
|
+
|
|
3179
|
+
### Changed files (name-status)
|
|
3180
|
+
${changedFiles && changedFiles.length > 0 ? changedFiles : "(no changes in range)"}
|
|
3181
|
+
|
|
3182
|
+
No changed file matches this spec's relatedPaths, so no hunks are inlined. "No related change" is a real signal — but before concluding, scan the name-status list for anything that could plausibly reach this spec and fetch its hunk with \`${CHANGED_FILE_DIFF_TOOL}\`.
|
|
3183
|
+
`;
|
|
3184
|
+
else diffBlock = `## Source changes since ${baseLabel}${rangeNote} (git diff, scoped to this spec's relatedPaths, may be truncated)
|
|
3302
3185
|
|
|
3303
|
-
|
|
3186
|
+
### Changed files (name-status)
|
|
3187
|
+
${changedFiles ?? "(unavailable)"}
|
|
3188
|
+
|
|
3189
|
+
### Patch
|
|
3190
|
+
\`\`\`diff
|
|
3191
|
+
${diffPatch}
|
|
3192
|
+
\`\`\`
|
|
3193
|
+
`;
|
|
3194
|
+
const driftBlock = driftIssues && driftIssues.length > 0 ? `## Spec↔code drift audit findings
|
|
3195
|
+
|
|
3196
|
+
A separate read-only audit compared the spec against the current source. Treat these as hints, not verdicts:
|
|
3197
|
+
|
|
3198
|
+
${driftIssues.map((i) => `- [${i.severity}] (${DRAFT_CATEGORY_LABEL[i.category]}${i.stepId ? `, step ${i.stepId}` : ""}) ${i.message}${i.detail ? ` — ${i.detail}` : ""}`).join("\n")}
|
|
3199
|
+
` : "";
|
|
3200
|
+
return `You are analyzing a failing E2E regression test against the source changes since a known-good baseline. Your job is a root-cause CALL, not a fix: decide which of three categories explains the failure, using the source diff as your primary context.
|
|
3201
|
+
|
|
3202
|
+
${languageBlock}## The three categories
|
|
3304
3203
|
|
|
3305
3204
|
The question that separates them: **is the behavior the spec describes still what the product intends?**
|
|
3306
3205
|
|
|
@@ -3317,14 +3216,19 @@ You can call \`Grep\`, \`Glob\`, and \`Read\` against the current repository (po
|
|
|
3317
3216
|
- read the changed files in full when the truncated patch is not enough,
|
|
3318
3217
|
- check whether the element/flow the spec describes still exists in the source.
|
|
3319
3218
|
|
|
3219
|
+
You can also call \`${CHANGED_FILE_DIFF_TOOL}\` with a file path to fetch that file's diff hunk for this run's base...HEAD range. The inline patch below is scoped to this spec's relatedPaths — files OUTSIDE that scope still appear in "Changed files (name-status)" but their hunks are not inlined. Before blaming (or ruling out) such a file, fetch its diff with this tool; Read only shows you its post-change state, not what changed.
|
|
3220
|
+
|
|
3320
3221
|
You have **up to 12 tool turns**. Do NOT write, edit, run shell commands, or hit the network.
|
|
3321
3222
|
|
|
3322
3223
|
## Decision guidance
|
|
3323
3224
|
|
|
3225
|
+
${lastGreen ? `The baseline is the commit where THIS spec last passed, so the range strictly covers the window in which it broke: the cause is either inside these changes or outside the code entirely (flaky timing, environment, an external service, test data). The range may mix several unrelated merges — most of the diff is noise; what matters is the specific change you can tie to the failing step.` : `The baseline is a fixed ref (typically the PR base): the spec is NOT guaranteed to have passed there, so the range is not guaranteed to contain the cause.`}
|
|
3226
|
+
|
|
3324
3227
|
- Diff touches only attributes/identifiers the test selects on (labels, testids, class names, timing) while the user-visible flow is intact → TEST_DRIFT.
|
|
3325
3228
|
- Diff intentionally removes/reworks the UI or flow that a spec step verifies (component deleted, page restructured, copy redefined, feature flag flipped) → SPEC_CHANGE.
|
|
3326
3229
|
- Diff UNINTENTIONALLY breaks behavior the spec still intends — e.g. a refactor that drops a side effect, an inverted condition, a regression hiding inside a cleanup commit — → PRODUCT_BUG, citing the diff hunk as evidence. A product bug is often introduced BY the diff; what separates it from SPEC_CHANGE is intent: does the change read as a deliberate redesign of what the spec verifies, or as collateral damage?
|
|
3327
|
-
- Diff is unrelated to the failing step (or there is no relevant diff) and the test was passing before → lean PRODUCT_BUG; first rule out timing/data flakiness and infrastructure errors (daemon not running, network down, missing credentials) — those read as UNKNOWN with low confidence, not PRODUCT_BUG
|
|
3230
|
+
${lastGreen ? `- No change in the range explains the failing step (after checking the inline patch, the name-status list, and any hunks you fetched) → the cause is outside the code: answer UNKNOWN with low confidence and name the suspected external cause (flaky timing, environment, external service, test data). Do NOT default to PRODUCT_BUG here — under this baseline a product regression must be tied to an in-range change.` : `- Diff is unrelated to the failing step (or there is no relevant diff) and the test was passing before → lean PRODUCT_BUG; first rule out timing/data flakiness and infrastructure errors (daemon not running, network down, missing credentials) — those read as UNKNOWN with low confidence, not PRODUCT_BUG.`}${range ? `
|
|
3231
|
+
- This range spans ${range.commitCount} commit${range.commitCount === 1 ? "" : "s"} over ${range.days} day${range.days === 1 ? "" : "s"}. The wider the range, the more unrelated changes are mixed in: SPEC_CHANGE and TEST_DRIFT still require citing the specific hunk — do not infer intent from the bulk of a large diff, and lower confidence when the evidence is spread thin.` : ""}
|
|
3328
3232
|
- The drift audit findings (when present) flag spec↔code mismatches; an ERROR there usually supports TEST_DRIFT or SPEC_CHANGE over PRODUCT_BUG.
|
|
3329
3233
|
|
|
3330
3234
|
## Sub-diagnosis vocabulary
|
|
@@ -3364,32 +3268,15 @@ Your **final** assistant message must start with \`{\` and end with \`}\` — a
|
|
|
3364
3268
|
- 0.4-0.7: plausible but another category could explain it
|
|
3365
3269
|
- < 0.4: answer UNKNOWN instead of guessing
|
|
3366
3270
|
|
|
3367
|
-
Evidence rules: TEST_DRIFT and SPEC_CHANGE require at least one concrete \`file\` reference (diff hunk or file:line you actually read). PRODUCT_BUG should explain why the diff does NOT account for the failure.
|
|
3271
|
+
Evidence rules: TEST_DRIFT and SPEC_CHANGE require at least one concrete \`file\` reference (diff hunk or file:line you actually read). PRODUCT_BUG should cite the in-range change that unintentionally broke the behavior when one exists; ${lastGreen ? "under this last-green baseline, if no in-range change explains the failure, that is UNKNOWN (external cause), not PRODUCT_BUG" : "when no such change exists, explain why the diff does NOT account for the failure"}.
|
|
3368
3272
|
|
|
3369
3273
|
## Test Spec (spec.yaml)
|
|
3370
3274
|
${specYaml}
|
|
3371
3275
|
|
|
3372
|
-
${
|
|
3276
|
+
${executionBlock}
|
|
3373
3277
|
|
|
3374
|
-
${
|
|
3375
|
-
|
|
3376
|
-
### Changed files (name-status)
|
|
3377
|
-
${changedFiles ?? "(unavailable)"}
|
|
3378
|
-
|
|
3379
|
-
### Patch
|
|
3380
|
-
\`\`\`diff
|
|
3381
|
-
${diffPatch}
|
|
3382
|
-
\`\`\`
|
|
3383
|
-
` : `## Source changes
|
|
3384
|
-
|
|
3385
|
-
No diff context is available (the base ref could not be resolved, or there are no changes). Classify from the failure log, the spec, and what you can read in the repository — and be correspondingly more conservative: prefer UNKNOWN over a confident SPEC_CHANGE/PRODUCT_BUG call without diff evidence.
|
|
3386
|
-
`}
|
|
3387
|
-
${driftIssues && driftIssues.length > 0 ? `## Spec↔code drift audit findings
|
|
3388
|
-
|
|
3389
|
-
A separate read-only audit compared the spec against the current source. Treat these as hints, not verdicts:
|
|
3390
|
-
|
|
3391
|
-
${driftIssues.map((i) => `- [${i.severity}] (${DRAFT_CATEGORY_LABEL[i.category]}${i.stepId ? `, step ${i.stepId}` : ""}) ${i.message}${i.detail ? ` — ${i.detail}` : ""}`).join("\n")}
|
|
3392
|
-
` : ""}`;
|
|
3278
|
+
${diffBlock}
|
|
3279
|
+
${driftBlock}`;
|
|
3393
3280
|
}
|
|
3394
3281
|
/**
|
|
3395
3282
|
* Render the execution-evidence section the model needs to classify the
|
|
@@ -3421,19 +3308,45 @@ ${liveTranscriptExcerpt}`);
|
|
|
3421
3308
|
//#endregion
|
|
3422
3309
|
//#region src/report/analyze.ts
|
|
3423
3310
|
/**
|
|
3311
|
+
* In-process MCP server exposing one tool: the diff hunk of a named changed
|
|
3312
|
+
* file. The inline patch in the prompt is only the relatedPaths-scoped seed;
|
|
3313
|
+
* this is the pull side — the model fetches hunks for files outside that
|
|
3314
|
+
* scope (or truncated inside it) only when it decides they matter, so the
|
|
3315
|
+
* full diff never has to ride in the prompt. Read-only over data already
|
|
3316
|
+
* captured in memory: no shell, no git access granted. The server/tool
|
|
3317
|
+
* names must compose to CHANGED_FILE_DIFF_TOOL (prompt.ts), which is how
|
|
3318
|
+
* the prompt tells the model to call it.
|
|
3319
|
+
*/
|
|
3320
|
+
function buildDiffMcpServer(getFileDiff) {
|
|
3321
|
+
return createSdkMcpServer({
|
|
3322
|
+
name: "diff",
|
|
3323
|
+
version: "1.0.0",
|
|
3324
|
+
tools: [tool("changed_file_diff", "Return the unified diff (base...HEAD) of one changed file from this run's diff range. Works for ANY file listed in 'Changed files (name-status)', including files outside the spec's relatedPaths scope whose hunks are not in the inline patch.", { path: z.string().describe("File path exactly as it appears in the name-status list") }, async ({ path }) => {
|
|
3325
|
+
const hunk = getFileDiff(path);
|
|
3326
|
+
if (hunk) info(` diff tool: ${path}`);
|
|
3327
|
+
return { content: [{
|
|
3328
|
+
type: "text",
|
|
3329
|
+
text: hunk ?? `No diff found for "${path}" in this run's diff range. Check the exact path in the name-status list (paths are relative to the working directory).`
|
|
3330
|
+
}] };
|
|
3331
|
+
})]
|
|
3332
|
+
});
|
|
3333
|
+
}
|
|
3334
|
+
/**
|
|
3424
3335
|
* Classify one failing spec into TEST_DRIFT / SPEC_CHANGE / PRODUCT_BUG /
|
|
3425
3336
|
* UNKNOWN. Same resilience contract as diagnose(): read-only tools, JSON-only
|
|
3426
3337
|
* final message, and any parse failure degrades to UNKNOWN with confidence 0
|
|
3427
3338
|
* rather than throwing — the report must always render.
|
|
3428
3339
|
*/
|
|
3429
|
-
async function analyzeFailure(input, options
|
|
3340
|
+
async function analyzeFailure(input, options) {
|
|
3430
3341
|
const { result: raw, isError } = await invokeClaudeStreaming({
|
|
3431
3342
|
prompt: buildFailureAnalysisPrompt(input),
|
|
3432
3343
|
allowedTools: [
|
|
3433
3344
|
"Read",
|
|
3434
3345
|
"Grep",
|
|
3435
|
-
"Glob"
|
|
3346
|
+
"Glob",
|
|
3347
|
+
CHANGED_FILE_DIFF_TOOL
|
|
3436
3348
|
],
|
|
3349
|
+
mcpServers: { diff: buildDiffMcpServer(options.getFileDiff) },
|
|
3437
3350
|
silenceBashLog: true,
|
|
3438
3351
|
maxTurns: 12,
|
|
3439
3352
|
...options.model ? { model: options.model } : {},
|
|
@@ -3504,15 +3417,194 @@ function normaliseFailureAnalysis(parsed) {
|
|
|
3504
3417
|
} : { detail });
|
|
3505
3418
|
if (evidence.length >= 3) break;
|
|
3506
3419
|
}
|
|
3507
|
-
return {
|
|
3508
|
-
label,
|
|
3509
|
-
confidence,
|
|
3510
|
-
subDiagnosis,
|
|
3511
|
-
headline,
|
|
3512
|
-
recommendation,
|
|
3513
|
-
evidence,
|
|
3514
|
-
reasoning
|
|
3515
|
-
};
|
|
3420
|
+
return {
|
|
3421
|
+
label,
|
|
3422
|
+
confidence,
|
|
3423
|
+
subDiagnosis,
|
|
3424
|
+
headline,
|
|
3425
|
+
recommendation,
|
|
3426
|
+
evidence,
|
|
3427
|
+
reasoning
|
|
3428
|
+
};
|
|
3429
|
+
}
|
|
3430
|
+
//#endregion
|
|
3431
|
+
//#region src/drift/affected.ts
|
|
3432
|
+
const execFileP = promisify(execFile);
|
|
3433
|
+
/**
|
|
3434
|
+
* GITHUB_BASE_REF holds a bare branch name (e.g. "main"); the local checkout
|
|
3435
|
+
* only has it as a remote-tracking ref, so prefix `origin/` unless already
|
|
3436
|
+
* qualified. Shared by `ccqa drift`'s resolveBaseRef and `ccqa run`'s
|
|
3437
|
+
* resolveAnalysisBase so the rule can't drift between them.
|
|
3438
|
+
*/
|
|
3439
|
+
function normalizeGithubBaseRef(ref) {
|
|
3440
|
+
return ref.startsWith("origin/") ? ref : `origin/${ref}`;
|
|
3441
|
+
}
|
|
3442
|
+
/**
|
|
3443
|
+
* Resolve the base ref to diff against for `ccqa drift --changed`.
|
|
3444
|
+
* Precedence: explicit override > GITHUB_BASE_REF > origin/main.
|
|
3445
|
+
*
|
|
3446
|
+
* Note: this is the `ccqa drift` rule. `ccqa run` resolves its baseline via
|
|
3447
|
+
* `src/run/git-context.ts` instead, which has no origin/main fallback — see
|
|
3448
|
+
* the rationale there.
|
|
3449
|
+
*/
|
|
3450
|
+
function resolveBaseRef(explicit) {
|
|
3451
|
+
if (explicit && explicit.length > 0) return explicit;
|
|
3452
|
+
const ghBase = process.env["GITHUB_BASE_REF"];
|
|
3453
|
+
if (ghBase && ghBase.length > 0) return normalizeGithubBaseRef(ghBase);
|
|
3454
|
+
return "origin/main";
|
|
3455
|
+
}
|
|
3456
|
+
/**
|
|
3457
|
+
* Run `git diff --name-status base...HEAD` from `cwd` and return one entry per
|
|
3458
|
+
* changed file. Renames are reported under their NEW path with status
|
|
3459
|
+
* "renamed" — the OLD path is dropped because the spec mapping is against the
|
|
3460
|
+
* post-rename layout.
|
|
3461
|
+
*
|
|
3462
|
+
* Paths are re-rooted to be relative to `cwd`, not the git repo root. In a
|
|
3463
|
+
* monorepo where `cwd` is a sub-package (e.g. `apps/foo`), git emits paths
|
|
3464
|
+
* relative to the repo root, but specs declare relatedPaths relative to
|
|
3465
|
+
* their own package. Changes outside `cwd` are kept under their repo-root
|
|
3466
|
+
* path and flagged `outsideCwd` — they only scope a spec in when the spec
|
|
3467
|
+
* explicitly declares a repo-root-relative glob, so an unrelated PR can
|
|
3468
|
+
* never accidentally match the app-relative globs.
|
|
3469
|
+
*/
|
|
3470
|
+
async function getChangedFiles(base, cwd) {
|
|
3471
|
+
const [{ stdout: rootOut }, { stdout: diffOut }] = await Promise.all([execFileP("git", ["rev-parse", "--show-toplevel"], { cwd }), execFileP("git", [
|
|
3472
|
+
"diff",
|
|
3473
|
+
"--name-status",
|
|
3474
|
+
"-M",
|
|
3475
|
+
`${base}...HEAD`
|
|
3476
|
+
], {
|
|
3477
|
+
cwd,
|
|
3478
|
+
maxBuffer: 32 * 1024 * 1024
|
|
3479
|
+
})]);
|
|
3480
|
+
return rerootChangedFiles(parseGitDiffOutput(diffOut), rootOut.trim(), cwd);
|
|
3481
|
+
}
|
|
3482
|
+
/**
|
|
3483
|
+
* Convert paths in `entries` from git-repo-root relative to `cwd` relative.
|
|
3484
|
+
* Entries outside `cwd` keep their repo-root path and are flagged
|
|
3485
|
+
* `outsideCwd`. Exported for unit tests.
|
|
3486
|
+
*/
|
|
3487
|
+
function rerootChangedFiles(entries, repoRoot, cwd) {
|
|
3488
|
+
const prefix = relative(repoRoot, cwd);
|
|
3489
|
+
if (!prefix) return entries;
|
|
3490
|
+
const out = [];
|
|
3491
|
+
for (const e of entries) {
|
|
3492
|
+
const rel = relative(prefix, e.path);
|
|
3493
|
+
if (rel.startsWith("..") || rel === "") out.push({
|
|
3494
|
+
...e,
|
|
3495
|
+
outsideCwd: true
|
|
3496
|
+
});
|
|
3497
|
+
else out.push({
|
|
3498
|
+
...e,
|
|
3499
|
+
path: rel
|
|
3500
|
+
});
|
|
3501
|
+
}
|
|
3502
|
+
return out;
|
|
3503
|
+
}
|
|
3504
|
+
function parseGitDiffOutput(stdout) {
|
|
3505
|
+
const out = [];
|
|
3506
|
+
for (const line of stdout.split("\n")) {
|
|
3507
|
+
if (!line.trim()) continue;
|
|
3508
|
+
const parts = line.split(" ");
|
|
3509
|
+
const code = parts[0];
|
|
3510
|
+
if (!code) continue;
|
|
3511
|
+
if (code.startsWith("R")) {
|
|
3512
|
+
const newPath = parts[2];
|
|
3513
|
+
if (newPath) out.push({
|
|
3514
|
+
path: newPath,
|
|
3515
|
+
status: "renamed"
|
|
3516
|
+
});
|
|
3517
|
+
continue;
|
|
3518
|
+
}
|
|
3519
|
+
if (code.startsWith("C")) {
|
|
3520
|
+
const newPath = parts[2];
|
|
3521
|
+
if (newPath) out.push({
|
|
3522
|
+
path: newPath,
|
|
3523
|
+
status: "added"
|
|
3524
|
+
});
|
|
3525
|
+
continue;
|
|
3526
|
+
}
|
|
3527
|
+
const path = parts[1];
|
|
3528
|
+
if (!path) continue;
|
|
3529
|
+
switch (code[0]) {
|
|
3530
|
+
case "A":
|
|
3531
|
+
out.push({
|
|
3532
|
+
path,
|
|
3533
|
+
status: "added"
|
|
3534
|
+
});
|
|
3535
|
+
break;
|
|
3536
|
+
case "M":
|
|
3537
|
+
case "T":
|
|
3538
|
+
out.push({
|
|
3539
|
+
path,
|
|
3540
|
+
status: "modified"
|
|
3541
|
+
});
|
|
3542
|
+
break;
|
|
3543
|
+
case "D":
|
|
3544
|
+
out.push({
|
|
3545
|
+
path,
|
|
3546
|
+
status: "deleted"
|
|
3547
|
+
});
|
|
3548
|
+
break;
|
|
3549
|
+
default: out.push({
|
|
3550
|
+
path,
|
|
3551
|
+
status: "modified"
|
|
3552
|
+
});
|
|
3553
|
+
}
|
|
3554
|
+
}
|
|
3555
|
+
return out;
|
|
3556
|
+
}
|
|
3557
|
+
/** Normalize a leading `./` away so diff paths and relatedPaths globs compare. */
|
|
3558
|
+
function stripLeadingDotSlash(s) {
|
|
3559
|
+
return s.startsWith("./") ? s.slice(2) : s;
|
|
3560
|
+
}
|
|
3561
|
+
const REGEX_CACHE = /* @__PURE__ */ new Map();
|
|
3562
|
+
/** Compiles `pattern` to a RegExp, memoized so repeated `--changed` matches don't re-build. */
|
|
3563
|
+
function compileGlob(pattern) {
|
|
3564
|
+
const cached = REGEX_CACHE.get(pattern);
|
|
3565
|
+
if (cached) return cached;
|
|
3566
|
+
const compiled = globToRegExp(stripLeadingDotSlash(pattern));
|
|
3567
|
+
REGEX_CACHE.set(pattern, compiled);
|
|
3568
|
+
return compiled;
|
|
3569
|
+
}
|
|
3570
|
+
function globToRegExp(pattern) {
|
|
3571
|
+
let re = "^";
|
|
3572
|
+
let i = 0;
|
|
3573
|
+
while (i < pattern.length) {
|
|
3574
|
+
const ch = pattern[i];
|
|
3575
|
+
if (ch === "?") {
|
|
3576
|
+
re += "[^/]";
|
|
3577
|
+
i++;
|
|
3578
|
+
continue;
|
|
3579
|
+
}
|
|
3580
|
+
if (ch !== "*") {
|
|
3581
|
+
re += /[.+^${}()|[\]\\]/.test(ch) ? "\\" + ch : ch;
|
|
3582
|
+
i++;
|
|
3583
|
+
continue;
|
|
3584
|
+
}
|
|
3585
|
+
if (pattern[i + 1] !== "*") {
|
|
3586
|
+
re += "[^/]*";
|
|
3587
|
+
i++;
|
|
3588
|
+
continue;
|
|
3589
|
+
}
|
|
3590
|
+
const hasLeadingSlash = re.endsWith("/");
|
|
3591
|
+
const hasTrailingSlash = pattern[i + 2] === "/";
|
|
3592
|
+
if (hasLeadingSlash) re = re.slice(0, -1);
|
|
3593
|
+
if (hasLeadingSlash || hasTrailingSlash) re += "(?:/?.*)?";
|
|
3594
|
+
else re += ".*";
|
|
3595
|
+
i += hasTrailingSlash ? 3 : 2;
|
|
3596
|
+
}
|
|
3597
|
+
return new RegExp(re + "$");
|
|
3598
|
+
}
|
|
3599
|
+
/**
|
|
3600
|
+
* Returns true if `changedPath` is covered by any of `relatedPaths`. An empty
|
|
3601
|
+
* `relatedPaths` returns false — callers handle the "unscoped spec" case
|
|
3602
|
+
* separately (treat the spec as always-affected) before calling this.
|
|
3603
|
+
*/
|
|
3604
|
+
function isPathAffectedBy(changedPath, relatedPaths) {
|
|
3605
|
+
const stripped = stripLeadingDotSlash(changedPath);
|
|
3606
|
+
for (const pattern of relatedPaths) if (compileGlob(pattern).test(stripped)) return true;
|
|
3607
|
+
return false;
|
|
3516
3608
|
}
|
|
3517
3609
|
/**
|
|
3518
3610
|
* Capture the PR diff used as context for failure analysis. `--relative`
|
|
@@ -3598,17 +3690,20 @@ function splitPatchByFile(patch) {
|
|
|
3598
3690
|
/**
|
|
3599
3691
|
* Scope a full patch down to the files a spec depends on, then truncate so
|
|
3600
3692
|
* the analysis prompt stays bounded. `relatedPaths` null/empty means the
|
|
3601
|
-
* spec is unscoped — keep the whole patch (still truncated).
|
|
3602
|
-
*
|
|
3693
|
+
* spec is unscoped — keep the whole patch (still truncated). When
|
|
3694
|
+
* relatedPaths are declared but nothing in the diff matches, the result is
|
|
3695
|
+
* the empty string: "no related change" is itself a signal the prompt
|
|
3696
|
+
* renders explicitly, and the model can inspect any unmatched file's hunk
|
|
3697
|
+
* via the on-demand diff tool — inlining the full unrelated diff (the old
|
|
3698
|
+
* fallback) just burned the prompt budget, especially under wide last-green
|
|
3699
|
+
* baselines. Callers scoping the same patch for many specs can pass
|
|
3700
|
+
* pre-split sections instead.
|
|
3603
3701
|
*/
|
|
3604
3702
|
function scopePatchForSpec(patch, relatedPaths, caps = {}) {
|
|
3605
3703
|
const perFile = caps.perFile ?? 8192;
|
|
3606
3704
|
const total = caps.total ?? 49152;
|
|
3607
3705
|
let sections = typeof patch === "string" ? splitPatchByFile(patch) : patch;
|
|
3608
|
-
if (relatedPaths && relatedPaths.length > 0)
|
|
3609
|
-
const scoped = sections.filter((s) => isPathAffectedBy(s.path, relatedPaths));
|
|
3610
|
-
if (scoped.length > 0) sections = scoped;
|
|
3611
|
-
}
|
|
3706
|
+
if (relatedPaths && relatedPaths.length > 0) sections = sections.filter((s) => isPathAffectedBy(s.path, relatedPaths));
|
|
3612
3707
|
const parts = [];
|
|
3613
3708
|
let used = 0;
|
|
3614
3709
|
let droppedFiles = 0;
|
|
@@ -3627,6 +3722,286 @@ function scopePatchForSpec(patch, relatedPaths, caps = {}) {
|
|
|
3627
3722
|
return parts.join("\n");
|
|
3628
3723
|
}
|
|
3629
3724
|
//#endregion
|
|
3725
|
+
//#region src/run/diff-provider.ts
|
|
3726
|
+
/**
|
|
3727
|
+
* Cap on one on-demand file-diff response. Larger than the inline seed's
|
|
3728
|
+
* per-file cap (the model explicitly asked for this file), but still bounded
|
|
3729
|
+
* so a generated-file hunk can't blow the context — the truncation note
|
|
3730
|
+
* points at Read for the file's full current state.
|
|
3731
|
+
*/
|
|
3732
|
+
const FILE_DIFF_RESPONSE_CAP = 16 * 1024;
|
|
3733
|
+
/** Find `path`'s section in a split patch and cap it. Exported for tests. */
|
|
3734
|
+
function lookupFileDiff(sections, path) {
|
|
3735
|
+
const normalized = stripLeadingDotSlash(path);
|
|
3736
|
+
const section = sections.find((s) => s.path === normalized);
|
|
3737
|
+
if (!section) return null;
|
|
3738
|
+
if (section.body.length <= 16384) return section.body;
|
|
3739
|
+
return `${section.body.slice(0, FILE_DIFF_RESPONSE_CAP)}\n[truncated: ${section.body.length - FILE_DIFF_RESPONSE_CAP} more chars — Read the file for its full current state]`;
|
|
3740
|
+
}
|
|
3741
|
+
/**
|
|
3742
|
+
* Best-effort width of the base..HEAD range. Two-dot rev-list matches what
|
|
3743
|
+
* the three-dot diff shows: commits on the HEAD side since the merge base.
|
|
3744
|
+
*/
|
|
3745
|
+
async function measureRange(sha, cwd) {
|
|
3746
|
+
try {
|
|
3747
|
+
const [{ stdout: count }, { stdout: baseTime }, { stdout: headTime }] = await Promise.all([
|
|
3748
|
+
execFileP("git", [
|
|
3749
|
+
"rev-list",
|
|
3750
|
+
"--count",
|
|
3751
|
+
`${sha}..HEAD`
|
|
3752
|
+
], { cwd }),
|
|
3753
|
+
execFileP("git", [
|
|
3754
|
+
"log",
|
|
3755
|
+
"-1",
|
|
3756
|
+
"--format=%ct",
|
|
3757
|
+
sha
|
|
3758
|
+
], { cwd }),
|
|
3759
|
+
execFileP("git", [
|
|
3760
|
+
"log",
|
|
3761
|
+
"-1",
|
|
3762
|
+
"--format=%ct",
|
|
3763
|
+
"HEAD"
|
|
3764
|
+
], { cwd })
|
|
3765
|
+
]);
|
|
3766
|
+
const seconds = Number(headTime.trim()) - Number(baseTime.trim());
|
|
3767
|
+
return {
|
|
3768
|
+
commitCount: Number(count.trim()),
|
|
3769
|
+
days: Math.max(0, Math.round(seconds / 86400))
|
|
3770
|
+
};
|
|
3771
|
+
} catch {
|
|
3772
|
+
return null;
|
|
3773
|
+
}
|
|
3774
|
+
}
|
|
3775
|
+
function createDiffProvider(args) {
|
|
3776
|
+
const { resolveBase, cwd } = args;
|
|
3777
|
+
const captures = /* @__PURE__ */ new Map();
|
|
3778
|
+
let relatedPathsIndex = null;
|
|
3779
|
+
function capture(sha) {
|
|
3780
|
+
const cached = captures.get(sha);
|
|
3781
|
+
if (cached) return cached;
|
|
3782
|
+
const pending = (async () => {
|
|
3783
|
+
const [result, range] = await Promise.all([capturePrDiff(sha, cwd), measureRange(sha, cwd)]);
|
|
3784
|
+
if (!result.ok) return {
|
|
3785
|
+
sections: null,
|
|
3786
|
+
nameStatus: null,
|
|
3787
|
+
error: result.error,
|
|
3788
|
+
range
|
|
3789
|
+
};
|
|
3790
|
+
const { patch, nameStatus } = result.diff;
|
|
3791
|
+
return {
|
|
3792
|
+
sections: patch.length > 0 ? splitPatchByFile(patch) : [],
|
|
3793
|
+
nameStatus,
|
|
3794
|
+
error: null,
|
|
3795
|
+
range
|
|
3796
|
+
};
|
|
3797
|
+
})();
|
|
3798
|
+
captures.set(sha, pending);
|
|
3799
|
+
return pending;
|
|
3800
|
+
}
|
|
3801
|
+
/** relatedPaths for every spec, read once from the feature tree. */
|
|
3802
|
+
function relatedPaths() {
|
|
3803
|
+
relatedPathsIndex ??= listFeatureTree(cwd).then((tree) => new Map(tree.flatMap((f) => f.specs.map((s) => [specKey({
|
|
3804
|
+
featureName: f.featureName,
|
|
3805
|
+
specName: s.specName
|
|
3806
|
+
}), s.relatedPaths ?? null]))));
|
|
3807
|
+
return relatedPathsIndex;
|
|
3808
|
+
}
|
|
3809
|
+
return { async forSpec(spec) {
|
|
3810
|
+
const resolved = await resolveBase(spec);
|
|
3811
|
+
if (!resolved.ok) return resolved;
|
|
3812
|
+
const [captured, index] = await Promise.all([capture(resolved.base.sha), relatedPaths()]);
|
|
3813
|
+
const scope = index.get(specKey(spec)) ?? null;
|
|
3814
|
+
const sections = captured.sections;
|
|
3815
|
+
return {
|
|
3816
|
+
ok: true,
|
|
3817
|
+
base: resolved.base,
|
|
3818
|
+
patch: sections ? scopePatchForSpec(sections, scope) : null,
|
|
3819
|
+
nameStatus: captured.nameStatus,
|
|
3820
|
+
error: captured.error,
|
|
3821
|
+
range: captured.range,
|
|
3822
|
+
fileDiff: (path) => sections ? lookupFileDiff(sections, path) : null
|
|
3823
|
+
};
|
|
3824
|
+
} };
|
|
3825
|
+
}
|
|
3826
|
+
//#endregion
|
|
3827
|
+
//#region src/run/errors.ts
|
|
3828
|
+
/**
|
|
3829
|
+
* Usage error (bad flag combination, broken profile, failed `git diff`, …)
|
|
3830
|
+
* thrown by the `run` pipeline and the helpers it calls, e.g.
|
|
3831
|
+
* `collectChangedSpecs`. `executeRun` never calls `process.exit`, so each
|
|
3832
|
+
* host maps this itself: the CLI action catches it and exits with
|
|
3833
|
+
* `exitCode`; the hub runner records it as a run-level error.
|
|
3834
|
+
*/
|
|
3835
|
+
var RunUsageError = class extends Error {
|
|
3836
|
+
exitCode = 2;
|
|
3837
|
+
constructor(message) {
|
|
3838
|
+
super(message);
|
|
3839
|
+
this.name = "RunUsageError";
|
|
3840
|
+
}
|
|
3841
|
+
};
|
|
3842
|
+
//#endregion
|
|
3843
|
+
//#region src/run/git-context.ts
|
|
3844
|
+
/** The `--failure-analysis` value that selects per-spec hub-ledger baselines. */
|
|
3845
|
+
const LAST_GREEN = "last-green";
|
|
3846
|
+
/** Resolve `ref` to a full commit sha, or null when it does not exist locally. */
|
|
3847
|
+
async function resolveCommitSha(ref, cwd) {
|
|
3848
|
+
try {
|
|
3849
|
+
const { stdout } = await execFileP("git", [
|
|
3850
|
+
"rev-parse",
|
|
3851
|
+
"--verify",
|
|
3852
|
+
"--quiet",
|
|
3853
|
+
`${ref}^{commit}`
|
|
3854
|
+
], { cwd });
|
|
3855
|
+
return stdout.trim() || null;
|
|
3856
|
+
} catch {
|
|
3857
|
+
return null;
|
|
3858
|
+
}
|
|
3859
|
+
}
|
|
3860
|
+
/**
|
|
3861
|
+
* Resolve a `[base]` flag value (from `--failure-analysis [base]` or
|
|
3862
|
+
* `--changed [base]`) to a verified baseline, failing fast — before any spec
|
|
3863
|
+
* runs — when it cannot be resolved.
|
|
3864
|
+
*
|
|
3865
|
+
* - a string value is an explicit ref;
|
|
3866
|
+
* - bare `true` derives the ref from GITHUB_BASE_REF (pull_request events)
|
|
3867
|
+
* and errors outside that context;
|
|
3868
|
+
* - the ref must resolve to a local commit, so a shallow CI checkout that
|
|
3869
|
+
* never fetched the base surfaces here as an actionable error instead of
|
|
3870
|
+
* an empty diff downstream.
|
|
3871
|
+
*
|
|
3872
|
+
* `flagName` only shapes the error messages.
|
|
3873
|
+
*/
|
|
3874
|
+
async function resolveAnalysisBase(flagValue, flagName, cwd) {
|
|
3875
|
+
let ref;
|
|
3876
|
+
let source;
|
|
3877
|
+
if (flagValue === "last-green") throw new RunUsageError(`${flagName}=${LAST_GREEN} is not supported — last-green baselines are per-spec and only apply to --failure-analysis`);
|
|
3878
|
+
if (typeof flagValue === "string") {
|
|
3879
|
+
ref = flagValue;
|
|
3880
|
+
source = "explicit";
|
|
3881
|
+
} else {
|
|
3882
|
+
const ghBase = process.env["GITHUB_BASE_REF"];
|
|
3883
|
+
if (!ghBase) throw new RunUsageError(`${flagName} without a base needs GITHUB_BASE_REF (a pull_request workflow); outside that context pass the base explicitly, e.g. ${flagName}=origin/main`);
|
|
3884
|
+
ref = normalizeGithubBaseRef(ghBase);
|
|
3885
|
+
source = "github-base-ref";
|
|
3886
|
+
}
|
|
3887
|
+
const sha = await resolveCommitSha(ref, cwd);
|
|
3888
|
+
if (sha === null) throw new RunUsageError(`${flagName}: '${ref}' is not a resolvable git ref in this checkout. If this is CI, the base may not be fetched (try fetch-depth: 0). If '${ref}' was meant as a spec target, put spec targets before flags or use ${flagName}=<ref>.`);
|
|
3889
|
+
return {
|
|
3890
|
+
ref,
|
|
3891
|
+
sha,
|
|
3892
|
+
source
|
|
3893
|
+
};
|
|
3894
|
+
}
|
|
3895
|
+
//#endregion
|
|
3896
|
+
//#region src/cli/git-branch.ts
|
|
3897
|
+
/** Best-effort current branch: CI env vars first, then git, else null. */
|
|
3898
|
+
async function detectBranch(cwd) {
|
|
3899
|
+
const fromEnv = process.env.GITHUB_HEAD_REF || process.env.GITHUB_REF_NAME;
|
|
3900
|
+
if (fromEnv) return fromEnv;
|
|
3901
|
+
try {
|
|
3902
|
+
const { stdout } = await execFileP("git", [
|
|
3903
|
+
"rev-parse",
|
|
3904
|
+
"--abbrev-ref",
|
|
3905
|
+
"HEAD"
|
|
3906
|
+
], { cwd });
|
|
3907
|
+
const branch = stdout.trim();
|
|
3908
|
+
return branch && branch !== "HEAD" ? branch : null;
|
|
3909
|
+
} catch {
|
|
3910
|
+
return null;
|
|
3911
|
+
}
|
|
3912
|
+
}
|
|
3913
|
+
/** Best-effort current commit SHA, or null (e.g. not a git repo). */
|
|
3914
|
+
async function getGitHead(cwd) {
|
|
3915
|
+
try {
|
|
3916
|
+
const { stdout } = await execFileP("git", ["rev-parse", "HEAD"], { cwd });
|
|
3917
|
+
return stdout.trim() || null;
|
|
3918
|
+
} catch {
|
|
3919
|
+
return null;
|
|
3920
|
+
}
|
|
3921
|
+
}
|
|
3922
|
+
//#endregion
|
|
3923
|
+
//#region src/run/last-green.ts
|
|
3924
|
+
/**
|
|
3925
|
+
* The repo's default branch name (e.g. "main"), from origin's HEAD ref.
|
|
3926
|
+
* Falls back to "main" when origin isn't configured — the lookup then just
|
|
3927
|
+
* queries a possibly-empty ledger bucket, which is harmless.
|
|
3928
|
+
*/
|
|
3929
|
+
async function detectDefaultBranch(cwd) {
|
|
3930
|
+
try {
|
|
3931
|
+
const { stdout } = await execFileP("git", [
|
|
3932
|
+
"symbolic-ref",
|
|
3933
|
+
"--short",
|
|
3934
|
+
"refs/remotes/origin/HEAD"
|
|
3935
|
+
], { cwd });
|
|
3936
|
+
const ref = stdout.trim();
|
|
3937
|
+
return ref.startsWith("origin/") ? ref.slice(7) : ref || "main";
|
|
3938
|
+
} catch {
|
|
3939
|
+
return "main";
|
|
3940
|
+
}
|
|
3941
|
+
}
|
|
3942
|
+
/**
|
|
3943
|
+
* Fetch the last-green ledger for this run — one hub round trip, logged as
|
|
3944
|
+
* the run's analysis-base meta line. Fails fast (RunUsageError) when the hub
|
|
3945
|
+
* can't serve it: `--failure-analysis=last-green` explicitly opted into
|
|
3946
|
+
* hub-backed baselines, so a broken hub connection is a usage error, never a
|
|
3947
|
+
* silent no-baseline run.
|
|
3948
|
+
*/
|
|
3949
|
+
async function fetchLastGreenLedger(hubCtx, profile, cwd) {
|
|
3950
|
+
const [fallbackBranch, detectedBranch] = await Promise.all([detectDefaultBranch(cwd), detectBranch(cwd)]);
|
|
3951
|
+
const branch = detectedBranch ?? fallbackBranch;
|
|
3952
|
+
let entries;
|
|
3953
|
+
try {
|
|
3954
|
+
entries = await hubCtx.hub.getLastGreen(hubCtx.project, {
|
|
3955
|
+
branch,
|
|
3956
|
+
fallbackBranch,
|
|
3957
|
+
...profile ? { profile } : {}
|
|
3958
|
+
});
|
|
3959
|
+
} catch (err) {
|
|
3960
|
+
throw new RunUsageError(`--failure-analysis=${LAST_GREEN}: could not fetch the last-green ledger from the hub: ${err instanceof Error ? err.message : String(err)}`);
|
|
3961
|
+
}
|
|
3962
|
+
const n = Object.keys(entries).length;
|
|
3963
|
+
const scope = branch === fallbackBranch ? branch : `${branch} → ${fallbackBranch}`;
|
|
3964
|
+
meta("analysis-base", `${LAST_GREEN} (${n} spec baseline${n === 1 ? "" : "s"}, branch ${scope})`);
|
|
3965
|
+
return entries;
|
|
3966
|
+
}
|
|
3967
|
+
/**
|
|
3968
|
+
* Per-spec baseline resolver for `--failure-analysis=last-green`. A spec
|
|
3969
|
+
* missing from the ledger (never green on a pushed run yet) or whose
|
|
3970
|
+
* baseline commit isn't in this checkout resolves to a skip — the run
|
|
3971
|
+
* continues; only that spec's classification is withheld, with the reason
|
|
3972
|
+
* recorded in its report row. Sha existence checks are memoized per commit.
|
|
3973
|
+
*/
|
|
3974
|
+
function createLastGreenResolver(entries, cwd) {
|
|
3975
|
+
const shaChecks = /* @__PURE__ */ new Map();
|
|
3976
|
+
const checkSha = (sha) => {
|
|
3977
|
+
const cached = shaChecks.get(sha);
|
|
3978
|
+
if (cached) return cached;
|
|
3979
|
+
const pending = resolveCommitSha(sha, cwd);
|
|
3980
|
+
shaChecks.set(sha, pending);
|
|
3981
|
+
return pending;
|
|
3982
|
+
};
|
|
3983
|
+
return async (spec) => {
|
|
3984
|
+
const entry = entries[specKey(spec)];
|
|
3985
|
+
if (!entry) return {
|
|
3986
|
+
ok: false,
|
|
3987
|
+
skip: "no last-green baseline for this spec on the hub yet (recorded once the spec passes on a pushed run)"
|
|
3988
|
+
};
|
|
3989
|
+
const sha = await checkSha(entry.gitHead);
|
|
3990
|
+
if (!sha) return {
|
|
3991
|
+
ok: false,
|
|
3992
|
+
skip: `last-green commit ${entry.gitHead.slice(0, 12)} is not in this checkout (shallow clone? try fetch-depth: 0)`
|
|
3993
|
+
};
|
|
3994
|
+
return {
|
|
3995
|
+
ok: true,
|
|
3996
|
+
base: {
|
|
3997
|
+
ref: LAST_GREEN,
|
|
3998
|
+
sha,
|
|
3999
|
+
source: "last-green"
|
|
4000
|
+
}
|
|
4001
|
+
};
|
|
4002
|
+
};
|
|
4003
|
+
}
|
|
4004
|
+
//#endregion
|
|
3630
4005
|
//#region src/report/github-format.ts
|
|
3631
4006
|
/**
|
|
3632
4007
|
* Build GitHub Actions `::error::` annotation lines for every failed spec in
|
|
@@ -4584,33 +4959,6 @@ function resolvePromptLocalPath(name, cwd) {
|
|
|
4584
4959
|
return join(cwd ?? process.cwd(), PROMPT_LOCAL_PATHS[name]);
|
|
4585
4960
|
}
|
|
4586
4961
|
//#endregion
|
|
4587
|
-
//#region src/cli/git-branch.ts
|
|
4588
|
-
/** Best-effort current branch: CI env vars first, then git, else null. */
|
|
4589
|
-
async function detectBranch(cwd) {
|
|
4590
|
-
const fromEnv = process.env.GITHUB_HEAD_REF || process.env.GITHUB_REF_NAME;
|
|
4591
|
-
if (fromEnv) return fromEnv;
|
|
4592
|
-
try {
|
|
4593
|
-
const { stdout } = await execFileP("git", [
|
|
4594
|
-
"rev-parse",
|
|
4595
|
-
"--abbrev-ref",
|
|
4596
|
-
"HEAD"
|
|
4597
|
-
], { cwd });
|
|
4598
|
-
const branch = stdout.trim();
|
|
4599
|
-
return branch && branch !== "HEAD" ? branch : null;
|
|
4600
|
-
} catch {
|
|
4601
|
-
return null;
|
|
4602
|
-
}
|
|
4603
|
-
}
|
|
4604
|
-
/** Best-effort current commit SHA, or null (e.g. not a git repo). */
|
|
4605
|
-
async function getGitHead(cwd) {
|
|
4606
|
-
try {
|
|
4607
|
-
const { stdout } = await execFileP("git", ["rev-parse", "HEAD"], { cwd });
|
|
4608
|
-
return stdout.trim() || null;
|
|
4609
|
-
} catch {
|
|
4610
|
-
return null;
|
|
4611
|
-
}
|
|
4612
|
-
}
|
|
4613
|
-
//#endregion
|
|
4614
4962
|
//#region src/cli/hub.ts
|
|
4615
4963
|
/**
|
|
4616
4964
|
* `ccqa hub` — the client side of the ccqa hub (a results/secret control
|
|
@@ -5540,22 +5888,14 @@ async function runLiveSpecs(specs, opts) {
|
|
|
5540
5888
|
const userPromptBundle = await loadPromptBundleFromHub(opts.hubContext ?? null, "live");
|
|
5541
5889
|
if (userPromptBundle !== null) meta("prompt", userPromptBundle.loaded.join(" + "));
|
|
5542
5890
|
const userPromptSuffix = userPromptBundle?.text ?? null;
|
|
5543
|
-
const
|
|
5891
|
+
const diffProvider = opts.diffProvider ?? null;
|
|
5892
|
+
const failureAnalysisEnabled = diffProvider != null;
|
|
5544
5893
|
const driftAuditEnabled = failureAnalysisEnabled && opts.driftAudit !== false;
|
|
5545
5894
|
const auth = failureAnalysisEnabled ? driftAuthAvailable() : {
|
|
5546
5895
|
ok: false,
|
|
5547
5896
|
reason: "disabled"
|
|
5548
5897
|
};
|
|
5549
5898
|
if (failureAnalysisEnabled && !auth.ok) info(`failure analysis skipped (${auth.reason})`);
|
|
5550
|
-
const baseRef = resolveBaseRef(opts.base);
|
|
5551
|
-
let diff = {
|
|
5552
|
-
ok: false,
|
|
5553
|
-
error: "diff not captured"
|
|
5554
|
-
};
|
|
5555
|
-
if (failureAnalysisEnabled && auth.ok) {
|
|
5556
|
-
diff = await capturePrDiff(baseRef, cwd);
|
|
5557
|
-
if (!diff.ok) info(`failure analysis: source diff unavailable (${diff.error}) — analyzing without diff context`);
|
|
5558
|
-
}
|
|
5559
5899
|
const reportDir = opts.reportDir ?? ".";
|
|
5560
5900
|
const concurrency = Math.max(1, opts.concurrency ?? 1);
|
|
5561
5901
|
const built = await runPool(specs, concurrency, (spec, i) => {
|
|
@@ -5577,10 +5917,8 @@ async function runLiveSpecs(specs, opts) {
|
|
|
5577
5917
|
};
|
|
5578
5918
|
const row = await buildLiveReportRow(outcome, {
|
|
5579
5919
|
auth,
|
|
5580
|
-
|
|
5581
|
-
baseRef,
|
|
5920
|
+
diffProvider,
|
|
5582
5921
|
reportDir,
|
|
5583
|
-
failureAnalysisEnabled,
|
|
5584
5922
|
driftAuditEnabled
|
|
5585
5923
|
}, opts, cwd);
|
|
5586
5924
|
await opts.report?.upsert(row);
|
|
@@ -5616,26 +5954,29 @@ async function buildLiveReportRow(r, ctx, opts, cwd) {
|
|
|
5616
5954
|
reportDir: ctx.reportDir
|
|
5617
5955
|
});
|
|
5618
5956
|
const driftForSpec = ctx.driftAuditEnabled && r.result.status === "failed" ? await runDriftAuditOne(r, opts, cwd) : null;
|
|
5619
|
-
const analysis = ctx.
|
|
5957
|
+
const analysis = ctx.diffProvider && r.result.status === "failed" ? await analyzeOneLiveFailure(r, ctx.diffProvider, driftForSpec, ctx.auth, opts, cwd) : void 0;
|
|
5620
5958
|
return {
|
|
5621
5959
|
...base,
|
|
5622
5960
|
driftIssues: driftForSpec,
|
|
5623
|
-
...analysisFieldsFor(analysis, r.result.status
|
|
5961
|
+
...analysisFieldsFor(analysis, r.result.status)
|
|
5624
5962
|
};
|
|
5625
5963
|
}
|
|
5626
5964
|
/**
|
|
5627
5965
|
* Merge analysis-related fields into the report row. The unattempted-failure
|
|
5628
5966
|
* branch exists so the report distinguishes "we tried and gave up" (auth /
|
|
5629
|
-
* spec.yaml missing) from "we deliberately did not run the classifier"
|
|
5967
|
+
* spec.yaml missing) from "we deliberately did not run the classifier" —
|
|
5968
|
+
* `a` is undefined for a failed spec exactly when analysis was not requested
|
|
5969
|
+
* (no diffProvider), so no separate flag is needed.
|
|
5630
5970
|
*/
|
|
5631
|
-
function analysisFieldsFor(a, status
|
|
5971
|
+
function analysisFieldsFor(a, status) {
|
|
5632
5972
|
if (a) return {
|
|
5633
5973
|
analysis: a.analysis,
|
|
5634
5974
|
analysisSkipped: a.analysisSkipped,
|
|
5635
5975
|
failureLogExcerpt: a.failureLogExcerpt,
|
|
5636
|
-
diffExcerpt: a.diffExcerpt
|
|
5976
|
+
diffExcerpt: a.diffExcerpt,
|
|
5977
|
+
...a.analysisBase ? { analysisBase: a.analysisBase } : {}
|
|
5637
5978
|
};
|
|
5638
|
-
if (
|
|
5979
|
+
if (status === "failed") return { analysisSkipped: "skipped: --failure-analysis not enabled" };
|
|
5639
5980
|
return {};
|
|
5640
5981
|
}
|
|
5641
5982
|
/**
|
|
@@ -5825,11 +6166,13 @@ function logBatchCost(runs) {
|
|
|
5825
6166
|
/**
|
|
5826
6167
|
* Classify one failed live run via `analyzeFailure` — same prompt as the
|
|
5827
6168
|
* deterministic path (Issue #47), fed the live transcript instead of the
|
|
5828
|
-
* vitest log. `auth
|
|
5829
|
-
* shared
|
|
5830
|
-
*
|
|
6169
|
+
* vitest log. `auth` is hoisted once by the caller; the diff comes from the
|
|
6170
|
+
* shared provider, already scoped to this spec's relatedPaths and truncated
|
|
6171
|
+
* (the live path used to feed the whole unscoped patch — in a monorepo that
|
|
6172
|
+
* ballooned the prompt with unrelated changes). Auth-unavailable /
|
|
6173
|
+
* no-failed-step degrade to `analysisSkipped` rather than throwing.
|
|
5831
6174
|
*/
|
|
5832
|
-
async function analyzeOneLiveFailure(r,
|
|
6175
|
+
async function analyzeOneLiveFailure(r, diffProvider, driftForSpec, auth, opts, cwd) {
|
|
5833
6176
|
const key = `${r.featureName}/${r.specName}`;
|
|
5834
6177
|
if (!auth.ok) return {
|
|
5835
6178
|
analysis: null,
|
|
@@ -5845,19 +6188,33 @@ async function analyzeOneLiveFailure(r, diff, baseRef, driftForSpec, auth, opts,
|
|
|
5845
6188
|
failureLogExcerpt: null,
|
|
5846
6189
|
diffExcerpt: null
|
|
5847
6190
|
};
|
|
6191
|
+
const specDiff = await diffProvider.forSpec({
|
|
6192
|
+
featureName: r.featureName,
|
|
6193
|
+
specName: r.specName
|
|
6194
|
+
});
|
|
6195
|
+
if (!specDiff.ok) return {
|
|
6196
|
+
analysis: null,
|
|
6197
|
+
analysisSkipped: specDiff.skip,
|
|
6198
|
+
failureLogExcerpt: excerpt,
|
|
6199
|
+
diffExcerpt: null
|
|
6200
|
+
};
|
|
6201
|
+
if (specDiff.error) info(`failure analysis: source diff unavailable (${specDiff.error}) — analyzing without diff context`);
|
|
5848
6202
|
const outcome = await analyzeFailure({
|
|
5849
6203
|
liveTranscriptExcerpt: excerpt,
|
|
5850
6204
|
specYaml: r.specYaml,
|
|
5851
|
-
diffPatch:
|
|
5852
|
-
changedFiles:
|
|
5853
|
-
baseRef:
|
|
6205
|
+
diffPatch: specDiff.patch,
|
|
6206
|
+
changedFiles: specDiff.nameStatus,
|
|
6207
|
+
baseRef: specDiff.base.ref,
|
|
6208
|
+
baseSource: specDiff.base.source,
|
|
6209
|
+
range: specDiff.range,
|
|
5854
6210
|
driftIssues: driftForSpec,
|
|
5855
6211
|
...opts.language ? { outputLanguage: opts.language } : {},
|
|
5856
6212
|
...opts.triageUserPrompt ? { triageUserPrompt: opts.triageUserPrompt } : {},
|
|
5857
6213
|
...opts.customPrompt ? { customPrompt: opts.customPrompt } : {}
|
|
5858
6214
|
}, {
|
|
5859
6215
|
...opts.model ? { model: opts.model } : {},
|
|
5860
|
-
cwd
|
|
6216
|
+
cwd,
|
|
6217
|
+
getFileDiff: specDiff.fileDiff
|
|
5861
6218
|
});
|
|
5862
6219
|
const pct = Math.round(outcome.analysis.confidence * 100);
|
|
5863
6220
|
const headline = outcome.analysis.headline.trim() || (outcome.analysis.reasoning.split("\n")[0] ?? "").trim();
|
|
@@ -5866,7 +6223,11 @@ async function analyzeOneLiveFailure(r, diff, baseRef, driftForSpec, auth, opts,
|
|
|
5866
6223
|
analysis: outcome.analysis,
|
|
5867
6224
|
analysisSkipped: null,
|
|
5868
6225
|
failureLogExcerpt: excerpt,
|
|
5869
|
-
diffExcerpt:
|
|
6226
|
+
diffExcerpt: specDiff.patch,
|
|
6227
|
+
analysisBase: {
|
|
6228
|
+
ref: specDiff.base.ref,
|
|
6229
|
+
sha: specDiff.base.sha
|
|
6230
|
+
}
|
|
5870
6231
|
};
|
|
5871
6232
|
}
|
|
5872
6233
|
function count(steps, target) {
|
|
@@ -8937,27 +9298,12 @@ function stripCodeFences(text) {
|
|
|
8937
9298
|
return m && m[1] !== void 0 ? m[1] : text;
|
|
8938
9299
|
}
|
|
8939
9300
|
//#endregion
|
|
8940
|
-
//#region src/run/errors.ts
|
|
8941
|
-
/**
|
|
8942
|
-
* Usage error (bad flag combination, broken profile, failed `git diff`, …)
|
|
8943
|
-
* thrown by the `run` pipeline and the helpers it calls, e.g.
|
|
8944
|
-
* `collectChangedSpecs`. `executeRun` never calls `process.exit`, so each
|
|
8945
|
-
* host maps this itself: the CLI action catches it and exits with
|
|
8946
|
-
* `exitCode`; the hub runner records it as a run-level error.
|
|
8947
|
-
*/
|
|
8948
|
-
var RunUsageError = class extends Error {
|
|
8949
|
-
exitCode = 2;
|
|
8950
|
-
constructor(message) {
|
|
8951
|
-
super(message);
|
|
8952
|
-
this.name = "RunUsageError";
|
|
8953
|
-
}
|
|
8954
|
-
};
|
|
8955
|
-
//#endregion
|
|
8956
9301
|
//#region src/cli/changed-specs.ts
|
|
8957
9302
|
/**
|
|
8958
|
-
* Filter specs to those affected by the git diff against the
|
|
8959
|
-
*
|
|
8960
|
-
* LLM new-file router (kept off here for predictable CI
|
|
9303
|
+
* Filter specs to those affected by the git diff against the `--changed
|
|
9304
|
+
* [base]` baseline. Powers `ccqa run --changed`; mirrors `ccqa drift
|
|
9305
|
+
* --changed` minus the LLM new-file router (kept off here for predictable CI
|
|
9306
|
+
* cost).
|
|
8961
9307
|
*
|
|
8962
9308
|
* A spec is "affected" when it has no `relatedPaths` (conservatively
|
|
8963
9309
|
* included), any changed file matches one of its `relatedPaths` globs, or it
|
|
@@ -8965,14 +9311,14 @@ var RunUsageError = class extends Error {
|
|
|
8965
9311
|
*/
|
|
8966
9312
|
async function collectChangedSpecs(specs, opts) {
|
|
8967
9313
|
const { cwd, base } = opts;
|
|
8968
|
-
const
|
|
9314
|
+
const resolved = await resolveAnalysisBase(base, "--changed", cwd);
|
|
8969
9315
|
let changed;
|
|
8970
9316
|
try {
|
|
8971
|
-
changed = await getChangedFiles(
|
|
9317
|
+
changed = await getChangedFiles(resolved.sha, cwd);
|
|
8972
9318
|
} catch (e) {
|
|
8973
|
-
throw new RunUsageError(`failed to run 'git diff' against ${
|
|
9319
|
+
throw new RunUsageError(`failed to run 'git diff' against ${resolved.ref}: ${e.message}`);
|
|
8974
9320
|
}
|
|
8975
|
-
meta("changed-base",
|
|
9321
|
+
meta("changed-base", `${resolved.ref} (${resolved.sha.slice(0, 12)})`);
|
|
8976
9322
|
meta("changed-files", changed.length);
|
|
8977
9323
|
return filterAffectedSpecs(specs, changed, cwd);
|
|
8978
9324
|
}
|
|
@@ -9036,6 +9382,27 @@ function dedupeSpecs(specs) {
|
|
|
9036
9382
|
async function executeRun(targets, opts) {
|
|
9037
9383
|
if (opts.changed && targets.length > 0) throw new RunUsageError("--changed and an explicit spec target cannot be combined");
|
|
9038
9384
|
const cwd = opts.cwd ?? process.cwd();
|
|
9385
|
+
const wantsLastGreen = opts.failureAnalysis === LAST_GREEN;
|
|
9386
|
+
const [head, fixedBase] = await Promise.all([getGitHead(cwd), opts.failureAnalysis && !wantsLastGreen ? resolveAnalysisBase(opts.failureAnalysis, "--failure-analysis", cwd) : null]);
|
|
9387
|
+
const git = {
|
|
9388
|
+
head,
|
|
9389
|
+
base: wantsLastGreen ? {
|
|
9390
|
+
ref: LAST_GREEN,
|
|
9391
|
+
sha: null,
|
|
9392
|
+
source: "last-green"
|
|
9393
|
+
} : fixedBase
|
|
9394
|
+
};
|
|
9395
|
+
let diffProvider = null;
|
|
9396
|
+
if (fixedBase) {
|
|
9397
|
+
diffProvider = createDiffProvider({
|
|
9398
|
+
resolveBase: async () => ({
|
|
9399
|
+
ok: true,
|
|
9400
|
+
base: fixedBase
|
|
9401
|
+
}),
|
|
9402
|
+
cwd
|
|
9403
|
+
});
|
|
9404
|
+
meta("analysis-base", `${fixedBase.ref} (${fixedBase.sha.slice(0, 12)}, ${fixedBase.source})`);
|
|
9405
|
+
}
|
|
9039
9406
|
let projectForProfile;
|
|
9040
9407
|
try {
|
|
9041
9408
|
if (opts.profile !== void 0) {
|
|
@@ -9071,7 +9438,17 @@ async function executeRun(targets, opts) {
|
|
|
9071
9438
|
} catch {
|
|
9072
9439
|
hubCtx = null;
|
|
9073
9440
|
}
|
|
9074
|
-
|
|
9441
|
+
if (wantsLastGreen && hubCtx == null) throw new RunUsageError(`--failure-analysis=${LAST_GREEN} requires a hub connection (--hub-url/--hub-token or CCQA_HUB_URL/CCQA_HUB_TOKEN)`);
|
|
9442
|
+
const ledgerHub = wantsLastGreen ? hubCtx : null;
|
|
9443
|
+
const [customPrompt, triageUserPrompt, ledgerEntries] = await Promise.all([
|
|
9444
|
+
fetchCustomPrompt(hubCtx),
|
|
9445
|
+
fetchTriageUserPrompt(hubCtx),
|
|
9446
|
+
ledgerHub ? fetchLastGreenLedger(ledgerHub, opts.profile, cwd) : null
|
|
9447
|
+
]);
|
|
9448
|
+
if (ledgerEntries) diffProvider = createDiffProvider({
|
|
9449
|
+
resolveBase: createLastGreenResolver(ledgerEntries, cwd),
|
|
9450
|
+
cwd
|
|
9451
|
+
});
|
|
9075
9452
|
const triageUserPromptHash = triageUserPrompt ? hashTriageUserPrompt(triageUserPrompt) : null;
|
|
9076
9453
|
const enumerateAll = () => listAllSpecsWithSpecFile(cwd);
|
|
9077
9454
|
let specs = dedupeSpecs((await Promise.all((targets.length ? targets : [void 0]).map((t) => resolveSpecTargets(t, enumerateAll, cwd)))).flat());
|
|
@@ -9079,7 +9456,7 @@ async function executeRun(targets, opts) {
|
|
|
9079
9456
|
const before = specs.length;
|
|
9080
9457
|
specs = await collectChangedSpecs(specs, {
|
|
9081
9458
|
cwd,
|
|
9082
|
-
base: opts.
|
|
9459
|
+
base: opts.changed
|
|
9083
9460
|
});
|
|
9084
9461
|
meta("changed-scoped", `${specs.length} of ${before} spec${before === 1 ? "" : "s"}`);
|
|
9085
9462
|
}
|
|
@@ -9121,6 +9498,7 @@ async function executeRun(targets, opts) {
|
|
|
9121
9498
|
project: hubCtx.project,
|
|
9122
9499
|
...branch ? { branch } : {},
|
|
9123
9500
|
...opts.profile ? { profile: opts.profile } : {},
|
|
9501
|
+
...git.head ? { gitHead: git.head } : {},
|
|
9124
9502
|
kind: "run"
|
|
9125
9503
|
});
|
|
9126
9504
|
hubRunId = opened.id;
|
|
@@ -9141,11 +9519,7 @@ async function executeRun(targets, opts) {
|
|
|
9141
9519
|
warn(`hub: could not open incremental run (${errMessage(err)}); continuing with local report only`);
|
|
9142
9520
|
}
|
|
9143
9521
|
const incrementalReport = createIncrementalReport(reportDir, buildReportEnvelope({
|
|
9144
|
-
|
|
9145
|
-
ok: false,
|
|
9146
|
-
error: "diff not yet captured"
|
|
9147
|
-
},
|
|
9148
|
-
baseRef: null,
|
|
9522
|
+
git,
|
|
9149
9523
|
customPromptVersion: customPrompt?.customPromptVersion ?? null,
|
|
9150
9524
|
triageUserPromptHash,
|
|
9151
9525
|
opts
|
|
@@ -9177,13 +9551,12 @@ async function executeRun(targets, opts) {
|
|
|
9177
9551
|
...opts.language ? { language: opts.language } : {},
|
|
9178
9552
|
...opts.out && liveSpecs.length === 1 ? { out: opts.out } : {},
|
|
9179
9553
|
cwd,
|
|
9180
|
-
...opts.base ? { base: opts.base } : {},
|
|
9181
9554
|
reportDir,
|
|
9182
9555
|
...typeof opts.retry === "number" ? { retry: opts.retry } : {},
|
|
9183
9556
|
concurrency: opts.concurrency ?? 1,
|
|
9184
9557
|
...opts.profile ? { profile: opts.profile } : {},
|
|
9185
9558
|
...opts.driftAudit !== false ? { driftAudit: true } : {},
|
|
9186
|
-
|
|
9559
|
+
diffProvider,
|
|
9187
9560
|
hubContext: hubCtx,
|
|
9188
9561
|
customPrompt,
|
|
9189
9562
|
triageUserPrompt,
|
|
@@ -9195,7 +9568,7 @@ async function executeRun(targets, opts) {
|
|
|
9195
9568
|
if (externalRows.some((r) => r.status === "failed")) overallExitCode = 1;
|
|
9196
9569
|
let report;
|
|
9197
9570
|
{
|
|
9198
|
-
const detReport = await analyzeDeterministicSummaries(det.summaries, opts, cwd, reportDir, customPrompt, triageUserPrompt);
|
|
9571
|
+
const detReport = await analyzeDeterministicSummaries(det.summaries, opts, cwd, reportDir, customPrompt, triageUserPrompt, diffProvider);
|
|
9199
9572
|
report = await writeUnifiedReport({
|
|
9200
9573
|
reportDir,
|
|
9201
9574
|
results: [
|
|
@@ -9203,8 +9576,7 @@ async function executeRun(targets, opts) {
|
|
|
9203
9576
|
...externalRows,
|
|
9204
9577
|
...live.reportResults
|
|
9205
9578
|
],
|
|
9206
|
-
|
|
9207
|
-
baseRef: detReport.baseRef,
|
|
9579
|
+
git,
|
|
9208
9580
|
customPromptVersion: detReport.customPromptVersion,
|
|
9209
9581
|
triageUserPromptHash,
|
|
9210
9582
|
opts
|
|
@@ -9213,8 +9585,7 @@ async function executeRun(targets, opts) {
|
|
|
9213
9585
|
if (hubRunId) {
|
|
9214
9586
|
const finalStatus = overallExitCode === 0 ? "passed" : "failed";
|
|
9215
9587
|
const reportMeta = buildReportEnvelope({
|
|
9216
|
-
|
|
9217
|
-
baseRef: detReport.baseRef,
|
|
9588
|
+
git,
|
|
9218
9589
|
customPromptVersion: detReport.customPromptVersion,
|
|
9219
9590
|
triageUserPromptHash,
|
|
9220
9591
|
opts
|
|
@@ -9412,24 +9783,15 @@ function failedSpec(s) {
|
|
|
9412
9783
|
* failure analysis when `--report` is on; degrades (no throw) when Claude
|
|
9413
9784
|
* auth or git diff aren't available. Caller writes the HTML / JSON.
|
|
9414
9785
|
*/
|
|
9415
|
-
async function analyzeDeterministicSummaries(summaries, opts, cwd, reportDir, customPrompt, triageUserPrompt) {
|
|
9416
|
-
const failureAnalysisEnabled =
|
|
9786
|
+
async function analyzeDeterministicSummaries(summaries, opts, cwd, reportDir, customPrompt, triageUserPrompt, diffProvider) {
|
|
9787
|
+
const failureAnalysisEnabled = diffProvider != null;
|
|
9417
9788
|
const driftAuditEnabled = failureAnalysisEnabled && opts.driftAudit !== false;
|
|
9418
|
-
const auth = failureAnalysisEnabled
|
|
9789
|
+
const auth = failureAnalysisEnabled ? driftAuthAvailable() : {
|
|
9419
9790
|
ok: false,
|
|
9420
9791
|
reason: "skipped by flags"
|
|
9421
9792
|
};
|
|
9422
9793
|
const failed = summaries.filter(failedSpec);
|
|
9423
9794
|
if (failureAnalysisEnabled && !auth.ok && failed.length > 0) info(`failure analysis skipped (${auth.reason})`);
|
|
9424
|
-
const baseRef = resolveBaseRef(opts.base);
|
|
9425
|
-
let diff = {
|
|
9426
|
-
ok: false,
|
|
9427
|
-
error: "diff not captured (no failures)"
|
|
9428
|
-
};
|
|
9429
|
-
if (failed.length > 0) {
|
|
9430
|
-
diff = await capturePrDiff(baseRef, cwd);
|
|
9431
|
-
if (!diff.ok) info(`drift-report: source diff unavailable (${diff.error}) — analyzing without diff context`);
|
|
9432
|
-
}
|
|
9433
9795
|
const tree = failed.length > 0 ? await listFeatureTree(cwd) : [];
|
|
9434
9796
|
const specInfoByKey = new Map(tree.flatMap((f) => f.specs.map((sp) => [`${f.featureName}/${sp.specName}`, sp])));
|
|
9435
9797
|
const findSpecInfo = (s) => specInfoByKey.get(`${s.featureName}/${s.specName}`) ?? null;
|
|
@@ -9456,9 +9818,9 @@ async function analyzeDeterministicSummaries(summaries, opts, cwd, reportDir, cu
|
|
|
9456
9818
|
onSpecStart: (t) => info(`drift audit: ${t.featureName}/${t.specName}`)
|
|
9457
9819
|
});
|
|
9458
9820
|
}
|
|
9459
|
-
const patchSections = diff.ok && diff.diff.patch.length > 0 ? splitPatchByFile(diff.diff.patch) : null;
|
|
9460
9821
|
const allBlocks = await loadAllBlocks(cwd);
|
|
9461
9822
|
let printedHeader = false;
|
|
9823
|
+
let warnedDiffUnavailable = false;
|
|
9462
9824
|
const results = [];
|
|
9463
9825
|
for (const s of summaries) {
|
|
9464
9826
|
const assertions = collectAssertions(s);
|
|
@@ -9493,14 +9855,23 @@ async function analyzeDeterministicSummaries(summaries, opts, cwd, reportDir, cu
|
|
|
9493
9855
|
});
|
|
9494
9856
|
continue;
|
|
9495
9857
|
}
|
|
9496
|
-
const
|
|
9497
|
-
|
|
9858
|
+
const specDiffResult = diffProvider ? await diffProvider.forSpec({
|
|
9859
|
+
featureName: s.featureName,
|
|
9860
|
+
specName: s.specName
|
|
9861
|
+
}) : null;
|
|
9862
|
+
const specDiff = specDiffResult?.ok ? specDiffResult : null;
|
|
9863
|
+
if (specDiff?.error && !warnedDiffUnavailable) {
|
|
9864
|
+
warnedDiffUnavailable = true;
|
|
9865
|
+
info(`failure analysis: source diff unavailable (${specDiff.error}) — analyzing without diff context`);
|
|
9866
|
+
}
|
|
9867
|
+
const diffExcerpt = specDiff?.patch ?? null;
|
|
9498
9868
|
const driftResult = driftResults.find((r) => r.target.featureName === s.featureName && r.target.specName === s.specName);
|
|
9499
9869
|
const driftIssues = driftResult?.ok ? driftResult.issues : null;
|
|
9500
9870
|
const failureLog = buildFailureLog(s);
|
|
9501
9871
|
let analysis = null;
|
|
9502
9872
|
let analysisSkipped = null;
|
|
9503
|
-
if (!
|
|
9873
|
+
if (!specDiffResult) analysisSkipped = "skipped: --failure-analysis not enabled";
|
|
9874
|
+
else if (!specDiffResult.ok) analysisSkipped = specDiffResult.skip;
|
|
9504
9875
|
else if (!auth.ok) analysisSkipped = auth.reason;
|
|
9505
9876
|
else if (specYaml === null) analysisSkipped = "no spec.yaml found for this spec";
|
|
9506
9877
|
else {
|
|
@@ -9511,15 +9882,18 @@ async function analyzeDeterministicSummaries(summaries, opts, cwd, reportDir, cu
|
|
|
9511
9882
|
specYaml,
|
|
9512
9883
|
failureLog,
|
|
9513
9884
|
diffPatch: diffExcerpt,
|
|
9514
|
-
changedFiles:
|
|
9515
|
-
baseRef:
|
|
9885
|
+
changedFiles: specDiffResult.nameStatus,
|
|
9886
|
+
baseRef: specDiffResult.base.ref,
|
|
9887
|
+
baseSource: specDiffResult.base.source,
|
|
9888
|
+
range: specDiffResult.range,
|
|
9516
9889
|
driftIssues,
|
|
9517
9890
|
...opts.language ? { outputLanguage: opts.language } : {},
|
|
9518
9891
|
...triageUserPrompt ? { triageUserPrompt } : {},
|
|
9519
9892
|
...customPrompt ? { customPrompt } : {}
|
|
9520
9893
|
}, {
|
|
9521
9894
|
...opts.model ? { model: opts.model } : {},
|
|
9522
|
-
cwd
|
|
9895
|
+
cwd,
|
|
9896
|
+
getFileDiff: specDiffResult.fileDiff
|
|
9523
9897
|
});
|
|
9524
9898
|
analysis = outcome.analysis;
|
|
9525
9899
|
if (!printedHeader) {
|
|
@@ -9537,6 +9911,10 @@ async function analyzeDeterministicSummaries(summaries, opts, cwd, reportDir, cu
|
|
|
9537
9911
|
status: "failed",
|
|
9538
9912
|
analysis,
|
|
9539
9913
|
analysisSkipped,
|
|
9914
|
+
...specDiff ? { analysisBase: {
|
|
9915
|
+
ref: specDiff.base.ref,
|
|
9916
|
+
sha: specDiff.base.sha
|
|
9917
|
+
} } : {},
|
|
9540
9918
|
driftIssues,
|
|
9541
9919
|
failureLogExcerpt: failureLog.length > 0 ? failureLog : null,
|
|
9542
9920
|
diffExcerpt,
|
|
@@ -9546,8 +9924,6 @@ async function analyzeDeterministicSummaries(summaries, opts, cwd, reportDir, cu
|
|
|
9546
9924
|
}
|
|
9547
9925
|
return {
|
|
9548
9926
|
results,
|
|
9549
|
-
diff,
|
|
9550
|
-
baseRef,
|
|
9551
9927
|
customPromptVersion: customPrompt?.customPromptVersion ?? null
|
|
9552
9928
|
};
|
|
9553
9929
|
}
|
|
@@ -9559,30 +9935,33 @@ async function analyzeDeterministicSummaries(summaries, opts, cwd, reportDir, cu
|
|
|
9559
9935
|
* final report.json stays byte-identical (existing e2e goldens compare it).
|
|
9560
9936
|
*/
|
|
9561
9937
|
function buildReportEnvelope(args) {
|
|
9562
|
-
const {
|
|
9938
|
+
const { git, customPromptVersion, triageUserPromptHash, opts } = args;
|
|
9563
9939
|
return {
|
|
9564
9940
|
schemaVersion: 1,
|
|
9565
9941
|
kind: "run",
|
|
9566
9942
|
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9567
9943
|
runId: process.env["GITHUB_RUN_ID"] ?? null,
|
|
9568
9944
|
git: {
|
|
9569
|
-
head:
|
|
9570
|
-
base:
|
|
9945
|
+
head: git.head,
|
|
9946
|
+
base: git.base?.ref ?? null,
|
|
9947
|
+
...git.base ? {
|
|
9948
|
+
baseSha: git.base.sha,
|
|
9949
|
+
baseSource: git.base.source
|
|
9950
|
+
} : {}
|
|
9571
9951
|
},
|
|
9572
9952
|
model: opts.model ?? null,
|
|
9573
9953
|
language: opts.language ?? null,
|
|
9574
|
-
promptVersion: "
|
|
9954
|
+
promptVersion: "6",
|
|
9575
9955
|
customPromptVersion,
|
|
9576
9956
|
...triageUserPromptHash !== null ? { triageUserPromptHash } : {}
|
|
9577
9957
|
};
|
|
9578
9958
|
}
|
|
9579
9959
|
/** Write the unified JSON (+ optional GitHub-annotation) report for one run. Returns the report data. */
|
|
9580
9960
|
async function writeUnifiedReport(args) {
|
|
9581
|
-
const { reportDir, results,
|
|
9961
|
+
const { reportDir, results, git, customPromptVersion, triageUserPromptHash, opts } = args;
|
|
9582
9962
|
const data = {
|
|
9583
9963
|
...buildReportEnvelope({
|
|
9584
|
-
|
|
9585
|
-
baseRef,
|
|
9964
|
+
git,
|
|
9586
9965
|
customPromptVersion,
|
|
9587
9966
|
triageUserPromptHash,
|
|
9588
9967
|
opts
|
|
@@ -9896,7 +10275,7 @@ function installTeardownSignalHandlers(teardown) {
|
|
|
9896
10275
|
}
|
|
9897
10276
|
//#endregion
|
|
9898
10277
|
//#region src/cli/run.ts
|
|
9899
|
-
const runCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("run").argument("[targets...]", "Specs to run, space-separated: each '<feature>/<spec>', '<feature>', or omit for all. Duplicates are de-duped.").description("Run specs. Each spec's execution mode comes from its spec.yaml `mode:` field (default deterministic; set `mode: live` to have Claude drive agent-browser live per step). Deterministic specs replay the recorded test.spec.ts under vitest. A structured report (report.json + evidence) is always written; use --push-report to also stream it to a hub.").option("--report [dir]", `Directory for the structured run results (report.json + evidence PNGs) that are always written. Default: ${DEFAULT_REPORT_DIR}/. Pass this only to change the location.`).option("--push-report", "Incrementally push the run report to the hub as the run progresses (open → patch per spec → finalize). Requires --hub-url/--hub-token (or CCQA_HUB_URL/CCQA_HUB_TOKEN). Without it, hub credentials are used only to fetch variables/sessions/prompts, not to push.").option("--changed", "Restrict execution to specs whose relatedPaths intersect the git diff against
|
|
10278
|
+
const runCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("run").argument("[targets...]", "Specs to run, space-separated: each '<feature>/<spec>', '<feature>', or omit for all. Duplicates are de-duped.").description("Run specs. Each spec's execution mode comes from its spec.yaml `mode:` field (default deterministic; set `mode: live` to have Claude drive agent-browser live per step). Deterministic specs replay the recorded test.spec.ts under vitest. A structured report (report.json + evidence) is always written; use --push-report to also stream it to a hub.").option("--report [dir]", `Directory for the structured run results (report.json + evidence PNGs) that are always written. Default: ${DEFAULT_REPORT_DIR}/. Pass this only to change the location.`).option("--push-report", "Incrementally push the run report to the hub as the run progresses (open → patch per spec → finalize). Requires --hub-url/--hub-token (or CCQA_HUB_URL/CCQA_HUB_TOKEN). Without it, hub credentials are used only to fetch variables/sessions/prompts, not to push.").option("--changed [base]", "Restrict execution to specs whose relatedPaths intersect the git diff against [base]. Without a value the base comes from $GITHUB_BASE_REF (pull_request CI); elsewhere pass it explicitly (e.g. --changed=origin/main). Cannot be combined with an explicit spec id.").option("--failure-analysis [base]", "Classify each failure (TEST_DRIFT / SPEC_CHANGE / PRODUCT_BUG) against the source diff since [base]. Without a value the base comes from $GITHUB_BASE_REF (pull_request CI); elsewhere pass it explicitly (e.g. --failure-analysis=origin/main), or pass 'last-green' to diff each spec against the commit where it last passed (per-spec baselines from the hub; requires a hub connection). Off by default — no Claude calls without it.").option("--no-drift-audit", "With --failure-analysis: skip the spec↔code drift audit shown in the report.").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--format <fmt>", "Additional output format alongside HTML when --report is set: 'text' (default), 'json' (writes report.json), 'github' (GitHub Actions annotations on stdout).", (raw) => {
|
|
9900
10279
|
if (REPORT_FORMATS.includes(raw)) return raw;
|
|
9901
10280
|
throw new Error(`--format must be one of ${REPORT_FORMATS.join(" | ")}`);
|
|
9902
10281
|
}, "text").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--no-evidence", `(deterministic only) Skip step-boundary evidence capture (PNG + meta JSON written to ${DEFAULT_REPORT_DIR}/${EVIDENCE_SUBDIR}/ by default).`).option("--retry <n>", "(live only) Retry each failed step up to N more times before recording failure. Default 0.", (raw) => {
|
|
@@ -13390,6 +13769,11 @@ z.object({ error: z.object({
|
|
|
13390
13769
|
code: z.string(),
|
|
13391
13770
|
message: z.string()
|
|
13392
13771
|
}) });
|
|
13772
|
+
z.object({
|
|
13773
|
+
gitHead: z.string(),
|
|
13774
|
+
runId: z.string(),
|
|
13775
|
+
at: z.string()
|
|
13776
|
+
});
|
|
13393
13777
|
/**
|
|
13394
13778
|
* A triage-learning job. Grading failing specs in the hub UI produces the
|
|
13395
13779
|
* "actual cause" labels this reads; the job turns them into an improved
|
|
@@ -13505,6 +13889,7 @@ function createPushRunHandler(config) {
|
|
|
13505
13889
|
};
|
|
13506
13890
|
await config.storage.artifacts.putDir(run.id, dir);
|
|
13507
13891
|
await config.storage.runs.create(run);
|
|
13892
|
+
await updateLastGreenLedger(config.storage, run, report.results);
|
|
13508
13893
|
sendJson(ctx.res, 201, run);
|
|
13509
13894
|
} finally {
|
|
13510
13895
|
await rm(dir, {
|
|
@@ -13515,14 +13900,17 @@ function createPushRunHandler(config) {
|
|
|
13515
13900
|
};
|
|
13516
13901
|
}
|
|
13517
13902
|
/**
|
|
13518
|
-
* POST /api/v1/runs/open?project=&branch=&profile=&kind= — start a
|
|
13519
|
-
* run with no report yet. Unlike `POST /runs`, nothing is pushed up
|
|
13520
|
-
* the caller patches results in as they finish (`PATCH /runs/:id`), so
|
|
13521
|
-
* interrupted run still leaves a partial report on the hub instead of
|
|
13903
|
+
* POST /api/v1/runs/open?project=&branch=&profile=&kind=&gitHead= — start a
|
|
13904
|
+
* "running" run with no report yet. Unlike `POST /runs`, nothing is pushed up
|
|
13905
|
+
* front: the caller patches results in as they finish (`PATCH /runs/:id`), so
|
|
13906
|
+
* an interrupted run still leaves a partial report on the hub instead of
|
|
13907
|
+
* none. `gitHead` (optional) attributes the run to a commit from the start —
|
|
13908
|
+
* without it an interrupted run would never learn its commit.
|
|
13522
13909
|
*/
|
|
13523
13910
|
function createOpenRunHandler(config) {
|
|
13524
13911
|
return async (ctx) => {
|
|
13525
13912
|
const { project, branch, profile, kind } = parseRunScope(ctx);
|
|
13913
|
+
const gitHead = ctx.url.searchParams.get("gitHead");
|
|
13526
13914
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
13527
13915
|
const run = {
|
|
13528
13916
|
id: randomUUID(),
|
|
@@ -13537,7 +13925,7 @@ function createOpenRunHandler(config) {
|
|
|
13537
13925
|
passed: 0,
|
|
13538
13926
|
failed: 0
|
|
13539
13927
|
},
|
|
13540
|
-
gitHead: null,
|
|
13928
|
+
gitHead: gitHead || null,
|
|
13541
13929
|
promptVersion: "",
|
|
13542
13930
|
ciRunId: null,
|
|
13543
13931
|
reportCreatedAt: now,
|
|
@@ -13553,10 +13941,7 @@ const PatchRunRequestSchema = z.object({
|
|
|
13553
13941
|
done: z.boolean().optional(),
|
|
13554
13942
|
finalStatus: z.enum(["passed", "failed"]).optional(),
|
|
13555
13943
|
reportMeta: z.object({
|
|
13556
|
-
git:
|
|
13557
|
-
head: z.string().nullable(),
|
|
13558
|
-
base: z.string().nullable()
|
|
13559
|
-
}).partial().optional(),
|
|
13944
|
+
git: GitEnvelopeSchema.partial().optional(),
|
|
13560
13945
|
model: z.string().nullable().optional(),
|
|
13561
13946
|
language: z.string().nullable().optional(),
|
|
13562
13947
|
promptVersion: z.string().optional(),
|
|
@@ -13580,6 +13965,38 @@ function countSpecs(results) {
|
|
|
13580
13965
|
};
|
|
13581
13966
|
}
|
|
13582
13967
|
/**
|
|
13968
|
+
* Advance the last-green ledger for every passed spec of a terminal
|
|
13969
|
+
* `kind: "run"` run. Spec-level, not run-level: a run with one chronically
|
|
13970
|
+
* failing spec still moves the baseline of every spec that did pass.
|
|
13971
|
+
* Best-effort — a ledger failure must not fail the push; the ledger is an
|
|
13972
|
+
* accelerator for `--failure-analysis=last-green`, not part of the run
|
|
13973
|
+
* record. Runs without a branch or gitHead can't be placed in the ledger and
|
|
13974
|
+
* are skipped.
|
|
13975
|
+
*
|
|
13976
|
+
* Ordering caveat (known approximation): `at` is the run's reportCreatedAt —
|
|
13977
|
+
* open time for incremental runs, report time for immutable pushes. When two
|
|
13978
|
+
* runs on the same branch+profile overlap, "newest at wins" can pick either
|
|
13979
|
+
* of the two genuinely-green commits, since the hub has no git ancestry to
|
|
13980
|
+
* order them properly. Accepted: CI serializes per branch in practice, and a
|
|
13981
|
+
* baseline can only ever point at a commit where the spec really passed.
|
|
13982
|
+
*/
|
|
13983
|
+
async function updateLastGreenLedger(storage, run, results) {
|
|
13984
|
+
const { gitHead, branch } = run;
|
|
13985
|
+
if (run.kind !== "run" || !gitHead || !branch) return;
|
|
13986
|
+
const passed = results.filter((r) => r.status === "passed");
|
|
13987
|
+
if (passed.length === 0) return;
|
|
13988
|
+
const entries = Object.fromEntries(passed.map((r) => [`${r.feature}/${r.spec}`, {
|
|
13989
|
+
gitHead,
|
|
13990
|
+
runId: run.id,
|
|
13991
|
+
at: run.reportCreatedAt
|
|
13992
|
+
}]));
|
|
13993
|
+
try {
|
|
13994
|
+
await storage.lastGreen.merge(run.project, run.profile ?? "default", branch, entries);
|
|
13995
|
+
} catch (err) {
|
|
13996
|
+
console.error(`hub: last-green ledger update failed for run "${run.id}": ${err instanceof Error ? err.message : String(err)}`);
|
|
13997
|
+
}
|
|
13998
|
+
}
|
|
13999
|
+
/**
|
|
13583
14000
|
* PATCH /api/v1/runs/:id — incrementally add spec results (and evidence) to a
|
|
13584
14001
|
* "running" run. Once the run is terminal (`done: true` was sent, or it was
|
|
13585
14002
|
* pushed immutably via `POST /runs`), further patches are rejected with 409.
|
|
@@ -13601,6 +14018,7 @@ function createPatchRunHandler(config) {
|
|
|
13601
14018
|
if (!parsed.success) throw new HttpError(400, "invalid_body", `request body is invalid: ${parsed.error.issues[0]?.message ?? "schema mismatch"}`);
|
|
13602
14019
|
const { rows, evidence, done, finalStatus, reportMeta } = parsed.data;
|
|
13603
14020
|
let specs = run.specs;
|
|
14021
|
+
let mergedResults = [];
|
|
13604
14022
|
await config.storage.artifacts.updateJsonFile(id, "report.json", (current) => {
|
|
13605
14023
|
const base = current ?? {
|
|
13606
14024
|
schemaVersion: 1,
|
|
@@ -13620,7 +14038,9 @@ function createPatchRunHandler(config) {
|
|
|
13620
14038
|
...base,
|
|
13621
14039
|
...reportMeta?.git ? { git: {
|
|
13622
14040
|
head: reportMeta.git.head ?? base.git.head,
|
|
13623
|
-
base: reportMeta.git.base ?? base.git.base
|
|
14041
|
+
base: reportMeta.git.base ?? base.git.base,
|
|
14042
|
+
baseSha: reportMeta.git.baseSha ?? base.git.baseSha ?? null,
|
|
14043
|
+
baseSource: reportMeta.git.baseSource ?? base.git.baseSource ?? null
|
|
13624
14044
|
} } : {},
|
|
13625
14045
|
...reportMeta?.model !== void 0 ? { model: reportMeta.model } : {},
|
|
13626
14046
|
...reportMeta?.language !== void 0 ? { language: reportMeta.language } : {},
|
|
@@ -13630,6 +14050,7 @@ function createPatchRunHandler(config) {
|
|
|
13630
14050
|
};
|
|
13631
14051
|
const merged = mergeResults(current?.results ?? [], rows);
|
|
13632
14052
|
specs = countSpecs(merged);
|
|
14053
|
+
mergedResults = merged;
|
|
13633
14054
|
return {
|
|
13634
14055
|
...envelope,
|
|
13635
14056
|
results: merged
|
|
@@ -13643,6 +14064,7 @@ function createPatchRunHandler(config) {
|
|
|
13643
14064
|
...reportMeta?.promptVersion ? { promptVersion: reportMeta.promptVersion } : {}
|
|
13644
14065
|
} : { specs };
|
|
13645
14066
|
const updated = await config.storage.runs.update(id, patch);
|
|
14067
|
+
if (done) await updateLastGreenLedger(config.storage, updated, mergedResults);
|
|
13646
14068
|
sendJson(ctx.res, 200, updated);
|
|
13647
14069
|
};
|
|
13648
14070
|
}
|
|
@@ -13759,9 +14181,11 @@ function parseRunScope(ctx) {
|
|
|
13759
14181
|
};
|
|
13760
14182
|
}
|
|
13761
14183
|
/**
|
|
13762
|
-
* A branch is a free-form label (e.g. `feature/foo`),
|
|
13763
|
-
*
|
|
13764
|
-
*
|
|
14184
|
+
* A branch is a free-form label (e.g. `feature/foo`), so `/` is allowed —
|
|
14185
|
+
* only length is bounded (a sanity cap; the last-green ledger separately
|
|
14186
|
+
* hash-truncates long percent-encoded names into a safe filename, see
|
|
14187
|
+
* paths.ts). Run records store it verbatim. null when the client didn't
|
|
14188
|
+
* send one. Exported for the last-green handler.
|
|
13765
14189
|
*/
|
|
13766
14190
|
function requireBranch(raw) {
|
|
13767
14191
|
if (raw === null || raw === "") return null;
|
|
@@ -14014,6 +14438,32 @@ function createListProfilesHandler(storage) {
|
|
|
14014
14438
|
};
|
|
14015
14439
|
}
|
|
14016
14440
|
//#endregion
|
|
14441
|
+
//#region src/hub/api/handlers/last-green.ts
|
|
14442
|
+
/**
|
|
14443
|
+
* GET /api/v1/projects/:project/last-green?profile=&branch=&fallbackBranch=
|
|
14444
|
+
*
|
|
14445
|
+
* Returns the last-green ledger entries for one project/profile, keyed by
|
|
14446
|
+
* "feature/spec". `branch` is the caller's current branch; `fallbackBranch`
|
|
14447
|
+
* (optional, typically the default branch) is overlaid *under* it, so a PR
|
|
14448
|
+
* branch with no greens of its own still inherits the default branch's
|
|
14449
|
+
* baselines while its own greens take precedence. One round trip serves the
|
|
14450
|
+
* whole run.
|
|
14451
|
+
*/
|
|
14452
|
+
function createGetLastGreenHandler(storage) {
|
|
14453
|
+
return async (ctx) => {
|
|
14454
|
+
const project = requireSafeSegment(ctx.params.project, "project");
|
|
14455
|
+
const profile = requireSafeSegment(ctx.url.searchParams.get("profile") ?? "default", "profile");
|
|
14456
|
+
const branch = requireBranch(ctx.url.searchParams.get("branch"));
|
|
14457
|
+
if (!branch) throw new HttpError(400, "missing_param", "branch query parameter is required");
|
|
14458
|
+
const fallbackBranch = requireBranch(ctx.url.searchParams.get("fallbackBranch"));
|
|
14459
|
+
const [primary, fallback] = await Promise.all([storage.lastGreen.get(project, profile, branch), fallbackBranch && fallbackBranch !== branch ? storage.lastGreen.get(project, profile, fallbackBranch) : Promise.resolve({})]);
|
|
14460
|
+
sendJson(ctx.res, 200, { entries: {
|
|
14461
|
+
...fallback,
|
|
14462
|
+
...primary
|
|
14463
|
+
} });
|
|
14464
|
+
};
|
|
14465
|
+
}
|
|
14466
|
+
//#endregion
|
|
14017
14467
|
//#region src/hub/api/handlers/prompts.ts
|
|
14018
14468
|
const MAX_PROMPT_BODY_BYTES = 256 * 1024;
|
|
14019
14469
|
/** Validate the `:project` route param (prompts are project-scoped, not per-profile). */
|
|
@@ -18022,7 +18472,7 @@ function createLearningWorker(deps) {
|
|
|
18022
18472
|
const generatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
18023
18473
|
const customPrompt = {
|
|
18024
18474
|
schemaVersion: 1,
|
|
18025
|
-
basePromptVersion: "
|
|
18475
|
+
basePromptVersion: "6",
|
|
18026
18476
|
customPromptVersion: `${generatedAt}-c${cases.length}`,
|
|
18027
18477
|
generatedAt,
|
|
18028
18478
|
guidance
|
|
@@ -18141,6 +18591,7 @@ function registerRoutes(router, config, queue) {
|
|
|
18141
18591
|
router.put("/api/v1/runs/:id/triage/actual-causes", createImportActualCausesHandler(storage));
|
|
18142
18592
|
router.get("/api/v1/projects", createListProjectsHandler(storage));
|
|
18143
18593
|
router.get("/api/v1/projects/:project/profiles", createListProfilesHandler(storage));
|
|
18594
|
+
router.get("/api/v1/projects/:project/last-green", createGetLastGreenHandler(storage));
|
|
18144
18595
|
const sessionConfig = {
|
|
18145
18596
|
store: storage.sessions,
|
|
18146
18597
|
encryptionKey: config.encryptionKey
|
|
@@ -18373,6 +18824,10 @@ function perspectivesKindDir(root) {
|
|
|
18373
18824
|
function perspectivesPath(root, project) {
|
|
18374
18825
|
return join(perspectivesKindDir(root), `${project}.json`);
|
|
18375
18826
|
}
|
|
18827
|
+
function lastGreenPath(root, project, profile, branch) {
|
|
18828
|
+
const encoded = encodeURIComponent(branch);
|
|
18829
|
+
return join(root, "last-green", project, profile, `${encoded.length <= 200 ? encoded : `${encoded.slice(0, 64)}-${createHash("sha256").update(branch).digest("hex").slice(0, 32)}`}.json`);
|
|
18830
|
+
}
|
|
18376
18831
|
//#endregion
|
|
18377
18832
|
//#region src/hub/core/storage/file/artifact-store.ts
|
|
18378
18833
|
/**
|
|
@@ -18460,6 +18915,25 @@ function createFileJobStore(root) {
|
|
|
18460
18915
|
};
|
|
18461
18916
|
}
|
|
18462
18917
|
//#endregion
|
|
18918
|
+
//#region src/hub/core/storage/file/last-green-store.ts
|
|
18919
|
+
function createFileLastGreenStore(root) {
|
|
18920
|
+
return {
|
|
18921
|
+
async get(project, profile, branch) {
|
|
18922
|
+
return await readJson(lastGreenPath(root, project, profile, branch)) ?? {};
|
|
18923
|
+
},
|
|
18924
|
+
async merge(project, profile, branch, entries) {
|
|
18925
|
+
await updateJson(lastGreenPath(root, project, profile, branch), (current) => {
|
|
18926
|
+
const out = { ...current ?? {} };
|
|
18927
|
+
for (const [key, entry] of Object.entries(entries)) {
|
|
18928
|
+
const prev = out[key];
|
|
18929
|
+
if (!prev || prev.at <= entry.at) out[key] = entry;
|
|
18930
|
+
}
|
|
18931
|
+
return out;
|
|
18932
|
+
});
|
|
18933
|
+
}
|
|
18934
|
+
};
|
|
18935
|
+
}
|
|
18936
|
+
//#endregion
|
|
18463
18937
|
//#region src/hub/core/storage/file/perspectives-store.ts
|
|
18464
18938
|
/**
|
|
18465
18939
|
* Defense-in-depth path validation: the HTTP layer already checks the project
|
|
@@ -18713,7 +19187,8 @@ function createFileHubStorage(dataDir) {
|
|
|
18713
19187
|
triage: createFileTriageStore(dataDir),
|
|
18714
19188
|
prompts: createFilePromptStore(dataDir),
|
|
18715
19189
|
perspectives: createFilePerspectivesStore(dataDir),
|
|
18716
|
-
jobs: createFileJobStore(dataDir)
|
|
19190
|
+
jobs: createFileJobStore(dataDir),
|
|
19191
|
+
lastGreen: createFileLastGreenStore(dataDir)
|
|
18717
19192
|
};
|
|
18718
19193
|
}
|
|
18719
19194
|
//#endregion
|