ccqa 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/bin/ccqa.mjs +840 -357
- package/dist/hub-client/index.d.mts +41 -0
- package/dist/hub-client/index.mjs +10 -0
- package/dist/package.json +11 -1
- package/package.json +11 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -11,10 +11,10 @@ import { basename, dirname, isAbsolute, join, normalize, posix, relative, resolv
|
|
|
11
11
|
import { parse, stringify } from "yaml";
|
|
12
12
|
import { ZodError, z } from "zod";
|
|
13
13
|
import { execFile, spawn, spawnSync } from "node:child_process";
|
|
14
|
-
import { query } from "@anthropic-ai/claude-agent-sdk";
|
|
14
|
+
import { createSdkMcpServer, query, tool } from "@anthropic-ai/claude-agent-sdk";
|
|
15
15
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
16
|
-
import { promisify } from "node:util";
|
|
17
16
|
import { createCipheriv, createDecipheriv, createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
|
|
17
|
+
import { promisify } from "node:util";
|
|
18
18
|
import { gunzipSync, gzipSync } from "node:zlib";
|
|
19
19
|
import { createInterface } from "node:readline";
|
|
20
20
|
import { createInterface as createInterface$1 } from "node:readline/promises";
|
|
@@ -679,10 +679,10 @@ function bundledVitestConfigPath() {
|
|
|
679
679
|
}
|
|
680
680
|
//#endregion
|
|
681
681
|
//#region src/runtime/spawn-vitest.ts
|
|
682
|
-
const require$
|
|
682
|
+
const require$2 = createRequire(import.meta.url);
|
|
683
683
|
function resolveVitestBin() {
|
|
684
|
-
const pkgPath = require$
|
|
685
|
-
const pkg = require$
|
|
684
|
+
const pkgPath = require$2.resolve("vitest/package.json");
|
|
685
|
+
const pkg = require$2(pkgPath);
|
|
686
686
|
const binRel = typeof pkg.bin === "string" ? pkg.bin : pkg.bin?.vitest;
|
|
687
687
|
if (!binRel) throw new Error(`vitest package.json has no bin entry (resolved at ${pkgPath})`);
|
|
688
688
|
return resolve(dirname(pkgPath), binRel);
|
|
@@ -1194,6 +1194,50 @@ function opt(key, value) {
|
|
|
1194
1194
|
return value ? { [key]: value } : {};
|
|
1195
1195
|
}
|
|
1196
1196
|
//#endregion
|
|
1197
|
+
//#region src/claude/native-binary.ts
|
|
1198
|
+
const require$1 = createRequire(import.meta.url);
|
|
1199
|
+
/**
|
|
1200
|
+
* The agent SDK launches Claude through a native `claude` binary that ships in
|
|
1201
|
+
* a per-platform package (`@anthropic-ai/claude-agent-sdk-<platform>-<cpu>`),
|
|
1202
|
+
* declared as an *optional* dependency of the SDK. Optional means a consumer's
|
|
1203
|
+
* lockfile can omit it without any install-time error — and then every Claude
|
|
1204
|
+
* call fails at runtime with a message that never reaches our logs. Resolving
|
|
1205
|
+
* the package up front lets us say so once, in a line that names the fix.
|
|
1206
|
+
*
|
|
1207
|
+
* ccqa's own package.json repeats these packages in `optionalDependencies` for
|
|
1208
|
+
* the same reason: a second declaration gives the resolver another chance to
|
|
1209
|
+
* record them. Keep that list's version range in step with the SDK's.
|
|
1210
|
+
*/
|
|
1211
|
+
function nativeBinaryPackage(platform = process.platform, arch = process.arch, musl = isMusl(platform)) {
|
|
1212
|
+
return `@anthropic-ai/claude-agent-sdk-${platform}-${arch === "arm64" ? "arm64" : "x64"}${platform === "linux" && musl ? "-musl" : ""}`;
|
|
1213
|
+
}
|
|
1214
|
+
/**
|
|
1215
|
+
* musl builds (Alpine and friends) need their own binary. Node doesn't expose
|
|
1216
|
+
* the libc flavour directly; the absence of `glibcVersionRuntime` in the
|
|
1217
|
+
* process report is the usual proxy.
|
|
1218
|
+
*/
|
|
1219
|
+
function isMusl(platform) {
|
|
1220
|
+
if (platform !== "linux") return false;
|
|
1221
|
+
return !(process.report?.getReport?.())?.header?.glibcVersionRuntime;
|
|
1222
|
+
}
|
|
1223
|
+
/**
|
|
1224
|
+
* Name of the platform package this host needs, or `null` when it resolves.
|
|
1225
|
+
* The per-platform packages have no `exports`, so the manifest is reachable.
|
|
1226
|
+
*/
|
|
1227
|
+
function missingNativeBinaryPackage(resolve = require$1.resolve) {
|
|
1228
|
+
const pkg = nativeBinaryPackage();
|
|
1229
|
+
try {
|
|
1230
|
+
resolve(`${pkg}/package.json`);
|
|
1231
|
+
return null;
|
|
1232
|
+
} catch {
|
|
1233
|
+
return pkg;
|
|
1234
|
+
}
|
|
1235
|
+
}
|
|
1236
|
+
/** Advice shown when the binary is absent — the package name plus how to fix it. */
|
|
1237
|
+
function missingNativeBinaryMessage(pkg) {
|
|
1238
|
+
return `${pkg} is not installed. The Claude Agent SDK needs it to start Claude on this platform, so every Claude-backed command (run in live mode, drift, diagnose) will fail. It ships as an optional dependency of the SDK, which a lockfile can drop silently: reinstall without omitting optional dependencies, or add it to your project as a direct dependency pinned to the same version as @anthropic-ai/claude-agent-sdk.`;
|
|
1239
|
+
}
|
|
1240
|
+
//#endregion
|
|
1197
1241
|
//#region src/claude/invoke.ts
|
|
1198
1242
|
function resolveModel(explicit) {
|
|
1199
1243
|
if (explicit) return explicit;
|
|
@@ -1230,8 +1274,20 @@ function resolveEndpointEnv() {
|
|
|
1230
1274
|
}
|
|
1231
1275
|
return endpointEnv;
|
|
1232
1276
|
}
|
|
1277
|
+
let nativeBinaryWarned = false;
|
|
1278
|
+
/**
|
|
1279
|
+
* Warn once per process when the SDK's per-platform native binary is missing:
|
|
1280
|
+
* every Claude call is about to fail, and the opaque per-step errors alone are
|
|
1281
|
+
* expensive to trace back to a lockfile that dropped an optional dependency.
|
|
1282
|
+
*/
|
|
1283
|
+
function warnOnceIfNativeBinaryMissing() {
|
|
1284
|
+
if (nativeBinaryWarned) return;
|
|
1285
|
+
nativeBinaryWarned = true;
|
|
1286
|
+
const missing = missingNativeBinaryPackage();
|
|
1287
|
+
if (missing) warn(missingNativeBinaryMessage(missing));
|
|
1288
|
+
}
|
|
1233
1289
|
async function invokeClaudeStreaming(options, onEvent) {
|
|
1234
|
-
const { prompt, systemPrompt, allowedTools, disableBuiltinTools = false, disableThinking = false, maxTurns, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, relaxAbConstraints = false } = options;
|
|
1290
|
+
const { prompt, systemPrompt, allowedTools, disableBuiltinTools = false, disableThinking = false, mcpServers, maxTurns, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, relaxAbConstraints = false } = options;
|
|
1235
1291
|
const resolvedModel = resolveModel(model);
|
|
1236
1292
|
const hasEndpointEnv = Object.keys(resolveEndpointEnv()).length > 0;
|
|
1237
1293
|
const mergedEnv = env || hasEndpointEnv ? {
|
|
@@ -1253,6 +1309,7 @@ async function invokeClaudeStreaming(options, onEvent) {
|
|
|
1253
1309
|
...resolvedModel ? { model: resolvedModel } : {},
|
|
1254
1310
|
...cwd ? { cwd } : {},
|
|
1255
1311
|
...mergedEnv ? { env: mergedEnv } : {},
|
|
1312
|
+
...mcpServers ? { mcpServers } : {},
|
|
1256
1313
|
...disableBuiltinTools ? { tools: [] } : {},
|
|
1257
1314
|
...disableThinking ? { thinking: { type: "disabled" } } : {},
|
|
1258
1315
|
hooks: onAbAction || onAbActionFailed ? {
|
|
@@ -1312,8 +1369,10 @@ async function invokeClaudeStreaming(options, onEvent) {
|
|
|
1312
1369
|
}] }]
|
|
1313
1370
|
} : void 0
|
|
1314
1371
|
};
|
|
1372
|
+
warnOnceIfNativeBinaryMissing();
|
|
1315
1373
|
let result = "";
|
|
1316
1374
|
let isError = false;
|
|
1375
|
+
let errorDetail = null;
|
|
1317
1376
|
let cost = {
|
|
1318
1377
|
totalCostUsd: null,
|
|
1319
1378
|
durationMs: null,
|
|
@@ -1336,18 +1395,24 @@ async function invokeClaudeStreaming(options, onEvent) {
|
|
|
1336
1395
|
}
|
|
1337
1396
|
}
|
|
1338
1397
|
if (msg.type === "result") {
|
|
1339
|
-
result = msg.subtype === "success" ? msg.result : "";
|
|
1340
1398
|
isError = msg.is_error ?? false;
|
|
1399
|
+
if (msg.subtype === "success") result = msg.result;
|
|
1400
|
+
else {
|
|
1401
|
+
result = "";
|
|
1402
|
+
errorDetail = `SDK reported ${msg.subtype}`;
|
|
1403
|
+
}
|
|
1341
1404
|
cost = extractInvocationCost(msg);
|
|
1342
1405
|
}
|
|
1343
1406
|
}
|
|
1344
1407
|
} catch (err) {
|
|
1345
1408
|
isError = true;
|
|
1346
|
-
|
|
1409
|
+
errorDetail = err instanceof Error ? err.message : String(err);
|
|
1410
|
+
if (!result) result = errorDetail;
|
|
1347
1411
|
}
|
|
1348
1412
|
return {
|
|
1349
1413
|
result,
|
|
1350
1414
|
isError,
|
|
1415
|
+
errorDetail,
|
|
1351
1416
|
cost
|
|
1352
1417
|
};
|
|
1353
1418
|
}
|
|
@@ -2220,171 +2285,6 @@ async function checkSpec(target, opts) {
|
|
|
2220
2285
|
};
|
|
2221
2286
|
}
|
|
2222
2287
|
//#endregion
|
|
2223
|
-
//#region src/drift/affected.ts
|
|
2224
|
-
const execFileP = promisify(execFile);
|
|
2225
|
-
/**
|
|
2226
|
-
* Resolve the base ref to diff against for `ccqa drift --changed`.
|
|
2227
|
-
* Precedence: explicit override > GITHUB_BASE_REF > origin/main.
|
|
2228
|
-
*/
|
|
2229
|
-
function resolveBaseRef(explicit) {
|
|
2230
|
-
if (explicit && explicit.length > 0) return explicit;
|
|
2231
|
-
const ghBase = process.env["GITHUB_BASE_REF"];
|
|
2232
|
-
if (ghBase && ghBase.length > 0) return ghBase.startsWith("origin/") ? ghBase : `origin/${ghBase}`;
|
|
2233
|
-
return "origin/main";
|
|
2234
|
-
}
|
|
2235
|
-
/**
|
|
2236
|
-
* Run `git diff --name-status base...HEAD` from `cwd` and return one entry per
|
|
2237
|
-
* changed file. Renames are reported under their NEW path with status
|
|
2238
|
-
* "renamed" — the OLD path is dropped because the spec mapping is against the
|
|
2239
|
-
* post-rename layout.
|
|
2240
|
-
*
|
|
2241
|
-
* Paths are re-rooted to be relative to `cwd`, not the git repo root. In a
|
|
2242
|
-
* monorepo where `cwd` is a sub-package (e.g. `apps/foo`), git emits paths
|
|
2243
|
-
* relative to the repo root, but specs declare relatedPaths relative to
|
|
2244
|
-
* their own package. Changes outside `cwd` are kept under their repo-root
|
|
2245
|
-
* path and flagged `outsideCwd` — they only scope a spec in when the spec
|
|
2246
|
-
* explicitly declares a repo-root-relative glob, so an unrelated PR can
|
|
2247
|
-
* never accidentally match the app-relative globs.
|
|
2248
|
-
*/
|
|
2249
|
-
async function getChangedFiles(base, cwd) {
|
|
2250
|
-
const [{ stdout: rootOut }, { stdout: diffOut }] = await Promise.all([execFileP("git", ["rev-parse", "--show-toplevel"], { cwd }), execFileP("git", [
|
|
2251
|
-
"diff",
|
|
2252
|
-
"--name-status",
|
|
2253
|
-
"-M",
|
|
2254
|
-
`${base}...HEAD`
|
|
2255
|
-
], {
|
|
2256
|
-
cwd,
|
|
2257
|
-
maxBuffer: 32 * 1024 * 1024
|
|
2258
|
-
})]);
|
|
2259
|
-
return rerootChangedFiles(parseGitDiffOutput(diffOut), rootOut.trim(), cwd);
|
|
2260
|
-
}
|
|
2261
|
-
/**
|
|
2262
|
-
* Convert paths in `entries` from git-repo-root relative to `cwd` relative.
|
|
2263
|
-
* Entries outside `cwd` keep their repo-root path and are flagged
|
|
2264
|
-
* `outsideCwd`. Exported for unit tests.
|
|
2265
|
-
*/
|
|
2266
|
-
function rerootChangedFiles(entries, repoRoot, cwd) {
|
|
2267
|
-
const prefix = relative(repoRoot, cwd);
|
|
2268
|
-
if (!prefix) return entries;
|
|
2269
|
-
const out = [];
|
|
2270
|
-
for (const e of entries) {
|
|
2271
|
-
const rel = relative(prefix, e.path);
|
|
2272
|
-
if (rel.startsWith("..") || rel === "") out.push({
|
|
2273
|
-
...e,
|
|
2274
|
-
outsideCwd: true
|
|
2275
|
-
});
|
|
2276
|
-
else out.push({
|
|
2277
|
-
...e,
|
|
2278
|
-
path: rel
|
|
2279
|
-
});
|
|
2280
|
-
}
|
|
2281
|
-
return out;
|
|
2282
|
-
}
|
|
2283
|
-
function parseGitDiffOutput(stdout) {
|
|
2284
|
-
const out = [];
|
|
2285
|
-
for (const line of stdout.split("\n")) {
|
|
2286
|
-
if (!line.trim()) continue;
|
|
2287
|
-
const parts = line.split(" ");
|
|
2288
|
-
const code = parts[0];
|
|
2289
|
-
if (!code) continue;
|
|
2290
|
-
if (code.startsWith("R")) {
|
|
2291
|
-
const newPath = parts[2];
|
|
2292
|
-
if (newPath) out.push({
|
|
2293
|
-
path: newPath,
|
|
2294
|
-
status: "renamed"
|
|
2295
|
-
});
|
|
2296
|
-
continue;
|
|
2297
|
-
}
|
|
2298
|
-
if (code.startsWith("C")) {
|
|
2299
|
-
const newPath = parts[2];
|
|
2300
|
-
if (newPath) out.push({
|
|
2301
|
-
path: newPath,
|
|
2302
|
-
status: "added"
|
|
2303
|
-
});
|
|
2304
|
-
continue;
|
|
2305
|
-
}
|
|
2306
|
-
const path = parts[1];
|
|
2307
|
-
if (!path) continue;
|
|
2308
|
-
switch (code[0]) {
|
|
2309
|
-
case "A":
|
|
2310
|
-
out.push({
|
|
2311
|
-
path,
|
|
2312
|
-
status: "added"
|
|
2313
|
-
});
|
|
2314
|
-
break;
|
|
2315
|
-
case "M":
|
|
2316
|
-
case "T":
|
|
2317
|
-
out.push({
|
|
2318
|
-
path,
|
|
2319
|
-
status: "modified"
|
|
2320
|
-
});
|
|
2321
|
-
break;
|
|
2322
|
-
case "D":
|
|
2323
|
-
out.push({
|
|
2324
|
-
path,
|
|
2325
|
-
status: "deleted"
|
|
2326
|
-
});
|
|
2327
|
-
break;
|
|
2328
|
-
default: out.push({
|
|
2329
|
-
path,
|
|
2330
|
-
status: "modified"
|
|
2331
|
-
});
|
|
2332
|
-
}
|
|
2333
|
-
}
|
|
2334
|
-
return out;
|
|
2335
|
-
}
|
|
2336
|
-
function stripLeadingDotSlash(s) {
|
|
2337
|
-
return s.startsWith("./") ? s.slice(2) : s;
|
|
2338
|
-
}
|
|
2339
|
-
const REGEX_CACHE = /* @__PURE__ */ new Map();
|
|
2340
|
-
/** Compiles `pattern` to a RegExp, memoized so repeated `--changed` matches don't re-build. */
|
|
2341
|
-
function compileGlob(pattern) {
|
|
2342
|
-
const cached = REGEX_CACHE.get(pattern);
|
|
2343
|
-
if (cached) return cached;
|
|
2344
|
-
const compiled = globToRegExp(stripLeadingDotSlash(pattern));
|
|
2345
|
-
REGEX_CACHE.set(pattern, compiled);
|
|
2346
|
-
return compiled;
|
|
2347
|
-
}
|
|
2348
|
-
function globToRegExp(pattern) {
|
|
2349
|
-
let re = "^";
|
|
2350
|
-
let i = 0;
|
|
2351
|
-
while (i < pattern.length) {
|
|
2352
|
-
const ch = pattern[i];
|
|
2353
|
-
if (ch === "?") {
|
|
2354
|
-
re += "[^/]";
|
|
2355
|
-
i++;
|
|
2356
|
-
continue;
|
|
2357
|
-
}
|
|
2358
|
-
if (ch !== "*") {
|
|
2359
|
-
re += /[.+^${}()|[\]\\]/.test(ch) ? "\\" + ch : ch;
|
|
2360
|
-
i++;
|
|
2361
|
-
continue;
|
|
2362
|
-
}
|
|
2363
|
-
if (pattern[i + 1] !== "*") {
|
|
2364
|
-
re += "[^/]*";
|
|
2365
|
-
i++;
|
|
2366
|
-
continue;
|
|
2367
|
-
}
|
|
2368
|
-
const hasLeadingSlash = re.endsWith("/");
|
|
2369
|
-
const hasTrailingSlash = pattern[i + 2] === "/";
|
|
2370
|
-
if (hasLeadingSlash) re = re.slice(0, -1);
|
|
2371
|
-
if (hasLeadingSlash || hasTrailingSlash) re += "(?:/?.*)?";
|
|
2372
|
-
else re += ".*";
|
|
2373
|
-
i += hasTrailingSlash ? 3 : 2;
|
|
2374
|
-
}
|
|
2375
|
-
return new RegExp(re + "$");
|
|
2376
|
-
}
|
|
2377
|
-
/**
|
|
2378
|
-
* Returns true if `changedPath` is covered by any of `relatedPaths`. An empty
|
|
2379
|
-
* `relatedPaths` returns false — callers handle the "unscoped spec" case
|
|
2380
|
-
* separately (treat the spec as always-affected) before calling this.
|
|
2381
|
-
*/
|
|
2382
|
-
function isPathAffectedBy(changedPath, relatedPaths) {
|
|
2383
|
-
const stripped = stripLeadingDotSlash(changedPath);
|
|
2384
|
-
for (const pattern of relatedPaths) if (compileGlob(pattern).test(stripped)) return true;
|
|
2385
|
-
return false;
|
|
2386
|
-
}
|
|
2387
|
-
//#endregion
|
|
2388
2288
|
//#region src/drift/auth.ts
|
|
2389
2289
|
/**
|
|
2390
2290
|
* Probe whether the host has any credential the Anthropic SDK can pick up:
|
|
@@ -3089,6 +2989,10 @@ const ReportSpecResultSchema = z.object({
|
|
|
3089
2989
|
assertions: z.array(ReportAssertionSchema).nullable(),
|
|
3090
2990
|
analysis: FailureAnalysisSchema.nullable(),
|
|
3091
2991
|
analysisSkipped: z.string().nullable(),
|
|
2992
|
+
analysisBase: z.object({
|
|
2993
|
+
ref: z.string(),
|
|
2994
|
+
sha: z.string()
|
|
2995
|
+
}).nullable().optional(),
|
|
3092
2996
|
driftIssues: z.array(DraftIssueSchema).nullable(),
|
|
3093
2997
|
failureLogExcerpt: z.string().nullable(),
|
|
3094
2998
|
diffExcerpt: z.string().nullable(),
|
|
@@ -3097,15 +3001,33 @@ const ReportSpecResultSchema = z.object({
|
|
|
3097
3001
|
artifacts: z.array(ReportArtifactSchema).optional(),
|
|
3098
3002
|
liveRun: LiveReportRunSchema.nullable()
|
|
3099
3003
|
});
|
|
3004
|
+
/**
|
|
3005
|
+
* Which rule produced the analysis baseline: "explicit" (a value was
|
|
3006
|
+
* passed), "github-base-ref" (derived from a pull_request event), or
|
|
3007
|
+
* "last-green" (per-spec baselines from the hub ledger — `baseSha` is then
|
|
3008
|
+
* null and each analyzed row carries its own `analysisBase`). Lets accuracy
|
|
3009
|
+
* numbers be stratified by baseline provenance. Single source of truth —
|
|
3010
|
+
* `src/run/git-context.ts`'s `BaseSource` type and the hub's PATCH schema
|
|
3011
|
+
* both derive from this.
|
|
3012
|
+
*/
|
|
3013
|
+
const BaseSourceSchema = z.enum([
|
|
3014
|
+
"explicit",
|
|
3015
|
+
"github-base-ref",
|
|
3016
|
+
"last-green"
|
|
3017
|
+
]);
|
|
3018
|
+
/** The report envelope's git block; also referenced by the hub's PATCH reportMeta schema. */
|
|
3019
|
+
const GitEnvelopeSchema = z.object({
|
|
3020
|
+
head: z.string().nullable(),
|
|
3021
|
+
base: z.string().nullable(),
|
|
3022
|
+
baseSha: z.string().nullable().optional(),
|
|
3023
|
+
baseSource: BaseSourceSchema.nullable().optional()
|
|
3024
|
+
});
|
|
3100
3025
|
const RunReportDataSchema = z.object({
|
|
3101
3026
|
schemaVersion: z.literal(1),
|
|
3102
3027
|
kind: z.enum(["run", "drift"]).default("run"),
|
|
3103
3028
|
createdAt: z.string(),
|
|
3104
3029
|
runId: z.string().nullable(),
|
|
3105
|
-
git:
|
|
3106
|
-
head: z.string().nullable(),
|
|
3107
|
-
base: z.string().nullable()
|
|
3108
|
-
}),
|
|
3030
|
+
git: GitEnvelopeSchema,
|
|
3109
3031
|
model: z.string().nullable(),
|
|
3110
3032
|
language: z.string().nullable().default(null),
|
|
3111
3033
|
promptVersion: z.string(),
|
|
@@ -3230,6 +3152,14 @@ function hashTriageUserPrompt(text) {
|
|
|
3230
3152
|
}
|
|
3231
3153
|
//#endregion
|
|
3232
3154
|
//#region src/report/prompt.ts
|
|
3155
|
+
/**
|
|
3156
|
+
* Fully-qualified name of the on-demand file-diff tool, as the model calls
|
|
3157
|
+
* it. Lives here (not analyze.ts) because the prompt text below references
|
|
3158
|
+
* it — one source of truth. The name is the SDK's `mcp__<server>__<tool>`
|
|
3159
|
+
* composition of the server ("diff") and tool ("changed_file_diff") that
|
|
3160
|
+
* analyze.ts registers; changing either side must keep the two in sync.
|
|
3161
|
+
*/
|
|
3162
|
+
const CHANGED_FILE_DIFF_TOOL = "mcp__diff__changed_file_diff";
|
|
3233
3163
|
function buildFailureAnalysisPrompt(input) {
|
|
3234
3164
|
const { script, specYaml, failureLog, liveTranscriptExcerpt, diffPatch, changedFiles, baseRef, driftIssues, outputLanguage = "auto", triageUserPrompt, customPrompt } = input;
|
|
3235
3165
|
const triageUserPromptBlock = buildTriageUserPromptBlock(triageUserPrompt);
|
|
@@ -3253,6 +3183,8 @@ You can call \`Grep\`, \`Glob\`, and \`Read\` against the current repository (po
|
|
|
3253
3183
|
- read the changed files in full when the truncated patch is not enough,
|
|
3254
3184
|
- check whether the element/flow the spec describes still exists in the source.
|
|
3255
3185
|
|
|
3186
|
+
You can also call \`${CHANGED_FILE_DIFF_TOOL}\` with a file path to fetch that file's diff hunk for this run's base...HEAD range. The inline patch below is scoped to this spec's relatedPaths — files OUTSIDE that scope still appear in "Changed files (name-status)" but their hunks are not inlined. Before blaming (or ruling out) such a file, fetch its diff with this tool; Read only shows you its post-change state, not what changed.
|
|
3187
|
+
|
|
3256
3188
|
You have **up to 12 tool turns**. Do NOT write, edit, run shell commands, or hit the network.
|
|
3257
3189
|
|
|
3258
3190
|
## Decision guidance
|
|
@@ -3357,19 +3289,45 @@ ${liveTranscriptExcerpt}`);
|
|
|
3357
3289
|
//#endregion
|
|
3358
3290
|
//#region src/report/analyze.ts
|
|
3359
3291
|
/**
|
|
3292
|
+
* In-process MCP server exposing one tool: the diff hunk of a named changed
|
|
3293
|
+
* file. The inline patch in the prompt is only the relatedPaths-scoped seed;
|
|
3294
|
+
* this is the pull side — the model fetches hunks for files outside that
|
|
3295
|
+
* scope (or truncated inside it) only when it decides they matter, so the
|
|
3296
|
+
* full diff never has to ride in the prompt. Read-only over data already
|
|
3297
|
+
* captured in memory: no shell, no git access granted. The server/tool
|
|
3298
|
+
* names must compose to CHANGED_FILE_DIFF_TOOL (prompt.ts), which is how
|
|
3299
|
+
* the prompt tells the model to call it.
|
|
3300
|
+
*/
|
|
3301
|
+
function buildDiffMcpServer(getFileDiff) {
|
|
3302
|
+
return createSdkMcpServer({
|
|
3303
|
+
name: "diff",
|
|
3304
|
+
version: "1.0.0",
|
|
3305
|
+
tools: [tool("changed_file_diff", "Return the unified diff (base...HEAD) of one changed file from this run's diff range. Works for ANY file listed in 'Changed files (name-status)', including files outside the spec's relatedPaths scope whose hunks are not in the inline patch.", { path: z.string().describe("File path exactly as it appears in the name-status list") }, async ({ path }) => {
|
|
3306
|
+
const hunk = getFileDiff(path);
|
|
3307
|
+
if (hunk) info(` diff tool: ${path}`);
|
|
3308
|
+
return { content: [{
|
|
3309
|
+
type: "text",
|
|
3310
|
+
text: hunk ?? `No diff found for "${path}" in this run's diff range. Check the exact path in the name-status list (paths are relative to the working directory).`
|
|
3311
|
+
}] };
|
|
3312
|
+
})]
|
|
3313
|
+
});
|
|
3314
|
+
}
|
|
3315
|
+
/**
|
|
3360
3316
|
* Classify one failing spec into TEST_DRIFT / SPEC_CHANGE / PRODUCT_BUG /
|
|
3361
3317
|
* UNKNOWN. Same resilience contract as diagnose(): read-only tools, JSON-only
|
|
3362
3318
|
* final message, and any parse failure degrades to UNKNOWN with confidence 0
|
|
3363
3319
|
* rather than throwing — the report must always render.
|
|
3364
3320
|
*/
|
|
3365
|
-
async function analyzeFailure(input, options
|
|
3321
|
+
async function analyzeFailure(input, options) {
|
|
3366
3322
|
const { result: raw, isError } = await invokeClaudeStreaming({
|
|
3367
3323
|
prompt: buildFailureAnalysisPrompt(input),
|
|
3368
3324
|
allowedTools: [
|
|
3369
3325
|
"Read",
|
|
3370
3326
|
"Grep",
|
|
3371
|
-
"Glob"
|
|
3327
|
+
"Glob",
|
|
3328
|
+
CHANGED_FILE_DIFF_TOOL
|
|
3372
3329
|
],
|
|
3330
|
+
mcpServers: { diff: buildDiffMcpServer(options.getFileDiff) },
|
|
3373
3331
|
silenceBashLog: true,
|
|
3374
3332
|
maxTurns: 12,
|
|
3375
3333
|
...options.model ? { model: options.model } : {},
|
|
@@ -3411,44 +3369,223 @@ function unknownAnalysis(reasoning) {
|
|
|
3411
3369
|
reasoning
|
|
3412
3370
|
};
|
|
3413
3371
|
}
|
|
3414
|
-
const LABELS = new Set(PREDICTED_LABELS);
|
|
3415
|
-
const SUB_SET = new Set(SUB_DIAGNOSES);
|
|
3372
|
+
const LABELS = new Set(PREDICTED_LABELS);
|
|
3373
|
+
const SUB_SET = new Set(SUB_DIAGNOSES);
|
|
3374
|
+
/**
|
|
3375
|
+
* Manual, lenient normalisation (mirrors diagnose's normaliseResult): a
|
|
3376
|
+
* missing/extra field should degrade gracefully, not reject the whole
|
|
3377
|
+
* prediction — only an unrecognisable label makes the candidate unusable.
|
|
3378
|
+
*/
|
|
3379
|
+
function normaliseFailureAnalysis(parsed) {
|
|
3380
|
+
if (!isObject(parsed)) return null;
|
|
3381
|
+
const label = parsed["label"];
|
|
3382
|
+
if (typeof label !== "string" || !LABELS.has(label)) return null;
|
|
3383
|
+
const confidence = typeof parsed["confidence"] === "number" ? clamp(parsed["confidence"], 0, 1) : 0;
|
|
3384
|
+
const reasoning = typeof parsed["reasoning"] === "string" ? parsed["reasoning"] : "";
|
|
3385
|
+
const headline = typeof parsed["headline"] === "string" ? parsed["headline"] : "";
|
|
3386
|
+
const recommendation = typeof parsed["recommendation"] === "string" ? parsed["recommendation"] : "";
|
|
3387
|
+
const rawSub = parsed["subDiagnosis"];
|
|
3388
|
+
const subDiagnosis = typeof rawSub === "string" && SUB_SET.has(rawSub) ? rawSub : "NONE";
|
|
3389
|
+
const evidence = [];
|
|
3390
|
+
if (Array.isArray(parsed["evidence"])) for (const item of parsed["evidence"]) {
|
|
3391
|
+
if (!isObject(item)) continue;
|
|
3392
|
+
const detail = typeof item["detail"] === "string" ? item["detail"] : null;
|
|
3393
|
+
if (detail === null) continue;
|
|
3394
|
+
const file = typeof item["file"] === "string" ? item["file"] : void 0;
|
|
3395
|
+
evidence.push(file !== void 0 ? {
|
|
3396
|
+
file,
|
|
3397
|
+
detail
|
|
3398
|
+
} : { detail });
|
|
3399
|
+
if (evidence.length >= 3) break;
|
|
3400
|
+
}
|
|
3401
|
+
return {
|
|
3402
|
+
label,
|
|
3403
|
+
confidence,
|
|
3404
|
+
subDiagnosis,
|
|
3405
|
+
headline,
|
|
3406
|
+
recommendation,
|
|
3407
|
+
evidence,
|
|
3408
|
+
reasoning
|
|
3409
|
+
};
|
|
3410
|
+
}
|
|
3411
|
+
//#endregion
|
|
3412
|
+
//#region src/drift/affected.ts
|
|
3413
|
+
const execFileP = promisify(execFile);
|
|
3414
|
+
/**
|
|
3415
|
+
* GITHUB_BASE_REF holds a bare branch name (e.g. "main"); the local checkout
|
|
3416
|
+
* only has it as a remote-tracking ref, so prefix `origin/` unless already
|
|
3417
|
+
* qualified. Shared by `ccqa drift`'s resolveBaseRef and `ccqa run`'s
|
|
3418
|
+
* resolveAnalysisBase so the rule can't drift between them.
|
|
3419
|
+
*/
|
|
3420
|
+
function normalizeGithubBaseRef(ref) {
|
|
3421
|
+
return ref.startsWith("origin/") ? ref : `origin/${ref}`;
|
|
3422
|
+
}
|
|
3423
|
+
/**
|
|
3424
|
+
* Resolve the base ref to diff against for `ccqa drift --changed`.
|
|
3425
|
+
* Precedence: explicit override > GITHUB_BASE_REF > origin/main.
|
|
3426
|
+
*
|
|
3427
|
+
* Note: this is the `ccqa drift` rule. `ccqa run` resolves its baseline via
|
|
3428
|
+
* `src/run/git-context.ts` instead, which has no origin/main fallback — see
|
|
3429
|
+
* the rationale there.
|
|
3430
|
+
*/
|
|
3431
|
+
function resolveBaseRef(explicit) {
|
|
3432
|
+
if (explicit && explicit.length > 0) return explicit;
|
|
3433
|
+
const ghBase = process.env["GITHUB_BASE_REF"];
|
|
3434
|
+
if (ghBase && ghBase.length > 0) return normalizeGithubBaseRef(ghBase);
|
|
3435
|
+
return "origin/main";
|
|
3436
|
+
}
|
|
3437
|
+
/**
|
|
3438
|
+
* Run `git diff --name-status base...HEAD` from `cwd` and return one entry per
|
|
3439
|
+
* changed file. Renames are reported under their NEW path with status
|
|
3440
|
+
* "renamed" — the OLD path is dropped because the spec mapping is against the
|
|
3441
|
+
* post-rename layout.
|
|
3442
|
+
*
|
|
3443
|
+
* Paths are re-rooted to be relative to `cwd`, not the git repo root. In a
|
|
3444
|
+
* monorepo where `cwd` is a sub-package (e.g. `apps/foo`), git emits paths
|
|
3445
|
+
* relative to the repo root, but specs declare relatedPaths relative to
|
|
3446
|
+
* their own package. Changes outside `cwd` are kept under their repo-root
|
|
3447
|
+
* path and flagged `outsideCwd` — they only scope a spec in when the spec
|
|
3448
|
+
* explicitly declares a repo-root-relative glob, so an unrelated PR can
|
|
3449
|
+
* never accidentally match the app-relative globs.
|
|
3450
|
+
*/
|
|
3451
|
+
async function getChangedFiles(base, cwd) {
|
|
3452
|
+
const [{ stdout: rootOut }, { stdout: diffOut }] = await Promise.all([execFileP("git", ["rev-parse", "--show-toplevel"], { cwd }), execFileP("git", [
|
|
3453
|
+
"diff",
|
|
3454
|
+
"--name-status",
|
|
3455
|
+
"-M",
|
|
3456
|
+
`${base}...HEAD`
|
|
3457
|
+
], {
|
|
3458
|
+
cwd,
|
|
3459
|
+
maxBuffer: 32 * 1024 * 1024
|
|
3460
|
+
})]);
|
|
3461
|
+
return rerootChangedFiles(parseGitDiffOutput(diffOut), rootOut.trim(), cwd);
|
|
3462
|
+
}
|
|
3463
|
+
/**
|
|
3464
|
+
* Convert paths in `entries` from git-repo-root relative to `cwd` relative.
|
|
3465
|
+
* Entries outside `cwd` keep their repo-root path and are flagged
|
|
3466
|
+
* `outsideCwd`. Exported for unit tests.
|
|
3467
|
+
*/
|
|
3468
|
+
function rerootChangedFiles(entries, repoRoot, cwd) {
|
|
3469
|
+
const prefix = relative(repoRoot, cwd);
|
|
3470
|
+
if (!prefix) return entries;
|
|
3471
|
+
const out = [];
|
|
3472
|
+
for (const e of entries) {
|
|
3473
|
+
const rel = relative(prefix, e.path);
|
|
3474
|
+
if (rel.startsWith("..") || rel === "") out.push({
|
|
3475
|
+
...e,
|
|
3476
|
+
outsideCwd: true
|
|
3477
|
+
});
|
|
3478
|
+
else out.push({
|
|
3479
|
+
...e,
|
|
3480
|
+
path: rel
|
|
3481
|
+
});
|
|
3482
|
+
}
|
|
3483
|
+
return out;
|
|
3484
|
+
}
|
|
3485
|
+
function parseGitDiffOutput(stdout) {
|
|
3486
|
+
const out = [];
|
|
3487
|
+
for (const line of stdout.split("\n")) {
|
|
3488
|
+
if (!line.trim()) continue;
|
|
3489
|
+
const parts = line.split(" ");
|
|
3490
|
+
const code = parts[0];
|
|
3491
|
+
if (!code) continue;
|
|
3492
|
+
if (code.startsWith("R")) {
|
|
3493
|
+
const newPath = parts[2];
|
|
3494
|
+
if (newPath) out.push({
|
|
3495
|
+
path: newPath,
|
|
3496
|
+
status: "renamed"
|
|
3497
|
+
});
|
|
3498
|
+
continue;
|
|
3499
|
+
}
|
|
3500
|
+
if (code.startsWith("C")) {
|
|
3501
|
+
const newPath = parts[2];
|
|
3502
|
+
if (newPath) out.push({
|
|
3503
|
+
path: newPath,
|
|
3504
|
+
status: "added"
|
|
3505
|
+
});
|
|
3506
|
+
continue;
|
|
3507
|
+
}
|
|
3508
|
+
const path = parts[1];
|
|
3509
|
+
if (!path) continue;
|
|
3510
|
+
switch (code[0]) {
|
|
3511
|
+
case "A":
|
|
3512
|
+
out.push({
|
|
3513
|
+
path,
|
|
3514
|
+
status: "added"
|
|
3515
|
+
});
|
|
3516
|
+
break;
|
|
3517
|
+
case "M":
|
|
3518
|
+
case "T":
|
|
3519
|
+
out.push({
|
|
3520
|
+
path,
|
|
3521
|
+
status: "modified"
|
|
3522
|
+
});
|
|
3523
|
+
break;
|
|
3524
|
+
case "D":
|
|
3525
|
+
out.push({
|
|
3526
|
+
path,
|
|
3527
|
+
status: "deleted"
|
|
3528
|
+
});
|
|
3529
|
+
break;
|
|
3530
|
+
default: out.push({
|
|
3531
|
+
path,
|
|
3532
|
+
status: "modified"
|
|
3533
|
+
});
|
|
3534
|
+
}
|
|
3535
|
+
}
|
|
3536
|
+
return out;
|
|
3537
|
+
}
|
|
3538
|
+
/** Normalize a leading `./` away so diff paths and relatedPaths globs compare. */
|
|
3539
|
+
function stripLeadingDotSlash(s) {
|
|
3540
|
+
return s.startsWith("./") ? s.slice(2) : s;
|
|
3541
|
+
}
|
|
3542
|
+
const REGEX_CACHE = /* @__PURE__ */ new Map();
|
|
3543
|
+
/** Compiles `pattern` to a RegExp, memoized so repeated `--changed` matches don't re-build. */
|
|
3544
|
+
function compileGlob(pattern) {
|
|
3545
|
+
const cached = REGEX_CACHE.get(pattern);
|
|
3546
|
+
if (cached) return cached;
|
|
3547
|
+
const compiled = globToRegExp(stripLeadingDotSlash(pattern));
|
|
3548
|
+
REGEX_CACHE.set(pattern, compiled);
|
|
3549
|
+
return compiled;
|
|
3550
|
+
}
|
|
3551
|
+
function globToRegExp(pattern) {
|
|
3552
|
+
let re = "^";
|
|
3553
|
+
let i = 0;
|
|
3554
|
+
while (i < pattern.length) {
|
|
3555
|
+
const ch = pattern[i];
|
|
3556
|
+
if (ch === "?") {
|
|
3557
|
+
re += "[^/]";
|
|
3558
|
+
i++;
|
|
3559
|
+
continue;
|
|
3560
|
+
}
|
|
3561
|
+
if (ch !== "*") {
|
|
3562
|
+
re += /[.+^${}()|[\]\\]/.test(ch) ? "\\" + ch : ch;
|
|
3563
|
+
i++;
|
|
3564
|
+
continue;
|
|
3565
|
+
}
|
|
3566
|
+
if (pattern[i + 1] !== "*") {
|
|
3567
|
+
re += "[^/]*";
|
|
3568
|
+
i++;
|
|
3569
|
+
continue;
|
|
3570
|
+
}
|
|
3571
|
+
const hasLeadingSlash = re.endsWith("/");
|
|
3572
|
+
const hasTrailingSlash = pattern[i + 2] === "/";
|
|
3573
|
+
if (hasLeadingSlash) re = re.slice(0, -1);
|
|
3574
|
+
if (hasLeadingSlash || hasTrailingSlash) re += "(?:/?.*)?";
|
|
3575
|
+
else re += ".*";
|
|
3576
|
+
i += hasTrailingSlash ? 3 : 2;
|
|
3577
|
+
}
|
|
3578
|
+
return new RegExp(re + "$");
|
|
3579
|
+
}
|
|
3416
3580
|
/**
|
|
3417
|
-
*
|
|
3418
|
-
*
|
|
3419
|
-
*
|
|
3581
|
+
* Returns true if `changedPath` is covered by any of `relatedPaths`. An empty
|
|
3582
|
+
* `relatedPaths` returns false — callers handle the "unscoped spec" case
|
|
3583
|
+
* separately (treat the spec as always-affected) before calling this.
|
|
3420
3584
|
*/
|
|
3421
|
-
function
|
|
3422
|
-
|
|
3423
|
-
const
|
|
3424
|
-
|
|
3425
|
-
const confidence = typeof parsed["confidence"] === "number" ? clamp(parsed["confidence"], 0, 1) : 0;
|
|
3426
|
-
const reasoning = typeof parsed["reasoning"] === "string" ? parsed["reasoning"] : "";
|
|
3427
|
-
const headline = typeof parsed["headline"] === "string" ? parsed["headline"] : "";
|
|
3428
|
-
const recommendation = typeof parsed["recommendation"] === "string" ? parsed["recommendation"] : "";
|
|
3429
|
-
const rawSub = parsed["subDiagnosis"];
|
|
3430
|
-
const subDiagnosis = typeof rawSub === "string" && SUB_SET.has(rawSub) ? rawSub : "NONE";
|
|
3431
|
-
const evidence = [];
|
|
3432
|
-
if (Array.isArray(parsed["evidence"])) for (const item of parsed["evidence"]) {
|
|
3433
|
-
if (!isObject(item)) continue;
|
|
3434
|
-
const detail = typeof item["detail"] === "string" ? item["detail"] : null;
|
|
3435
|
-
if (detail === null) continue;
|
|
3436
|
-
const file = typeof item["file"] === "string" ? item["file"] : void 0;
|
|
3437
|
-
evidence.push(file !== void 0 ? {
|
|
3438
|
-
file,
|
|
3439
|
-
detail
|
|
3440
|
-
} : { detail });
|
|
3441
|
-
if (evidence.length >= 3) break;
|
|
3442
|
-
}
|
|
3443
|
-
return {
|
|
3444
|
-
label,
|
|
3445
|
-
confidence,
|
|
3446
|
-
subDiagnosis,
|
|
3447
|
-
headline,
|
|
3448
|
-
recommendation,
|
|
3449
|
-
evidence,
|
|
3450
|
-
reasoning
|
|
3451
|
-
};
|
|
3585
|
+
function isPathAffectedBy(changedPath, relatedPaths) {
|
|
3586
|
+
const stripped = stripLeadingDotSlash(changedPath);
|
|
3587
|
+
for (const pattern of relatedPaths) if (compileGlob(pattern).test(stripped)) return true;
|
|
3588
|
+
return false;
|
|
3452
3589
|
}
|
|
3453
3590
|
/**
|
|
3454
3591
|
* Capture the PR diff used as context for failure analysis. `--relative`
|
|
@@ -3563,6 +3700,249 @@ function scopePatchForSpec(patch, relatedPaths, caps = {}) {
|
|
|
3563
3700
|
return parts.join("\n");
|
|
3564
3701
|
}
|
|
3565
3702
|
//#endregion
|
|
3703
|
+
//#region src/run/diff-provider.ts
|
|
3704
|
+
/**
|
|
3705
|
+
* Cap on one on-demand file-diff response. Larger than the inline seed's
|
|
3706
|
+
* per-file cap (the model explicitly asked for this file), but still bounded
|
|
3707
|
+
* so a generated-file hunk can't blow the context — the truncation note
|
|
3708
|
+
* points at Read for the file's full current state.
|
|
3709
|
+
*/
|
|
3710
|
+
const FILE_DIFF_RESPONSE_CAP = 16 * 1024;
|
|
3711
|
+
/** Find `path`'s section in a split patch and cap it. Exported for tests. */
|
|
3712
|
+
function lookupFileDiff(sections, path) {
|
|
3713
|
+
const normalized = stripLeadingDotSlash(path);
|
|
3714
|
+
const section = sections.find((s) => s.path === normalized);
|
|
3715
|
+
if (!section) return null;
|
|
3716
|
+
if (section.body.length <= 16384) return section.body;
|
|
3717
|
+
return `${section.body.slice(0, FILE_DIFF_RESPONSE_CAP)}\n[truncated: ${section.body.length - FILE_DIFF_RESPONSE_CAP} more chars — Read the file for its full current state]`;
|
|
3718
|
+
}
|
|
3719
|
+
function createDiffProvider(args) {
|
|
3720
|
+
const { resolveBase, cwd } = args;
|
|
3721
|
+
const captures = /* @__PURE__ */ new Map();
|
|
3722
|
+
let relatedPathsIndex = null;
|
|
3723
|
+
function capture(sha) {
|
|
3724
|
+
const cached = captures.get(sha);
|
|
3725
|
+
if (cached) return cached;
|
|
3726
|
+
const pending = (async () => {
|
|
3727
|
+
const result = await capturePrDiff(sha, cwd);
|
|
3728
|
+
if (!result.ok) return {
|
|
3729
|
+
sections: null,
|
|
3730
|
+
nameStatus: null,
|
|
3731
|
+
error: result.error
|
|
3732
|
+
};
|
|
3733
|
+
const { patch, nameStatus } = result.diff;
|
|
3734
|
+
return {
|
|
3735
|
+
sections: patch.length > 0 ? splitPatchByFile(patch) : [],
|
|
3736
|
+
nameStatus,
|
|
3737
|
+
error: null
|
|
3738
|
+
};
|
|
3739
|
+
})();
|
|
3740
|
+
captures.set(sha, pending);
|
|
3741
|
+
return pending;
|
|
3742
|
+
}
|
|
3743
|
+
/** relatedPaths for every spec, read once from the feature tree. */
|
|
3744
|
+
function relatedPaths() {
|
|
3745
|
+
relatedPathsIndex ??= listFeatureTree(cwd).then((tree) => new Map(tree.flatMap((f) => f.specs.map((s) => [specKey({
|
|
3746
|
+
featureName: f.featureName,
|
|
3747
|
+
specName: s.specName
|
|
3748
|
+
}), s.relatedPaths ?? null]))));
|
|
3749
|
+
return relatedPathsIndex;
|
|
3750
|
+
}
|
|
3751
|
+
return { async forSpec(spec) {
|
|
3752
|
+
const resolved = await resolveBase(spec);
|
|
3753
|
+
if (!resolved.ok) return resolved;
|
|
3754
|
+
const [captured, index] = await Promise.all([capture(resolved.base.sha), relatedPaths()]);
|
|
3755
|
+
const scope = index.get(specKey(spec)) ?? null;
|
|
3756
|
+
const sections = captured.sections;
|
|
3757
|
+
return {
|
|
3758
|
+
ok: true,
|
|
3759
|
+
base: resolved.base,
|
|
3760
|
+
patch: sections ? scopePatchForSpec(sections, scope) : null,
|
|
3761
|
+
nameStatus: captured.nameStatus,
|
|
3762
|
+
error: captured.error,
|
|
3763
|
+
fileDiff: (path) => sections ? lookupFileDiff(sections, path) : null
|
|
3764
|
+
};
|
|
3765
|
+
} };
|
|
3766
|
+
}
|
|
3767
|
+
//#endregion
|
|
3768
|
+
//#region src/run/errors.ts
|
|
3769
|
+
/**
|
|
3770
|
+
* Usage error (bad flag combination, broken profile, failed `git diff`, …)
|
|
3771
|
+
* thrown by the `run` pipeline and the helpers it calls, e.g.
|
|
3772
|
+
* `collectChangedSpecs`. `executeRun` never calls `process.exit`, so each
|
|
3773
|
+
* host maps this itself: the CLI action catches it and exits with
|
|
3774
|
+
* `exitCode`; the hub runner records it as a run-level error.
|
|
3775
|
+
*/
|
|
3776
|
+
var RunUsageError = class extends Error {
|
|
3777
|
+
exitCode = 2;
|
|
3778
|
+
constructor(message) {
|
|
3779
|
+
super(message);
|
|
3780
|
+
this.name = "RunUsageError";
|
|
3781
|
+
}
|
|
3782
|
+
};
|
|
3783
|
+
//#endregion
|
|
3784
|
+
//#region src/run/git-context.ts
|
|
3785
|
+
/** The `--failure-analysis` value that selects per-spec hub-ledger baselines. */
|
|
3786
|
+
const LAST_GREEN = "last-green";
|
|
3787
|
+
/** Resolve `ref` to a full commit sha, or null when it does not exist locally. */
|
|
3788
|
+
async function resolveCommitSha(ref, cwd) {
|
|
3789
|
+
try {
|
|
3790
|
+
const { stdout } = await execFileP("git", [
|
|
3791
|
+
"rev-parse",
|
|
3792
|
+
"--verify",
|
|
3793
|
+
"--quiet",
|
|
3794
|
+
`${ref}^{commit}`
|
|
3795
|
+
], { cwd });
|
|
3796
|
+
return stdout.trim() || null;
|
|
3797
|
+
} catch {
|
|
3798
|
+
return null;
|
|
3799
|
+
}
|
|
3800
|
+
}
|
|
3801
|
+
/**
|
|
3802
|
+
* Resolve a `[base]` flag value (from `--failure-analysis [base]` or
|
|
3803
|
+
* `--changed [base]`) to a verified baseline, failing fast — before any spec
|
|
3804
|
+
* runs — when it cannot be resolved.
|
|
3805
|
+
*
|
|
3806
|
+
* - a string value is an explicit ref;
|
|
3807
|
+
* - bare `true` derives the ref from GITHUB_BASE_REF (pull_request events)
|
|
3808
|
+
* and errors outside that context;
|
|
3809
|
+
* - the ref must resolve to a local commit, so a shallow CI checkout that
|
|
3810
|
+
* never fetched the base surfaces here as an actionable error instead of
|
|
3811
|
+
* an empty diff downstream.
|
|
3812
|
+
*
|
|
3813
|
+
* `flagName` only shapes the error messages.
|
|
3814
|
+
*/
|
|
3815
|
+
async function resolveAnalysisBase(flagValue, flagName, cwd) {
|
|
3816
|
+
let ref;
|
|
3817
|
+
let source;
|
|
3818
|
+
if (flagValue === "last-green") throw new RunUsageError(`${flagName}=${LAST_GREEN} is not supported — last-green baselines are per-spec and only apply to --failure-analysis`);
|
|
3819
|
+
if (typeof flagValue === "string") {
|
|
3820
|
+
ref = flagValue;
|
|
3821
|
+
source = "explicit";
|
|
3822
|
+
} else {
|
|
3823
|
+
const ghBase = process.env["GITHUB_BASE_REF"];
|
|
3824
|
+
if (!ghBase) throw new RunUsageError(`${flagName} without a base needs GITHUB_BASE_REF (a pull_request workflow); outside that context pass the base explicitly, e.g. ${flagName}=origin/main`);
|
|
3825
|
+
ref = normalizeGithubBaseRef(ghBase);
|
|
3826
|
+
source = "github-base-ref";
|
|
3827
|
+
}
|
|
3828
|
+
const sha = await resolveCommitSha(ref, cwd);
|
|
3829
|
+
if (sha === null) throw new RunUsageError(`${flagName}: '${ref}' is not a resolvable git ref in this checkout. If this is CI, the base may not be fetched (try fetch-depth: 0). If '${ref}' was meant as a spec target, put spec targets before flags or use ${flagName}=<ref>.`);
|
|
3830
|
+
return {
|
|
3831
|
+
ref,
|
|
3832
|
+
sha,
|
|
3833
|
+
source
|
|
3834
|
+
};
|
|
3835
|
+
}
|
|
3836
|
+
//#endregion
|
|
3837
|
+
//#region src/cli/git-branch.ts
|
|
3838
|
+
/** Best-effort current branch: CI env vars first, then git, else null. */
|
|
3839
|
+
async function detectBranch(cwd) {
|
|
3840
|
+
const fromEnv = process.env.GITHUB_HEAD_REF || process.env.GITHUB_REF_NAME;
|
|
3841
|
+
if (fromEnv) return fromEnv;
|
|
3842
|
+
try {
|
|
3843
|
+
const { stdout } = await execFileP("git", [
|
|
3844
|
+
"rev-parse",
|
|
3845
|
+
"--abbrev-ref",
|
|
3846
|
+
"HEAD"
|
|
3847
|
+
], { cwd });
|
|
3848
|
+
const branch = stdout.trim();
|
|
3849
|
+
return branch && branch !== "HEAD" ? branch : null;
|
|
3850
|
+
} catch {
|
|
3851
|
+
return null;
|
|
3852
|
+
}
|
|
3853
|
+
}
|
|
3854
|
+
/** Best-effort current commit SHA, or null (e.g. not a git repo). */
|
|
3855
|
+
async function getGitHead(cwd) {
|
|
3856
|
+
try {
|
|
3857
|
+
const { stdout } = await execFileP("git", ["rev-parse", "HEAD"], { cwd });
|
|
3858
|
+
return stdout.trim() || null;
|
|
3859
|
+
} catch {
|
|
3860
|
+
return null;
|
|
3861
|
+
}
|
|
3862
|
+
}
|
|
3863
|
+
//#endregion
|
|
3864
|
+
//#region src/run/last-green.ts
|
|
3865
|
+
/**
|
|
3866
|
+
* The repo's default branch name (e.g. "main"), from origin's HEAD ref.
|
|
3867
|
+
* Falls back to "main" when origin isn't configured — the lookup then just
|
|
3868
|
+
* queries a possibly-empty ledger bucket, which is harmless.
|
|
3869
|
+
*/
|
|
3870
|
+
async function detectDefaultBranch(cwd) {
|
|
3871
|
+
try {
|
|
3872
|
+
const { stdout } = await execFileP("git", [
|
|
3873
|
+
"symbolic-ref",
|
|
3874
|
+
"--short",
|
|
3875
|
+
"refs/remotes/origin/HEAD"
|
|
3876
|
+
], { cwd });
|
|
3877
|
+
const ref = stdout.trim();
|
|
3878
|
+
return ref.startsWith("origin/") ? ref.slice(7) : ref || "main";
|
|
3879
|
+
} catch {
|
|
3880
|
+
return "main";
|
|
3881
|
+
}
|
|
3882
|
+
}
|
|
3883
|
+
/**
|
|
3884
|
+
* Fetch the last-green ledger for this run — one hub round trip, logged as
|
|
3885
|
+
* the run's analysis-base meta line. Fails fast (RunUsageError) when the hub
|
|
3886
|
+
* can't serve it: `--failure-analysis=last-green` explicitly opted into
|
|
3887
|
+
* hub-backed baselines, so a broken hub connection is a usage error, never a
|
|
3888
|
+
* silent no-baseline run.
|
|
3889
|
+
*/
|
|
3890
|
+
async function fetchLastGreenLedger(hubCtx, profile, cwd) {
|
|
3891
|
+
const [fallbackBranch, detectedBranch] = await Promise.all([detectDefaultBranch(cwd), detectBranch(cwd)]);
|
|
3892
|
+
const branch = detectedBranch ?? fallbackBranch;
|
|
3893
|
+
let entries;
|
|
3894
|
+
try {
|
|
3895
|
+
entries = await hubCtx.hub.getLastGreen(hubCtx.project, {
|
|
3896
|
+
branch,
|
|
3897
|
+
fallbackBranch,
|
|
3898
|
+
...profile ? { profile } : {}
|
|
3899
|
+
});
|
|
3900
|
+
} catch (err) {
|
|
3901
|
+
throw new RunUsageError(`--failure-analysis=${LAST_GREEN}: could not fetch the last-green ledger from the hub: ${err instanceof Error ? err.message : String(err)}`);
|
|
3902
|
+
}
|
|
3903
|
+
const n = Object.keys(entries).length;
|
|
3904
|
+
const scope = branch === fallbackBranch ? branch : `${branch} → ${fallbackBranch}`;
|
|
3905
|
+
meta("analysis-base", `${LAST_GREEN} (${n} spec baseline${n === 1 ? "" : "s"}, branch ${scope})`);
|
|
3906
|
+
return entries;
|
|
3907
|
+
}
|
|
3908
|
+
/**
|
|
3909
|
+
* Per-spec baseline resolver for `--failure-analysis=last-green`. A spec
|
|
3910
|
+
* missing from the ledger (never green on a pushed run yet) or whose
|
|
3911
|
+
* baseline commit isn't in this checkout resolves to a skip — the run
|
|
3912
|
+
* continues; only that spec's classification is withheld, with the reason
|
|
3913
|
+
* recorded in its report row. Sha existence checks are memoized per commit.
|
|
3914
|
+
*/
|
|
3915
|
+
function createLastGreenResolver(entries, cwd) {
|
|
3916
|
+
const shaChecks = /* @__PURE__ */ new Map();
|
|
3917
|
+
const checkSha = (sha) => {
|
|
3918
|
+
const cached = shaChecks.get(sha);
|
|
3919
|
+
if (cached) return cached;
|
|
3920
|
+
const pending = resolveCommitSha(sha, cwd);
|
|
3921
|
+
shaChecks.set(sha, pending);
|
|
3922
|
+
return pending;
|
|
3923
|
+
};
|
|
3924
|
+
return async (spec) => {
|
|
3925
|
+
const entry = entries[specKey(spec)];
|
|
3926
|
+
if (!entry) return {
|
|
3927
|
+
ok: false,
|
|
3928
|
+
skip: "no last-green baseline for this spec on the hub yet (recorded once the spec passes on a pushed run)"
|
|
3929
|
+
};
|
|
3930
|
+
const sha = await checkSha(entry.gitHead);
|
|
3931
|
+
if (!sha) return {
|
|
3932
|
+
ok: false,
|
|
3933
|
+
skip: `last-green commit ${entry.gitHead.slice(0, 12)} is not in this checkout (shallow clone? try fetch-depth: 0)`
|
|
3934
|
+
};
|
|
3935
|
+
return {
|
|
3936
|
+
ok: true,
|
|
3937
|
+
base: {
|
|
3938
|
+
ref: LAST_GREEN,
|
|
3939
|
+
sha,
|
|
3940
|
+
source: "last-green"
|
|
3941
|
+
}
|
|
3942
|
+
};
|
|
3943
|
+
};
|
|
3944
|
+
}
|
|
3945
|
+
//#endregion
|
|
3566
3946
|
//#region src/report/github-format.ts
|
|
3567
3947
|
/**
|
|
3568
3948
|
* Build GitHub Actions `::error::` annotation lines for every failed spec in
|
|
@@ -4520,33 +4900,6 @@ function resolvePromptLocalPath(name, cwd) {
|
|
|
4520
4900
|
return join(cwd ?? process.cwd(), PROMPT_LOCAL_PATHS[name]);
|
|
4521
4901
|
}
|
|
4522
4902
|
//#endregion
|
|
4523
|
-
//#region src/cli/git-branch.ts
|
|
4524
|
-
/** Best-effort current branch: CI env vars first, then git, else null. */
|
|
4525
|
-
async function detectBranch(cwd) {
|
|
4526
|
-
const fromEnv = process.env.GITHUB_HEAD_REF || process.env.GITHUB_REF_NAME;
|
|
4527
|
-
if (fromEnv) return fromEnv;
|
|
4528
|
-
try {
|
|
4529
|
-
const { stdout } = await execFileP("git", [
|
|
4530
|
-
"rev-parse",
|
|
4531
|
-
"--abbrev-ref",
|
|
4532
|
-
"HEAD"
|
|
4533
|
-
], { cwd });
|
|
4534
|
-
const branch = stdout.trim();
|
|
4535
|
-
return branch && branch !== "HEAD" ? branch : null;
|
|
4536
|
-
} catch {
|
|
4537
|
-
return null;
|
|
4538
|
-
}
|
|
4539
|
-
}
|
|
4540
|
-
/** Best-effort current commit SHA, or null (e.g. not a git repo). */
|
|
4541
|
-
async function getGitHead(cwd) {
|
|
4542
|
-
try {
|
|
4543
|
-
const { stdout } = await execFileP("git", ["rev-parse", "HEAD"], { cwd });
|
|
4544
|
-
return stdout.trim() || null;
|
|
4545
|
-
} catch {
|
|
4546
|
-
return null;
|
|
4547
|
-
}
|
|
4548
|
-
}
|
|
4549
|
-
//#endregion
|
|
4550
4903
|
//#region src/cli/hub.ts
|
|
4551
4904
|
/**
|
|
4552
4905
|
* `ccqa hub` — the client side of the ccqa hub (a results/secret control
|
|
@@ -5101,6 +5454,7 @@ async function runLiveExecutor(input) {
|
|
|
5101
5454
|
const transcriptParts = [];
|
|
5102
5455
|
const commandParts = [];
|
|
5103
5456
|
let isError = false;
|
|
5457
|
+
let errorDetail = null;
|
|
5104
5458
|
let cost = emptyStepCost();
|
|
5105
5459
|
try {
|
|
5106
5460
|
const result = await invokeClaudeStreaming({
|
|
@@ -5120,6 +5474,7 @@ async function runLiveExecutor(input) {
|
|
|
5120
5474
|
}
|
|
5121
5475
|
});
|
|
5122
5476
|
isError = result.isError;
|
|
5477
|
+
errorDetail = result.errorDetail;
|
|
5123
5478
|
cost = {
|
|
5124
5479
|
totalCostUsd: result.cost.totalCostUsd,
|
|
5125
5480
|
durationApiMs: result.cost.durationApiMs,
|
|
@@ -5132,7 +5487,8 @@ async function runLiveExecutor(input) {
|
|
|
5132
5487
|
};
|
|
5133
5488
|
} catch (err) {
|
|
5134
5489
|
isError = true;
|
|
5135
|
-
|
|
5490
|
+
errorDetail = err instanceof Error ? err.message : String(err);
|
|
5491
|
+
transcriptParts.push(`[ccqa] invokeClaudeStreaming threw: ${errorDetail}`);
|
|
5136
5492
|
}
|
|
5137
5493
|
const transcript = transcriptParts.join("\n");
|
|
5138
5494
|
const after = takeScreenshot(input.sessionName, paths.afterPng, { fullPage: true });
|
|
@@ -5141,6 +5497,7 @@ async function runLiveExecutor(input) {
|
|
|
5141
5497
|
const { status, reasoning } = judgeStepOutcome({
|
|
5142
5498
|
step,
|
|
5143
5499
|
isError,
|
|
5500
|
+
errorDetail,
|
|
5144
5501
|
judged: findLastStepResult(transcript)
|
|
5145
5502
|
});
|
|
5146
5503
|
return {
|
|
@@ -5219,11 +5576,14 @@ function sumStepCosts(steps) {
|
|
|
5219
5576
|
* Kept as a pure helper so the executor loop stays readable and the
|
|
5220
5577
|
* branches are individually testable.
|
|
5221
5578
|
*/
|
|
5222
|
-
function judgeStepOutcome({ step, isError, judged }) {
|
|
5223
|
-
if (isError)
|
|
5224
|
-
|
|
5225
|
-
|
|
5226
|
-
|
|
5579
|
+
function judgeStepOutcome({ step, isError, errorDetail, judged }) {
|
|
5580
|
+
if (isError) {
|
|
5581
|
+
const detail = errorDetail ? `: ${errorDetail}` : "";
|
|
5582
|
+
return {
|
|
5583
|
+
status: "failed",
|
|
5584
|
+
reasoning: judged?.reasoning ? `agent error${detail}; last reasoning: ${judged.reasoning}` : `Claude invocation returned an error${detail}`
|
|
5585
|
+
};
|
|
5586
|
+
}
|
|
5227
5587
|
if (!judged) return {
|
|
5228
5588
|
status: "failed",
|
|
5229
5589
|
reasoning: "STEP_RESULT missing"
|
|
@@ -5469,22 +5829,14 @@ async function runLiveSpecs(specs, opts) {
|
|
|
5469
5829
|
const userPromptBundle = await loadPromptBundleFromHub(opts.hubContext ?? null, "live");
|
|
5470
5830
|
if (userPromptBundle !== null) meta("prompt", userPromptBundle.loaded.join(" + "));
|
|
5471
5831
|
const userPromptSuffix = userPromptBundle?.text ?? null;
|
|
5472
|
-
const
|
|
5832
|
+
const diffProvider = opts.diffProvider ?? null;
|
|
5833
|
+
const failureAnalysisEnabled = diffProvider != null;
|
|
5473
5834
|
const driftAuditEnabled = failureAnalysisEnabled && opts.driftAudit !== false;
|
|
5474
5835
|
const auth = failureAnalysisEnabled ? driftAuthAvailable() : {
|
|
5475
5836
|
ok: false,
|
|
5476
5837
|
reason: "disabled"
|
|
5477
5838
|
};
|
|
5478
5839
|
if (failureAnalysisEnabled && !auth.ok) info(`failure analysis skipped (${auth.reason})`);
|
|
5479
|
-
const baseRef = resolveBaseRef(opts.base);
|
|
5480
|
-
let diff = {
|
|
5481
|
-
ok: false,
|
|
5482
|
-
error: "diff not captured"
|
|
5483
|
-
};
|
|
5484
|
-
if (failureAnalysisEnabled && auth.ok) {
|
|
5485
|
-
diff = await capturePrDiff(baseRef, cwd);
|
|
5486
|
-
if (!diff.ok) info(`failure analysis: source diff unavailable (${diff.error}) — analyzing without diff context`);
|
|
5487
|
-
}
|
|
5488
5840
|
const reportDir = opts.reportDir ?? ".";
|
|
5489
5841
|
const concurrency = Math.max(1, opts.concurrency ?? 1);
|
|
5490
5842
|
const built = await runPool(specs, concurrency, (spec, i) => {
|
|
@@ -5506,10 +5858,8 @@ async function runLiveSpecs(specs, opts) {
|
|
|
5506
5858
|
};
|
|
5507
5859
|
const row = await buildLiveReportRow(outcome, {
|
|
5508
5860
|
auth,
|
|
5509
|
-
|
|
5510
|
-
baseRef,
|
|
5861
|
+
diffProvider,
|
|
5511
5862
|
reportDir,
|
|
5512
|
-
failureAnalysisEnabled,
|
|
5513
5863
|
driftAuditEnabled
|
|
5514
5864
|
}, opts, cwd);
|
|
5515
5865
|
await opts.report?.upsert(row);
|
|
@@ -5545,26 +5895,29 @@ async function buildLiveReportRow(r, ctx, opts, cwd) {
|
|
|
5545
5895
|
reportDir: ctx.reportDir
|
|
5546
5896
|
});
|
|
5547
5897
|
const driftForSpec = ctx.driftAuditEnabled && r.result.status === "failed" ? await runDriftAuditOne(r, opts, cwd) : null;
|
|
5548
|
-
const analysis = ctx.
|
|
5898
|
+
const analysis = ctx.diffProvider && r.result.status === "failed" ? await analyzeOneLiveFailure(r, ctx.diffProvider, driftForSpec, ctx.auth, opts, cwd) : void 0;
|
|
5549
5899
|
return {
|
|
5550
5900
|
...base,
|
|
5551
5901
|
driftIssues: driftForSpec,
|
|
5552
|
-
...analysisFieldsFor(analysis, r.result.status
|
|
5902
|
+
...analysisFieldsFor(analysis, r.result.status)
|
|
5553
5903
|
};
|
|
5554
5904
|
}
|
|
5555
5905
|
/**
|
|
5556
5906
|
* Merge analysis-related fields into the report row. The unattempted-failure
|
|
5557
5907
|
* branch exists so the report distinguishes "we tried and gave up" (auth /
|
|
5558
|
-
* spec.yaml missing) from "we deliberately did not run the classifier"
|
|
5908
|
+
* spec.yaml missing) from "we deliberately did not run the classifier" —
|
|
5909
|
+
* `a` is undefined for a failed spec exactly when analysis was not requested
|
|
5910
|
+
* (no diffProvider), so no separate flag is needed.
|
|
5559
5911
|
*/
|
|
5560
|
-
function analysisFieldsFor(a, status
|
|
5912
|
+
function analysisFieldsFor(a, status) {
|
|
5561
5913
|
if (a) return {
|
|
5562
5914
|
analysis: a.analysis,
|
|
5563
5915
|
analysisSkipped: a.analysisSkipped,
|
|
5564
5916
|
failureLogExcerpt: a.failureLogExcerpt,
|
|
5565
|
-
diffExcerpt: a.diffExcerpt
|
|
5917
|
+
diffExcerpt: a.diffExcerpt,
|
|
5918
|
+
...a.analysisBase ? { analysisBase: a.analysisBase } : {}
|
|
5566
5919
|
};
|
|
5567
|
-
if (
|
|
5920
|
+
if (status === "failed") return { analysisSkipped: "skipped: --failure-analysis not enabled" };
|
|
5568
5921
|
return {};
|
|
5569
5922
|
}
|
|
5570
5923
|
/**
|
|
@@ -5754,11 +6107,13 @@ function logBatchCost(runs) {
|
|
|
5754
6107
|
/**
|
|
5755
6108
|
* Classify one failed live run via `analyzeFailure` — same prompt as the
|
|
5756
6109
|
* deterministic path (Issue #47), fed the live transcript instead of the
|
|
5757
|
-
* vitest log. `auth
|
|
5758
|
-
* shared
|
|
5759
|
-
*
|
|
6110
|
+
* vitest log. `auth` is hoisted once by the caller; the diff comes from the
|
|
6111
|
+
* shared provider, already scoped to this spec's relatedPaths and truncated
|
|
6112
|
+
* (the live path used to feed the whole unscoped patch — in a monorepo that
|
|
6113
|
+
* ballooned the prompt with unrelated changes). Auth-unavailable /
|
|
6114
|
+
* no-failed-step degrade to `analysisSkipped` rather than throwing.
|
|
5760
6115
|
*/
|
|
5761
|
-
async function analyzeOneLiveFailure(r,
|
|
6116
|
+
async function analyzeOneLiveFailure(r, diffProvider, driftForSpec, auth, opts, cwd) {
|
|
5762
6117
|
const key = `${r.featureName}/${r.specName}`;
|
|
5763
6118
|
if (!auth.ok) return {
|
|
5764
6119
|
analysis: null,
|
|
@@ -5774,19 +6129,31 @@ async function analyzeOneLiveFailure(r, diff, baseRef, driftForSpec, auth, opts,
|
|
|
5774
6129
|
failureLogExcerpt: null,
|
|
5775
6130
|
diffExcerpt: null
|
|
5776
6131
|
};
|
|
6132
|
+
const specDiff = await diffProvider.forSpec({
|
|
6133
|
+
featureName: r.featureName,
|
|
6134
|
+
specName: r.specName
|
|
6135
|
+
});
|
|
6136
|
+
if (!specDiff.ok) return {
|
|
6137
|
+
analysis: null,
|
|
6138
|
+
analysisSkipped: specDiff.skip,
|
|
6139
|
+
failureLogExcerpt: excerpt,
|
|
6140
|
+
diffExcerpt: null
|
|
6141
|
+
};
|
|
6142
|
+
if (specDiff.error) info(`failure analysis: source diff unavailable (${specDiff.error}) — analyzing without diff context`);
|
|
5777
6143
|
const outcome = await analyzeFailure({
|
|
5778
6144
|
liveTranscriptExcerpt: excerpt,
|
|
5779
6145
|
specYaml: r.specYaml,
|
|
5780
|
-
diffPatch:
|
|
5781
|
-
changedFiles:
|
|
5782
|
-
baseRef:
|
|
6146
|
+
diffPatch: specDiff.patch,
|
|
6147
|
+
changedFiles: specDiff.nameStatus,
|
|
6148
|
+
baseRef: specDiff.base.ref,
|
|
5783
6149
|
driftIssues: driftForSpec,
|
|
5784
6150
|
...opts.language ? { outputLanguage: opts.language } : {},
|
|
5785
6151
|
...opts.triageUserPrompt ? { triageUserPrompt: opts.triageUserPrompt } : {},
|
|
5786
6152
|
...opts.customPrompt ? { customPrompt: opts.customPrompt } : {}
|
|
5787
6153
|
}, {
|
|
5788
6154
|
...opts.model ? { model: opts.model } : {},
|
|
5789
|
-
cwd
|
|
6155
|
+
cwd,
|
|
6156
|
+
getFileDiff: specDiff.fileDiff
|
|
5790
6157
|
});
|
|
5791
6158
|
const pct = Math.round(outcome.analysis.confidence * 100);
|
|
5792
6159
|
const headline = outcome.analysis.headline.trim() || (outcome.analysis.reasoning.split("\n")[0] ?? "").trim();
|
|
@@ -5795,7 +6162,11 @@ async function analyzeOneLiveFailure(r, diff, baseRef, driftForSpec, auth, opts,
|
|
|
5795
6162
|
analysis: outcome.analysis,
|
|
5796
6163
|
analysisSkipped: null,
|
|
5797
6164
|
failureLogExcerpt: excerpt,
|
|
5798
|
-
diffExcerpt:
|
|
6165
|
+
diffExcerpt: specDiff.patch,
|
|
6166
|
+
analysisBase: {
|
|
6167
|
+
ref: specDiff.base.ref,
|
|
6168
|
+
sha: specDiff.base.sha
|
|
6169
|
+
}
|
|
5799
6170
|
};
|
|
5800
6171
|
}
|
|
5801
6172
|
function count(steps, target) {
|
|
@@ -8866,27 +9237,12 @@ function stripCodeFences(text) {
|
|
|
8866
9237
|
return m && m[1] !== void 0 ? m[1] : text;
|
|
8867
9238
|
}
|
|
8868
9239
|
//#endregion
|
|
8869
|
-
//#region src/run/errors.ts
|
|
8870
|
-
/**
|
|
8871
|
-
* Usage error (bad flag combination, broken profile, failed `git diff`, …)
|
|
8872
|
-
* thrown by the `run` pipeline and the helpers it calls, e.g.
|
|
8873
|
-
* `collectChangedSpecs`. `executeRun` never calls `process.exit`, so each
|
|
8874
|
-
* host maps this itself: the CLI action catches it and exits with
|
|
8875
|
-
* `exitCode`; the hub runner records it as a run-level error.
|
|
8876
|
-
*/
|
|
8877
|
-
var RunUsageError = class extends Error {
|
|
8878
|
-
exitCode = 2;
|
|
8879
|
-
constructor(message) {
|
|
8880
|
-
super(message);
|
|
8881
|
-
this.name = "RunUsageError";
|
|
8882
|
-
}
|
|
8883
|
-
};
|
|
8884
|
-
//#endregion
|
|
8885
9240
|
//#region src/cli/changed-specs.ts
|
|
8886
9241
|
/**
|
|
8887
|
-
* Filter specs to those affected by the git diff against the
|
|
8888
|
-
*
|
|
8889
|
-
* LLM new-file router (kept off here for predictable CI
|
|
9242
|
+
* Filter specs to those affected by the git diff against the `--changed
|
|
9243
|
+
* [base]` baseline. Powers `ccqa run --changed`; mirrors `ccqa drift
|
|
9244
|
+
* --changed` minus the LLM new-file router (kept off here for predictable CI
|
|
9245
|
+
* cost).
|
|
8890
9246
|
*
|
|
8891
9247
|
* A spec is "affected" when it has no `relatedPaths` (conservatively
|
|
8892
9248
|
* included), any changed file matches one of its `relatedPaths` globs, or it
|
|
@@ -8894,14 +9250,14 @@ var RunUsageError = class extends Error {
|
|
|
8894
9250
|
*/
|
|
8895
9251
|
async function collectChangedSpecs(specs, opts) {
|
|
8896
9252
|
const { cwd, base } = opts;
|
|
8897
|
-
const
|
|
9253
|
+
const resolved = await resolveAnalysisBase(base, "--changed", cwd);
|
|
8898
9254
|
let changed;
|
|
8899
9255
|
try {
|
|
8900
|
-
changed = await getChangedFiles(
|
|
9256
|
+
changed = await getChangedFiles(resolved.sha, cwd);
|
|
8901
9257
|
} catch (e) {
|
|
8902
|
-
throw new RunUsageError(`failed to run 'git diff' against ${
|
|
9258
|
+
throw new RunUsageError(`failed to run 'git diff' against ${resolved.ref}: ${e.message}`);
|
|
8903
9259
|
}
|
|
8904
|
-
meta("changed-base",
|
|
9260
|
+
meta("changed-base", `${resolved.ref} (${resolved.sha.slice(0, 12)})`);
|
|
8905
9261
|
meta("changed-files", changed.length);
|
|
8906
9262
|
return filterAffectedSpecs(specs, changed, cwd);
|
|
8907
9263
|
}
|
|
@@ -8965,6 +9321,27 @@ function dedupeSpecs(specs) {
|
|
|
8965
9321
|
async function executeRun(targets, opts) {
|
|
8966
9322
|
if (opts.changed && targets.length > 0) throw new RunUsageError("--changed and an explicit spec target cannot be combined");
|
|
8967
9323
|
const cwd = opts.cwd ?? process.cwd();
|
|
9324
|
+
const wantsLastGreen = opts.failureAnalysis === LAST_GREEN;
|
|
9325
|
+
const [head, fixedBase] = await Promise.all([getGitHead(cwd), opts.failureAnalysis && !wantsLastGreen ? resolveAnalysisBase(opts.failureAnalysis, "--failure-analysis", cwd) : null]);
|
|
9326
|
+
const git = {
|
|
9327
|
+
head,
|
|
9328
|
+
base: wantsLastGreen ? {
|
|
9329
|
+
ref: LAST_GREEN,
|
|
9330
|
+
sha: null,
|
|
9331
|
+
source: "last-green"
|
|
9332
|
+
} : fixedBase
|
|
9333
|
+
};
|
|
9334
|
+
let diffProvider = null;
|
|
9335
|
+
if (fixedBase) {
|
|
9336
|
+
diffProvider = createDiffProvider({
|
|
9337
|
+
resolveBase: async () => ({
|
|
9338
|
+
ok: true,
|
|
9339
|
+
base: fixedBase
|
|
9340
|
+
}),
|
|
9341
|
+
cwd
|
|
9342
|
+
});
|
|
9343
|
+
meta("analysis-base", `${fixedBase.ref} (${fixedBase.sha.slice(0, 12)}, ${fixedBase.source})`);
|
|
9344
|
+
}
|
|
8968
9345
|
let projectForProfile;
|
|
8969
9346
|
try {
|
|
8970
9347
|
if (opts.profile !== void 0) {
|
|
@@ -9000,7 +9377,17 @@ async function executeRun(targets, opts) {
|
|
|
9000
9377
|
} catch {
|
|
9001
9378
|
hubCtx = null;
|
|
9002
9379
|
}
|
|
9003
|
-
|
|
9380
|
+
if (wantsLastGreen && hubCtx == null) throw new RunUsageError(`--failure-analysis=${LAST_GREEN} requires a hub connection (--hub-url/--hub-token or CCQA_HUB_URL/CCQA_HUB_TOKEN)`);
|
|
9381
|
+
const ledgerHub = wantsLastGreen ? hubCtx : null;
|
|
9382
|
+
const [customPrompt, triageUserPrompt, ledgerEntries] = await Promise.all([
|
|
9383
|
+
fetchCustomPrompt(hubCtx),
|
|
9384
|
+
fetchTriageUserPrompt(hubCtx),
|
|
9385
|
+
ledgerHub ? fetchLastGreenLedger(ledgerHub, opts.profile, cwd) : null
|
|
9386
|
+
]);
|
|
9387
|
+
if (ledgerEntries) diffProvider = createDiffProvider({
|
|
9388
|
+
resolveBase: createLastGreenResolver(ledgerEntries, cwd),
|
|
9389
|
+
cwd
|
|
9390
|
+
});
|
|
9004
9391
|
const triageUserPromptHash = triageUserPrompt ? hashTriageUserPrompt(triageUserPrompt) : null;
|
|
9005
9392
|
const enumerateAll = () => listAllSpecsWithSpecFile(cwd);
|
|
9006
9393
|
let specs = dedupeSpecs((await Promise.all((targets.length ? targets : [void 0]).map((t) => resolveSpecTargets(t, enumerateAll, cwd)))).flat());
|
|
@@ -9008,7 +9395,7 @@ async function executeRun(targets, opts) {
|
|
|
9008
9395
|
const before = specs.length;
|
|
9009
9396
|
specs = await collectChangedSpecs(specs, {
|
|
9010
9397
|
cwd,
|
|
9011
|
-
base: opts.
|
|
9398
|
+
base: opts.changed
|
|
9012
9399
|
});
|
|
9013
9400
|
meta("changed-scoped", `${specs.length} of ${before} spec${before === 1 ? "" : "s"}`);
|
|
9014
9401
|
}
|
|
@@ -9050,6 +9437,7 @@ async function executeRun(targets, opts) {
|
|
|
9050
9437
|
project: hubCtx.project,
|
|
9051
9438
|
...branch ? { branch } : {},
|
|
9052
9439
|
...opts.profile ? { profile: opts.profile } : {},
|
|
9440
|
+
...git.head ? { gitHead: git.head } : {},
|
|
9053
9441
|
kind: "run"
|
|
9054
9442
|
});
|
|
9055
9443
|
hubRunId = opened.id;
|
|
@@ -9070,11 +9458,7 @@ async function executeRun(targets, opts) {
|
|
|
9070
9458
|
warn(`hub: could not open incremental run (${errMessage(err)}); continuing with local report only`);
|
|
9071
9459
|
}
|
|
9072
9460
|
const incrementalReport = createIncrementalReport(reportDir, buildReportEnvelope({
|
|
9073
|
-
|
|
9074
|
-
ok: false,
|
|
9075
|
-
error: "diff not yet captured"
|
|
9076
|
-
},
|
|
9077
|
-
baseRef: null,
|
|
9461
|
+
git,
|
|
9078
9462
|
customPromptVersion: customPrompt?.customPromptVersion ?? null,
|
|
9079
9463
|
triageUserPromptHash,
|
|
9080
9464
|
opts
|
|
@@ -9106,13 +9490,12 @@ async function executeRun(targets, opts) {
|
|
|
9106
9490
|
...opts.language ? { language: opts.language } : {},
|
|
9107
9491
|
...opts.out && liveSpecs.length === 1 ? { out: opts.out } : {},
|
|
9108
9492
|
cwd,
|
|
9109
|
-
...opts.base ? { base: opts.base } : {},
|
|
9110
9493
|
reportDir,
|
|
9111
9494
|
...typeof opts.retry === "number" ? { retry: opts.retry } : {},
|
|
9112
9495
|
concurrency: opts.concurrency ?? 1,
|
|
9113
9496
|
...opts.profile ? { profile: opts.profile } : {},
|
|
9114
9497
|
...opts.driftAudit !== false ? { driftAudit: true } : {},
|
|
9115
|
-
|
|
9498
|
+
diffProvider,
|
|
9116
9499
|
hubContext: hubCtx,
|
|
9117
9500
|
customPrompt,
|
|
9118
9501
|
triageUserPrompt,
|
|
@@ -9124,7 +9507,7 @@ async function executeRun(targets, opts) {
|
|
|
9124
9507
|
if (externalRows.some((r) => r.status === "failed")) overallExitCode = 1;
|
|
9125
9508
|
let report;
|
|
9126
9509
|
{
|
|
9127
|
-
const detReport = await analyzeDeterministicSummaries(det.summaries, opts, cwd, reportDir, customPrompt, triageUserPrompt);
|
|
9510
|
+
const detReport = await analyzeDeterministicSummaries(det.summaries, opts, cwd, reportDir, customPrompt, triageUserPrompt, diffProvider);
|
|
9128
9511
|
report = await writeUnifiedReport({
|
|
9129
9512
|
reportDir,
|
|
9130
9513
|
results: [
|
|
@@ -9132,8 +9515,7 @@ async function executeRun(targets, opts) {
|
|
|
9132
9515
|
...externalRows,
|
|
9133
9516
|
...live.reportResults
|
|
9134
9517
|
],
|
|
9135
|
-
|
|
9136
|
-
baseRef: detReport.baseRef,
|
|
9518
|
+
git,
|
|
9137
9519
|
customPromptVersion: detReport.customPromptVersion,
|
|
9138
9520
|
triageUserPromptHash,
|
|
9139
9521
|
opts
|
|
@@ -9142,8 +9524,7 @@ async function executeRun(targets, opts) {
|
|
|
9142
9524
|
if (hubRunId) {
|
|
9143
9525
|
const finalStatus = overallExitCode === 0 ? "passed" : "failed";
|
|
9144
9526
|
const reportMeta = buildReportEnvelope({
|
|
9145
|
-
|
|
9146
|
-
baseRef: detReport.baseRef,
|
|
9527
|
+
git,
|
|
9147
9528
|
customPromptVersion: detReport.customPromptVersion,
|
|
9148
9529
|
triageUserPromptHash,
|
|
9149
9530
|
opts
|
|
@@ -9341,24 +9722,15 @@ function failedSpec(s) {
|
|
|
9341
9722
|
* failure analysis when `--report` is on; degrades (no throw) when Claude
|
|
9342
9723
|
* auth or git diff aren't available. Caller writes the HTML / JSON.
|
|
9343
9724
|
*/
|
|
9344
|
-
async function analyzeDeterministicSummaries(summaries, opts, cwd, reportDir, customPrompt, triageUserPrompt) {
|
|
9345
|
-
const failureAnalysisEnabled =
|
|
9725
|
+
async function analyzeDeterministicSummaries(summaries, opts, cwd, reportDir, customPrompt, triageUserPrompt, diffProvider) {
|
|
9726
|
+
const failureAnalysisEnabled = diffProvider != null;
|
|
9346
9727
|
const driftAuditEnabled = failureAnalysisEnabled && opts.driftAudit !== false;
|
|
9347
|
-
const auth = failureAnalysisEnabled
|
|
9728
|
+
const auth = failureAnalysisEnabled ? driftAuthAvailable() : {
|
|
9348
9729
|
ok: false,
|
|
9349
9730
|
reason: "skipped by flags"
|
|
9350
9731
|
};
|
|
9351
9732
|
const failed = summaries.filter(failedSpec);
|
|
9352
9733
|
if (failureAnalysisEnabled && !auth.ok && failed.length > 0) info(`failure analysis skipped (${auth.reason})`);
|
|
9353
|
-
const baseRef = resolveBaseRef(opts.base);
|
|
9354
|
-
let diff = {
|
|
9355
|
-
ok: false,
|
|
9356
|
-
error: "diff not captured (no failures)"
|
|
9357
|
-
};
|
|
9358
|
-
if (failed.length > 0) {
|
|
9359
|
-
diff = await capturePrDiff(baseRef, cwd);
|
|
9360
|
-
if (!diff.ok) info(`drift-report: source diff unavailable (${diff.error}) — analyzing without diff context`);
|
|
9361
|
-
}
|
|
9362
9734
|
const tree = failed.length > 0 ? await listFeatureTree(cwd) : [];
|
|
9363
9735
|
const specInfoByKey = new Map(tree.flatMap((f) => f.specs.map((sp) => [`${f.featureName}/${sp.specName}`, sp])));
|
|
9364
9736
|
const findSpecInfo = (s) => specInfoByKey.get(`${s.featureName}/${s.specName}`) ?? null;
|
|
@@ -9385,9 +9757,9 @@ async function analyzeDeterministicSummaries(summaries, opts, cwd, reportDir, cu
|
|
|
9385
9757
|
onSpecStart: (t) => info(`drift audit: ${t.featureName}/${t.specName}`)
|
|
9386
9758
|
});
|
|
9387
9759
|
}
|
|
9388
|
-
const patchSections = diff.ok && diff.diff.patch.length > 0 ? splitPatchByFile(diff.diff.patch) : null;
|
|
9389
9760
|
const allBlocks = await loadAllBlocks(cwd);
|
|
9390
9761
|
let printedHeader = false;
|
|
9762
|
+
let warnedDiffUnavailable = false;
|
|
9391
9763
|
const results = [];
|
|
9392
9764
|
for (const s of summaries) {
|
|
9393
9765
|
const assertions = collectAssertions(s);
|
|
@@ -9422,14 +9794,23 @@ async function analyzeDeterministicSummaries(summaries, opts, cwd, reportDir, cu
|
|
|
9422
9794
|
});
|
|
9423
9795
|
continue;
|
|
9424
9796
|
}
|
|
9425
|
-
const
|
|
9426
|
-
|
|
9797
|
+
const specDiffResult = diffProvider ? await diffProvider.forSpec({
|
|
9798
|
+
featureName: s.featureName,
|
|
9799
|
+
specName: s.specName
|
|
9800
|
+
}) : null;
|
|
9801
|
+
const specDiff = specDiffResult?.ok ? specDiffResult : null;
|
|
9802
|
+
if (specDiff?.error && !warnedDiffUnavailable) {
|
|
9803
|
+
warnedDiffUnavailable = true;
|
|
9804
|
+
info(`failure analysis: source diff unavailable (${specDiff.error}) — analyzing without diff context`);
|
|
9805
|
+
}
|
|
9806
|
+
const diffExcerpt = specDiff?.patch ?? null;
|
|
9427
9807
|
const driftResult = driftResults.find((r) => r.target.featureName === s.featureName && r.target.specName === s.specName);
|
|
9428
9808
|
const driftIssues = driftResult?.ok ? driftResult.issues : null;
|
|
9429
9809
|
const failureLog = buildFailureLog(s);
|
|
9430
9810
|
let analysis = null;
|
|
9431
9811
|
let analysisSkipped = null;
|
|
9432
|
-
if (!
|
|
9812
|
+
if (!specDiffResult) analysisSkipped = "skipped: --failure-analysis not enabled";
|
|
9813
|
+
else if (!specDiffResult.ok) analysisSkipped = specDiffResult.skip;
|
|
9433
9814
|
else if (!auth.ok) analysisSkipped = auth.reason;
|
|
9434
9815
|
else if (specYaml === null) analysisSkipped = "no spec.yaml found for this spec";
|
|
9435
9816
|
else {
|
|
@@ -9440,15 +9821,16 @@ async function analyzeDeterministicSummaries(summaries, opts, cwd, reportDir, cu
|
|
|
9440
9821
|
specYaml,
|
|
9441
9822
|
failureLog,
|
|
9442
9823
|
diffPatch: diffExcerpt,
|
|
9443
|
-
changedFiles:
|
|
9444
|
-
baseRef:
|
|
9824
|
+
changedFiles: specDiffResult.nameStatus,
|
|
9825
|
+
baseRef: specDiffResult.base.ref,
|
|
9445
9826
|
driftIssues,
|
|
9446
9827
|
...opts.language ? { outputLanguage: opts.language } : {},
|
|
9447
9828
|
...triageUserPrompt ? { triageUserPrompt } : {},
|
|
9448
9829
|
...customPrompt ? { customPrompt } : {}
|
|
9449
9830
|
}, {
|
|
9450
9831
|
...opts.model ? { model: opts.model } : {},
|
|
9451
|
-
cwd
|
|
9832
|
+
cwd,
|
|
9833
|
+
getFileDiff: specDiffResult.fileDiff
|
|
9452
9834
|
});
|
|
9453
9835
|
analysis = outcome.analysis;
|
|
9454
9836
|
if (!printedHeader) {
|
|
@@ -9466,6 +9848,10 @@ async function analyzeDeterministicSummaries(summaries, opts, cwd, reportDir, cu
|
|
|
9466
9848
|
status: "failed",
|
|
9467
9849
|
analysis,
|
|
9468
9850
|
analysisSkipped,
|
|
9851
|
+
...specDiff ? { analysisBase: {
|
|
9852
|
+
ref: specDiff.base.ref,
|
|
9853
|
+
sha: specDiff.base.sha
|
|
9854
|
+
} } : {},
|
|
9469
9855
|
driftIssues,
|
|
9470
9856
|
failureLogExcerpt: failureLog.length > 0 ? failureLog : null,
|
|
9471
9857
|
diffExcerpt,
|
|
@@ -9475,8 +9861,6 @@ async function analyzeDeterministicSummaries(summaries, opts, cwd, reportDir, cu
|
|
|
9475
9861
|
}
|
|
9476
9862
|
return {
|
|
9477
9863
|
results,
|
|
9478
|
-
diff,
|
|
9479
|
-
baseRef,
|
|
9480
9864
|
customPromptVersion: customPrompt?.customPromptVersion ?? null
|
|
9481
9865
|
};
|
|
9482
9866
|
}
|
|
@@ -9488,30 +9872,33 @@ async function analyzeDeterministicSummaries(summaries, opts, cwd, reportDir, cu
|
|
|
9488
9872
|
* final report.json stays byte-identical (existing e2e goldens compare it).
|
|
9489
9873
|
*/
|
|
9490
9874
|
function buildReportEnvelope(args) {
|
|
9491
|
-
const {
|
|
9875
|
+
const { git, customPromptVersion, triageUserPromptHash, opts } = args;
|
|
9492
9876
|
return {
|
|
9493
9877
|
schemaVersion: 1,
|
|
9494
9878
|
kind: "run",
|
|
9495
9879
|
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9496
9880
|
runId: process.env["GITHUB_RUN_ID"] ?? null,
|
|
9497
9881
|
git: {
|
|
9498
|
-
head:
|
|
9499
|
-
base:
|
|
9882
|
+
head: git.head,
|
|
9883
|
+
base: git.base?.ref ?? null,
|
|
9884
|
+
...git.base ? {
|
|
9885
|
+
baseSha: git.base.sha,
|
|
9886
|
+
baseSource: git.base.source
|
|
9887
|
+
} : {}
|
|
9500
9888
|
},
|
|
9501
9889
|
model: opts.model ?? null,
|
|
9502
9890
|
language: opts.language ?? null,
|
|
9503
|
-
promptVersion: "
|
|
9891
|
+
promptVersion: "5",
|
|
9504
9892
|
customPromptVersion,
|
|
9505
9893
|
...triageUserPromptHash !== null ? { triageUserPromptHash } : {}
|
|
9506
9894
|
};
|
|
9507
9895
|
}
|
|
9508
9896
|
/** Write the unified JSON (+ optional GitHub-annotation) report for one run. Returns the report data. */
|
|
9509
9897
|
async function writeUnifiedReport(args) {
|
|
9510
|
-
const { reportDir, results,
|
|
9898
|
+
const { reportDir, results, git, customPromptVersion, triageUserPromptHash, opts } = args;
|
|
9511
9899
|
const data = {
|
|
9512
9900
|
...buildReportEnvelope({
|
|
9513
|
-
|
|
9514
|
-
baseRef,
|
|
9901
|
+
git,
|
|
9515
9902
|
customPromptVersion,
|
|
9516
9903
|
triageUserPromptHash,
|
|
9517
9904
|
opts
|
|
@@ -9825,7 +10212,7 @@ function installTeardownSignalHandlers(teardown) {
|
|
|
9825
10212
|
}
|
|
9826
10213
|
//#endregion
|
|
9827
10214
|
//#region src/cli/run.ts
|
|
9828
|
-
const runCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("run").argument("[targets...]", "Specs to run, space-separated: each '<feature>/<spec>', '<feature>', or omit for all. Duplicates are de-duped.").description("Run specs. Each spec's execution mode comes from its spec.yaml `mode:` field (default deterministic; set `mode: live` to have Claude drive agent-browser live per step). Deterministic specs replay the recorded test.spec.ts under vitest. A structured report (report.json + evidence) is always written; use --push-report to also stream it to a hub.").option("--report [dir]", `Directory for the structured run results (report.json + evidence PNGs) that are always written. Default: ${DEFAULT_REPORT_DIR}/. Pass this only to change the location.`).option("--push-report", "Incrementally push the run report to the hub as the run progresses (open → patch per spec → finalize). Requires --hub-url/--hub-token (or CCQA_HUB_URL/CCQA_HUB_TOKEN). Without it, hub credentials are used only to fetch variables/sessions/prompts, not to push.").option("--changed", "Restrict execution to specs whose relatedPaths intersect the git diff against
|
|
10215
|
+
const runCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("run").argument("[targets...]", "Specs to run, space-separated: each '<feature>/<spec>', '<feature>', or omit for all. Duplicates are de-duped.").description("Run specs. Each spec's execution mode comes from its spec.yaml `mode:` field (default deterministic; set `mode: live` to have Claude drive agent-browser live per step). Deterministic specs replay the recorded test.spec.ts under vitest. A structured report (report.json + evidence) is always written; use --push-report to also stream it to a hub.").option("--report [dir]", `Directory for the structured run results (report.json + evidence PNGs) that are always written. Default: ${DEFAULT_REPORT_DIR}/. Pass this only to change the location.`).option("--push-report", "Incrementally push the run report to the hub as the run progresses (open → patch per spec → finalize). Requires --hub-url/--hub-token (or CCQA_HUB_URL/CCQA_HUB_TOKEN). Without it, hub credentials are used only to fetch variables/sessions/prompts, not to push.").option("--changed [base]", "Restrict execution to specs whose relatedPaths intersect the git diff against [base]. Without a value the base comes from $GITHUB_BASE_REF (pull_request CI); elsewhere pass it explicitly (e.g. --changed=origin/main). Cannot be combined with an explicit spec id.").option("--failure-analysis [base]", "Classify each failure (TEST_DRIFT / SPEC_CHANGE / PRODUCT_BUG) against the source diff since [base]. Without a value the base comes from $GITHUB_BASE_REF (pull_request CI); elsewhere pass it explicitly (e.g. --failure-analysis=origin/main), or pass 'last-green' to diff each spec against the commit where it last passed (per-spec baselines from the hub; requires a hub connection). Off by default — no Claude calls without it.").option("--no-drift-audit", "With --failure-analysis: skip the spec↔code drift audit shown in the report.").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--format <fmt>", "Additional output format alongside HTML when --report is set: 'text' (default), 'json' (writes report.json), 'github' (GitHub Actions annotations on stdout).", (raw) => {
|
|
9829
10216
|
if (REPORT_FORMATS.includes(raw)) return raw;
|
|
9830
10217
|
throw new Error(`--format must be one of ${REPORT_FORMATS.join(" | ")}`);
|
|
9831
10218
|
}, "text").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--no-evidence", `(deterministic only) Skip step-boundary evidence capture (PNG + meta JSON written to ${DEFAULT_REPORT_DIR}/${EVIDENCE_SUBDIR}/ by default).`).option("--retry <n>", "(live only) Retry each failed step up to N more times before recording failure. Default 0.", (raw) => {
|
|
@@ -13319,6 +13706,11 @@ z.object({ error: z.object({
|
|
|
13319
13706
|
code: z.string(),
|
|
13320
13707
|
message: z.string()
|
|
13321
13708
|
}) });
|
|
13709
|
+
z.object({
|
|
13710
|
+
gitHead: z.string(),
|
|
13711
|
+
runId: z.string(),
|
|
13712
|
+
at: z.string()
|
|
13713
|
+
});
|
|
13322
13714
|
/**
|
|
13323
13715
|
* A triage-learning job. Grading failing specs in the hub UI produces the
|
|
13324
13716
|
* "actual cause" labels this reads; the job turns them into an improved
|
|
@@ -13434,6 +13826,7 @@ function createPushRunHandler(config) {
|
|
|
13434
13826
|
};
|
|
13435
13827
|
await config.storage.artifacts.putDir(run.id, dir);
|
|
13436
13828
|
await config.storage.runs.create(run);
|
|
13829
|
+
await updateLastGreenLedger(config.storage, run, report.results);
|
|
13437
13830
|
sendJson(ctx.res, 201, run);
|
|
13438
13831
|
} finally {
|
|
13439
13832
|
await rm(dir, {
|
|
@@ -13444,14 +13837,17 @@ function createPushRunHandler(config) {
|
|
|
13444
13837
|
};
|
|
13445
13838
|
}
|
|
13446
13839
|
/**
|
|
13447
|
-
* POST /api/v1/runs/open?project=&branch=&profile=&kind= — start a
|
|
13448
|
-
* run with no report yet. Unlike `POST /runs`, nothing is pushed up
|
|
13449
|
-
* the caller patches results in as they finish (`PATCH /runs/:id`), so
|
|
13450
|
-
* interrupted run still leaves a partial report on the hub instead of
|
|
13840
|
+
* POST /api/v1/runs/open?project=&branch=&profile=&kind=&gitHead= — start a
|
|
13841
|
+
* "running" run with no report yet. Unlike `POST /runs`, nothing is pushed up
|
|
13842
|
+
* front: the caller patches results in as they finish (`PATCH /runs/:id`), so
|
|
13843
|
+
* an interrupted run still leaves a partial report on the hub instead of
|
|
13844
|
+
* none. `gitHead` (optional) attributes the run to a commit from the start —
|
|
13845
|
+
* without it an interrupted run would never learn its commit.
|
|
13451
13846
|
*/
|
|
13452
13847
|
function createOpenRunHandler(config) {
|
|
13453
13848
|
return async (ctx) => {
|
|
13454
13849
|
const { project, branch, profile, kind } = parseRunScope(ctx);
|
|
13850
|
+
const gitHead = ctx.url.searchParams.get("gitHead");
|
|
13455
13851
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
13456
13852
|
const run = {
|
|
13457
13853
|
id: randomUUID(),
|
|
@@ -13466,7 +13862,7 @@ function createOpenRunHandler(config) {
|
|
|
13466
13862
|
passed: 0,
|
|
13467
13863
|
failed: 0
|
|
13468
13864
|
},
|
|
13469
|
-
gitHead: null,
|
|
13865
|
+
gitHead: gitHead || null,
|
|
13470
13866
|
promptVersion: "",
|
|
13471
13867
|
ciRunId: null,
|
|
13472
13868
|
reportCreatedAt: now,
|
|
@@ -13482,10 +13878,7 @@ const PatchRunRequestSchema = z.object({
|
|
|
13482
13878
|
done: z.boolean().optional(),
|
|
13483
13879
|
finalStatus: z.enum(["passed", "failed"]).optional(),
|
|
13484
13880
|
reportMeta: z.object({
|
|
13485
|
-
git:
|
|
13486
|
-
head: z.string().nullable(),
|
|
13487
|
-
base: z.string().nullable()
|
|
13488
|
-
}).partial().optional(),
|
|
13881
|
+
git: GitEnvelopeSchema.partial().optional(),
|
|
13489
13882
|
model: z.string().nullable().optional(),
|
|
13490
13883
|
language: z.string().nullable().optional(),
|
|
13491
13884
|
promptVersion: z.string().optional(),
|
|
@@ -13509,6 +13902,38 @@ function countSpecs(results) {
|
|
|
13509
13902
|
};
|
|
13510
13903
|
}
|
|
13511
13904
|
/**
|
|
13905
|
+
* Advance the last-green ledger for every passed spec of a terminal
|
|
13906
|
+
* `kind: "run"` run. Spec-level, not run-level: a run with one chronically
|
|
13907
|
+
* failing spec still moves the baseline of every spec that did pass.
|
|
13908
|
+
* Best-effort — a ledger failure must not fail the push; the ledger is an
|
|
13909
|
+
* accelerator for `--failure-analysis=last-green`, not part of the run
|
|
13910
|
+
* record. Runs without a branch or gitHead can't be placed in the ledger and
|
|
13911
|
+
* are skipped.
|
|
13912
|
+
*
|
|
13913
|
+
* Ordering caveat (known approximation): `at` is the run's reportCreatedAt —
|
|
13914
|
+
* open time for incremental runs, report time for immutable pushes. When two
|
|
13915
|
+
* runs on the same branch+profile overlap, "newest at wins" can pick either
|
|
13916
|
+
* of the two genuinely-green commits, since the hub has no git ancestry to
|
|
13917
|
+
* order them properly. Accepted: CI serializes per branch in practice, and a
|
|
13918
|
+
* baseline can only ever point at a commit where the spec really passed.
|
|
13919
|
+
*/
|
|
13920
|
+
async function updateLastGreenLedger(storage, run, results) {
|
|
13921
|
+
const { gitHead, branch } = run;
|
|
13922
|
+
if (run.kind !== "run" || !gitHead || !branch) return;
|
|
13923
|
+
const passed = results.filter((r) => r.status === "passed");
|
|
13924
|
+
if (passed.length === 0) return;
|
|
13925
|
+
const entries = Object.fromEntries(passed.map((r) => [`${r.feature}/${r.spec}`, {
|
|
13926
|
+
gitHead,
|
|
13927
|
+
runId: run.id,
|
|
13928
|
+
at: run.reportCreatedAt
|
|
13929
|
+
}]));
|
|
13930
|
+
try {
|
|
13931
|
+
await storage.lastGreen.merge(run.project, run.profile ?? "default", branch, entries);
|
|
13932
|
+
} catch (err) {
|
|
13933
|
+
console.error(`hub: last-green ledger update failed for run "${run.id}": ${err instanceof Error ? err.message : String(err)}`);
|
|
13934
|
+
}
|
|
13935
|
+
}
|
|
13936
|
+
/**
|
|
13512
13937
|
* PATCH /api/v1/runs/:id — incrementally add spec results (and evidence) to a
|
|
13513
13938
|
* "running" run. Once the run is terminal (`done: true` was sent, or it was
|
|
13514
13939
|
* pushed immutably via `POST /runs`), further patches are rejected with 409.
|
|
@@ -13530,6 +13955,7 @@ function createPatchRunHandler(config) {
|
|
|
13530
13955
|
if (!parsed.success) throw new HttpError(400, "invalid_body", `request body is invalid: ${parsed.error.issues[0]?.message ?? "schema mismatch"}`);
|
|
13531
13956
|
const { rows, evidence, done, finalStatus, reportMeta } = parsed.data;
|
|
13532
13957
|
let specs = run.specs;
|
|
13958
|
+
let mergedResults = [];
|
|
13533
13959
|
await config.storage.artifacts.updateJsonFile(id, "report.json", (current) => {
|
|
13534
13960
|
const base = current ?? {
|
|
13535
13961
|
schemaVersion: 1,
|
|
@@ -13549,7 +13975,9 @@ function createPatchRunHandler(config) {
|
|
|
13549
13975
|
...base,
|
|
13550
13976
|
...reportMeta?.git ? { git: {
|
|
13551
13977
|
head: reportMeta.git.head ?? base.git.head,
|
|
13552
|
-
base: reportMeta.git.base ?? base.git.base
|
|
13978
|
+
base: reportMeta.git.base ?? base.git.base,
|
|
13979
|
+
baseSha: reportMeta.git.baseSha ?? base.git.baseSha ?? null,
|
|
13980
|
+
baseSource: reportMeta.git.baseSource ?? base.git.baseSource ?? null
|
|
13553
13981
|
} } : {},
|
|
13554
13982
|
...reportMeta?.model !== void 0 ? { model: reportMeta.model } : {},
|
|
13555
13983
|
...reportMeta?.language !== void 0 ? { language: reportMeta.language } : {},
|
|
@@ -13559,6 +13987,7 @@ function createPatchRunHandler(config) {
|
|
|
13559
13987
|
};
|
|
13560
13988
|
const merged = mergeResults(current?.results ?? [], rows);
|
|
13561
13989
|
specs = countSpecs(merged);
|
|
13990
|
+
mergedResults = merged;
|
|
13562
13991
|
return {
|
|
13563
13992
|
...envelope,
|
|
13564
13993
|
results: merged
|
|
@@ -13572,6 +14001,7 @@ function createPatchRunHandler(config) {
|
|
|
13572
14001
|
...reportMeta?.promptVersion ? { promptVersion: reportMeta.promptVersion } : {}
|
|
13573
14002
|
} : { specs };
|
|
13574
14003
|
const updated = await config.storage.runs.update(id, patch);
|
|
14004
|
+
if (done) await updateLastGreenLedger(config.storage, updated, mergedResults);
|
|
13575
14005
|
sendJson(ctx.res, 200, updated);
|
|
13576
14006
|
};
|
|
13577
14007
|
}
|
|
@@ -13688,9 +14118,11 @@ function parseRunScope(ctx) {
|
|
|
13688
14118
|
};
|
|
13689
14119
|
}
|
|
13690
14120
|
/**
|
|
13691
|
-
* A branch is a free-form label (e.g. `feature/foo`),
|
|
13692
|
-
*
|
|
13693
|
-
*
|
|
14121
|
+
* A branch is a free-form label (e.g. `feature/foo`), so `/` is allowed —
|
|
14122
|
+
* only length is bounded (a sanity cap; the last-green ledger separately
|
|
14123
|
+
* hash-truncates long percent-encoded names into a safe filename, see
|
|
14124
|
+
* paths.ts). Run records store it verbatim. null when the client didn't
|
|
14125
|
+
* send one. Exported for the last-green handler.
|
|
13694
14126
|
*/
|
|
13695
14127
|
function requireBranch(raw) {
|
|
13696
14128
|
if (raw === null || raw === "") return null;
|
|
@@ -13943,6 +14375,32 @@ function createListProfilesHandler(storage) {
|
|
|
13943
14375
|
};
|
|
13944
14376
|
}
|
|
13945
14377
|
//#endregion
|
|
14378
|
+
//#region src/hub/api/handlers/last-green.ts
|
|
14379
|
+
/**
|
|
14380
|
+
* GET /api/v1/projects/:project/last-green?profile=&branch=&fallbackBranch=
|
|
14381
|
+
*
|
|
14382
|
+
* Returns the last-green ledger entries for one project/profile, keyed by
|
|
14383
|
+
* "feature/spec". `branch` is the caller's current branch; `fallbackBranch`
|
|
14384
|
+
* (optional, typically the default branch) is overlaid *under* it, so a PR
|
|
14385
|
+
* branch with no greens of its own still inherits the default branch's
|
|
14386
|
+
* baselines while its own greens take precedence. One round trip serves the
|
|
14387
|
+
* whole run.
|
|
14388
|
+
*/
|
|
14389
|
+
function createGetLastGreenHandler(storage) {
|
|
14390
|
+
return async (ctx) => {
|
|
14391
|
+
const project = requireSafeSegment(ctx.params.project, "project");
|
|
14392
|
+
const profile = requireSafeSegment(ctx.url.searchParams.get("profile") ?? "default", "profile");
|
|
14393
|
+
const branch = requireBranch(ctx.url.searchParams.get("branch"));
|
|
14394
|
+
if (!branch) throw new HttpError(400, "missing_param", "branch query parameter is required");
|
|
14395
|
+
const fallbackBranch = requireBranch(ctx.url.searchParams.get("fallbackBranch"));
|
|
14396
|
+
const [primary, fallback] = await Promise.all([storage.lastGreen.get(project, profile, branch), fallbackBranch && fallbackBranch !== branch ? storage.lastGreen.get(project, profile, fallbackBranch) : Promise.resolve({})]);
|
|
14397
|
+
sendJson(ctx.res, 200, { entries: {
|
|
14398
|
+
...fallback,
|
|
14399
|
+
...primary
|
|
14400
|
+
} });
|
|
14401
|
+
};
|
|
14402
|
+
}
|
|
14403
|
+
//#endregion
|
|
13946
14404
|
//#region src/hub/api/handlers/prompts.ts
|
|
13947
14405
|
const MAX_PROMPT_BODY_BYTES = 256 * 1024;
|
|
13948
14406
|
/** Validate the `:project` route param (prompts are project-scoped, not per-profile). */
|
|
@@ -17951,7 +18409,7 @@ function createLearningWorker(deps) {
|
|
|
17951
18409
|
const generatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
17952
18410
|
const customPrompt = {
|
|
17953
18411
|
schemaVersion: 1,
|
|
17954
|
-
basePromptVersion: "
|
|
18412
|
+
basePromptVersion: "5",
|
|
17955
18413
|
customPromptVersion: `${generatedAt}-c${cases.length}`,
|
|
17956
18414
|
generatedAt,
|
|
17957
18415
|
guidance
|
|
@@ -18070,6 +18528,7 @@ function registerRoutes(router, config, queue) {
|
|
|
18070
18528
|
router.put("/api/v1/runs/:id/triage/actual-causes", createImportActualCausesHandler(storage));
|
|
18071
18529
|
router.get("/api/v1/projects", createListProjectsHandler(storage));
|
|
18072
18530
|
router.get("/api/v1/projects/:project/profiles", createListProfilesHandler(storage));
|
|
18531
|
+
router.get("/api/v1/projects/:project/last-green", createGetLastGreenHandler(storage));
|
|
18073
18532
|
const sessionConfig = {
|
|
18074
18533
|
store: storage.sessions,
|
|
18075
18534
|
encryptionKey: config.encryptionKey
|
|
@@ -18302,6 +18761,10 @@ function perspectivesKindDir(root) {
|
|
|
18302
18761
|
function perspectivesPath(root, project) {
|
|
18303
18762
|
return join(perspectivesKindDir(root), `${project}.json`);
|
|
18304
18763
|
}
|
|
18764
|
+
function lastGreenPath(root, project, profile, branch) {
|
|
18765
|
+
const encoded = encodeURIComponent(branch);
|
|
18766
|
+
return join(root, "last-green", project, profile, `${encoded.length <= 200 ? encoded : `${encoded.slice(0, 64)}-${createHash("sha256").update(branch).digest("hex").slice(0, 32)}`}.json`);
|
|
18767
|
+
}
|
|
18305
18768
|
//#endregion
|
|
18306
18769
|
//#region src/hub/core/storage/file/artifact-store.ts
|
|
18307
18770
|
/**
|
|
@@ -18389,6 +18852,25 @@ function createFileJobStore(root) {
|
|
|
18389
18852
|
};
|
|
18390
18853
|
}
|
|
18391
18854
|
//#endregion
|
|
18855
|
+
//#region src/hub/core/storage/file/last-green-store.ts
|
|
18856
|
+
function createFileLastGreenStore(root) {
|
|
18857
|
+
return {
|
|
18858
|
+
async get(project, profile, branch) {
|
|
18859
|
+
return await readJson(lastGreenPath(root, project, profile, branch)) ?? {};
|
|
18860
|
+
},
|
|
18861
|
+
async merge(project, profile, branch, entries) {
|
|
18862
|
+
await updateJson(lastGreenPath(root, project, profile, branch), (current) => {
|
|
18863
|
+
const out = { ...current ?? {} };
|
|
18864
|
+
for (const [key, entry] of Object.entries(entries)) {
|
|
18865
|
+
const prev = out[key];
|
|
18866
|
+
if (!prev || prev.at <= entry.at) out[key] = entry;
|
|
18867
|
+
}
|
|
18868
|
+
return out;
|
|
18869
|
+
});
|
|
18870
|
+
}
|
|
18871
|
+
};
|
|
18872
|
+
}
|
|
18873
|
+
//#endregion
|
|
18392
18874
|
//#region src/hub/core/storage/file/perspectives-store.ts
|
|
18393
18875
|
/**
|
|
18394
18876
|
* Defense-in-depth path validation: the HTTP layer already checks the project
|
|
@@ -18642,7 +19124,8 @@ function createFileHubStorage(dataDir) {
|
|
|
18642
19124
|
triage: createFileTriageStore(dataDir),
|
|
18643
19125
|
prompts: createFilePromptStore(dataDir),
|
|
18644
19126
|
perspectives: createFilePerspectivesStore(dataDir),
|
|
18645
|
-
jobs: createFileJobStore(dataDir)
|
|
19127
|
+
jobs: createFileJobStore(dataDir),
|
|
19128
|
+
lastGreen: createFileLastGreenStore(dataDir)
|
|
18646
19129
|
};
|
|
18647
19130
|
}
|
|
18648
19131
|
//#endregion
|