harnery 0.37.0 → 0.38.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/dist/commander.d.ts +9 -0
  2. package/dist/commander.d.ts.map +1 -1
  3. package/dist/commander.js +7 -2
  4. package/dist/commands/admission.d.ts +21 -0
  5. package/dist/commands/admission.d.ts.map +1 -0
  6. package/dist/commands/admission.js +565 -0
  7. package/dist/commands/agents.d.ts +7 -0
  8. package/dist/commands/agents.d.ts.map +1 -1
  9. package/dist/commands/agents.js +61 -1
  10. package/dist/commands/artifacts.d.ts.map +1 -1
  11. package/dist/commands/artifacts.js +48 -2
  12. package/dist/commands/browse-ai.d.ts +2 -2
  13. package/dist/commands/browse-ai.d.ts.map +1 -1
  14. package/dist/commands/browse-ai.js +6 -4
  15. package/dist/commands/browse.d.ts.map +1 -1
  16. package/dist/commands/browse.js +349 -21
  17. package/dist/commands/fetch.js +1 -0
  18. package/dist/commands/qa-record.d.ts +144 -0
  19. package/dist/commands/qa-record.d.ts.map +1 -0
  20. package/dist/commands/qa-record.js +0 -0
  21. package/dist/commands/qa-run.d.ts +7 -3
  22. package/dist/commands/qa-run.d.ts.map +1 -1
  23. package/dist/commands/qa-run.js +268 -13
  24. package/dist/commands/qa-status.d.ts +71 -0
  25. package/dist/commands/qa-status.d.ts.map +1 -0
  26. package/dist/commands/qa-status.js +490 -0
  27. package/dist/commands/qa-verify.d.ts +40 -0
  28. package/dist/commands/qa-verify.d.ts.map +1 -0
  29. package/dist/commands/qa-verify.js +180 -0
  30. package/dist/commands/review-pack.d.ts +4 -0
  31. package/dist/commands/review-pack.d.ts.map +1 -0
  32. package/dist/commands/review-pack.js +1001 -0
  33. package/dist/core/agents/qa-signal.d.ts +111 -0
  34. package/dist/core/agents/qa-signal.d.ts.map +1 -0
  35. package/dist/core/agents/qa-signal.js +231 -0
  36. package/dist/core/agents/session-name-display.d.ts +20 -5
  37. package/dist/core/agents/session-name-display.d.ts.map +1 -1
  38. package/dist/core/agents/session-name-display.js +67 -7
  39. package/dist/core/agents/state/heartbeat-reader.d.ts +7 -0
  40. package/dist/core/agents/state/heartbeat-reader.d.ts.map +1 -1
  41. package/dist/core/agents/state/heartbeat-writer.d.ts +11 -0
  42. package/dist/core/agents/state/heartbeat-writer.d.ts.map +1 -1
  43. package/dist/core/agents/state/heartbeat-writer.js +17 -0
  44. package/dist/core/agents/state/live-coordination-view.d.ts.map +1 -1
  45. package/dist/core/agents/state/live-coordination-view.js +1 -0
  46. package/dist/core/agents/state/live-coordination-writer.js +5 -0
  47. package/dist/core/artifacts/constants.d.ts +1 -1
  48. package/dist/core/artifacts/constants.js +1 -1
  49. package/dist/core/artifacts/index.d.ts +57 -6
  50. package/dist/core/artifacts/index.d.ts.map +1 -1
  51. package/dist/core/artifacts/index.js +265 -10
  52. package/dist/core/config.d.ts +8 -0
  53. package/dist/core/config.d.ts.map +1 -1
  54. package/dist/core/config.js +16 -0
  55. package/dist/core/diagnostics/bundle.d.ts +16 -0
  56. package/dist/core/diagnostics/bundle.d.ts.map +1 -1
  57. package/dist/core/diagnostics/bundle.js +101 -7
  58. package/dist/core/events/v3/bootstrap.d.ts.map +1 -1
  59. package/dist/core/events/v3/bootstrap.js +10 -0
  60. package/dist/core/events/v3/coordination-view.d.ts +3 -0
  61. package/dist/core/events/v3/coordination-view.d.ts.map +1 -1
  62. package/dist/core/events/v3/coordination-view.js +69 -8
  63. package/dist/core/events/v3/producers/intake.d.ts.map +1 -1
  64. package/dist/core/events/v3/producers/intake.js +9 -2
  65. package/dist/core/events/v3/producers/recorder.d.ts +23 -0
  66. package/dist/core/events/v3/producers/recorder.d.ts.map +1 -1
  67. package/dist/core/events/v3/producers/recorder.js +264 -18
  68. package/dist/core/hooks/cli.js +120 -27
  69. package/dist/core/hooks/resolve/transcript.d.ts.map +1 -1
  70. package/dist/core/hooks/resolve/transcript.js +10 -3
  71. package/dist/core/hooks/session-name-presence.d.ts.map +1 -1
  72. package/dist/core/hooks/session-name-presence.js +4 -1
  73. package/dist/core/qa-artifacts.d.ts +20 -0
  74. package/dist/core/qa-artifacts.d.ts.map +1 -0
  75. package/dist/core/qa-artifacts.js +110 -0
  76. package/dist/core/resources/contract.d.ts +6 -0
  77. package/dist/core/resources/contract.d.ts.map +1 -1
  78. package/dist/core/resources/sampler.d.ts +7 -0
  79. package/dist/core/resources/sampler.d.ts.map +1 -1
  80. package/dist/core/resources/sampler.js +106 -5
  81. package/dist/lib/admission.d.ts +71 -0
  82. package/dist/lib/admission.d.ts.map +1 -0
  83. package/dist/lib/admission.js +264 -0
  84. package/dist/lib/agent-browser/client.d.ts +1 -1
  85. package/dist/lib/agent-browser/client.d.ts.map +1 -1
  86. package/dist/lib/agent-browser/client.js +1 -5
  87. package/dist/lib/browser/capture-fidelity.d.ts +39 -0
  88. package/dist/lib/browser/capture-fidelity.d.ts.map +1 -0
  89. package/dist/lib/browser/capture-fidelity.js +84 -0
  90. package/dist/lib/browser/client.d.ts +41 -1
  91. package/dist/lib/browser/client.d.ts.map +1 -1
  92. package/dist/lib/browser/client.js +167 -9
  93. package/dist/lib/browser/critique.d.ts +38 -1
  94. package/dist/lib/browser/critique.d.ts.map +1 -1
  95. package/dist/lib/browser/critique.js +34 -6
  96. package/dist/lib/browser/index.d.ts +4 -2
  97. package/dist/lib/browser/index.d.ts.map +1 -1
  98. package/dist/lib/browser/index.js +2 -0
  99. package/dist/lib/browser/page-review-judge.d.ts +64 -0
  100. package/dist/lib/browser/page-review-judge.d.ts.map +1 -0
  101. package/dist/lib/browser/page-review-judge.js +270 -0
  102. package/dist/lib/browser/page-review-pack.d.ts +613 -0
  103. package/dist/lib/browser/page-review-pack.d.ts.map +1 -0
  104. package/dist/lib/browser/page-review-pack.js +1751 -0
  105. package/dist/lib/browser/qa-run-contracts.d.ts +214 -10
  106. package/dist/lib/browser/qa-run-contracts.d.ts.map +1 -1
  107. package/dist/lib/browser/qa-run-contracts.js +136 -1
  108. package/dist/lib/browser/qa-run.d.ts +100 -10
  109. package/dist/lib/browser/qa-run.d.ts.map +1 -1
  110. package/dist/lib/browser/qa-run.js +768 -169
  111. package/dist/lib/browser/request-diagnostics.d.ts +13 -0
  112. package/dist/lib/browser/request-diagnostics.d.ts.map +1 -0
  113. package/dist/lib/browser/request-diagnostics.js +18 -0
  114. package/dist/lib/browser/tiling.d.ts +19 -0
  115. package/dist/lib/browser/tiling.d.ts.map +1 -1
  116. package/dist/lib/browser/tiling.js +28 -0
  117. package/dist/lib/cookies/client.d.ts +9 -0
  118. package/dist/lib/cookies/client.d.ts.map +1 -1
  119. package/dist/lib/cookies/client.js +197 -44
  120. package/dist/lib/cookies/extra.d.ts +18 -0
  121. package/dist/lib/cookies/extra.d.ts.map +1 -0
  122. package/dist/lib/cookies/extra.js +14 -0
  123. package/dist/lib/cookies/index.d.ts +2 -1
  124. package/dist/lib/cookies/index.d.ts.map +1 -1
  125. package/dist/lib/cookies/index.js +2 -1
  126. package/dist/lib/durable-job.d.ts +124 -0
  127. package/dist/lib/durable-job.d.ts.map +1 -0
  128. package/dist/lib/durable-job.js +296 -0
  129. package/dist/lib/http/client.d.ts +7 -1
  130. package/dist/lib/http/client.d.ts.map +1 -1
  131. package/dist/lib/http/client.js +2 -0
  132. package/dist/lib/instructions/templates.d.ts.map +1 -1
  133. package/dist/lib/instructions/templates.js +5 -2
  134. package/package.json +8 -2
  135. package/src/commander.ts +50 -2
  136. package/src/commands/admission.ts +699 -0
  137. package/src/commands/agents.ts +87 -1
  138. package/src/commands/artifacts.ts +97 -21
  139. package/src/commands/browse-ai.ts +10 -5
  140. package/src/commands/browse.ts +481 -20
  141. package/src/commands/fetch.ts +1 -0
  142. package/src/commands/qa-record.ts +682 -0
  143. package/src/commands/qa-run.ts +335 -16
  144. package/src/commands/qa-status.ts +608 -0
  145. package/src/commands/qa-verify.ts +238 -0
  146. package/src/commands/review-pack.ts +1281 -0
  147. package/src/core/agents/qa-signal.ts +261 -0
  148. package/src/core/agents/session-name-display.ts +78 -7
  149. package/src/core/agents/state/heartbeat-reader.ts +7 -0
  150. package/src/core/agents/state/heartbeat-writer.ts +23 -0
  151. package/src/core/agents/state/live-coordination-view.ts +1 -0
  152. package/src/core/agents/state/live-coordination-writer.ts +5 -0
  153. package/src/core/artifacts/constants.ts +1 -1
  154. package/src/core/artifacts/index.ts +370 -21
  155. package/src/core/config.ts +23 -0
  156. package/src/core/diagnostics/bundle.ts +119 -11
  157. package/src/core/events/v3/bootstrap.ts +10 -0
  158. package/src/core/events/v3/coordination-view.ts +100 -11
  159. package/src/core/events/v3/producers/intake.ts +9 -2
  160. package/src/core/events/v3/producers/recorder.ts +312 -18
  161. package/src/core/hooks/cli.ts +140 -32
  162. package/src/core/hooks/resolve/transcript.ts +10 -3
  163. package/src/core/hooks/session-name-presence.ts +6 -1
  164. package/src/core/qa-artifacts.ts +126 -0
  165. package/src/core/resources/contract.ts +7 -0
  166. package/src/core/resources/sampler.ts +138 -6
  167. package/src/lib/admission.ts +347 -0
  168. package/src/lib/agent-browser/client.ts +2 -10
  169. package/src/lib/browser/capture-fidelity.ts +98 -0
  170. package/src/lib/browser/client.ts +206 -10
  171. package/src/lib/browser/critique.ts +62 -7
  172. package/src/lib/browser/index.ts +36 -0
  173. package/src/lib/browser/page-review-judge.ts +360 -0
  174. package/src/lib/browser/page-review-pack.ts +2384 -0
  175. package/src/lib/browser/qa-run-contracts.ts +366 -3
  176. package/src/lib/browser/qa-run.ts +868 -190
  177. package/src/lib/browser/request-diagnostics.ts +27 -0
  178. package/src/lib/browser/tiling.ts +32 -0
  179. package/src/lib/cookies/client.ts +228 -42
  180. package/src/lib/cookies/extra.ts +28 -0
  181. package/src/lib/cookies/index.ts +2 -0
  182. package/src/lib/durable-job.ts +407 -0
  183. package/src/lib/http/client.ts +13 -1
  184. package/src/lib/instructions/templates.ts +5 -2
@@ -11,32 +11,126 @@
11
11
  //
12
12
  // Toolkit tier: this module must not import src/core (layering check).
13
13
 
14
- import { execFile } from "node:child_process";
15
- import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
16
- import { join } from "node:path";
14
+ import { spawn } from "node:child_process";
15
+ import { randomUUID } from "node:crypto";
16
+ import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
17
+ import { cpus, freemem, loadavg, totalmem } from "node:os";
18
+ import { basename, join } from "node:path";
19
+ import { type CritiqueProvider, DEFAULT_CRITIQUE_RUBRIC } from "./critique.js";
20
+ import { type JudgedContext, judgePageReviewPack, toCritiqueRecords } from "./page-review-judge.js";
21
+ import {
22
+ finalizePageReviewPack,
23
+ gateHitsFromEnvelope,
24
+ PAGE_REVIEW_DEFAULT_RETENTION_MINUTES,
25
+ PAGE_REVIEW_FINDINGS_FILENAME,
26
+ PAGE_REVIEW_PACK_DIRNAME,
27
+ PAGE_REVIEW_PACK_SCHEMA,
28
+ PAGE_REVIEW_REVIEW_FILENAME,
29
+ type PageReviewContextRecord,
30
+ type PageReviewCritiqueRecord,
31
+ type PageReviewGateRecord,
32
+ readPackContext,
33
+ readPackDom,
34
+ readPackFullPage,
35
+ readPackManifest,
36
+ readPackSignature,
37
+ } from "./page-review-pack.js";
17
38
  import type { QaManifest } from "./qa-plan.js";
39
+ import { type PersistedCritique, QA_CRITIQUE_CONTRACT_VERSION, rubricDigest } from "./qa-reuse.js";
18
40
  import {
41
+ computeJobDigest,
19
42
  computeVerdict,
20
43
  mergeCoverage,
21
44
  QA_RUN_RESULT_SCHEMA_VERSION,
45
+ QA_RUN_STATUS_SCHEMA_VERSION,
22
46
  type QaRunBlocker,
23
47
  type QaRunCommandOutcome,
24
48
  type QaRunContext,
49
+ type QaRunCritiqueLatency,
25
50
  type QaRunCritiqueOutcome,
51
+ type QaRunCritiquePool,
52
+ type QaRunHostSample,
26
53
  type QaRunJob,
27
54
  type QaRunResult,
55
+ type QaRunReviewPack,
56
+ type QaRunStage,
57
+ type QaRunStatusDocument,
58
+ type QaRunStatusState,
28
59
  } from "./qa-run-contracts.js";
60
+ import { type QaSnapshotStoreOptions, saveQaSnapshot } from "./qa-snapshot.js";
29
61
 
30
- /** Set on critique children unless the job permits metered critique: the
31
- * host's critique provider must stay on subscription-backed headless
32
- * harnesses and surface exhaustion instead of falling back to a metered API. */
62
+ /** Set on the runner's own environment before the host's critique provider is
63
+ * loaded for the judge stage, unless the job permits metered critique: the
64
+ * provider must stay on subscription-backed headless harnesses and surface
65
+ * exhaustion instead of falling back to a metered API. */
33
66
  export const QA_RUN_HEADLESS_ONLY_ENV = "HARNERY_CRITIQUE_HEADLESS_ONLY";
34
67
 
35
68
  /** Result document written into the run's output directory. */
36
69
  export const QA_RUN_RESULT_FILENAME = "page-qa-result.json";
37
70
 
71
+ /** Pointer document written into the parent output directory after every
72
+ * run, naming the newest run's directory and verdict. Consumers resolve the
73
+ * current result through this pointer instead of guessing at loose files. */
74
+ export const QA_RUN_LATEST_FILENAME = "latest.json";
75
+
76
+ export interface QaRunLatestPointerInput {
77
+ run_id: string;
78
+ dir: string;
79
+ completed_at: string;
80
+ verdict: QaRunResult["verdict"];
81
+ }
82
+
83
+ /**
84
+ * Publish the parent directory's latest-result pointer without allowing an
85
+ * older completion to replace a newer one. Both runner and manual evidence
86
+ * use this writer so every producer preserves the same ordering invariant.
87
+ */
88
+ export function writeLatestPointer(outParent: string, input: QaRunLatestPointerInput): string {
89
+ const pointerPath = join(outParent, QA_RUN_LATEST_FILENAME);
90
+ let pointerIsNewer = true;
91
+ try {
92
+ const existing = JSON.parse(readFileSync(pointerPath, "utf8")) as {
93
+ completed_at?: unknown;
94
+ };
95
+ if (typeof existing.completed_at === "string") {
96
+ const existingCompletedAt = Date.parse(existing.completed_at);
97
+ const candidateCompletedAt = Date.parse(input.completed_at);
98
+ pointerIsNewer =
99
+ Number.isNaN(existingCompletedAt) || candidateCompletedAt >= existingCompletedAt;
100
+ }
101
+ } catch {
102
+ // No readable pointer yet: this result becomes the first one.
103
+ }
104
+
105
+ if (pointerIsNewer) {
106
+ const pointer = {
107
+ schema_version: 1,
108
+ ...input,
109
+ result: join(input.dir, QA_RUN_RESULT_FILENAME),
110
+ };
111
+ const pointerTmp = join(outParent, `.${QA_RUN_LATEST_FILENAME}.${input.run_id}.tmp`);
112
+ writeFileSync(pointerTmp, `${JSON.stringify(pointer, null, 2)}\n`);
113
+ renameSync(pointerTmp, pointerPath);
114
+ }
115
+ return pointerPath;
116
+ }
117
+
118
+ /** Live status document beside the result (QaRunStatusDocument): written at
119
+ * start, every stage boundary, and on a heartbeat timer, so a disconnected
120
+ * client can tell a running job from a dead one without guessing. */
121
+ export const QA_RUN_STATUS_FILENAME = "run-status.json";
122
+
123
+ /** The effective validated job, written into the run directory so a
124
+ * reconnecting client can re-derive the job digest (`qa-verify --job`). */
125
+ export const QA_RUN_JOB_FILENAME = "job.json";
126
+
127
+ const STATUS_HEARTBEAT_MS = 15_000;
128
+
38
129
  const DEFAULT_COMMAND_TIMEOUT_MS = 120_000;
130
+ const DEFAULT_RUN_DEADLINE_MS = 900_000;
39
131
  const DEFAULT_COMMAND_CONCURRENCY = 2;
132
+ /** Gate-hit rectangles forwarded to one context's capture child (`--review-pack-hit-rect`). */
133
+ const REVIEW_PACK_HIT_RECTS_PER_CONTEXT = 50;
40
134
  const EXEC_MAX_BUFFER = 16 * 1024 * 1024;
41
135
 
42
136
  /** The planner's deterministic vocabulary is an executable contract, not a
@@ -70,8 +164,31 @@ export interface QaRunExecResult {
70
164
  /** Injectable child-process executor. `argv[0]` is the executable. */
71
165
  export type QaRunExec = (argv: string[], options: QaRunExecOptions) => Promise<QaRunExecResult>;
72
166
 
73
- /** Default executor: execFile with argv arrays only (never a shell string),
74
- * closed stdin, bounded output buffers, and the policy timeout. */
167
+ /** Grace between the timeout's SIGTERM and the follow-up SIGKILL. A child
168
+ * that catches SIGTERM (Bun installs a handler by default) gets this long to
169
+ * exit before the kill is made non-negotiable. */
170
+ export const QA_RUN_KILL_GRACE_MS = 5_000;
171
+
172
+ /** After the group SIGKILL, how long to wait for stdio to drain before
173
+ * destroying the streams and settling anyway. An escaped grandchild (setsid)
174
+ * can hold the pipes open forever; the result must not wait on it. */
175
+ const KILL_DRAIN_MS = 2_000;
176
+
177
+ /** Default executor: spawn with argv arrays only (never a shell string),
178
+ * closed stdin, bounded output buffers, and the policy timeout.
179
+ *
180
+ * Timeout enforcement is escalated and group-wide, via `spawn` rather than
181
+ * `execFile` for two live-verified reasons. First, a child that catches
182
+ * SIGTERM while awaiting its own grandchildren turns a single polite kill
183
+ * into an unbounded wait (a critique command outlived its 120s cap by 10x).
184
+ * Second, `execFile` resolves only when the child's stdio closes, and an
185
+ * orphaned grandchild inheriting the pipe keeps it open after the child is
186
+ * dead — so even a delivered kill did not settle the call. The child is
187
+ * therefore spawned detached into its own process group; at the deadline the
188
+ * whole group gets SIGTERM, then SIGKILL after a grace, and the result
189
+ * settles on exit with whatever output drained, never waiting on a pipe an
190
+ * orphan still holds. A timed-out command reports an error even if the child
191
+ * then exits 0 — a result produced after the deadline cannot be trusted. */
75
192
  export const defaultQaRunExec: QaRunExec = (argv, options) =>
76
193
  new Promise((resolvePromise) => {
77
194
  const [command, ...args] = argv;
@@ -79,47 +196,122 @@ export const defaultQaRunExec: QaRunExec = (argv, options) =>
79
196
  resolvePromise({ exitCode: null, stdout: "", stderr: "", error: "empty argv" });
80
197
  return;
81
198
  }
82
- const child = execFile(
83
- command,
84
- args,
85
- {
86
- timeout: options.timeoutMs,
87
- maxBuffer: EXEC_MAX_BUFFER,
88
- env: options.env,
89
- killSignal: "SIGTERM",
90
- },
91
- (err, stdout, stderr) => {
92
- const out = String(stdout ?? "");
93
- const errOut = String(stderr ?? "");
94
- if (!err) {
95
- resolvePromise({ exitCode: 0, stdout: out, stderr: errOut });
96
- return;
97
- }
98
- const failure = err as NodeJS.ErrnoException & {
99
- code?: number | string;
100
- killed?: boolean;
101
- signal?: NodeJS.Signals | null;
102
- };
103
- if (typeof failure.code === "number") {
104
- // Completed with a nonzero exit code: a real outcome, not an error.
105
- resolvePromise({ exitCode: failure.code, stdout: out, stderr: errOut });
199
+ // Windows has no process groups; the direct-child kill is the best
200
+ // available fallback there.
201
+ const groupKill = process.platform !== "win32";
202
+ const child = spawn(command, args, {
203
+ env: options.env,
204
+ stdio: ["ignore", "pipe", "pipe"],
205
+ ...(groupKill ? { detached: true } : {}),
206
+ });
207
+
208
+ let stdout = "";
209
+ let stderr = "";
210
+ let outBytes = 0;
211
+ let timedOut = false;
212
+ let overflowed = false;
213
+ let settled = false;
214
+ let termTimer: ReturnType<typeof setTimeout> | undefined;
215
+ let killTimer: ReturnType<typeof setTimeout> | undefined;
216
+ let drainTimer: ReturnType<typeof setTimeout> | undefined;
217
+
218
+ const settle = (result: QaRunExecResult): void => {
219
+ if (settled) return;
220
+ settled = true;
221
+ clearTimeout(termTimer);
222
+ clearTimeout(killTimer);
223
+ clearTimeout(drainTimer);
224
+ resolvePromise(result);
225
+ };
226
+
227
+ const signalGroup = (signal: NodeJS.Signals): void => {
228
+ const pid = child.pid;
229
+ if (!pid) return;
230
+ if (groupKill) {
231
+ try {
232
+ process.kill(-pid, signal);
106
233
  return;
234
+ } catch {
235
+ // Group already gone, or the leader died before setpgid: fall
236
+ // through to the direct child so the kill still lands somewhere.
107
237
  }
108
- const reason =
109
- failure.killed || failure.signal
110
- ? `killed by ${failure.signal ?? "signal"} (timeout ${options.timeoutMs}ms)`
111
- : failure.message || "spawn failed";
112
- resolvePromise({ exitCode: null, stdout: out, stderr: errOut, error: reason });
113
- },
114
- );
115
- child.stdin?.end();
238
+ }
239
+ try {
240
+ child.kill(signal);
241
+ } catch {
242
+ // Nothing left to kill.
243
+ }
244
+ };
245
+
246
+ const collect = (chunk: Buffer | string, sink: "stdout" | "stderr"): void => {
247
+ const text = String(chunk);
248
+ outBytes += text.length;
249
+ if (sink === "stdout") stdout += text;
250
+ else stderr += text;
251
+ if (outBytes > EXEC_MAX_BUFFER && !overflowed) {
252
+ overflowed = true;
253
+ signalGroup("SIGKILL");
254
+ }
255
+ };
256
+ child.stdout?.on("data", (chunk) => collect(chunk, "stdout"));
257
+ child.stderr?.on("data", (chunk) => collect(chunk, "stderr"));
258
+
259
+ child.on("error", (err) => {
260
+ settle({ exitCode: null, stdout, stderr, error: err.message || "spawn failed" });
261
+ });
262
+
263
+ const finish = (code: number | null, signal: NodeJS.Signals | null): void => {
264
+ if (timedOut) {
265
+ settle({
266
+ exitCode: null,
267
+ stdout,
268
+ stderr,
269
+ error: `timed out after ${options.timeoutMs}ms (process group killed)`,
270
+ });
271
+ } else if (overflowed) {
272
+ settle({
273
+ exitCode: null,
274
+ stdout,
275
+ stderr,
276
+ error: `output exceeded ${EXEC_MAX_BUFFER} bytes (process group killed)`,
277
+ });
278
+ } else if (signal) {
279
+ settle({ exitCode: null, stdout, stderr, error: `killed by ${signal}` });
280
+ } else {
281
+ settle({ exitCode: code, stdout, stderr });
282
+ }
283
+ };
284
+
285
+ // "close" is the clean path: process exited AND stdio drained. "exit"
286
+ // arms the drain failsafe so an orphan holding the pipes cannot postpone
287
+ // the result forever.
288
+ child.on("close", (code, signal) => finish(code, signal));
289
+ child.on("exit", (code, signal) => {
290
+ drainTimer = setTimeout(() => {
291
+ child.stdout?.destroy();
292
+ child.stderr?.destroy();
293
+ finish(code, signal);
294
+ }, KILL_DRAIN_MS);
295
+ drainTimer.unref?.();
296
+ });
297
+
298
+ termTimer = setTimeout(() => {
299
+ timedOut = true;
300
+ signalGroup("SIGTERM");
301
+ }, options.timeoutMs);
302
+ killTimer = setTimeout(() => signalGroup("SIGKILL"), options.timeoutMs + QA_RUN_KILL_GRACE_MS);
303
+ termTimer.unref?.();
304
+ killTimer.unref?.();
116
305
  });
117
306
 
118
307
  export interface QaRunMatrixOptions {
119
308
  /** A validated job (see validateQaRunJob — the runner trusts its shape). */
120
309
  job: QaRunJob;
121
- /** Directory for artifacts and the result document. Created if missing. */
122
- outDir: string;
310
+ /** PARENT directory for run output. Every invocation creates its own
311
+ * `run-<run_id>/` beneath it for artifacts and the result document, and
312
+ * maintains `latest.json` in the parent — a reused parent can therefore
313
+ * never present an older run's result as the current one. */
314
+ outParent: string;
123
315
  /** argv prefix that reaches the host CLI's browse command, e.g.
124
316
  * `[process.execPath, cliScript, "browse"]`. */
125
317
  browseArgv: string[];
@@ -129,6 +321,41 @@ export interface QaRunMatrixOptions {
129
321
  childEnv?: NodeJS.ProcessEnv;
130
322
  /** Progress callback for human-facing per-stage lines. */
131
323
  onLog?: (message: string) => void;
324
+ /** Run ID override (tests). Default: crypto.randomUUID(). */
325
+ runId?: string;
326
+ /** Working-tree revision probe supplied by the caller (the CLI probes git
327
+ * once). Ignored when the job itself pins tested_revision. */
328
+ revisionProbe?: { tested_revision?: string; worktree_dirty?: boolean };
329
+ /** Machine-wide admission gate. When present the runner acquires a slot
330
+ * before any browser work, records the wait as wall_time_ms.queue (never
331
+ * part of total), and finalizes an incomplete result with an "admission"
332
+ * blocker when acquisition fails — the evidence trail survives a full
333
+ * queue. The returned function releases the slot; the runner calls it at
334
+ * finalize, and a crashed runner's slot is reclaimed by dead-PID pruning. */
335
+ admission?: {
336
+ resource: string;
337
+ acquire: (onWait: (message: string) => void) => Promise<() => void>;
338
+ /** Snapshot of the other holders of the resource, sampled alongside host
339
+ * pressure so an incomplete run names what it was competing with. */
340
+ holders?: () => Array<{ label: string; pid: number }>;
341
+ };
342
+ /** Host-injected vision call for the judge stage. The judge runs in this
343
+ * process over the pack on disk, after every capture browser has closed. */
344
+ critiqueProvider?: CritiqueProvider;
345
+ /** Lazy alternative to `critiqueProvider`; called once, after the
346
+ * headless-only environment has been applied. */
347
+ critiqueProviderLoader?: () => Promise<CritiqueProvider | undefined>;
348
+ /** QA snapshot store override (tests, host-managed cache locations). */
349
+ snapshotStore?: QaSnapshotStoreOptions;
350
+ /** Renders the command a reader can run to judge the pack later; shown in
351
+ * the pack's review.md when the judge did not run. */
352
+ reviewPackJudgeCommand?: (packDir: string) => string;
353
+ /** True when the run writes into the managed artifact store: the pack's
354
+ * manifest then says `managed: true` and the expiry sweep may delete it
355
+ * once `policy.review_pack_retention_minutes` (default 90) have passed
356
+ * since the judge finished. A pack under an explicit out dir stays
357
+ * unmanaged and is never deleted automatically. */
358
+ reviewPackManaged?: boolean;
132
359
  }
133
360
 
134
361
  interface TimedExec {
@@ -264,49 +491,190 @@ function parseConsoleFailures(envelope: Record<string, unknown> | undefined): st
264
491
  return failures;
265
492
  }
266
493
 
267
- interface EnvelopeCritique {
268
- tiles?: number;
269
- provider?: boolean;
270
- outcome?: string;
271
- findings?: Array<{ severity?: string; category?: string; description?: string }>;
272
- provider_meta?: Record<string, unknown>;
273
- error?: string;
274
- }
275
-
276
- function providerLabel(critique: EnvelopeCritique | undefined): string {
277
- const meta = critique?.provider_meta;
278
- if (meta) {
279
- for (const key of ["provider", "route", "model"]) {
280
- const value = meta[key];
281
- if (typeof value === "string" && value.length > 0) return value;
282
- }
494
+ /**
495
+ * Lift per-backend vision latency out of the envelope's provider_meta. The
496
+ * shape is host-owned (`providers[name].latency_ms = {count, p50, p95}`), so
497
+ * read defensively: a malformed entry or one with zero calls is dropped.
498
+ * Without this the runner kept only the provider label, and a slow tile was
499
+ * visible in `ps` and nowhere in the result document.
500
+ */
501
+ function critiqueLatency(
502
+ providerMeta: Record<string, unknown> | undefined,
503
+ ): Record<string, QaRunCritiqueLatency> | undefined {
504
+ const providers = providerMeta?.providers;
505
+ if (!providers || typeof providers !== "object" || Array.isArray(providers)) return undefined;
506
+ const out: Record<string, QaRunCritiqueLatency> = {};
507
+ for (const [name, state] of Object.entries(providers as Record<string, unknown>)) {
508
+ const lat = (state as { latency_ms?: unknown } | null)?.latency_ms as
509
+ | { count?: unknown; p50?: unknown; p95?: unknown }
510
+ | undefined;
511
+ if (!lat || typeof lat !== "object") continue;
512
+ const { count, p50, p95 } = lat;
513
+ if (typeof count !== "number" || count <= 0) continue;
514
+ if (typeof p50 !== "number" || typeof p95 !== "number") continue;
515
+ out[name] = { count, p50, p95 };
283
516
  }
284
- return critique?.provider ? "host" : "none";
517
+ return Object.keys(out).length > 0 ? out : undefined;
285
518
  }
286
519
 
287
520
  /**
288
521
  * Execute the whole QA matrix for one validated job and return the result
289
522
  * (also written to `<outDir>/page-qa-result.json`). Stages: plan →
290
- * deterministic gates (bounded pool) → interactions (serial) → critique (one
291
- * context at a time; the critique provider owns tile concurrency) → snapshot.
292
- * The verdict is computeVerdict over everything recorded — fail-closed.
523
+ * deterministic gates (bounded pool) → interactions (serial) → capture (each
524
+ * context rendered once into the run's page review pack through the same
525
+ * pool, browser closed) → critique (one in-process pool of vision calls over
526
+ * every tile of every context, no browser open) → snapshot (persisted from
527
+ * the pack's files in signoff). The verdict is computeVerdict over
528
+ * everything recorded — fail-closed.
293
529
  */
294
530
  export async function runQaMatrix(options: QaRunMatrixOptions): Promise<QaRunResult> {
295
- const { job, outDir, browseArgv } = options;
531
+ const { job, outParent, browseArgv } = options;
296
532
  const exec = options.exec ?? defaultQaRunExec;
297
533
  const log = options.onLog ?? (() => {});
298
534
  const timeoutMs = job.policy?.command_timeout_ms ?? DEFAULT_COMMAND_TIMEOUT_MS;
299
535
  const concurrency = job.policy?.command_concurrency ?? DEFAULT_COMMAND_CONCURRENCY;
536
+
537
+ const hostSample = (): QaRunHostSample => {
538
+ const holders = options.admission?.holders?.() ?? undefined;
539
+ return {
540
+ captured_at: new Date().toISOString(),
541
+ loadavg_1m: loadavg()[0] ?? 0,
542
+ free_mem_bytes: freemem(),
543
+ total_mem_bytes: totalmem(),
544
+ cpu_count: cpus().length,
545
+ // Exclude this run itself: a holder list that names the sampler tells
546
+ // the reader nothing about contention.
547
+ ...(holders ? { competing: holders.filter((holder) => holder.pid !== process.pid) } : {}),
548
+ };
549
+ };
550
+ const stageHostSamples: Partial<Record<QaRunStage, QaRunHostSample>> = {};
551
+
552
+ const runId = options.runId ?? randomUUID();
553
+ const outDir = join(outParent, `run-${runId}`);
300
554
  mkdirSync(outDir, { recursive: true });
555
+ const startedAtIso = new Date().toISOString();
556
+ writeFileSync(join(outDir, QA_RUN_JOB_FILENAME), `${JSON.stringify(job, null, 2)}\n`);
301
557
 
558
+ // Live status: state + stage + heartbeat, written atomically so a client
559
+ // that lost its terminal can distinguish this run being alive from dead.
560
+ let statusState: QaRunStatusState = "running";
561
+ let statusStage: QaRunStage | null = null;
562
+ let statusQueue: QaRunStatusDocument["queue"];
563
+ let statusVerdict: QaRunStatusDocument["verdict"];
564
+ const writeStatus = (): void => {
565
+ const status: QaRunStatusDocument = {
566
+ schema_version: QA_RUN_STATUS_SCHEMA_VERSION,
567
+ run_id: runId,
568
+ pid: process.pid,
569
+ state: statusState,
570
+ stage: statusStage,
571
+ started_at: startedAtIso,
572
+ updated_at: new Date().toISOString(),
573
+ ...(statusQueue ? { queue: statusQueue } : {}),
574
+ ...(statusVerdict ? { verdict: statusVerdict } : {}),
575
+ };
576
+ try {
577
+ const tmp = join(outDir, `.${QA_RUN_STATUS_FILENAME}.tmp`);
578
+ writeFileSync(tmp, `${JSON.stringify(status, null, 2)}\n`);
579
+ renameSync(tmp, join(outDir, QA_RUN_STATUS_FILENAME));
580
+ } catch {
581
+ // Status is advisory; the result document is the authoritative record.
582
+ }
583
+ };
584
+ const heartbeat = setInterval(writeStatus, STATUS_HEARTBEAT_MS);
585
+ heartbeat.unref();
586
+
587
+ /** Enter a stage: record the host pressure it starts under and refresh the
588
+ * live status document in one place. */
589
+ const enterStage = (stage: QaRunStage): void => {
590
+ statusStage = stage;
591
+ stageHostSamples[stage] = hostSample();
592
+ writeStatus();
593
+ };
594
+
595
+ const revision: Pick<
596
+ QaRunResult["run"],
597
+ "tested_revision" | "revision_source" | "worktree_dirty"
598
+ > =
599
+ job.tested_revision !== undefined
600
+ ? { tested_revision: job.tested_revision, revision_source: "job" }
601
+ : options.revisionProbe?.tested_revision !== undefined
602
+ ? {
603
+ tested_revision: options.revisionProbe.tested_revision,
604
+ revision_source: "git",
605
+ ...(options.revisionProbe.worktree_dirty !== undefined
606
+ ? { worktree_dirty: options.revisionProbe.worktree_dirty }
607
+ : {}),
608
+ }
609
+ : { revision_source: "unknown" };
610
+ const jobDigest = computeJobDigest(job);
611
+ const wall: QaRunResult["wall_time_ms"] = {
612
+ plan: 0,
613
+ gates: 0,
614
+ interactions: 0,
615
+ capture: 0,
616
+ critique: 0,
617
+ snapshot: 0,
618
+ total: 0,
619
+ };
620
+
621
+ // ------------------------------------------------------------- admission
622
+ // Queue wait happens before the runner clock starts: wall_time_ms.total
623
+ // stays pure runner time and the wait is reported as wall_time_ms.queue.
624
+ let releaseAdmission: (() => void) | undefined;
625
+ let admissionFailure: string | undefined;
626
+ if (options.admission) {
627
+ statusState = "queued";
628
+ statusQueue = {
629
+ resource: options.admission.resource,
630
+ waiting_since: new Date().toISOString(),
631
+ };
632
+ writeStatus();
633
+ const queueStart = Date.now();
634
+ try {
635
+ releaseAdmission = await options.admission.acquire((message) => {
636
+ log(message);
637
+ writeStatus();
638
+ });
639
+ } catch (err: unknown) {
640
+ admissionFailure = err instanceof Error ? err.message : String(err);
641
+ }
642
+ wall.queue = Date.now() - queueStart;
643
+ statusQueue = undefined;
644
+ }
645
+ statusState = "running";
646
+ writeStatus();
647
+
648
+ const hostStart = hostSample();
302
649
  const startedAt = Date.now();
303
- const wall = { plan: 0, gates: 0, interactions: 0, critique: 0, snapshot: 0, total: 0 };
304
650
  const blockers: QaRunBlocker[] = [];
651
+ // Overall deadline: the clock starts after admission (pure runner time) and
652
+ // is consulted before every command, so the worst overshoot is one command
653
+ // timeout plus the kill grace. Exceeding it fails closed as incomplete.
654
+ const runDeadlineMs = job.policy?.run_deadline_ms ?? DEFAULT_RUN_DEADLINE_MS;
655
+ let deadlineHit = false;
656
+ const pastDeadline = (): boolean => {
657
+ if (deadlineHit) return true;
658
+ if (Date.now() - startedAt < runDeadlineMs) return false;
659
+ deadlineHit = true;
660
+ blockers.push({
661
+ stage: "deadline",
662
+ reason:
663
+ `run deadline of ${runDeadlineMs}ms exceeded — remaining commands were skipped and ` +
664
+ "the result finalized as incomplete (raise policy.run_deadline_ms for a legitimately " +
665
+ "larger matrix)",
666
+ });
667
+ log(`deadline: ${runDeadlineMs}ms exceeded, skipping remaining commands`);
668
+ return true;
669
+ };
305
670
  const commands: QaRunCommandOutcome[] = [];
306
671
  const critique: QaRunCritiqueOutcome[] = [];
672
+ const stagesRun: QaRunStage[] = [];
307
673
  let manifest: QaManifest | null = null;
308
674
  let contexts: QaRunContext[] = [];
309
675
  let snapshot: QaRunResult["snapshot"] = { saved: false };
676
+ let critiquePool: QaRunCritiquePool | undefined;
677
+ let reviewPack: QaRunReviewPack | undefined;
310
678
 
311
679
  const baseEnv: NodeJS.ProcessEnv = { ...process.env, ...options.childEnv };
312
680
 
@@ -328,15 +696,44 @@ export async function runQaMatrix(options: QaRunMatrixOptions): Promise<QaRunRes
328
696
 
329
697
  const finalize = (): QaRunResult => {
330
698
  wall.total = Date.now() - startedAt;
699
+ // Prefix semantics: the last stage the run progressed THROUGH cleanly.
700
+ // Stages are pushed in execution order; the first blocked stage ends the
701
+ // clean prefix, so a gate-blocked run reports "plan" even though the
702
+ // (empty) interactions stage technically executed afterwards.
703
+ const blockedStages = new Set(blockers.map((blocker) => blocker.stage));
704
+ let lastCompletedStage: QaRunStage | null = null;
705
+ for (const stage of stagesRun) {
706
+ if (blockedStages.has(stage)) break;
707
+ lastCompletedStage = stage;
708
+ }
331
709
  const result: QaRunResult = {
332
710
  schema_version: QA_RUN_RESULT_SCHEMA_VERSION,
711
+ evidence_source: "runner",
712
+ run: {
713
+ run_id: runId,
714
+ started_at: startedAtIso,
715
+ completed_at: new Date().toISOString(),
716
+ ...revision,
717
+ job_digest: jobDigest,
718
+ out_dir: outDir,
719
+ },
720
+ host: {
721
+ start: hostStart,
722
+ finish: hostSample(),
723
+ ...(Object.keys(stageHostSamples).length > 0 ? { stages: stageHostSamples } : {}),
724
+ },
725
+ last_completed_stage: lastCompletedStage,
333
726
  target: job.target,
334
- ...(job.tested_revision !== undefined ? { tested_revision: job.tested_revision } : {}),
727
+ ...(revision.tested_revision !== undefined
728
+ ? { tested_revision: revision.tested_revision }
729
+ : {}),
335
730
  mode: job.mode,
336
731
  qa_plan: manifest,
337
732
  contexts,
338
733
  commands,
339
734
  critique,
735
+ ...(critiquePool ? { critique_pool: critiquePool } : {}),
736
+ ...(reviewPack ? { review_pack: reviewPack } : {}),
340
737
  snapshot,
341
738
  wall_time_ms: { ...wall },
342
739
  blockers,
@@ -349,9 +746,29 @@ export async function runQaMatrix(options: QaRunMatrixOptions): Promise<QaRunRes
349
746
  }),
350
747
  };
351
748
  writeFileSync(join(outDir, QA_RUN_RESULT_FILENAME), `${JSON.stringify(result, null, 2)}\n`);
749
+ writeLatestPointer(outParent, {
750
+ run_id: runId,
751
+ dir: basename(outDir),
752
+ completed_at: result.run.completed_at,
753
+ verdict: result.verdict,
754
+ });
755
+ statusState = "completed";
756
+ statusStage = null;
757
+ statusVerdict = result.verdict;
758
+ writeStatus();
759
+ clearInterval(heartbeat);
760
+ releaseAdmission?.();
352
761
  return result;
353
762
  };
354
763
 
764
+ if (admissionFailure !== undefined) {
765
+ blockers.push({
766
+ stage: "admission",
767
+ reason: `no admission slot for browser work: ${admissionFailure}`,
768
+ });
769
+ return finalize();
770
+ }
771
+
355
772
  // Base render arguments shared by every per-context invocation.
356
773
  const contextRenderArgs = (ctx: QaRunContext): string[] => [
357
774
  "--viewport",
@@ -360,6 +777,13 @@ export async function runQaMatrix(options: QaRunMatrixOptions): Promise<QaRunRes
360
777
  ...(ctx.args ?? []),
361
778
  ];
362
779
 
780
+ // The planner's tile ceiling and every critique child must agree on the
781
+ // per-context band cap, or the predicted cost and the real coverage drift.
782
+ const critiqueMaxTilesArgs =
783
+ job.policy?.critique_max_tiles !== undefined
784
+ ? ["--check-critique-max-tiles", String(job.policy.critique_max_tiles)]
785
+ : [];
786
+
363
787
  // ------------------------------------------------------------------ plan
364
788
  const planArgv = [
365
789
  ...browseArgv,
@@ -367,13 +791,16 @@ export async function runQaMatrix(options: QaRunMatrixOptions): Promise<QaRunRes
367
791
  "--qa-plan",
368
792
  "--json",
369
793
  "--no-screenshot",
794
+ ...critiqueMaxTilesArgs,
370
795
  ...(job.qa_hints?.scopes ?? []).flatMap((selector) => ["--qa-scope", selector]),
371
796
  ...(job.qa_hints?.states?.length ? ["--qa-states", job.qa_hints.states.join(",")] : []),
372
797
  ];
373
798
  log(`plan: ${job.target}`);
799
+ enterStage("plan");
374
800
  const planStart = Date.now();
375
801
  const plan = await timedExec(planArgv, baseEnv);
376
802
  wall.plan = Date.now() - planStart;
803
+ stagesRun.push("plan");
377
804
  let planEnvelope: Record<string, unknown> | undefined;
378
805
  if (!plan.res.error && plan.res.exitCode === 0) {
379
806
  try {
@@ -449,8 +876,10 @@ export async function runQaMatrix(options: QaRunMatrixOptions): Promise<QaRunRes
449
876
  const enforceConsole = manifest.checks.deterministic.includes("console");
450
877
  const checks = job.checks ?? [];
451
878
  const gateOutcomes = new Array<QaRunCommandOutcome>(contexts.length);
879
+ enterStage("gates");
452
880
  const gatesStart = Date.now();
453
881
  await runPool(contexts, concurrency, async (ctx, index) => {
882
+ if (pastDeadline()) return;
454
883
  const applicable = checks.filter(
455
884
  (check) => check.contexts === undefined || check.contexts.includes(ctx.id),
456
885
  );
@@ -466,6 +895,10 @@ export async function runQaMatrix(options: QaRunMatrixOptions): Promise<QaRunRes
466
895
  ...contextRenderArgs(ctx),
467
896
  "--out",
468
897
  outPrefix,
898
+ // When the manifest requires a visual pass, the capture stage writes the
899
+ // full-page PNG into the pack, so the gate trio carries no screenshot;
900
+ // nothing reads the gate PNG (the envelope JSON is the evidence).
901
+ ...(manifest.checks.visual !== "none" ? ["--no-screenshot"] : []),
469
902
  ...manifestGateArgs,
470
903
  ...applicable.flatMap((check) => check.args),
471
904
  ];
@@ -515,11 +948,14 @@ export async function runQaMatrix(options: QaRunMatrixOptions): Promise<QaRunRes
515
948
  };
516
949
  });
517
950
  wall.gates = Date.now() - gatesStart;
951
+ stagesRun.push("gates");
518
952
  // Manifest order regardless of completion order: outcomes were written by
519
- // context index, so a straight push preserves it.
520
- commands.push(...gateOutcomes);
953
+ // context index, so a straight push preserves it. Deadline-skipped slots
954
+ // are empty: the single deadline blocker is their record.
955
+ commands.push(...gateOutcomes.filter((outcome) => outcome !== undefined));
521
956
 
522
957
  // ---------------------------------------------------------- interactions
958
+ enterStage("interactions");
523
959
  const interactionsStart = Date.now();
524
960
  const declaredStates = new Set((job.interaction_states ?? []).map((state) => state.name));
525
961
  const manifestStates = [
@@ -538,6 +974,7 @@ export async function runQaMatrix(options: QaRunMatrixOptions): Promise<QaRunRes
538
974
  }
539
975
  }
540
976
  for (const state of job.interaction_states ?? []) {
977
+ if (pastDeadline()) break;
541
978
  const outPrefix = join(outDir, `interaction-${state.name}`);
542
979
  const argv = [
543
980
  ...browseArgv,
@@ -587,151 +1024,391 @@ export async function runQaMatrix(options: QaRunMatrixOptions): Promise<QaRunRes
587
1024
  });
588
1025
  }
589
1026
  wall.interactions = Date.now() - interactionsStart;
1027
+ stagesRun.push("interactions");
590
1028
 
591
- // -------------------------------------------------------------- critique
1029
+ // -------------------------------------------------------------- capture
1030
+ // Each context is rendered ONCE into the run's page review pack (full-page
1031
+ // screenshot, tiles as PNG files, DOM, signature) through the same bounded
1032
+ // pool the gates used, and its browser closes before any vision call.
592
1033
  const visual = manifest.checks.visual;
593
1034
  const cleanSoFar =
594
1035
  blockers.length === 0 && commands.every((command) => command.outcome === "passed");
595
- const critiqueStart = Date.now();
596
- const savedSnapshots = new Map<string, string>();
597
- if (cleanSoFar && visual !== "none") {
598
- const critiqueEnv: NodeJS.ProcessEnv = job.policy?.allow_metered_critique
599
- ? { ...baseEnv }
600
- : { ...baseEnv, [QA_RUN_HEADLESS_ONLY_ENV]: "1" };
601
- const scopeSelectors =
602
- visual === "full-page" ? [undefined] : manifest.scopes.map((s) => s.selector);
603
- for (const ctx of contexts) {
604
- let tilesTotal = 0;
605
- let tilesReviewed = 0;
606
- let tilesReused = 0;
607
- let provider = "none";
608
- let contextOutcome: QaRunCritiqueOutcome["outcome"] = "passed";
609
- const findings: QaRunCritiqueOutcome["findings"] = [];
610
- for (const [scopeIndex, selector] of scopeSelectors.entries()) {
611
- const suffix = scopeSelectors.length > 1 ? `-scope${scopeIndex}` : "";
612
- const outPrefix = join(outDir, `${ctx.id}-critique${suffix}`);
613
- const argv = [
614
- ...browseArgv,
615
- job.target,
616
- ...contextRenderArgs(ctx),
617
- "--out",
618
- outPrefix,
619
- ...(selector !== undefined ? ["--check-critique", selector] : ["--check-critique"]),
620
- "--check-critique-fail",
621
- ...(manifest.baseline_source !== "none" ? ["--qa-reuse"] : []),
622
- ...(job.mode === "signoff"
623
- ? ["--qa-snapshot", "--qa-theme", ctx.theme, "--qa-state", ctx.state]
624
- : []),
625
- ];
626
- log(`critique ${ctx.id}${selector !== undefined ? ` [${selector}]` : ""}`);
627
- const { res, wallTimeMs } = await timedExec(argv, critiqueEnv);
628
- const jsonPath = `${outPrefix}.json`;
629
- const envelope = readEnvelope(jsonPath);
630
- const envelopeCritique = envelope?.critique as EnvelopeCritique | undefined;
631
- const failures: string[] = [];
632
- let commandOutcome: QaRunCommandOutcome["outcome"];
633
- if (res.error || (res.exitCode !== 0 && res.exitCode !== 2) || !envelope) {
634
- commandOutcome = "unknown";
635
- const excerpt = execErrorExcerpt(res);
636
- const base = res.error
637
- ? res.error
638
- : !envelope
639
- ? `exit code ${res.exitCode ?? "null"}, missing JSON artifact ${jsonPath}`
640
- : `exit code ${res.exitCode ?? "null"}`;
641
- const reason = excerpt ? `${base}: ${excerpt}` : base;
1036
+ const packDir = join(outDir, PAGE_REVIEW_PACK_DIRNAME);
1037
+ const capturedRecords: PageReviewContextRecord[] = [];
1038
+ const judgeCommand = options.reviewPackJudgeCommand?.(packDir);
1039
+ // Gate rectangles ride along from each gate's JSON artifact so the pack's
1040
+ // inspection plan can point a runt, a contrast miss, or a clipped element
1041
+ // at the tiles that show it. A missing or unparseable artifact simply
1042
+ // contributes no hits; the gate's `failures` still carry the text.
1043
+ const gateRecords = (): PageReviewGateRecord[] =>
1044
+ commands
1045
+ .filter((command) => command.check_id !== "plan" && command.check_id !== "review-pack")
1046
+ .map((command) => {
1047
+ const hits = command.artifacts.json
1048
+ ? gateHitsFromEnvelope(readEnvelope(command.artifacts.json))
1049
+ : [];
1050
+ return {
1051
+ context_id: command.context_id,
1052
+ check_id: command.check_id,
1053
+ outcome: command.outcome,
1054
+ failures: command.failures,
1055
+ ...(hits.length > 0 ? { hits } : {}),
1056
+ };
1057
+ });
1058
+ const finalizePack = (
1059
+ stage: QaRunStage,
1060
+ critiqueRecords: PageReviewCritiqueRecord[] | null,
1061
+ pool?: { concurrency: number; wall_time_ms: number; provider: string },
1062
+ ): void => {
1063
+ try {
1064
+ finalizePageReviewPack({
1065
+ packDir,
1066
+ target: job.target,
1067
+ ...(revision.tested_revision !== undefined
1068
+ ? { tested_revision: revision.tested_revision }
1069
+ : {}),
1070
+ contexts: capturedRecords,
1071
+ gates: gateRecords(),
1072
+ critique: critiqueRecords,
1073
+ ...(pool ? { pool } : {}),
1074
+ ...(judgeCommand ? { judgeCommand } : {}),
1075
+ createdAt: startedAtIso,
1076
+ // The retention clock restarts at each finalize, so it ends
1077
+ // `retention_minutes` after the judge (or after capture when the
1078
+ // judge never runs).
1079
+ retention: {
1080
+ expires_at: new Date(
1081
+ Date.now() +
1082
+ (job.policy?.review_pack_retention_minutes ?? PAGE_REVIEW_DEFAULT_RETENTION_MINUTES) *
1083
+ 60_000,
1084
+ ).toISOString(),
1085
+ managed: options.reviewPackManaged === true,
1086
+ },
1087
+ });
1088
+ const written = readPackManifest(packDir);
1089
+ reviewPack = {
1090
+ schema: PAGE_REVIEW_PACK_SCHEMA,
1091
+ dir: packDir,
1092
+ review: join(packDir, PAGE_REVIEW_REVIEW_FILENAME),
1093
+ findings: join(packDir, PAGE_REVIEW_FINDINGS_FILENAME),
1094
+ ...(written.retention ? { expires_at: written.retention.expires_at } : {}),
1095
+ ...(written.size_bytes !== undefined ? { size_bytes: written.size_bytes } : {}),
1096
+ };
1097
+ } catch (err: unknown) {
1098
+ blockers.push({
1099
+ stage,
1100
+ reason: `page review pack could not be finalized: ${err instanceof Error ? err.message : String(err)}`,
1101
+ });
1102
+ }
1103
+ };
1104
+
1105
+ // The provider is loaded once, before capture, so the capture children can
1106
+ // clamp band height to the routed model's vision budget (what browse does
1107
+ // for --check-critique) and the judge reuses the same instance.
1108
+ const priorHeadlessOnly = process.env[QA_RUN_HEADLESS_ONLY_ENV];
1109
+ const restoreHeadlessOnly = (): void => {
1110
+ if (priorHeadlessOnly === undefined) delete process.env[QA_RUN_HEADLESS_ONLY_ENV];
1111
+ else process.env[QA_RUN_HEADLESS_ONLY_ENV] = priorHeadlessOnly;
1112
+ };
1113
+ let provider: CritiqueProvider | undefined;
1114
+ const runsVisual = cleanSoFar && visual !== "none";
1115
+ if (runsVisual) {
1116
+ if (!job.policy?.allow_metered_critique) process.env[QA_RUN_HEADLESS_ONLY_ENV] = "1";
1117
+ try {
1118
+ provider =
1119
+ options.critiqueProvider ??
1120
+ (options.critiqueProviderLoader ? await options.critiqueProviderLoader() : undefined);
1121
+ } catch (err: unknown) {
1122
+ provider = undefined;
1123
+ log(`critique provider failed to load: ${err instanceof Error ? err.message : String(err)}`);
1124
+ }
1125
+ }
1126
+ const bandArgs =
1127
+ provider?.tileBudgetPx !== undefined
1128
+ ? ["--check-critique-band", String(Math.max(200, Math.min(1400, provider.tileBudgetPx)))]
1129
+ : [];
1130
+ const scopeArgs =
1131
+ visual === "full-page"
1132
+ ? []
1133
+ : manifest.scopes.flatMap((s) => ["--review-pack-scope", s.selector]);
1134
+ // Gate-hit rectangles ride into each context's capture child so the bands
1135
+ // they land in are cut even past the tile cap (`hit band N` tiles). The
1136
+ // gates ran first, so the rects are known before the page is captured.
1137
+ const hitRectsByContext = new Map<string, string[]>();
1138
+ for (const gate of gateRecords()) {
1139
+ for (const hit of gate.hits ?? []) {
1140
+ const list = hitRectsByContext.get(gate.context_id) ?? [];
1141
+ if (list.length >= REVIEW_PACK_HIT_RECTS_PER_CONTEXT) break;
1142
+ const r = hit.rect;
1143
+ list.push(
1144
+ [r.x, r.y, Math.max(0, r.width), Math.max(0, r.height)].map((n) => Math.round(n)).join(","),
1145
+ );
1146
+ hitRectsByContext.set(gate.context_id, list);
1147
+ }
1148
+ }
1149
+ const hitRectArgs = (contextId: string): string[] =>
1150
+ (hitRectsByContext.get(contextId) ?? []).flatMap((spec) => ["--review-pack-hit-rect", spec]);
1151
+
1152
+ enterStage("capture");
1153
+ const captureStart = Date.now();
1154
+ if (runsVisual) {
1155
+ const captureOutcomes = new Array<QaRunCommandOutcome>(contexts.length);
1156
+ const captureRecords = new Array<PageReviewContextRecord | undefined>(contexts.length);
1157
+ await runPool(contexts, concurrency, async (ctx, index) => {
1158
+ if (pastDeadline()) return;
1159
+ const outPrefix = join(outDir, `${ctx.id}-capture`);
1160
+ const argv = [
1161
+ ...browseArgv,
1162
+ job.target,
1163
+ ...contextRenderArgs(ctx),
1164
+ "--out",
1165
+ outPrefix,
1166
+ "--no-screenshot",
1167
+ "--review-pack",
1168
+ packDir,
1169
+ "--review-pack-context",
1170
+ ctx.id,
1171
+ "--qa-theme",
1172
+ ctx.theme,
1173
+ "--qa-state",
1174
+ ctx.state,
1175
+ ...(visual === "scoped" ? ["--no-review-pack-bands"] : []),
1176
+ ...scopeArgs,
1177
+ ...critiqueMaxTilesArgs,
1178
+ ...bandArgs,
1179
+ ...hitRectArgs(ctx.id),
1180
+ ];
1181
+ log(
1182
+ `capture ${ctx.id}${scopeArgs.length > 0 ? ` [${manifest.scopes.map((s) => s.selector).join(",")}]` : ""}`,
1183
+ );
1184
+ const { res, wallTimeMs } = await timedExec(argv, baseEnv);
1185
+ const jsonPath = `${outPrefix}.json`;
1186
+ const envelope = readEnvelope(jsonPath);
1187
+ const report = envelope?.reviewPack as { context_id?: string } | undefined;
1188
+ const failures: string[] = [];
1189
+ let outcome: QaRunCommandOutcome["outcome"];
1190
+ let record: PageReviewContextRecord | undefined;
1191
+ if (!res.error && res.exitCode === 0 && envelope && report?.context_id) {
1192
+ try {
1193
+ record = readPackContext(packDir, report.context_id);
1194
+ outcome = "passed";
1195
+ } catch (err: unknown) {
1196
+ outcome = "unknown";
1197
+ const reason = `pack context unreadable: ${err instanceof Error ? err.message : String(err)}`;
642
1198
  failures.push(reason);
643
- blockers.push({
644
- stage: "critique",
645
- context_id: ctx.id,
646
- reason: `critique command did not complete: ${reason}`,
647
- });
648
- contextOutcome = "unknown";
649
- } else {
650
- tilesTotal += envelopeCritique?.tiles ?? 0;
651
- const reuse = envelope.qaReuse as
652
- | { tiles_reused?: number; tiles_reviewed?: number }
653
- | undefined;
654
- tilesReused += reuse?.tiles_reused ?? 0;
655
- tilesReviewed += reuse?.tiles_reviewed ?? envelopeCritique?.tiles ?? 0;
656
- provider = providerLabel(envelopeCritique);
657
- for (const finding of envelopeCritique?.findings ?? []) {
658
- findings.push({
659
- severity: finding.severity ?? "unknown",
660
- summary: finding.category
661
- ? `${finding.category}: ${finding.description ?? ""}`
662
- : (finding.description ?? ""),
663
- ...(selector !== undefined ? { selector } : {}),
664
- });
665
- }
666
- if (envelopeCritique?.outcome === "pass" && res.exitCode === 0) {
667
- commandOutcome = "passed";
668
- } else if (envelopeCritique?.outcome === "fail") {
669
- commandOutcome = "failed";
670
- failures.push(`critique found ${envelopeCritique.findings?.length ?? 0} defect(s)`);
671
- contextOutcome = "failed";
672
- } else {
673
- // "skipped" (no provider / exhausted headless list) or anything
674
- // unrecognized: the review did not happen, so nothing is proven.
675
- commandOutcome = "unknown";
676
- const detail = envelopeCritique?.error ?? "critique reported no conclusive outcome";
677
- failures.push(detail);
678
- blockers.push({
679
- stage: "critique",
680
- context_id: ctx.id,
681
- reason: job.policy?.allow_metered_critique
682
- ? `critique did not complete: ${detail}`
683
- : `critique did not complete under the headless-only policy ` +
684
- `(${QA_RUN_HEADLESS_ONLY_ENV}=1; permit metered fallback with ` +
685
- `policy.allow_metered_critique / --allow-metered): ${detail}`,
686
- });
687
- if (contextOutcome !== "failed") contextOutcome = "unknown";
688
- }
689
- if (job.mode === "signoff") {
690
- const saved = (envelope.qaPlan as { snapshotSaved?: { path?: string } } | undefined)
691
- ?.snapshotSaved;
692
- if (saved?.path) savedSnapshots.set(ctx.id, saved.path);
693
- }
1199
+ blockers.push({ stage: "capture", context_id: ctx.id, reason });
694
1200
  }
695
- commands.push({
1201
+ } else {
1202
+ outcome = "unknown";
1203
+ const excerpt = execErrorExcerpt(res);
1204
+ const base = res.error
1205
+ ? res.error
1206
+ : !envelope
1207
+ ? `exit code ${res.exitCode ?? "null"}, missing JSON artifact ${jsonPath}`
1208
+ : !report?.context_id
1209
+ ? `exit code ${res.exitCode ?? "null"}, envelope carries no reviewPack record`
1210
+ : `exit code ${res.exitCode ?? "null"}`;
1211
+ const reason = excerpt ? `${base}: ${excerpt}` : base;
1212
+ failures.push(reason);
1213
+ blockers.push({
1214
+ stage: "capture",
696
1215
  context_id: ctx.id,
697
- check_id: "critique",
698
- argv,
699
- exit_code: res.exitCode,
700
- outcome: commandOutcome,
701
- failures,
702
- artifacts: gatherArtifacts(outPrefix),
703
- wall_time_ms: wallTimeMs,
1216
+ reason: `capture command did not complete: ${reason}`,
704
1217
  });
705
1218
  }
706
- critique.push({
1219
+ captureOutcomes[index] = {
707
1220
  context_id: ctx.id,
708
- provider,
709
- tiles_total: tilesTotal,
710
- tiles_reviewed: tilesReviewed,
711
- tiles_reused: tilesReused,
712
- outcome: contextOutcome,
713
- findings,
714
- });
715
- if (job.mode === "signoff" && contextOutcome === "passed" && !savedSnapshots.has(ctx.id)) {
1221
+ check_id: "review-pack",
1222
+ argv,
1223
+ exit_code: res.exitCode,
1224
+ outcome,
1225
+ failures,
1226
+ artifacts: gatherArtifacts(outPrefix),
1227
+ wall_time_ms: wallTimeMs,
1228
+ };
1229
+ captureRecords[index] = record;
1230
+ });
1231
+ commands.push(...captureOutcomes.filter((outcome) => outcome !== undefined));
1232
+ capturedRecords.push(
1233
+ ...captureRecords.filter((record): record is PageReviewContextRecord => Boolean(record)),
1234
+ );
1235
+ // The pack is readable from here on even if the judge never runs.
1236
+ if (capturedRecords.length > 0) finalizePack("capture", null);
1237
+ }
1238
+ wall.capture = Date.now() - captureStart;
1239
+ if (runsVisual) stagesRun.push("capture");
1240
+
1241
+ // -------------------------------------------------------------- critique
1242
+ // One pool of vision calls across every captured context, from disk. No
1243
+ // browser is open during this stage.
1244
+ enterStage("critique");
1245
+ const critiqueStart = Date.now();
1246
+ const judgedById = new Map<string, JudgedContext>();
1247
+ if (runsVisual && capturedRecords.length > 0 && !pastDeadline()) {
1248
+ const judged = await judgePageReviewPack({
1249
+ packDir,
1250
+ provider,
1251
+ rubric: DEFAULT_CRITIQUE_RUBRIC,
1252
+ ...(job.policy?.critique_pool !== undefined ? { concurrency: job.policy.critique_pool } : {}),
1253
+ contextIds: capturedRecords.map((record) => record.id),
1254
+ ...(manifest.baseline_source !== "none"
1255
+ ? {
1256
+ reuse: {
1257
+ target: job.target,
1258
+ ...(options.snapshotStore ? { store: options.snapshotStore } : {}),
1259
+ },
1260
+ }
1261
+ : {}),
1262
+ deadlineAt: startedAt + runDeadlineMs,
1263
+ onLog: log,
1264
+ });
1265
+ const latency = critiqueLatency(judged.provider_meta);
1266
+ critiquePool = {
1267
+ concurrency: judged.pool.concurrency,
1268
+ tiles_total: judged.tiles_total,
1269
+ tiles_reviewed: judged.tiles_reviewed,
1270
+ tiles_reused: judged.tiles_reused,
1271
+ wall_time_ms: judged.pool.wall_time_ms,
1272
+ provider: judged.pool.provider,
1273
+ ...(latency ? { latency_ms: latency } : {}),
1274
+ };
1275
+ for (const row of judged.contexts) {
1276
+ judgedById.set(row.context_id, row);
1277
+ let contextOutcome: QaRunCritiqueOutcome["outcome"] =
1278
+ row.outcome === "pass" ? "passed" : row.outcome === "fail" ? "failed" : "unknown";
1279
+ if (row.outcome === "skipped") {
1280
+ const detail = row.error ?? "critique reported no conclusive outcome";
716
1281
  blockers.push({
717
- stage: "snapshot",
718
- context_id: ctx.id,
719
- reason: "signoff critique passed but no QA snapshot was persisted for the context",
1282
+ stage: "critique",
1283
+ context_id: row.context_id,
1284
+ reason: job.policy?.allow_metered_critique
1285
+ ? `critique did not complete: ${detail}`
1286
+ : `critique did not complete under the headless-only policy ` +
1287
+ `(${QA_RUN_HEADLESS_ONLY_ENV}=1; permit metered fallback with ` +
1288
+ `policy.allow_metered_critique / --allow-metered): ${detail}`,
1289
+ });
1290
+ } else if (row.outcome === "incomplete") {
1291
+ blockers.push({
1292
+ stage: "critique",
1293
+ context_id: row.context_id,
1294
+ reason:
1295
+ `judge did not review ${row.tiles_unjudged} of ${row.tiles_total} tile(s) before the ` +
1296
+ "run deadline; nothing is proven for them",
1297
+ });
1298
+ }
1299
+ // A capped capture holds the top of the page and nothing below it.
1300
+ // Signoff cannot rest on that; review mode keeps the row honest via
1301
+ // `coverage` and leaves the verdict to the tiles that were seen.
1302
+ if (job.mode === "signoff" && row.coverage.capped && contextOutcome !== "failed") {
1303
+ contextOutcome = "unknown";
1304
+ blockers.push({
1305
+ stage: "critique",
1306
+ context_id: row.context_id,
1307
+ reason:
1308
+ `critique coverage capped: ${row.coverage.bands_reviewed} of ${row.coverage.bands_total} bands ` +
1309
+ `reviewed (${row.coverage.reviewed_height_px} of ${row.coverage.page_height_px} px); raise ` +
1310
+ "policy.critique_max_tiles to review the whole page in signoff mode",
720
1311
  });
721
1312
  }
1313
+ const scopeById = new Map(row.record.tiles.map((tile) => [tile.id, tile.scope]));
1314
+ critique.push({
1315
+ context_id: row.context_id,
1316
+ provider: row.provider,
1317
+ tiles_total: row.tiles_total,
1318
+ tiles_reviewed: row.tiles_reviewed,
1319
+ tiles_reused: row.tiles_reused,
1320
+ outcome: contextOutcome,
1321
+ findings: row.findings.map((finding) => {
1322
+ const selector = scopeById.get(finding.tile_id);
1323
+ return {
1324
+ severity: finding.severity,
1325
+ summary: `${finding.category}: ${finding.description}`,
1326
+ tile: `${row.context_id}/${finding.tile_id}`,
1327
+ ...(selector !== undefined ? { selector } : {}),
1328
+ };
1329
+ }),
1330
+ coverage: row.coverage,
1331
+ });
722
1332
  }
1333
+ finalizePack("critique", toCritiqueRecords(judged), judged.pool);
723
1334
  }
1335
+ if (runsVisual) restoreHeadlessOnly();
724
1336
  wall.critique = Date.now() - critiqueStart;
1337
+ if (runsVisual) stagesRun.push("critique");
725
1338
 
726
1339
  // -------------------------------------------------------------- snapshot
727
- // Critique invocations carry --qa-snapshot in signoff mode. When the
728
- // manifest requires no visual pass (visual === "none"), a passing signoff
729
- // still needs its baseline persisted, so a dedicated snapshot pass runs.
1340
+ // Signoff persists each context's baseline from the pack's own files (no
1341
+ // browser). The critique rides along only when the whole tile set was
1342
+ // freshly judged, uncapped, and unscoped — a partial or scoped review must
1343
+ // never become the next baseline's finding record. When the manifest
1344
+ // required no visual pass, a dedicated browse --qa-snapshot pass still runs.
1345
+ enterStage("snapshot");
730
1346
  const snapshotStart = Date.now();
731
- if (job.mode === "signoff" && visual === "none" && blockers.length === 0) {
1347
+ const savedSnapshots = new Map<string, string>();
1348
+ if (job.mode === "signoff" && runsVisual) {
1349
+ for (const record of capturedRecords) {
1350
+ if (pastDeadline()) break;
1351
+ const row = judgedById.get(record.id);
1352
+ const persistCritique =
1353
+ row !== undefined &&
1354
+ (row.outcome === "pass" || row.outcome === "fail") &&
1355
+ !record.coverage.capped &&
1356
+ row.tiles_reused === 0 &&
1357
+ row.tiles_unjudged === 0 &&
1358
+ record.scopes.length === 0;
1359
+ const persisted: PersistedCritique | undefined =
1360
+ persistCritique && row
1361
+ ? {
1362
+ contract_version: QA_CRITIQUE_CONTRACT_VERSION,
1363
+ rubric_digest: rubricDigest(DEFAULT_CRITIQUE_RUBRIC),
1364
+ outcome: row.outcome as "pass" | "fail",
1365
+ findings: row.findings.map(({ tile_id: _tileId, ...finding }) => finding),
1366
+ tiles: record.tiles.map((tile) => ({
1367
+ index: tile.index,
1368
+ label: tile.label,
1369
+ x: tile.x,
1370
+ scrollY: tile.scrollY,
1371
+ width: tile.width,
1372
+ height: tile.height,
1373
+ })),
1374
+ }
1375
+ : undefined;
1376
+ try {
1377
+ const saved = saveQaSnapshot(
1378
+ job.target,
1379
+ { viewport: record.viewport, theme: record.theme, state: record.state },
1380
+ {
1381
+ signature: readPackSignature(packDir, record),
1382
+ domHtml: readPackDom(packDir, record),
1383
+ screenshotPng: readPackFullPage(packDir, record),
1384
+ ...(persisted ? { critique: persisted } : {}),
1385
+ },
1386
+ options.snapshotStore ?? {},
1387
+ );
1388
+ savedSnapshots.set(record.id, saved.path);
1389
+ log(`snapshot ${record.id}: saved${persisted ? " with critique" : ""}`);
1390
+ } catch (err: unknown) {
1391
+ blockers.push({
1392
+ stage: "snapshot",
1393
+ context_id: record.id,
1394
+ reason: `snapshot could not be persisted from the pack: ${err instanceof Error ? err.message : String(err)}`,
1395
+ });
1396
+ }
1397
+ }
1398
+ for (const row of critique) {
1399
+ if (row.outcome === "passed" && !savedSnapshots.has(row.context_id)) {
1400
+ blockers.push({
1401
+ stage: "snapshot",
1402
+ context_id: row.context_id,
1403
+ reason: "signoff critique passed but no QA snapshot was persisted for the context",
1404
+ });
1405
+ }
1406
+ }
1407
+ } else if (job.mode === "signoff" && visual === "none" && blockers.length === 0) {
732
1408
  const stillClean = commands.every((command) => command.outcome === "passed");
733
1409
  if (stillClean) {
734
1410
  for (const ctx of contexts) {
1411
+ if (pastDeadline()) break;
735
1412
  const outPrefix = join(outDir, `${ctx.id}-snapshot`);
736
1413
  const argv = [
737
1414
  ...browseArgv,
@@ -773,6 +1450,7 @@ export async function runQaMatrix(options: QaRunMatrixOptions): Promise<QaRunRes
773
1450
  }
774
1451
  }
775
1452
  wall.snapshot = Date.now() - snapshotStart;
1453
+ if (job.mode === "signoff") stagesRun.push("snapshot");
776
1454
 
777
1455
  if (job.mode === "signoff") {
778
1456
  const allSaved = contexts.length > 0 && contexts.every((ctx) => savedSnapshots.has(ctx.id));