harnery 0.37.0 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commander.d.ts +9 -0
- package/dist/commander.d.ts.map +1 -1
- package/dist/commander.js +7 -2
- package/dist/commands/admission.d.ts +21 -0
- package/dist/commands/admission.d.ts.map +1 -0
- package/dist/commands/admission.js +565 -0
- package/dist/commands/agents.d.ts +7 -0
- package/dist/commands/agents.d.ts.map +1 -1
- package/dist/commands/agents.js +61 -1
- package/dist/commands/artifacts.d.ts.map +1 -1
- package/dist/commands/artifacts.js +48 -2
- package/dist/commands/browse-ai.d.ts +2 -2
- package/dist/commands/browse-ai.d.ts.map +1 -1
- package/dist/commands/browse-ai.js +6 -4
- package/dist/commands/browse.d.ts.map +1 -1
- package/dist/commands/browse.js +349 -21
- package/dist/commands/fetch.js +1 -0
- package/dist/commands/qa-record.d.ts +144 -0
- package/dist/commands/qa-record.d.ts.map +1 -0
- package/dist/commands/qa-record.js +0 -0
- package/dist/commands/qa-run.d.ts +7 -3
- package/dist/commands/qa-run.d.ts.map +1 -1
- package/dist/commands/qa-run.js +268 -13
- package/dist/commands/qa-status.d.ts +71 -0
- package/dist/commands/qa-status.d.ts.map +1 -0
- package/dist/commands/qa-status.js +490 -0
- package/dist/commands/qa-verify.d.ts +40 -0
- package/dist/commands/qa-verify.d.ts.map +1 -0
- package/dist/commands/qa-verify.js +180 -0
- package/dist/commands/review-pack.d.ts +4 -0
- package/dist/commands/review-pack.d.ts.map +1 -0
- package/dist/commands/review-pack.js +1001 -0
- package/dist/core/agents/qa-signal.d.ts +111 -0
- package/dist/core/agents/qa-signal.d.ts.map +1 -0
- package/dist/core/agents/qa-signal.js +231 -0
- package/dist/core/agents/session-name-display.d.ts +20 -5
- package/dist/core/agents/session-name-display.d.ts.map +1 -1
- package/dist/core/agents/session-name-display.js +67 -7
- package/dist/core/agents/state/heartbeat-reader.d.ts +7 -0
- package/dist/core/agents/state/heartbeat-reader.d.ts.map +1 -1
- package/dist/core/agents/state/heartbeat-writer.d.ts +11 -0
- package/dist/core/agents/state/heartbeat-writer.d.ts.map +1 -1
- package/dist/core/agents/state/heartbeat-writer.js +17 -0
- package/dist/core/agents/state/live-coordination-view.d.ts.map +1 -1
- package/dist/core/agents/state/live-coordination-view.js +1 -0
- package/dist/core/agents/state/live-coordination-writer.js +5 -0
- package/dist/core/artifacts/constants.d.ts +1 -1
- package/dist/core/artifacts/constants.js +1 -1
- package/dist/core/artifacts/index.d.ts +57 -6
- package/dist/core/artifacts/index.d.ts.map +1 -1
- package/dist/core/artifacts/index.js +265 -10
- package/dist/core/config.d.ts +8 -0
- package/dist/core/config.d.ts.map +1 -1
- package/dist/core/config.js +16 -0
- package/dist/core/diagnostics/bundle.d.ts +16 -0
- package/dist/core/diagnostics/bundle.d.ts.map +1 -1
- package/dist/core/diagnostics/bundle.js +101 -7
- package/dist/core/events/v3/bootstrap.d.ts.map +1 -1
- package/dist/core/events/v3/bootstrap.js +10 -0
- package/dist/core/events/v3/coordination-view.d.ts +3 -0
- package/dist/core/events/v3/coordination-view.d.ts.map +1 -1
- package/dist/core/events/v3/coordination-view.js +69 -8
- package/dist/core/events/v3/producers/intake.d.ts.map +1 -1
- package/dist/core/events/v3/producers/intake.js +9 -2
- package/dist/core/events/v3/producers/recorder.d.ts +23 -0
- package/dist/core/events/v3/producers/recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/recorder.js +264 -18
- package/dist/core/hooks/cli.js +120 -27
- package/dist/core/hooks/resolve/transcript.d.ts.map +1 -1
- package/dist/core/hooks/resolve/transcript.js +10 -3
- package/dist/core/hooks/session-name-presence.d.ts.map +1 -1
- package/dist/core/hooks/session-name-presence.js +4 -1
- package/dist/core/qa-artifacts.d.ts +20 -0
- package/dist/core/qa-artifacts.d.ts.map +1 -0
- package/dist/core/qa-artifacts.js +110 -0
- package/dist/core/resources/contract.d.ts +6 -0
- package/dist/core/resources/contract.d.ts.map +1 -1
- package/dist/core/resources/sampler.d.ts +7 -0
- package/dist/core/resources/sampler.d.ts.map +1 -1
- package/dist/core/resources/sampler.js +106 -5
- package/dist/lib/admission.d.ts +71 -0
- package/dist/lib/admission.d.ts.map +1 -0
- package/dist/lib/admission.js +264 -0
- package/dist/lib/agent-browser/client.d.ts +1 -1
- package/dist/lib/agent-browser/client.d.ts.map +1 -1
- package/dist/lib/agent-browser/client.js +1 -5
- package/dist/lib/browser/capture-fidelity.d.ts +39 -0
- package/dist/lib/browser/capture-fidelity.d.ts.map +1 -0
- package/dist/lib/browser/capture-fidelity.js +84 -0
- package/dist/lib/browser/client.d.ts +41 -1
- package/dist/lib/browser/client.d.ts.map +1 -1
- package/dist/lib/browser/client.js +167 -9
- package/dist/lib/browser/critique.d.ts +38 -1
- package/dist/lib/browser/critique.d.ts.map +1 -1
- package/dist/lib/browser/critique.js +34 -6
- package/dist/lib/browser/index.d.ts +4 -2
- package/dist/lib/browser/index.d.ts.map +1 -1
- package/dist/lib/browser/index.js +2 -0
- package/dist/lib/browser/page-review-judge.d.ts +64 -0
- package/dist/lib/browser/page-review-judge.d.ts.map +1 -0
- package/dist/lib/browser/page-review-judge.js +270 -0
- package/dist/lib/browser/page-review-pack.d.ts +613 -0
- package/dist/lib/browser/page-review-pack.d.ts.map +1 -0
- package/dist/lib/browser/page-review-pack.js +1751 -0
- package/dist/lib/browser/qa-run-contracts.d.ts +214 -10
- package/dist/lib/browser/qa-run-contracts.d.ts.map +1 -1
- package/dist/lib/browser/qa-run-contracts.js +136 -1
- package/dist/lib/browser/qa-run.d.ts +100 -10
- package/dist/lib/browser/qa-run.d.ts.map +1 -1
- package/dist/lib/browser/qa-run.js +768 -169
- package/dist/lib/browser/request-diagnostics.d.ts +13 -0
- package/dist/lib/browser/request-diagnostics.d.ts.map +1 -0
- package/dist/lib/browser/request-diagnostics.js +18 -0
- package/dist/lib/browser/tiling.d.ts +19 -0
- package/dist/lib/browser/tiling.d.ts.map +1 -1
- package/dist/lib/browser/tiling.js +28 -0
- package/dist/lib/cookies/client.d.ts +9 -0
- package/dist/lib/cookies/client.d.ts.map +1 -1
- package/dist/lib/cookies/client.js +197 -44
- package/dist/lib/cookies/extra.d.ts +18 -0
- package/dist/lib/cookies/extra.d.ts.map +1 -0
- package/dist/lib/cookies/extra.js +14 -0
- package/dist/lib/cookies/index.d.ts +2 -1
- package/dist/lib/cookies/index.d.ts.map +1 -1
- package/dist/lib/cookies/index.js +2 -1
- package/dist/lib/durable-job.d.ts +124 -0
- package/dist/lib/durable-job.d.ts.map +1 -0
- package/dist/lib/durable-job.js +296 -0
- package/dist/lib/http/client.d.ts +7 -1
- package/dist/lib/http/client.d.ts.map +1 -1
- package/dist/lib/http/client.js +2 -0
- package/dist/lib/instructions/templates.d.ts.map +1 -1
- package/dist/lib/instructions/templates.js +5 -2
- package/package.json +8 -2
- package/src/commander.ts +50 -2
- package/src/commands/admission.ts +699 -0
- package/src/commands/agents.ts +87 -1
- package/src/commands/artifacts.ts +97 -21
- package/src/commands/browse-ai.ts +10 -5
- package/src/commands/browse.ts +481 -20
- package/src/commands/fetch.ts +1 -0
- package/src/commands/qa-record.ts +682 -0
- package/src/commands/qa-run.ts +335 -16
- package/src/commands/qa-status.ts +608 -0
- package/src/commands/qa-verify.ts +238 -0
- package/src/commands/review-pack.ts +1281 -0
- package/src/core/agents/qa-signal.ts +261 -0
- package/src/core/agents/session-name-display.ts +78 -7
- package/src/core/agents/state/heartbeat-reader.ts +7 -0
- package/src/core/agents/state/heartbeat-writer.ts +23 -0
- package/src/core/agents/state/live-coordination-view.ts +1 -0
- package/src/core/agents/state/live-coordination-writer.ts +5 -0
- package/src/core/artifacts/constants.ts +1 -1
- package/src/core/artifacts/index.ts +370 -21
- package/src/core/config.ts +23 -0
- package/src/core/diagnostics/bundle.ts +119 -11
- package/src/core/events/v3/bootstrap.ts +10 -0
- package/src/core/events/v3/coordination-view.ts +100 -11
- package/src/core/events/v3/producers/intake.ts +9 -2
- package/src/core/events/v3/producers/recorder.ts +312 -18
- package/src/core/hooks/cli.ts +140 -32
- package/src/core/hooks/resolve/transcript.ts +10 -3
- package/src/core/hooks/session-name-presence.ts +6 -1
- package/src/core/qa-artifacts.ts +126 -0
- package/src/core/resources/contract.ts +7 -0
- package/src/core/resources/sampler.ts +138 -6
- package/src/lib/admission.ts +347 -0
- package/src/lib/agent-browser/client.ts +2 -10
- package/src/lib/browser/capture-fidelity.ts +98 -0
- package/src/lib/browser/client.ts +206 -10
- package/src/lib/browser/critique.ts +62 -7
- package/src/lib/browser/index.ts +36 -0
- package/src/lib/browser/page-review-judge.ts +360 -0
- package/src/lib/browser/page-review-pack.ts +2384 -0
- package/src/lib/browser/qa-run-contracts.ts +366 -3
- package/src/lib/browser/qa-run.ts +868 -190
- package/src/lib/browser/request-diagnostics.ts +27 -0
- package/src/lib/browser/tiling.ts +32 -0
- package/src/lib/cookies/client.ts +228 -42
- package/src/lib/cookies/extra.ts +28 -0
- package/src/lib/cookies/index.ts +2 -0
- package/src/lib/durable-job.ts +407 -0
- package/src/lib/http/client.ts +13 -1
- package/src/lib/instructions/templates.ts +5 -2
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import type { QaContext, QaManifest } from "./qa-plan.js";
|
|
2
2
|
export declare const QA_RUN_JOB_SCHEMA_VERSION: 1;
|
|
3
|
-
export declare const QA_RUN_RESULT_SCHEMA_VERSION:
|
|
3
|
+
export declare const QA_RUN_RESULT_SCHEMA_VERSION: 4;
|
|
4
|
+
/** How a result's evidence was produced. `runner` means the qa-run matrix
|
|
5
|
+
* executed the checks itself. `manual` means an operator or agent performed
|
|
6
|
+
* the checks by hand and recorded them (qa-record); such a result can report
|
|
7
|
+
* a defect but can never claim a pass, because nothing re-executable proved
|
|
8
|
+
* the absence of one. */
|
|
9
|
+
export type QaRunEvidenceSource = "runner" | "manual";
|
|
4
10
|
/** One rendering context the runner will capture and check. */
|
|
5
11
|
export interface QaRunContext {
|
|
6
12
|
/** Stable ID, unique within the job; also the artifact prefix component.
|
|
@@ -46,6 +52,26 @@ export interface QaRunPolicy {
|
|
|
46
52
|
command_concurrency?: number;
|
|
47
53
|
/** Per-command timeout in milliseconds (default 120000). */
|
|
48
54
|
command_timeout_ms?: number;
|
|
55
|
+
/** Overall runner deadline in milliseconds (default 900000). When the run
|
|
56
|
+
* exceeds it, remaining commands are skipped, a `deadline` blocker is
|
|
57
|
+
* recorded, and the result finalizes as incomplete — releasing the
|
|
58
|
+
* admission slot instead of holding it open-endedly. */
|
|
59
|
+
run_deadline_ms?: number;
|
|
60
|
+
/** Full-page critique bands per context (browse `--check-critique-max-tiles`,
|
|
61
|
+
* default 24). Raise it when a tall page must be reviewed end to end and the
|
|
62
|
+
* per-tile cost is accepted; each critique row's `coverage` records what the
|
|
63
|
+
* run actually saw, which is why this knob can stay out of the job digest. */
|
|
64
|
+
critique_max_tiles?: number;
|
|
65
|
+
/** Vision calls in flight during the judge stage, across every context of
|
|
66
|
+
* the run (1 to 16). Default: the host provider's own concurrency. The
|
|
67
|
+
* capture stage closes every browser before judging starts, so this knob
|
|
68
|
+
* costs model-call parallelism, never browser memory. */
|
|
69
|
+
critique_pool?: number;
|
|
70
|
+
/** Minutes the run's page review pack lives after the judge finishes before
|
|
71
|
+
* the whole pack directory is deleted (default 90; 1 to 43200). The result
|
|
72
|
+
* document keeps every finding inline, so a run stays reportable after its
|
|
73
|
+
* pack is gone. */
|
|
74
|
+
review_pack_retention_minutes?: number;
|
|
49
75
|
}
|
|
50
76
|
export interface QaRunJob {
|
|
51
77
|
schema_version: typeof QA_RUN_JOB_SCHEMA_VERSION;
|
|
@@ -86,6 +112,26 @@ export interface QaRunCommandOutcome {
|
|
|
86
112
|
};
|
|
87
113
|
wall_time_ms: number;
|
|
88
114
|
}
|
|
115
|
+
/** Vision-call latency one critique backend reported over the judge pool,
|
|
116
|
+
* in milliseconds over the tile calls it served. The percentiles are the
|
|
117
|
+
* provider's own sample across every context (one pool, one sample). */
|
|
118
|
+
export interface QaRunCritiqueLatency {
|
|
119
|
+
count: number;
|
|
120
|
+
p50: number;
|
|
121
|
+
p95: number;
|
|
122
|
+
}
|
|
123
|
+
/** What share of the page the critique tiles covered, lifted from the browse
|
|
124
|
+
* envelope. `capped` means the tiler dropped bands past its per-context
|
|
125
|
+
* maximum; in signoff mode that is a blocker, in review mode a flag. Across
|
|
126
|
+
* several scope commands the heights take the worst case and the band counts
|
|
127
|
+
* add. */
|
|
128
|
+
export interface QaRunCritiqueCoverage {
|
|
129
|
+
page_height_px: number;
|
|
130
|
+
reviewed_height_px: number;
|
|
131
|
+
bands_total: number;
|
|
132
|
+
bands_reviewed: number;
|
|
133
|
+
capped: boolean;
|
|
134
|
+
}
|
|
89
135
|
export interface QaRunCritiqueOutcome {
|
|
90
136
|
context_id: string;
|
|
91
137
|
provider: string;
|
|
@@ -97,16 +143,131 @@ export interface QaRunCritiqueOutcome {
|
|
|
97
143
|
severity: string;
|
|
98
144
|
summary: string;
|
|
99
145
|
selector?: string;
|
|
146
|
+
tile?: string;
|
|
100
147
|
}>;
|
|
148
|
+
/** Tile coverage of the page. Absent when the capture died or reported none. */
|
|
149
|
+
coverage?: QaRunCritiqueCoverage;
|
|
150
|
+
}
|
|
151
|
+
/** The judge stage as one unit: every context's tiles through one bounded
|
|
152
|
+
* pool of vision calls, with no browser open. `latency_ms` is the backends'
|
|
153
|
+
* own per-call sample over the whole pool, keyed by backend name. */
|
|
154
|
+
export interface QaRunCritiquePool {
|
|
155
|
+
concurrency: number;
|
|
156
|
+
tiles_total: number;
|
|
157
|
+
tiles_reviewed: number;
|
|
158
|
+
tiles_reused: number;
|
|
159
|
+
wall_time_ms: number;
|
|
160
|
+
provider: string;
|
|
161
|
+
latency_ms?: Record<string, QaRunCritiqueLatency>;
|
|
162
|
+
}
|
|
163
|
+
/** Where the run's page review pack lives: the on-disk evidence an agent can
|
|
164
|
+
* review without a browser (tiles, DOM, coverage, `review.md`, and the
|
|
165
|
+
* delegated-review `findings.json`). */
|
|
166
|
+
export interface QaRunReviewPack {
|
|
167
|
+
schema: string;
|
|
168
|
+
dir: string;
|
|
169
|
+
review: string;
|
|
170
|
+
findings: string;
|
|
171
|
+
/** When the pack directory is deleted (ISO). Absent when the run never
|
|
172
|
+
* reached the point of knowing (capture failed before any context landed). */
|
|
173
|
+
expires_at?: string;
|
|
174
|
+
/** Bytes on disk at finalize time. */
|
|
175
|
+
size_bytes?: number;
|
|
101
176
|
}
|
|
102
177
|
export interface QaRunBlocker {
|
|
103
|
-
stage: "validate" | "plan" | "gates" | "interactions" | "critique" | "snapshot" | "result";
|
|
178
|
+
stage: "validate" | "admission" | "plan" | "gates" | "interactions" | "capture" | "critique" | "snapshot" | "deadline" | "result";
|
|
104
179
|
context_id?: string;
|
|
105
180
|
reason: string;
|
|
106
181
|
}
|
|
107
182
|
export type QaRunVerdict = "passed" | "failed" | "incomplete";
|
|
183
|
+
/** Runner stages in execution order. `last_completed_stage` names the last
|
|
184
|
+
* one that finished without contributing a blocker. */
|
|
185
|
+
export declare const QA_RUN_STAGES: readonly ["plan", "gates", "interactions", "capture", "critique", "snapshot"];
|
|
186
|
+
export type QaRunStage = (typeof QA_RUN_STAGES)[number];
|
|
187
|
+
/** Identity of one runner invocation. This block is what makes a result
|
|
188
|
+
* verifiable evidence rather than a loose file: a consumer matches it against
|
|
189
|
+
* the invocation being reported (see assessQaRunEvidence) instead of trusting
|
|
190
|
+
* whatever sits in a reused directory. */
|
|
191
|
+
export interface QaRunIdentity {
|
|
192
|
+
/** Minted per invocation (crypto.randomUUID). */
|
|
193
|
+
run_id: string;
|
|
194
|
+
/** ISO-8601 UTC bounds of the invocation. */
|
|
195
|
+
started_at: string;
|
|
196
|
+
completed_at: string;
|
|
197
|
+
/** Git SHA or content identifier of what was tested, when resolvable. */
|
|
198
|
+
tested_revision?: string;
|
|
199
|
+
/** Where tested_revision came from: the job document, a git probe of the
|
|
200
|
+
* working directory, or nowhere (`unknown`, tested_revision absent). */
|
|
201
|
+
revision_source: "job" | "git" | "unknown";
|
|
202
|
+
/** `git status --porcelain` was non-empty when the run started — a revision
|
|
203
|
+
* alone does not prove content. Absent when no git probe ran. */
|
|
204
|
+
worktree_dirty?: boolean;
|
|
205
|
+
/** SHA-256 over the effective validated job (computeJobDigest). */
|
|
206
|
+
job_digest: string;
|
|
207
|
+
/** Absolute run directory the result was written into. A result found
|
|
208
|
+
* elsewhere has been moved or copied and fails evidence assessment. */
|
|
209
|
+
out_dir: string;
|
|
210
|
+
}
|
|
211
|
+
export declare const QA_RUN_STATUS_SCHEMA_VERSION: 1;
|
|
212
|
+
export type QaRunStatusState = "launching" | "queued" | "running" | "completed";
|
|
213
|
+
/** Live status document (`run-status.json`) beside the result in every run
|
|
214
|
+
* directory. Written at start, every stage boundary, and on a heartbeat
|
|
215
|
+
* timer, so a client that lost its terminal (e.g. a Windows-to-WSL bridge
|
|
216
|
+
* disconnect) can mechanically distinguish a running job from a dead one:
|
|
217
|
+
* non-terminal state + dead PID + no result document = dead. The result
|
|
218
|
+
* document stays authoritative once the run completes. */
|
|
219
|
+
export interface QaRunStatusDocument {
|
|
220
|
+
schema_version: typeof QA_RUN_STATUS_SCHEMA_VERSION;
|
|
221
|
+
run_id: string;
|
|
222
|
+
/** PID of the process executing the matrix (the detach parent records the
|
|
223
|
+
* child's PID in its initial `launching` write; the child overwrites). */
|
|
224
|
+
pid: number;
|
|
225
|
+
state: QaRunStatusState;
|
|
226
|
+
/** Stage currently executing; null before plan and after completion. */
|
|
227
|
+
stage: QaRunStage | null;
|
|
228
|
+
started_at: string;
|
|
229
|
+
/** Heartbeat. Stage boundaries and a periodic timer both refresh it. */
|
|
230
|
+
updated_at: string;
|
|
231
|
+
/** Present only while state is "queued". */
|
|
232
|
+
queue?: {
|
|
233
|
+
resource: string;
|
|
234
|
+
waiting_since: string;
|
|
235
|
+
};
|
|
236
|
+
/** Present only once state is "completed". */
|
|
237
|
+
verdict?: QaRunVerdict;
|
|
238
|
+
}
|
|
239
|
+
/** Host-pressure sample. Captured at start, at every stage boundary, and at
|
|
240
|
+
* finish, so an incomplete run carries the load context that produced it and
|
|
241
|
+
* names the other heavy jobs it was competing with. */
|
|
242
|
+
export interface QaRunHostSample {
|
|
243
|
+
captured_at: string;
|
|
244
|
+
loadavg_1m: number;
|
|
245
|
+
free_mem_bytes: number;
|
|
246
|
+
total_mem_bytes: number;
|
|
247
|
+
cpu_count: number;
|
|
248
|
+
/** Other holders of the admission resource at sample time. Present only
|
|
249
|
+
* when the run queued; an empty array means the run had the host to
|
|
250
|
+
* itself. This is what turns "it was slow" into "it was slow because these
|
|
251
|
+
* three jobs held slots". */
|
|
252
|
+
competing?: Array<{
|
|
253
|
+
label: string;
|
|
254
|
+
pid: number;
|
|
255
|
+
}>;
|
|
256
|
+
}
|
|
108
257
|
export interface QaRunResult {
|
|
109
258
|
schema_version: typeof QA_RUN_RESULT_SCHEMA_VERSION;
|
|
259
|
+
/** Runner-executed or hand-recorded. A manual result never reads passed. */
|
|
260
|
+
evidence_source: QaRunEvidenceSource;
|
|
261
|
+
run: QaRunIdentity;
|
|
262
|
+
host: {
|
|
263
|
+
start: QaRunHostSample;
|
|
264
|
+
finish: QaRunHostSample;
|
|
265
|
+
/** Sample taken as each stage began, so a stall is attributable to the
|
|
266
|
+
* stage that was running and the load at that moment. */
|
|
267
|
+
stages?: Partial<Record<QaRunStage, QaRunHostSample>>;
|
|
268
|
+
};
|
|
269
|
+
/** Null when the run never completed a stage cleanly (e.g. plan failed). */
|
|
270
|
+
last_completed_stage: QaRunStage | null;
|
|
110
271
|
target: string;
|
|
111
272
|
tested_revision?: string;
|
|
112
273
|
mode: "signoff" | "review";
|
|
@@ -116,6 +277,11 @@ export interface QaRunResult {
|
|
|
116
277
|
contexts: QaRunContext[];
|
|
117
278
|
commands: QaRunCommandOutcome[];
|
|
118
279
|
critique: QaRunCritiqueOutcome[];
|
|
280
|
+
/** Present once the judge stage ran (even when it skipped for lack of a
|
|
281
|
+
* provider); absent when the run stopped before it. */
|
|
282
|
+
critique_pool?: QaRunCritiquePool;
|
|
283
|
+
/** Present once the capture stage wrote at least one context. */
|
|
284
|
+
review_pack?: QaRunReviewPack;
|
|
119
285
|
snapshot: {
|
|
120
286
|
saved: boolean;
|
|
121
287
|
path?: string;
|
|
@@ -124,9 +290,17 @@ export interface QaRunResult {
|
|
|
124
290
|
plan: number;
|
|
125
291
|
gates: number;
|
|
126
292
|
interactions: number;
|
|
293
|
+
/** Browser time: rendering every context into the review pack. */
|
|
294
|
+
capture: number;
|
|
295
|
+
/** Judge time: vision calls over the pack, no browser open. */
|
|
127
296
|
critique: number;
|
|
128
297
|
snapshot: number;
|
|
298
|
+
/** Runner stages only — admission queue wait is deliberately excluded
|
|
299
|
+
* (see `queue`) so total stays pure runner time. */
|
|
129
300
|
total: number;
|
|
301
|
+
/** Milliseconds spent waiting for a machine-wide admission slot before
|
|
302
|
+
* any browser work. Absent when the run did not queue. */
|
|
303
|
+
queue?: number;
|
|
130
304
|
};
|
|
131
305
|
blockers: QaRunBlocker[];
|
|
132
306
|
verdict: QaRunVerdict;
|
|
@@ -153,21 +327,51 @@ export declare function contextIdFor(context: QaContext): string;
|
|
|
153
327
|
* returned list is what the runner executes and what the result reports.
|
|
154
328
|
*/
|
|
155
329
|
export declare function mergeCoverage(manifest: QaManifest, job: QaRunJob): QaRunContext[];
|
|
330
|
+
/** SHA-256 hex digest over the effective validated job (after the CLI merges
|
|
331
|
+
* its authoritative `target` and `mode`). `policy` is excluded: concurrency,
|
|
332
|
+
* timeout, and metered-critique knobs change how the run executes, not what
|
|
333
|
+
* it proves, and CLI flags mutate them after the job file is read — including
|
|
334
|
+
* them would make the same job file verify differently across invocations. */
|
|
335
|
+
export declare function computeJobDigest(job: QaRunJob): string;
|
|
336
|
+
export interface QaEvidenceExpectations {
|
|
337
|
+
/** Exact run ID the caller expects (from the invocation it just made). */
|
|
338
|
+
run_id?: string;
|
|
339
|
+
/** Revision the evidence must have tested. A result whose revision_source
|
|
340
|
+
* is `unknown` cannot satisfy a revision expectation — fail-closed. */
|
|
341
|
+
tested_revision?: string;
|
|
342
|
+
/** Digest of the effective job the evidence must have run. */
|
|
343
|
+
job_digest?: string;
|
|
344
|
+
/** ISO-8601 floor: a run started before this instant is stale. */
|
|
345
|
+
not_started_before?: string;
|
|
346
|
+
/** Maximum age of completed_at in milliseconds, evaluated against `now`. */
|
|
347
|
+
max_age_ms?: number;
|
|
348
|
+
/** Evaluation instant for max_age_ms (ISO-8601; default: current time). */
|
|
349
|
+
now?: string;
|
|
350
|
+
/** Directory the result file was read from. Compared against the recorded
|
|
351
|
+
* run.out_dir: a moved or copied result is not evidence for its new home. */
|
|
352
|
+
found_in_dir?: string;
|
|
353
|
+
}
|
|
354
|
+
export interface QaEvidenceAssessment {
|
|
355
|
+
fresh: boolean;
|
|
356
|
+
/** Empty when fresh; each entry names one independent staleness reason. */
|
|
357
|
+
reasons: string[];
|
|
358
|
+
run?: QaRunIdentity;
|
|
359
|
+
verdict?: QaRunVerdict;
|
|
360
|
+
}
|
|
156
361
|
/**
|
|
157
|
-
*
|
|
158
|
-
*
|
|
159
|
-
*
|
|
160
|
-
*
|
|
161
|
-
* - any failed or unknown command/critique outcome → failed when the check
|
|
162
|
-
* completed and found a defect, incomplete when the outcome is unknown;
|
|
163
|
-
* - signoff mode additionally requires the snapshot to have been saved.
|
|
164
|
-
* `passed` is only reachable when every input proves out.
|
|
362
|
+
* Assess whether a result document is fresh evidence for the invocation the
|
|
363
|
+
* caller has in mind. Fail-closed: a document without a verifiable identity
|
|
364
|
+
* block (schema v1 or foreign JSON) is stale by definition, and every
|
|
365
|
+
* expectation mismatch is reported, not just the first.
|
|
165
366
|
*/
|
|
367
|
+
export declare function assessQaRunEvidence(document: unknown, expectations?: QaEvidenceExpectations): QaEvidenceAssessment;
|
|
166
368
|
export declare function computeVerdict(input: {
|
|
167
369
|
mode: QaRunJob["mode"];
|
|
168
370
|
blockers: QaRunBlocker[];
|
|
169
371
|
commands: QaRunCommandOutcome[];
|
|
170
372
|
critique: QaRunCritiqueOutcome[];
|
|
171
373
|
snapshotSaved: boolean;
|
|
374
|
+
/** Defaults to `runner`. `manual` caps the verdict at incomplete. */
|
|
375
|
+
evidenceSource?: QaRunEvidenceSource;
|
|
172
376
|
}): QaRunVerdict;
|
|
173
377
|
//# sourceMappingURL=qa-run-contracts.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"qa-run-contracts.d.ts","sourceRoot":"","sources":["../../../src/lib/browser/qa-run-contracts.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"qa-run-contracts.d.ts","sourceRoot":"","sources":["../../../src/lib/browser/qa-run-contracts.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1D,eAAO,MAAM,yBAAyB,EAAG,CAAU,CAAC;AACpD,eAAO,MAAM,4BAA4B,EAAG,CAAU,CAAC;AAEvD;;;;yBAIyB;AACzB,MAAM,MAAM,mBAAmB,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEtD,+DAA+D;AAC/D,MAAM,WAAW,YAAY;IAC3B;4DACwD;IACxD,EAAE,EAAE,MAAM,CAAC;IACX,8DAA8D;IAC9D,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,OAAO,GAAG,MAAM,CAAC;IACxB,oDAAoD;IACpD,KAAK,EAAE,MAAM,CAAC;IACd;4EACwE;IACxE,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;4BAE4B;AAC5B,MAAM,WAAW,UAAU;IACzB,2EAA2E;IAC3E,EAAE,EAAE,MAAM,CAAC;IACX,sFAAsF;IACtF,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED;qEACqE;AACrE,MAAM,WAAW,gBAAgB;IAC/B,iFAAiF;IACjF,IAAI,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB;iFAC6E;IAC7E,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B;0EACsE;IACtE,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC;4CACwC;IACxC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,4DAA4D;IAC5D,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;4DAGwD;IACxD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;kFAG8E;IAC9E,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;6DAGyD;IACzD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;uBAGmB;IACnB,6BAA6B,CAAC,EAAE,MAAM,CAAC;CACxC;AAED,MAAM,WAAW,QAAQ;IACvB,cAAc,EAAE,OAAO,yBAAyB,CAAC;IACjD,uEAAuE;IACvE,MAAM,EAAE,MAAM,CAAC;IACf,6DAA6D;IAC7D,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,0EAA0E;IAC1E,IAAI,EAAE,SAAS,GAAG,QAAQ,CAAC;IAC3B,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,YAAY,EAAE,CAAC;IAC1B;mFAC+E;IAC/E,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC;IACtB,kBAAkB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACxC,mEAAmE;IACnE,QAAQ,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IACpD,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,8CAA8C;AAC9C,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB;8DAC0D;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,OAAO,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;IACzC,oEAAoE;IACpE,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,SAAS,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1D,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;wEAEwE;AACxE,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;UAIU;AACV,MAAM,WAAW,qBAAqB;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;IACzC,QAAQ,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACzF,gFAAgF;IAChF,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CAClC;AAED;;qEAEqE;AACrE,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;CACnD;AAED;;wCAEwC;AACxC,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB;kFAC8E;IAC9E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,sCAAsC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EACD,UAAU,GACV,WAAW,GACX,MAAM,GACN,OAAO,GACP,cAAc,GACd,SAAS,GACT,UAAU,GACV,UAAU,GACV,UAAU,GACV,QAAQ,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG,QAAQ,GAAG,YAAY,CAAC;AAE9D;uDACuD;AACvD,eAAO,MAAM,aAAa,+EAOhB,CAAC;AACX,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC;AAExD;;;0CAG0C;AAC1C,MAAM,WAAW,aAAa;IAC5B,iDAAiD;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,6CAA6C;IAC7C,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,yEAAyE;IACzE,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;4EACwE;IACxE,eAAe,EAAE,KAAK,GAAG,KAAK,GAAG,SAAS,CAAC;IAC3C;qEACiE;IACjE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,mEAAmE;IACnE,UAAU,EAAE,MAAM,CAAC;IACnB;2EACuE;IACvE,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,eAAO,MAAM,4BAA4B,EAAG,CAAU,CAAC;AAEvD,MAAM,MAAM,gBAAgB,GAAG,WAAW,GAAG,QAAQ,GAAG,SAAS,GAAG,WAAW,CAAC;AAEhF;;;;;0DAK0D;AAC1D,MAAM,WAAW,mBAAmB;IAClC,cAAc,EAAE,OAAO,4BAA4B,CAAC;IACpD,MAAM,EAAE,MAAM,CAAC;IACf;8EAC0E;IAC1E,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,gBAAgB,CAAC;IACxB,wEAAwE;IACxE,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,UAAU,EAAE,MAAM,CAAC;IACnB,4CAA4C;IAC5C,KAAK,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;IACpD,8CAA8C;IAC9C,OAAO,CAAC,EAAE,YAAY,CAAC;CACxB;AAED;;uDAEuD;AACvD,MAAM,WAAW,eAAe;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB;;;iCAG6B;IAC7B,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACnD;AAED,MAAM,WAAW,WAAW;IAC1B,cAAc,EAAE,OAAO,4BAA4B,CAAC;IACpD,4EAA4E;IAC5E,eAAe,EAAE,mBAAmB,CAAC;IACrC,GAAG,EAAE,aAAa,CAAC;IACnB,IAAI,EAAE;QACJ,KAAK,EAAE,eAAe,CAAC;QACvB,MAAM,EAAE,eAAe,CAAC;QACxB;iEACyD;QACzD,MAAM,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC,CAAC;KACvD,CAAC;IACF,4EAA4E;IAC5E,oBAAoB,EAAE,UAAU,GAAG,IAAI,CAAC;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,EAAE,SAAS,GAAG,QAAQ,CAAC;IAC3B,4EAA4E;IAC5E,OAAO,EAAE,UAAU,GAAG,IAAI,CAAC;IAC3B,mEAAmE;IACnE,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,QAAQ,EAAE,mBAAmB,EAAE,CAAC;IAChC,QAAQ,EAAE,oBAAoB,EAAE,CAAC;IACjC;2DACuD;IACvD,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAClC,iEAAiE;IACjE,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B,QAAQ,EAAE;QAAE,KAAK,EAAE,OAAO,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5C,YAAY,EAAE;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,EAAE,MAAM,CAAC;QACrB,kEAAkE;QAClE,OAAO,EAAE,MAAM,CAAC;QAChB,+DAA+D;QAC/D,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB;4DACoD;QACpD,KAAK,EAAE,MAAM,CAAC;QACd;kEAC0D;QAC1D,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,OAAO,EAAE,YAAY,CAAC;CACvB;AAeD,MAAM,MAAM,kBAAkB,GAAG;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,GAAG,EAAE,QAAQ,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC;AAqC/F;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,kBAAkB,CAoInE;AAMD,wBAAgB,YAAY,CAAC,OAAO,EAAE,SAAS,GAAG,MAAM,CAEvD;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE,QAAQ,GAAG,YAAY,EAAE,CAgBjF;AAkCD;;;;8EAI8E;AAC9E,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,QAAQ,GAAG,MAAM,CAKtD;AAED,MAAM,WAAW,sBAAsB;IACrC,0EAA0E;IAC1E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;2EACuE;IACvE,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,8DAA8D;IAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kEAAkE;IAClE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,4EAA4E;IAC5E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,2EAA2E;IAC3E,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;iFAC6E;IAC7E,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,OAAO,CAAC;IACf,2EAA2E;IAC3E,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,CAAC,EAAE,aAAa,CAAC;IACpB,OAAO,CAAC,EAAE,YAAY,CAAC;CACxB;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,OAAO,EACjB,YAAY,GAAE,sBAA2B,GACxC,oBAAoB,CAsEtB;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE;IACpC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IACvB,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,QAAQ,EAAE,mBAAmB,EAAE,CAAC;IAChC,QAAQ,EAAE,oBAAoB,EAAE,CAAC;IACjC,aAAa,EAAE,OAAO,CAAC;IACvB,qEAAqE;IACrE,cAAc,CAAC,EAAE,mBAAmB,CAAC;CACtC,GAAG,YAAY,CAcf"}
|
|
@@ -16,8 +16,20 @@
|
|
|
16
16
|
// secret-shaped fields before any browser or model work starts.
|
|
17
17
|
//
|
|
18
18
|
// Toolkit tier: this module must not import src/core (layering check).
|
|
19
|
+
import { createHash } from "node:crypto";
|
|
19
20
|
export const QA_RUN_JOB_SCHEMA_VERSION = 1;
|
|
20
|
-
export const QA_RUN_RESULT_SCHEMA_VERSION =
|
|
21
|
+
export const QA_RUN_RESULT_SCHEMA_VERSION = 4;
|
|
22
|
+
/** Runner stages in execution order. `last_completed_stage` names the last
|
|
23
|
+
* one that finished without contributing a blocker. */
|
|
24
|
+
export const QA_RUN_STAGES = [
|
|
25
|
+
"plan",
|
|
26
|
+
"gates",
|
|
27
|
+
"interactions",
|
|
28
|
+
"capture",
|
|
29
|
+
"critique",
|
|
30
|
+
"snapshot",
|
|
31
|
+
];
|
|
32
|
+
export const QA_RUN_STATUS_SCHEMA_VERSION = 1;
|
|
21
33
|
// ---------------------------------------------------------------------------
|
|
22
34
|
// Job validation
|
|
23
35
|
// ---------------------------------------------------------------------------
|
|
@@ -162,6 +174,30 @@ export function validateQaRunJob(value) {
|
|
|
162
174
|
errors.push("policy.command_timeout_ms must be an integer ≥ 1000");
|
|
163
175
|
}
|
|
164
176
|
}
|
|
177
|
+
if (p?.run_deadline_ms !== undefined) {
|
|
178
|
+
const n = p.run_deadline_ms;
|
|
179
|
+
if (typeof n !== "number" || !Number.isInteger(n) || n < 10_000) {
|
|
180
|
+
errors.push("policy.run_deadline_ms must be an integer ≥ 10000");
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
if (p?.critique_max_tiles !== undefined) {
|
|
184
|
+
const n = p.critique_max_tiles;
|
|
185
|
+
if (typeof n !== "number" || !Number.isInteger(n) || n < 1 || n > 200) {
|
|
186
|
+
errors.push("policy.critique_max_tiles must be an integer between 1 and 200");
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
if (p?.critique_pool !== undefined) {
|
|
190
|
+
const n = p.critique_pool;
|
|
191
|
+
if (typeof n !== "number" || !Number.isInteger(n) || n < 1 || n > 16) {
|
|
192
|
+
errors.push("policy.critique_pool must be an integer between 1 and 16");
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
if (p?.review_pack_retention_minutes !== undefined) {
|
|
196
|
+
const n = p.review_pack_retention_minutes;
|
|
197
|
+
if (typeof n !== "number" || !Number.isInteger(n) || n < 1 || n > 43_200) {
|
|
198
|
+
errors.push("policy.review_pack_retention_minutes must be an integer between 1 and 43200");
|
|
199
|
+
}
|
|
200
|
+
}
|
|
165
201
|
}
|
|
166
202
|
scanForSecrets(job, "job", errors);
|
|
167
203
|
if (errors.length > 0)
|
|
@@ -211,11 +247,110 @@ export function mergeCoverage(manifest, job) {
|
|
|
211
247
|
* - signoff mode additionally requires the snapshot to have been saved.
|
|
212
248
|
* `passed` is only reachable when every input proves out.
|
|
213
249
|
*/
|
|
250
|
+
// ---------------------------------------------------------------------------
|
|
251
|
+
// Run identity: job digest + evidence assessment
|
|
252
|
+
// ---------------------------------------------------------------------------
|
|
253
|
+
/** Recursively key-sort plain objects so the digest is stable under key
|
|
254
|
+
* order. Arrays keep their order — it is semantically meaningful (contexts,
|
|
255
|
+
* argv arrays). */
|
|
256
|
+
function canonicalize(value) {
|
|
257
|
+
if (Array.isArray(value))
|
|
258
|
+
return value.map(canonicalize);
|
|
259
|
+
if (value && typeof value === "object") {
|
|
260
|
+
const record = value;
|
|
261
|
+
const sorted = {};
|
|
262
|
+
for (const key of Object.keys(record).sort())
|
|
263
|
+
sorted[key] = canonicalize(record[key]);
|
|
264
|
+
return sorted;
|
|
265
|
+
}
|
|
266
|
+
return value;
|
|
267
|
+
}
|
|
268
|
+
/** SHA-256 hex digest over the effective validated job (after the CLI merges
|
|
269
|
+
* its authoritative `target` and `mode`). `policy` is excluded: concurrency,
|
|
270
|
+
* timeout, and metered-critique knobs change how the run executes, not what
|
|
271
|
+
* it proves, and CLI flags mutate them after the job file is read — including
|
|
272
|
+
* them would make the same job file verify differently across invocations. */
|
|
273
|
+
export function computeJobDigest(job) {
|
|
274
|
+
const { policy: _policy, ...identityBearing } = job;
|
|
275
|
+
return createHash("sha256")
|
|
276
|
+
.update(JSON.stringify(canonicalize(identityBearing)))
|
|
277
|
+
.digest("hex");
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* Assess whether a result document is fresh evidence for the invocation the
|
|
281
|
+
* caller has in mind. Fail-closed: a document without a verifiable identity
|
|
282
|
+
* block (schema v1 or foreign JSON) is stale by definition, and every
|
|
283
|
+
* expectation mismatch is reported, not just the first.
|
|
284
|
+
*/
|
|
285
|
+
export function assessQaRunEvidence(document, expectations = {}) {
|
|
286
|
+
const reasons = [];
|
|
287
|
+
if (!document || typeof document !== "object" || Array.isArray(document)) {
|
|
288
|
+
return { fresh: false, reasons: ["document is not a result object"] };
|
|
289
|
+
}
|
|
290
|
+
const result = document;
|
|
291
|
+
if (result.schema_version !== QA_RUN_RESULT_SCHEMA_VERSION) {
|
|
292
|
+
return {
|
|
293
|
+
fresh: false,
|
|
294
|
+
reasons: [
|
|
295
|
+
`schema_version is ${JSON.stringify(result.schema_version)}, not ` +
|
|
296
|
+
`${QA_RUN_RESULT_SCHEMA_VERSION} — pre-identity results carry nothing to verify`,
|
|
297
|
+
],
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
const run = result.run;
|
|
301
|
+
if (!run ||
|
|
302
|
+
typeof run.run_id !== "string" ||
|
|
303
|
+
typeof run.started_at !== "string" ||
|
|
304
|
+
typeof run.completed_at !== "string" ||
|
|
305
|
+
typeof run.job_digest !== "string") {
|
|
306
|
+
return { fresh: false, reasons: ["result carries no complete run-identity block"] };
|
|
307
|
+
}
|
|
308
|
+
if (expectations.run_id !== undefined && run.run_id !== expectations.run_id) {
|
|
309
|
+
reasons.push(`run_id ${run.run_id} is not the expected ${expectations.run_id}`);
|
|
310
|
+
}
|
|
311
|
+
if (expectations.job_digest !== undefined && run.job_digest !== expectations.job_digest) {
|
|
312
|
+
reasons.push("job_digest does not match the expected job definition");
|
|
313
|
+
}
|
|
314
|
+
if (expectations.tested_revision !== undefined) {
|
|
315
|
+
if (run.revision_source === "unknown" || run.tested_revision === undefined) {
|
|
316
|
+
reasons.push(`a revision expectation was given but the run recorded revision_source "unknown"`);
|
|
317
|
+
}
|
|
318
|
+
else if (run.tested_revision !== expectations.tested_revision) {
|
|
319
|
+
reasons.push(`tested_revision ${run.tested_revision} is not the expected ${expectations.tested_revision}`);
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
if (expectations.not_started_before !== undefined) {
|
|
323
|
+
if (Date.parse(run.started_at) < Date.parse(expectations.not_started_before)) {
|
|
324
|
+
reasons.push(`run started ${run.started_at}, before the freshness floor ${expectations.not_started_before}`);
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
if (expectations.max_age_ms !== undefined) {
|
|
328
|
+
const now = expectations.now !== undefined ? Date.parse(expectations.now) : Date.now();
|
|
329
|
+
const age = now - Date.parse(run.completed_at);
|
|
330
|
+
if (Number.isNaN(age) || age > expectations.max_age_ms) {
|
|
331
|
+
reasons.push(`run completed ${run.completed_at}, older than the ${expectations.max_age_ms}ms maximum age`);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
if (expectations.found_in_dir !== undefined && run.out_dir !== expectations.found_in_dir) {
|
|
335
|
+
reasons.push(`result was found in ${expectations.found_in_dir} but records out_dir ${run.out_dir} — ` +
|
|
336
|
+
"a moved or copied result is not evidence for its new location");
|
|
337
|
+
}
|
|
338
|
+
return {
|
|
339
|
+
fresh: reasons.length === 0,
|
|
340
|
+
reasons,
|
|
341
|
+
run: run,
|
|
342
|
+
...(result.verdict !== undefined ? { verdict: result.verdict } : {}),
|
|
343
|
+
};
|
|
344
|
+
}
|
|
214
345
|
export function computeVerdict(input) {
|
|
215
346
|
const failed = input.commands.some((command) => command.outcome === "failed") ||
|
|
216
347
|
input.critique.some((entry) => entry.outcome === "failed");
|
|
217
348
|
if (failed)
|
|
218
349
|
return "failed";
|
|
350
|
+
// Hand-recorded evidence can prove a defect but never its absence: nothing
|
|
351
|
+
// re-executable ran, so a manual result stops at incomplete by contract.
|
|
352
|
+
if (input.evidenceSource === "manual")
|
|
353
|
+
return "incomplete";
|
|
219
354
|
const unknown = input.commands.some((command) => command.outcome === "unknown") ||
|
|
220
355
|
input.critique.some((entry) => entry.outcome === "unknown");
|
|
221
356
|
if (input.blockers.length > 0 || unknown)
|
|
@@ -1,10 +1,36 @@
|
|
|
1
|
+
import { type CritiqueProvider } from "./critique.js";
|
|
1
2
|
import { type QaRunJob, type QaRunResult } from "./qa-run-contracts.js";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
*
|
|
3
|
+
import { type QaSnapshotStoreOptions } from "./qa-snapshot.js";
|
|
4
|
+
/** Set on the runner's own environment before the host's critique provider is
|
|
5
|
+
* loaded for the judge stage, unless the job permits metered critique: the
|
|
6
|
+
* provider must stay on subscription-backed headless harnesses and surface
|
|
7
|
+
* exhaustion instead of falling back to a metered API. */
|
|
5
8
|
export declare const QA_RUN_HEADLESS_ONLY_ENV = "HARNERY_CRITIQUE_HEADLESS_ONLY";
|
|
6
9
|
/** Result document written into the run's output directory. */
|
|
7
10
|
export declare const QA_RUN_RESULT_FILENAME = "page-qa-result.json";
|
|
11
|
+
/** Pointer document written into the parent output directory after every
|
|
12
|
+
* run, naming the newest run's directory and verdict. Consumers resolve the
|
|
13
|
+
* current result through this pointer instead of guessing at loose files. */
|
|
14
|
+
export declare const QA_RUN_LATEST_FILENAME = "latest.json";
|
|
15
|
+
export interface QaRunLatestPointerInput {
|
|
16
|
+
run_id: string;
|
|
17
|
+
dir: string;
|
|
18
|
+
completed_at: string;
|
|
19
|
+
verdict: QaRunResult["verdict"];
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Publish the parent directory's latest-result pointer without allowing an
|
|
23
|
+
* older completion to replace a newer one. Both runner and manual evidence
|
|
24
|
+
* use this writer so every producer preserves the same ordering invariant.
|
|
25
|
+
*/
|
|
26
|
+
export declare function writeLatestPointer(outParent: string, input: QaRunLatestPointerInput): string;
|
|
27
|
+
/** Live status document beside the result (QaRunStatusDocument): written at
|
|
28
|
+
* start, every stage boundary, and on a heartbeat timer, so a disconnected
|
|
29
|
+
* client can tell a running job from a dead one without guessing. */
|
|
30
|
+
export declare const QA_RUN_STATUS_FILENAME = "run-status.json";
|
|
31
|
+
/** The effective validated job, written into the run directory so a
|
|
32
|
+
* reconnecting client can re-derive the job digest (`qa-verify --job`). */
|
|
33
|
+
export declare const QA_RUN_JOB_FILENAME = "job.json";
|
|
8
34
|
export interface QaRunExecOptions {
|
|
9
35
|
/** Hard per-command timeout in milliseconds. */
|
|
10
36
|
timeoutMs: number;
|
|
@@ -22,14 +48,34 @@ export interface QaRunExecResult {
|
|
|
22
48
|
}
|
|
23
49
|
/** Injectable child-process executor. `argv[0]` is the executable. */
|
|
24
50
|
export type QaRunExec = (argv: string[], options: QaRunExecOptions) => Promise<QaRunExecResult>;
|
|
25
|
-
/**
|
|
26
|
-
*
|
|
51
|
+
/** Grace between the timeout's SIGTERM and the follow-up SIGKILL. A child
|
|
52
|
+
* that catches SIGTERM (Bun installs a handler by default) gets this long to
|
|
53
|
+
* exit before the kill is made non-negotiable. */
|
|
54
|
+
export declare const QA_RUN_KILL_GRACE_MS = 5000;
|
|
55
|
+
/** Default executor: spawn with argv arrays only (never a shell string),
|
|
56
|
+
* closed stdin, bounded output buffers, and the policy timeout.
|
|
57
|
+
*
|
|
58
|
+
* Timeout enforcement is escalated and group-wide, via `spawn` rather than
|
|
59
|
+
* `execFile` for two live-verified reasons. First, a child that catches
|
|
60
|
+
* SIGTERM while awaiting its own grandchildren turns a single polite kill
|
|
61
|
+
* into an unbounded wait (a critique command outlived its 120s cap by 10x).
|
|
62
|
+
* Second, `execFile` resolves only when the child's stdio closes, and an
|
|
63
|
+
* orphaned grandchild inheriting the pipe keeps it open after the child is
|
|
64
|
+
* dead — so even a delivered kill did not settle the call. The child is
|
|
65
|
+
* therefore spawned detached into its own process group; at the deadline the
|
|
66
|
+
* whole group gets SIGTERM, then SIGKILL after a grace, and the result
|
|
67
|
+
* settles on exit with whatever output drained, never waiting on a pipe an
|
|
68
|
+
* orphan still holds. A timed-out command reports an error even if the child
|
|
69
|
+
* then exits 0 — a result produced after the deadline cannot be trusted. */
|
|
27
70
|
export declare const defaultQaRunExec: QaRunExec;
|
|
28
71
|
export interface QaRunMatrixOptions {
|
|
29
72
|
/** A validated job (see validateQaRunJob — the runner trusts its shape). */
|
|
30
73
|
job: QaRunJob;
|
|
31
|
-
/**
|
|
32
|
-
|
|
74
|
+
/** PARENT directory for run output. Every invocation creates its own
|
|
75
|
+
* `run-<run_id>/` beneath it for artifacts and the result document, and
|
|
76
|
+
* maintains `latest.json` in the parent — a reused parent can therefore
|
|
77
|
+
* never present an older run's result as the current one. */
|
|
78
|
+
outParent: string;
|
|
33
79
|
/** argv prefix that reaches the host CLI's browse command, e.g.
|
|
34
80
|
* `[process.execPath, cliScript, "browse"]`. */
|
|
35
81
|
browseArgv: string[];
|
|
@@ -39,13 +85,57 @@ export interface QaRunMatrixOptions {
|
|
|
39
85
|
childEnv?: NodeJS.ProcessEnv;
|
|
40
86
|
/** Progress callback for human-facing per-stage lines. */
|
|
41
87
|
onLog?: (message: string) => void;
|
|
88
|
+
/** Run ID override (tests). Default: crypto.randomUUID(). */
|
|
89
|
+
runId?: string;
|
|
90
|
+
/** Working-tree revision probe supplied by the caller (the CLI probes git
|
|
91
|
+
* once). Ignored when the job itself pins tested_revision. */
|
|
92
|
+
revisionProbe?: {
|
|
93
|
+
tested_revision?: string;
|
|
94
|
+
worktree_dirty?: boolean;
|
|
95
|
+
};
|
|
96
|
+
/** Machine-wide admission gate. When present the runner acquires a slot
|
|
97
|
+
* before any browser work, records the wait as wall_time_ms.queue (never
|
|
98
|
+
* part of total), and finalizes an incomplete result with an "admission"
|
|
99
|
+
* blocker when acquisition fails — the evidence trail survives a full
|
|
100
|
+
* queue. The returned function releases the slot; the runner calls it at
|
|
101
|
+
* finalize, and a crashed runner's slot is reclaimed by dead-PID pruning. */
|
|
102
|
+
admission?: {
|
|
103
|
+
resource: string;
|
|
104
|
+
acquire: (onWait: (message: string) => void) => Promise<() => void>;
|
|
105
|
+
/** Snapshot of the other holders of the resource, sampled alongside host
|
|
106
|
+
* pressure so an incomplete run names what it was competing with. */
|
|
107
|
+
holders?: () => Array<{
|
|
108
|
+
label: string;
|
|
109
|
+
pid: number;
|
|
110
|
+
}>;
|
|
111
|
+
};
|
|
112
|
+
/** Host-injected vision call for the judge stage. The judge runs in this
|
|
113
|
+
* process over the pack on disk, after every capture browser has closed. */
|
|
114
|
+
critiqueProvider?: CritiqueProvider;
|
|
115
|
+
/** Lazy alternative to `critiqueProvider`; called once, after the
|
|
116
|
+
* headless-only environment has been applied. */
|
|
117
|
+
critiqueProviderLoader?: () => Promise<CritiqueProvider | undefined>;
|
|
118
|
+
/** QA snapshot store override (tests, host-managed cache locations). */
|
|
119
|
+
snapshotStore?: QaSnapshotStoreOptions;
|
|
120
|
+
/** Renders the command a reader can run to judge the pack later; shown in
|
|
121
|
+
* the pack's review.md when the judge did not run. */
|
|
122
|
+
reviewPackJudgeCommand?: (packDir: string) => string;
|
|
123
|
+
/** True when the run writes into the managed artifact store: the pack's
|
|
124
|
+
* manifest then says `managed: true` and the expiry sweep may delete it
|
|
125
|
+
* once `policy.review_pack_retention_minutes` (default 90) have passed
|
|
126
|
+
* since the judge finished. A pack under an explicit out dir stays
|
|
127
|
+
* unmanaged and is never deleted automatically. */
|
|
128
|
+
reviewPackManaged?: boolean;
|
|
42
129
|
}
|
|
43
130
|
/**
|
|
44
131
|
* Execute the whole QA matrix for one validated job and return the result
|
|
45
132
|
* (also written to `<outDir>/page-qa-result.json`). Stages: plan →
|
|
46
|
-
* deterministic gates (bounded pool) → interactions (serial) →
|
|
47
|
-
* context
|
|
48
|
-
*
|
|
133
|
+
* deterministic gates (bounded pool) → interactions (serial) → capture (each
|
|
134
|
+
* context rendered once into the run's page review pack through the same
|
|
135
|
+
* pool, browser closed) → critique (one in-process pool of vision calls over
|
|
136
|
+
* every tile of every context, no browser open) → snapshot (persisted from
|
|
137
|
+
* the pack's files in signoff). The verdict is computeVerdict over
|
|
138
|
+
* everything recorded — fail-closed.
|
|
49
139
|
*/
|
|
50
140
|
export declare function runQaMatrix(options: QaRunMatrixOptions): Promise<QaRunResult>;
|
|
51
141
|
//# sourceMappingURL=qa-run.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"qa-run.d.ts","sourceRoot":"","sources":["../../../src/lib/browser/qa-run.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"qa-run.d.ts","sourceRoot":"","sources":["../../../src/lib/browser/qa-run.ts"],"names":[],"mappings":"AAkBA,OAAO,EAAE,KAAK,gBAAgB,EAA2B,MAAM,eAAe,CAAC;AAqB/E,OAAO,EAaL,KAAK,QAAQ,EACb,KAAK,WAAW,EAKjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,KAAK,sBAAsB,EAAkB,MAAM,kBAAkB,CAAC;AAE/E;;;0DAG0D;AAC1D,eAAO,MAAM,wBAAwB,mCAAmC,CAAC;AAEzE,+DAA+D;AAC/D,eAAO,MAAM,sBAAsB,wBAAwB,CAAC;AAE5D;;6EAE6E;AAC7E,eAAO,MAAM,sBAAsB,gBAAgB,CAAC;AAEpD,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,WAAW,CAAC,SAAS,CAAC,CAAC;CACjC;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,uBAAuB,GAAG,MAAM,CA4B5F;AAED;;qEAEqE;AACrE,eAAO,MAAM,sBAAsB,oBAAoB,CAAC;AAExD;2EAC2E;AAC3E,eAAO,MAAM,mBAAmB,aAAa,CAAC;AAsB9C,MAAM,WAAW,gBAAgB;IAC/B,gDAAgD;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,kEAAkE;IAClE,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC;CACxB;AAED,MAAM,WAAW,eAAe;IAC9B,yEAAyE;IACzE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf;8CAC0C;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,sEAAsE;AACtE,MAAM,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC,eAAe,CAAC,CAAC;AAEhG;;kDAEkD;AAClD,eAAO,MAAM,oBAAoB,OAAQ,CAAC;AAO1C;;;;;;;;;;;;;;4EAc4E;AAC5E,eAAO,MAAM,gBAAgB,EAAE,SAiH3B,CAAC;AAEL,MAAM,WAAW,kBAAkB;IACjC,4EAA4E;IAC5E,GAAG,EAAE,QAAQ,CAAC;IACd;;;iEAG6D;IAC7D,SAAS,EAAE,MAAM,CAAC;IAClB;oDACgD;IAChD,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,2EAA2E;IAC3E,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,kEAAkE;IAClE,QAAQ,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IAC7B,0DAA0D;IAC1D,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,6DAA6D;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;kEAC8D;IAC9D,aAAa,CAAC,EAAE;QAAE,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IACvE;;;;;iFAK6E;IAC7E,SAAS,CAAC,EAAE;QACV,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,KAAK,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QACpE;6EACqE;QACrE,OAAO,CAAC,EAAE,MAAM,KAAK,CAAC;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,GAAG,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACvD,CAAC;IACF;gFAC4E;IAC5E,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC;qDACiD;IACjD,sBAAsB,CAAC,EAAE,MAAM,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAAC;IACrE,wEAAwE;IACxE,aAAa,CAAC,EAAE,sBAAsB,CAAC;IACvC;0DACsD;IACtD,sBAAsB,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC;IACrD;;;;uDAImD;IACnD,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAiKD;;;;;;;;;GASG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,WAAW,CAAC,CAo6BnF"}
|