agent-dealer 1.3.1 → 1.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle/server/dist/adapters/agent-health.test.js +52 -0
- package/bundle/server/dist/adapters/git-worktree.js +38 -1
- package/bundle/server/dist/adapters/github.js +50 -0
- package/bundle/server/dist/adapters/github.test.js +89 -1
- package/bundle/server/dist/capacity/service.js +66 -6
- package/bundle/server/dist/capacity/service.test.js +220 -0
- package/bundle/server/dist/coordinator/auto-merge.js +46 -9
- package/bundle/server/dist/coordinator/commands.js +89 -9
- package/bundle/server/dist/coordinator/commands.test.js +228 -2
- package/bundle/server/dist/coordinator/developer-effect.js +36 -0
- package/bundle/server/dist/coordinator/operator-verification.integration.test.js +102 -1
- package/bundle/server/dist/coordinator/product-scope-contract.test.js +249 -0
- package/bundle/server/dist/coordinator/prompts.js +57 -2
- package/bundle/server/dist/coordinator/prompts.test.js +236 -2
- package/bundle/server/dist/coordinator/reviewer-browser-probe.js +89 -10
- package/bundle/server/dist/coordinator/reviewer-browser-probe.test.js +100 -0
- package/bundle/server/dist/coordinator/reviewer-effect.js +30 -1
- package/bundle/server/dist/coordinator/reviewer-effect.test.js +54 -0
- package/bundle/server/dist/coordinator/reviewer-visual-evidence.js +594 -0
- package/bundle/server/dist/coordinator/reviewer-visual-evidence.test.js +496 -0
- package/bundle/server/dist/coordinator/source-reload.js +44 -9
- package/bundle/server/dist/coordinator/visual-qa-collection.test.js +619 -0
- package/bundle/server/dist/coordinator/visual-qa.js +630 -0
- package/bundle/server/dist/coordinator/visual-qa.test.js +493 -0
- package/bundle/server/dist/db/index.js +8 -0
- package/bundle/server/dist/db/schema.sql +5 -1
- package/bundle/server/dist/paths.js +24 -0
- package/bundle/server/dist/repository/issues.js +8 -1
- package/bundle/server/dist/repository/issues.test.js +34 -0
- package/bundle/server/dist/repository/runtime-availability.js +29 -11
- package/bundle/server/dist/routes/issues-execution-contract.test.js +293 -0
- package/bundle/server/dist/routes/issues-source-attachments.test.js +277 -0
- package/bundle/server/dist/routes/issues.js +23 -8
- package/bundle/server/dist/routes/issues.test.js +80 -0
- package/bundle/server/dist/runners/usage-cap.js +66 -15
- package/bundle/server/dist/runners/usage-cap.test.js +96 -2
- package/bundle/server/package.json +2 -2
- package/bundle/server/static-ui/assets/index-B_pNUDRY.js +64 -0
- package/bundle/server/static-ui/index.html +1 -1
- package/bundle/shared/dist/runtime-capacity.d.ts +85 -0
- package/bundle/shared/dist/runtime-capacity.js +19 -0
- package/bundle/shared/package.json +1 -1
- package/package.json +1 -1
- package/bundle/server/static-ui/assets/index-Bz_c-UbO.js +0 -64
|
@@ -0,0 +1,619 @@
|
|
|
1
|
+
// packages/server/src/coordinator/visual-qa-collection.test.ts
|
|
2
|
+
//
|
|
3
|
+
// NOT-381: coordinator-side visual-QA collection — artifact persistence, safe
|
|
4
|
+
// screenshot storage, transient-dir removal, rejection evidence, and the
|
|
5
|
+
// developer-to-reviewer handoff of the validated receipt.
|
|
6
|
+
//
|
|
7
|
+
// Direct `collectDeveloperVisualQa` tests use synthetic issue/session ids (the
|
|
8
|
+
// artifacts table carries no FKs); the closing integration test drives a real
|
|
9
|
+
// git repository + bare remote end to end through the coordinator.
|
|
10
|
+
import { test, before, beforeEach, after } from "node:test";
|
|
11
|
+
import assert from "node:assert/strict";
|
|
12
|
+
import { execFileSync } from "node:child_process";
|
|
13
|
+
import { randomUUID } from "node:crypto";
|
|
14
|
+
import fs from "node:fs";
|
|
15
|
+
import os from "node:os";
|
|
16
|
+
import path from "node:path";
|
|
17
|
+
process.env.AGENT_DEALER_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-visqa-home-"));
|
|
18
|
+
process.env.MAX_COORDINATOR_CONCURRENCY = "2";
|
|
19
|
+
process.env.COORDINATOR_HEARTBEAT_MS = "20";
|
|
20
|
+
process.env.COORDINATOR_FAIL_BACKOFF_MS = "0";
|
|
21
|
+
process.env.CHECKS_POLL_TIMEOUT_MS = "60";
|
|
22
|
+
process.env.CHECKS_POLL_INTERVAL_MS = "10";
|
|
23
|
+
process.env.REVIEWER_TIMEOUT_MS = "5000";
|
|
24
|
+
process.env.REVIEWER_PUBLISH_WAIT_ATTEMPTS = "3";
|
|
25
|
+
process.env.REVIEWER_PUBLISH_WAIT_INTERVAL_MS = "10";
|
|
26
|
+
const { migrate, getDb } = await import("../db/index.js");
|
|
27
|
+
const { createAgent } = await import("../repository/agents.js");
|
|
28
|
+
const { createIssue, getIssue } = await import("../repository/issues.js");
|
|
29
|
+
const { createWorkerSession } = await import("../repository/worker-sessions.js");
|
|
30
|
+
const { listArtifactsForIssue } = await import("../repository/artifacts-for-issue.js");
|
|
31
|
+
const { latestIssueArtifact } = await import("../repository/artifacts.js");
|
|
32
|
+
const { startWorkflow } = await import("./commands.js");
|
|
33
|
+
const { registerEffectHandler, resetEffectHandlers } = await import("./effect-registry.js");
|
|
34
|
+
const { runCoordinatorTick, drainCoordinator } = await import("./worker-loop.js");
|
|
35
|
+
const { runDeveloperEffect } = await import("./developer-effect.js");
|
|
36
|
+
const { runReviewerEffect } = await import("./reviewer-effect.js");
|
|
37
|
+
const { realDeveloperSpawn, realReviewerSpawn } = await import("./spawn.js");
|
|
38
|
+
const { realGithubAdapter } = await import("../adapters/github.js");
|
|
39
|
+
const { VISUAL_QA_DIR_NAME, VISUAL_QA_RECEIPT_KIND, VISUAL_QA_REJECTED_KIND, collectDeveloperVisualQa, readLatestVisualQa, } = await import("./visual-qa.js");
|
|
40
|
+
before(() => migrate());
|
|
41
|
+
beforeEach(() => getDb().exec("DELETE FROM review_publications; DELETE FROM work_items"));
|
|
42
|
+
after(() => resetEffectHandlers());
|
|
43
|
+
const HEAD_A = "a".repeat(40);
|
|
44
|
+
const HEAD_B = "b".repeat(40);
|
|
45
|
+
const TEST_DECK_ID = "00000000-0000-4000-a000-000000000099";
|
|
46
|
+
function writeWorktree(files) {
|
|
47
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-visqa-wt-"));
|
|
48
|
+
for (const [rel, content] of Object.entries(files)) {
|
|
49
|
+
const dest = path.join(dir, rel);
|
|
50
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
51
|
+
fs.writeFileSync(dest, content);
|
|
52
|
+
}
|
|
53
|
+
return dir;
|
|
54
|
+
}
|
|
55
|
+
function verifiedConclusion(head) {
|
|
56
|
+
return [
|
|
57
|
+
`Implemented the widget.`,
|
|
58
|
+
``,
|
|
59
|
+
`Visual QA: verified`,
|
|
60
|
+
`head: ${head}`,
|
|
61
|
+
`app: real (no mocks)`,
|
|
62
|
+
`scenario: /widgets/new — fill the form and submit`,
|
|
63
|
+
`viewports: 1440x900, 390x800`,
|
|
64
|
+
`commands: npm run dev; capture via chromium headless screenshot`,
|
|
65
|
+
`screenshots: ${VISUAL_QA_DIR_NAME}/desktop-1440x900.png, ${VISUAL_QA_DIR_NAME}/mobile-390x800.png`,
|
|
66
|
+
``,
|
|
67
|
+
].join("\n");
|
|
68
|
+
}
|
|
69
|
+
// Artifacts FK-reference issues + worker_sessions, so direct collection tests
|
|
70
|
+
// build real rows (synthetic UUIDs violate the constraint).
|
|
71
|
+
function freshIds() {
|
|
72
|
+
const dev = createAgent({ name: `dev-${randomUUID()}`, runtime: "claude_code", deckId: TEST_DECK_ID });
|
|
73
|
+
const rev = createAgent({ name: `rev-${randomUUID()}`, runtime: "claude_code", deckId: TEST_DECK_ID });
|
|
74
|
+
const issueId = createIssue({
|
|
75
|
+
title: "t",
|
|
76
|
+
description: "d",
|
|
77
|
+
acceptanceCriteria: "a",
|
|
78
|
+
repo: "github.com/acme/app",
|
|
79
|
+
baseBranch: "main",
|
|
80
|
+
developerAgentId: dev.id,
|
|
81
|
+
reviewerAgentId: rev.id,
|
|
82
|
+
maxReviewRounds: 1,
|
|
83
|
+
maxInfraAttempts: 1,
|
|
84
|
+
source: "manual",
|
|
85
|
+
}).id;
|
|
86
|
+
const sessionId = createWorkerSession({
|
|
87
|
+
issueId,
|
|
88
|
+
role: "developer",
|
|
89
|
+
round: 1,
|
|
90
|
+
agentId: dev.id,
|
|
91
|
+
runtime: "claude_code",
|
|
92
|
+
}).id;
|
|
93
|
+
return { issueId, sessionId };
|
|
94
|
+
}
|
|
95
|
+
function receiptContent(issueId) {
|
|
96
|
+
const art = latestIssueArtifact(issueId, VISUAL_QA_RECEIPT_KIND);
|
|
97
|
+
assert.ok(art?.contentJson, "expected a visual_qa_receipt artifact");
|
|
98
|
+
return JSON.parse(art.contentJson);
|
|
99
|
+
}
|
|
100
|
+
function rejectedContent(issueId) {
|
|
101
|
+
const art = latestIssueArtifact(issueId, VISUAL_QA_REJECTED_KIND);
|
|
102
|
+
assert.ok(art?.contentJson, "expected a visual_qa_rejected artifact");
|
|
103
|
+
return JSON.parse(art.contentJson);
|
|
104
|
+
}
|
|
105
|
+
// Direct collection: acceptance.
|
|
106
|
+
test("NOT-381: verified receipt copies screenshots to issue storage, persists blob refs, removes the worktree dir", () => {
|
|
107
|
+
const { issueId, sessionId } = freshIds();
|
|
108
|
+
const dir = writeWorktree({
|
|
109
|
+
[`${VISUAL_QA_DIR_NAME}/desktop-1440x900.png`]: "desktop-bytes",
|
|
110
|
+
[`${VISUAL_QA_DIR_NAME}/mobile-390x800.png`]: "mobile-bytes",
|
|
111
|
+
"feature.txt": "implemented",
|
|
112
|
+
});
|
|
113
|
+
try {
|
|
114
|
+
const result = collectDeveloperVisualQa({
|
|
115
|
+
issueId,
|
|
116
|
+
sessionId,
|
|
117
|
+
worktreePath: dir,
|
|
118
|
+
conclusion: verifiedConclusion(HEAD_A),
|
|
119
|
+
expectedHeadSha: HEAD_A,
|
|
120
|
+
});
|
|
121
|
+
assert.equal(result.kind, "receipt");
|
|
122
|
+
assert.equal(result.status, "verified");
|
|
123
|
+
const content = receiptContent(issueId);
|
|
124
|
+
assert.equal(content.status, "verified");
|
|
125
|
+
assert.equal(content.headSha, HEAD_A);
|
|
126
|
+
assert.equal(content.screenshots.length, 2);
|
|
127
|
+
for (const ref of content.screenshots) {
|
|
128
|
+
assert.ok(fs.existsSync(ref.blobPath), `blob must exist: ${ref.blobPath}`);
|
|
129
|
+
assert.ok(ref.blobPath.includes(issueId), "blob is namespaced to the issue");
|
|
130
|
+
assert.ok(ref.sizeBytes > 0);
|
|
131
|
+
assert.match(ref.sha256, /^[0-9a-f]{64}$/);
|
|
132
|
+
}
|
|
133
|
+
assert.equal(fs.readFileSync(content.screenshots[0].blobPath, "utf8"), "desktop-bytes");
|
|
134
|
+
assert.equal(fs.readFileSync(content.screenshots[1].blobPath, "utf8"), "mobile-bytes");
|
|
135
|
+
// The transient directory is gone; the rest of the worktree is untouched.
|
|
136
|
+
assert.equal(fs.existsSync(path.join(dir, VISUAL_QA_DIR_NAME)), false);
|
|
137
|
+
assert.equal(fs.readFileSync(path.join(dir, "feature.txt"), "utf8"), "implemented");
|
|
138
|
+
// The reviewer read path surfaces the same receipt.
|
|
139
|
+
const latest = readLatestVisualQa(issueId);
|
|
140
|
+
assert.ok(latest && latest.kind === "receipt");
|
|
141
|
+
assert.equal(latest.content.headSha, HEAD_A);
|
|
142
|
+
assert.equal(latestIssueArtifact(issueId, VISUAL_QA_REJECTED_KIND), null);
|
|
143
|
+
}
|
|
144
|
+
finally {
|
|
145
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
146
|
+
}
|
|
147
|
+
});
|
|
148
|
+
test("NOT-381: no Visual QA line collects nothing and persists nothing", () => {
|
|
149
|
+
const { issueId, sessionId } = freshIds();
|
|
150
|
+
const dir = writeWorktree({});
|
|
151
|
+
try {
|
|
152
|
+
const result = collectDeveloperVisualQa({
|
|
153
|
+
issueId,
|
|
154
|
+
sessionId,
|
|
155
|
+
worktreePath: dir,
|
|
156
|
+
conclusion: "Implemented the widget, no screenshots.",
|
|
157
|
+
expectedHeadSha: HEAD_A,
|
|
158
|
+
});
|
|
159
|
+
assert.deepEqual(result, { kind: "none" });
|
|
160
|
+
assert.equal(listArtifactsForIssue(issueId).length, 0);
|
|
161
|
+
assert.equal(readLatestVisualQa(issueId), null);
|
|
162
|
+
}
|
|
163
|
+
finally {
|
|
164
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
165
|
+
}
|
|
166
|
+
});
|
|
167
|
+
// Direct collection: every rejection fails loudly.
|
|
168
|
+
test("NOT-381: SHA mismatch rejects loudly and keeps the files for inspection", () => {
|
|
169
|
+
const { issueId, sessionId } = freshIds();
|
|
170
|
+
const dir = writeWorktree({ [`${VISUAL_QA_DIR_NAME}/desktop-1440x900.png`]: "x" });
|
|
171
|
+
try {
|
|
172
|
+
const result = collectDeveloperVisualQa({
|
|
173
|
+
issueId,
|
|
174
|
+
sessionId,
|
|
175
|
+
worktreePath: dir,
|
|
176
|
+
conclusion: verifiedConclusion(HEAD_B),
|
|
177
|
+
expectedHeadSha: HEAD_A,
|
|
178
|
+
});
|
|
179
|
+
assert.equal(result.kind, "rejected");
|
|
180
|
+
assert.match(result.reason, /bound to bbbbbbbb but the developer handoff is aaaaaaaa/);
|
|
181
|
+
const rejected = rejectedContent(issueId);
|
|
182
|
+
assert.equal(rejected.status, "verified");
|
|
183
|
+
assert.match(rejected.reason, /bound to bbbbbbbb but the developer handoff is aaaaaaaa/);
|
|
184
|
+
assert.equal(rejected.expectedHeadSha, HEAD_A);
|
|
185
|
+
assert.equal(latestIssueArtifact(issueId, VISUAL_QA_RECEIPT_KIND), null);
|
|
186
|
+
// Rejected files stay put for inspection (the dir is git-excluded upstream).
|
|
187
|
+
assert.equal(fs.existsSync(path.join(dir, VISUAL_QA_DIR_NAME, "desktop-1440x900.png")), true);
|
|
188
|
+
const latest = readLatestVisualQa(issueId);
|
|
189
|
+
assert.ok(latest && latest.kind === "rejected");
|
|
190
|
+
}
|
|
191
|
+
finally {
|
|
192
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
test("NOT-381: missing screenshot file rejects loudly", () => {
|
|
196
|
+
const { issueId, sessionId } = freshIds();
|
|
197
|
+
const dir = writeWorktree({ [`${VISUAL_QA_DIR_NAME}/desktop-1440x900.png`]: "x" });
|
|
198
|
+
try {
|
|
199
|
+
const result = collectDeveloperVisualQa({
|
|
200
|
+
issueId,
|
|
201
|
+
sessionId,
|
|
202
|
+
worktreePath: dir,
|
|
203
|
+
conclusion: verifiedConclusion(HEAD_A),
|
|
204
|
+
expectedHeadSha: HEAD_A,
|
|
205
|
+
});
|
|
206
|
+
assert.equal(result.kind, "rejected");
|
|
207
|
+
assert.match(result.reason, /screenshot is missing/);
|
|
208
|
+
assert.match(result.reason, /mobile-390x800\.png/);
|
|
209
|
+
assert.equal(latestIssueArtifact(issueId, VISUAL_QA_RECEIPT_KIND), null);
|
|
210
|
+
}
|
|
211
|
+
finally {
|
|
212
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
213
|
+
}
|
|
214
|
+
});
|
|
215
|
+
test("NOT-381: screenshot symlink rejects loudly", () => {
|
|
216
|
+
const { issueId, sessionId } = freshIds();
|
|
217
|
+
const dir = writeWorktree({
|
|
218
|
+
[`${VISUAL_QA_DIR_NAME}/desktop-1440x900.png`]: "x",
|
|
219
|
+
"real.png": "x",
|
|
220
|
+
});
|
|
221
|
+
fs.symlinkSync(path.join(dir, "real.png"), path.join(dir, VISUAL_QA_DIR_NAME, "mobile-390x800.png"));
|
|
222
|
+
try {
|
|
223
|
+
const result = collectDeveloperVisualQa({
|
|
224
|
+
issueId,
|
|
225
|
+
sessionId,
|
|
226
|
+
worktreePath: dir,
|
|
227
|
+
conclusion: verifiedConclusion(HEAD_A),
|
|
228
|
+
expectedHeadSha: HEAD_A,
|
|
229
|
+
});
|
|
230
|
+
assert.equal(result.kind, "rejected");
|
|
231
|
+
assert.match(result.reason, /is a symlink, refusing/);
|
|
232
|
+
assert.equal(latestIssueArtifact(issueId, VISUAL_QA_RECEIPT_KIND), null);
|
|
233
|
+
}
|
|
234
|
+
finally {
|
|
235
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
236
|
+
}
|
|
237
|
+
});
|
|
238
|
+
test("NOT-381: symlinked artifact directory rejects loudly and stores no blobs", () => {
|
|
239
|
+
const { issueId, sessionId } = freshIds();
|
|
240
|
+
const dir = writeWorktree({ "keep.txt": "x" });
|
|
241
|
+
const outside = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-visqa-out-"));
|
|
242
|
+
try {
|
|
243
|
+
fs.writeFileSync(path.join(outside, "desktop-1440x900.png"), "outside-bytes");
|
|
244
|
+
fs.writeFileSync(path.join(outside, "mobile-390x800.png"), "outside-bytes");
|
|
245
|
+
fs.symlinkSync(outside, path.join(dir, VISUAL_QA_DIR_NAME));
|
|
246
|
+
const result = collectDeveloperVisualQa({
|
|
247
|
+
issueId,
|
|
248
|
+
sessionId,
|
|
249
|
+
worktreePath: dir,
|
|
250
|
+
conclusion: verifiedConclusion(HEAD_A),
|
|
251
|
+
expectedHeadSha: HEAD_A,
|
|
252
|
+
});
|
|
253
|
+
assert.equal(result.kind, "rejected");
|
|
254
|
+
assert.match(result.reason, /visual-artifact directory is a symlink, refusing/);
|
|
255
|
+
assert.equal(latestIssueArtifact(issueId, VISUAL_QA_RECEIPT_KIND), null);
|
|
256
|
+
}
|
|
257
|
+
finally {
|
|
258
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
259
|
+
fs.rmSync(outside, { recursive: true, force: true });
|
|
260
|
+
}
|
|
261
|
+
});
|
|
262
|
+
test("NOT-381: symlinked subdirectory rejects loudly and stores no blobs", () => {
|
|
263
|
+
const { issueId, sessionId } = freshIds();
|
|
264
|
+
const dir = writeWorktree({ [`${VISUAL_QA_DIR_NAME}/desktop-1440x900.png`]: "x" });
|
|
265
|
+
const outside = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-visqa-out-"));
|
|
266
|
+
try {
|
|
267
|
+
fs.writeFileSync(path.join(outside, "mobile-390x800.png"), "outside-bytes");
|
|
268
|
+
fs.symlinkSync(outside, path.join(dir, VISUAL_QA_DIR_NAME, "sub"));
|
|
269
|
+
const conclusion = verifiedConclusion(HEAD_A).replace(`${VISUAL_QA_DIR_NAME}/mobile-390x800.png`, `${VISUAL_QA_DIR_NAME}/sub/mobile-390x800.png`);
|
|
270
|
+
const result = collectDeveloperVisualQa({
|
|
271
|
+
issueId,
|
|
272
|
+
sessionId,
|
|
273
|
+
worktreePath: dir,
|
|
274
|
+
conclusion,
|
|
275
|
+
expectedHeadSha: HEAD_A,
|
|
276
|
+
});
|
|
277
|
+
assert.equal(result.kind, "rejected");
|
|
278
|
+
assert.match(result.reason, /traverses a symlink, refusing/);
|
|
279
|
+
assert.equal(latestIssueArtifact(issueId, VISUAL_QA_RECEIPT_KIND), null);
|
|
280
|
+
}
|
|
281
|
+
finally {
|
|
282
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
283
|
+
fs.rmSync(outside, { recursive: true, force: true });
|
|
284
|
+
}
|
|
285
|
+
});
|
|
286
|
+
test("NOT-381: screenshot path traversal rejects loudly", () => {
|
|
287
|
+
const { issueId, sessionId } = freshIds();
|
|
288
|
+
const dir = writeWorktree({ "evil.png": "x", [`${VISUAL_QA_DIR_NAME}/desktop-1440x900.png`]: "x" });
|
|
289
|
+
try {
|
|
290
|
+
const conclusion = verifiedConclusion(HEAD_A).replace(`${VISUAL_QA_DIR_NAME}/mobile-390x800.png`, "../evil.png");
|
|
291
|
+
const result = collectDeveloperVisualQa({
|
|
292
|
+
issueId,
|
|
293
|
+
sessionId,
|
|
294
|
+
worktreePath: dir,
|
|
295
|
+
conclusion,
|
|
296
|
+
expectedHeadSha: HEAD_A,
|
|
297
|
+
});
|
|
298
|
+
assert.equal(result.kind, "rejected");
|
|
299
|
+
assert.match(result.reason, /escapes the visual-artifact directory/);
|
|
300
|
+
assert.equal(latestIssueArtifact(issueId, VISUAL_QA_RECEIPT_KIND), null);
|
|
301
|
+
}
|
|
302
|
+
finally {
|
|
303
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
304
|
+
}
|
|
305
|
+
});
|
|
306
|
+
test("NOT-381: missing required verified fields reject loudly", () => {
|
|
307
|
+
const { issueId, sessionId } = freshIds();
|
|
308
|
+
const dir = writeWorktree({});
|
|
309
|
+
try {
|
|
310
|
+
const result = collectDeveloperVisualQa({
|
|
311
|
+
issueId,
|
|
312
|
+
sessionId,
|
|
313
|
+
worktreePath: dir,
|
|
314
|
+
conclusion: `Visual QA: verified\nhead: ${HEAD_A}\napp: real\n`,
|
|
315
|
+
expectedHeadSha: HEAD_A,
|
|
316
|
+
});
|
|
317
|
+
assert.equal(result.kind, "rejected");
|
|
318
|
+
assert.match(result.reason, /scenario\/path/);
|
|
319
|
+
assert.equal(latestIssueArtifact(issueId, VISUAL_QA_RECEIPT_KIND), null);
|
|
320
|
+
}
|
|
321
|
+
finally {
|
|
322
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
323
|
+
}
|
|
324
|
+
});
|
|
325
|
+
test("NOT-381: unknown status rejects loudly", () => {
|
|
326
|
+
const { issueId, sessionId } = freshIds();
|
|
327
|
+
const dir = writeWorktree({});
|
|
328
|
+
try {
|
|
329
|
+
const result = collectDeveloperVisualQa({
|
|
330
|
+
issueId,
|
|
331
|
+
sessionId,
|
|
332
|
+
worktreePath: dir,
|
|
333
|
+
conclusion: "Done.\n\nVisual QA: done\n",
|
|
334
|
+
expectedHeadSha: HEAD_A,
|
|
335
|
+
});
|
|
336
|
+
assert.equal(result.kind, "rejected");
|
|
337
|
+
assert.match(result.reason, /unknown Visual QA status/);
|
|
338
|
+
assert.equal(latestIssueArtifact(issueId, VISUAL_QA_RECEIPT_KIND), null);
|
|
339
|
+
}
|
|
340
|
+
finally {
|
|
341
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
342
|
+
}
|
|
343
|
+
});
|
|
344
|
+
// Direct collection: unavailable / not_required states.
|
|
345
|
+
test("NOT-381: unavailable persists the concrete failed capability with no screenshot artifacts", () => {
|
|
346
|
+
const { issueId, sessionId } = freshIds();
|
|
347
|
+
const dir = writeWorktree({});
|
|
348
|
+
try {
|
|
349
|
+
const result = collectDeveloperVisualQa({
|
|
350
|
+
issueId,
|
|
351
|
+
sessionId,
|
|
352
|
+
worktreePath: dir,
|
|
353
|
+
conclusion: `Tried the probe.\n\nVisual QA: unavailable\nhead: ${HEAD_A}\ncapability: browser launch: chromium binary not found\n`,
|
|
354
|
+
expectedHeadSha: HEAD_A,
|
|
355
|
+
});
|
|
356
|
+
assert.equal(result.kind, "receipt");
|
|
357
|
+
assert.equal(result.status, "unavailable");
|
|
358
|
+
const content = receiptContent(issueId);
|
|
359
|
+
assert.equal(content.status, "unavailable");
|
|
360
|
+
assert.equal(content.capability, "browser launch: chromium binary not found");
|
|
361
|
+
assert.deepEqual(content.screenshots, []);
|
|
362
|
+
assert.equal(latestIssueArtifact(issueId, VISUAL_QA_REJECTED_KIND), null);
|
|
363
|
+
}
|
|
364
|
+
finally {
|
|
365
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
366
|
+
}
|
|
367
|
+
});
|
|
368
|
+
test("NOT-381: unavailable without a capability rejects loudly (never normalized into a pass)", () => {
|
|
369
|
+
const { issueId, sessionId } = freshIds();
|
|
370
|
+
const dir = writeWorktree({});
|
|
371
|
+
try {
|
|
372
|
+
const result = collectDeveloperVisualQa({
|
|
373
|
+
issueId,
|
|
374
|
+
sessionId,
|
|
375
|
+
worktreePath: dir,
|
|
376
|
+
conclusion: "Visual QA: not run\n",
|
|
377
|
+
expectedHeadSha: HEAD_A,
|
|
378
|
+
});
|
|
379
|
+
assert.equal(result.kind, "rejected");
|
|
380
|
+
assert.match(result.reason, /names no failed capability/);
|
|
381
|
+
assert.equal(latestIssueArtifact(issueId, VISUAL_QA_RECEIPT_KIND), null);
|
|
382
|
+
}
|
|
383
|
+
finally {
|
|
384
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
385
|
+
}
|
|
386
|
+
});
|
|
387
|
+
test("NOT-381: not_required persists with no screenshot artifacts even when the worker names files", () => {
|
|
388
|
+
const { issueId, sessionId } = freshIds();
|
|
389
|
+
const dir = writeWorktree({ [`${VISUAL_QA_DIR_NAME}/stray.png`]: "x" });
|
|
390
|
+
try {
|
|
391
|
+
const result = collectDeveloperVisualQa({
|
|
392
|
+
issueId,
|
|
393
|
+
sessionId,
|
|
394
|
+
worktreePath: dir,
|
|
395
|
+
conclusion: `No UI changed.\n\nVisual QA: not_required\nscreenshots: ${VISUAL_QA_DIR_NAME}/stray.png\n`,
|
|
396
|
+
expectedHeadSha: HEAD_A,
|
|
397
|
+
});
|
|
398
|
+
assert.equal(result.kind, "receipt");
|
|
399
|
+
assert.equal(result.status, "not_required");
|
|
400
|
+
const content = receiptContent(issueId);
|
|
401
|
+
assert.deepEqual(content.screenshots, []);
|
|
402
|
+
assert.equal(latestIssueArtifact(issueId, VISUAL_QA_REJECTED_KIND), null);
|
|
403
|
+
}
|
|
404
|
+
finally {
|
|
405
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
406
|
+
}
|
|
407
|
+
});
|
|
408
|
+
test("NOT-381: readLatestVisualQa prefers the newest record across kinds (tie prefers the rejection)", () => {
|
|
409
|
+
const { issueId, sessionId } = freshIds();
|
|
410
|
+
const dir = writeWorktree({});
|
|
411
|
+
try {
|
|
412
|
+
// Receipt first, then a newer rejection for a later attempt.
|
|
413
|
+
collectDeveloperVisualQa({
|
|
414
|
+
issueId,
|
|
415
|
+
sessionId,
|
|
416
|
+
worktreePath: dir,
|
|
417
|
+
conclusion: "Visual QA: not_required\n",
|
|
418
|
+
expectedHeadSha: HEAD_A,
|
|
419
|
+
});
|
|
420
|
+
const first = readLatestVisualQa(issueId);
|
|
421
|
+
assert.ok(first && first.kind === "receipt");
|
|
422
|
+
collectDeveloperVisualQa({
|
|
423
|
+
issueId,
|
|
424
|
+
sessionId,
|
|
425
|
+
worktreePath: dir,
|
|
426
|
+
conclusion: `Visual QA: verified\nhead: ${HEAD_B}\napp: real\nscenario: x\nviewports: 1440x900\ncommands: y\nscreenshots: ${VISUAL_QA_DIR_NAME}/a.png\n`,
|
|
427
|
+
expectedHeadSha: HEAD_A,
|
|
428
|
+
});
|
|
429
|
+
const second = readLatestVisualQa(issueId);
|
|
430
|
+
assert.ok(second && second.kind === "rejected");
|
|
431
|
+
}
|
|
432
|
+
finally {
|
|
433
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
434
|
+
}
|
|
435
|
+
});
|
|
436
|
+
// Developer-to-reviewer integration: the exact branch's real head, desktop +
|
|
437
|
+
// mobile screenshots, a validated receipt bound to the published PR head, a
|
|
438
|
+
// clean worktree (no committed screenshots), and the receipt in the reviewer
|
|
439
|
+
// input.
|
|
440
|
+
let repo;
|
|
441
|
+
let remote;
|
|
442
|
+
function git(cwd, ...args) {
|
|
443
|
+
return execFileSync("git", args, { cwd, encoding: "utf8" }).trim();
|
|
444
|
+
}
|
|
445
|
+
before(() => {
|
|
446
|
+
repo = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-visqa-repo-"));
|
|
447
|
+
git(repo, "init", "-q", "-b", "main");
|
|
448
|
+
git(repo, "config", "user.email", "test@example.com");
|
|
449
|
+
git(repo, "config", "user.name", "Test");
|
|
450
|
+
fs.writeFileSync(path.join(repo, "README.md"), "hello\n");
|
|
451
|
+
git(repo, "add", ".");
|
|
452
|
+
git(repo, "commit", "-q", "-m", "init");
|
|
453
|
+
remote = fs.mkdtempSync(path.join(os.tmpdir(), "dealer-visqa-remote-"));
|
|
454
|
+
execFileSync("git", ["init", "-q", "--bare", "-b", "main", remote]);
|
|
455
|
+
git(repo, "remote", "add", "origin", remote);
|
|
456
|
+
git(repo, "push", "-q", "origin", "main");
|
|
457
|
+
});
|
|
458
|
+
after(() => {
|
|
459
|
+
fs.rmSync(repo, { recursive: true, force: true });
|
|
460
|
+
fs.rmSync(remote, { recursive: true, force: true });
|
|
461
|
+
});
|
|
462
|
+
const okDeckCallTool = async (name, _args) => ({
|
|
463
|
+
content: [
|
|
464
|
+
{
|
|
465
|
+
type: "text",
|
|
466
|
+
text: JSON.stringify({ id: TEST_DECK_ID, name: "test-deck" }),
|
|
467
|
+
},
|
|
468
|
+
],
|
|
469
|
+
});
|
|
470
|
+
async function makeIssue() {
|
|
471
|
+
const dev = createAgent({ name: `dev-${Math.random()}`, runtime: "claude_code", deckId: TEST_DECK_ID });
|
|
472
|
+
const rev = createAgent({ name: `rev-${Math.random()}`, runtime: "claude_code", deckId: TEST_DECK_ID });
|
|
473
|
+
return createIssue({
|
|
474
|
+
title: "Add widget",
|
|
475
|
+
description: "Build the widget.",
|
|
476
|
+
acceptanceCriteria: "Widget renders.",
|
|
477
|
+
repo,
|
|
478
|
+
baseBranch: "main",
|
|
479
|
+
developerAgentId: dev.id,
|
|
480
|
+
reviewerAgentId: rev.id,
|
|
481
|
+
maxReviewRounds: 3,
|
|
482
|
+
maxInfraAttempts: 3,
|
|
483
|
+
source: "manual",
|
|
484
|
+
}).id;
|
|
485
|
+
}
|
|
486
|
+
async function pump(max = 20) {
|
|
487
|
+
for (let i = 0; i < max; i++) {
|
|
488
|
+
const started = await runCoordinatorTick({ leaseOwner: "pump" });
|
|
489
|
+
await drainCoordinator();
|
|
490
|
+
if (started === 0)
|
|
491
|
+
return;
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
function fakeGithub() {
|
|
495
|
+
const prsByBranch = new Map();
|
|
496
|
+
const branchByNumber = new Map();
|
|
497
|
+
let nextNumber = 100;
|
|
498
|
+
const remoteHead = (branch) => git(remote, "rev-parse", branch);
|
|
499
|
+
const adapter = {
|
|
500
|
+
async viewPr({ branch, number }) {
|
|
501
|
+
const resolved = branch ?? (number != null ? branchByNumber.get(number) : undefined);
|
|
502
|
+
if (!resolved)
|
|
503
|
+
return null;
|
|
504
|
+
const pr = prsByBranch.get(resolved);
|
|
505
|
+
if (!pr)
|
|
506
|
+
return null;
|
|
507
|
+
if (number != null && number !== pr.number)
|
|
508
|
+
return null;
|
|
509
|
+
return { number: pr.number, url: pr.url, baseRefName: pr.base, headRefName: resolved, headRefOid: remoteHead(resolved), isDraft: true };
|
|
510
|
+
},
|
|
511
|
+
async createDraftPr({ base, head }) {
|
|
512
|
+
if (!head)
|
|
513
|
+
throw new Error("fakeGithub.createDraftPr requires an explicit --head");
|
|
514
|
+
const number = nextNumber++;
|
|
515
|
+
const url = `https://github.com/o/r/pull/${number}`;
|
|
516
|
+
prsByBranch.set(head, { number, url, base });
|
|
517
|
+
branchByNumber.set(number, head);
|
|
518
|
+
return { ok: true, number, url };
|
|
519
|
+
},
|
|
520
|
+
async checksSnapshot() {
|
|
521
|
+
return "success";
|
|
522
|
+
},
|
|
523
|
+
async publishReview() {
|
|
524
|
+
return { ok: true, event: "APPROVE", usedCommentFallback: false };
|
|
525
|
+
},
|
|
526
|
+
};
|
|
527
|
+
return adapter;
|
|
528
|
+
}
|
|
529
|
+
/** The worker implements, captures desktop + mobile at the final HEAD, and
|
|
530
|
+
* closes with a structured receipt naming that exact SHA. */
|
|
531
|
+
const visualSpawn = async (input) => {
|
|
532
|
+
fs.writeFileSync(path.join(input.cwd, "feature.txt"), "implemented\n");
|
|
533
|
+
git(input.cwd, "add", ".");
|
|
534
|
+
git(input.cwd, "-c", "user.email=agent@test", "-c", "user.name=Agent", "commit", "-q", "-m", "implement");
|
|
535
|
+
const head = git(input.cwd, "rev-parse", "HEAD");
|
|
536
|
+
const visualDir = path.join(input.cwd, VISUAL_QA_DIR_NAME);
|
|
537
|
+
fs.mkdirSync(visualDir, { recursive: true });
|
|
538
|
+
fs.writeFileSync(path.join(visualDir, "desktop-1440x900.png"), "desktop-bytes");
|
|
539
|
+
fs.writeFileSync(path.join(visualDir, "mobile-390x800.png"), "mobile-bytes");
|
|
540
|
+
const transcript = [
|
|
541
|
+
`Implemented the widget with real-app visual verification.`,
|
|
542
|
+
``,
|
|
543
|
+
`Visual QA: verified`,
|
|
544
|
+
`head: ${head}`,
|
|
545
|
+
`app: real (no mocks)`,
|
|
546
|
+
`scenario: /widgets/new — fill the form and submit`,
|
|
547
|
+
`viewports: 1440x900, 390x800`,
|
|
548
|
+
`commands: npm run dev:api & npm run dev:web -- --port 4321; capture via chromium headless screenshot`,
|
|
549
|
+
`screenshots: ${VISUAL_QA_DIR_NAME}/desktop-1440x900.png, ${VISUAL_QA_DIR_NAME}/mobile-390x800.png`,
|
|
550
|
+
``,
|
|
551
|
+
].join("\n");
|
|
552
|
+
return { exitCode: 0, transcript, logPath: "/dev/null", timedOut: false };
|
|
553
|
+
};
|
|
554
|
+
function shasFromPrompt(prompt) {
|
|
555
|
+
const baseSha = prompt.match(/"baseSha" to exactly "([0-9a-f]+)"/)?.[1];
|
|
556
|
+
const headSha = prompt.match(/"headSha" to exactly "([0-9a-f]+)"/)?.[1];
|
|
557
|
+
if (!baseSha || !headSha)
|
|
558
|
+
throw new Error("could not extract SHAs from reviewer prompt");
|
|
559
|
+
return { baseSha, headSha };
|
|
560
|
+
}
|
|
561
|
+
test("NOT-381: developer screenshots hand off a validated receipt bound to the PR head into the reviewer input", async () => {
|
|
562
|
+
const issueId = await makeIssue();
|
|
563
|
+
const reviewerPrompts = [];
|
|
564
|
+
const approvingSpawn = async (input) => {
|
|
565
|
+
reviewerPrompts.push(input.prompt);
|
|
566
|
+
const { baseSha, headSha } = shasFromPrompt(input.prompt);
|
|
567
|
+
const body = {
|
|
568
|
+
verdict: "approved",
|
|
569
|
+
baseSha,
|
|
570
|
+
headSha,
|
|
571
|
+
acceptanceCriteriaAssessment: "Met.",
|
|
572
|
+
evidenceAssessment: "Visual receipt bound to this head; code reviewed.",
|
|
573
|
+
findings: [],
|
|
574
|
+
risks: [],
|
|
575
|
+
};
|
|
576
|
+
return { exitCode: 0, transcript: `Review done.\n\`\`\`json\n${JSON.stringify(body)}\n\`\`\`\n`, logPath: "/dev/null", timedOut: false };
|
|
577
|
+
};
|
|
578
|
+
// One shared in-memory PR store across both handlers — the reviewer looks
|
|
579
|
+
// the PR up by number, so a second store would miss.
|
|
580
|
+
const github = fakeGithub();
|
|
581
|
+
registerEffectHandler("developer", (ctx) => runDeveloperEffect(ctx, { deckCallTool: okDeckCallTool, spawn: visualSpawn, github }));
|
|
582
|
+
registerEffectHandler("reviewer", (ctx) => runReviewerEffect(ctx, { deckCallTool: okDeckCallTool, spawn: approvingSpawn, github }));
|
|
583
|
+
startWorkflow(issueId);
|
|
584
|
+
await pump(1);
|
|
585
|
+
// Developer handed off cleanly at the published PR head.
|
|
586
|
+
const afterDev = getIssue(issueId);
|
|
587
|
+
assert.equal(afterDev.status, "reviewing");
|
|
588
|
+
assert.ok(afterDev.headSha);
|
|
589
|
+
const prHead = afterDev.headSha;
|
|
590
|
+
assert.ok(afterDev.prNumber);
|
|
591
|
+
// The receipt names the exact PR head and both screenshot blob references
|
|
592
|
+
// exist in Dealer storage.
|
|
593
|
+
const content = receiptContent(issueId);
|
|
594
|
+
assert.equal(content.status, "verified");
|
|
595
|
+
assert.equal(content.headSha, prHead);
|
|
596
|
+
assert.equal(content.screenshots.length, 2);
|
|
597
|
+
const blobBytes = content.screenshots.map((s) => {
|
|
598
|
+
assert.ok(fs.existsSync(s.blobPath), `blob must exist: ${s.blobPath}`);
|
|
599
|
+
return fs.readFileSync(s.blobPath, "utf8");
|
|
600
|
+
});
|
|
601
|
+
assert.deepEqual(blobBytes.sort(), ["desktop-bytes", "mobile-bytes"]);
|
|
602
|
+
assert.equal(latestIssueArtifact(issueId, VISUAL_QA_REJECTED_KIND), null);
|
|
603
|
+
// Clean worktree state: screenshots were never committed — the pushed branch
|
|
604
|
+
// tree carries no visual-artifact directory.
|
|
605
|
+
const branch = `issue-${issueId}`;
|
|
606
|
+
const tree = git(repo, "ls-tree", "-r", branch, "--name-only");
|
|
607
|
+
assert.doesNotMatch(tree, new RegExp(VISUAL_QA_DIR_NAME));
|
|
608
|
+
assert.match(tree, /feature\.txt/);
|
|
609
|
+
assert.equal(git(remote, "rev-parse", branch), prHead);
|
|
610
|
+
await pump(1);
|
|
611
|
+
assert.equal(reviewerPrompts.length, 1, "reviewer ran exactly once");
|
|
612
|
+
const reviewerPrompt = reviewerPrompts[0];
|
|
613
|
+
assert.match(reviewerPrompt, /## Visual QA \(developer receipt — SHA-bound to this head\)/);
|
|
614
|
+
assert.ok(reviewerPrompt.includes(prHead), "reviewer input names the exact PR head");
|
|
615
|
+
assert.match(reviewerPrompt, /desktop-1440x900\.png/);
|
|
616
|
+
assert.match(reviewerPrompt, /mobile-390x800\.png/);
|
|
617
|
+
assert.match(reviewerPrompt, /real \(no mocks\)/);
|
|
618
|
+
assert.match(reviewerPrompt, /1440x900, 390x800/);
|
|
619
|
+
});
|