@joeytheman/gemini-mcp-tool 1.3.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +127 -35
- package/dist/__tests__/agyModelSteps.test.d.ts +2 -0
- package/dist/__tests__/agyModelSteps.test.d.ts.map +1 -0
- package/dist/__tests__/agyModelSteps.test.js +58 -0
- package/dist/__tests__/agyModelSteps.test.js.map +1 -0
- package/dist/__tests__/ask-gemini.test.js +99 -37
- package/dist/__tests__/ask-gemini.test.js.map +1 -1
- package/dist/__tests__/capture-script.chromium.test.d.ts +2 -0
- package/dist/__tests__/capture-script.chromium.test.d.ts.map +1 -0
- package/dist/__tests__/capture-script.chromium.test.js +235 -0
- package/dist/__tests__/capture-script.chromium.test.js.map +1 -0
- package/dist/__tests__/layout-snippet.chromium.test.d.ts +2 -0
- package/dist/__tests__/layout-snippet.chromium.test.d.ts.map +1 -0
- package/dist/__tests__/layout-snippet.chromium.test.js +333 -0
- package/dist/__tests__/layout-snippet.chromium.test.js.map +1 -0
- package/dist/__tests__/layout-snippet.test.d.ts +2 -0
- package/dist/__tests__/layout-snippet.test.d.ts.map +1 -0
- package/dist/__tests__/layout-snippet.test.js +12 -0
- package/dist/__tests__/layout-snippet.test.js.map +1 -0
- package/dist/__tests__/live-pass.test.d.ts +2 -0
- package/dist/__tests__/live-pass.test.d.ts.map +1 -0
- package/dist/__tests__/live-pass.test.js +1249 -0
- package/dist/__tests__/live-pass.test.js.map +1 -0
- package/dist/__tests__/screen-review.test.d.ts +2 -0
- package/dist/__tests__/screen-review.test.d.ts.map +1 -0
- package/dist/__tests__/screen-review.test.js +553 -0
- package/dist/__tests__/screen-review.test.js.map +1 -0
- package/dist/__tests__/server-progress.test.d.ts +2 -0
- package/dist/__tests__/server-progress.test.d.ts.map +1 -0
- package/dist/__tests__/server-progress.test.js +63 -0
- package/dist/__tests__/server-progress.test.js.map +1 -0
- package/dist/__tests__/server.test.js +17 -19
- package/dist/__tests__/server.test.js.map +1 -1
- package/dist/__tests__/simple-tools.test.d.ts +2 -0
- package/dist/__tests__/simple-tools.test.d.ts.map +1 -0
- package/dist/__tests__/simple-tools.test.js +30 -0
- package/dist/__tests__/simple-tools.test.js.map +1 -0
- package/dist/__tests__/tools.test.js +91 -0
- package/dist/__tests__/tools.test.js.map +1 -1
- package/dist/__tests__/utils/agyExecutor.test.d.ts +2 -0
- package/dist/__tests__/utils/agyExecutor.test.d.ts.map +1 -0
- package/dist/__tests__/utils/agyExecutor.test.js +731 -0
- package/dist/__tests__/utils/agyExecutor.test.js.map +1 -0
- package/dist/__tests__/utils/agyTranscriptRecovery.test.d.ts +2 -0
- package/dist/__tests__/utils/agyTranscriptRecovery.test.d.ts.map +1 -0
- package/dist/__tests__/utils/agyTranscriptRecovery.test.js +127 -0
- package/dist/__tests__/utils/agyTranscriptRecovery.test.js.map +1 -0
- package/dist/__tests__/utils/chunkCache.test.js +26 -2
- package/dist/__tests__/utils/chunkCache.test.js.map +1 -1
- package/dist/__tests__/utils/commandExecutor.test.js +185 -7
- package/dist/__tests__/utils/commandExecutor.test.js.map +1 -1
- package/dist/__tests__/utils/fileReferences.test.d.ts +2 -0
- package/dist/__tests__/utils/fileReferences.test.d.ts.map +1 -0
- package/dist/__tests__/utils/fileReferences.test.js +87 -0
- package/dist/__tests__/utils/fileReferences.test.js.map +1 -0
- package/dist/__tests__/utils/gitStatusGuard.test.d.ts +2 -0
- package/dist/__tests__/utils/gitStatusGuard.test.d.ts.map +1 -0
- package/dist/__tests__/utils/gitStatusGuard.test.js +106 -0
- package/dist/__tests__/utils/gitStatusGuard.test.js.map +1 -0
- package/dist/__tests__/utils/progressUpdates.test.d.ts +2 -0
- package/dist/__tests__/utils/progressUpdates.test.d.ts.map +1 -0
- package/dist/__tests__/utils/progressUpdates.test.js +62 -0
- package/dist/__tests__/utils/progressUpdates.test.js.map +1 -0
- package/dist/__tests__/utils/responseCache.test.js +31 -11
- package/dist/__tests__/utils/responseCache.test.js.map +1 -1
- package/dist/__tests__/utils/test-helpers.d.ts +4 -0
- package/dist/__tests__/utils/test-helpers.d.ts.map +1 -1
- package/dist/__tests__/utils/test-helpers.js +16 -0
- package/dist/__tests__/utils/test-helpers.js.map +1 -1
- package/dist/constants.d.ts +91 -26
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +101 -28
- package/dist/constants.js.map +1 -1
- package/dist/index.js +20 -95
- package/dist/index.js.map +1 -1
- package/dist/layoutCheckSnippet.generated.d.ts +2 -0
- package/dist/layoutCheckSnippet.generated.d.ts.map +1 -0
- package/dist/layoutCheckSnippet.generated.js +3 -0
- package/dist/layoutCheckSnippet.generated.js.map +1 -0
- package/dist/tools/ask-gemini.tool.d.ts +34 -0
- package/dist/tools/ask-gemini.tool.d.ts.map +1 -1
- package/dist/tools/ask-gemini.tool.js +40 -23
- package/dist/tools/ask-gemini.tool.js.map +1 -1
- package/dist/tools/brainstorm.tool.d.ts.map +1 -1
- package/dist/tools/brainstorm.tool.js +7 -5
- package/dist/tools/brainstorm.tool.js.map +1 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +3 -1
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/live-pass.tool.d.ts +206 -0
- package/dist/tools/live-pass.tool.d.ts.map +1 -0
- package/dist/tools/live-pass.tool.js +670 -0
- package/dist/tools/live-pass.tool.js.map +1 -0
- package/dist/tools/registry.d.ts +6 -4
- package/dist/tools/registry.d.ts.map +1 -1
- package/dist/tools/registry.js +14 -11
- package/dist/tools/registry.js.map +1 -1
- package/dist/tools/screen-review.tool.d.ts +124 -0
- package/dist/tools/screen-review.tool.d.ts.map +1 -0
- package/dist/tools/screen-review.tool.js +449 -0
- package/dist/tools/screen-review.tool.js.map +1 -0
- package/dist/tools/simple-tools.d.ts.map +1 -1
- package/dist/tools/simple-tools.js +9 -3
- package/dist/tools/simple-tools.js.map +1 -1
- package/dist/utils/agyExecutor.d.ts +76 -0
- package/dist/utils/agyExecutor.d.ts.map +1 -0
- package/dist/utils/agyExecutor.js +449 -0
- package/dist/utils/agyExecutor.js.map +1 -0
- package/dist/utils/agyModelSteps.d.ts +5 -0
- package/dist/utils/agyModelSteps.d.ts.map +1 -0
- package/dist/utils/agyModelSteps.js +33 -0
- package/dist/utils/agyModelSteps.js.map +1 -0
- package/dist/utils/agyReport.d.ts +19 -0
- package/dist/utils/agyReport.d.ts.map +1 -0
- package/dist/utils/agyReport.js +62 -0
- package/dist/utils/agyReport.js.map +1 -0
- package/dist/utils/agyTranscriptRecovery.d.ts +16 -0
- package/dist/utils/agyTranscriptRecovery.d.ts.map +1 -0
- package/dist/utils/agyTranscriptRecovery.js +149 -0
- package/dist/utils/agyTranscriptRecovery.js.map +1 -0
- package/dist/utils/artifactFile.d.ts +6 -0
- package/dist/utils/artifactFile.d.ts.map +1 -0
- package/dist/utils/artifactFile.js +55 -0
- package/dist/utils/artifactFile.js.map +1 -0
- package/dist/utils/captureScript.d.ts +24 -0
- package/dist/utils/captureScript.d.ts.map +1 -0
- package/dist/utils/captureScript.js +83 -0
- package/dist/utils/captureScript.js.map +1 -0
- package/dist/utils/chunkCache.d.ts.map +1 -1
- package/dist/utils/chunkCache.js +19 -5
- package/dist/utils/chunkCache.js.map +1 -1
- package/dist/utils/commandExecutor.d.ts +7 -1
- package/dist/utils/commandExecutor.d.ts.map +1 -1
- package/dist/utils/commandExecutor.js +162 -22
- package/dist/utils/commandExecutor.js.map +1 -1
- package/dist/utils/fileReferences.d.ts +45 -0
- package/dist/utils/fileReferences.d.ts.map +1 -0
- package/dist/utils/fileReferences.js +169 -0
- package/dist/utils/fileReferences.js.map +1 -0
- package/dist/utils/gitCheckout.d.ts +14 -0
- package/dist/utils/gitCheckout.d.ts.map +1 -0
- package/dist/utils/gitCheckout.js +36 -0
- package/dist/utils/gitCheckout.js.map +1 -0
- package/dist/utils/gitStatusGuard.d.ts +35 -0
- package/dist/utils/gitStatusGuard.d.ts.map +1 -0
- package/dist/utils/gitStatusGuard.js +82 -0
- package/dist/utils/gitStatusGuard.js.map +1 -0
- package/dist/utils/layoutCheck.d.ts +58 -0
- package/dist/utils/layoutCheck.d.ts.map +1 -0
- package/dist/utils/layoutCheck.js +84 -0
- package/dist/utils/layoutCheck.js.map +1 -0
- package/dist/utils/logger.d.ts +0 -1
- package/dist/utils/logger.d.ts.map +1 -1
- package/dist/utils/logger.js +0 -6
- package/dist/utils/logger.js.map +1 -1
- package/dist/utils/progressUpdates.d.ts +14 -0
- package/dist/utils/progressUpdates.d.ts.map +1 -0
- package/dist/utils/progressUpdates.js +40 -0
- package/dist/utils/progressUpdates.js.map +1 -0
- package/dist/utils/responseCache.d.ts +3 -3
- package/dist/utils/responseCache.d.ts.map +1 -1
- package/dist/utils/responseCache.js +11 -6
- package/dist/utils/responseCache.js.map +1 -1
- package/package.json +19 -16
- package/dist/__tests__/utils/geminiExecutor.test.d.ts +0 -2
- package/dist/__tests__/utils/geminiExecutor.test.d.ts.map +0 -1
- package/dist/__tests__/utils/geminiExecutor.test.js +0 -337
- package/dist/__tests__/utils/geminiExecutor.test.js.map +0 -1
- package/dist/utils/geminiExecutor.d.ts +0 -17
- package/dist/utils/geminiExecutor.d.ts.map +0 -1
- package/dist/utils/geminiExecutor.js +0 -238
- package/dist/utils/geminiExecutor.js.map +0 -1
|
@@ -0,0 +1,1249 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import * as fs from 'fs';
|
|
3
|
+
import * as path from 'path';
|
|
4
|
+
import { execFileSync } from 'child_process';
|
|
5
|
+
vi.mock('../utils/logger.js', () => ({
|
|
6
|
+
Logger: { debug: vi.fn(), warn: vi.fn(), error: vi.fn(), log: vi.fn(), toolInvocation: vi.fn() },
|
|
7
|
+
}));
|
|
8
|
+
const mockExecuteAgyJson = vi.fn();
|
|
9
|
+
vi.mock('../utils/agyExecutor.js', async (importOriginal) => {
|
|
10
|
+
const actual = await importOriginal();
|
|
11
|
+
return { ...actual, executeAgyJson: (...args) => mockExecuteAgyJson(...args) };
|
|
12
|
+
});
|
|
13
|
+
const cleanSnapshot = (root = '/repo') => ({
|
|
14
|
+
root,
|
|
15
|
+
lines: [],
|
|
16
|
+
hashes: new Map(),
|
|
17
|
+
truncated: false,
|
|
18
|
+
});
|
|
19
|
+
const mockSnapshotWorkingTree = vi.fn().mockReturnValue(cleanSnapshot());
|
|
20
|
+
const mockCountAgyModelSteps = vi.fn(async (_conversationId) => null);
|
|
21
|
+
vi.mock('../utils/agyModelSteps.js', async (importOriginal) => {
|
|
22
|
+
const actual = await importOriginal();
|
|
23
|
+
return { ...actual, countAgyModelSteps: (id) => mockCountAgyModelSteps(id) };
|
|
24
|
+
});
|
|
25
|
+
vi.mock('../utils/gitStatusGuard.js', async (importOriginal) => {
|
|
26
|
+
const actual = await importOriginal();
|
|
27
|
+
return { ...actual, snapshotWorkingTree: (...args) => mockSnapshotWorkingTree(...args) };
|
|
28
|
+
});
|
|
29
|
+
import { livePassTool, livePassArgsSchema, livePassResultSchema, LIVE_PASS_RESULT_JSON_SCHEMA, buildLivePassPrompt, frameFileName, prescribedFrameNames, prescribedTargetFiles, formatLayoutSection, readLayoutChecks, validateFrames, checkCoverage, deriveVerdict, runPublishCommand, writeCaptureScripts, } from '../tools/live-pass.tool.js';
|
|
30
|
+
import { ERROR_MESSAGES, LAYOUT_CHECK_SNIPPET, LIVE_PASS, MODELS } from '../constants.js';
|
|
31
|
+
import { readLayoutCheck } from '../utils/layoutCheck.js';
|
|
32
|
+
import { buildCaptureScript, SCROLL_OVERFLOW_SNIPPET } from '../utils/captureScript.js';
|
|
33
|
+
import { makeTempDir, cleanupTempDirs } from './utils/test-helpers.js';
|
|
34
|
+
// ------------------------------------------------------------------ helpers
|
|
35
|
+
const makeWorkDir = () => makeTempDir('live-pass');
|
|
36
|
+
const PNG = fs.readFileSync(new URL('./fixtures/images/pixel.png', import.meta.url));
|
|
37
|
+
const JPEG = fs.readFileSync(new URL('./fixtures/images/pixel.jpg', import.meta.url));
|
|
38
|
+
const HOME_TARGET = {
|
|
39
|
+
screen: 'home',
|
|
40
|
+
state: 'default',
|
|
41
|
+
viewport: '390x844',
|
|
42
|
+
expected: "heading 'Hello'",
|
|
43
|
+
};
|
|
44
|
+
function makeFrame(overrides = {}) {
|
|
45
|
+
return {
|
|
46
|
+
file: '/tmp/frame.png',
|
|
47
|
+
screen: 'home',
|
|
48
|
+
state: 'default',
|
|
49
|
+
expected: "heading 'Hello'",
|
|
50
|
+
viewport: '390x844',
|
|
51
|
+
verdict: 'PASS',
|
|
52
|
+
observed: 'Shows the Hello heading above the list.',
|
|
53
|
+
...overrides,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
function makeResult(overrides = {}) {
|
|
57
|
+
return {
|
|
58
|
+
verdict: 'PASS',
|
|
59
|
+
summary: 'Everything rendered.',
|
|
60
|
+
served_branch_check: "footer reads 'BRANCH: smoke'",
|
|
61
|
+
frames: [makeFrame()],
|
|
62
|
+
findings: [],
|
|
63
|
+
regressions: [],
|
|
64
|
+
resolved: [],
|
|
65
|
+
changes: [],
|
|
66
|
+
skipped: [],
|
|
67
|
+
...overrides,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
const CLEAN_VALIDATION = { checks: [], okFiles: [], problems: [], notes: [], unreportedCaptures: [] };
|
|
71
|
+
const CLEAN_COVERAGE = { uncovered: [], skipped: [], extraFrames: [] };
|
|
72
|
+
function writeFrameFile(artifactDir, name, contents) {
|
|
73
|
+
fs.mkdirSync(artifactDir, { recursive: true });
|
|
74
|
+
const file = path.join(artifactDir, name);
|
|
75
|
+
fs.writeFileSync(file, contents ?? (/\.png$/i.test(name) ? PNG : /\.jpe?g$/i.test(name) ? JPEG : 'not an image'));
|
|
76
|
+
return file;
|
|
77
|
+
}
|
|
78
|
+
function writeManifest(artifactDir, frames) {
|
|
79
|
+
fs.writeFileSync(path.join(artifactDir, LIVE_PASS.MANIFEST_FILE), JSON.stringify({ run_started: new Date().toISOString(), driver: 'playwright', frames, skipped: [] }));
|
|
80
|
+
}
|
|
81
|
+
function makeLayoutReport(overrides = {}) {
|
|
82
|
+
return {
|
|
83
|
+
check: LIVE_PASS.LAYOUT_CHECK_ID,
|
|
84
|
+
viewport: { width: 390, height: 844 },
|
|
85
|
+
found: 0,
|
|
86
|
+
entries: [],
|
|
87
|
+
clamped: [],
|
|
88
|
+
...overrides,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
const SPILL_ENTRY = {
|
|
92
|
+
kind: 'text-spill',
|
|
93
|
+
text: 'Uploaded to the studio wins wall',
|
|
94
|
+
testID: 'label',
|
|
95
|
+
rect: { x: 0, y: 0, width: 320, height: 20 },
|
|
96
|
+
};
|
|
97
|
+
function writeLayoutFile(artifactDir, frameName, contents) {
|
|
98
|
+
fs.mkdirSync(artifactDir, { recursive: true });
|
|
99
|
+
const file = path.join(artifactDir, frameName.replace(/\.png$/, LIVE_PASS.LAYOUT_FILE_SUFFIX));
|
|
100
|
+
fs.writeFileSync(file, typeof contents === 'string' ? contents : JSON.stringify(contents, null, 2));
|
|
101
|
+
return file;
|
|
102
|
+
}
|
|
103
|
+
function envelope(structured, overrides = {}) {
|
|
104
|
+
return {
|
|
105
|
+
conversation_id: 'conv-abc',
|
|
106
|
+
status: 'SUCCESS',
|
|
107
|
+
response: '',
|
|
108
|
+
structured_output: structured,
|
|
109
|
+
num_turns: 3,
|
|
110
|
+
duration_seconds: 42,
|
|
111
|
+
usage: { input_tokens: 16000, output_tokens: 900, total_tokens: 16900 },
|
|
112
|
+
denied_actions: [],
|
|
113
|
+
...overrides,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
function run(args) {
|
|
117
|
+
return livePassTool.execute(args);
|
|
118
|
+
}
|
|
119
|
+
beforeEach(() => {
|
|
120
|
+
vi.clearAllMocks();
|
|
121
|
+
mockSnapshotWorkingTree.mockReturnValue(cleanSnapshot());
|
|
122
|
+
});
|
|
123
|
+
afterEach(cleanupTempDirs);
|
|
124
|
+
// ------------------------------------------------------------------- schema
|
|
125
|
+
describe('livePassArgsSchema', () => {
|
|
126
|
+
const minimal = {
|
|
127
|
+
targets: [HOME_TARGET],
|
|
128
|
+
brief: 'Base URL http://localhost:8123',
|
|
129
|
+
workingDirectory: '/tmp/project',
|
|
130
|
+
driver: 'playwright',
|
|
131
|
+
};
|
|
132
|
+
it('asks the caller\'s brief for the start URL and any link a user arrives by, with its source', () => {
|
|
133
|
+
expect(livePassArgsSchema.shape.brief.description).toContain("the app's start URL or app id, any link a user arrives by (an invite, email or share link) with where it comes from");
|
|
134
|
+
expect(livePassArgsSchema.shape.brief.description).not.toMatch(/entry url/i);
|
|
135
|
+
});
|
|
136
|
+
it('applies the documented defaults', () => {
|
|
137
|
+
const parsed = livePassArgsSchema.parse(minimal);
|
|
138
|
+
expect(parsed.yolo).toBe(true);
|
|
139
|
+
expect(parsed.printTimeout).toBe(LIVE_PASS.DEFAULT_PRINT_TIMEOUT);
|
|
140
|
+
expect(parsed.printTimeout).toBe('15m');
|
|
141
|
+
expect(parsed.model).toBe(MODELS.MEDIUM);
|
|
142
|
+
expect(parsed.model).toBe('Gemini 3.8 Flash (Medium)');
|
|
143
|
+
});
|
|
144
|
+
it('requires a driver and at least one target', () => {
|
|
145
|
+
expect(() => livePassArgsSchema.parse({ ...minimal, driver: undefined })).toThrow();
|
|
146
|
+
expect(() => livePassArgsSchema.parse({ ...minimal, targets: [] })).toThrow();
|
|
147
|
+
});
|
|
148
|
+
it('rejects a duplicate screen/state/viewport triple', () => {
|
|
149
|
+
expect(() => livePassArgsSchema.parse({
|
|
150
|
+
...minimal,
|
|
151
|
+
targets: [HOME_TARGET, { ...HOME_TARGET, screen: ' Home ' }],
|
|
152
|
+
})).toThrow(/duplicate target/);
|
|
153
|
+
});
|
|
154
|
+
it('keeps two states of the same screen', () => {
|
|
155
|
+
const parsed = livePassArgsSchema.parse({
|
|
156
|
+
...minimal,
|
|
157
|
+
targets: [HOME_TARGET, { ...HOME_TARGET, state: 'empty' }],
|
|
158
|
+
});
|
|
159
|
+
expect(parsed.targets).toHaveLength(2);
|
|
160
|
+
});
|
|
161
|
+
it('rejects a playwright target whose viewport is not WxH, and lets maestro name a form factor', () => {
|
|
162
|
+
const phone = { ...HOME_TARGET, viewport: 'iPhone 15' };
|
|
163
|
+
const web = livePassArgsSchema.safeParse({ ...minimal, targets: [HOME_TARGET, phone] });
|
|
164
|
+
const native = livePassArgsSchema.safeParse({ ...minimal, driver: 'maestro', targets: [phone] });
|
|
165
|
+
expect(web.success).toBe(false);
|
|
166
|
+
expect(web.error?.issues).toEqual([expect.objectContaining({
|
|
167
|
+
path: ['targets', 1, 'viewport'],
|
|
168
|
+
message: `${ERROR_MESSAGES.LIVE_PASS_WEB_VIEWPORT}. Received 'iPhone 15'.`,
|
|
169
|
+
})]);
|
|
170
|
+
expect(native.success).toBe(true);
|
|
171
|
+
});
|
|
172
|
+
});
|
|
173
|
+
describe('LIVE_PASS_RESULT_JSON_SCHEMA', () => {
|
|
174
|
+
it('has no $schema key and requires every result section', () => {
|
|
175
|
+
expect(LIVE_PASS_RESULT_JSON_SCHEMA).not.toHaveProperty('$schema');
|
|
176
|
+
expect(LIVE_PASS_RESULT_JSON_SCHEMA.required).toEqual(expect.arrayContaining([
|
|
177
|
+
'verdict', 'summary', 'served_branch_check', 'frames',
|
|
178
|
+
'findings', 'regressions', 'resolved', 'changes', 'skipped',
|
|
179
|
+
]));
|
|
180
|
+
});
|
|
181
|
+
it('requires expected and observed on every frame', () => {
|
|
182
|
+
const frames = LIVE_PASS_RESULT_JSON_SCHEMA.properties.frames;
|
|
183
|
+
expect(frames.items.required).toEqual(expect.arrayContaining(['file', 'expected', 'observed', 'verdict']));
|
|
184
|
+
});
|
|
185
|
+
it('offers link_source on a frame without requiring it', () => {
|
|
186
|
+
const frames = LIVE_PASS_RESULT_JSON_SCHEMA.properties.frames;
|
|
187
|
+
expect(frames.items.properties.link_source.type).toBe('string');
|
|
188
|
+
expect(frames.items.required).not.toContain('link_source');
|
|
189
|
+
});
|
|
190
|
+
});
|
|
191
|
+
// ------------------------------------------------------------------- prompt
|
|
192
|
+
describe('frameFileName', () => {
|
|
193
|
+
it('slugifies every segment so spaces and slashes cannot collide', () => {
|
|
194
|
+
expect(frameFileName({ screen: 'Settings/Profile', state: 'sheet open', viewport: '390x844' }))
|
|
195
|
+
.toBe('settings-profile-sheet-open-390x844.png');
|
|
196
|
+
expect(frameFileName({ screen: ' Order Detail ', state: 'Empty!', viewport: '1280 x 800' }))
|
|
197
|
+
.toBe('order-detail-empty-1280-x-800.png');
|
|
198
|
+
});
|
|
199
|
+
it('suffixes a name two different targets would otherwise slugify onto', () => {
|
|
200
|
+
const names = prescribedFrameNames([
|
|
201
|
+
{ screen: 'settings/profile', state: 'default', viewport: '390x844' },
|
|
202
|
+
{ screen: 'settings-profile', state: 'default', viewport: '390x844' },
|
|
203
|
+
]);
|
|
204
|
+
expect([...names.values()]).toEqual([
|
|
205
|
+
'settings-profile-default-390x844.png',
|
|
206
|
+
'settings-profile-default-390x844-2.png',
|
|
207
|
+
]);
|
|
208
|
+
});
|
|
209
|
+
it('gives two viewports of the same screen and state different files', () => {
|
|
210
|
+
const mobile = frameFileName({ screen: 'home', state: 'default', viewport: '390x844' });
|
|
211
|
+
const desktop = frameFileName({ screen: 'home', state: 'default', viewport: '1280x800' });
|
|
212
|
+
expect(mobile).not.toBe(desktop);
|
|
213
|
+
});
|
|
214
|
+
});
|
|
215
|
+
describe('buildLivePassPrompt', () => {
|
|
216
|
+
const base = {
|
|
217
|
+
brief: 'Start URL http://localhost:8123; marker: footer BRANCH: feat/orders',
|
|
218
|
+
targets: [HOME_TARGET, { ...HOME_TARGET, state: 'offline', expected: 'offline notice plus Retry' }],
|
|
219
|
+
workingDirectory: '/work/tree',
|
|
220
|
+
artifactDir: '/work/tree/.live-pass/run1',
|
|
221
|
+
checkout: { branch: 'feat/orders', head: '0123456789abcdef0123456789abcdef01234567' },
|
|
222
|
+
};
|
|
223
|
+
it('renders the targets as a numbered list with their expected state', () => {
|
|
224
|
+
const prompt = buildLivePassPrompt({ ...base, driver: 'playwright', resumed: false });
|
|
225
|
+
expect(prompt).toContain('1. screen: home | state: default | viewport: 390x844');
|
|
226
|
+
expect(prompt).toContain("expected: heading 'Hello'");
|
|
227
|
+
expect(prompt).toContain('2. screen: home | state: offline | viewport: 390x844');
|
|
228
|
+
expect(prompt).toContain('expected: offline notice plus Retry');
|
|
229
|
+
});
|
|
230
|
+
it('carries the playwright procedure and the evidence discipline', () => {
|
|
231
|
+
const prompt = buildLivePassPrompt({ ...base, driver: 'playwright', resumed: false });
|
|
232
|
+
expect(prompt).toContain('Capture it: browser_run_code_unsafe with filename set to the exact absolute path printed as "capture:" for that target below. Never pass code, never write, edit or read a capture file');
|
|
233
|
+
expect(prompt).toContain('/work/tree/.live-pass/run1');
|
|
234
|
+
expect(prompt).toContain('If a capture returns an error, add the target to "skipped" with the error text. Never fall back to browser_take_screenshot.');
|
|
235
|
+
expect(prompt).toContain('The capture sets the target\'s viewport and grows a scrolling screen taller, a few times at most and up to 4000 px, until its scroll area fits; a screen that still scrolls after that is shot as it stands.');
|
|
236
|
+
expect(prompt).toContain('an added "; layout not measured: <error>" or "; viewport not restored: <error>" still saved the frame.');
|
|
237
|
+
expect(prompt).not.toContain('full height');
|
|
238
|
+
expect(prompt).not.toContain('browser_resize');
|
|
239
|
+
expect(prompt).not.toContain('fullPage');
|
|
240
|
+
expect(prompt).not.toContain('320x568');
|
|
241
|
+
expect(prompt).toContain('answer is ONLY the schema JSON');
|
|
242
|
+
expect(prompt).toContain('a spinner, blank page, skeleton, error, or the wrong screen is a FAIL');
|
|
243
|
+
expect(prompt).toContain('PASSES when it shows that state with everything "expected" names');
|
|
244
|
+
expect(prompt).toContain('Something "expected" does not ask for is at most a MINOR finding');
|
|
245
|
+
expect(prompt).toContain(LIVE_PASS.MANIFEST_FILE);
|
|
246
|
+
expect(prompt).not.toContain('list_devices');
|
|
247
|
+
});
|
|
248
|
+
it('prints one prescribed absolute file path per target, for both drivers', () => {
|
|
249
|
+
for (const driver of ['playwright', 'maestro']) {
|
|
250
|
+
const prompt = buildLivePassPrompt({ ...base, driver, resumed: false });
|
|
251
|
+
expect(prompt).toContain(` file: /work/tree/.live-pass/run1/${frameFileName(base.targets[0])}`);
|
|
252
|
+
expect(prompt).toContain(` file: /work/tree/.live-pass/run1/${frameFileName(base.targets[1])}`);
|
|
253
|
+
// The viewport is part of every prescribed name, so two viewports of one
|
|
254
|
+
// screen/state can never land in the same file.
|
|
255
|
+
expect(prompt).toContain('home-default-390x844.png');
|
|
256
|
+
expect(prompt).toContain('home-offline-390x844.png');
|
|
257
|
+
expect(prompt).toContain('a file that answers two targets is treated as no evidence at all');
|
|
258
|
+
expect(prompt).toContain('never reuse another target\'s file');
|
|
259
|
+
}
|
|
260
|
+
});
|
|
261
|
+
it('carries the maestro procedure instead for the maestro driver', () => {
|
|
262
|
+
const prompt = buildLivePassPrompt({ ...base, driver: 'maestro', resumed: false });
|
|
263
|
+
expect(prompt).toContain('list_devices');
|
|
264
|
+
expect(prompt).toContain('xcrun simctl io');
|
|
265
|
+
expect(prompt).toContain('Never use take_screenshot for evidence');
|
|
266
|
+
expect(prompt).not.toContain('browser_take_screenshot');
|
|
267
|
+
expect(prompt).not.toContain('browser_run_code_unsafe');
|
|
268
|
+
});
|
|
269
|
+
it('adds the delta paragraph only for a resumed pass', () => {
|
|
270
|
+
const fresh = buildLivePassPrompt({ ...base, driver: 'playwright', resumed: false });
|
|
271
|
+
const resumed = buildLivePassPrompt({ ...base, driver: 'playwright', resumed: true });
|
|
272
|
+
expect(fresh).not.toContain('Delta reporting');
|
|
273
|
+
expect(fresh).toContain('This is a fresh pass');
|
|
274
|
+
expect(resumed).toContain('Delta reporting');
|
|
275
|
+
expect(resumed).toContain('Never put an improvement in "findings" or "regressions"');
|
|
276
|
+
});
|
|
277
|
+
});
|
|
278
|
+
describe('buildLivePassPrompt: context Gemini used to dig for', () => {
|
|
279
|
+
const base = {
|
|
280
|
+
brief: 'Start URL http://localhost:8123; marker: footer BRANCH: feat/orders',
|
|
281
|
+
targets: [HOME_TARGET, { ...HOME_TARGET, state: 'offline', expected: 'offline notice plus Retry' }],
|
|
282
|
+
workingDirectory: '/work/tree',
|
|
283
|
+
artifactDir: '/work/tree/.live-pass/run1',
|
|
284
|
+
checkout: { branch: 'feat/orders', head: '0123456789abcdef0123456789abcdef01234567' },
|
|
285
|
+
resumed: false,
|
|
286
|
+
};
|
|
287
|
+
it('prints the branch and HEAD read in code, and forbids running git to find them', () => {
|
|
288
|
+
const prompt = buildLivePassPrompt({ ...base, driver: 'playwright' });
|
|
289
|
+
expect(prompt).toContain('- Checkout under test: feat/orders at 0123456789abcdef0123456789abcdef01234567. Read for you, so never run git.');
|
|
290
|
+
expect(prompt).toContain('The checkout above is context. Set BLOCKED over it only when the marker names a branch or commit that contradicts it. A detached HEAD never contradicts a branch name, and a short commit hash matches the full one it begins.');
|
|
291
|
+
});
|
|
292
|
+
it('names a detached HEAD instead of a branch', () => {
|
|
293
|
+
const prompt = buildLivePassPrompt({ ...base, driver: 'playwright', checkout: { branch: null, head: 'abc1234' } });
|
|
294
|
+
expect(prompt).toContain('- Checkout under test: a detached HEAD at abc1234.');
|
|
295
|
+
});
|
|
296
|
+
it('falls back to the brief marker when the working directory is not a git checkout', () => {
|
|
297
|
+
const prompt = buildLivePassPrompt({ ...base, driver: 'playwright', checkout: null });
|
|
298
|
+
expect(prompt).toContain("- Checkout under test: not a git checkout with a commit, so rely on the brief's marker alone.");
|
|
299
|
+
});
|
|
300
|
+
it('carries a parameter cheat sheet for every playwright tool the procedure uses', () => {
|
|
301
|
+
const prompt = buildLivePassPrompt({ ...base, driver: 'playwright' });
|
|
302
|
+
expect(prompt).toContain('## Tool cheat sheet (the installed parameters: never read tool schemas or Playwright source)');
|
|
303
|
+
for (const line of [
|
|
304
|
+
'- browser_navigate {url}',
|
|
305
|
+
'- browser_click {target, element?, doubleClick?, button?, modifiers?}',
|
|
306
|
+
'- browser_type {target, text, submit?, slowly?}',
|
|
307
|
+
'- browser_fill_form {fields: [{target, name, type: textbox | checkbox | radio | combobox | slider, value}]}',
|
|
308
|
+
'- browser_select_option {target, values: [...]}',
|
|
309
|
+
'- browser_press_key {key}',
|
|
310
|
+
'- browser_handle_dialog {accept, promptText?}',
|
|
311
|
+
'- browser_wait_for {text? | textGone? | time?}',
|
|
312
|
+
'- browser_find {text | regex}',
|
|
313
|
+
'- browser_snapshot {target?, depth?, boxes?}',
|
|
314
|
+
'- browser_run_code_unsafe {filename}: filename is absolute.',
|
|
315
|
+
'- browser_evaluate {function, filename}: the return value is written to filename as JSON instead of returned to you.',
|
|
316
|
+
]) {
|
|
317
|
+
expect(prompt).toContain(line);
|
|
318
|
+
}
|
|
319
|
+
expect(prompt).not.toContain('- browser_take_screenshot');
|
|
320
|
+
expect(prompt).toContain('a unique selector such as [data-testid="save"]');
|
|
321
|
+
expect(prompt).not.toContain('- inspect_screen {device_id}');
|
|
322
|
+
});
|
|
323
|
+
it('carries the maestro tool cheat sheet for the maestro driver instead', () => {
|
|
324
|
+
const prompt = buildLivePassPrompt({ ...base, driver: 'maestro' });
|
|
325
|
+
for (const line of [
|
|
326
|
+
'- list_devices {}',
|
|
327
|
+
'- run {device_id, yaml, env?}',
|
|
328
|
+
'- inspect_screen {device_id}',
|
|
329
|
+
'- take_screenshot {device_id}: returns an image inline; never evidence.',
|
|
330
|
+
'- cheat_sheet {}',
|
|
331
|
+
]) {
|
|
332
|
+
expect(prompt).toContain(line);
|
|
333
|
+
}
|
|
334
|
+
expect(prompt).not.toContain('- browser_click {');
|
|
335
|
+
});
|
|
336
|
+
it('tells a web pass to answer native dialogs right after the triggering click', () => {
|
|
337
|
+
const prompt = buildLivePassPrompt({ ...base, driver: 'playwright' });
|
|
338
|
+
expect(prompt).toContain('Native dialogs such as window.confirm and window.alert block the page: accept or dismiss each with browser_handle_dialog right after the click that opened it.');
|
|
339
|
+
});
|
|
340
|
+
it.each(['playwright', 'maestro'])('forbids code, database and git reads for the %s driver', (driver) => {
|
|
341
|
+
const prompt = buildLivePassPrompt({ ...base, driver });
|
|
342
|
+
expect(prompt).toContain('Never read product source, tests, docs, config, tool schemas or tool source. Never query a database.');
|
|
343
|
+
expect(prompt).toContain("Never run a shell command except the capture command the procedure names and removing a stale file at a target's own \"file:\" path.");
|
|
344
|
+
expect(prompt).toContain('Never read code to work out why something failed. Report what you saw.');
|
|
345
|
+
expect(prompt).toContain('never run a git command.');
|
|
346
|
+
expect(prompt).toContain('It is only where you run: read, list and glob nothing outside the artifact directory.');
|
|
347
|
+
expect(prompt).not.toContain('Working directory (read-only)');
|
|
348
|
+
});
|
|
349
|
+
it.each(['playwright', 'maestro'])('opens the %s app once and reaches every other target by taps from the previous screen', (driver) => {
|
|
350
|
+
const prompt = buildLivePassPrompt({ ...base, driver });
|
|
351
|
+
expect(prompt).toContain('## User path\n- The pass opens the app once, at its start URL (or by launching it on a device), as the brief names it.');
|
|
352
|
+
expect(prompt).toContain('- Every other target is reached by taps from the previous screen: sign in through the sign-in form, tap through the navigation, and fill and submit forms. Any other URL or deep link in the brief is context, never a navigation target.');
|
|
353
|
+
expect(prompt).toContain('- A reload, only where the brief makes one part of the path, opens the URL the page is already on.');
|
|
354
|
+
expect(prompt).toContain('- No script that clicks, navigates, or sets storage, cookies or app state for you.');
|
|
355
|
+
});
|
|
356
|
+
it.each(['playwright', 'maestro'])('lets a %s target begin at its own link only when the brief names the link the user arrives by, and has the frame quote its source', (driver) => {
|
|
357
|
+
const prompt = buildLivePassPrompt({ ...base, driver });
|
|
358
|
+
expect(prompt).toContain('- A target may begin at a link of its own only when the brief says the user arrives by that link, such as an invite, email or share link, and names where the link comes from. Open it as following that link would, and quote the brief\'s words naming the source in that frame\'s "link_source".');
|
|
359
|
+
});
|
|
360
|
+
it.each(['playwright', 'maestro'])('leaves no per-target entry URL or URL jump anywhere in the %s prompt', (driver) => {
|
|
361
|
+
const prompt = buildLivePassPrompt({ ...base, driver });
|
|
362
|
+
expect(prompt).not.toMatch(/entry url/i);
|
|
363
|
+
expect(prompt).not.toContain('deeper URL');
|
|
364
|
+
expect(prompt).not.toContain('walk on from wherever');
|
|
365
|
+
expect(prompt).not.toContain('Never use openLink to skip a user step.');
|
|
366
|
+
});
|
|
367
|
+
it('sends a web pass\'s browser_navigate only where the user path allows, and keeps browser_evaluate and run_code to the printed save functions and capture files', () => {
|
|
368
|
+
const prompt = buildLivePassPrompt({ ...base, driver: 'playwright' });
|
|
369
|
+
expect(prompt).not.toContain("browser_navigate to the target's route");
|
|
370
|
+
expect(prompt).toContain('browser_navigate opens only the URLs the "User path" section allows.\nFor each target, in order:\n- Reach it as the "User path" section says.');
|
|
371
|
+
expect(prompt).toContain('browser_evaluate runs only the "save layout:" functions and browser_run_code_unsafe only the "capture:" files. Neither is used to click, type, navigate, or set storage, cookies or app state.');
|
|
372
|
+
});
|
|
373
|
+
it('has a maestro pass launch the app only at the start and openLink only for a link arrival', () => {
|
|
374
|
+
const prompt = buildLivePassPrompt({ ...base, driver: 'maestro' });
|
|
375
|
+
expect(prompt).toContain('- Reach each state as the "User path" section says, with inline YAML flows: launchApp only to open the app at the start, then tapOn and inputText; openLink only for a link arrival that section allows.');
|
|
376
|
+
});
|
|
377
|
+
it('locates web controls by testID, then browser_find, then a scoped snapshot', () => {
|
|
378
|
+
const prompt = buildLivePassPrompt({ ...base, driver: 'playwright' });
|
|
379
|
+
expect(prompt).toContain('Locate each control cheaply, in this order: a testID the brief names, as target [data-testid="<testID>"]; then browser_find with its visible label; then browser_snapshot with a target and a small depth scoped to that region. Never take a full-page snapshot just to look around');
|
|
380
|
+
});
|
|
381
|
+
it.each(['playwright', 'maestro'])('reports a %s dead end with the broken step, and a missing fact as skipped', (driver) => {
|
|
382
|
+
const prompt = buildLivePassPrompt({ ...base, driver });
|
|
383
|
+
expect(prompt).toContain('add it to "skipped" with the step where the path broke, such as "step 3: tapped Save, nothing happened". A dead end a user would hit is also a finding.');
|
|
384
|
+
expect(prompt).toContain('add the target to "skipped" with "brief missing: <the fact>". Never search the code for it.');
|
|
385
|
+
});
|
|
386
|
+
it('prints each web target its own capture file, layout file and save function, and neither page snippet', () => {
|
|
387
|
+
const prompt = buildLivePassPrompt({ ...base, driver: 'playwright' });
|
|
388
|
+
expect(prompt).toContain([
|
|
389
|
+
' file: /work/tree/.live-pass/run1/home-default-390x844.png',
|
|
390
|
+
' capture: /work/tree/.live-pass/run1/home-default-390x844.capture.js',
|
|
391
|
+
' layout: /work/tree/.live-pass/run1/home-default-390x844.layout.json',
|
|
392
|
+
' save layout: () => window.__livePassLayout?.["home-default-390x844.png"]',
|
|
393
|
+
].join('\n'));
|
|
394
|
+
expect(prompt).toContain([
|
|
395
|
+
' file: /work/tree/.live-pass/run1/home-offline-390x844.png',
|
|
396
|
+
' capture: /work/tree/.live-pass/run1/home-offline-390x844.capture.js',
|
|
397
|
+
' layout: /work/tree/.live-pass/run1/home-offline-390x844.layout.json',
|
|
398
|
+
' save layout: () => window.__livePassLayout?.["home-offline-390x844.png"]',
|
|
399
|
+
].join('\n'));
|
|
400
|
+
expect(prompt).toContain('browser_evaluate with function set to the "save layout:" function printed for that target, copied verbatim, and filename set to the exact absolute path printed as "layout:" for that target. Do not open that file or report on it');
|
|
401
|
+
expect(prompt).not.toContain(LAYOUT_CHECK_SNIPPET);
|
|
402
|
+
expect(prompt).not.toContain(SCROLL_OVERFLOW_SNIPPET);
|
|
403
|
+
expect(prompt).not.toContain('getBoundingClientRect');
|
|
404
|
+
});
|
|
405
|
+
it('asks a maestro pass for no capture file or layout check', () => {
|
|
406
|
+
const prompt = buildLivePassPrompt({ ...base, driver: 'maestro' });
|
|
407
|
+
expect(prompt).not.toContain('capture:');
|
|
408
|
+
expect(prompt).not.toContain('layout:');
|
|
409
|
+
expect(prompt).not.toContain(LAYOUT_CHECK_SNIPPET);
|
|
410
|
+
expect(prompt).not.toContain(SCROLL_OVERFLOW_SNIPPET);
|
|
411
|
+
});
|
|
412
|
+
});
|
|
413
|
+
describe('prescribedTargetFiles', () => {
|
|
414
|
+
it('puts each capture and layout file next to its frame, sharing the frame stem and its collision suffix', () => {
|
|
415
|
+
const targets = [
|
|
416
|
+
{ ...HOME_TARGET, screen: 'settings/profile' },
|
|
417
|
+
{ ...HOME_TARGET, screen: 'settings-profile' },
|
|
418
|
+
];
|
|
419
|
+
const files = prescribedTargetFiles(targets, '/art');
|
|
420
|
+
expect([...files.values()]).toEqual([
|
|
421
|
+
{
|
|
422
|
+
frameName: 'settings-profile-default-390x844.png',
|
|
423
|
+
frame: '/art/settings-profile-default-390x844.png',
|
|
424
|
+
capture: '/art/settings-profile-default-390x844.capture.js',
|
|
425
|
+
layout: '/art/settings-profile-default-390x844.layout.json',
|
|
426
|
+
},
|
|
427
|
+
{
|
|
428
|
+
frameName: 'settings-profile-default-390x844-2.png',
|
|
429
|
+
frame: '/art/settings-profile-default-390x844-2.png',
|
|
430
|
+
capture: '/art/settings-profile-default-390x844-2.capture.js',
|
|
431
|
+
layout: '/art/settings-profile-default-390x844-2.layout.json',
|
|
432
|
+
},
|
|
433
|
+
]);
|
|
434
|
+
});
|
|
435
|
+
});
|
|
436
|
+
// ------------------------------------------------------------ layout checks
|
|
437
|
+
describe('readLayoutCheck', () => {
|
|
438
|
+
const frameName = frameFileName(HOME_TARGET);
|
|
439
|
+
it('accepts a fresh, schema-valid file inside the artifact directory', () => {
|
|
440
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
441
|
+
const report = makeLayoutReport({ found: 1, entries: [SPILL_ENTRY] });
|
|
442
|
+
const file = writeLayoutFile(artifactDir, frameName, report);
|
|
443
|
+
const check = readLayoutCheck(file, artifactDir, Date.now());
|
|
444
|
+
expect(check).toEqual({ file, status: 'OK', report });
|
|
445
|
+
});
|
|
446
|
+
it('reports a file that was never written as MISSING', () => {
|
|
447
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
448
|
+
fs.mkdirSync(artifactDir, { recursive: true });
|
|
449
|
+
const file = path.join(artifactDir, 'home-default-390x844.layout.json');
|
|
450
|
+
expect(readLayoutCheck(file, artifactDir, Date.now())).toEqual({ file, status: 'MISSING', problem: 'not written' });
|
|
451
|
+
});
|
|
452
|
+
it('reports a file left over from an earlier run as STALE', () => {
|
|
453
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
454
|
+
const file = writeLayoutFile(artifactDir, frameName, makeLayoutReport());
|
|
455
|
+
const yesterday = new Date(Date.now() - 24 * 60 * 60 * 1000);
|
|
456
|
+
fs.utimesSync(file, yesterday, yesterday);
|
|
457
|
+
expect(readLayoutCheck(file, artifactDir, Date.now())).toEqual({ file, status: 'STALE', problem: 'predates this run' });
|
|
458
|
+
});
|
|
459
|
+
it('accepts a file written within the freshness tolerance before the run clock', () => {
|
|
460
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
461
|
+
const file = writeLayoutFile(artifactDir, frameName, makeLayoutReport());
|
|
462
|
+
const runStartMs = fs.statSync(file).mtimeMs + LIVE_PASS.FRESHNESS_TOLERANCE_MS - 1;
|
|
463
|
+
expect(readLayoutCheck(file, artifactDir, runStartMs).status).toBe('OK');
|
|
464
|
+
});
|
|
465
|
+
it('reports a prescribed path symlinked out of the artifact directory as OUTSIDE', () => {
|
|
466
|
+
const workDir = makeWorkDir();
|
|
467
|
+
const artifactDir = path.join(workDir, 'frames');
|
|
468
|
+
fs.mkdirSync(artifactDir, { recursive: true });
|
|
469
|
+
const outside = writeLayoutFile(makeWorkDir(), frameName, makeLayoutReport());
|
|
470
|
+
const prescribed = path.join(artifactDir, path.basename(outside));
|
|
471
|
+
fs.symlinkSync(outside, prescribed);
|
|
472
|
+
expect(readLayoutCheck(prescribed, artifactDir, Date.now())).toEqual({
|
|
473
|
+
file: outside,
|
|
474
|
+
status: 'OUTSIDE',
|
|
475
|
+
problem: 'resolves outside the artifact directory',
|
|
476
|
+
});
|
|
477
|
+
});
|
|
478
|
+
it('reports an empty file as MALFORMED', () => {
|
|
479
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
480
|
+
const file = writeLayoutFile(artifactDir, frameName, '');
|
|
481
|
+
expect(readLayoutCheck(file, artifactDir, Date.now())).toEqual({ file, status: 'MALFORMED', problem: 'empty' });
|
|
482
|
+
});
|
|
483
|
+
it('reports a result measured at another width than the frame as MISMATCHED', () => {
|
|
484
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
485
|
+
const file = writeLayoutFile(artifactDir, frameName, makeLayoutReport({ viewport: { width: 1280, height: 800 } }));
|
|
486
|
+
expect(readLayoutCheck(file, artifactDir, Date.now(), 390)).toEqual({
|
|
487
|
+
file,
|
|
488
|
+
status: 'MISMATCHED',
|
|
489
|
+
problem: 'measured at width 1280, but the frame is 390 wide',
|
|
490
|
+
});
|
|
491
|
+
});
|
|
492
|
+
it('accepts a result measured at the frame width whatever the height, since a scrolling frame is resized taller', () => {
|
|
493
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
494
|
+
const report = makeLayoutReport({ viewport: { width: 390, height: 2210 } });
|
|
495
|
+
const file = writeLayoutFile(artifactDir, frameName, report);
|
|
496
|
+
expect(readLayoutCheck(file, artifactDir, Date.now(), 390)).toEqual({ file, status: 'OK', report });
|
|
497
|
+
});
|
|
498
|
+
it('reports a file that is not JSON as MALFORMED', () => {
|
|
499
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
500
|
+
const file = writeLayoutFile(artifactDir, frameName, 'undefined');
|
|
501
|
+
expect(readLayoutCheck(file, artifactDir, Date.now())).toEqual({ file, status: 'MALFORMED', problem: 'not valid JSON' });
|
|
502
|
+
});
|
|
503
|
+
it.each([
|
|
504
|
+
['another check id', { ...makeLayoutReport(), check: 'layout-check/0' }, 'check'],
|
|
505
|
+
['an unknown kind', makeLayoutReport({ found: 1, entries: [{ ...SPILL_ENTRY, kind: 'ugly' }] }), 'entries.0.kind'],
|
|
506
|
+
['text longer than 80 characters', makeLayoutReport({ found: 1, entries: [{ ...SPILL_ENTRY, text: 'x'.repeat(81) }] }), 'entries.0.text'],
|
|
507
|
+
['more than 50 entries', makeLayoutReport({ found: 51, entries: Array(51).fill(SPILL_ENTRY) }), 'entries'],
|
|
508
|
+
['a defect kind in the clamped list', makeLayoutReport({ clamped: [SPILL_ENTRY] }), 'clamped.0.kind'],
|
|
509
|
+
])('reports a result with %s as MALFORMED', (_label, contents, issuePath) => {
|
|
510
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
511
|
+
const file = writeLayoutFile(artifactDir, frameName, contents);
|
|
512
|
+
const check = readLayoutCheck(file, artifactDir, Date.now());
|
|
513
|
+
expect(check.status).toBe('MALFORMED');
|
|
514
|
+
expect(check.report).toBeUndefined();
|
|
515
|
+
expect(check.problem).toContain(`not a ${LIVE_PASS.LAYOUT_CHECK_ID} result (${issuePath}:`);
|
|
516
|
+
});
|
|
517
|
+
});
|
|
518
|
+
describe('formatLayoutSection', () => {
|
|
519
|
+
it('lists each frame\'s entries, its intentional caps apart, a clean frame, and a frame never run', () => {
|
|
520
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
521
|
+
const offline = { ...HOME_TARGET, state: 'offline' };
|
|
522
|
+
const empty = { ...HOME_TARGET, state: 'empty' };
|
|
523
|
+
writeLayoutFile(artifactDir, frameFileName(HOME_TARGET), makeLayoutReport({
|
|
524
|
+
found: 1,
|
|
525
|
+
entries: [SPILL_ENTRY],
|
|
526
|
+
clamped: [{ kind: 'text-clamped', text: 'Costume checklist', testID: null, rect: { x: 0, y: 40, width: 200, height: 40 } }],
|
|
527
|
+
}));
|
|
528
|
+
writeLayoutFile(artifactDir, frameFileName(offline), makeLayoutReport());
|
|
529
|
+
const section = formatLayoutSection(readLayoutChecks([HOME_TARGET, offline, empty], artifactDir, Date.now()));
|
|
530
|
+
expect(section).toBe([
|
|
531
|
+
'## Layout checks (measured in the page)',
|
|
532
|
+
'- home / default / 390x844: 1 entry',
|
|
533
|
+
' - text-spill "Uploaded to the studio wins wall" testID=label at 0,0 320x20',
|
|
534
|
+
' - capped (line clamp / ellipsis): text-clamped "Costume checklist" at 0,40 200x40',
|
|
535
|
+
'- home / offline / 390x844: clean',
|
|
536
|
+
`- home / empty / 390x844: not run (MISSING: ${path.join(artifactDir, 'home-empty-390x844.layout.json')} not written)`,
|
|
537
|
+
].join('\n'));
|
|
538
|
+
});
|
|
539
|
+
it('shows a frame measured at the wrong width as not run, with both widths', () => {
|
|
540
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
541
|
+
const file = writeLayoutFile(artifactDir, frameFileName(HOME_TARGET), makeLayoutReport({
|
|
542
|
+
viewport: { width: 1280, height: 800 }, found: 1, entries: [SPILL_ENTRY],
|
|
543
|
+
}));
|
|
544
|
+
const section = formatLayoutSection(readLayoutChecks([HOME_TARGET], artifactDir, Date.now()));
|
|
545
|
+
expect(section).toBe([
|
|
546
|
+
'## Layout checks (measured in the page)',
|
|
547
|
+
`- home / default / 390x844: not run (MISMATCHED: ${file} measured at width 1280, but the frame is 390 wide)`,
|
|
548
|
+
].join('\n'));
|
|
549
|
+
});
|
|
550
|
+
it('marks a frame a framework dev overlay covered as polluted', () => {
|
|
551
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
552
|
+
const overlay = { kind: 'dev-overlay', text: 'Uncaught Error', testID: null, rect: { x: 0, y: 0, width: 390, height: 844 } };
|
|
553
|
+
writeLayoutFile(artifactDir, frameFileName(HOME_TARGET), makeLayoutReport({ found: 1, entries: [overlay] }));
|
|
554
|
+
const section = formatLayoutSection(readLayoutChecks([HOME_TARGET], artifactDir, Date.now()));
|
|
555
|
+
expect(section).toBe([
|
|
556
|
+
'## Layout checks (measured in the page)',
|
|
557
|
+
'- home / default / 390x844: 1 entry (a framework dev overlay was on screen: this frame is polluted)',
|
|
558
|
+
' - dev-overlay "Uncaught Error" at 0,0 390x844',
|
|
559
|
+
].join('\n'));
|
|
560
|
+
});
|
|
561
|
+
it('says when the snippet found more entries than it listed', () => {
|
|
562
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
563
|
+
writeLayoutFile(artifactDir, frameFileName(HOME_TARGET), makeLayoutReport({ found: 73, entries: [SPILL_ENTRY] }));
|
|
564
|
+
const section = formatLayoutSection(readLayoutChecks([HOME_TARGET], artifactDir, Date.now()));
|
|
565
|
+
expect(section).toContain('- home / default / 390x844: 73 entries (73 found, first 1 listed)');
|
|
566
|
+
});
|
|
567
|
+
});
|
|
568
|
+
// --------------------------------------------------------------- validation
|
|
569
|
+
describe('validateFrames', () => {
|
|
570
|
+
it('accepts a fresh, non-empty frame inside the artifact directory', () => {
|
|
571
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
572
|
+
const file = writeFrameFile(artifactDir, 'home-default.png');
|
|
573
|
+
writeManifest(artifactDir, [{ file }]);
|
|
574
|
+
const validation = validateFrames(makeResult({ frames: [makeFrame({ file })] }), artifactDir, Date.now());
|
|
575
|
+
expect(validation.checks[0].status).toBe('OK');
|
|
576
|
+
expect(validation.okFiles).toEqual([file]);
|
|
577
|
+
expect(validation.problems).toEqual([]);
|
|
578
|
+
});
|
|
579
|
+
it.each(['jpg', 'jpeg'])('accepts a real .%s frame', extension => {
|
|
580
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
581
|
+
const file = writeFrameFile(artifactDir, `home-default.${extension}`);
|
|
582
|
+
writeManifest(artifactDir, [{ file }]);
|
|
583
|
+
const validation = validateFrames(makeResult({ frames: [makeFrame({ file })] }), artifactDir, Date.now());
|
|
584
|
+
expect(validation.checks[0].status).toBe('OK');
|
|
585
|
+
expect(validation.okFiles).toEqual([file]);
|
|
586
|
+
});
|
|
587
|
+
it.each([
|
|
588
|
+
['MISSING', (dir) => path.join(dir, 'never-written.png')],
|
|
589
|
+
['EMPTY', (dir) => {
|
|
590
|
+
const file = path.join(dir, 'empty.png');
|
|
591
|
+
fs.writeFileSync(file, '');
|
|
592
|
+
return file;
|
|
593
|
+
}],
|
|
594
|
+
['STALE', (dir) => {
|
|
595
|
+
const file = writeFrameFile(dir, 'stale.png');
|
|
596
|
+
const yesterday = new Date(Date.now() - 24 * 60 * 60 * 1000);
|
|
597
|
+
fs.utimesSync(file, yesterday, yesterday);
|
|
598
|
+
return file;
|
|
599
|
+
}],
|
|
600
|
+
])('flags a %s frame', (status, makeFile) => {
|
|
601
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
602
|
+
fs.mkdirSync(artifactDir, { recursive: true });
|
|
603
|
+
const file = makeFile(artifactDir);
|
|
604
|
+
writeManifest(artifactDir, [{ file }]);
|
|
605
|
+
const validation = validateFrames(makeResult({ frames: [makeFrame({ file })] }), artifactDir, Date.now());
|
|
606
|
+
expect(validation.checks[0].status).toBe(status);
|
|
607
|
+
expect(validation.okFiles).toEqual([]);
|
|
608
|
+
expect(validation.problems.join('\n')).toContain(status);
|
|
609
|
+
});
|
|
610
|
+
it('marks every frame after the first that reuses an evidence file as DUPLICATE', () => {
|
|
611
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
612
|
+
const file = writeFrameFile(artifactDir, 'home-default-390x844.png');
|
|
613
|
+
writeManifest(artifactDir, [{ file }, { file }]);
|
|
614
|
+
const wide = { ...HOME_TARGET, viewport: '1280x800' };
|
|
615
|
+
const validation = validateFrames(makeResult({ frames: [makeFrame({ file }), makeFrame({ file, viewport: '1280x800' })] }), artifactDir, Date.now(), [HOME_TARGET, wide]);
|
|
616
|
+
expect(validation.checks.map(check => check.status)).toEqual(['OK', 'DUPLICATE']);
|
|
617
|
+
expect(validation.okFiles).toEqual([file]);
|
|
618
|
+
expect(validation.problems.join('\n')).toContain('reused evidence file');
|
|
619
|
+
});
|
|
620
|
+
it('notes a frame saved under a name other than the one prescribed, without failing it', () => {
|
|
621
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
622
|
+
const file = writeFrameFile(artifactDir, 'whatever-i-felt-like.png');
|
|
623
|
+
writeManifest(artifactDir, [{ file }]);
|
|
624
|
+
const validation = validateFrames(makeResult({ frames: [makeFrame({ file })] }), artifactDir, Date.now(), [HOME_TARGET]);
|
|
625
|
+
expect(validation.checks[0].status).toBe('OK');
|
|
626
|
+
expect(validation.checks[0].misnamed).toBe(true);
|
|
627
|
+
expect(validation.problems).toEqual([]);
|
|
628
|
+
expect(validation.notes.join('\n')).toContain('MISNAMED');
|
|
629
|
+
expect(validation.notes.join('\n')).toContain(frameFileName(HOME_TARGET));
|
|
630
|
+
});
|
|
631
|
+
it.each([
|
|
632
|
+
['a capture file', 'home-default-390x844.capture.js'],
|
|
633
|
+
['a layout file', 'home-default-390x844.layout.json'],
|
|
634
|
+
['text with a PNG extension', 'home-default-390x844.png'],
|
|
635
|
+
['text with a JPEG extension', 'home-default-390x844.jpg'],
|
|
636
|
+
])('rejects %s reported as a frame as NOT_IMAGE, and fails the pass', (_label, name) => {
|
|
637
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
638
|
+
const file = writeFrameFile(artifactDir, name, 'not an image');
|
|
639
|
+
writeManifest(artifactDir, [{ file }]);
|
|
640
|
+
const result = makeResult({ frames: [makeFrame({ file })] });
|
|
641
|
+
const validation = validateFrames(result, artifactDir, Date.now(), [HOME_TARGET]);
|
|
642
|
+
const decision = deriveVerdict({ result, validation, coverage: CLEAN_COVERAGE, deniedActions: '' });
|
|
643
|
+
expect(validation.checks[0].status).toBe('NOT_IMAGE');
|
|
644
|
+
expect(validation.okFiles).toEqual([]);
|
|
645
|
+
expect(validation.problems).toEqual([`NOT_IMAGE: ${file} (home / default)`]);
|
|
646
|
+
expect(decision.verdict).toBe('FAIL');
|
|
647
|
+
expect(decision.reasons).toEqual([`NOT_IMAGE: ${file} (home / default)`]);
|
|
648
|
+
});
|
|
649
|
+
it('rejects JPEG bytes renamed to .png', () => {
|
|
650
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
651
|
+
const file = writeFrameFile(artifactDir, 'home-default-390x844.png', JPEG);
|
|
652
|
+
writeManifest(artifactDir, [{ file }]);
|
|
653
|
+
const validation = validateFrames(makeResult({ frames: [makeFrame({ file })] }), artifactDir, Date.now());
|
|
654
|
+
expect(validation.checks[0].status).toBe('NOT_IMAGE');
|
|
655
|
+
expect(validation.okFiles).toEqual([]);
|
|
656
|
+
});
|
|
657
|
+
it('treats a directory reported as a frame as MISSING', () => {
|
|
658
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
659
|
+
const file = path.join(artifactDir, 'home-default.png');
|
|
660
|
+
fs.mkdirSync(file, { recursive: true });
|
|
661
|
+
writeManifest(artifactDir, [{ file }]);
|
|
662
|
+
const validation = validateFrames(makeResult({ frames: [makeFrame({ file })] }), artifactDir, Date.now());
|
|
663
|
+
expect(validation.checks[0].status).toBe('MISSING');
|
|
664
|
+
expect(validation.okFiles).toEqual([]);
|
|
665
|
+
});
|
|
666
|
+
it('flags a frame written outside the artifact directory', () => {
|
|
667
|
+
const workDir = makeWorkDir();
|
|
668
|
+
const artifactDir = path.join(workDir, 'frames');
|
|
669
|
+
fs.mkdirSync(artifactDir, { recursive: true });
|
|
670
|
+
const outside = writeFrameFile(workDir, 'escaped.png');
|
|
671
|
+
writeManifest(artifactDir, [{ file: outside }]);
|
|
672
|
+
const validation = validateFrames(makeResult({ frames: [makeFrame({ file: outside })] }), artifactDir, Date.now());
|
|
673
|
+
expect(validation.checks[0].status).toBe('OUTSIDE');
|
|
674
|
+
});
|
|
675
|
+
it('reports a missing manifest and captures that were never reported', () => {
|
|
676
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
677
|
+
const reported = writeFrameFile(artifactDir, 'home-default.png');
|
|
678
|
+
writeFrameFile(artifactDir, 'orphan.png');
|
|
679
|
+
const validation = validateFrames(makeResult({ frames: [makeFrame({ file: reported })] }), artifactDir, Date.now());
|
|
680
|
+
expect(validation.problems.join('\n')).toContain('was not written');
|
|
681
|
+
expect(validation.unreportedCaptures).toEqual([path.join(artifactDir, 'orphan.png')]);
|
|
682
|
+
});
|
|
683
|
+
it('rejects a manifest left over from an earlier run in a reused artifact directory', () => {
|
|
684
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
685
|
+
const file = writeFrameFile(artifactDir, 'home-default-390x844.png');
|
|
686
|
+
writeManifest(artifactDir, [{ file }]);
|
|
687
|
+
const yesterday = new Date(Date.now() - 24 * 60 * 60 * 1000);
|
|
688
|
+
fs.utimesSync(path.join(artifactDir, LIVE_PASS.MANIFEST_FILE), yesterday, yesterday);
|
|
689
|
+
const validation = validateFrames(makeResult({ frames: [makeFrame({ file })] }), artifactDir, Date.now(), [HOME_TARGET]);
|
|
690
|
+
expect(validation.problems.join('\n')).toContain('predates this run');
|
|
691
|
+
});
|
|
692
|
+
it('reports a manifest whose frame count disagrees with the answer', () => {
|
|
693
|
+
const artifactDir = path.join(makeWorkDir(), 'frames');
|
|
694
|
+
const file = writeFrameFile(artifactDir, 'home-default.png');
|
|
695
|
+
writeManifest(artifactDir, []);
|
|
696
|
+
const validation = validateFrames(makeResult({ frames: [makeFrame({ file })] }), artifactDir, Date.now());
|
|
697
|
+
expect(validation.problems.join('\n')).toContain('lists 0 frames');
|
|
698
|
+
});
|
|
699
|
+
});
|
|
700
|
+
describe('checkCoverage', () => {
|
|
701
|
+
it('matches a target to its frame exactly', () => {
|
|
702
|
+
const coverage = checkCoverage([HOME_TARGET], makeResult());
|
|
703
|
+
expect(coverage.uncovered).toEqual([]);
|
|
704
|
+
expect(coverage.extraFrames).toEqual([]);
|
|
705
|
+
});
|
|
706
|
+
it('matches despite case and whitespace differences', () => {
|
|
707
|
+
const result = makeResult({
|
|
708
|
+
frames: [makeFrame({ screen: ' HOME ', state: 'Default', viewport: '390 x 844' })],
|
|
709
|
+
});
|
|
710
|
+
expect(checkCoverage([HOME_TARGET], result).uncovered).toEqual([]);
|
|
711
|
+
});
|
|
712
|
+
it('reports a target with no frame as uncovered', () => {
|
|
713
|
+
const missing = { ...HOME_TARGET, screen: 'settings' };
|
|
714
|
+
const coverage = checkCoverage([HOME_TARGET, missing], makeResult());
|
|
715
|
+
expect(coverage.uncovered).toEqual([missing]);
|
|
716
|
+
});
|
|
717
|
+
it('reports a target only present in skipped as skipped', () => {
|
|
718
|
+
const missing = { ...HOME_TARGET, screen: 'settings' };
|
|
719
|
+
const result = makeResult({ skipped: [{ target: 'settings / default', reason: 'route 500s' }] });
|
|
720
|
+
const coverage = checkCoverage([HOME_TARGET, missing], result);
|
|
721
|
+
expect(coverage.uncovered).toEqual([]);
|
|
722
|
+
expect(coverage.skipped).toEqual([{ target: missing, reason: 'route 500s' }]);
|
|
723
|
+
});
|
|
724
|
+
it('keeps a frame that matches no requested target as an extra', () => {
|
|
725
|
+
const result = makeResult({
|
|
726
|
+
frames: [makeFrame(), makeFrame({ screen: 'bonus', file: '/tmp/bonus.png' })],
|
|
727
|
+
});
|
|
728
|
+
expect(checkCoverage([HOME_TARGET], result).extraFrames).toEqual(['/tmp/bonus.png']);
|
|
729
|
+
});
|
|
730
|
+
});
|
|
731
|
+
describe('deriveVerdict', () => {
|
|
732
|
+
it('downgrades a reported PASS when a frame failed', () => {
|
|
733
|
+
const result = makeResult({ frames: [makeFrame({ verdict: 'FAIL' })] });
|
|
734
|
+
const decision = deriveVerdict({ result, validation: CLEAN_VALIDATION, coverage: CLEAN_COVERAGE, deniedActions: '' });
|
|
735
|
+
expect(decision.verdict).toBe('FAIL');
|
|
736
|
+
expect(decision.reported).toBe('PASS');
|
|
737
|
+
expect(decision.reasons.join('\n')).toContain('frame FAIL: home / default');
|
|
738
|
+
});
|
|
739
|
+
it('downgrades a reported PASS when a target was never covered', () => {
|
|
740
|
+
const decision = deriveVerdict({
|
|
741
|
+
result: makeResult(),
|
|
742
|
+
validation: CLEAN_VALIDATION,
|
|
743
|
+
coverage: { ...CLEAN_COVERAGE, uncovered: [{ ...HOME_TARGET, screen: 'settings' }] },
|
|
744
|
+
deniedActions: '',
|
|
745
|
+
});
|
|
746
|
+
expect(decision.verdict).toBe('FAIL');
|
|
747
|
+
expect(decision.reasons.join('\n')).toContain('uncovered target: settings');
|
|
748
|
+
});
|
|
749
|
+
it('downgrades a reported PASS when a frame did not validate', () => {
|
|
750
|
+
const decision = deriveVerdict({
|
|
751
|
+
result: makeResult(),
|
|
752
|
+
validation: { ...CLEAN_VALIDATION, problems: ['STALE: /tmp/frame.png (home / default)'] },
|
|
753
|
+
coverage: CLEAN_COVERAGE,
|
|
754
|
+
deniedActions: '',
|
|
755
|
+
});
|
|
756
|
+
expect(decision.verdict).toBe('FAIL');
|
|
757
|
+
});
|
|
758
|
+
it.each([
|
|
759
|
+
['BLOCKER', 'FAIL'],
|
|
760
|
+
['MAJOR', 'FAIL'],
|
|
761
|
+
['MINOR', 'PASS'],
|
|
762
|
+
])('treats a %s finding as %s', (severity, expected) => {
|
|
763
|
+
const result = makeResult({ findings: [{ severity, screen: 'home', description: 'label clipped' }] });
|
|
764
|
+
const decision = deriveVerdict({ result, validation: CLEAN_VALIDATION, coverage: CLEAN_COVERAGE, deniedActions: '' });
|
|
765
|
+
expect(decision.verdict).toBe(expected);
|
|
766
|
+
});
|
|
767
|
+
it('fails a target Gemini skipped: a reason explains missing evidence, it does not excuse it', () => {
|
|
768
|
+
const decision = deriveVerdict({
|
|
769
|
+
result: makeResult(),
|
|
770
|
+
validation: CLEAN_VALIDATION,
|
|
771
|
+
coverage: {
|
|
772
|
+
...CLEAN_COVERAGE,
|
|
773
|
+
skipped: [{ target: { ...HOME_TARGET, screen: 'settings' }, reason: 'simulator busy' }],
|
|
774
|
+
},
|
|
775
|
+
deniedActions: '',
|
|
776
|
+
});
|
|
777
|
+
expect(decision.verdict).toBe('FAIL');
|
|
778
|
+
expect(decision.reasons.join('\n')).toContain('skipped target: settings / default (simulator busy)');
|
|
779
|
+
});
|
|
780
|
+
it('is BLOCKED whenever Gemini reported BLOCKED', () => {
|
|
781
|
+
const result = makeResult({ verdict: 'BLOCKED', frames: [] });
|
|
782
|
+
const decision = deriveVerdict({ result, validation: CLEAN_VALIDATION, coverage: CLEAN_COVERAGE, deniedActions: '' });
|
|
783
|
+
expect(decision.verdict).toBe('BLOCKED');
|
|
784
|
+
});
|
|
785
|
+
it('is BLOCKED whenever an action was denied, whatever Gemini reported', () => {
|
|
786
|
+
const decision = deriveVerdict({
|
|
787
|
+
result: makeResult(),
|
|
788
|
+
validation: CLEAN_VALIDATION,
|
|
789
|
+
coverage: CLEAN_COVERAGE,
|
|
790
|
+
deniedActions: 'RunCommand (command)',
|
|
791
|
+
});
|
|
792
|
+
expect(decision.verdict).toBe('BLOCKED');
|
|
793
|
+
expect(decision.reasons.join('\n')).toContain('RunCommand (command)');
|
|
794
|
+
});
|
|
795
|
+
it('fails on a regression and passes on resolved or neutral changes', () => {
|
|
796
|
+
const regressed = deriveVerdict({
|
|
797
|
+
result: makeResult({ regressions: [{ severity: 'MAJOR', screen: 'home', description: 'Retry no longer works' }] }),
|
|
798
|
+
validation: CLEAN_VALIDATION,
|
|
799
|
+
coverage: CLEAN_COVERAGE,
|
|
800
|
+
deniedActions: '',
|
|
801
|
+
});
|
|
802
|
+
const improved = deriveVerdict({
|
|
803
|
+
result: makeResult({
|
|
804
|
+
resolved: [{ screen: 'home', description: 'clipped label now wraps' }],
|
|
805
|
+
changes: [{ screen: 'home', description: 'Retry moved below the message' }],
|
|
806
|
+
}),
|
|
807
|
+
validation: CLEAN_VALIDATION,
|
|
808
|
+
coverage: CLEAN_COVERAGE,
|
|
809
|
+
deniedActions: '',
|
|
810
|
+
});
|
|
811
|
+
expect(regressed.verdict).toBe('FAIL');
|
|
812
|
+
expect(improved.verdict).toBe('PASS');
|
|
813
|
+
});
|
|
814
|
+
});
|
|
815
|
+
// --------------------------------------------------------------- publishing
|
|
816
|
+
describe('runPublishCommand', () => {
|
|
817
|
+
it('substitutes shell-quoted file paths', async () => {
|
|
818
|
+
const dir = makeWorkDir();
|
|
819
|
+
const file = writeFrameFile(dir, "it's here.png");
|
|
820
|
+
const result = await runPublishCommand('echo {files}', [file], dir);
|
|
821
|
+
expect(result.ran).toBe(true);
|
|
822
|
+
expect(result.exitCode).toBe(0);
|
|
823
|
+
expect(result.stdout).toBe(file);
|
|
824
|
+
});
|
|
825
|
+
it('does not run without a command or without OK frames', async () => {
|
|
826
|
+
const dir = makeWorkDir();
|
|
827
|
+
expect((await runPublishCommand(undefined, ['/tmp/a.png'], dir)).ran).toBe(false);
|
|
828
|
+
const skipped = await runPublishCommand('echo {files}', [], dir);
|
|
829
|
+
expect(skipped.ran).toBe(false);
|
|
830
|
+
expect(skipped.skippedReason).toContain('no frames validated OK');
|
|
831
|
+
});
|
|
832
|
+
it('does not hang on a command that reads stdin', async () => {
|
|
833
|
+
const dir = makeWorkDir();
|
|
834
|
+
const result = await runPublishCommand('cat; echo done', ['/tmp/a.png'], dir);
|
|
835
|
+
expect(result.ran).toBe(true);
|
|
836
|
+
expect(result.stdout).toBe('done');
|
|
837
|
+
});
|
|
838
|
+
it('reports a non-zero exit instead of throwing', async () => {
|
|
839
|
+
const dir = makeWorkDir();
|
|
840
|
+
const result = await runPublishCommand('echo boom >&2; exit 3', ['/tmp/a.png'], dir);
|
|
841
|
+
expect(result.exitCode).toBe(3);
|
|
842
|
+
expect(result.stderr).toContain('boom');
|
|
843
|
+
});
|
|
844
|
+
});
|
|
845
|
+
// ------------------------------------------------------------------ execute
|
|
846
|
+
describe('live-pass execute', () => {
|
|
847
|
+
it('rejects a working directory that is a root or does not exist', async () => {
|
|
848
|
+
const base = { targets: [HOME_TARGET], brief: 'b', driver: 'playwright' };
|
|
849
|
+
await expect(run({ ...base, workingDirectory: '/' })).rejects.toThrow(ERROR_MESSAGES.INVALID_WORKING_DIRECTORY);
|
|
850
|
+
await expect(run({ ...base, workingDirectory: '/no/such/dir/here' })).rejects.toThrow(ERROR_MESSAGES.INVALID_WORKING_DIRECTORY);
|
|
851
|
+
expect(mockExecuteAgyJson).not.toHaveBeenCalled();
|
|
852
|
+
});
|
|
853
|
+
it.each([
|
|
854
|
+
['outside the working directory', (workDir) => path.join(path.dirname(workDir), 'elsewhere')],
|
|
855
|
+
['equal to the working directory', (workDir) => workDir],
|
|
856
|
+
['relative', () => '.live-pass/run1'],
|
|
857
|
+
])('rejects an artifactDir that is %s', async (_label, makeDir) => {
|
|
858
|
+
const workDir = makeWorkDir();
|
|
859
|
+
await expect(run({
|
|
860
|
+
targets: [HOME_TARGET], brief: 'b', driver: 'playwright',
|
|
861
|
+
workingDirectory: workDir, artifactDir: makeDir(workDir),
|
|
862
|
+
})).rejects.toThrow(ERROR_MESSAGES.LIVE_PASS_ARTIFACT_DIR);
|
|
863
|
+
expect(mockExecuteAgyJson).not.toHaveBeenCalled();
|
|
864
|
+
});
|
|
865
|
+
it('accepts an artifactDir reached through a symlinked spelling of the worktree', async () => {
|
|
866
|
+
const workDir = makeWorkDir();
|
|
867
|
+
const link = path.join(makeWorkDir(), 'link');
|
|
868
|
+
fs.symlinkSync(workDir, link);
|
|
869
|
+
mockExecuteAgyJson.mockResolvedValue(envelope(makeResult()));
|
|
870
|
+
await run({
|
|
871
|
+
targets: [HOME_TARGET], brief: 'b', driver: 'playwright',
|
|
872
|
+
workingDirectory: link, artifactDir: path.join(link, '.live-pass', 'run1'),
|
|
873
|
+
});
|
|
874
|
+
const options = mockExecuteAgyJson.mock.calls[0][1];
|
|
875
|
+
expect(options.cwd).toBe(workDir);
|
|
876
|
+
expect(options.includeDirectories[1]).toBe(path.join(workDir, '.live-pass', 'run1'));
|
|
877
|
+
});
|
|
878
|
+
it('refuses the default artifact directory when .live-pass is a symlink out of the worktree', async () => {
|
|
879
|
+
const workDir = makeWorkDir();
|
|
880
|
+
const outside = makeWorkDir();
|
|
881
|
+
fs.symlinkSync(outside, path.join(workDir, LIVE_PASS.DEFAULT_ARTIFACT_SEGMENT));
|
|
882
|
+
await expect(run({
|
|
883
|
+
targets: [HOME_TARGET], brief: 'b', driver: 'playwright', workingDirectory: workDir,
|
|
884
|
+
})).rejects.toThrow(ERROR_MESSAGES.LIVE_PASS_ARTIFACT_DIR);
|
|
885
|
+
expect(mockExecuteAgyJson).not.toHaveBeenCalled();
|
|
886
|
+
});
|
|
887
|
+
it('reports uninvited edits when the agy call itself fails', async () => {
|
|
888
|
+
const workDir = makeWorkDir();
|
|
889
|
+
mockSnapshotWorkingTree
|
|
890
|
+
.mockReturnValueOnce(cleanSnapshot(workDir))
|
|
891
|
+
.mockReturnValueOnce({
|
|
892
|
+
root: workDir,
|
|
893
|
+
lines: [' M src/app/index.tsx'],
|
|
894
|
+
hashes: new Map(),
|
|
895
|
+
truncated: false,
|
|
896
|
+
});
|
|
897
|
+
mockExecuteAgyJson.mockRejectedValue(new Error('Error: timed out waiting for response'));
|
|
898
|
+
await expect(run({
|
|
899
|
+
targets: [HOME_TARGET], brief: 'b', driver: 'playwright', workingDirectory: workDir,
|
|
900
|
+
})).rejects.toThrow(/timed out[\s\S]*Working tree[\s\S]*src\/app\/index\.tsx/);
|
|
901
|
+
});
|
|
902
|
+
it('defaults the artifact directory to a timestamped folder under the worktree', async () => {
|
|
903
|
+
const workDir = makeWorkDir();
|
|
904
|
+
mockExecuteAgyJson.mockResolvedValue(envelope(makeResult()));
|
|
905
|
+
await run({ targets: [HOME_TARGET], brief: 'b', driver: 'playwright', workingDirectory: workDir });
|
|
906
|
+
const options = mockExecuteAgyJson.mock.calls[0][1];
|
|
907
|
+
const artifactDir = options.includeDirectories[1];
|
|
908
|
+
expect(artifactDir.startsWith(path.join(workDir, LIVE_PASS.DEFAULT_ARTIFACT_SEGMENT))).toBe(true);
|
|
909
|
+
expect(fs.existsSync(artifactDir)).toBe(true);
|
|
910
|
+
});
|
|
911
|
+
it('forwards the agy options the pass depends on', async () => {
|
|
912
|
+
const workDir = makeWorkDir();
|
|
913
|
+
const artifactDir = path.join(workDir, '.live-pass', 'run1');
|
|
914
|
+
mockExecuteAgyJson.mockResolvedValue(envelope(makeResult()));
|
|
915
|
+
await run({
|
|
916
|
+
targets: [HOME_TARGET], brief: 'b', driver: 'playwright',
|
|
917
|
+
workingDirectory: workDir, artifactDir, conversationId: 'conv-abc',
|
|
918
|
+
});
|
|
919
|
+
const [prompt, options] = mockExecuteAgyJson.mock.calls[0];
|
|
920
|
+
expect(prompt).toContain('Delta reporting');
|
|
921
|
+
expect(options).toMatchObject({
|
|
922
|
+
cwd: workDir,
|
|
923
|
+
noCache: true,
|
|
924
|
+
conversationId: 'conv-abc',
|
|
925
|
+
yolo: true,
|
|
926
|
+
model: MODELS.MEDIUM,
|
|
927
|
+
printTimeout: LIVE_PASS.DEFAULT_PRINT_TIMEOUT,
|
|
928
|
+
});
|
|
929
|
+
expect(options.includeDirectories).toEqual([workDir, artifactDir]);
|
|
930
|
+
expect(JSON.parse(options.jsonSchema).properties).toHaveProperty('frames');
|
|
931
|
+
});
|
|
932
|
+
it('reports PASS with frames, PR markdown and no image data', async () => {
|
|
933
|
+
const workDir = makeWorkDir();
|
|
934
|
+
const artifactDir = path.join(workDir, '.live-pass', 'run1');
|
|
935
|
+
let file = '';
|
|
936
|
+
mockExecuteAgyJson.mockImplementation(async () => {
|
|
937
|
+
file = writeFrameFile(artifactDir, 'home-default-390x844.png');
|
|
938
|
+
writeManifest(artifactDir, [{ file }]);
|
|
939
|
+
return envelope(makeResult({ frames: [makeFrame({ file })] }));
|
|
940
|
+
});
|
|
941
|
+
const report = await run({
|
|
942
|
+
targets: [HOME_TARGET], brief: 'b', driver: 'playwright',
|
|
943
|
+
workingDirectory: workDir, artifactDir, publishCommand: 'echo ""',
|
|
944
|
+
});
|
|
945
|
+
expect(report).toContain('[GEMINI_CONVERSATION_ID=conv-abc]');
|
|
946
|
+
expect(report).toContain('# Live pass: PASS');
|
|
947
|
+
expect(report).toContain(file);
|
|
948
|
+
expect(report).toContain('## PR markdown');
|
|
949
|
+
expect(report).toContain(';
|
|
950
|
+
expect(report).not.toMatch(/data:image|base64/);
|
|
951
|
+
expect(report).not.toContain('WARNING');
|
|
952
|
+
});
|
|
953
|
+
it('fails two targets answered by one screenshot, even when Gemini reports PASS', async () => {
|
|
954
|
+
const workDir = makeWorkDir();
|
|
955
|
+
const artifactDir = path.join(workDir, '.live-pass', 'run1');
|
|
956
|
+
const wide = { ...HOME_TARGET, viewport: '1280x800' };
|
|
957
|
+
mockExecuteAgyJson.mockImplementation(async () => {
|
|
958
|
+
const file = writeFrameFile(artifactDir, frameFileName(HOME_TARGET));
|
|
959
|
+
writeManifest(artifactDir, [{ file }, { file }]);
|
|
960
|
+
return envelope(makeResult({
|
|
961
|
+
frames: [makeFrame({ file }), makeFrame({ file, viewport: '1280x800' })],
|
|
962
|
+
}));
|
|
963
|
+
});
|
|
964
|
+
const report = await run({
|
|
965
|
+
targets: [HOME_TARGET, wide], brief: 'b', driver: 'playwright',
|
|
966
|
+
workingDirectory: workDir, artifactDir,
|
|
967
|
+
});
|
|
968
|
+
expect(report).toContain('# Live pass: FAIL (Gemini reported PASS)');
|
|
969
|
+
expect(report).toContain('reused evidence file');
|
|
970
|
+
});
|
|
971
|
+
it('passes the same two targets when each has its own screenshot', async () => {
|
|
972
|
+
const workDir = makeWorkDir();
|
|
973
|
+
const artifactDir = path.join(workDir, '.live-pass', 'run1');
|
|
974
|
+
const wide = { ...HOME_TARGET, viewport: '1280x800' };
|
|
975
|
+
mockExecuteAgyJson.mockImplementation(async () => {
|
|
976
|
+
const mobile = writeFrameFile(artifactDir, frameFileName(HOME_TARGET));
|
|
977
|
+
const desktop = writeFrameFile(artifactDir, frameFileName(wide));
|
|
978
|
+
writeManifest(artifactDir, [{ file: mobile }, { file: desktop }]);
|
|
979
|
+
return envelope(makeResult({
|
|
980
|
+
frames: [makeFrame({ file: mobile }), makeFrame({ file: desktop, viewport: '1280x800' })],
|
|
981
|
+
}));
|
|
982
|
+
});
|
|
983
|
+
const report = await run({
|
|
984
|
+
targets: [HOME_TARGET, wide], brief: 'b', driver: 'playwright',
|
|
985
|
+
workingDirectory: workDir, artifactDir,
|
|
986
|
+
});
|
|
987
|
+
expect(report).toContain('# Live pass: PASS');
|
|
988
|
+
expect(report).not.toContain('reused evidence file');
|
|
989
|
+
expect(report).not.toContain('MISNAMED');
|
|
990
|
+
});
|
|
991
|
+
it('derives FAIL from an uncovered target even when Gemini reports PASS', async () => {
|
|
992
|
+
const workDir = makeWorkDir();
|
|
993
|
+
const artifactDir = path.join(workDir, '.live-pass', 'run1');
|
|
994
|
+
mockExecuteAgyJson.mockImplementation(async () => {
|
|
995
|
+
const file = writeFrameFile(artifactDir, 'home-default-390x844.png');
|
|
996
|
+
writeManifest(artifactDir, [{ file }]);
|
|
997
|
+
return envelope(makeResult({ frames: [makeFrame({ file })] }));
|
|
998
|
+
});
|
|
999
|
+
const report = await run({
|
|
1000
|
+
targets: [HOME_TARGET, { ...HOME_TARGET, screen: 'missing', state: 'not-found', expected: '404 text' }],
|
|
1001
|
+
brief: 'b', driver: 'playwright', workingDirectory: workDir, artifactDir,
|
|
1002
|
+
});
|
|
1003
|
+
expect(report).toContain('# Live pass: FAIL (Gemini reported PASS)');
|
|
1004
|
+
expect(report).toContain('uncovered target: missing / not-found');
|
|
1005
|
+
});
|
|
1006
|
+
it('passes a resumed re-pass whose only earlier defect is now fixed', async () => {
|
|
1007
|
+
const workDir = makeWorkDir();
|
|
1008
|
+
const artifactDir = path.join(workDir, '.live-pass', 'run2');
|
|
1009
|
+
mockExecuteAgyJson.mockImplementation(async () => {
|
|
1010
|
+
const file = writeFrameFile(artifactDir, 'home-default-390x844.png');
|
|
1011
|
+
writeManifest(artifactDir, [{ file }]);
|
|
1012
|
+
return envelope(makeResult({
|
|
1013
|
+
frames: [makeFrame({ file })],
|
|
1014
|
+
resolved: [{ screen: 'home', description: 'the clipped Retry label now wraps' }],
|
|
1015
|
+
}));
|
|
1016
|
+
});
|
|
1017
|
+
const report = await run({
|
|
1018
|
+
targets: [HOME_TARGET], brief: 'b', driver: 'playwright',
|
|
1019
|
+
workingDirectory: workDir, artifactDir, conversationId: 'conv-abc',
|
|
1020
|
+
});
|
|
1021
|
+
expect(report).toContain('# Live pass: PASS');
|
|
1022
|
+
expect(report).toContain('the clipped Retry label now wraps');
|
|
1023
|
+
});
|
|
1024
|
+
it('fails a resumed re-pass that reports a regression', async () => {
|
|
1025
|
+
const workDir = makeWorkDir();
|
|
1026
|
+
const artifactDir = path.join(workDir, '.live-pass', 'run2');
|
|
1027
|
+
mockExecuteAgyJson.mockImplementation(async () => {
|
|
1028
|
+
const file = writeFrameFile(artifactDir, 'home-default-390x844.png');
|
|
1029
|
+
writeManifest(artifactDir, [{ file }]);
|
|
1030
|
+
return envelope(makeResult({
|
|
1031
|
+
frames: [makeFrame({ file })],
|
|
1032
|
+
regressions: [{ severity: 'MAJOR', screen: 'home', description: 'Retry no longer reloads' }],
|
|
1033
|
+
}));
|
|
1034
|
+
});
|
|
1035
|
+
const report = await run({
|
|
1036
|
+
targets: [HOME_TARGET], brief: 'b', driver: 'playwright',
|
|
1037
|
+
workingDirectory: workDir, artifactDir, conversationId: 'conv-abc',
|
|
1038
|
+
});
|
|
1039
|
+
expect(report).toContain('# Live pass: FAIL');
|
|
1040
|
+
expect(report).toContain('Retry no longer reloads');
|
|
1041
|
+
});
|
|
1042
|
+
it('blocks and skips publishing when actions were denied', async () => {
|
|
1043
|
+
const workDir = makeWorkDir();
|
|
1044
|
+
const artifactDir = path.join(workDir, '.live-pass', 'run1');
|
|
1045
|
+
const marker = path.join(workDir, 'published.txt');
|
|
1046
|
+
mockExecuteAgyJson.mockResolvedValue(envelope(makeResult({ verdict: 'BLOCKED', frames: [] }), {
|
|
1047
|
+
denied_actions: [{ action: 'command', display_name: 'RunCommand' }],
|
|
1048
|
+
}));
|
|
1049
|
+
const report = await run({
|
|
1050
|
+
targets: [HOME_TARGET], brief: 'b', driver: 'playwright',
|
|
1051
|
+
workingDirectory: workDir, artifactDir, publishCommand: `touch ${marker}`,
|
|
1052
|
+
});
|
|
1053
|
+
expect(report).toContain('# Live pass: BLOCKED');
|
|
1054
|
+
expect(report).toContain('RunCommand (command)');
|
|
1055
|
+
expect(fs.existsSync(marker)).toBe(false);
|
|
1056
|
+
});
|
|
1057
|
+
it('blocks when the answer does not match the result schema', async () => {
|
|
1058
|
+
const workDir = makeWorkDir();
|
|
1059
|
+
mockExecuteAgyJson.mockResolvedValue(envelope({ verdict: 'MAYBE' }));
|
|
1060
|
+
const report = await run({
|
|
1061
|
+
targets: [HOME_TARGET], brief: 'b', driver: 'playwright', workingDirectory: workDir,
|
|
1062
|
+
});
|
|
1063
|
+
expect(report).toContain('# Live pass: BLOCKED');
|
|
1064
|
+
expect(report).toContain('Schema issues');
|
|
1065
|
+
expect(report).toContain('Raw answer (truncated)');
|
|
1066
|
+
});
|
|
1067
|
+
it('still reports uninvited edits when the answer is off-schema', async () => {
|
|
1068
|
+
const workDir = makeWorkDir();
|
|
1069
|
+
mockSnapshotWorkingTree
|
|
1070
|
+
.mockReturnValueOnce(cleanSnapshot(workDir))
|
|
1071
|
+
.mockReturnValueOnce({ root: workDir, lines: [' M src/app/index.tsx'], hashes: new Map(), truncated: false });
|
|
1072
|
+
mockExecuteAgyJson.mockResolvedValue(envelope({ verdict: 'MAYBE' }));
|
|
1073
|
+
const report = await run({
|
|
1074
|
+
targets: [HOME_TARGET], brief: 'b', driver: 'playwright', workingDirectory: workDir,
|
|
1075
|
+
});
|
|
1076
|
+
expect(report).toContain('# Live pass: BLOCKED');
|
|
1077
|
+
expect(report).toContain('## Working tree');
|
|
1078
|
+
expect(report).toContain('src/app/index.tsx');
|
|
1079
|
+
});
|
|
1080
|
+
it('warns when the working tree changed during the pass', async () => {
|
|
1081
|
+
const workDir = makeWorkDir();
|
|
1082
|
+
const artifactDir = path.join(workDir, '.live-pass', 'run1');
|
|
1083
|
+
mockSnapshotWorkingTree
|
|
1084
|
+
.mockReturnValueOnce(cleanSnapshot(workDir))
|
|
1085
|
+
.mockReturnValueOnce({ root: workDir, lines: [' M src/app/index.tsx'], hashes: new Map(), truncated: false });
|
|
1086
|
+
mockExecuteAgyJson.mockImplementation(async () => {
|
|
1087
|
+
const file = writeFrameFile(artifactDir, 'home-default-390x844.png');
|
|
1088
|
+
writeManifest(artifactDir, [{ file }]);
|
|
1089
|
+
return envelope(makeResult({ frames: [makeFrame({ file })] }));
|
|
1090
|
+
});
|
|
1091
|
+
const report = await run({
|
|
1092
|
+
targets: [HOME_TARGET], brief: 'b', driver: 'playwright',
|
|
1093
|
+
workingDirectory: workDir, artifactDir,
|
|
1094
|
+
});
|
|
1095
|
+
expect(report).toContain('## Working tree');
|
|
1096
|
+
expect(report).toContain('WARNING');
|
|
1097
|
+
expect(report).toContain('src/app/index.tsx');
|
|
1098
|
+
});
|
|
1099
|
+
});
|
|
1100
|
+
describe('live-pass execute: checkout, turns and layout checks', () => {
|
|
1101
|
+
function passWithOneFrame(artifactDir, layout) {
|
|
1102
|
+
mockExecuteAgyJson.mockImplementation(async () => {
|
|
1103
|
+
const file = writeFrameFile(artifactDir, frameFileName(HOME_TARGET));
|
|
1104
|
+
if (layout !== undefined)
|
|
1105
|
+
writeLayoutFile(artifactDir, frameFileName(HOME_TARGET), layout);
|
|
1106
|
+
writeManifest(artifactDir, [{ file }]);
|
|
1107
|
+
return envelope(makeResult({ frames: [makeFrame({ file })] }), { num_turns: 9 });
|
|
1108
|
+
});
|
|
1109
|
+
}
|
|
1110
|
+
it('reads the branch and HEAD of a git working directory into the prompt', async () => {
|
|
1111
|
+
const workDir = makeWorkDir();
|
|
1112
|
+
const git = (...args) => execFileSync('git', args, { cwd: workDir, encoding: 'utf-8' }).trim();
|
|
1113
|
+
git('init', '--quiet', '--initial-branch', 'feat/served');
|
|
1114
|
+
git('-c', 'user.email=t@example.com', '-c', 'user.name=t', 'commit', '--quiet', '--allow-empty', '-m', 'seed');
|
|
1115
|
+
const head = git('rev-parse', 'HEAD');
|
|
1116
|
+
mockExecuteAgyJson.mockResolvedValue(envelope(makeResult()));
|
|
1117
|
+
await run({ targets: [HOME_TARGET], brief: 'b', driver: 'playwright', workingDirectory: workDir });
|
|
1118
|
+
expect(mockExecuteAgyJson.mock.calls[0][0]).toContain(`- Checkout under test: feat/served at ${head}. Read for you, so never run git.`);
|
|
1119
|
+
});
|
|
1120
|
+
it('runs on a working directory that is not a git checkout', async () => {
|
|
1121
|
+
const workDir = makeWorkDir();
|
|
1122
|
+
mockExecuteAgyJson.mockResolvedValue(envelope(makeResult()));
|
|
1123
|
+
await run({ targets: [HOME_TARGET], brief: 'b', driver: 'playwright', workingDirectory: workDir });
|
|
1124
|
+
expect(mockExecuteAgyJson.mock.calls[0][0]).toContain("- Checkout under test: not a git checkout with a commit, so rely on the brief's marker alone.");
|
|
1125
|
+
});
|
|
1126
|
+
it("prints Gemini's model steps per frame, read from agy's store for this conversation, under the usage line", async () => {
|
|
1127
|
+
const workDir = makeWorkDir();
|
|
1128
|
+
const artifactDir = path.join(workDir, '.live-pass', 'run1');
|
|
1129
|
+
passWithOneFrame(artifactDir);
|
|
1130
|
+
mockCountAgyModelSteps.mockResolvedValueOnce(18);
|
|
1131
|
+
const report = await run({ targets: [HOME_TARGET], brief: 'b', driver: 'playwright', workingDirectory: workDir, artifactDir });
|
|
1132
|
+
expect(mockCountAgyModelSteps).toHaveBeenLastCalledWith('conv-abc');
|
|
1133
|
+
expect(report).toMatch(/\nUsage: 9 turns [^\n]*\n\nGemini steps: 18 \(18 per frame, 1 frame\)\n/);
|
|
1134
|
+
});
|
|
1135
|
+
it('prints the quoted link source under a frame that began at a link the user arrives by', async () => {
|
|
1136
|
+
const workDir = makeWorkDir();
|
|
1137
|
+
const artifactDir = path.join(workDir, '.live-pass', 'run1');
|
|
1138
|
+
mockExecuteAgyJson.mockImplementation(async () => {
|
|
1139
|
+
const file = writeFrameFile(artifactDir, frameFileName(HOME_TARGET));
|
|
1140
|
+
writeManifest(artifactDir, [{ file }]);
|
|
1141
|
+
return envelope(makeResult({ frames: [makeFrame({ file, link_source: 'the invite email the studio owner sends' })] }));
|
|
1142
|
+
});
|
|
1143
|
+
const report = await run({ targets: [HOME_TARGET], brief: 'b', driver: 'playwright', workingDirectory: workDir, artifactDir });
|
|
1144
|
+
expect(report).toContain(` file: ${path.join(artifactDir, frameFileName(HOME_TARGET))}\n arrived by link: the invite email the studio owner sends\n expected: heading 'Hello'`);
|
|
1145
|
+
});
|
|
1146
|
+
it('prints no link line under a frame reached by taps', async () => {
|
|
1147
|
+
const workDir = makeWorkDir();
|
|
1148
|
+
const artifactDir = path.join(workDir, '.live-pass', 'run1');
|
|
1149
|
+
passWithOneFrame(artifactDir);
|
|
1150
|
+
const report = await run({ targets: [HOME_TARGET], brief: 'b', driver: 'playwright', workingDirectory: workDir, artifactDir });
|
|
1151
|
+
expect(report).toContain(` file: ${path.join(artifactDir, frameFileName(HOME_TARGET))}\n expected: heading 'Hello'`);
|
|
1152
|
+
expect(report).not.toContain('arrived by link');
|
|
1153
|
+
});
|
|
1154
|
+
it('reports layout defects for the caller to judge without changing a PASS', async () => {
|
|
1155
|
+
const workDir = makeWorkDir();
|
|
1156
|
+
const artifactDir = path.join(workDir, '.live-pass', 'run1');
|
|
1157
|
+
passWithOneFrame(artifactDir, makeLayoutReport({ found: 1, entries: [SPILL_ENTRY] }));
|
|
1158
|
+
const report = await run({ targets: [HOME_TARGET], brief: 'b', driver: 'playwright', workingDirectory: workDir, artifactDir });
|
|
1159
|
+
expect(report).toContain('# Live pass: PASS\n');
|
|
1160
|
+
expect(report).toContain('## Why\n- none');
|
|
1161
|
+
expect(report).toContain('## Layout checks (measured in the page)\n- home / default / 390x844: 1 entry\n - text-spill "Uploaded to the studio wins wall" testID=label at 0,0 320x20');
|
|
1162
|
+
});
|
|
1163
|
+
it.each([
|
|
1164
|
+
['missing', undefined, 'MISSING'],
|
|
1165
|
+
['malformed', '{"check":', 'MALFORMED'],
|
|
1166
|
+
])('shows a %s layout file as not run, and still passes', async (_label, layout, status) => {
|
|
1167
|
+
const workDir = makeWorkDir();
|
|
1168
|
+
const artifactDir = path.join(workDir, '.live-pass', 'run1');
|
|
1169
|
+
passWithOneFrame(artifactDir, layout);
|
|
1170
|
+
const report = await run({ targets: [HOME_TARGET], brief: 'b', driver: 'playwright', workingDirectory: workDir, artifactDir });
|
|
1171
|
+
expect(report).toContain('# Live pass: PASS\n');
|
|
1172
|
+
expect(report).toContain(`- home / default / 390x844: not run (${status}: `);
|
|
1173
|
+
expect(report).not.toContain('layout.json (home');
|
|
1174
|
+
});
|
|
1175
|
+
it('never feeds layout entries into the derived verdict', async () => {
|
|
1176
|
+
const workDir = makeWorkDir();
|
|
1177
|
+
const clean = path.join(workDir, '.live-pass', 'clean');
|
|
1178
|
+
const defective = path.join(workDir, '.live-pass', 'defective');
|
|
1179
|
+
passWithOneFrame(clean, makeLayoutReport());
|
|
1180
|
+
const cleanReport = await run({ targets: [HOME_TARGET], brief: 'b', driver: 'playwright', workingDirectory: workDir, artifactDir: clean });
|
|
1181
|
+
passWithOneFrame(defective, makeLayoutReport({ found: 50, entries: Array(50).fill(SPILL_ENTRY) }));
|
|
1182
|
+
const defectiveReport = await run({ targets: [HOME_TARGET], brief: 'b', driver: 'playwright', workingDirectory: workDir, artifactDir: defective });
|
|
1183
|
+
expect(defectiveReport).toContain('- home / default / 390x844: 50 entries');
|
|
1184
|
+
expect(cleanReport).toContain('- home / default / 390x844: clean');
|
|
1185
|
+
const verdictAndWhy = (report) => report.slice(report.indexOf('# Live pass'), report.indexOf('## Served-branch check'));
|
|
1186
|
+
expect(verdictAndWhy(defectiveReport).replace(defective, clean)).toBe(verdictAndWhy(cleanReport));
|
|
1187
|
+
});
|
|
1188
|
+
it('writes one capture script per web target at its prescribed path before agy runs', async () => {
|
|
1189
|
+
const workDir = makeWorkDir();
|
|
1190
|
+
const artifactDir = path.join(workDir, '.live-pass', 'run1');
|
|
1191
|
+
const wide = { ...HOME_TARGET, viewport: '1280x800' };
|
|
1192
|
+
let writtenBeforeAgy = [];
|
|
1193
|
+
mockExecuteAgyJson.mockImplementation(async () => {
|
|
1194
|
+
writtenBeforeAgy = fs.readdirSync(artifactDir).sort();
|
|
1195
|
+
return envelope(makeResult());
|
|
1196
|
+
});
|
|
1197
|
+
await run({ targets: [HOME_TARGET, wide], brief: 'b', driver: 'playwright', workingDirectory: workDir, artifactDir });
|
|
1198
|
+
expect(writtenBeforeAgy).toEqual(['home-default-1280x800.capture.js', 'home-default-390x844.capture.js']);
|
|
1199
|
+
for (const [stem, viewport] of [['home-default-390x844', { width: 390, height: 844 }], ['home-default-1280x800', { width: 1280, height: 800 }]]) {
|
|
1200
|
+
expect(fs.readFileSync(path.join(artifactDir, `${stem}.capture.js`), 'utf-8')).toBe(buildCaptureScript({
|
|
1201
|
+
viewport, frameFile: path.join(artifactDir, `${stem}.png`), key: `${stem}.png`,
|
|
1202
|
+
}));
|
|
1203
|
+
}
|
|
1204
|
+
});
|
|
1205
|
+
it('replaces a symlink at a capture path with a regular file, leaving the link\'s target untouched', () => {
|
|
1206
|
+
const artifactDir = makeWorkDir();
|
|
1207
|
+
const outside = path.join(makeWorkDir(), 'victim.txt');
|
|
1208
|
+
fs.writeFileSync(outside, 'keep me');
|
|
1209
|
+
const capture = path.join(artifactDir, 'home-default-390x844.capture.js');
|
|
1210
|
+
fs.symlinkSync(outside, capture);
|
|
1211
|
+
writeCaptureScripts([HOME_TARGET], artifactDir);
|
|
1212
|
+
expect(fs.readFileSync(outside, 'utf-8')).toBe('keep me');
|
|
1213
|
+
expect(fs.lstatSync(capture).isFile()).toBe(true);
|
|
1214
|
+
});
|
|
1215
|
+
it('writes no capture script for a maestro pass', async () => {
|
|
1216
|
+
const workDir = makeWorkDir();
|
|
1217
|
+
const artifactDir = path.join(workDir, '.live-pass', 'run1');
|
|
1218
|
+
mockExecuteAgyJson.mockResolvedValue(envelope(makeResult()));
|
|
1219
|
+
await run({ targets: [HOME_TARGET], brief: 'b', driver: 'maestro', workingDirectory: workDir, artifactDir });
|
|
1220
|
+
expect(fs.readdirSync(artifactDir)).toEqual([]);
|
|
1221
|
+
});
|
|
1222
|
+
it('refuses to write a capture script for a viewport that is not WxH', () => {
|
|
1223
|
+
const artifactDir = makeWorkDir();
|
|
1224
|
+
expect(() => writeCaptureScripts([{ ...HOME_TARGET, viewport: 'iPhone 15' }], artifactDir))
|
|
1225
|
+
.toThrow(`${ERROR_MESSAGES.LIVE_PASS_WEB_VIEWPORT}. Received 'iPhone 15'.`);
|
|
1226
|
+
expect(fs.readdirSync(artifactDir)).toEqual([]);
|
|
1227
|
+
});
|
|
1228
|
+
it('adds no layout section to a maestro pass', async () => {
|
|
1229
|
+
const workDir = makeWorkDir();
|
|
1230
|
+
const artifactDir = path.join(workDir, '.live-pass', 'run1');
|
|
1231
|
+
passWithOneFrame(artifactDir);
|
|
1232
|
+
const report = await run({ targets: [HOME_TARGET], brief: 'b', driver: 'maestro', workingDirectory: workDir, artifactDir });
|
|
1233
|
+
expect(report).not.toContain('Layout checks');
|
|
1234
|
+
});
|
|
1235
|
+
});
|
|
1236
|
+
describe('livePassResultSchema', () => {
|
|
1237
|
+
it('accepts a fresh pass with empty delta arrays', () => {
|
|
1238
|
+
expect(livePassResultSchema.safeParse(makeResult()).success).toBe(true);
|
|
1239
|
+
});
|
|
1240
|
+
it('accepts a frame with a link_source and one without', () => {
|
|
1241
|
+
const result = makeResult({ frames: [makeFrame(), makeFrame({ file: '/tmp/invite.png', link_source: 'the share link a teammate texts' })] });
|
|
1242
|
+
expect(livePassResultSchema.safeParse(result).success).toBe(true);
|
|
1243
|
+
});
|
|
1244
|
+
it('rejects a frame verdict outside PASS/FAIL', () => {
|
|
1245
|
+
const result = makeResult({ frames: [makeFrame({ verdict: 'BLOCKED' })] });
|
|
1246
|
+
expect(livePassResultSchema.safeParse(result).success).toBe(false);
|
|
1247
|
+
});
|
|
1248
|
+
});
|
|
1249
|
+
//# sourceMappingURL=live-pass.test.js.map
|