@storysnap/cli 2.0.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/client.d.ts +34 -1
- package/dist/api/client.js +3 -1
- package/dist/cli/commands/test.d.ts +1 -0
- package/dist/cli/commands/test.js +119 -62
- package/dist/cli/commands/update.d.ts +1 -0
- package/dist/cli/commands/update.js +62 -30
- package/dist/cli/environment-warning.d.ts +17 -0
- package/dist/cli/environment-warning.js +42 -0
- package/dist/cli/index.js +11 -3
- package/dist/cli/review.d.ts +9 -0
- package/dist/cli/review.js +10 -0
- package/dist/cli/select-stories.d.ts +6 -1
- package/dist/cli/select-stories.js +16 -0
- package/dist/cli/verdict.d.ts +17 -0
- package/dist/cli/verdict.js +22 -0
- package/dist/config/schema.d.ts +11 -0
- package/dist/config/schema.js +18 -0
- package/dist/core/capture-environment.d.ts +19 -0
- package/dist/core/capture-environment.js +102 -0
- package/dist/core/changed.js +10 -23
- package/dist/core/compare.d.ts +3 -1
- package/dist/core/compare.js +30 -10
- package/dist/core/git-context.d.ts +25 -0
- package/dist/core/git-context.js +75 -0
- package/dist/core/git.d.ts +11 -0
- package/dist/core/git.js +25 -0
- package/dist/core/hash.d.ts +7 -0
- package/dist/core/hash.js +11 -0
- package/dist/core/modes.d.ts +50 -0
- package/dist/core/modes.js +124 -0
- package/dist/core/screenshot.d.ts +91 -4
- package/dist/core/screenshot.js +266 -47
- package/dist/core/story-parameters.d.ts +48 -0
- package/dist/core/story-parameters.js +122 -0
- package/dist/core/story-render.d.ts +31 -0
- package/dist/core/story-render.js +75 -0
- package/dist/core/storybook.d.ts +16 -2
- package/dist/core/storybook.js +18 -6
- package/dist/core/viewports.d.ts +14 -0
- package/dist/core/viewports.js +27 -0
- package/package.json +9 -10
package/dist/api/client.d.ts
CHANGED
|
@@ -6,6 +6,10 @@ export interface TestRunInput {
|
|
|
6
6
|
storybookUrl?: string;
|
|
7
7
|
branch?: string;
|
|
8
8
|
commit?: string;
|
|
9
|
+
/** The branch `branch` will merge into; its baselines fill in for missing ones. */
|
|
10
|
+
baseBranch?: string;
|
|
11
|
+
/** The commit `branch` forked from `baseBranch` at. */
|
|
12
|
+
mergeBase?: string;
|
|
9
13
|
/** pixelmatch per-pixel colour sensitivity (0-1). */
|
|
10
14
|
threshold?: number;
|
|
11
15
|
/** Percentage of the image allowed to differ before a story fails (0-100). */
|
|
@@ -18,6 +22,16 @@ export interface CreateResultInput {
|
|
|
18
22
|
browser: string;
|
|
19
23
|
viewportWidth: number;
|
|
20
24
|
viewportHeight: number;
|
|
25
|
+
/** The mode it was captured in; omitted without one. */
|
|
26
|
+
mode?: string;
|
|
27
|
+
/** SHA-256 of the PNG, so the server can recognise a screenshot approved on another branch. */
|
|
28
|
+
contentHash?: string;
|
|
29
|
+
/** The story rendered inconsistently while it was captured. */
|
|
30
|
+
flaky?: boolean;
|
|
31
|
+
/** Overrides the run's pixel threshold for this story. */
|
|
32
|
+
threshold?: number;
|
|
33
|
+
/** Overrides the run's tolerance for this story. */
|
|
34
|
+
maxDiffPercent?: number;
|
|
21
35
|
}
|
|
22
36
|
export interface ResultUploadTarget {
|
|
23
37
|
resultId: string;
|
|
@@ -31,6 +45,12 @@ export interface TestRun {
|
|
|
31
45
|
status: 'pending' | 'running' | 'passed' | 'failed' | 'error';
|
|
32
46
|
branch?: string;
|
|
33
47
|
commit?: string;
|
|
48
|
+
baseBranch?: string;
|
|
49
|
+
mergeBase?: string;
|
|
50
|
+
/** The branch review this run is a push of; absent on the default branch. */
|
|
51
|
+
reviewId?: string | null;
|
|
52
|
+
/** Its page in the dashboard, when the server knows where that is. Only on creation. */
|
|
53
|
+
reviewUrl?: string | null;
|
|
34
54
|
totalStories: number;
|
|
35
55
|
passedStories: number;
|
|
36
56
|
failedStories: number;
|
|
@@ -51,6 +71,8 @@ export interface TestResult {
|
|
|
51
71
|
browser: string;
|
|
52
72
|
viewportWidth: number;
|
|
53
73
|
viewportHeight: number;
|
|
74
|
+
/** '' when the story was captured without modes. */
|
|
75
|
+
mode?: string;
|
|
54
76
|
status: 'pending' | 'pass' | 'fail' | 'new' | 'error';
|
|
55
77
|
diffPercentage?: number;
|
|
56
78
|
diffPixels?: number;
|
|
@@ -58,6 +80,11 @@ export interface TestResult {
|
|
|
58
80
|
baselineUrl?: string;
|
|
59
81
|
diffUrl?: string;
|
|
60
82
|
errorMessage?: string;
|
|
83
|
+
flaky?: boolean;
|
|
84
|
+
/** What the screenshot was captured with; null when not recorded. */
|
|
85
|
+
captureEnvironment?: string | null;
|
|
86
|
+
/** What its baseline was captured with; null when not recorded or there is none. */
|
|
87
|
+
baselineCaptureEnvironment?: string | null;
|
|
61
88
|
}
|
|
62
89
|
export interface TestRunWithResults extends TestRun {
|
|
63
90
|
results: TestResult[];
|
|
@@ -70,6 +97,7 @@ export interface BaselineUploadUrl {
|
|
|
70
97
|
export interface Baseline {
|
|
71
98
|
id: string;
|
|
72
99
|
projectId: string;
|
|
100
|
+
branch?: string;
|
|
73
101
|
storyId: string;
|
|
74
102
|
browser: string;
|
|
75
103
|
viewportWidth: number;
|
|
@@ -109,7 +137,7 @@ export declare class ApiClient {
|
|
|
109
137
|
uploadImage(uploadUrl: string, imageBuffer: Buffer): Promise<void>;
|
|
110
138
|
getTestRun(id: string): Promise<TestRunWithResults>;
|
|
111
139
|
listTestRuns(limit?: number, offset?: number): Promise<TestRun[]>;
|
|
112
|
-
getBaselineUploadUrl(storyId: string, browser: string, viewportWidth: number, viewportHeight: number): Promise<BaselineUploadUrl>;
|
|
140
|
+
getBaselineUploadUrl(storyId: string, browser: string, viewportWidth: number, viewportHeight: number, branch?: string, mode?: string): Promise<BaselineUploadUrl>;
|
|
113
141
|
uploadBaseline(uploadUrl: string, imageBuffer: Buffer): Promise<void>;
|
|
114
142
|
/**
|
|
115
143
|
* Points the baseline record at an already-uploaded image. Without this the
|
|
@@ -120,7 +148,12 @@ export declare class ApiClient {
|
|
|
120
148
|
browser: string;
|
|
121
149
|
viewportWidth: number;
|
|
122
150
|
viewportHeight: number;
|
|
151
|
+
/** The mode it was captured in; omitted without one. */
|
|
152
|
+
mode?: string;
|
|
123
153
|
storageKey: string;
|
|
154
|
+
/** Omitted, the server uses the project's default branch. */
|
|
155
|
+
branch?: string;
|
|
156
|
+
contentHash?: string;
|
|
124
157
|
}): Promise<Baseline>;
|
|
125
158
|
listBaselines(limit?: number, offset?: number, storyId?: string, browser?: string): Promise<Baseline[]>;
|
|
126
159
|
approveBaseline(baselineId: string, testResultId?: string): Promise<Baseline>;
|
package/dist/api/client.js
CHANGED
|
@@ -99,12 +99,14 @@ export class ApiClient {
|
|
|
99
99
|
const response = await this.request('GET', `/api/v1/test-runs?limit=${limit}&offset=${offset}`);
|
|
100
100
|
return response.data;
|
|
101
101
|
}
|
|
102
|
-
async getBaselineUploadUrl(storyId, browser, viewportWidth, viewportHeight) {
|
|
102
|
+
async getBaselineUploadUrl(storyId, browser, viewportWidth, viewportHeight, branch, mode) {
|
|
103
103
|
const response = await this.request('POST', '/api/v1/baselines/upload-url', {
|
|
104
104
|
storyId,
|
|
105
105
|
browser,
|
|
106
106
|
viewportWidth,
|
|
107
107
|
viewportHeight,
|
|
108
|
+
branch,
|
|
109
|
+
mode,
|
|
108
110
|
contentType: 'image/png',
|
|
109
111
|
});
|
|
110
112
|
return response.data;
|
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
import { loadConfig } from '../../config/loader.js';
|
|
2
2
|
import { loadCredentials } from '../../config/credentials.js';
|
|
3
|
-
import { captureScreenshots,
|
|
3
|
+
import { captureLabel, captureScreenshots, groupCaptures, parseScreenshotKey } from '../../core/screenshot.js';
|
|
4
4
|
import { mapWithConcurrency } from '../../core/concurrency.js';
|
|
5
|
+
import { detectGitContext, describeGitContext } from '../../core/git-context.js';
|
|
6
|
+
import { hashScreenshot } from '../../core/hash.js';
|
|
5
7
|
import { compareAllScreenshots, DEFAULT_MAX_DIFF_PERCENT } from '../../core/compare.js';
|
|
6
8
|
import { createStorage } from '../../storage/index.js';
|
|
7
9
|
import { createReporter } from '../reporter.js';
|
|
8
|
-
import { selectStories } from '../select-stories.js';
|
|
10
|
+
import { selectStories, withoutSkipped } from '../select-stories.js';
|
|
11
|
+
import { describeReview } from '../review.js';
|
|
12
|
+
import { apiRunVerdict } from '../verdict.js';
|
|
13
|
+
import { reportEnvironmentMismatches } from '../environment-warning.js';
|
|
9
14
|
import { ApiClient } from '../../api/client.js';
|
|
10
15
|
function parseBrowsers(browserArg) {
|
|
11
16
|
if (!browserArg) {
|
|
@@ -69,24 +74,32 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
|
|
|
69
74
|
const client = new ApiClient({ apiUrl, apiToken });
|
|
70
75
|
// Screenshots are captured here, not by the server: the worker has no route
|
|
71
76
|
// to a Storybook running on this machine or on a CI runner.
|
|
72
|
-
const
|
|
73
|
-
if (!
|
|
77
|
+
const selected = await selectStories(reporter, config, options, 'tests');
|
|
78
|
+
if (!selected) {
|
|
74
79
|
return;
|
|
75
80
|
}
|
|
76
|
-
const totalTests = stories.length * config.browsers.length;
|
|
77
81
|
reporter.startSpinner('Capturing screenshots...');
|
|
78
|
-
const screenshots = await captureScreenshots(config,
|
|
82
|
+
const screenshots = await captureScreenshots(config, selected, (completed, total, story, browser) => {
|
|
79
83
|
reporter.progress(completed, total, `${story.title}/${story.name} (${browser})`);
|
|
80
84
|
});
|
|
81
85
|
reporter.succeedSpinner('Screenshots captured');
|
|
86
|
+
const stories = withoutSkipped(reporter, selected, screenshots);
|
|
87
|
+
if (!stories) {
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
// One per capture: a story has one per viewport, a failed one has one.
|
|
91
|
+
const totalTests = screenshots.size;
|
|
92
|
+
const gitContext = await detectGitContext({
|
|
93
|
+
overrides: { branch: options.branch, commit: options.commit, baseBranch: options.baseBranch },
|
|
94
|
+
});
|
|
95
|
+
reporter.info(describeGitContext(gitContext));
|
|
82
96
|
reporter.startSpinner('Creating test run...');
|
|
83
97
|
// Only sent when the config actually declares them. Left undefined the server
|
|
84
98
|
// applies the project's own defaults, which is what makes `PATCH /projects/me`
|
|
85
99
|
// worth having; a config that does declare them overrides for this run.
|
|
86
100
|
const testRun = await client.createTestRun({
|
|
87
101
|
storybookUrl: config.storybookUrl,
|
|
88
|
-
|
|
89
|
-
commit: options.commit,
|
|
102
|
+
...gitContext,
|
|
90
103
|
threshold: config.threshold,
|
|
91
104
|
maxDiffPercent: config.maxDiffPercent,
|
|
92
105
|
});
|
|
@@ -115,22 +128,24 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
|
|
|
115
128
|
});
|
|
116
129
|
reporter.succeedSpinner('Tests complete');
|
|
117
130
|
// Report results
|
|
118
|
-
reportApiResults(reporter, completedTestRun, options, totalTests - uploaded);
|
|
131
|
+
reportApiResults(reporter, completedTestRun, options, totalTests - uploaded, config.viewport, describeReview(testRun));
|
|
119
132
|
}
|
|
120
133
|
/**
|
|
121
134
|
* Registers each captured screenshot with the API and PUTs the PNG to the
|
|
122
135
|
* presigned URL it hands back. Returns how many made it.
|
|
123
136
|
*/
|
|
124
137
|
async function uploadScreenshots(client, testRunId, config, stories, screenshots, reporter) {
|
|
125
|
-
const total =
|
|
138
|
+
const total = screenshots.size;
|
|
126
139
|
let uploaded = 0;
|
|
127
140
|
let attempted = 0;
|
|
128
|
-
//
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
141
|
+
// One task per capture - a story has one per viewport - and two requests
|
|
142
|
+
// per capture, so they share the capture concurrency.
|
|
143
|
+
const captures = groupCaptures(screenshots);
|
|
144
|
+
const tasks = stories.flatMap((story) => config.browsers.flatMap((browser) => (captures.get(`${story.id}::${browser}`) ?? [undefined]).map((screenshot) => ({ story, browser, screenshot }))));
|
|
145
|
+
await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser, screenshot }) => {
|
|
146
|
+
const label = screenshot ? captureLabel({ viewport: { width: screenshot.width, height: screenshot.height }, mode: screenshot.mode }, config.viewport) : '';
|
|
132
147
|
if (!screenshot || screenshot.buffer.length === 0) {
|
|
133
|
-
reporter.warning(`Failed to capture ${story.id} (${browser}): ${screenshot?.error ?? 'no screenshot'}`);
|
|
148
|
+
reporter.warning(`Failed to capture ${story.id} (${browser})${label}: ${screenshot?.error ?? 'no screenshot'}`);
|
|
134
149
|
attempted++;
|
|
135
150
|
return;
|
|
136
151
|
}
|
|
@@ -140,17 +155,23 @@ async function uploadScreenshots(client, testRunId, config, stories, screenshots
|
|
|
140
155
|
storyTitle: story.title,
|
|
141
156
|
storyName: story.name,
|
|
142
157
|
browser,
|
|
143
|
-
viewportWidth:
|
|
144
|
-
viewportHeight:
|
|
158
|
+
viewportWidth: screenshot.width,
|
|
159
|
+
viewportHeight: screenshot.height,
|
|
160
|
+
mode: screenshot.mode,
|
|
161
|
+
contentHash: hashScreenshot(screenshot.buffer),
|
|
162
|
+
flaky: screenshot.flaky,
|
|
163
|
+
// The story's own tolerance, from its parameters; the run's otherwise.
|
|
164
|
+
threshold: screenshot.threshold,
|
|
165
|
+
maxDiffPercent: screenshot.maxDiffPercent,
|
|
145
166
|
});
|
|
146
167
|
await client.uploadImage(target.uploadUrl, screenshot.buffer);
|
|
147
168
|
uploaded++;
|
|
148
169
|
}
|
|
149
170
|
catch (error) {
|
|
150
|
-
reporter.warning(`Failed to upload ${story.id} (${browser}): ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
171
|
+
reporter.warning(`Failed to upload ${story.id} (${browser})${label}: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
151
172
|
}
|
|
152
173
|
attempted++;
|
|
153
|
-
reporter.progress(attempted, total, `${story.title}/${story.name} (${browser})`);
|
|
174
|
+
reporter.progress(attempted, total, `${story.title}/${story.name} (${browser})${label}`);
|
|
154
175
|
});
|
|
155
176
|
return uploaded;
|
|
156
177
|
}
|
|
@@ -163,16 +184,20 @@ async function runLocalMode(reporter, config, options) {
|
|
|
163
184
|
// Create storage
|
|
164
185
|
const storage = createStorage(config.storage);
|
|
165
186
|
// Discover stories
|
|
166
|
-
const
|
|
167
|
-
if (!
|
|
187
|
+
const selected = await selectStories(reporter, config, options, 'tests');
|
|
188
|
+
if (!selected) {
|
|
168
189
|
return;
|
|
169
190
|
}
|
|
170
191
|
// Capture screenshots
|
|
171
192
|
reporter.startSpinner('Capturing screenshots...');
|
|
172
|
-
const screenshots = await captureScreenshots(config,
|
|
193
|
+
const screenshots = await captureScreenshots(config, selected, (completed, total, story, browser) => {
|
|
173
194
|
reporter.progress(completed, total, `${story.title}/${story.name} (${browser})`);
|
|
174
195
|
});
|
|
175
196
|
reporter.succeedSpinner('Screenshots captured');
|
|
197
|
+
const stories = withoutSkipped(reporter, selected, screenshots);
|
|
198
|
+
if (!stories) {
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
176
201
|
// Compare against baselines
|
|
177
202
|
reporter.startSpinner('Comparing against baselines...');
|
|
178
203
|
const results = await compareAllScreenshots(screenshots, {
|
|
@@ -181,9 +206,10 @@ async function runLocalMode(reporter, config, options) {
|
|
|
181
206
|
storage,
|
|
182
207
|
localSnapshotsDir: config.localSnapshotsDir,
|
|
183
208
|
concurrency: config.concurrency,
|
|
209
|
+
viewport: config.viewport,
|
|
184
210
|
}, (completed, total, key) => {
|
|
185
|
-
const { storyId, browser } = parseScreenshotKey(key);
|
|
186
|
-
reporter.progress(completed, total, `${storyId} (${browser})`);
|
|
211
|
+
const { storyId, browser, viewport, mode } = parseScreenshotKey(key);
|
|
212
|
+
reporter.progress(completed, total, `${storyId} (${browser})${captureLabel({ viewport, mode }, config.viewport)}`);
|
|
187
213
|
});
|
|
188
214
|
reporter.succeedSpinner('Comparison complete');
|
|
189
215
|
// Build summary
|
|
@@ -191,9 +217,11 @@ async function runLocalMode(reporter, config, options) {
|
|
|
191
217
|
// Report results
|
|
192
218
|
reporter.header('Results');
|
|
193
219
|
for (const result of summary.results) {
|
|
194
|
-
reportLocalResult(result.story, result.comparison);
|
|
220
|
+
reportLocalResult(result.story, result.comparison, config.viewport);
|
|
195
221
|
}
|
|
196
222
|
reporter.reportSummary(summary, config.maxDiffPercent ?? DEFAULT_MAX_DIFF_PERCENT);
|
|
223
|
+
reportFlakyCount(summary.results.filter((result) => result.comparison.flaky).length);
|
|
224
|
+
reportEnvironmentMismatches(summary.results.map((result) => result.comparison));
|
|
197
225
|
// A failed comparison exits non-zero whether or not --ci was passed.
|
|
198
226
|
//
|
|
199
227
|
// It used to exit 0 without --ci, which meant any pipeline that forgot the
|
|
@@ -208,10 +236,10 @@ async function runLocalMode(reporter, config, options) {
|
|
|
208
236
|
process.exit(1);
|
|
209
237
|
}
|
|
210
238
|
}
|
|
211
|
-
function reportLocalResult(story, comparison) {
|
|
239
|
+
function reportLocalResult(story, comparison, configured) {
|
|
212
240
|
const storyName = `${story.title}/${story.name}`;
|
|
213
241
|
const browserSuffix = comparison.browser !== 'chromium' ? ` (${comparison.browser})` : '';
|
|
214
|
-
const displayName = `${storyName}${browserSuffix}`;
|
|
242
|
+
const displayName = `${storyName}${browserSuffix}${comparison.viewport ? captureLabel({ viewport: comparison.viewport, mode: comparison.mode }, configured) : ''}`;
|
|
215
243
|
switch (comparison.status) {
|
|
216
244
|
case 'pass':
|
|
217
245
|
console.log('\x1b[32m ✓\x1b[0m', `\x1b[2m${displayName}\x1b[0m`);
|
|
@@ -232,13 +260,23 @@ function reportLocalResult(story, comparison) {
|
|
|
232
260
|
}
|
|
233
261
|
break;
|
|
234
262
|
}
|
|
263
|
+
if (comparison.flaky) {
|
|
264
|
+
console.log(FLAKY_NOTE);
|
|
265
|
+
}
|
|
235
266
|
}
|
|
236
|
-
|
|
267
|
+
const FLAKY_NOTE = '\x1b[33m ⚠ flaky: rendered inconsistently while capturing\x1b[0m';
|
|
268
|
+
/** Points at the flaky stories, if any, below a run's summary. */
|
|
269
|
+
function reportFlakyCount(count) {
|
|
270
|
+
if (count > 0) {
|
|
271
|
+
console.log(`\x1b[33m ${count} flaky ${count === 1 ? 'story' : 'stories'}\x1b[0m`, '\x1b[2m- they rendered differently between screenshots or only captured on a retry\x1b[0m');
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
function reportApiResults(reporter, testRun, options, missing, configured, review) {
|
|
237
275
|
reporter.header('Results');
|
|
238
276
|
for (const result of testRun.results) {
|
|
239
277
|
const storyName = result.storyName || result.storyId;
|
|
240
278
|
const browserSuffix = result.browser && result.browser !== 'chromium' ? ` (${result.browser})` : '';
|
|
241
|
-
const displayName = `${storyName}${browserSuffix}`;
|
|
279
|
+
const displayName = `${storyName}${browserSuffix}${captureLabel({ viewport: { width: result.viewportWidth, height: result.viewportHeight }, mode: result.mode }, configured)}`;
|
|
242
280
|
switch (result.status) {
|
|
243
281
|
case 'pass':
|
|
244
282
|
console.log('\x1b[32m ✓\x1b[0m', `\x1b[2m${displayName}\x1b[0m`);
|
|
@@ -265,6 +303,9 @@ function reportApiResults(reporter, testRun, options, missing) {
|
|
|
265
303
|
}
|
|
266
304
|
break;
|
|
267
305
|
}
|
|
306
|
+
if (result.flaky) {
|
|
307
|
+
console.log(FLAKY_NOTE);
|
|
308
|
+
}
|
|
268
309
|
}
|
|
269
310
|
reporter.newLine();
|
|
270
311
|
console.log('\x1b[1mSummary:\x1b[0m');
|
|
@@ -288,13 +329,32 @@ function reportApiResults(reporter, testRun, options, missing) {
|
|
|
288
329
|
parts.push(`\x1b[31m${testRun.errorStories} errors\x1b[0m`);
|
|
289
330
|
}
|
|
290
331
|
console.log(` ${parts.join(', ')} of ${testRun.totalStories} stories`);
|
|
332
|
+
reportFlakyCount(testRun.results.filter((result) => result.flaky).length);
|
|
333
|
+
reportEnvironmentMismatches(testRun.results);
|
|
291
334
|
reporter.newLine();
|
|
292
|
-
if (
|
|
293
|
-
console.log(
|
|
335
|
+
if (review) {
|
|
336
|
+
console.log(`\x1b[1mReview:\x1b[0m ${review}`);
|
|
337
|
+
reporter.newLine();
|
|
338
|
+
}
|
|
339
|
+
const verdict = apiRunVerdict(testRun, missing);
|
|
340
|
+
if (verdict === 'failed' && testRun.failedStories === 0) {
|
|
341
|
+
// Nothing differed; some stories just couldn't be compared, and approving
|
|
342
|
+
// won't help with those.
|
|
343
|
+
console.log('\x1b[31mVisual regression tests failed.\x1b[0m', '\x1b[2mSome stories could not be compared - see the errors above.\x1b[0m');
|
|
344
|
+
}
|
|
345
|
+
else if (verdict === 'failed') {
|
|
346
|
+
console.log('\x1b[31mVisual regression tests failed.\x1b[0m', review
|
|
347
|
+
? '\x1b[2mApprove the changes in the review above, or fix them and push again.\x1b[0m'
|
|
348
|
+
: '\x1b[2mReview changes in the dashboard or run "storysnap update" to update baselines.\x1b[0m');
|
|
294
349
|
}
|
|
295
|
-
else if (
|
|
350
|
+
else if (verdict === 'missing') {
|
|
296
351
|
console.log(`\x1b[31m${missing} screenshot(s) were not captured or uploaded.\x1b[0m`, '\x1b[2mSee the warnings above.\x1b[0m');
|
|
297
352
|
}
|
|
353
|
+
else if (verdict === 'new') {
|
|
354
|
+
console.log('\x1b[33mNew stories detected.\x1b[0m', review
|
|
355
|
+
? '\x1b[2mApprove them in the review above to create their baselines.\x1b[0m'
|
|
356
|
+
: '\x1b[2mApprove them in the dashboard or run "storysnap update" to create baselines.\x1b[0m');
|
|
357
|
+
}
|
|
298
358
|
else {
|
|
299
359
|
console.log('\x1b[32mAll visual regression tests passed!\x1b[0m');
|
|
300
360
|
}
|
|
@@ -311,45 +371,42 @@ function reportApiResults(reporter, testRun, options, missing) {
|
|
|
311
371
|
}
|
|
312
372
|
function buildSummary(stories, browsers, results) {
|
|
313
373
|
const summary = {
|
|
314
|
-
total:
|
|
374
|
+
total: 0,
|
|
315
375
|
passed: 0,
|
|
316
376
|
failed: 0,
|
|
317
377
|
newStories: 0,
|
|
318
378
|
errors: 0,
|
|
319
379
|
results: [],
|
|
320
380
|
};
|
|
381
|
+
// A story has one result per viewport it was captured at.
|
|
382
|
+
const byStory = new Map();
|
|
383
|
+
for (const comparison of results.values()) {
|
|
384
|
+
const key = `${comparison.storyId}::${comparison.browser}`;
|
|
385
|
+
byStory.set(key, [...(byStory.get(key) ?? []), comparison]);
|
|
386
|
+
}
|
|
321
387
|
for (const story of stories) {
|
|
322
388
|
for (const browser of browsers) {
|
|
323
|
-
const
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
summary.
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
summary.failed++;
|
|
344
|
-
break;
|
|
345
|
-
case 'new':
|
|
346
|
-
summary.newStories++;
|
|
347
|
-
break;
|
|
348
|
-
case 'error':
|
|
349
|
-
summary.errors++;
|
|
350
|
-
break;
|
|
389
|
+
const comparisons = byStory.get(`${story.id}::${browser}`) ?? [
|
|
390
|
+
{ storyId: story.id, browser, status: 'error', errorMessage: 'No comparison result' },
|
|
391
|
+
];
|
|
392
|
+
for (const comparison of comparisons) {
|
|
393
|
+
summary.total++;
|
|
394
|
+
switch (comparison.status) {
|
|
395
|
+
case 'pass':
|
|
396
|
+
summary.passed++;
|
|
397
|
+
break;
|
|
398
|
+
case 'fail':
|
|
399
|
+
summary.failed++;
|
|
400
|
+
break;
|
|
401
|
+
case 'new':
|
|
402
|
+
summary.newStories++;
|
|
403
|
+
break;
|
|
404
|
+
case 'error':
|
|
405
|
+
summary.errors++;
|
|
406
|
+
break;
|
|
407
|
+
}
|
|
408
|
+
summary.results.push({ story, comparison });
|
|
351
409
|
}
|
|
352
|
-
summary.results.push({ story, comparison });
|
|
353
410
|
}
|
|
354
411
|
}
|
|
355
412
|
return summary;
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { loadConfig } from '../../config/loader.js';
|
|
2
2
|
import { loadCredentials } from '../../config/credentials.js';
|
|
3
|
-
import {
|
|
4
|
-
import { captureScreenshots,
|
|
3
|
+
import { getCaptureKey } from '../../core/storybook.js';
|
|
4
|
+
import { captureLabel, captureScreenshots, groupCaptures } from '../../core/screenshot.js';
|
|
5
5
|
import { mapWithConcurrency } from '../../core/concurrency.js';
|
|
6
|
+
import { detectGitContext } from '../../core/git-context.js';
|
|
7
|
+
import { hashScreenshot } from '../../core/hash.js';
|
|
6
8
|
import { createStorage } from '../../storage/index.js';
|
|
7
9
|
import { createReporter } from '../reporter.js';
|
|
8
|
-
import { selectStories } from '../select-stories.js';
|
|
10
|
+
import { selectStories, withoutSkipped } from '../select-stories.js';
|
|
9
11
|
import { ApiClient } from '../../api/client.js';
|
|
10
12
|
function parseBrowsers(browserArg) {
|
|
11
13
|
if (!browserArg) {
|
|
@@ -71,35 +73,43 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
|
|
|
71
73
|
reporter.info(`Browsers: ${config.browsers.join(', ')}`);
|
|
72
74
|
reporter.newLine();
|
|
73
75
|
const client = new ApiClient({ apiUrl, apiToken });
|
|
76
|
+
// Baselines land on this branch only; other branches keep theirs.
|
|
77
|
+
const { branch } = await detectGitContext({ overrides: { branch: options.branch } });
|
|
78
|
+
reporter.info(branch ? `Branch: ${branch}` : "Branch: not detected - updating the project's default branch (pass --branch to name one)");
|
|
74
79
|
// Discover stories
|
|
75
|
-
const
|
|
76
|
-
if (!
|
|
80
|
+
const selected = await selectStories(reporter, config, options, 'baselines');
|
|
81
|
+
if (!selected) {
|
|
77
82
|
return;
|
|
78
83
|
}
|
|
79
|
-
const totalTests = stories.length * config.browsers.length;
|
|
80
84
|
// Capture screenshots
|
|
81
85
|
reporter.startSpinner('Capturing screenshots...');
|
|
82
|
-
const screenshots = await captureScreenshots(config,
|
|
86
|
+
const screenshots = await captureScreenshots(config, selected, (completed, total, story, browser) => {
|
|
83
87
|
reporter.progress(completed, total, `${story.title}/${story.name} (${browser})`);
|
|
84
88
|
});
|
|
85
89
|
reporter.succeedSpinner('Screenshots captured');
|
|
90
|
+
const stories = withoutSkipped(reporter, selected, screenshots);
|
|
91
|
+
if (!stories) {
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
// One per capture: a story has one per viewport, a failed one has one.
|
|
95
|
+
const totalTests = screenshots.size;
|
|
86
96
|
// Upload baselines via API
|
|
87
97
|
reporter.startSpinner('Uploading baselines via API...');
|
|
88
98
|
let uploaded = 0;
|
|
89
99
|
const failures = [];
|
|
90
100
|
// Uploads are network round trips, so they share the capture concurrency.
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
const
|
|
101
|
+
const captures = groupCaptures(screenshots);
|
|
102
|
+
const tasks = stories.flatMap((story) => config.browsers.flatMap((browser) => (captures.get(`${story.id}::${browser}`) ?? [undefined]).map((screenshot) => ({ story, browser, screenshot }))));
|
|
103
|
+
await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser, screenshot }) => {
|
|
104
|
+
const label = `${story.title}/${story.name} (${browser})${screenshot ? captureLabel({ viewport: { width: screenshot.width, height: screenshot.height }, mode: screenshot.mode }, config.viewport) : ''}`;
|
|
95
105
|
if (!screenshot || screenshot.buffer.length === 0) {
|
|
96
|
-
failures.push(`${
|
|
106
|
+
failures.push(`${label}: capture failed - ${screenshot?.error ?? 'no screenshot'}`);
|
|
97
107
|
return;
|
|
98
108
|
}
|
|
99
109
|
if (!options.dryRun) {
|
|
100
110
|
try {
|
|
101
111
|
// Get presigned upload URL from API
|
|
102
|
-
const uploadInfo = await client.getBaselineUploadUrl(story.id, browser,
|
|
112
|
+
const uploadInfo = await client.getBaselineUploadUrl(story.id, browser, screenshot.width, screenshot.height, branch, screenshot.mode);
|
|
103
113
|
// Upload directly to S3 using presigned URL
|
|
104
114
|
await client.uploadBaseline(uploadInfo.url, screenshot.buffer);
|
|
105
115
|
// Record the baseline. The upload alone leaves an orphaned object -
|
|
@@ -107,21 +117,25 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
|
|
|
107
117
|
await client.registerBaseline({
|
|
108
118
|
storyId: story.id,
|
|
109
119
|
browser,
|
|
110
|
-
viewportWidth:
|
|
111
|
-
viewportHeight:
|
|
120
|
+
viewportWidth: screenshot.width,
|
|
121
|
+
viewportHeight: screenshot.height,
|
|
122
|
+
mode: screenshot.mode,
|
|
112
123
|
storageKey: uploadInfo.storageKey,
|
|
124
|
+
branch,
|
|
125
|
+
contentHash: hashScreenshot(screenshot.buffer),
|
|
113
126
|
});
|
|
114
127
|
uploaded++;
|
|
115
128
|
}
|
|
116
129
|
catch (error) {
|
|
117
|
-
failures.push(`${
|
|
130
|
+
failures.push(`${label}: upload failed - ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
118
131
|
}
|
|
119
132
|
}
|
|
120
133
|
else {
|
|
121
134
|
uploaded++;
|
|
122
135
|
}
|
|
123
|
-
reporter.progress(uploaded + failures.length, totalTests,
|
|
136
|
+
reporter.progress(uploaded + failures.length, totalTests, label);
|
|
124
137
|
});
|
|
138
|
+
warnFlaky(reporter, screenshots, config.viewport);
|
|
125
139
|
finishUpdate(reporter, uploaded, totalTests, failures, options);
|
|
126
140
|
}
|
|
127
141
|
async function runLocalMode(reporter, config, options) {
|
|
@@ -133,49 +147,67 @@ async function runLocalMode(reporter, config, options) {
|
|
|
133
147
|
// Create storage
|
|
134
148
|
const storage = createStorage(config.storage);
|
|
135
149
|
// Discover stories
|
|
136
|
-
const
|
|
137
|
-
if (!
|
|
150
|
+
const selected = await selectStories(reporter, config, options, 'baselines');
|
|
151
|
+
if (!selected) {
|
|
138
152
|
return;
|
|
139
153
|
}
|
|
140
|
-
const totalTests = stories.length * config.browsers.length;
|
|
141
154
|
// Capture screenshots
|
|
142
155
|
reporter.startSpinner('Capturing screenshots...');
|
|
143
|
-
const screenshots = await captureScreenshots(config,
|
|
156
|
+
const screenshots = await captureScreenshots(config, selected, (completed, total, story, browser) => {
|
|
144
157
|
reporter.progress(completed, total, `${story.title}/${story.name} (${browser})`);
|
|
145
158
|
});
|
|
146
159
|
reporter.succeedSpinner('Screenshots captured');
|
|
160
|
+
const stories = withoutSkipped(reporter, selected, screenshots);
|
|
161
|
+
if (!stories) {
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
// One per capture: a story has one per viewport, a failed one has one.
|
|
165
|
+
const totalTests = screenshots.size;
|
|
147
166
|
// Upload baselines
|
|
148
167
|
reporter.startSpinner('Uploading baselines...');
|
|
149
168
|
let uploaded = 0;
|
|
150
169
|
const failures = [];
|
|
151
170
|
// Writes to S3 are network round trips, so they share the capture
|
|
152
171
|
// concurrency; local writes simply finish sooner.
|
|
153
|
-
const
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
const
|
|
172
|
+
const captures = groupCaptures(screenshots);
|
|
173
|
+
const tasks = stories.flatMap((story) => config.browsers.flatMap((browser) => (captures.get(`${story.id}::${browser}`) ?? [undefined]).map((screenshot) => ({ story, browser, screenshot }))));
|
|
174
|
+
await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser, screenshot }) => {
|
|
175
|
+
const label = `${story.title}/${story.name} (${browser})${screenshot ? captureLabel({ viewport: { width: screenshot.width, height: screenshot.height }, mode: screenshot.mode }, config.viewport) : ''}`;
|
|
157
176
|
if (!screenshot || screenshot.buffer.length === 0) {
|
|
158
|
-
failures.push(`${
|
|
177
|
+
failures.push(`${label}: capture failed - ${screenshot?.error ?? 'no screenshot'}`);
|
|
159
178
|
return;
|
|
160
179
|
}
|
|
161
|
-
//
|
|
162
|
-
const key =
|
|
180
|
+
// Named by story, browser and - when not the configured one - viewport.
|
|
181
|
+
const key = getCaptureKey(screenshot, config.viewport);
|
|
163
182
|
if (!options.dryRun) {
|
|
164
183
|
try {
|
|
165
184
|
await storage.upload(key, screenshot.buffer);
|
|
166
185
|
uploaded++;
|
|
167
186
|
}
|
|
168
187
|
catch (error) {
|
|
169
|
-
failures.push(`${
|
|
188
|
+
failures.push(`${label}: upload failed - ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
170
189
|
}
|
|
171
190
|
}
|
|
172
191
|
else {
|
|
173
192
|
uploaded++;
|
|
174
193
|
}
|
|
175
|
-
reporter.progress(uploaded + failures.length, totalTests,
|
|
194
|
+
reporter.progress(uploaded + failures.length, totalTests, label);
|
|
176
195
|
});
|
|
196
|
+
warnFlaky(reporter, screenshots, config.viewport);
|
|
177
197
|
finishUpdate(reporter, uploaded, totalTests, failures, options);
|
|
178
198
|
}
|
|
199
|
+
/**
|
|
200
|
+
* Names the baselines captured from a story that rendered inconsistently, as
|
|
201
|
+
* the next test run may not reproduce them.
|
|
202
|
+
*/
|
|
203
|
+
function warnFlaky(reporter, screenshots, configured) {
|
|
204
|
+
for (const screenshot of screenshots.values()) {
|
|
205
|
+
if (screenshot.flaky && screenshot.buffer.length > 0) {
|
|
206
|
+
reporter.warning(`${screenshot.storyId} (${screenshot.browser})${captureLabel({ viewport: { width: screenshot.width, height: screenshot.height }, mode: screenshot.mode }, configured)} is flaky: it rendered differently between screenshots ` +
|
|
207
|
+
'or only captured on a retry, so later runs may not match this baseline');
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
}
|
|
179
211
|
/**
|
|
180
212
|
* Reports the outcome and exits non-zero if any baseline was not saved. A run
|
|
181
213
|
* that quietly drops baselines and exits 0 lets a pipeline go green with an
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/** The parts of a result that say where it and its baseline were captured. */
|
|
2
|
+
export interface EnvironmentFields {
|
|
3
|
+
status: string;
|
|
4
|
+
captureEnvironment?: string | null;
|
|
5
|
+
baselineCaptureEnvironment?: string | null;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Lines warning that baselines were captured somewhere other than this run,
|
|
9
|
+
* one per pair of environments, or none. Different machines render fonts and
|
|
10
|
+
* edges differently, so failures in such a run may be nothing but that.
|
|
11
|
+
* Baselines from before environments were recorded say nothing either way
|
|
12
|
+
* and are left out.
|
|
13
|
+
*/
|
|
14
|
+
export declare function describeEnvironmentMismatches(results: EnvironmentFields[]): string[];
|
|
15
|
+
/** Prints the warnings, with what to do about them, if there are any. */
|
|
16
|
+
export declare function reportEnvironmentMismatches(results: EnvironmentFields[]): void;
|
|
17
|
+
//# sourceMappingURL=environment-warning.d.ts.map
|