@storysnap/cli 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/client.d.ts +18 -1
- package/dist/api/client.js +2 -1
- package/dist/cli/commands/test.js +103 -59
- package/dist/cli/commands/update.js +55 -30
- package/dist/cli/environment-warning.d.ts +17 -0
- package/dist/cli/environment-warning.js +42 -0
- package/dist/cli/index.js +5 -1
- package/dist/cli/select-stories.d.ts +6 -1
- package/dist/cli/select-stories.js +16 -0
- package/dist/cli/verdict.d.ts +17 -0
- package/dist/cli/verdict.js +22 -0
- package/dist/config/schema.d.ts +11 -0
- package/dist/config/schema.js +18 -0
- package/dist/core/capture-environment.d.ts +19 -0
- package/dist/core/capture-environment.js +102 -0
- package/dist/core/compare.d.ts +3 -1
- package/dist/core/compare.js +30 -10
- package/dist/core/hash.d.ts +5 -1
- package/dist/core/hash.js +7 -2
- package/dist/core/modes.d.ts +50 -0
- package/dist/core/modes.js +124 -0
- package/dist/core/screenshot.d.ts +91 -4
- package/dist/core/screenshot.js +266 -47
- package/dist/core/story-parameters.d.ts +48 -0
- package/dist/core/story-parameters.js +122 -0
- package/dist/core/story-render.d.ts +31 -0
- package/dist/core/story-render.js +75 -0
- package/dist/core/storybook.d.ts +16 -2
- package/dist/core/storybook.js +18 -6
- package/dist/core/viewports.d.ts +14 -0
- package/dist/core/viewports.js +27 -0
- package/package.json +9 -10
package/dist/api/client.d.ts
CHANGED
|
@@ -22,8 +22,16 @@ export interface CreateResultInput {
|
|
|
22
22
|
browser: string;
|
|
23
23
|
viewportWidth: number;
|
|
24
24
|
viewportHeight: number;
|
|
25
|
+
/** The mode it was captured in; omitted without one. */
|
|
26
|
+
mode?: string;
|
|
25
27
|
/** SHA-256 of the PNG, so the server can recognise a screenshot approved on another branch. */
|
|
26
28
|
contentHash?: string;
|
|
29
|
+
/** The story rendered inconsistently while it was captured. */
|
|
30
|
+
flaky?: boolean;
|
|
31
|
+
/** Overrides the run's pixel threshold for this story. */
|
|
32
|
+
threshold?: number;
|
|
33
|
+
/** Overrides the run's tolerance for this story. */
|
|
34
|
+
maxDiffPercent?: number;
|
|
27
35
|
}
|
|
28
36
|
export interface ResultUploadTarget {
|
|
29
37
|
resultId: string;
|
|
@@ -63,6 +71,8 @@ export interface TestResult {
|
|
|
63
71
|
browser: string;
|
|
64
72
|
viewportWidth: number;
|
|
65
73
|
viewportHeight: number;
|
|
74
|
+
/** '' when the story was captured without modes. */
|
|
75
|
+
mode?: string;
|
|
66
76
|
status: 'pending' | 'pass' | 'fail' | 'new' | 'error';
|
|
67
77
|
diffPercentage?: number;
|
|
68
78
|
diffPixels?: number;
|
|
@@ -70,6 +80,11 @@ export interface TestResult {
|
|
|
70
80
|
baselineUrl?: string;
|
|
71
81
|
diffUrl?: string;
|
|
72
82
|
errorMessage?: string;
|
|
83
|
+
flaky?: boolean;
|
|
84
|
+
/** What the screenshot was captured with; null when not recorded. */
|
|
85
|
+
captureEnvironment?: string | null;
|
|
86
|
+
/** What its baseline was captured with; null when not recorded or there is none. */
|
|
87
|
+
baselineCaptureEnvironment?: string | null;
|
|
73
88
|
}
|
|
74
89
|
export interface TestRunWithResults extends TestRun {
|
|
75
90
|
results: TestResult[];
|
|
@@ -122,7 +137,7 @@ export declare class ApiClient {
|
|
|
122
137
|
uploadImage(uploadUrl: string, imageBuffer: Buffer): Promise<void>;
|
|
123
138
|
getTestRun(id: string): Promise<TestRunWithResults>;
|
|
124
139
|
listTestRuns(limit?: number, offset?: number): Promise<TestRun[]>;
|
|
125
|
-
getBaselineUploadUrl(storyId: string, browser: string, viewportWidth: number, viewportHeight: number, branch?: string): Promise<BaselineUploadUrl>;
|
|
140
|
+
getBaselineUploadUrl(storyId: string, browser: string, viewportWidth: number, viewportHeight: number, branch?: string, mode?: string): Promise<BaselineUploadUrl>;
|
|
126
141
|
uploadBaseline(uploadUrl: string, imageBuffer: Buffer): Promise<void>;
|
|
127
142
|
/**
|
|
128
143
|
* Points the baseline record at an already-uploaded image. Without this the
|
|
@@ -133,6 +148,8 @@ export declare class ApiClient {
|
|
|
133
148
|
browser: string;
|
|
134
149
|
viewportWidth: number;
|
|
135
150
|
viewportHeight: number;
|
|
151
|
+
/** The mode it was captured in; omitted without one. */
|
|
152
|
+
mode?: string;
|
|
136
153
|
storageKey: string;
|
|
137
154
|
/** Omitted, the server uses the project's default branch. */
|
|
138
155
|
branch?: string;
|
package/dist/api/client.js
CHANGED
|
@@ -99,13 +99,14 @@ export class ApiClient {
|
|
|
99
99
|
const response = await this.request('GET', `/api/v1/test-runs?limit=${limit}&offset=${offset}`);
|
|
100
100
|
return response.data;
|
|
101
101
|
}
|
|
102
|
-
async getBaselineUploadUrl(storyId, browser, viewportWidth, viewportHeight, branch) {
|
|
102
|
+
async getBaselineUploadUrl(storyId, browser, viewportWidth, viewportHeight, branch, mode) {
|
|
103
103
|
const response = await this.request('POST', '/api/v1/baselines/upload-url', {
|
|
104
104
|
storyId,
|
|
105
105
|
browser,
|
|
106
106
|
viewportWidth,
|
|
107
107
|
viewportHeight,
|
|
108
108
|
branch,
|
|
109
|
+
mode,
|
|
109
110
|
contentType: 'image/png',
|
|
110
111
|
});
|
|
111
112
|
return response.data;
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
import { loadConfig } from '../../config/loader.js';
|
|
2
2
|
import { loadCredentials } from '../../config/credentials.js';
|
|
3
|
-
import { captureScreenshots,
|
|
3
|
+
import { captureLabel, captureScreenshots, groupCaptures, parseScreenshotKey } from '../../core/screenshot.js';
|
|
4
4
|
import { mapWithConcurrency } from '../../core/concurrency.js';
|
|
5
5
|
import { detectGitContext, describeGitContext } from '../../core/git-context.js';
|
|
6
6
|
import { hashScreenshot } from '../../core/hash.js';
|
|
7
7
|
import { compareAllScreenshots, DEFAULT_MAX_DIFF_PERCENT } from '../../core/compare.js';
|
|
8
8
|
import { createStorage } from '../../storage/index.js';
|
|
9
9
|
import { createReporter } from '../reporter.js';
|
|
10
|
-
import { selectStories } from '../select-stories.js';
|
|
10
|
+
import { selectStories, withoutSkipped } from '../select-stories.js';
|
|
11
11
|
import { describeReview } from '../review.js';
|
|
12
|
+
import { apiRunVerdict } from '../verdict.js';
|
|
13
|
+
import { reportEnvironmentMismatches } from '../environment-warning.js';
|
|
12
14
|
import { ApiClient } from '../../api/client.js';
|
|
13
15
|
function parseBrowsers(browserArg) {
|
|
14
16
|
if (!browserArg) {
|
|
@@ -72,16 +74,21 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
|
|
|
72
74
|
const client = new ApiClient({ apiUrl, apiToken });
|
|
73
75
|
// Screenshots are captured here, not by the server: the worker has no route
|
|
74
76
|
// to a Storybook running on this machine or on a CI runner.
|
|
75
|
-
const
|
|
76
|
-
if (!
|
|
77
|
+
const selected = await selectStories(reporter, config, options, 'tests');
|
|
78
|
+
if (!selected) {
|
|
77
79
|
return;
|
|
78
80
|
}
|
|
79
|
-
const totalTests = stories.length * config.browsers.length;
|
|
80
81
|
reporter.startSpinner('Capturing screenshots...');
|
|
81
|
-
const screenshots = await captureScreenshots(config,
|
|
82
|
+
const screenshots = await captureScreenshots(config, selected, (completed, total, story, browser) => {
|
|
82
83
|
reporter.progress(completed, total, `${story.title}/${story.name} (${browser})`);
|
|
83
84
|
});
|
|
84
85
|
reporter.succeedSpinner('Screenshots captured');
|
|
86
|
+
const stories = withoutSkipped(reporter, selected, screenshots);
|
|
87
|
+
if (!stories) {
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
// One per capture: a story has one per viewport, a failed one has one.
|
|
91
|
+
const totalTests = screenshots.size;
|
|
85
92
|
const gitContext = await detectGitContext({
|
|
86
93
|
overrides: { branch: options.branch, commit: options.commit, baseBranch: options.baseBranch },
|
|
87
94
|
});
|
|
@@ -121,22 +128,24 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
|
|
|
121
128
|
});
|
|
122
129
|
reporter.succeedSpinner('Tests complete');
|
|
123
130
|
// Report results
|
|
124
|
-
reportApiResults(reporter, completedTestRun, options, totalTests - uploaded, describeReview(testRun));
|
|
131
|
+
reportApiResults(reporter, completedTestRun, options, totalTests - uploaded, config.viewport, describeReview(testRun));
|
|
125
132
|
}
|
|
126
133
|
/**
|
|
127
134
|
* Registers each captured screenshot with the API and PUTs the PNG to the
|
|
128
135
|
* presigned URL it hands back. Returns how many made it.
|
|
129
136
|
*/
|
|
130
137
|
async function uploadScreenshots(client, testRunId, config, stories, screenshots, reporter) {
|
|
131
|
-
const total =
|
|
138
|
+
const total = screenshots.size;
|
|
132
139
|
let uploaded = 0;
|
|
133
140
|
let attempted = 0;
|
|
134
|
-
//
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
141
|
+
// One task per capture - a story has one per viewport - and two requests
|
|
142
|
+
// per capture, so they share the capture concurrency.
|
|
143
|
+
const captures = groupCaptures(screenshots);
|
|
144
|
+
const tasks = stories.flatMap((story) => config.browsers.flatMap((browser) => (captures.get(`${story.id}::${browser}`) ?? [undefined]).map((screenshot) => ({ story, browser, screenshot }))));
|
|
145
|
+
await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser, screenshot }) => {
|
|
146
|
+
const label = screenshot ? captureLabel({ viewport: { width: screenshot.width, height: screenshot.height }, mode: screenshot.mode }, config.viewport) : '';
|
|
138
147
|
if (!screenshot || screenshot.buffer.length === 0) {
|
|
139
|
-
reporter.warning(`Failed to capture ${story.id} (${browser}): ${screenshot?.error ?? 'no screenshot'}`);
|
|
148
|
+
reporter.warning(`Failed to capture ${story.id} (${browser})${label}: ${screenshot?.error ?? 'no screenshot'}`);
|
|
140
149
|
attempted++;
|
|
141
150
|
return;
|
|
142
151
|
}
|
|
@@ -146,18 +155,23 @@ async function uploadScreenshots(client, testRunId, config, stories, screenshots
|
|
|
146
155
|
storyTitle: story.title,
|
|
147
156
|
storyName: story.name,
|
|
148
157
|
browser,
|
|
149
|
-
viewportWidth:
|
|
150
|
-
viewportHeight:
|
|
158
|
+
viewportWidth: screenshot.width,
|
|
159
|
+
viewportHeight: screenshot.height,
|
|
160
|
+
mode: screenshot.mode,
|
|
151
161
|
contentHash: hashScreenshot(screenshot.buffer),
|
|
162
|
+
flaky: screenshot.flaky,
|
|
163
|
+
// The story's own tolerance, from its parameters; the run's otherwise.
|
|
164
|
+
threshold: screenshot.threshold,
|
|
165
|
+
maxDiffPercent: screenshot.maxDiffPercent,
|
|
152
166
|
});
|
|
153
167
|
await client.uploadImage(target.uploadUrl, screenshot.buffer);
|
|
154
168
|
uploaded++;
|
|
155
169
|
}
|
|
156
170
|
catch (error) {
|
|
157
|
-
reporter.warning(`Failed to upload ${story.id} (${browser}): ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
171
|
+
reporter.warning(`Failed to upload ${story.id} (${browser})${label}: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
158
172
|
}
|
|
159
173
|
attempted++;
|
|
160
|
-
reporter.progress(attempted, total, `${story.title}/${story.name} (${browser})`);
|
|
174
|
+
reporter.progress(attempted, total, `${story.title}/${story.name} (${browser})${label}`);
|
|
161
175
|
});
|
|
162
176
|
return uploaded;
|
|
163
177
|
}
|
|
@@ -170,16 +184,20 @@ async function runLocalMode(reporter, config, options) {
|
|
|
170
184
|
// Create storage
|
|
171
185
|
const storage = createStorage(config.storage);
|
|
172
186
|
// Discover stories
|
|
173
|
-
const
|
|
174
|
-
if (!
|
|
187
|
+
const selected = await selectStories(reporter, config, options, 'tests');
|
|
188
|
+
if (!selected) {
|
|
175
189
|
return;
|
|
176
190
|
}
|
|
177
191
|
// Capture screenshots
|
|
178
192
|
reporter.startSpinner('Capturing screenshots...');
|
|
179
|
-
const screenshots = await captureScreenshots(config,
|
|
193
|
+
const screenshots = await captureScreenshots(config, selected, (completed, total, story, browser) => {
|
|
180
194
|
reporter.progress(completed, total, `${story.title}/${story.name} (${browser})`);
|
|
181
195
|
});
|
|
182
196
|
reporter.succeedSpinner('Screenshots captured');
|
|
197
|
+
const stories = withoutSkipped(reporter, selected, screenshots);
|
|
198
|
+
if (!stories) {
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
183
201
|
// Compare against baselines
|
|
184
202
|
reporter.startSpinner('Comparing against baselines...');
|
|
185
203
|
const results = await compareAllScreenshots(screenshots, {
|
|
@@ -188,9 +206,10 @@ async function runLocalMode(reporter, config, options) {
|
|
|
188
206
|
storage,
|
|
189
207
|
localSnapshotsDir: config.localSnapshotsDir,
|
|
190
208
|
concurrency: config.concurrency,
|
|
209
|
+
viewport: config.viewport,
|
|
191
210
|
}, (completed, total, key) => {
|
|
192
|
-
const { storyId, browser } = parseScreenshotKey(key);
|
|
193
|
-
reporter.progress(completed, total, `${storyId} (${browser})`);
|
|
211
|
+
const { storyId, browser, viewport, mode } = parseScreenshotKey(key);
|
|
212
|
+
reporter.progress(completed, total, `${storyId} (${browser})${captureLabel({ viewport, mode }, config.viewport)}`);
|
|
194
213
|
});
|
|
195
214
|
reporter.succeedSpinner('Comparison complete');
|
|
196
215
|
// Build summary
|
|
@@ -198,9 +217,11 @@ async function runLocalMode(reporter, config, options) {
|
|
|
198
217
|
// Report results
|
|
199
218
|
reporter.header('Results');
|
|
200
219
|
for (const result of summary.results) {
|
|
201
|
-
reportLocalResult(result.story, result.comparison);
|
|
220
|
+
reportLocalResult(result.story, result.comparison, config.viewport);
|
|
202
221
|
}
|
|
203
222
|
reporter.reportSummary(summary, config.maxDiffPercent ?? DEFAULT_MAX_DIFF_PERCENT);
|
|
223
|
+
reportFlakyCount(summary.results.filter((result) => result.comparison.flaky).length);
|
|
224
|
+
reportEnvironmentMismatches(summary.results.map((result) => result.comparison));
|
|
204
225
|
// A failed comparison exits non-zero whether or not --ci was passed.
|
|
205
226
|
//
|
|
206
227
|
// It used to exit 0 without --ci, which meant any pipeline that forgot the
|
|
@@ -215,10 +236,10 @@ async function runLocalMode(reporter, config, options) {
|
|
|
215
236
|
process.exit(1);
|
|
216
237
|
}
|
|
217
238
|
}
|
|
218
|
-
function reportLocalResult(story, comparison) {
|
|
239
|
+
function reportLocalResult(story, comparison, configured) {
|
|
219
240
|
const storyName = `${story.title}/${story.name}`;
|
|
220
241
|
const browserSuffix = comparison.browser !== 'chromium' ? ` (${comparison.browser})` : '';
|
|
221
|
-
const displayName = `${storyName}${browserSuffix}`;
|
|
242
|
+
const displayName = `${storyName}${browserSuffix}${comparison.viewport ? captureLabel({ viewport: comparison.viewport, mode: comparison.mode }, configured) : ''}`;
|
|
222
243
|
switch (comparison.status) {
|
|
223
244
|
case 'pass':
|
|
224
245
|
console.log('\x1b[32m ✓\x1b[0m', `\x1b[2m${displayName}\x1b[0m`);
|
|
@@ -239,13 +260,23 @@ function reportLocalResult(story, comparison) {
|
|
|
239
260
|
}
|
|
240
261
|
break;
|
|
241
262
|
}
|
|
263
|
+
if (comparison.flaky) {
|
|
264
|
+
console.log(FLAKY_NOTE);
|
|
265
|
+
}
|
|
242
266
|
}
|
|
243
|
-
|
|
267
|
+
const FLAKY_NOTE = '\x1b[33m ⚠ flaky: rendered inconsistently while capturing\x1b[0m';
|
|
268
|
+
/** Points at the flaky stories, if any, below a run's summary. */
|
|
269
|
+
function reportFlakyCount(count) {
|
|
270
|
+
if (count > 0) {
|
|
271
|
+
console.log(`\x1b[33m ${count} flaky ${count === 1 ? 'story' : 'stories'}\x1b[0m`, '\x1b[2m- they rendered differently between screenshots or only captured on a retry\x1b[0m');
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
function reportApiResults(reporter, testRun, options, missing, configured, review) {
|
|
244
275
|
reporter.header('Results');
|
|
245
276
|
for (const result of testRun.results) {
|
|
246
277
|
const storyName = result.storyName || result.storyId;
|
|
247
278
|
const browserSuffix = result.browser && result.browser !== 'chromium' ? ` (${result.browser})` : '';
|
|
248
|
-
const displayName = `${storyName}${browserSuffix}`;
|
|
279
|
+
const displayName = `${storyName}${browserSuffix}${captureLabel({ viewport: { width: result.viewportWidth, height: result.viewportHeight }, mode: result.mode }, configured)}`;
|
|
249
280
|
switch (result.status) {
|
|
250
281
|
case 'pass':
|
|
251
282
|
console.log('\x1b[32m ✓\x1b[0m', `\x1b[2m${displayName}\x1b[0m`);
|
|
@@ -272,6 +303,9 @@ function reportApiResults(reporter, testRun, options, missing, review) {
|
|
|
272
303
|
}
|
|
273
304
|
break;
|
|
274
305
|
}
|
|
306
|
+
if (result.flaky) {
|
|
307
|
+
console.log(FLAKY_NOTE);
|
|
308
|
+
}
|
|
275
309
|
}
|
|
276
310
|
reporter.newLine();
|
|
277
311
|
console.log('\x1b[1mSummary:\x1b[0m');
|
|
@@ -295,19 +329,32 @@ function reportApiResults(reporter, testRun, options, missing, review) {
|
|
|
295
329
|
parts.push(`\x1b[31m${testRun.errorStories} errors\x1b[0m`);
|
|
296
330
|
}
|
|
297
331
|
console.log(` ${parts.join(', ')} of ${testRun.totalStories} stories`);
|
|
332
|
+
reportFlakyCount(testRun.results.filter((result) => result.flaky).length);
|
|
333
|
+
reportEnvironmentMismatches(testRun.results);
|
|
298
334
|
reporter.newLine();
|
|
299
335
|
if (review) {
|
|
300
336
|
console.log(`\x1b[1mReview:\x1b[0m ${review}`);
|
|
301
337
|
reporter.newLine();
|
|
302
338
|
}
|
|
303
|
-
|
|
339
|
+
const verdict = apiRunVerdict(testRun, missing);
|
|
340
|
+
if (verdict === 'failed' && testRun.failedStories === 0) {
|
|
341
|
+
// Nothing differed; some stories just couldn't be compared, and approving
|
|
342
|
+
// won't help with those.
|
|
343
|
+
console.log('\x1b[31mVisual regression tests failed.\x1b[0m', '\x1b[2mSome stories could not be compared - see the errors above.\x1b[0m');
|
|
344
|
+
}
|
|
345
|
+
else if (verdict === 'failed') {
|
|
304
346
|
console.log('\x1b[31mVisual regression tests failed.\x1b[0m', review
|
|
305
347
|
? '\x1b[2mApprove the changes in the review above, or fix them and push again.\x1b[0m'
|
|
306
348
|
: '\x1b[2mReview changes in the dashboard or run "storysnap update" to update baselines.\x1b[0m');
|
|
307
349
|
}
|
|
308
|
-
else if (
|
|
350
|
+
else if (verdict === 'missing') {
|
|
309
351
|
console.log(`\x1b[31m${missing} screenshot(s) were not captured or uploaded.\x1b[0m`, '\x1b[2mSee the warnings above.\x1b[0m');
|
|
310
352
|
}
|
|
353
|
+
else if (verdict === 'new') {
|
|
354
|
+
console.log('\x1b[33mNew stories detected.\x1b[0m', review
|
|
355
|
+
? '\x1b[2mApprove them in the review above to create their baselines.\x1b[0m'
|
|
356
|
+
: '\x1b[2mApprove them in the dashboard or run "storysnap update" to create baselines.\x1b[0m');
|
|
357
|
+
}
|
|
311
358
|
else {
|
|
312
359
|
console.log('\x1b[32mAll visual regression tests passed!\x1b[0m');
|
|
313
360
|
}
|
|
@@ -324,45 +371,42 @@ function reportApiResults(reporter, testRun, options, missing, review) {
|
|
|
324
371
|
}
|
|
325
372
|
function buildSummary(stories, browsers, results) {
|
|
326
373
|
const summary = {
|
|
327
|
-
total:
|
|
374
|
+
total: 0,
|
|
328
375
|
passed: 0,
|
|
329
376
|
failed: 0,
|
|
330
377
|
newStories: 0,
|
|
331
378
|
errors: 0,
|
|
332
379
|
results: [],
|
|
333
380
|
};
|
|
381
|
+
// A story has one result per viewport it was captured at.
|
|
382
|
+
const byStory = new Map();
|
|
383
|
+
for (const comparison of results.values()) {
|
|
384
|
+
const key = `${comparison.storyId}::${comparison.browser}`;
|
|
385
|
+
byStory.set(key, [...(byStory.get(key) ?? []), comparison]);
|
|
386
|
+
}
|
|
334
387
|
for (const story of stories) {
|
|
335
388
|
for (const browser of browsers) {
|
|
336
|
-
const
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
summary.
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
summary.failed++;
|
|
357
|
-
break;
|
|
358
|
-
case 'new':
|
|
359
|
-
summary.newStories++;
|
|
360
|
-
break;
|
|
361
|
-
case 'error':
|
|
362
|
-
summary.errors++;
|
|
363
|
-
break;
|
|
389
|
+
const comparisons = byStory.get(`${story.id}::${browser}`) ?? [
|
|
390
|
+
{ storyId: story.id, browser, status: 'error', errorMessage: 'No comparison result' },
|
|
391
|
+
];
|
|
392
|
+
for (const comparison of comparisons) {
|
|
393
|
+
summary.total++;
|
|
394
|
+
switch (comparison.status) {
|
|
395
|
+
case 'pass':
|
|
396
|
+
summary.passed++;
|
|
397
|
+
break;
|
|
398
|
+
case 'fail':
|
|
399
|
+
summary.failed++;
|
|
400
|
+
break;
|
|
401
|
+
case 'new':
|
|
402
|
+
summary.newStories++;
|
|
403
|
+
break;
|
|
404
|
+
case 'error':
|
|
405
|
+
summary.errors++;
|
|
406
|
+
break;
|
|
407
|
+
}
|
|
408
|
+
summary.results.push({ story, comparison });
|
|
364
409
|
}
|
|
365
|
-
summary.results.push({ story, comparison });
|
|
366
410
|
}
|
|
367
411
|
}
|
|
368
412
|
return summary;
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { loadConfig } from '../../config/loader.js';
|
|
2
2
|
import { loadCredentials } from '../../config/credentials.js';
|
|
3
|
-
import {
|
|
4
|
-
import { captureScreenshots,
|
|
3
|
+
import { getCaptureKey } from '../../core/storybook.js';
|
|
4
|
+
import { captureLabel, captureScreenshots, groupCaptures } from '../../core/screenshot.js';
|
|
5
5
|
import { mapWithConcurrency } from '../../core/concurrency.js';
|
|
6
6
|
import { detectGitContext } from '../../core/git-context.js';
|
|
7
7
|
import { hashScreenshot } from '../../core/hash.js';
|
|
8
8
|
import { createStorage } from '../../storage/index.js';
|
|
9
9
|
import { createReporter } from '../reporter.js';
|
|
10
|
-
import { selectStories } from '../select-stories.js';
|
|
10
|
+
import { selectStories, withoutSkipped } from '../select-stories.js';
|
|
11
11
|
import { ApiClient } from '../../api/client.js';
|
|
12
12
|
function parseBrowsers(browserArg) {
|
|
13
13
|
if (!browserArg) {
|
|
@@ -77,34 +77,39 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
|
|
|
77
77
|
const { branch } = await detectGitContext({ overrides: { branch: options.branch } });
|
|
78
78
|
reporter.info(branch ? `Branch: ${branch}` : "Branch: not detected - updating the project's default branch (pass --branch to name one)");
|
|
79
79
|
// Discover stories
|
|
80
|
-
const
|
|
81
|
-
if (!
|
|
80
|
+
const selected = await selectStories(reporter, config, options, 'baselines');
|
|
81
|
+
if (!selected) {
|
|
82
82
|
return;
|
|
83
83
|
}
|
|
84
|
-
const totalTests = stories.length * config.browsers.length;
|
|
85
84
|
// Capture screenshots
|
|
86
85
|
reporter.startSpinner('Capturing screenshots...');
|
|
87
|
-
const screenshots = await captureScreenshots(config,
|
|
86
|
+
const screenshots = await captureScreenshots(config, selected, (completed, total, story, browser) => {
|
|
88
87
|
reporter.progress(completed, total, `${story.title}/${story.name} (${browser})`);
|
|
89
88
|
});
|
|
90
89
|
reporter.succeedSpinner('Screenshots captured');
|
|
90
|
+
const stories = withoutSkipped(reporter, selected, screenshots);
|
|
91
|
+
if (!stories) {
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
// One per capture: a story has one per viewport, a failed one has one.
|
|
95
|
+
const totalTests = screenshots.size;
|
|
91
96
|
// Upload baselines via API
|
|
92
97
|
reporter.startSpinner('Uploading baselines via API...');
|
|
93
98
|
let uploaded = 0;
|
|
94
99
|
const failures = [];
|
|
95
100
|
// Uploads are network round trips, so they share the capture concurrency.
|
|
96
|
-
const
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
const
|
|
101
|
+
const captures = groupCaptures(screenshots);
|
|
102
|
+
const tasks = stories.flatMap((story) => config.browsers.flatMap((browser) => (captures.get(`${story.id}::${browser}`) ?? [undefined]).map((screenshot) => ({ story, browser, screenshot }))));
|
|
103
|
+
await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser, screenshot }) => {
|
|
104
|
+
const label = `${story.title}/${story.name} (${browser})${screenshot ? captureLabel({ viewport: { width: screenshot.width, height: screenshot.height }, mode: screenshot.mode }, config.viewport) : ''}`;
|
|
100
105
|
if (!screenshot || screenshot.buffer.length === 0) {
|
|
101
|
-
failures.push(`${
|
|
106
|
+
failures.push(`${label}: capture failed - ${screenshot?.error ?? 'no screenshot'}`);
|
|
102
107
|
return;
|
|
103
108
|
}
|
|
104
109
|
if (!options.dryRun) {
|
|
105
110
|
try {
|
|
106
111
|
// Get presigned upload URL from API
|
|
107
|
-
const uploadInfo = await client.getBaselineUploadUrl(story.id, browser,
|
|
112
|
+
const uploadInfo = await client.getBaselineUploadUrl(story.id, browser, screenshot.width, screenshot.height, branch, screenshot.mode);
|
|
108
113
|
// Upload directly to S3 using presigned URL
|
|
109
114
|
await client.uploadBaseline(uploadInfo.url, screenshot.buffer);
|
|
110
115
|
// Record the baseline. The upload alone leaves an orphaned object -
|
|
@@ -112,8 +117,9 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
|
|
|
112
117
|
await client.registerBaseline({
|
|
113
118
|
storyId: story.id,
|
|
114
119
|
browser,
|
|
115
|
-
viewportWidth:
|
|
116
|
-
viewportHeight:
|
|
120
|
+
viewportWidth: screenshot.width,
|
|
121
|
+
viewportHeight: screenshot.height,
|
|
122
|
+
mode: screenshot.mode,
|
|
117
123
|
storageKey: uploadInfo.storageKey,
|
|
118
124
|
branch,
|
|
119
125
|
contentHash: hashScreenshot(screenshot.buffer),
|
|
@@ -121,14 +127,15 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
|
|
|
121
127
|
uploaded++;
|
|
122
128
|
}
|
|
123
129
|
catch (error) {
|
|
124
|
-
failures.push(`${
|
|
130
|
+
failures.push(`${label}: upload failed - ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
125
131
|
}
|
|
126
132
|
}
|
|
127
133
|
else {
|
|
128
134
|
uploaded++;
|
|
129
135
|
}
|
|
130
|
-
reporter.progress(uploaded + failures.length, totalTests,
|
|
136
|
+
reporter.progress(uploaded + failures.length, totalTests, label);
|
|
131
137
|
});
|
|
138
|
+
warnFlaky(reporter, screenshots, config.viewport);
|
|
132
139
|
finishUpdate(reporter, uploaded, totalTests, failures, options);
|
|
133
140
|
}
|
|
134
141
|
async function runLocalMode(reporter, config, options) {
|
|
@@ -140,49 +147,67 @@ async function runLocalMode(reporter, config, options) {
|
|
|
140
147
|
// Create storage
|
|
141
148
|
const storage = createStorage(config.storage);
|
|
142
149
|
// Discover stories
|
|
143
|
-
const
|
|
144
|
-
if (!
|
|
150
|
+
const selected = await selectStories(reporter, config, options, 'baselines');
|
|
151
|
+
if (!selected) {
|
|
145
152
|
return;
|
|
146
153
|
}
|
|
147
|
-
const totalTests = stories.length * config.browsers.length;
|
|
148
154
|
// Capture screenshots
|
|
149
155
|
reporter.startSpinner('Capturing screenshots...');
|
|
150
|
-
const screenshots = await captureScreenshots(config,
|
|
156
|
+
const screenshots = await captureScreenshots(config, selected, (completed, total, story, browser) => {
|
|
151
157
|
reporter.progress(completed, total, `${story.title}/${story.name} (${browser})`);
|
|
152
158
|
});
|
|
153
159
|
reporter.succeedSpinner('Screenshots captured');
|
|
160
|
+
const stories = withoutSkipped(reporter, selected, screenshots);
|
|
161
|
+
if (!stories) {
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
// One per capture: a story has one per viewport, a failed one has one.
|
|
165
|
+
const totalTests = screenshots.size;
|
|
154
166
|
// Upload baselines
|
|
155
167
|
reporter.startSpinner('Uploading baselines...');
|
|
156
168
|
let uploaded = 0;
|
|
157
169
|
const failures = [];
|
|
158
170
|
// Writes to S3 are network round trips, so they share the capture
|
|
159
171
|
// concurrency; local writes simply finish sooner.
|
|
160
|
-
const
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
const
|
|
172
|
+
const captures = groupCaptures(screenshots);
|
|
173
|
+
const tasks = stories.flatMap((story) => config.browsers.flatMap((browser) => (captures.get(`${story.id}::${browser}`) ?? [undefined]).map((screenshot) => ({ story, browser, screenshot }))));
|
|
174
|
+
await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser, screenshot }) => {
|
|
175
|
+
const label = `${story.title}/${story.name} (${browser})${screenshot ? captureLabel({ viewport: { width: screenshot.width, height: screenshot.height }, mode: screenshot.mode }, config.viewport) : ''}`;
|
|
164
176
|
if (!screenshot || screenshot.buffer.length === 0) {
|
|
165
|
-
failures.push(`${
|
|
177
|
+
failures.push(`${label}: capture failed - ${screenshot?.error ?? 'no screenshot'}`);
|
|
166
178
|
return;
|
|
167
179
|
}
|
|
168
|
-
//
|
|
169
|
-
const key =
|
|
180
|
+
// Named by story, browser and - when not the configured one - viewport.
|
|
181
|
+
const key = getCaptureKey(screenshot, config.viewport);
|
|
170
182
|
if (!options.dryRun) {
|
|
171
183
|
try {
|
|
172
184
|
await storage.upload(key, screenshot.buffer);
|
|
173
185
|
uploaded++;
|
|
174
186
|
}
|
|
175
187
|
catch (error) {
|
|
176
|
-
failures.push(`${
|
|
188
|
+
failures.push(`${label}: upload failed - ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
177
189
|
}
|
|
178
190
|
}
|
|
179
191
|
else {
|
|
180
192
|
uploaded++;
|
|
181
193
|
}
|
|
182
|
-
reporter.progress(uploaded + failures.length, totalTests,
|
|
194
|
+
reporter.progress(uploaded + failures.length, totalTests, label);
|
|
183
195
|
});
|
|
196
|
+
warnFlaky(reporter, screenshots, config.viewport);
|
|
184
197
|
finishUpdate(reporter, uploaded, totalTests, failures, options);
|
|
185
198
|
}
|
|
199
|
+
/**
|
|
200
|
+
* Names the baselines captured from a story that rendered inconsistently, as
|
|
201
|
+
* the next test run may not reproduce them.
|
|
202
|
+
*/
|
|
203
|
+
function warnFlaky(reporter, screenshots, configured) {
|
|
204
|
+
for (const screenshot of screenshots.values()) {
|
|
205
|
+
if (screenshot.flaky && screenshot.buffer.length > 0) {
|
|
206
|
+
reporter.warning(`${screenshot.storyId} (${screenshot.browser})${captureLabel({ viewport: { width: screenshot.width, height: screenshot.height }, mode: screenshot.mode }, configured)} is flaky: it rendered differently between screenshots ` +
|
|
207
|
+
'or only captured on a retry, so later runs may not match this baseline');
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
}
|
|
186
211
|
/**
|
|
187
212
|
* Reports the outcome and exits non-zero if any baseline was not saved. A run
|
|
188
213
|
* that quietly drops baselines and exits 0 lets a pipeline go green with an
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/** The parts of a result that say where it and its baseline were captured. */
|
|
2
|
+
export interface EnvironmentFields {
|
|
3
|
+
status: string;
|
|
4
|
+
captureEnvironment?: string | null;
|
|
5
|
+
baselineCaptureEnvironment?: string | null;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Lines warning that baselines were captured somewhere other than this run,
|
|
9
|
+
* one per pair of environments, or none. Different machines render fonts and
|
|
10
|
+
* edges differently, so failures in such a run may be nothing but that.
|
|
11
|
+
* Baselines from before environments were recorded say nothing either way
|
|
12
|
+
* and are left out.
|
|
13
|
+
*/
|
|
14
|
+
export declare function describeEnvironmentMismatches(results: EnvironmentFields[]): string[];
|
|
15
|
+
/** Prints the warnings, with what to do about them, if there are any. */
|
|
16
|
+
export declare function reportEnvironmentMismatches(results: EnvironmentFields[]): void;
|
|
17
|
+
//# sourceMappingURL=environment-warning.d.ts.map
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
/**
|
|
3
|
+
* Lines warning that baselines were captured somewhere other than this run,
|
|
4
|
+
* one per pair of environments, or none. Different machines render fonts and
|
|
5
|
+
* edges differently, so failures in such a run may be nothing but that.
|
|
6
|
+
* Baselines from before environments were recorded say nothing either way
|
|
7
|
+
* and are left out.
|
|
8
|
+
*/
|
|
9
|
+
export function describeEnvironmentMismatches(results) {
|
|
10
|
+
const pairs = new Map();
|
|
11
|
+
for (const result of results) {
|
|
12
|
+
const { captureEnvironment: current, baselineCaptureEnvironment: baseline } = result;
|
|
13
|
+
if (!current || !baseline || current === baseline) {
|
|
14
|
+
continue;
|
|
15
|
+
}
|
|
16
|
+
const key = `${baseline}\n${current}`;
|
|
17
|
+
const pair = pairs.get(key) ?? { baseline, current, count: 0, failed: 0 };
|
|
18
|
+
pair.count++;
|
|
19
|
+
if (result.status === 'fail')
|
|
20
|
+
pair.failed++;
|
|
21
|
+
pairs.set(key, pair);
|
|
22
|
+
}
|
|
23
|
+
return [...pairs.values()].map(({ baseline, current, count, failed }) => {
|
|
24
|
+
const subject = `${count} ${count === 1 ? 'baseline was' : 'baselines were'} captured with ${baseline}`;
|
|
25
|
+
const failures = failed > 0 ? ` (${failed} of them failed)` : '';
|
|
26
|
+
return `${subject}, this run with ${current}${failures}`;
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
/** Prints the warnings, with what to do about them, if there are any. */
|
|
30
|
+
export function reportEnvironmentMismatches(results) {
|
|
31
|
+
const lines = describeEnvironmentMismatches(results);
|
|
32
|
+
if (lines.length === 0) {
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
for (const line of lines) {
|
|
36
|
+
console.log(chalk.yellow(` ⚠ ${line}`));
|
|
37
|
+
}
|
|
38
|
+
console.log(chalk.dim(' Screenshots from different browsers or machines can differ in every glyph. Capture baselines\n' +
|
|
39
|
+
' and runs in the same place - the storysnap-capture image does this - see\n' +
|
|
40
|
+
' https://github.com/Kosai106/storysnap/blob/main/docs/capture-image.md'));
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=environment-warning.js.map
|
package/dist/cli/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import { createRequire } from 'node:module';
|
|
2
3
|
import { Command, InvalidArgumentError } from 'commander';
|
|
3
4
|
import { testCommand } from './commands/test.js';
|
|
4
5
|
import { updateCommand } from './commands/update.js';
|
|
@@ -13,11 +14,14 @@ function parseConcurrency(value) {
|
|
|
13
14
|
}
|
|
14
15
|
return n;
|
|
15
16
|
}
|
|
17
|
+
// From dist/cli/index.js - and src/cli/index.ts under tests - the package's
|
|
18
|
+
// own package.json is two levels up; release-please keeps its version current.
|
|
19
|
+
const { version } = createRequire(import.meta.url)('../../package.json');
|
|
16
20
|
const program = new Command();
|
|
17
21
|
program
|
|
18
22
|
.name('storysnap')
|
|
19
23
|
.description('Visual regression testing tool for Storybook')
|
|
20
|
-
.version(
|
|
24
|
+
.version(version);
|
|
21
25
|
program
|
|
22
26
|
.command('init')
|
|
23
27
|
.description('Initialize StorySnap configuration')
|