@storysnap/cli 2.1.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -22,8 +22,16 @@ export interface CreateResultInput {
22
22
  browser: string;
23
23
  viewportWidth: number;
24
24
  viewportHeight: number;
25
+ /** The mode it was captured in; omitted without one. */
26
+ mode?: string;
25
27
  /** SHA-256 of the PNG, so the server can recognise a screenshot approved on another branch. */
26
28
  contentHash?: string;
29
+ /** The story rendered inconsistently while it was captured. */
30
+ flaky?: boolean;
31
+ /** Overrides the run's pixel threshold for this story. */
32
+ threshold?: number;
33
+ /** Overrides the run's tolerance for this story. */
34
+ maxDiffPercent?: number;
27
35
  }
28
36
  export interface ResultUploadTarget {
29
37
  resultId: string;
@@ -63,6 +71,8 @@ export interface TestResult {
63
71
  browser: string;
64
72
  viewportWidth: number;
65
73
  viewportHeight: number;
74
+ /** '' when the story was captured without modes. */
75
+ mode?: string;
66
76
  status: 'pending' | 'pass' | 'fail' | 'new' | 'error';
67
77
  diffPercentage?: number;
68
78
  diffPixels?: number;
@@ -70,6 +80,11 @@ export interface TestResult {
70
80
  baselineUrl?: string;
71
81
  diffUrl?: string;
72
82
  errorMessage?: string;
83
+ flaky?: boolean;
84
+ /** What the screenshot was captured with; null when not recorded. */
85
+ captureEnvironment?: string | null;
86
+ /** What its baseline was captured with; null when not recorded or there is none. */
87
+ baselineCaptureEnvironment?: string | null;
73
88
  }
74
89
  export interface TestRunWithResults extends TestRun {
75
90
  results: TestResult[];
@@ -122,7 +137,7 @@ export declare class ApiClient {
122
137
  uploadImage(uploadUrl: string, imageBuffer: Buffer): Promise<void>;
123
138
  getTestRun(id: string): Promise<TestRunWithResults>;
124
139
  listTestRuns(limit?: number, offset?: number): Promise<TestRun[]>;
125
- getBaselineUploadUrl(storyId: string, browser: string, viewportWidth: number, viewportHeight: number, branch?: string): Promise<BaselineUploadUrl>;
140
+ getBaselineUploadUrl(storyId: string, browser: string, viewportWidth: number, viewportHeight: number, branch?: string, mode?: string): Promise<BaselineUploadUrl>;
126
141
  uploadBaseline(uploadUrl: string, imageBuffer: Buffer): Promise<void>;
127
142
  /**
128
143
  * Points the baseline record at an already-uploaded image. Without this the
@@ -133,6 +148,8 @@ export declare class ApiClient {
133
148
  browser: string;
134
149
  viewportWidth: number;
135
150
  viewportHeight: number;
151
+ /** The mode it was captured in; omitted without one. */
152
+ mode?: string;
136
153
  storageKey: string;
137
154
  /** Omitted, the server uses the project's default branch. */
138
155
  branch?: string;
@@ -99,13 +99,14 @@ export class ApiClient {
99
99
  const response = await this.request('GET', `/api/v1/test-runs?limit=${limit}&offset=${offset}`);
100
100
  return response.data;
101
101
  }
102
- async getBaselineUploadUrl(storyId, browser, viewportWidth, viewportHeight, branch) {
102
+ async getBaselineUploadUrl(storyId, browser, viewportWidth, viewportHeight, branch, mode) {
103
103
  const response = await this.request('POST', '/api/v1/baselines/upload-url', {
104
104
  storyId,
105
105
  browser,
106
106
  viewportWidth,
107
107
  viewportHeight,
108
108
  branch,
109
+ mode,
109
110
  contentType: 'image/png',
110
111
  });
111
112
  return response.data;
@@ -1,14 +1,16 @@
1
1
  import { loadConfig } from '../../config/loader.js';
2
2
  import { loadCredentials } from '../../config/credentials.js';
3
- import { captureScreenshots, getScreenshotKey, parseScreenshotKey } from '../../core/screenshot.js';
3
+ import { captureLabel, captureScreenshots, groupCaptures, parseScreenshotKey } from '../../core/screenshot.js';
4
4
  import { mapWithConcurrency } from '../../core/concurrency.js';
5
5
  import { detectGitContext, describeGitContext } from '../../core/git-context.js';
6
6
  import { hashScreenshot } from '../../core/hash.js';
7
7
  import { compareAllScreenshots, DEFAULT_MAX_DIFF_PERCENT } from '../../core/compare.js';
8
8
  import { createStorage } from '../../storage/index.js';
9
9
  import { createReporter } from '../reporter.js';
10
- import { selectStories } from '../select-stories.js';
10
+ import { selectStories, withoutSkipped } from '../select-stories.js';
11
11
  import { describeReview } from '../review.js';
12
+ import { apiRunVerdict } from '../verdict.js';
13
+ import { reportEnvironmentMismatches } from '../environment-warning.js';
12
14
  import { ApiClient } from '../../api/client.js';
13
15
  function parseBrowsers(browserArg) {
14
16
  if (!browserArg) {
@@ -72,16 +74,21 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
72
74
  const client = new ApiClient({ apiUrl, apiToken });
73
75
  // Screenshots are captured here, not by the server: the worker has no route
74
76
  // to a Storybook running on this machine or on a CI runner.
75
- const stories = await selectStories(reporter, config, options, 'tests');
76
- if (!stories) {
77
+ const selected = await selectStories(reporter, config, options, 'tests');
78
+ if (!selected) {
77
79
  return;
78
80
  }
79
- const totalTests = stories.length * config.browsers.length;
80
81
  reporter.startSpinner('Capturing screenshots...');
81
- const screenshots = await captureScreenshots(config, stories, (completed, total, story, browser) => {
82
+ const screenshots = await captureScreenshots(config, selected, (completed, total, story, browser) => {
82
83
  reporter.progress(completed, total, `${story.title}/${story.name} (${browser})`);
83
84
  });
84
85
  reporter.succeedSpinner('Screenshots captured');
86
+ const stories = withoutSkipped(reporter, selected, screenshots);
87
+ if (!stories) {
88
+ return;
89
+ }
90
+ // One per capture: a story has one per viewport, a failed one has one.
91
+ const totalTests = screenshots.size;
85
92
  const gitContext = await detectGitContext({
86
93
  overrides: { branch: options.branch, commit: options.commit, baseBranch: options.baseBranch },
87
94
  });
@@ -121,22 +128,24 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
121
128
  });
122
129
  reporter.succeedSpinner('Tests complete');
123
130
  // Report results
124
- reportApiResults(reporter, completedTestRun, options, totalTests - uploaded, describeReview(testRun));
131
+ reportApiResults(reporter, completedTestRun, options, totalTests - uploaded, config.viewport, describeReview(testRun));
125
132
  }
126
133
  /**
127
134
  * Registers each captured screenshot with the API and PUTs the PNG to the
128
135
  * presigned URL it hands back. Returns how many made it.
129
136
  */
130
137
  async function uploadScreenshots(client, testRunId, config, stories, screenshots, reporter) {
131
- const total = stories.length * config.browsers.length;
138
+ const total = screenshots.size;
132
139
  let uploaded = 0;
133
140
  let attempted = 0;
134
- // Two requests per story, so they share the capture concurrency.
135
- const tasks = stories.flatMap((story) => config.browsers.map((browser) => ({ story, browser })));
136
- await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser }) => {
137
- const screenshot = screenshots.get(getScreenshotKey(story.id, browser));
141
+ // One task per capture - a story has one per viewport - and two requests
142
+ // per capture, so they share the capture concurrency.
143
+ const captures = groupCaptures(screenshots);
144
+ const tasks = stories.flatMap((story) => config.browsers.flatMap((browser) => (captures.get(`${story.id}::${browser}`) ?? [undefined]).map((screenshot) => ({ story, browser, screenshot }))));
145
+ await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser, screenshot }) => {
146
+ const label = screenshot ? captureLabel({ viewport: { width: screenshot.width, height: screenshot.height }, mode: screenshot.mode }, config.viewport) : '';
138
147
  if (!screenshot || screenshot.buffer.length === 0) {
139
- reporter.warning(`Failed to capture ${story.id} (${browser}): ${screenshot?.error ?? 'no screenshot'}`);
148
+ reporter.warning(`Failed to capture ${story.id} (${browser})${label}: ${screenshot?.error ?? 'no screenshot'}`);
140
149
  attempted++;
141
150
  return;
142
151
  }
@@ -146,18 +155,23 @@ async function uploadScreenshots(client, testRunId, config, stories, screenshots
146
155
  storyTitle: story.title,
147
156
  storyName: story.name,
148
157
  browser,
149
- viewportWidth: config.viewport.width,
150
- viewportHeight: config.viewport.height,
158
+ viewportWidth: screenshot.width,
159
+ viewportHeight: screenshot.height,
160
+ mode: screenshot.mode,
151
161
  contentHash: hashScreenshot(screenshot.buffer),
162
+ flaky: screenshot.flaky,
163
+ // The story's own tolerance, from its parameters; the run's otherwise.
164
+ threshold: screenshot.threshold,
165
+ maxDiffPercent: screenshot.maxDiffPercent,
152
166
  });
153
167
  await client.uploadImage(target.uploadUrl, screenshot.buffer);
154
168
  uploaded++;
155
169
  }
156
170
  catch (error) {
157
- reporter.warning(`Failed to upload ${story.id} (${browser}): ${error instanceof Error ? error.message : 'Unknown error'}`);
171
+ reporter.warning(`Failed to upload ${story.id} (${browser})${label}: ${error instanceof Error ? error.message : 'Unknown error'}`);
158
172
  }
159
173
  attempted++;
160
- reporter.progress(attempted, total, `${story.title}/${story.name} (${browser})`);
174
+ reporter.progress(attempted, total, `${story.title}/${story.name} (${browser})${label}`);
161
175
  });
162
176
  return uploaded;
163
177
  }
@@ -170,16 +184,20 @@ async function runLocalMode(reporter, config, options) {
170
184
  // Create storage
171
185
  const storage = createStorage(config.storage);
172
186
  // Discover stories
173
- const stories = await selectStories(reporter, config, options, 'tests');
174
- if (!stories) {
187
+ const selected = await selectStories(reporter, config, options, 'tests');
188
+ if (!selected) {
175
189
  return;
176
190
  }
177
191
  // Capture screenshots
178
192
  reporter.startSpinner('Capturing screenshots...');
179
- const screenshots = await captureScreenshots(config, stories, (completed, total, story, browser) => {
193
+ const screenshots = await captureScreenshots(config, selected, (completed, total, story, browser) => {
180
194
  reporter.progress(completed, total, `${story.title}/${story.name} (${browser})`);
181
195
  });
182
196
  reporter.succeedSpinner('Screenshots captured');
197
+ const stories = withoutSkipped(reporter, selected, screenshots);
198
+ if (!stories) {
199
+ return;
200
+ }
183
201
  // Compare against baselines
184
202
  reporter.startSpinner('Comparing against baselines...');
185
203
  const results = await compareAllScreenshots(screenshots, {
@@ -188,9 +206,10 @@ async function runLocalMode(reporter, config, options) {
188
206
  storage,
189
207
  localSnapshotsDir: config.localSnapshotsDir,
190
208
  concurrency: config.concurrency,
209
+ viewport: config.viewport,
191
210
  }, (completed, total, key) => {
192
- const { storyId, browser } = parseScreenshotKey(key);
193
- reporter.progress(completed, total, `${storyId} (${browser})`);
211
+ const { storyId, browser, viewport, mode } = parseScreenshotKey(key);
212
+ reporter.progress(completed, total, `${storyId} (${browser})${captureLabel({ viewport, mode }, config.viewport)}`);
194
213
  });
195
214
  reporter.succeedSpinner('Comparison complete');
196
215
  // Build summary
@@ -198,9 +217,11 @@ async function runLocalMode(reporter, config, options) {
198
217
  // Report results
199
218
  reporter.header('Results');
200
219
  for (const result of summary.results) {
201
- reportLocalResult(result.story, result.comparison);
220
+ reportLocalResult(result.story, result.comparison, config.viewport);
202
221
  }
203
222
  reporter.reportSummary(summary, config.maxDiffPercent ?? DEFAULT_MAX_DIFF_PERCENT);
223
+ reportFlakyCount(summary.results.filter((result) => result.comparison.flaky).length);
224
+ reportEnvironmentMismatches(summary.results.map((result) => result.comparison));
204
225
  // A failed comparison exits non-zero whether or not --ci was passed.
205
226
  //
206
227
  // It used to exit 0 without --ci, which meant any pipeline that forgot the
@@ -215,10 +236,10 @@ async function runLocalMode(reporter, config, options) {
215
236
  process.exit(1);
216
237
  }
217
238
  }
218
- function reportLocalResult(story, comparison) {
239
+ function reportLocalResult(story, comparison, configured) {
219
240
  const storyName = `${story.title}/${story.name}`;
220
241
  const browserSuffix = comparison.browser !== 'chromium' ? ` (${comparison.browser})` : '';
221
- const displayName = `${storyName}${browserSuffix}`;
242
+ const displayName = `${storyName}${browserSuffix}${comparison.viewport ? captureLabel({ viewport: comparison.viewport, mode: comparison.mode }, configured) : ''}`;
222
243
  switch (comparison.status) {
223
244
  case 'pass':
224
245
  console.log('\x1b[32m ✓\x1b[0m', `\x1b[2m${displayName}\x1b[0m`);
@@ -239,13 +260,23 @@ function reportLocalResult(story, comparison) {
239
260
  }
240
261
  break;
241
262
  }
263
+ if (comparison.flaky) {
264
+ console.log(FLAKY_NOTE);
265
+ }
242
266
  }
243
- function reportApiResults(reporter, testRun, options, missing, review) {
267
+ const FLAKY_NOTE = '\x1b[33m ⚠ flaky: rendered inconsistently while capturing\x1b[0m';
268
+ /** Points at the flaky stories, if any, below a run's summary. */
269
+ function reportFlakyCount(count) {
270
+ if (count > 0) {
271
+ console.log(`\x1b[33m ${count} flaky ${count === 1 ? 'story' : 'stories'}\x1b[0m`, '\x1b[2m- they rendered differently between screenshots or only captured on a retry\x1b[0m');
272
+ }
273
+ }
274
+ function reportApiResults(reporter, testRun, options, missing, configured, review) {
244
275
  reporter.header('Results');
245
276
  for (const result of testRun.results) {
246
277
  const storyName = result.storyName || result.storyId;
247
278
  const browserSuffix = result.browser && result.browser !== 'chromium' ? ` (${result.browser})` : '';
248
- const displayName = `${storyName}${browserSuffix}`;
279
+ const displayName = `${storyName}${browserSuffix}${captureLabel({ viewport: { width: result.viewportWidth, height: result.viewportHeight }, mode: result.mode }, configured)}`;
249
280
  switch (result.status) {
250
281
  case 'pass':
251
282
  console.log('\x1b[32m ✓\x1b[0m', `\x1b[2m${displayName}\x1b[0m`);
@@ -272,6 +303,9 @@ function reportApiResults(reporter, testRun, options, missing, review) {
272
303
  }
273
304
  break;
274
305
  }
306
+ if (result.flaky) {
307
+ console.log(FLAKY_NOTE);
308
+ }
275
309
  }
276
310
  reporter.newLine();
277
311
  console.log('\x1b[1mSummary:\x1b[0m');
@@ -295,19 +329,32 @@ function reportApiResults(reporter, testRun, options, missing, review) {
295
329
  parts.push(`\x1b[31m${testRun.errorStories} errors\x1b[0m`);
296
330
  }
297
331
  console.log(` ${parts.join(', ')} of ${testRun.totalStories} stories`);
332
+ reportFlakyCount(testRun.results.filter((result) => result.flaky).length);
333
+ reportEnvironmentMismatches(testRun.results);
298
334
  reporter.newLine();
299
335
  if (review) {
300
336
  console.log(`\x1b[1mReview:\x1b[0m ${review}`);
301
337
  reporter.newLine();
302
338
  }
303
- if (testRun.status === 'failed') {
339
+ const verdict = apiRunVerdict(testRun, missing);
340
+ if (verdict === 'failed' && testRun.failedStories === 0) {
341
+ // Nothing differed; some stories just couldn't be compared, and approving
342
+ // won't help with those.
343
+ console.log('\x1b[31mVisual regression tests failed.\x1b[0m', '\x1b[2mSome stories could not be compared - see the errors above.\x1b[0m');
344
+ }
345
+ else if (verdict === 'failed') {
304
346
  console.log('\x1b[31mVisual regression tests failed.\x1b[0m', review
305
347
  ? '\x1b[2mApprove the changes in the review above, or fix them and push again.\x1b[0m'
306
348
  : '\x1b[2mReview changes in the dashboard or run "storysnap update" to update baselines.\x1b[0m');
307
349
  }
308
- else if (missing > 0) {
350
+ else if (verdict === 'missing') {
309
351
  console.log(`\x1b[31m${missing} screenshot(s) were not captured or uploaded.\x1b[0m`, '\x1b[2mSee the warnings above.\x1b[0m');
310
352
  }
353
+ else if (verdict === 'new') {
354
+ console.log('\x1b[33mNew stories detected.\x1b[0m', review
355
+ ? '\x1b[2mApprove them in the review above to create their baselines.\x1b[0m'
356
+ : '\x1b[2mApprove them in the dashboard or run "storysnap update" to create baselines.\x1b[0m');
357
+ }
311
358
  else {
312
359
  console.log('\x1b[32mAll visual regression tests passed!\x1b[0m');
313
360
  }
@@ -324,45 +371,42 @@ function reportApiResults(reporter, testRun, options, missing, review) {
324
371
  }
325
372
  function buildSummary(stories, browsers, results) {
326
373
  const summary = {
327
- total: stories.length * browsers.length,
374
+ total: 0,
328
375
  passed: 0,
329
376
  failed: 0,
330
377
  newStories: 0,
331
378
  errors: 0,
332
379
  results: [],
333
380
  };
381
+ // A story has one result per viewport it was captured at.
382
+ const byStory = new Map();
383
+ for (const comparison of results.values()) {
384
+ const key = `${comparison.storyId}::${comparison.browser}`;
385
+ byStory.set(key, [...(byStory.get(key) ?? []), comparison]);
386
+ }
334
387
  for (const story of stories) {
335
388
  for (const browser of browsers) {
336
- const key = getScreenshotKey(story.id, browser);
337
- const comparison = results.get(key);
338
- if (!comparison) {
339
- summary.errors++;
340
- summary.results.push({
341
- story,
342
- comparison: {
343
- storyId: story.id,
344
- browser,
345
- status: 'error',
346
- errorMessage: 'No comparison result',
347
- },
348
- });
349
- continue;
350
- }
351
- switch (comparison.status) {
352
- case 'pass':
353
- summary.passed++;
354
- break;
355
- case 'fail':
356
- summary.failed++;
357
- break;
358
- case 'new':
359
- summary.newStories++;
360
- break;
361
- case 'error':
362
- summary.errors++;
363
- break;
389
+ const comparisons = byStory.get(`${story.id}::${browser}`) ?? [
390
+ { storyId: story.id, browser, status: 'error', errorMessage: 'No comparison result' },
391
+ ];
392
+ for (const comparison of comparisons) {
393
+ summary.total++;
394
+ switch (comparison.status) {
395
+ case 'pass':
396
+ summary.passed++;
397
+ break;
398
+ case 'fail':
399
+ summary.failed++;
400
+ break;
401
+ case 'new':
402
+ summary.newStories++;
403
+ break;
404
+ case 'error':
405
+ summary.errors++;
406
+ break;
407
+ }
408
+ summary.results.push({ story, comparison });
364
409
  }
365
- summary.results.push({ story, comparison });
366
410
  }
367
411
  }
368
412
  return summary;
@@ -1,13 +1,13 @@
1
1
  import { loadConfig } from '../../config/loader.js';
2
2
  import { loadCredentials } from '../../config/credentials.js';
3
- import { getStoryKey } from '../../core/storybook.js';
4
- import { captureScreenshots, getScreenshotKey } from '../../core/screenshot.js';
3
+ import { getCaptureKey } from '../../core/storybook.js';
4
+ import { captureLabel, captureScreenshots, groupCaptures } from '../../core/screenshot.js';
5
5
  import { mapWithConcurrency } from '../../core/concurrency.js';
6
6
  import { detectGitContext } from '../../core/git-context.js';
7
7
  import { hashScreenshot } from '../../core/hash.js';
8
8
  import { createStorage } from '../../storage/index.js';
9
9
  import { createReporter } from '../reporter.js';
10
- import { selectStories } from '../select-stories.js';
10
+ import { selectStories, withoutSkipped } from '../select-stories.js';
11
11
  import { ApiClient } from '../../api/client.js';
12
12
  function parseBrowsers(browserArg) {
13
13
  if (!browserArg) {
@@ -77,34 +77,39 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
77
77
  const { branch } = await detectGitContext({ overrides: { branch: options.branch } });
78
78
  reporter.info(branch ? `Branch: ${branch}` : "Branch: not detected - updating the project's default branch (pass --branch to name one)");
79
79
  // Discover stories
80
- const stories = await selectStories(reporter, config, options, 'baselines');
81
- if (!stories) {
80
+ const selected = await selectStories(reporter, config, options, 'baselines');
81
+ if (!selected) {
82
82
  return;
83
83
  }
84
- const totalTests = stories.length * config.browsers.length;
85
84
  // Capture screenshots
86
85
  reporter.startSpinner('Capturing screenshots...');
87
- const screenshots = await captureScreenshots(config, stories, (completed, total, story, browser) => {
86
+ const screenshots = await captureScreenshots(config, selected, (completed, total, story, browser) => {
88
87
  reporter.progress(completed, total, `${story.title}/${story.name} (${browser})`);
89
88
  });
90
89
  reporter.succeedSpinner('Screenshots captured');
90
+ const stories = withoutSkipped(reporter, selected, screenshots);
91
+ if (!stories) {
92
+ return;
93
+ }
94
+ // One per capture: a story has one per viewport, a failed one has one.
95
+ const totalTests = screenshots.size;
91
96
  // Upload baselines via API
92
97
  reporter.startSpinner('Uploading baselines via API...');
93
98
  let uploaded = 0;
94
99
  const failures = [];
95
100
  // Uploads are network round trips, so they share the capture concurrency.
96
- const tasks = stories.flatMap((story) => config.browsers.map((browser) => ({ story, browser })));
97
- await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser }) => {
98
- const screenshotKey = getScreenshotKey(story.id, browser);
99
- const screenshot = screenshots.get(screenshotKey);
101
+ const captures = groupCaptures(screenshots);
102
+ const tasks = stories.flatMap((story) => config.browsers.flatMap((browser) => (captures.get(`${story.id}::${browser}`) ?? [undefined]).map((screenshot) => ({ story, browser, screenshot }))));
103
+ await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser, screenshot }) => {
104
+ const label = `${story.title}/${story.name} (${browser})${screenshot ? captureLabel({ viewport: { width: screenshot.width, height: screenshot.height }, mode: screenshot.mode }, config.viewport) : ''}`;
100
105
  if (!screenshot || screenshot.buffer.length === 0) {
101
- failures.push(`${story.title}/${story.name} (${browser}): capture failed - ${screenshot?.error ?? 'no screenshot'}`);
106
+ failures.push(`${label}: capture failed - ${screenshot?.error ?? 'no screenshot'}`);
102
107
  return;
103
108
  }
104
109
  if (!options.dryRun) {
105
110
  try {
106
111
  // Get presigned upload URL from API
107
- const uploadInfo = await client.getBaselineUploadUrl(story.id, browser, config.viewport.width, config.viewport.height, branch);
112
+ const uploadInfo = await client.getBaselineUploadUrl(story.id, browser, screenshot.width, screenshot.height, branch, screenshot.mode);
108
113
  // Upload directly to S3 using presigned URL
109
114
  await client.uploadBaseline(uploadInfo.url, screenshot.buffer);
110
115
  // Record the baseline. The upload alone leaves an orphaned object -
@@ -112,8 +117,9 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
112
117
  await client.registerBaseline({
113
118
  storyId: story.id,
114
119
  browser,
115
- viewportWidth: config.viewport.width,
116
- viewportHeight: config.viewport.height,
120
+ viewportWidth: screenshot.width,
121
+ viewportHeight: screenshot.height,
122
+ mode: screenshot.mode,
117
123
  storageKey: uploadInfo.storageKey,
118
124
  branch,
119
125
  contentHash: hashScreenshot(screenshot.buffer),
@@ -121,14 +127,15 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
121
127
  uploaded++;
122
128
  }
123
129
  catch (error) {
124
- failures.push(`${story.title}/${story.name} (${browser}): upload failed - ${error instanceof Error ? error.message : 'Unknown error'}`);
130
+ failures.push(`${label}: upload failed - ${error instanceof Error ? error.message : 'Unknown error'}`);
125
131
  }
126
132
  }
127
133
  else {
128
134
  uploaded++;
129
135
  }
130
- reporter.progress(uploaded + failures.length, totalTests, `${story.title}/${story.name} (${browser})`);
136
+ reporter.progress(uploaded + failures.length, totalTests, label);
131
137
  });
138
+ warnFlaky(reporter, screenshots, config.viewport);
132
139
  finishUpdate(reporter, uploaded, totalTests, failures, options);
133
140
  }
134
141
  async function runLocalMode(reporter, config, options) {
@@ -140,49 +147,67 @@ async function runLocalMode(reporter, config, options) {
140
147
  // Create storage
141
148
  const storage = createStorage(config.storage);
142
149
  // Discover stories
143
- const stories = await selectStories(reporter, config, options, 'baselines');
144
- if (!stories) {
150
+ const selected = await selectStories(reporter, config, options, 'baselines');
151
+ if (!selected) {
145
152
  return;
146
153
  }
147
- const totalTests = stories.length * config.browsers.length;
148
154
  // Capture screenshots
149
155
  reporter.startSpinner('Capturing screenshots...');
150
- const screenshots = await captureScreenshots(config, stories, (completed, total, story, browser) => {
156
+ const screenshots = await captureScreenshots(config, selected, (completed, total, story, browser) => {
151
157
  reporter.progress(completed, total, `${story.title}/${story.name} (${browser})`);
152
158
  });
153
159
  reporter.succeedSpinner('Screenshots captured');
160
+ const stories = withoutSkipped(reporter, selected, screenshots);
161
+ if (!stories) {
162
+ return;
163
+ }
164
+ // One per capture: a story has one per viewport, a failed one has one.
165
+ const totalTests = screenshots.size;
154
166
  // Upload baselines
155
167
  reporter.startSpinner('Uploading baselines...');
156
168
  let uploaded = 0;
157
169
  const failures = [];
158
170
  // Writes to S3 are network round trips, so they share the capture
159
171
  // concurrency; local writes simply finish sooner.
160
- const tasks = stories.flatMap((story) => config.browsers.map((browser) => ({ story, browser })));
161
- await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser }) => {
162
- const screenshotKey = getScreenshotKey(story.id, browser);
163
- const screenshot = screenshots.get(screenshotKey);
172
+ const captures = groupCaptures(screenshots);
173
+ const tasks = stories.flatMap((story) => config.browsers.flatMap((browser) => (captures.get(`${story.id}::${browser}`) ?? [undefined]).map((screenshot) => ({ story, browser, screenshot }))));
174
+ await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser, screenshot }) => {
175
+ const label = `${story.title}/${story.name} (${browser})${screenshot ? captureLabel({ viewport: { width: screenshot.width, height: screenshot.height }, mode: screenshot.mode }, config.viewport) : ''}`;
164
176
  if (!screenshot || screenshot.buffer.length === 0) {
165
- failures.push(`${story.title}/${story.name} (${browser}): capture failed - ${screenshot?.error ?? 'no screenshot'}`);
177
+ failures.push(`${label}: capture failed - ${screenshot?.error ?? 'no screenshot'}`);
166
178
  return;
167
179
  }
168
- // Use browser-aware storage key
169
- const key = getStoryKey(story.id, browser);
180
+ // Named by story, browser and - when not the configured one - viewport.
181
+ const key = getCaptureKey(screenshot, config.viewport);
170
182
  if (!options.dryRun) {
171
183
  try {
172
184
  await storage.upload(key, screenshot.buffer);
173
185
  uploaded++;
174
186
  }
175
187
  catch (error) {
176
- failures.push(`${story.title}/${story.name} (${browser}): upload failed - ${error instanceof Error ? error.message : 'Unknown error'}`);
188
+ failures.push(`${label}: upload failed - ${error instanceof Error ? error.message : 'Unknown error'}`);
177
189
  }
178
190
  }
179
191
  else {
180
192
  uploaded++;
181
193
  }
182
- reporter.progress(uploaded + failures.length, totalTests, `${story.title}/${story.name} (${browser})`);
194
+ reporter.progress(uploaded + failures.length, totalTests, label);
183
195
  });
196
+ warnFlaky(reporter, screenshots, config.viewport);
184
197
  finishUpdate(reporter, uploaded, totalTests, failures, options);
185
198
  }
199
+ /**
200
+ * Names the baselines captured from a story that rendered inconsistently, as
201
+ * the next test run may not reproduce them.
202
+ */
203
+ function warnFlaky(reporter, screenshots, configured) {
204
+ for (const screenshot of screenshots.values()) {
205
+ if (screenshot.flaky && screenshot.buffer.length > 0) {
206
+ reporter.warning(`${screenshot.storyId} (${screenshot.browser})${captureLabel({ viewport: { width: screenshot.width, height: screenshot.height }, mode: screenshot.mode }, configured)} is flaky: it rendered differently between screenshots ` +
207
+ 'or only captured on a retry, so later runs may not match this baseline');
208
+ }
209
+ }
210
+ }
186
211
  /**
187
212
  * Reports the outcome and exits non-zero if any baseline was not saved. A run
188
213
  * that quietly drops baselines and exits 0 lets a pipeline go green with an
@@ -0,0 +1,17 @@
1
+ /** The parts of a result that say where it and its baseline were captured. */
2
+ export interface EnvironmentFields {
3
+ status: string;
4
+ captureEnvironment?: string | null;
5
+ baselineCaptureEnvironment?: string | null;
6
+ }
7
+ /**
8
+ * Lines warning that baselines were captured somewhere other than this run,
9
+ * one per pair of environments, or none. Different machines render fonts and
10
+ * edges differently, so failures in such a run may be nothing but that.
11
+ * Baselines from before environments were recorded say nothing either way
12
+ * and are left out.
13
+ */
14
+ export declare function describeEnvironmentMismatches(results: EnvironmentFields[]): string[];
15
+ /** Prints the warnings, with what to do about them, if there are any. */
16
+ export declare function reportEnvironmentMismatches(results: EnvironmentFields[]): void;
17
+ //# sourceMappingURL=environment-warning.d.ts.map
@@ -0,0 +1,42 @@
1
+ import chalk from 'chalk';
2
+ /**
3
+ * Lines warning that baselines were captured somewhere other than this run,
4
+ * one per pair of environments, or none. Different machines render fonts and
5
+ * edges differently, so failures in such a run may be nothing but that.
6
+ * Baselines from before environments were recorded say nothing either way
7
+ * and are left out.
8
+ */
9
+ export function describeEnvironmentMismatches(results) {
10
+ const pairs = new Map();
11
+ for (const result of results) {
12
+ const { captureEnvironment: current, baselineCaptureEnvironment: baseline } = result;
13
+ if (!current || !baseline || current === baseline) {
14
+ continue;
15
+ }
16
+ const key = `${baseline}\n${current}`;
17
+ const pair = pairs.get(key) ?? { baseline, current, count: 0, failed: 0 };
18
+ pair.count++;
19
+ if (result.status === 'fail')
20
+ pair.failed++;
21
+ pairs.set(key, pair);
22
+ }
23
+ return [...pairs.values()].map(({ baseline, current, count, failed }) => {
24
+ const subject = `${count} ${count === 1 ? 'baseline was' : 'baselines were'} captured with ${baseline}`;
25
+ const failures = failed > 0 ? ` (${failed} of them failed)` : '';
26
+ return `${subject}, this run with ${current}${failures}`;
27
+ });
28
+ }
29
+ /** Prints the warnings, with what to do about them, if there are any. */
30
+ export function reportEnvironmentMismatches(results) {
31
+ const lines = describeEnvironmentMismatches(results);
32
+ if (lines.length === 0) {
33
+ return;
34
+ }
35
+ for (const line of lines) {
36
+ console.log(chalk.yellow(` ⚠ ${line}`));
37
+ }
38
+ console.log(chalk.dim(' Screenshots from different browsers or machines can differ in every glyph. Capture baselines\n' +
39
+ ' and runs in the same place - the storysnap-capture image does this - see\n' +
40
+ ' https://github.com/Kosai106/storysnap/blob/main/docs/capture-image.md'));
41
+ }
42
+ //# sourceMappingURL=environment-warning.js.map
package/dist/cli/index.js CHANGED
@@ -1,4 +1,5 @@
1
1
  #!/usr/bin/env node
2
+ import { createRequire } from 'node:module';
2
3
  import { Command, InvalidArgumentError } from 'commander';
3
4
  import { testCommand } from './commands/test.js';
4
5
  import { updateCommand } from './commands/update.js';
@@ -13,11 +14,14 @@ function parseConcurrency(value) {
13
14
  }
14
15
  return n;
15
16
  }
17
+ // From dist/cli/index.js - and src/cli/index.ts under tests - the package's
18
+ // own package.json is two levels up; release-please keeps its version current.
19
+ const { version } = createRequire(import.meta.url)('../../package.json');
16
20
  const program = new Command();
17
21
  program
18
22
  .name('storysnap')
19
23
  .description('Visual regression testing tool for Storybook')
20
- .version('1.0.0');
24
+ .version(version);
21
25
  program
22
26
  .command('init')
23
27
  .description('Initialize StorySnap configuration')