@storysnap/cli 2.0.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/api/client.d.ts +34 -1
  2. package/dist/api/client.js +3 -1
  3. package/dist/cli/commands/test.d.ts +1 -0
  4. package/dist/cli/commands/test.js +119 -62
  5. package/dist/cli/commands/update.d.ts +1 -0
  6. package/dist/cli/commands/update.js +62 -30
  7. package/dist/cli/environment-warning.d.ts +17 -0
  8. package/dist/cli/environment-warning.js +42 -0
  9. package/dist/cli/index.js +11 -3
  10. package/dist/cli/review.d.ts +9 -0
  11. package/dist/cli/review.js +10 -0
  12. package/dist/cli/select-stories.d.ts +6 -1
  13. package/dist/cli/select-stories.js +16 -0
  14. package/dist/cli/verdict.d.ts +17 -0
  15. package/dist/cli/verdict.js +22 -0
  16. package/dist/config/schema.d.ts +11 -0
  17. package/dist/config/schema.js +18 -0
  18. package/dist/core/capture-environment.d.ts +19 -0
  19. package/dist/core/capture-environment.js +102 -0
  20. package/dist/core/changed.js +10 -23
  21. package/dist/core/compare.d.ts +3 -1
  22. package/dist/core/compare.js +30 -10
  23. package/dist/core/git-context.d.ts +25 -0
  24. package/dist/core/git-context.js +75 -0
  25. package/dist/core/git.d.ts +11 -0
  26. package/dist/core/git.js +25 -0
  27. package/dist/core/hash.d.ts +7 -0
  28. package/dist/core/hash.js +11 -0
  29. package/dist/core/modes.d.ts +50 -0
  30. package/dist/core/modes.js +124 -0
  31. package/dist/core/screenshot.d.ts +91 -4
  32. package/dist/core/screenshot.js +266 -47
  33. package/dist/core/story-parameters.d.ts +48 -0
  34. package/dist/core/story-parameters.js +122 -0
  35. package/dist/core/story-render.d.ts +31 -0
  36. package/dist/core/story-render.js +75 -0
  37. package/dist/core/storybook.d.ts +16 -2
  38. package/dist/core/storybook.js +18 -6
  39. package/dist/core/viewports.d.ts +14 -0
  40. package/dist/core/viewports.js +27 -0
  41. package/package.json +9 -10
@@ -6,6 +6,10 @@ export interface TestRunInput {
6
6
  storybookUrl?: string;
7
7
  branch?: string;
8
8
  commit?: string;
9
+ /** The branch `branch` will merge into; its baselines fill in for missing ones. */
10
+ baseBranch?: string;
11
+ /** The commit `branch` forked from `baseBranch` at. */
12
+ mergeBase?: string;
9
13
  /** pixelmatch per-pixel colour sensitivity (0-1). */
10
14
  threshold?: number;
11
15
  /** Percentage of the image allowed to differ before a story fails (0-100). */
@@ -18,6 +22,16 @@ export interface CreateResultInput {
18
22
  browser: string;
19
23
  viewportWidth: number;
20
24
  viewportHeight: number;
25
+ /** The mode it was captured in; omitted without one. */
26
+ mode?: string;
27
+ /** SHA-256 of the PNG, so the server can recognise a screenshot approved on another branch. */
28
+ contentHash?: string;
29
+ /** The story rendered inconsistently while it was captured. */
30
+ flaky?: boolean;
31
+ /** Overrides the run's pixel threshold for this story. */
32
+ threshold?: number;
33
+ /** Overrides the run's tolerance for this story. */
34
+ maxDiffPercent?: number;
21
35
  }
22
36
  export interface ResultUploadTarget {
23
37
  resultId: string;
@@ -31,6 +45,12 @@ export interface TestRun {
31
45
  status: 'pending' | 'running' | 'passed' | 'failed' | 'error';
32
46
  branch?: string;
33
47
  commit?: string;
48
+ baseBranch?: string;
49
+ mergeBase?: string;
50
+ /** The branch review this run is a push of; absent on the default branch. */
51
+ reviewId?: string | null;
52
+ /** Its page in the dashboard, when the server knows where that is. Only on creation. */
53
+ reviewUrl?: string | null;
34
54
  totalStories: number;
35
55
  passedStories: number;
36
56
  failedStories: number;
@@ -51,6 +71,8 @@ export interface TestResult {
51
71
  browser: string;
52
72
  viewportWidth: number;
53
73
  viewportHeight: number;
74
+ /** '' when the story was captured without modes. */
75
+ mode?: string;
54
76
  status: 'pending' | 'pass' | 'fail' | 'new' | 'error';
55
77
  diffPercentage?: number;
56
78
  diffPixels?: number;
@@ -58,6 +80,11 @@ export interface TestResult {
58
80
  baselineUrl?: string;
59
81
  diffUrl?: string;
60
82
  errorMessage?: string;
83
+ flaky?: boolean;
84
+ /** What the screenshot was captured with; null when not recorded. */
85
+ captureEnvironment?: string | null;
86
+ /** What its baseline was captured with; null when not recorded or there is none. */
87
+ baselineCaptureEnvironment?: string | null;
61
88
  }
62
89
  export interface TestRunWithResults extends TestRun {
63
90
  results: TestResult[];
@@ -70,6 +97,7 @@ export interface BaselineUploadUrl {
70
97
  export interface Baseline {
71
98
  id: string;
72
99
  projectId: string;
100
+ branch?: string;
73
101
  storyId: string;
74
102
  browser: string;
75
103
  viewportWidth: number;
@@ -109,7 +137,7 @@ export declare class ApiClient {
109
137
  uploadImage(uploadUrl: string, imageBuffer: Buffer): Promise<void>;
110
138
  getTestRun(id: string): Promise<TestRunWithResults>;
111
139
  listTestRuns(limit?: number, offset?: number): Promise<TestRun[]>;
112
- getBaselineUploadUrl(storyId: string, browser: string, viewportWidth: number, viewportHeight: number): Promise<BaselineUploadUrl>;
140
+ getBaselineUploadUrl(storyId: string, browser: string, viewportWidth: number, viewportHeight: number, branch?: string, mode?: string): Promise<BaselineUploadUrl>;
113
141
  uploadBaseline(uploadUrl: string, imageBuffer: Buffer): Promise<void>;
114
142
  /**
115
143
  * Points the baseline record at an already-uploaded image. Without this the
@@ -120,7 +148,12 @@ export declare class ApiClient {
120
148
  browser: string;
121
149
  viewportWidth: number;
122
150
  viewportHeight: number;
151
+ /** The mode it was captured in; omitted without one. */
152
+ mode?: string;
123
153
  storageKey: string;
154
+ /** Omitted, the server uses the project's default branch. */
155
+ branch?: string;
156
+ contentHash?: string;
124
157
  }): Promise<Baseline>;
125
158
  listBaselines(limit?: number, offset?: number, storyId?: string, browser?: string): Promise<Baseline[]>;
126
159
  approveBaseline(baselineId: string, testResultId?: string): Promise<Baseline>;
@@ -99,12 +99,14 @@ export class ApiClient {
99
99
  const response = await this.request('GET', `/api/v1/test-runs?limit=${limit}&offset=${offset}`);
100
100
  return response.data;
101
101
  }
102
- async getBaselineUploadUrl(storyId, browser, viewportWidth, viewportHeight) {
102
+ async getBaselineUploadUrl(storyId, browser, viewportWidth, viewportHeight, branch, mode) {
103
103
  const response = await this.request('POST', '/api/v1/baselines/upload-url', {
104
104
  storyId,
105
105
  browser,
106
106
  viewportWidth,
107
107
  viewportHeight,
108
+ branch,
109
+ mode,
108
110
  contentType: 'image/png',
109
111
  });
110
112
  return response.data;
@@ -6,6 +6,7 @@ export interface TestOptions extends StorySelectionOptions {
6
6
  concurrency?: number;
7
7
  branch?: string;
8
8
  commit?: string;
9
+ baseBranch?: string;
9
10
  ci?: boolean;
10
11
  verbose?: boolean;
11
12
  local?: boolean;
@@ -1,11 +1,16 @@
1
1
  import { loadConfig } from '../../config/loader.js';
2
2
  import { loadCredentials } from '../../config/credentials.js';
3
- import { captureScreenshots, getScreenshotKey, parseScreenshotKey } from '../../core/screenshot.js';
3
+ import { captureLabel, captureScreenshots, groupCaptures, parseScreenshotKey } from '../../core/screenshot.js';
4
4
  import { mapWithConcurrency } from '../../core/concurrency.js';
5
+ import { detectGitContext, describeGitContext } from '../../core/git-context.js';
6
+ import { hashScreenshot } from '../../core/hash.js';
5
7
  import { compareAllScreenshots, DEFAULT_MAX_DIFF_PERCENT } from '../../core/compare.js';
6
8
  import { createStorage } from '../../storage/index.js';
7
9
  import { createReporter } from '../reporter.js';
8
- import { selectStories } from '../select-stories.js';
10
+ import { selectStories, withoutSkipped } from '../select-stories.js';
11
+ import { describeReview } from '../review.js';
12
+ import { apiRunVerdict } from '../verdict.js';
13
+ import { reportEnvironmentMismatches } from '../environment-warning.js';
9
14
  import { ApiClient } from '../../api/client.js';
10
15
  function parseBrowsers(browserArg) {
11
16
  if (!browserArg) {
@@ -69,24 +74,32 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
69
74
  const client = new ApiClient({ apiUrl, apiToken });
70
75
  // Screenshots are captured here, not by the server: the worker has no route
71
76
  // to a Storybook running on this machine or on a CI runner.
72
- const stories = await selectStories(reporter, config, options, 'tests');
73
- if (!stories) {
77
+ const selected = await selectStories(reporter, config, options, 'tests');
78
+ if (!selected) {
74
79
  return;
75
80
  }
76
- const totalTests = stories.length * config.browsers.length;
77
81
  reporter.startSpinner('Capturing screenshots...');
78
- const screenshots = await captureScreenshots(config, stories, (completed, total, story, browser) => {
82
+ const screenshots = await captureScreenshots(config, selected, (completed, total, story, browser) => {
79
83
  reporter.progress(completed, total, `${story.title}/${story.name} (${browser})`);
80
84
  });
81
85
  reporter.succeedSpinner('Screenshots captured');
86
+ const stories = withoutSkipped(reporter, selected, screenshots);
87
+ if (!stories) {
88
+ return;
89
+ }
90
+ // One per capture: a story has one per viewport, a failed one has one.
91
+ const totalTests = screenshots.size;
92
+ const gitContext = await detectGitContext({
93
+ overrides: { branch: options.branch, commit: options.commit, baseBranch: options.baseBranch },
94
+ });
95
+ reporter.info(describeGitContext(gitContext));
82
96
  reporter.startSpinner('Creating test run...');
83
97
  // Only sent when the config actually declares them. Left undefined the server
84
98
  // applies the project's own defaults, which is what makes `PATCH /projects/me`
85
99
  // worth having; a config that does declare them overrides for this run.
86
100
  const testRun = await client.createTestRun({
87
101
  storybookUrl: config.storybookUrl,
88
- branch: options.branch,
89
- commit: options.commit,
102
+ ...gitContext,
90
103
  threshold: config.threshold,
91
104
  maxDiffPercent: config.maxDiffPercent,
92
105
  });
@@ -115,22 +128,24 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
115
128
  });
116
129
  reporter.succeedSpinner('Tests complete');
117
130
  // Report results
118
- reportApiResults(reporter, completedTestRun, options, totalTests - uploaded);
131
+ reportApiResults(reporter, completedTestRun, options, totalTests - uploaded, config.viewport, describeReview(testRun));
119
132
  }
120
133
  /**
121
134
  * Registers each captured screenshot with the API and PUTs the PNG to the
122
135
  * presigned URL it hands back. Returns how many made it.
123
136
  */
124
137
  async function uploadScreenshots(client, testRunId, config, stories, screenshots, reporter) {
125
- const total = stories.length * config.browsers.length;
138
+ const total = screenshots.size;
126
139
  let uploaded = 0;
127
140
  let attempted = 0;
128
- // Two requests per story, so they share the capture concurrency.
129
- const tasks = stories.flatMap((story) => config.browsers.map((browser) => ({ story, browser })));
130
- await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser }) => {
131
- const screenshot = screenshots.get(getScreenshotKey(story.id, browser));
141
+ // One task per capture - a story has one per viewport - and two requests
142
+ // per capture, so they share the capture concurrency.
143
+ const captures = groupCaptures(screenshots);
144
+ const tasks = stories.flatMap((story) => config.browsers.flatMap((browser) => (captures.get(`${story.id}::${browser}`) ?? [undefined]).map((screenshot) => ({ story, browser, screenshot }))));
145
+ await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser, screenshot }) => {
146
+ const label = screenshot ? captureLabel({ viewport: { width: screenshot.width, height: screenshot.height }, mode: screenshot.mode }, config.viewport) : '';
132
147
  if (!screenshot || screenshot.buffer.length === 0) {
133
- reporter.warning(`Failed to capture ${story.id} (${browser}): ${screenshot?.error ?? 'no screenshot'}`);
148
+ reporter.warning(`Failed to capture ${story.id} (${browser})${label}: ${screenshot?.error ?? 'no screenshot'}`);
134
149
  attempted++;
135
150
  return;
136
151
  }
@@ -140,17 +155,23 @@ async function uploadScreenshots(client, testRunId, config, stories, screenshots
140
155
  storyTitle: story.title,
141
156
  storyName: story.name,
142
157
  browser,
143
- viewportWidth: config.viewport.width,
144
- viewportHeight: config.viewport.height,
158
+ viewportWidth: screenshot.width,
159
+ viewportHeight: screenshot.height,
160
+ mode: screenshot.mode,
161
+ contentHash: hashScreenshot(screenshot.buffer),
162
+ flaky: screenshot.flaky,
163
+ // The story's own tolerance, from its parameters; the run's otherwise.
164
+ threshold: screenshot.threshold,
165
+ maxDiffPercent: screenshot.maxDiffPercent,
145
166
  });
146
167
  await client.uploadImage(target.uploadUrl, screenshot.buffer);
147
168
  uploaded++;
148
169
  }
149
170
  catch (error) {
150
- reporter.warning(`Failed to upload ${story.id} (${browser}): ${error instanceof Error ? error.message : 'Unknown error'}`);
171
+ reporter.warning(`Failed to upload ${story.id} (${browser})${label}: ${error instanceof Error ? error.message : 'Unknown error'}`);
151
172
  }
152
173
  attempted++;
153
- reporter.progress(attempted, total, `${story.title}/${story.name} (${browser})`);
174
+ reporter.progress(attempted, total, `${story.title}/${story.name} (${browser})${label}`);
154
175
  });
155
176
  return uploaded;
156
177
  }
@@ -163,16 +184,20 @@ async function runLocalMode(reporter, config, options) {
163
184
  // Create storage
164
185
  const storage = createStorage(config.storage);
165
186
  // Discover stories
166
- const stories = await selectStories(reporter, config, options, 'tests');
167
- if (!stories) {
187
+ const selected = await selectStories(reporter, config, options, 'tests');
188
+ if (!selected) {
168
189
  return;
169
190
  }
170
191
  // Capture screenshots
171
192
  reporter.startSpinner('Capturing screenshots...');
172
- const screenshots = await captureScreenshots(config, stories, (completed, total, story, browser) => {
193
+ const screenshots = await captureScreenshots(config, selected, (completed, total, story, browser) => {
173
194
  reporter.progress(completed, total, `${story.title}/${story.name} (${browser})`);
174
195
  });
175
196
  reporter.succeedSpinner('Screenshots captured');
197
+ const stories = withoutSkipped(reporter, selected, screenshots);
198
+ if (!stories) {
199
+ return;
200
+ }
176
201
  // Compare against baselines
177
202
  reporter.startSpinner('Comparing against baselines...');
178
203
  const results = await compareAllScreenshots(screenshots, {
@@ -181,9 +206,10 @@ async function runLocalMode(reporter, config, options) {
181
206
  storage,
182
207
  localSnapshotsDir: config.localSnapshotsDir,
183
208
  concurrency: config.concurrency,
209
+ viewport: config.viewport,
184
210
  }, (completed, total, key) => {
185
- const { storyId, browser } = parseScreenshotKey(key);
186
- reporter.progress(completed, total, `${storyId} (${browser})`);
211
+ const { storyId, browser, viewport, mode } = parseScreenshotKey(key);
212
+ reporter.progress(completed, total, `${storyId} (${browser})${captureLabel({ viewport, mode }, config.viewport)}`);
187
213
  });
188
214
  reporter.succeedSpinner('Comparison complete');
189
215
  // Build summary
@@ -191,9 +217,11 @@ async function runLocalMode(reporter, config, options) {
191
217
  // Report results
192
218
  reporter.header('Results');
193
219
  for (const result of summary.results) {
194
- reportLocalResult(result.story, result.comparison);
220
+ reportLocalResult(result.story, result.comparison, config.viewport);
195
221
  }
196
222
  reporter.reportSummary(summary, config.maxDiffPercent ?? DEFAULT_MAX_DIFF_PERCENT);
223
+ reportFlakyCount(summary.results.filter((result) => result.comparison.flaky).length);
224
+ reportEnvironmentMismatches(summary.results.map((result) => result.comparison));
197
225
  // A failed comparison exits non-zero whether or not --ci was passed.
198
226
  //
199
227
  // It used to exit 0 without --ci, which meant any pipeline that forgot the
@@ -208,10 +236,10 @@ async function runLocalMode(reporter, config, options) {
208
236
  process.exit(1);
209
237
  }
210
238
  }
211
- function reportLocalResult(story, comparison) {
239
+ function reportLocalResult(story, comparison, configured) {
212
240
  const storyName = `${story.title}/${story.name}`;
213
241
  const browserSuffix = comparison.browser !== 'chromium' ? ` (${comparison.browser})` : '';
214
- const displayName = `${storyName}${browserSuffix}`;
242
+ const displayName = `${storyName}${browserSuffix}${comparison.viewport ? captureLabel({ viewport: comparison.viewport, mode: comparison.mode }, configured) : ''}`;
215
243
  switch (comparison.status) {
216
244
  case 'pass':
217
245
  console.log('\x1b[32m ✓\x1b[0m', `\x1b[2m${displayName}\x1b[0m`);
@@ -232,13 +260,23 @@ function reportLocalResult(story, comparison) {
232
260
  }
233
261
  break;
234
262
  }
263
+ if (comparison.flaky) {
264
+ console.log(FLAKY_NOTE);
265
+ }
235
266
  }
236
- function reportApiResults(reporter, testRun, options, missing) {
267
+ const FLAKY_NOTE = '\x1b[33m ⚠ flaky: rendered inconsistently while capturing\x1b[0m';
268
+ /** Points at the flaky stories, if any, below a run's summary. */
269
+ function reportFlakyCount(count) {
270
+ if (count > 0) {
271
+ console.log(`\x1b[33m ${count} flaky ${count === 1 ? 'story' : 'stories'}\x1b[0m`, '\x1b[2m- they rendered differently between screenshots or only captured on a retry\x1b[0m');
272
+ }
273
+ }
274
+ function reportApiResults(reporter, testRun, options, missing, configured, review) {
237
275
  reporter.header('Results');
238
276
  for (const result of testRun.results) {
239
277
  const storyName = result.storyName || result.storyId;
240
278
  const browserSuffix = result.browser && result.browser !== 'chromium' ? ` (${result.browser})` : '';
241
- const displayName = `${storyName}${browserSuffix}`;
279
+ const displayName = `${storyName}${browserSuffix}${captureLabel({ viewport: { width: result.viewportWidth, height: result.viewportHeight }, mode: result.mode }, configured)}`;
242
280
  switch (result.status) {
243
281
  case 'pass':
244
282
  console.log('\x1b[32m ✓\x1b[0m', `\x1b[2m${displayName}\x1b[0m`);
@@ -265,6 +303,9 @@ function reportApiResults(reporter, testRun, options, missing) {
265
303
  }
266
304
  break;
267
305
  }
306
+ if (result.flaky) {
307
+ console.log(FLAKY_NOTE);
308
+ }
268
309
  }
269
310
  reporter.newLine();
270
311
  console.log('\x1b[1mSummary:\x1b[0m');
@@ -288,13 +329,32 @@ function reportApiResults(reporter, testRun, options, missing) {
288
329
  parts.push(`\x1b[31m${testRun.errorStories} errors\x1b[0m`);
289
330
  }
290
331
  console.log(` ${parts.join(', ')} of ${testRun.totalStories} stories`);
332
+ reportFlakyCount(testRun.results.filter((result) => result.flaky).length);
333
+ reportEnvironmentMismatches(testRun.results);
291
334
  reporter.newLine();
292
- if (testRun.status === 'failed') {
293
- console.log('\x1b[31mVisual regression tests failed.\x1b[0m', '\x1b[2mReview changes in the dashboard or run "storysnap update" to update baselines.\x1b[0m');
335
+ if (review) {
336
+ console.log(`\x1b[1mReview:\x1b[0m ${review}`);
337
+ reporter.newLine();
338
+ }
339
+ const verdict = apiRunVerdict(testRun, missing);
340
+ if (verdict === 'failed' && testRun.failedStories === 0) {
341
+ // Nothing differed; some stories just couldn't be compared, and approving
342
+ // won't help with those.
343
+ console.log('\x1b[31mVisual regression tests failed.\x1b[0m', '\x1b[2mSome stories could not be compared - see the errors above.\x1b[0m');
344
+ }
345
+ else if (verdict === 'failed') {
346
+ console.log('\x1b[31mVisual regression tests failed.\x1b[0m', review
347
+ ? '\x1b[2mApprove the changes in the review above, or fix them and push again.\x1b[0m'
348
+ : '\x1b[2mReview changes in the dashboard or run "storysnap update" to update baselines.\x1b[0m');
294
349
  }
295
- else if (missing > 0) {
350
+ else if (verdict === 'missing') {
296
351
  console.log(`\x1b[31m${missing} screenshot(s) were not captured or uploaded.\x1b[0m`, '\x1b[2mSee the warnings above.\x1b[0m');
297
352
  }
353
+ else if (verdict === 'new') {
354
+ console.log('\x1b[33mNew stories detected.\x1b[0m', review
355
+ ? '\x1b[2mApprove them in the review above to create their baselines.\x1b[0m'
356
+ : '\x1b[2mApprove them in the dashboard or run "storysnap update" to create baselines.\x1b[0m');
357
+ }
298
358
  else {
299
359
  console.log('\x1b[32mAll visual regression tests passed!\x1b[0m');
300
360
  }
@@ -311,45 +371,42 @@ function reportApiResults(reporter, testRun, options, missing) {
311
371
  }
312
372
  function buildSummary(stories, browsers, results) {
313
373
  const summary = {
314
- total: stories.length * browsers.length,
374
+ total: 0,
315
375
  passed: 0,
316
376
  failed: 0,
317
377
  newStories: 0,
318
378
  errors: 0,
319
379
  results: [],
320
380
  };
381
+ // A story has one result per viewport it was captured at.
382
+ const byStory = new Map();
383
+ for (const comparison of results.values()) {
384
+ const key = `${comparison.storyId}::${comparison.browser}`;
385
+ byStory.set(key, [...(byStory.get(key) ?? []), comparison]);
386
+ }
321
387
  for (const story of stories) {
322
388
  for (const browser of browsers) {
323
- const key = getScreenshotKey(story.id, browser);
324
- const comparison = results.get(key);
325
- if (!comparison) {
326
- summary.errors++;
327
- summary.results.push({
328
- story,
329
- comparison: {
330
- storyId: story.id,
331
- browser,
332
- status: 'error',
333
- errorMessage: 'No comparison result',
334
- },
335
- });
336
- continue;
337
- }
338
- switch (comparison.status) {
339
- case 'pass':
340
- summary.passed++;
341
- break;
342
- case 'fail':
343
- summary.failed++;
344
- break;
345
- case 'new':
346
- summary.newStories++;
347
- break;
348
- case 'error':
349
- summary.errors++;
350
- break;
389
+ const comparisons = byStory.get(`${story.id}::${browser}`) ?? [
390
+ { storyId: story.id, browser, status: 'error', errorMessage: 'No comparison result' },
391
+ ];
392
+ for (const comparison of comparisons) {
393
+ summary.total++;
394
+ switch (comparison.status) {
395
+ case 'pass':
396
+ summary.passed++;
397
+ break;
398
+ case 'fail':
399
+ summary.failed++;
400
+ break;
401
+ case 'new':
402
+ summary.newStories++;
403
+ break;
404
+ case 'error':
405
+ summary.errors++;
406
+ break;
407
+ }
408
+ summary.results.push({ story, comparison });
351
409
  }
352
- summary.results.push({ story, comparison });
353
410
  }
354
411
  }
355
412
  return summary;
@@ -4,6 +4,7 @@ export interface UpdateOptions extends StorySelectionOptions {
4
4
  browser?: string;
5
5
  storybookUrl?: string;
6
6
  concurrency?: number;
7
+ branch?: string;
7
8
  verbose?: boolean;
8
9
  dryRun?: boolean;
9
10
  local?: boolean;
@@ -1,11 +1,13 @@
1
1
  import { loadConfig } from '../../config/loader.js';
2
2
  import { loadCredentials } from '../../config/credentials.js';
3
- import { getStoryKey } from '../../core/storybook.js';
4
- import { captureScreenshots, getScreenshotKey } from '../../core/screenshot.js';
3
+ import { getCaptureKey } from '../../core/storybook.js';
4
+ import { captureLabel, captureScreenshots, groupCaptures } from '../../core/screenshot.js';
5
5
  import { mapWithConcurrency } from '../../core/concurrency.js';
6
+ import { detectGitContext } from '../../core/git-context.js';
7
+ import { hashScreenshot } from '../../core/hash.js';
6
8
  import { createStorage } from '../../storage/index.js';
7
9
  import { createReporter } from '../reporter.js';
8
- import { selectStories } from '../select-stories.js';
10
+ import { selectStories, withoutSkipped } from '../select-stories.js';
9
11
  import { ApiClient } from '../../api/client.js';
10
12
  function parseBrowsers(browserArg) {
11
13
  if (!browserArg) {
@@ -71,35 +73,43 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
71
73
  reporter.info(`Browsers: ${config.browsers.join(', ')}`);
72
74
  reporter.newLine();
73
75
  const client = new ApiClient({ apiUrl, apiToken });
76
+ // Baselines land on this branch only; other branches keep theirs.
77
+ const { branch } = await detectGitContext({ overrides: { branch: options.branch } });
78
+ reporter.info(branch ? `Branch: ${branch}` : "Branch: not detected - updating the project's default branch (pass --branch to name one)");
74
79
  // Discover stories
75
- const stories = await selectStories(reporter, config, options, 'baselines');
76
- if (!stories) {
80
+ const selected = await selectStories(reporter, config, options, 'baselines');
81
+ if (!selected) {
77
82
  return;
78
83
  }
79
- const totalTests = stories.length * config.browsers.length;
80
84
  // Capture screenshots
81
85
  reporter.startSpinner('Capturing screenshots...');
82
- const screenshots = await captureScreenshots(config, stories, (completed, total, story, browser) => {
86
+ const screenshots = await captureScreenshots(config, selected, (completed, total, story, browser) => {
83
87
  reporter.progress(completed, total, `${story.title}/${story.name} (${browser})`);
84
88
  });
85
89
  reporter.succeedSpinner('Screenshots captured');
90
+ const stories = withoutSkipped(reporter, selected, screenshots);
91
+ if (!stories) {
92
+ return;
93
+ }
94
+ // One per capture: a story has one per viewport, a failed one has one.
95
+ const totalTests = screenshots.size;
86
96
  // Upload baselines via API
87
97
  reporter.startSpinner('Uploading baselines via API...');
88
98
  let uploaded = 0;
89
99
  const failures = [];
90
100
  // Uploads are network round trips, so they share the capture concurrency.
91
- const tasks = stories.flatMap((story) => config.browsers.map((browser) => ({ story, browser })));
92
- await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser }) => {
93
- const screenshotKey = getScreenshotKey(story.id, browser);
94
- const screenshot = screenshots.get(screenshotKey);
101
+ const captures = groupCaptures(screenshots);
102
+ const tasks = stories.flatMap((story) => config.browsers.flatMap((browser) => (captures.get(`${story.id}::${browser}`) ?? [undefined]).map((screenshot) => ({ story, browser, screenshot }))));
103
+ await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser, screenshot }) => {
104
+ const label = `${story.title}/${story.name} (${browser})${screenshot ? captureLabel({ viewport: { width: screenshot.width, height: screenshot.height }, mode: screenshot.mode }, config.viewport) : ''}`;
95
105
  if (!screenshot || screenshot.buffer.length === 0) {
96
- failures.push(`${story.title}/${story.name} (${browser}): capture failed - ${screenshot?.error ?? 'no screenshot'}`);
106
+ failures.push(`${label}: capture failed - ${screenshot?.error ?? 'no screenshot'}`);
97
107
  return;
98
108
  }
99
109
  if (!options.dryRun) {
100
110
  try {
101
111
  // Get presigned upload URL from API
102
- const uploadInfo = await client.getBaselineUploadUrl(story.id, browser, config.viewport.width, config.viewport.height);
112
+ const uploadInfo = await client.getBaselineUploadUrl(story.id, browser, screenshot.width, screenshot.height, branch, screenshot.mode);
103
113
  // Upload directly to S3 using presigned URL
104
114
  await client.uploadBaseline(uploadInfo.url, screenshot.buffer);
105
115
  // Record the baseline. The upload alone leaves an orphaned object -
@@ -107,21 +117,25 @@ async function runApiMode(reporter, config, apiUrl, apiToken, options) {
107
117
  await client.registerBaseline({
108
118
  storyId: story.id,
109
119
  browser,
110
- viewportWidth: config.viewport.width,
111
- viewportHeight: config.viewport.height,
120
+ viewportWidth: screenshot.width,
121
+ viewportHeight: screenshot.height,
122
+ mode: screenshot.mode,
112
123
  storageKey: uploadInfo.storageKey,
124
+ branch,
125
+ contentHash: hashScreenshot(screenshot.buffer),
113
126
  });
114
127
  uploaded++;
115
128
  }
116
129
  catch (error) {
117
- failures.push(`${story.title}/${story.name} (${browser}): upload failed - ${error instanceof Error ? error.message : 'Unknown error'}`);
130
+ failures.push(`${label}: upload failed - ${error instanceof Error ? error.message : 'Unknown error'}`);
118
131
  }
119
132
  }
120
133
  else {
121
134
  uploaded++;
122
135
  }
123
- reporter.progress(uploaded + failures.length, totalTests, `${story.title}/${story.name} (${browser})`);
136
+ reporter.progress(uploaded + failures.length, totalTests, label);
124
137
  });
138
+ warnFlaky(reporter, screenshots, config.viewport);
125
139
  finishUpdate(reporter, uploaded, totalTests, failures, options);
126
140
  }
127
141
  async function runLocalMode(reporter, config, options) {
@@ -133,49 +147,67 @@ async function runLocalMode(reporter, config, options) {
133
147
  // Create storage
134
148
  const storage = createStorage(config.storage);
135
149
  // Discover stories
136
- const stories = await selectStories(reporter, config, options, 'baselines');
137
- if (!stories) {
150
+ const selected = await selectStories(reporter, config, options, 'baselines');
151
+ if (!selected) {
138
152
  return;
139
153
  }
140
- const totalTests = stories.length * config.browsers.length;
141
154
  // Capture screenshots
142
155
  reporter.startSpinner('Capturing screenshots...');
143
- const screenshots = await captureScreenshots(config, stories, (completed, total, story, browser) => {
156
+ const screenshots = await captureScreenshots(config, selected, (completed, total, story, browser) => {
144
157
  reporter.progress(completed, total, `${story.title}/${story.name} (${browser})`);
145
158
  });
146
159
  reporter.succeedSpinner('Screenshots captured');
160
+ const stories = withoutSkipped(reporter, selected, screenshots);
161
+ if (!stories) {
162
+ return;
163
+ }
164
+ // One per capture: a story has one per viewport, a failed one has one.
165
+ const totalTests = screenshots.size;
147
166
  // Upload baselines
148
167
  reporter.startSpinner('Uploading baselines...');
149
168
  let uploaded = 0;
150
169
  const failures = [];
151
170
  // Writes to S3 are network round trips, so they share the capture
152
171
  // concurrency; local writes simply finish sooner.
153
- const tasks = stories.flatMap((story) => config.browsers.map((browser) => ({ story, browser })));
154
- await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser }) => {
155
- const screenshotKey = getScreenshotKey(story.id, browser);
156
- const screenshot = screenshots.get(screenshotKey);
172
+ const captures = groupCaptures(screenshots);
173
+ const tasks = stories.flatMap((story) => config.browsers.flatMap((browser) => (captures.get(`${story.id}::${browser}`) ?? [undefined]).map((screenshot) => ({ story, browser, screenshot }))));
174
+ await mapWithConcurrency(tasks, config.concurrency, async ({ story, browser, screenshot }) => {
175
+ const label = `${story.title}/${story.name} (${browser})${screenshot ? captureLabel({ viewport: { width: screenshot.width, height: screenshot.height }, mode: screenshot.mode }, config.viewport) : ''}`;
157
176
  if (!screenshot || screenshot.buffer.length === 0) {
158
- failures.push(`${story.title}/${story.name} (${browser}): capture failed - ${screenshot?.error ?? 'no screenshot'}`);
177
+ failures.push(`${label}: capture failed - ${screenshot?.error ?? 'no screenshot'}`);
159
178
  return;
160
179
  }
161
- // Use browser-aware storage key
162
- const key = getStoryKey(story.id, browser);
180
+ // Named by story, browser and - when not the configured one - viewport.
181
+ const key = getCaptureKey(screenshot, config.viewport);
163
182
  if (!options.dryRun) {
164
183
  try {
165
184
  await storage.upload(key, screenshot.buffer);
166
185
  uploaded++;
167
186
  }
168
187
  catch (error) {
169
- failures.push(`${story.title}/${story.name} (${browser}): upload failed - ${error instanceof Error ? error.message : 'Unknown error'}`);
188
+ failures.push(`${label}: upload failed - ${error instanceof Error ? error.message : 'Unknown error'}`);
170
189
  }
171
190
  }
172
191
  else {
173
192
  uploaded++;
174
193
  }
175
- reporter.progress(uploaded + failures.length, totalTests, `${story.title}/${story.name} (${browser})`);
194
+ reporter.progress(uploaded + failures.length, totalTests, label);
176
195
  });
196
+ warnFlaky(reporter, screenshots, config.viewport);
177
197
  finishUpdate(reporter, uploaded, totalTests, failures, options);
178
198
  }
199
+ /**
200
+ * Names the baselines captured from a story that rendered inconsistently, as
201
+ * the next test run may not reproduce them.
202
+ */
203
+ function warnFlaky(reporter, screenshots, configured) {
204
+ for (const screenshot of screenshots.values()) {
205
+ if (screenshot.flaky && screenshot.buffer.length > 0) {
206
+ reporter.warning(`${screenshot.storyId} (${screenshot.browser})${captureLabel({ viewport: { width: screenshot.width, height: screenshot.height }, mode: screenshot.mode }, configured)} is flaky: it rendered differently between screenshots ` +
207
+ 'or only captured on a retry, so later runs may not match this baseline');
208
+ }
209
+ }
210
+ }
179
211
  /**
180
212
  * Reports the outcome and exits non-zero if any baseline was not saved. A run
181
213
  * that quietly drops baselines and exits 0 lets a pipeline go green with an
@@ -0,0 +1,17 @@
1
+ /** The parts of a result that say where it and its baseline were captured. */
2
+ export interface EnvironmentFields {
3
+ status: string;
4
+ captureEnvironment?: string | null;
5
+ baselineCaptureEnvironment?: string | null;
6
+ }
7
+ /**
8
+ * Lines warning that baselines were captured somewhere other than this run,
9
+ * one per pair of environments, or none. Different machines render fonts and
10
+ * edges differently, so failures in such a run may be nothing but that.
11
+ * Baselines from before environments were recorded say nothing either way
12
+ * and are left out.
13
+ */
14
+ export declare function describeEnvironmentMismatches(results: EnvironmentFields[]): string[];
15
+ /** Prints the warnings, with what to do about them, if there are any. */
16
+ export declare function reportEnvironmentMismatches(results: EnvironmentFields[]): void;
17
+ //# sourceMappingURL=environment-warning.d.ts.map