@jxsuite/import 0.39.8 → 0.40.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/run.ts CHANGED
@@ -1,27 +1,24 @@
1
1
  /**
2
- * Programmatic import pipeline — the orchestrator behind both the jx-import CLI and the Studio
3
- * import endpoint. Runs the same phase sequence as the CLI (capture → styles → assets →
4
- * componentize → emit → verify), reporting progress through a callback instead of the console.
2
+ * The local import — the orchestrator behind both the jx-import CLI and the Studio import endpoint.
3
+ *
4
+ * Everything portable moved to `pipeline.ts`. What is left is what only a machine with a disk and a
5
+ * Chrome can do: the empty-directory guard, the `project.json` seed, the local write sink, the
6
+ * browser launch, and the verify phase (which builds the project with `@jxsuite/compiler` and
7
+ * serves it over `Bun.serve` to screenshot it — a hosted platform never executes a project's own
8
+ * JavaScript, so verification does not follow the pipeline anywhere).
5
9
  */
6
10
 
7
11
  import { existsSync, readdirSync } from "node:fs";
8
- import { isAbsolute } from "node:path";
9
- import { capturePage, closeBrowser, launchBrowser } from "./capture.ts";
10
- import { convertToJx } from "./to-jx.ts";
11
- import { emitMultiPageProject } from "./emit.ts";
12
- import { captureStyles } from "./style-capture.ts";
13
- import { diffAllStyles, kebabToCamel } from "./style-diff.ts";
14
- import { extractMedia } from "./media-extract.ts";
15
- import { applyStylesToTree } from "./apply-styles.ts";
16
- import { collectAssets } from "./asset-collect.ts";
17
- import { downloadAssets } from "./asset-download.ts";
18
- import { rewriteAssetUrls } from "./asset-rewrite.ts";
19
- import { applyTokens } from "./css-tokens.ts";
20
- import { crawlSite } from "./crawl.ts";
21
- import { detectLayout } from "./layout-detect.ts";
12
+ import { isAbsolute, join } from "node:path";
13
+ import { mkdir } from "node:fs/promises";
14
+ import { closeBrowser, launchBrowser } from "./browser-local.ts";
15
+ import { createLocalIo } from "./io.ts";
16
+ import { runImportPipeline } from "./pipeline.ts";
17
+ import type { BreakpointPolicy } from "./breakpoint-plan.ts";
18
+ import type { ImportPhase, ImportProgressEvent } from "./pipeline.ts";
22
19
  import type { Browser } from "puppeteer-core";
23
- import type { JxElement } from "@jxsuite/schema/types";
24
- import type { ComponentizeResult } from "./componentize.ts";
20
+
21
+ export type { ImportPhase, ImportProgressEvent } from "./pipeline.ts";
25
22
 
26
23
  export interface ImportSiteOptions {
27
24
  /** The page to import; must be http(s). */
@@ -34,6 +31,14 @@ export interface ImportSiteOptions {
34
31
  maxPages?: number;
35
32
  /** Skip styles/assets for pages above this node count (default 5000). */
36
33
  maxNodesPerPage?: number;
34
+ /**
35
+ * Which of the site's declared breakpoints the project keeps (default: three, evenly spaced).
36
+ *
37
+ * A real site declares as many breakpoints as it has accumulated frameworks; nine `$media`
38
+ * entries is nine canvas sizes in Studio and nine columns in every style editor, and nobody
39
+ * authors against that. See `breakpoint-plan.ts`.
40
+ */
41
+ breakpoints?: BreakpointPolicy;
37
42
  /** Capture computed styles (default true). */
38
43
  styles?: boolean;
39
44
  /** Download and rewrite assets (default true). */
@@ -46,87 +51,77 @@ export interface ImportSiteOptions {
46
51
  componentize?: false | { minInstances?: number; minDepth?: number };
47
52
  /** LLM refinement of component/prop names; false/undefined to skip. */
48
53
  ai?: false | { apiKey: string; baseUrl?: string | undefined; model?: string | undefined };
49
- /** Build + screenshot-diff the emitted project against the original; false to skip. */
50
- verify?: false | { threshold?: number };
54
+ /**
55
+ * Build + screenshot-diff the emitted project against the original; false to skip.
56
+ *
57
+ * `threshold` is pixelmatch's per-pixel COLOUR tolerance, not a bar; `minFidelity` is the bar,
58
+ * and the run reports `passed` against it. `fullPage` compares the whole scrollable page (default
59
+ * true) rather than the first viewport.
60
+ */
61
+ verify?: false | { threshold?: number; minFidelity?: number; fullPage?: boolean | undefined };
51
62
  /** Explicit browser binary (wins over CHROME_PATH and PATH discovery). */
52
63
  chromePath?: string;
53
64
  /** Aborts between phases (and between crawled pages). */
54
65
  signal?: AbortSignal;
55
66
  }
56
67
 
57
- export type ImportPhase =
58
- | "launch"
59
- | "capture"
60
- | "crawl"
61
- | "styles"
62
- | "assets"
63
- | "layout"
64
- | "components"
65
- | "ai"
66
- | "emit"
67
- | "verify"
68
- | "done";
69
-
70
- export interface ImportProgressEvent {
71
- phase: ImportPhase;
72
- message: string;
73
- current?: number;
74
- total?: number;
75
- }
76
-
77
68
  export interface ImportSiteResult {
78
69
  outDir: string;
79
70
  pages: { route: string; title: string; nodeCount: number }[];
80
71
  fileCount: number;
81
- verify: { averageFidelity: number; reportDir: string } | null;
72
+ verify: {
73
+ averageFidelity: number;
74
+ reportDir: string;
75
+ /**
76
+ * Whether the run met its bar — it built cleanly, every page rendered, and the average reached
77
+ * `minFidelity`. Nothing about verify could fail before this existed: a clone scoring 8%
78
+ * finished exactly like one scoring 95% (issue #232).
79
+ */
80
+ passed: boolean;
81
+ /** The bar `passed` was measured against. */
82
+ minFidelity: number;
83
+ /** Errors the compiler reported building the project — recorded, and now also enforced. */
84
+ buildErrors: string[];
85
+ /**
86
+ * Per page, because the average cannot name one.
87
+ *
88
+ * "Average fidelity 84%" is a fact nobody can act on; "the pricing page renders at 61%" is a
89
+ * decision — retry it, patch it by hand, or accept it. The verifier computed both and only the
90
+ * average was reported, so the actionable half was thrown away one level below the caller.
91
+ *
92
+ * The counts alongside it are what a percentage cannot say: a page that 404s on fifteen images
93
+ * scores badly, and only `failedRequests` says why.
94
+ */
95
+ pages: {
96
+ route: string;
97
+ fidelity: number;
98
+ consoleErrors: number;
99
+ failedRequests: number;
100
+ error?: string;
101
+ }[];
102
+ } | null;
82
103
  warnings: string[];
83
104
  }
84
105
 
85
- function throwIfAborted(signal?: AbortSignal): void {
86
- if (signal?.aborted) {
87
- throw new Error("Import aborted");
88
- }
89
- }
90
-
91
- function formatBytes(bytes: number): string {
92
- if (bytes < 1024) {
93
- return `${bytes} B`;
94
- }
95
- if (bytes < 1024 * 1024) {
96
- return `${(bytes / 1024).toFixed(1)} KB`;
97
- }
98
- return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
99
- }
100
-
101
106
  /**
102
- * Run the heuristic componentize pass and refine it with the LLM. Returns undefined when the AI
103
- * pass is off, componentization is disabled, or the heuristic found nothing (the emit step then
104
- * runs its own heuristic pass as usual).
107
+ * What the CLI and the OSS server ask for when the caller named no model.
108
+ *
109
+ * It lives at the call site rather than inside `ai-componentize.ts`, because a default belongs to
110
+ * whoever knows the provider. This path talks to an OpenAI-compatible endpoint; a backend brokering
111
+ * Workers AI would 404 on this id, which is exactly why the module below it demands one.
105
112
  */
106
- async function maybeAiComponentize(
107
- pageMap: Map<string, JxElement>,
108
- componentizeOptions: false | { minInstances?: number; minDepth?: number },
109
- ai: ImportSiteOptions["ai"],
110
- progress: (phase: ImportPhase, message: string) => void,
111
- ): Promise<ComponentizeResult | undefined> {
112
- if (!ai || componentizeOptions === false) {
113
- return undefined;
114
- }
115
- const { componentize } = await import("./componentize.ts");
116
- const { aiComponentize } = await import("./ai-componentize.ts");
117
- progress("components", "Running heuristic componentization...");
118
- const heuristic = componentize(pageMap, componentizeOptions);
119
- if (heuristic.components.size === 0) {
120
- return undefined;
113
+ const DEFAULT_AI_MODEL = "gpt-4o-mini";
114
+
115
+ /** The one file the seed writes, and the only thing the empty-directory guard tolerates. */
116
+ const PROJECT_FILE = "project.json";
117
+
118
+ /** A placeholder project name from the URL, replaced by the page title at emit. */
119
+ function seedName(url: string): string {
120
+ try {
121
+ return new URL(url).hostname.replace(/^www\./, "") || "Imported Site";
122
+ } catch {
123
+ return "Imported Site";
121
124
  }
122
- progress("ai", `Found ${heuristic.components.size} component(s), refining with AI...`);
123
- const refined = await aiComponentize(
124
- heuristic,
125
- { apiKey: ai.apiKey, baseUrl: ai.baseUrl, model: ai.model },
126
- (msg) => progress("ai", msg),
127
- );
128
- progress("ai", `AI refined ${refined.components.size} component(s)`);
129
- return refined;
130
125
  }
131
126
 
132
127
  /**
@@ -139,22 +134,7 @@ export async function importSite(
139
134
  options: ImportSiteOptions,
140
135
  onProgress?: (e: ImportProgressEvent) => void,
141
136
  ): Promise<ImportSiteResult> {
142
- const {
143
- url,
144
- outDir,
145
- maxDepth = 2,
146
- maxPages = 25,
147
- maxNodesPerPage = 5000,
148
- styles = true,
149
- assets = true,
150
- scroll = true,
151
- respectRobots = true,
152
- componentize: componentizeOptions = {},
153
- ai = false,
154
- verify = false,
155
- chromePath,
156
- signal,
157
- } = options;
137
+ const { url, outDir, maxDepth = 2, ai = false, verify = false, chromePath, signal } = options;
158
138
 
159
139
  if (!url.startsWith("http://") && !url.startsWith("https://")) {
160
140
  throw new Error("URL must start with http:// or https://");
@@ -162,16 +142,16 @@ export async function importSite(
162
142
  if (!isAbsolute(outDir)) {
163
143
  throw new Error(`outDir must be an absolute path, got "${outDir}"`);
164
144
  }
165
- if (existsSync(outDir) && readdirSync(outDir).length > 0) {
145
+ /* The guard admits the run's OWN seed and nothing else. It has to: the destination is created and
146
+ given a `project.json` before the browser launches, so a host can open the project and watch the
147
+ rest of the run arrive in its file tree — and a guard that could not tell that seed from a
148
+ stranger's files would refuse every import at its second statement. */
149
+ if (existsSync(outDir) && readdirSync(outDir).some((entry) => entry !== PROJECT_FILE)) {
166
150
  throw new Error(`Directory "${outDir}" is not empty`);
167
151
  }
168
152
 
169
- const progress = (
170
- phase: ImportPhase,
171
- message: string,
172
- counts?: { current?: number; total?: number },
173
- ) => {
174
- onProgress?.({ phase, message, ...counts });
153
+ const progress = (phase: ImportPhase, message: string) => {
154
+ onProgress?.({ phase, message });
175
155
  };
176
156
  const warnings: string[] = [];
177
157
  const warn = (phase: ImportPhase, message: string) => {
@@ -179,350 +159,175 @@ export async function importSite(
179
159
  progress(phase, `⚠ ${message}`);
180
160
  };
181
161
  const verifyThreshold = verify === false ? 0.15 : (verify.threshold ?? 0.15);
162
+ const verifyMinFidelity = verify === false ? 0 : (verify.minFidelity ?? 0);
163
+ const verifyFullPage = verify === false ? true : (verify.fullPage ?? true);
182
164
 
183
165
  try {
184
- if (maxDepth === 0) {
185
- return await runSinglePage();
186
- }
187
- return await runCrawl();
188
- } finally {
189
- await closeBrowser();
190
- }
166
+ await seedProject();
191
167
 
192
- // ─── Single-page mode: Phase 0-2 pipeline, no crawl overhead ────────────────
193
- async function runSinglePage(): Promise<ImportSiteResult> {
194
168
  progress("launch", "Launching browser...");
195
169
  const browser = await launchBrowser(
196
170
  chromePath === undefined ? {} : { executablePath: chromePath },
197
171
  );
198
172
 
199
- throwIfAborted(signal);
200
- progress("capture", "Capturing page...");
201
- const capture = await capturePage(url, browser, { scrollToBottom: scroll });
202
- progress("capture", `Captured: "${capture.title}" (${capture.links.length} links found)`);
203
-
204
- progress("capture", "Converting to Jx...");
205
- const jx = convertToJx(capture.bodyHtml);
206
- progress(
207
- "capture",
208
- `Converted: ${jx.nodeCount} nodes, ${jx.collectedStyles.length} inline stylesheets collected`,
209
- );
210
- if (jx.nodeCount > maxNodesPerPage) {
211
- warn("capture", `Large page (${jx.nodeCount} nodes). Studio may be slow.`);
212
- }
213
-
214
- let breakpoints: Record<string, string> | undefined;
215
- let styleTokens: Record<string, string> | undefined;
216
-
217
- if (styles) {
218
- throwIfAborted(signal);
219
- progress("styles", "Capturing computed styles...");
220
- const styleResult = await captureStyles(capture.page);
221
- progress(
222
- "styles",
223
- `Captured styles for ${styleResult.elements.length} elements, ${Object.keys(styleResult.uaDefaults).length} tag baselines`,
224
- );
225
-
226
- progress("styles", "Diffing against UA defaults...");
227
- const diffed = diffAllStyles(styleResult.elements, styleResult.uaDefaults);
228
- progress("styles", `${diffed.length} elements with non-default styles`);
229
-
230
- // Replace resolved values with var(--name) references
231
- const propCount = Object.keys(styleResult.customProperties).length;
232
- if (propCount > 0) {
233
- const tokenResult = applyTokens(diffed, styleResult.customProperties);
234
- if (tokenResult.replacements > 0) {
235
- styleTokens = tokenResult.tokens;
236
- progress(
237
- "styles",
238
- `Extracted ${Object.keys(tokenResult.tokens).length} design tokens (${tokenResult.replacements} replacements)`,
239
- );
240
- } else {
241
- progress(
242
- "styles",
243
- `${propCount} custom properties found, but none matched computed values`,
244
- );
245
- }
246
- }
247
-
248
- if (styleResult.mediaQueries.length > 0) {
249
- progress(
250
- "styles",
251
- `Extracting @media breakpoints (${styleResult.mediaQueries.length} queries found)...`,
252
- );
253
- const media = await extractMedia(
254
- capture.page,
255
- styleResult.elements,
256
- styleResult.uaDefaults,
257
- styleResult.mediaQueries,
258
- );
259
- const bpCount = Object.keys(media.breakpoints).length;
260
- if (bpCount > 0) {
261
- progress("styles", `${bpCount} breakpoints with style changes`);
262
- ({ breakpoints } = media);
263
- applyStylesToTree(jx.document, diffed, media.deltas);
264
- } else {
265
- progress("styles", "No responsive breakpoints with style changes");
266
- applyStylesToTree(jx.document, diffed);
267
- }
268
- } else {
269
- progress("styles", "No @media queries found");
270
- applyStylesToTree(jx.document, diffed);
271
- }
272
-
273
- // Apply <html>/<body> computed styles to the root wrapper
274
- if (Object.keys(styleResult.documentStyles).length > 0) {
275
- if (!jx.document.style) {
276
- jx.document.style = {};
277
- }
278
- for (const [prop, val] of Object.entries(styleResult.documentStyles)) {
279
- const camel = kebabToCamel(prop);
280
- jx.document.style[camel] = val;
281
- }
282
- progress(
173
+ const result = await runImportPipeline(
174
+ {
175
+ url,
176
+ browser,
177
+ io: createLocalIo(outDir),
178
+ maxDepth,
179
+ /* The model default is applied here so the pipeline can require one. See DEFAULT_AI_MODEL. */
180
+ ai: ai === false ? false : { ...ai, model: ai.model ?? DEFAULT_AI_MODEL },
181
+ referenceScreenshots: verify === false ? false : { fullPage: verifyFullPage },
182
+ ...pick(options, [
183
+ "maxPages",
184
+ "maxNodesPerPage",
185
+ "breakpoints",
283
186
  "styles",
284
- `Applied ${Object.keys(styleResult.documentStyles).length} document-level styles to root`,
285
- );
286
- }
287
- }
288
-
289
- let fontFaceRules: string[] | undefined;
290
- let fontRewriteMap: Map<string, string> | undefined;
291
-
292
- if (assets) {
293
- throwIfAborted(signal);
294
- progress("assets", "Collecting asset URLs...");
295
- const collected = await collectAssets(capture.page);
296
- progress(
297
- "assets",
298
- `Found ${collected.assets.length} assets (${collected.inlineSvgCount} inline SVGs kept, ${collected.stylesheets.length} stylesheets retained)`,
299
- );
300
-
301
- const allFontRules = collected.stylesheets.flatMap((s) => s.fontFaceRules);
302
- if (allFontRules.length > 0) {
303
- fontFaceRules = allFontRules;
304
- progress("assets", `Found ${allFontRules.length} @font-face rules`);
305
- }
306
-
307
- if (collected.assets.length > 0) {
308
- progress("assets", "Downloading assets...");
309
- const downloaded = await downloadAssets(collected.assets, outDir, url);
310
- progress(
311
187
  "assets",
312
- `Downloaded ${downloaded.rewriteMap.size} assets (${formatBytes(downloaded.totalBytes)})`,
313
- );
314
- if (downloaded.failed.length > 0) {
315
- warn("assets", `${downloaded.failed.length} assets failed to download`);
316
- }
317
- if (downloaded.skipped.length > 0) {
318
- progress("assets", `Skipped ${downloaded.skipped.length} tracking/analytics URLs`);
319
- }
320
-
321
- if (fontFaceRules) {
322
- fontRewriteMap = new Map<string, string>();
323
- for (const [originalUrl, localPath] of downloaded.rewriteMap) {
324
- if (localPath.includes("/fonts/") || /\.(woff2?|ttf|otf|eot)$/i.test(localPath)) {
325
- fontRewriteMap.set(originalUrl, localPath);
326
- }
327
- }
328
- }
329
-
330
- progress("assets", "Rewriting asset URLs...");
331
- const rewrites = rewriteAssetUrls(jx.document, downloaded.rewriteMap, url);
332
- progress("assets", `Rewrote ${rewrites} URL references`);
333
- }
334
- }
335
-
336
- // Capture the reference screenshot before closing the page (for verify)
337
- let referenceScreenshot: Buffer | undefined;
338
- if (verify !== false) {
339
- progress("verify", "Capturing reference screenshot...");
340
- const { captureReferenceScreenshot } = await import("./verify.ts");
341
- referenceScreenshot = await captureReferenceScreenshot(capture.page);
342
- }
343
-
344
- await capture.page.close();
345
-
346
- throwIfAborted(signal);
347
- const pageMap = new Map([["pages/index.json", jx.document]]);
348
- const precomputedComponents = await maybeAiComponentize(
349
- pageMap,
350
- componentizeOptions,
351
- ai,
352
- progress,
353
- );
354
-
355
- throwIfAborted(signal);
356
- progress("emit", "Writing project...");
357
- const { files } = await emitMultiPageProject({
358
- outDir,
359
- title: capture.title,
360
- pages: pageMap,
361
- sourceUrl: url,
362
- breakpoints,
363
- componentizeOptions: precomputedComponents ? false : componentizeOptions,
364
- precomputedComponents,
365
- fontFaceRules,
366
- fontRewriteMap,
367
- styleTokens,
368
- });
369
- progress("emit", `Wrote ${files.length} files`);
370
-
371
- let verifySummary: ImportSiteResult["verify"] = null;
372
- if (verify !== false && referenceScreenshot) {
373
- verifySummary = await runVerify(
374
- new Map([["pages/index.json", { sourceUrl: url, screenshot: referenceScreenshot }]]),
375
- browser,
376
- );
377
- }
378
-
379
- progress("done", `Imported ${url} → ${outDir}`);
380
- return {
381
- outDir,
382
- pages: [{ route: "pages/index.json", title: capture.title, nodeCount: jx.nodeCount }],
383
- fileCount: files.length,
384
- verify: verifySummary,
385
- warnings,
386
- };
387
- }
388
-
389
- // ─── Multi-page crawl mode (Phase 3) ────────────────────────────────────────
390
- async function runCrawl(): Promise<ImportSiteResult> {
391
- progress("launch", "Launching browser...");
392
- await launchBrowser(chromePath === undefined ? {} : { executablePath: chromePath });
393
-
394
- throwIfAborted(signal);
395
- const result = await crawlSite({
396
- url,
397
- outDir,
398
- maxDepth,
399
- maxPages,
400
- maxNodesPerPage,
401
- skipStyles: !styles,
402
- skipAssets: !assets,
403
- respectRobots,
404
- noScroll: !scroll,
405
- captureScreenshots: verify !== false,
406
- onProgress: (msg) => progress("crawl", msg.trim()),
407
- ...(signal === undefined ? {} : { signal }),
408
- });
409
-
410
- progress(
411
- "crawl",
412
- `Crawled ${result.pages.length} page${result.pages.length === 1 ? "" : "s"}`,
413
- { current: result.pages.length, total: maxPages },
188
+ "scroll",
189
+ "respectRobots",
190
+ "componentize",
191
+ "signal",
192
+ ]),
193
+ },
194
+ onProgress,
414
195
  );
415
- if (result.skippedByRobots.length > 0) {
416
- progress("crawl", `Skipped ${result.skippedByRobots.length} URLs (robots.txt)`);
417
- }
418
- if (result.skippedByNodeCap.length > 0) {
419
- warn(
420
- "crawl",
421
- `${result.skippedByNodeCap.length} pages exceeded node cap (styles/assets skipped)`,
422
- );
423
- }
424
-
425
- const pageMap = new Map<string, JxElement>();
426
- for (const page of result.pages) {
427
- pageMap.set(page.route, page.jx.document);
428
- }
429
-
430
- // Layout detection
431
- let layout: JxElement | undefined;
432
- if (result.pages.length >= 2) {
433
- throwIfAborted(signal);
434
- progress("layout", "Detecting shared layout (header/footer)...");
435
- const layoutResult = detectLayout(pageMap);
436
- if (layoutResult) {
437
- ({ layout } = layoutResult);
438
- const sharedCount = (Array.isArray(layout.children) ? layout.children.length : 0) - 1; // Minus the slot
439
- progress("layout", `Found shared layout with ${sharedCount} shared elements`);
440
- for (const [route, stripped] of layoutResult.strippedPages) {
441
- pageMap.set(route, stripped);
442
- }
443
- } else {
444
- progress("layout", "No shared layout detected");
445
- }
446
- }
447
-
448
- throwIfAborted(signal);
449
- const precomputedComponents = await maybeAiComponentize(
450
- pageMap,
451
- componentizeOptions,
452
- ai,
453
- progress,
454
- );
455
-
456
- throwIfAborted(signal);
457
- progress("emit", "Writing project...");
458
- const title = result.pages[0]?.title || new URL(url).hostname;
459
- const { files } = await emitMultiPageProject({
460
- outDir,
461
- title,
462
- sourceUrl: url,
463
- pages: pageMap,
464
- layout,
465
- breakpoints: result.breakpoints,
466
- componentizeOptions: precomputedComponents ? false : componentizeOptions,
467
- precomputedComponents,
468
- fontFaceRules: result.fontFaceRules.length > 0 ? result.fontFaceRules : undefined,
469
- fontRewriteMap: result.fontRewriteMap.size > 0 ? result.fontRewriteMap : undefined,
470
- styleTokens: result.styleTokens,
471
- });
472
- progress("emit", `Wrote ${files.length} files`);
196
+ warnings.push(...result.warnings);
473
197
 
474
198
  let verifySummary: ImportSiteResult["verify"] = null;
475
199
  if (verify !== false) {
476
- const verifyPages = new Map<string, { sourceUrl: string; screenshot: Buffer | string }>();
477
- for (const page of result.pages) {
478
- if (page.screenshot) {
479
- verifyPages.set(page.route, { sourceUrl: page.url, screenshot: page.screenshot });
480
- }
481
- }
482
- if (verifyPages.size === 0) {
200
+ if (result.references.size === 0) {
483
201
  progress("verify", "No reference screenshots captured — skipping verify");
484
202
  } else {
485
- verifySummary = await runVerify(verifyPages);
203
+ /* Only the single-page run hands the verifier its browser. A crawl's pages were captured
204
+ and closed one at a time, so by now the connection has outlived every page it opened and
205
+ `verifyProject` launching its own is the same cost either way. */
206
+ verifySummary = await runVerify(result.references, maxDepth === 0 ? browser : undefined);
486
207
  }
487
208
  }
488
209
 
489
210
  progress("done", `Imported ${url} → ${outDir}`);
490
211
  return {
491
212
  outDir,
492
- pages: result.pages.map((p) => ({
493
- route: p.route,
494
- title: p.title,
495
- nodeCount: p.jx.nodeCount,
496
- })),
497
- fileCount: files.length,
213
+ pages: result.pages,
214
+ fileCount: result.files.length,
498
215
  verify: verifySummary,
499
216
  warnings,
500
217
  };
218
+ } finally {
219
+ await closeBrowser();
220
+ }
221
+
222
+ /**
223
+ * Create the destination and give it a `project.json` valid enough to open.
224
+ *
225
+ * The emit phase rewrites this file completely, so nothing here is a guess at what the import
226
+ * will find — it is the minimum that makes the directory a PROJECT, so the caller can open it now
227
+ * instead of at the end. Everything a crawl writes then lands in a file tree somebody is
228
+ * watching.
229
+ */
230
+ async function seedProject(): Promise<void> {
231
+ await mkdir(outDir, { recursive: true });
232
+ const seed = { name: seedName(url), imports: {}, images: { optimize: false } };
233
+ await Bun.write(join(outDir, PROJECT_FILE), `${JSON.stringify(seed, null, 2)}\n`);
234
+ onProgress?.({
235
+ message: `Created ${outDir}`,
236
+ phase: "seed",
237
+ root: outDir,
238
+ });
501
239
  }
502
240
 
503
241
  // ─── Phase 5: build + screenshot-diff against the original ─────────────────
504
242
  async function runVerify(
505
- verifyPages: Map<string, { sourceUrl: string; screenshot: Buffer | string }>,
243
+ references: Map<string, { sourceUrl: string; screenshot: Uint8Array }>,
506
244
  browser?: Browser,
507
245
  ): Promise<ImportSiteResult["verify"]> {
508
246
  throwIfAborted(signal);
509
247
  progress("verify", "Verifying against the original...");
510
248
  const { verifyProject } = await import("./verify.ts");
249
+ const verifyPages = new Map<string, { sourceUrl: string; screenshot: Buffer | string }>();
250
+ for (const [route, ref] of references) {
251
+ verifyPages.set(route, { sourceUrl: ref.sourceUrl, screenshot: Buffer.from(ref.screenshot) });
252
+ }
511
253
  const verifyResult = await verifyProject({
512
254
  projectDir: outDir,
513
255
  pages: verifyPages,
514
256
  threshold: verifyThreshold,
257
+ minFidelity: verifyMinFidelity,
258
+ fullPage: verifyFullPage,
515
259
  onProgress: (msg) => progress("verify", msg),
516
260
  ...(browser === undefined ? {} : { browser }),
517
261
  });
262
+ /*
263
+ * Build errors were logged inside the verifier and dropped there. They are warnings on the
264
+ * import as a whole: a project that did not compile is not a clone of anything, however the
265
+ * pixels happen to score.
266
+ */
267
+ for (const error of verifyResult.buildErrors) {
268
+ warn("verify", `Build error: ${error}`);
269
+ }
518
270
  for (const page of verifyResult.pages) {
519
271
  const status = page.error ? `ERROR: ${page.error}` : `${page.fidelity}% fidelity`;
520
- progress("verify", `${page.route} — ${status}`);
272
+ const misses =
273
+ page.failedRequests.length > 0 ? `, ${page.failedRequests.length} failed request(s)` : "";
274
+ progress("verify", `${page.route} — ${status}${misses}`);
521
275
  }
522
276
  progress("verify", `Average fidelity: ${verifyResult.averageFidelity}%`);
277
+ if (!verifyResult.passed) {
278
+ warn(
279
+ "verify",
280
+ `Verification failed: average fidelity ${verifyResult.averageFidelity}% ` +
281
+ `(minimum ${verifyMinFidelity}%). See ${verifyResult.reportDir}/report.json`,
282
+ );
283
+ }
523
284
  return {
524
285
  averageFidelity: verifyResult.averageFidelity,
525
286
  reportDir: verifyResult.reportDir,
287
+ passed: verifyResult.passed,
288
+ minFidelity: verifyMinFidelity,
289
+ buildErrors: verifyResult.buildErrors,
290
+ pages: verifyResult.pages.map((page) => {
291
+ const entry: {
292
+ route: string;
293
+ fidelity: number;
294
+ consoleErrors: number;
295
+ failedRequests: number;
296
+ error?: string;
297
+ } = {
298
+ route: page.route,
299
+ fidelity: page.fidelity,
300
+ consoleErrors: page.consoleErrors.length,
301
+ failedRequests: page.failedRequests.length,
302
+ };
303
+ if (page.error !== undefined) {
304
+ entry.error = page.error;
305
+ }
306
+ return entry;
307
+ }),
526
308
  };
527
309
  }
528
310
  }
311
+
312
+ function throwIfAborted(signal?: AbortSignal): void {
313
+ if (signal?.aborted) {
314
+ throw new Error("Import aborted");
315
+ }
316
+ }
317
+
318
+ /**
319
+ * Forward the options this wrapper shares with the pipeline, and only the ones actually given.
320
+ *
321
+ * `exactOptionalPropertyTypes` is on, so `{ signal: undefined }` is a different thing from an
322
+ * absent `signal` and the pipeline would reject it. Copying present keys is also what keeps every
323
+ * default in ONE place — restating `maxPages = 25` here is how the CLI and the cloud drift apart.
324
+ */
325
+ function pick<T extends object, K extends keyof T>(source: T, keys: readonly K[]): Pick<T, K> {
326
+ const out = {} as Pick<T, K>;
327
+ for (const key of keys) {
328
+ if (source[key] !== undefined) {
329
+ out[key] = source[key];
330
+ }
331
+ }
332
+ return out;
333
+ }