@jxsuite/import 0.39.8 → 0.40.1

package/src/capture.ts

@@ -1,5 +1,44 @@
-import { launch } from "puppeteer-core";
-import type { Browser, Page } from "puppeteer-core";
+/**
+ * Page capture, and the browser surface the whole import is written against.
+ *
+ * That surface is declared STRUCTURALLY here rather than imported from `puppeteer-core`, and the
+ * reason is bundling, not taste: `puppeteer-core` is unloadable in workerd, and a Worker bundler
+ * follows value imports transitively, so a single one anywhere in `pipeline.ts`'s graph would fail
+ * the deploy. Naming only `newPage` — and, on a page, the five calls the phases actually make —
+ * lets the same code drive a local Chrome (`browser-local.ts`) or a `@cloudflare/puppeteer`
+ * session, both of which provide exactly this and more.
+ */
+
+/**
+ * What `evaluate` hands the in-page function.
+ *
+ * A homomorphic mapped type rather than plain `Params`, because puppeteer's own `evaluate` declares
+ * its parameters through one (it unwraps `JSHandle`s there). Two generic signatures only relate
+ * when they are shaped alike, and with a bare tuple puppeteer's `Page` stops being assignable to
+ * `ImportPage` — which is the one thing this interface has to be true of.
+ */
+export type PageArgs<Params extends unknown[]> = { [K in keyof Params]: Params[K] };
+
+export interface ImportPage {
+  goto: (
+    url: string,
+    options?: {
+      waitUntil?: "load" | "domcontentloaded" | "networkidle0" | "networkidle2";
+      timeout?: number;
+    },
+  ) => Promise<unknown>;
+  setViewport: (viewport: { width: number; height: number }) => Promise<unknown>;
+  evaluate: <Params extends unknown[], R>(
+    fn: (...params: PageArgs<Params>) => R,
+    ...args: Params
+  ) => Promise<Awaited<R>>;
+  screenshot: (options?: { fullPage?: boolean; type?: "png" }) => Promise<Uint8Array>;
+  close: () => Promise<unknown>;
+}
+
+export interface ImportBrowser {
+  newPage: () => Promise<ImportPage>;
+}
 
 export interface CaptureResult {
   url: string;
@@ -7,61 +46,8 @@ export interface CaptureResult {
   bodyHtml: string;
   /** Discovered same-origin links for the crawler (Phase 3). */
   links: string[];
-  /** The puppeteer Page, kept open for style capture (Phase 1). Caller must close. */
-  page: Page;
-}
-
-const DEFAULT_CHROME_PATHS = [
-  "google-chrome-stable",
-  "google-chrome",
-  "chromium-browser",
-  "chromium",
-];
-
-function findChrome(executablePath?: string): string {
-  if (executablePath) {
-    return executablePath;
-  }
-  const env = process.env.CHROME_PATH;
-  if (env) {
-    return env;
-  }
-
-  for (const name of DEFAULT_CHROME_PATHS) {
-    const path = Bun.which(name);
-    if (path) {
-      return path;
-    }
-  }
-  throw new Error(
-    "Could not find Chrome/Chromium. Set CHROME_PATH or install google-chrome-stable.",
-  );
-}
-
-let _browser: Browser | null = null;
-
-export interface LaunchOptions {
-  /** Explicit browser binary. Wins over CHROME_PATH and PATH discovery. */
-  executablePath?: string;
-}
-
-export async function launchBrowser(options?: LaunchOptions): Promise<Browser> {
-  if (_browser?.connected) {
-    return _browser;
-  }
-  _browser = await launch({
-    executablePath: findChrome(options?.executablePath),
-    headless: true,
-    args: ["--no-sandbox", "--disable-gpu"],
-  });
-  return _browser;
-}
-
-export async function closeBrowser(): Promise<void> {
-  if (_browser?.connected) {
-    await _browser.close();
-    _browser = null;
-  }
+  /** The page, kept open for style capture (Phase 1). Caller must close. */
+  page: ImportPage;
 }
 
 export interface CaptureOptions {
@@ -73,7 +59,7 @@ export interface CaptureOptions {
  * Scroll the page to the bottom in steps to trigger lazy-loaded images and intersection-observer
  * content, then scroll back to top. Settles between steps to let content render.
  */
-async function scrollToRevealAll(page: Page): Promise<void> {
+async function scrollToRevealAll(page: ImportPage): Promise<void> {
   await page.evaluate(async () => {
     const delay = (ms: number) =>
       new Promise<void>((r) => {
@@ -111,15 +97,18 @@ async function scrollToRevealAll(page: Page): Promise<void> {
 /**
  * Capture a page's DOM. Returns the page object still open — the caller is responsible for closing
  * it (after style capture in Phase 1, or immediately).
+ *
+ * The browser is a parameter and has no default. It used to fall back to launching a local Chrome,
+ * and that fallback was the value import of `puppeteer-core` that kept this module — and everything
+ * that reaches it — out of a Worker.
  */
 export async function capturePage(
   url: string,
-  browser?: Browser,
+  browser: ImportBrowser,
   options?: CaptureOptions,
 ): Promise<CaptureResult> {
   const { scrollToBottom = true } = options ?? {};
-  const br = browser ?? (await launchBrowser());
-  const page: Page = await br.newPage();
+  const page = await browser.newPage();
   await page.setViewport({ width: 1440, height: 900 });
 
   await page.goto(url, { waitUntil: "networkidle0", timeout: 30_000 });

package/src/cli.ts

@@ -3,6 +3,13 @@
 import { resolve } from "node:path";
 import { homedir } from "node:os";
 import { importSite } from "./run.ts";
+import {
+  DEFAULT_BREAKPOINT_POLICY,
+  MAX_BREAKPOINTS,
+  MAX_BREAKPOINT_WIDTH,
+  MIN_BREAKPOINT_WIDTH,
+} from "./breakpoint-plan.ts";
+import type { BreakpointPolicy, BreakpointRounding } from "./breakpoint-plan.ts";
 
 function usage(): never {
   console.log(`Usage: jx-import <url> [options]
@@ -14,6 +21,10 @@ Options:
   --depth <n>              Max crawl depth (default: 2, 0 = single page)
   --max-pages <n>          Max pages to capture (default: 25)
   --max-nodes-per-page <n> Skip styles/assets for pages above this (default: 5000)
+  --max-breakpoints <n>    Keep at most n of the site's breakpoints (default: 3, 0 = keep all)
+  --breakpoints <list>     Keep these widths instead, e.g. 640,1024,1440
+  --breakpoint-rounding <r> How a kept width matches a declared one: nearest|down|up
+                           (default: nearest)
   --no-styles              Skip CSS capture
   --no-assets              Skip asset download
   --no-crawl               Single page only (equivalent to --depth 0)
@@ -25,7 +36,10 @@ Options:
   --ai-components          Use LLM to refine component names and props (Phase 4 AI pass)
   --ai-model <model>       Model for AI componentization (default: gpt-4o-mini)
   --verify                 After import, build and screenshot-diff vs original (Phase 5)
-  --verify-threshold <n>   Pixel diff threshold 0..1 (default: 0.15)
+  --min-fidelity <n>       Fail (exit 1) below this average fidelity 0..100 (default: 25)
+  --verify-threshold <n>   Per-pixel colour tolerance 0..1 for the diff (default: 0.15).
+                           Moves the score; it is NOT the pass bar — --min-fidelity is.
+  --verify-viewport-only   Diff only the first viewport instead of the whole page
 
 Environment:
   OPENAI_API_KEY           Required for --ai-components
@@ -37,6 +51,8 @@ Examples:
   jx-import https://example.com --depth 1 --max-pages 10
   jx-import https://example.com --no-crawl
   jx-import https://example.com --out sites/my-clone --no-styles
+  jx-import https://example.com --breakpoints 640,1024,1440
+  jx-import https://example.com --max-breakpoints 0
   jx-import https://example.com --verify
   jx-import https://example.com --ai-components`);
   process.exit(1);
@@ -73,7 +89,60 @@ const verifyThreshold =
   verifyThresholdIdx !== -1 && args[verifyThresholdIdx + 1]
     ? Number(args[verifyThresholdIdx + 1]) || 0.15
     : 0.15;
+/*
+ * The floor below which the output is not a clone of anything.
+ *
+ * It is deliberately low rather than a quality bar: a real import of a complicated site lands well
+ * under 100 for reasons nobody can fix from here (a rotating hero, a font that renders a hair
+ * differently), and failing those would train people to pass `--min-fidelity 0` and stop looking.
+ * Under a quarter of pixels matching is a different question — that is the 8%-fidelity Wix clone
+ * that printed `Done!` and exited 0, which is worse than a failure because the only way to see it
+ * is to open the result (issue #232).
+ */
+const minFidelity = parseIntArg(args, "--min-fidelity", 25);
+const verifyFullPage = !args.includes("--verify-viewport-only");
 
+/**
+ * The breakpoint policy from the flags.
+ *
+ * `--breakpoints` wins over `--max-breakpoints` because it is the more specific answer: naming the
+ * widths says everything a count could have.
+ */
+function breakpointPolicy(): BreakpointPolicy {
+  const roundingIdx = args.indexOf("--breakpoint-rounding");
+  const roundingArg = roundingIdx === -1 ? "" : (args[roundingIdx + 1] ?? "");
+  const rounding: BreakpointRounding =
+    roundingArg === "down" || roundingArg === "up" ? roundingArg : "nearest";
+
+  const widthsIdx = args.indexOf("--breakpoints");
+  const widthsArg = widthsIdx === -1 ? "" : (args[widthsIdx + 1] ?? "");
+  if (widthsArg) {
+    const widths = widthsArg
+      .split(",")
+      .map((part) => Math.trunc(Number(part.trim())))
+      .filter((n) => Number.isFinite(n) && n >= MIN_BREAKPOINT_WIDTH && n <= MAX_BREAKPOINT_WIDTH);
+    if (widths.length === 0) {
+      console.error(
+        `Error: --breakpoints needs widths between ${MIN_BREAKPOINT_WIDTH} and ` +
+          `${MAX_BREAKPOINT_WIDTH}, e.g. --breakpoints 640,1024,1440`,
+      );
+      process.exit(1);
+    }
+    return { mode: "explicit", rounding, widths };
+  }
+
+  if (!args.includes("--max-breakpoints")) {
+    return DEFAULT_BREAKPOINT_POLICY;
+  }
+  const count = parseIntArg(args, "--max-breakpoints", 3);
+  // Zero is the only spelling of "keep all" a count can have, and it reads better than omitting the
+  // Flag when a script is choosing between the two.
+  return count <= 0
+    ? { mode: "all" }
+    : { count: Math.min(MAX_BREAKPOINTS, count), mode: "limit", rounding };
+}
+
+const breakpoints = breakpointPolicy();
 const maxDepth = noCrawl ? 0 : parseIntArg(args, "--depth", 2);
 const maxPages = parseIntArg(args, "--max-pages", 25);
 const maxNodesPerPage = parseIntArg(args, "--max-nodes-per-page", 5000);
@@ -116,6 +185,7 @@ async function main() {
       {
         url: url as string,
         outDir,
+        breakpoints,
         maxDepth,
         maxPages,
         maxNodesPerPage,
@@ -125,7 +195,9 @@ async function main() {
         respectRobots: !args.includes("--no-robots"),
         componentize: noComponents ? false : { minInstances, minDepth },
         ai,
-        verify: doVerify ? { threshold: verifyThreshold } : false,
+        verify: doVerify
+          ? { threshold: verifyThreshold, minFidelity, fullPage: verifyFullPage }
+          : false,
       },
       (e) => console.log(`  ${e.message}`),
     );
@@ -136,9 +208,41 @@ async function main() {
     }
     if (result.verify) {
       console.log(`  Average fidelity: ${result.verify.averageFidelity}%`);
+      for (const page of result.verify.pages) {
+        const misses = [
+          page.failedRequests > 0 ? `${page.failedRequests} failed request(s)` : "",
+          page.consoleErrors > 0 ? `${page.consoleErrors} console error(s)` : "",
+        ].filter(Boolean);
+        if (misses.length > 0) {
+          console.log(`    ${page.route}: ${misses.join(", ")}`);
+        }
+      }
       console.log(`  Report: ${result.verify.reportDir}/report.json`);
     }
 
+    /*
+     * A verify that cannot fail is a report, not a gate. An import nobody opens and looks at is
+     * exactly the case the exit code exists for, so a build error or a fidelity under the floor
+     * ends the run non-zero and says which one it was.
+     */
+    if (result.verify && !result.verify.passed) {
+      for (const error of result.verify.buildErrors) {
+        console.error(`Build error: ${error}`);
+      }
+      console.error(
+        `\nVerification FAILED — the project is at ${outDir}, but it does not match the original.`,
+      );
+      if (result.verify.buildErrors.length === 0) {
+        console.error(
+          `  Average fidelity ${result.verify.averageFidelity}% is below the ` +
+            `${result.verify.minFidelity}% minimum. Pass --min-fidelity to change it.`,
+        );
+      }
+      console.error(`  Report: ${result.verify.reportDir}/report.json`);
+      process.exitCode = 1;
+      return;
+    }
+
     console.log(`\nDone! Open in Studio:`);
     console.log(
       `  http://localhost:3000/packages/studio/index.html?project=${outDir}/project.json`,

package/src/componentize.ts

@@ -54,6 +54,16 @@ function structuralHash(node: JxElement | string): string {
   return `<${tag}>${childHashes.join(",")}`;
 }
 
+/**
+ * Attributes that may not become a component prop.
+ *
+ * A prop is something a person fills in per instance, and a class is not one: `emit.ts` strips the
+ * source site's classes on the way out (`strip-classes.ts`), so a class that happened to differ
+ * between two instances of the same block would otherwise mint a `${state.class}` prop whose value
+ * is then deleted — a prop with a name nobody chose, wired to nothing.
+ */
+const PRESENTATION_ATTRS = new Set(["class", "className"]);
+
 function collectLeafValues(node: JxElement | string, prefix: string, out: Map<string, string>) {
   if (typeof node === "string") {
     out.set(`${prefix}.$text`, node);
@@ -64,6 +74,9 @@ function collectLeafValues(node: JxElement | string, prefix: string, out: Map<st
   }
   if (node.attributes) {
     for (const [k, v] of Object.entries(node.attributes)) {
+      if (PRESENTATION_ATTRS.has(k)) {
+        continue;
+      }
       if (typeof v === "string" || typeof v === "number" || typeof v === "boolean") {
         out.set(`${prefix}.attr.${k}`, String(v));
       }

package/src/crawl.ts

@@ -1,10 +1,17 @@
-import { launchBrowser, capturePage } from "./capture.ts";
-import type { CaptureResult } from "./capture.ts";
+import { capturePage } from "./capture.ts";
+import type { CaptureResult, ImportBrowser } from "./capture.ts";
+import type { ImportIo } from "./io.ts";
 import { convertToJx } from "./to-jx.ts";
 import type { ToJxResult } from "./to-jx.ts";
 import { captureStyles } from "./style-capture.ts";
 import { diffAllStyles, kebabToCamel } from "./style-diff.ts";
 import { extractMedia } from "./media-extract.ts";
+import {
+  DEFAULT_BREAKPOINT_POLICY,
+  analyzeMediaQueries,
+  planBreakpoints,
+} from "./breakpoint-plan.ts";
+import type { Breakpoint, BreakpointPolicy } from "./breakpoint-plan.ts";
 import { applyStylesToTree } from "./apply-styles.ts";
 import { collectAssets } from "./asset-collect.ts";
 import { downloadAssets } from "./asset-download.ts";
@@ -13,7 +20,10 @@ import { applyTokens } from "./css-tokens.ts";
 
 export interface CrawlOptions {
   url: string;
-  outDir: string;
+  /** The browser to capture with. The crawl never launches one — see `browser-local.ts`. */
+  browser: ImportBrowser;
+  /** Where downloaded assets go. */
+  io: ImportIo;
   maxDepth: number;
   maxPages: number;
   maxNodesPerPage: number;
@@ -24,6 +34,16 @@ export interface CrawlOptions {
   noScroll?: boolean;
   /** Capture a reference screenshot of each page before closing (for --verify). */
   captureScreenshots?: boolean;
+  /**
+   * Capture the whole scrollable page rather than the viewport (default true).
+   *
+   * It has to agree with what `verifyProject` screenshots on the other side. A viewport reference
+   * diffed against a full-page render is compared by padding the shorter image, so the two would
+   * disagree over everything below the fold before a single style was compared.
+   */
+  fullPageScreenshots?: boolean;
+  /** Which of the site's declared breakpoints the project keeps (see `breakpoint-plan.ts`). */
+  breakpoints?: BreakpointPolicy | undefined;
   onProgress?: (msg: string) => void;
   /** Abort the crawl between pages; the loop throws before capturing the next page. */
   signal?: AbortSignal;
@@ -36,8 +56,8 @@ export interface CrawledPage {
   jx: ToJxResult;
   depth: number;
   links: string[];
-  /** Reference screenshot PNG buffer, present when captureScreenshots is true. */
-  screenshot?: Buffer | undefined;
+  /** Reference screenshot PNG bytes, present when captureScreenshots is true. */
+  screenshot?: Uint8Array | undefined;
 }
 
 export interface CrawlResult {
@@ -171,18 +191,19 @@ function isDisallowed(url: string, disallowedPaths: Set<string>): boolean {
 export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
   const {
     url,
-    outDir,
+    browser,
+    io,
     maxDepth,
     maxPages,
     maxNodesPerPage,
     skipStyles,
     skipAssets,
     respectRobots,
+    fullPageScreenshots = true,
     onProgress = console.log,
   } = options;
 
   const { origin } = new URL(url);
-  const browser = await launchBrowser();
 
   // Fetch robots.txt
   let disallowedPaths = new Set<string>();
@@ -202,6 +223,9 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
   const skippedByRobots: string[] = [];
   const skippedByNodeCap: string[] = [];
   let mergedBreakpoints: Record<string, string> | undefined;
+  /* Decided on the first page that declares a width query, then reused. See ExtractMediaOptions.plan
+     — a per-page plan would union back up to the count the policy just refused. */
+  let breakpointPlan: Breakpoint[] | undefined;
   let mergedTokens: Record<string, string> | undefined;
   const allFontFaceRules: string[] = [];
   const fontRewriteMap = new Map<string, string>();
@@ -241,9 +265,9 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
         `  ⚠ Page exceeds node cap (${jx.nodeCount} > ${maxNodesPerPage}), skipping styles/assets`,
       );
       skippedByNodeCap.push(entry.url);
-      let screenshot: Buffer | undefined;
+      let screenshot: Uint8Array | undefined;
       if (options.captureScreenshots) {
-        screenshot = Buffer.from(await capture.page.screenshot({ type: "png" }));
+        screenshot = await capture.page.screenshot({ fullPage: fullPageScreenshots, type: "png" });
       }
       await capture.page.close();
 
@@ -277,11 +301,25 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
         }
 
         if (styleResult.mediaQueries.length > 0) {
+          if (!breakpointPlan) {
+            const declared = analyzeMediaQueries(styleResult.mediaQueries);
+            breakpointPlan = planBreakpoints(
+              declared,
+              options.breakpoints ?? DEFAULT_BREAKPOINT_POLICY,
+            ).keep;
+            if (breakpointPlan.length < declared.length) {
+              onProgress?.(
+                `  ${declared.length} breakpoints declared, keeping ` +
+                  `${breakpointPlan.length}: ${breakpointPlan.map((b) => b.name).join(", ")}\n`,
+              );
+            }
+          }
           const media = await extractMedia(
             capture.page,
             styleResult.elements,
             styleResult.uaDefaults,
             styleResult.mediaQueries,
+            { plan: breakpointPlan },
           );
           const bpCount = Object.keys(media.breakpoints).length;
           if (bpCount > 0) {
@@ -327,7 +365,7 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
         }
 
         if (collected.assets.length > 0) {
-          const downloaded = await downloadAssets(collected.assets, outDir, entry.url);
+          const downloaded = await downloadAssets(collected.assets, io, entry.url);
           if (downloaded.rewriteMap.size > 0) {
             rewriteAssetUrls(jx.document, downloaded.rewriteMap, entry.url);
             // Collect font rewrites (R2)
@@ -345,9 +383,9 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
       }
     }
 
-    let screenshot: Buffer | undefined;
+    let screenshot: Uint8Array | undefined;
     if (options.captureScreenshots) {
-      screenshot = Buffer.from(await capture.page.screenshot({ type: "png" }));
+      screenshot = await capture.page.screenshot({ fullPage: fullPageScreenshots, type: "png" });
     }
     await capture.page.close();