@jxsuite/import 0.39.8 → 0.40.1

package/src/srcset.ts

@@ -0,0 +1,32 @@
+/**
+ * Where one `srcset` entry ends and the next begins.
+ *
+ * Its own module because two passes have to agree about it: `asset-collect.ts` decides from it what
+ * gets DOWNLOADED, and `asset-rewrite.ts` reassembles the attribute from it. If they disagreed the
+ * importer would fetch one set of URLs and rewrite another.
+ */
+
+/**
+ * A `srcset` separator is a comma that starts a NEW URL. A comma inside a URL never is.
+ *
+ * Splitting on every comma shreds any URL carrying commas in its own path, and Wix, Cloudinary and
+ * imgix all encode image transforms exactly that way — `.../fill/w_375,h_127,al_c,q_85/logo.png`
+ * became a dozen unfetchable fragments. On one Wix site that was 109 failed downloads, and because
+ * a failed download leaves the reference untouched, 46 `static.wixstatic.com` URLs stayed in the
+ * emitted page: the "clone" served its images from the host it had cloned (issue #231).
+ *
+ * The lookahead lists the ways a URL can begin — absolute, `data:`, protocol-relative, root- or
+ * dot-relative, or a bare `name.ext` — which is what distinguishes a separator from a transform
+ * parameter. No `g` flag: `String.split` needs none, and a sticky `lastIndex` shared across calls
+ * is a bug waiting to happen.
+ */
+export const SRCSET_SEPARATOR =
+  /,(?=\s*(?:https?:\/\/|data:|\/\/|\/|\.{1,2}\/|[A-Za-z0-9_-]+\.[A-Za-z]{2,5}[/?#\s]))/;
+
+/** The URL of each entry in a `srcset`, descriptors (`2x`, `800w`) dropped. */
+export function parseSrcset(srcset: string): string[] {
+  return srcset
+    .split(SRCSET_SEPARATOR)
+    .map((entry) => entry.trim().split(/\s+/)[0] ?? "")
+    .filter(Boolean);
+}

package/src/strip-classes.ts

@@ -0,0 +1,73 @@
+/**
+ * Strip the source site's CSS classes out of an imported tree.
+ *
+ * The importer never resolves a class to a style. Every visual fact it emits comes from
+ * `getComputedStyle` (`style-capture.ts` → `style-diff.ts` → `apply-styles.ts`), and the only CSS
+ * it writes is `public/assets/fonts.css`, built from `@font-face` rules alone — the original
+ * stylesheets are collected for their fonts and then discarded. So an imported `class="hero
+ * grid-cols-3 lg:pt-12"` names rules that do not exist in the emitted project: it is dead weight
+ * that survives into every page, layout and component a reader then has to edit around.
+ *
+ * It is worse than inert in one place. `componentize.ts` treats every attribute as a prop
+ * candidate, so a class that happens to differ between two instances of the same block becomes a
+ * `${state.class}` prop with a name nobody chose. That is handled at the source there; this module
+ * is what guarantees nothing reaches disk regardless.
+ *
+ * **This is not a claim that Jx is classless.** `className` is a first-class schema property
+ * (`specs/spec.md` §8.1) and the compiler emits generated classes for nested style rules (§9.2);
+ * neither changes. What is removed here is one specific thing: class names imported from a site
+ * whose stylesheets were not imported with them.
+ *
+ * `id`, `role` and `aria-*` are deliberately left alone — anchors and accessibility survive the
+ * round trip, and unlike a class they mean the same thing without the original CSS.
+ *
+ * @docs studio/projects/create
+ */
+
+import type { JxElement } from "@jxsuite/schema/types";
+
+/** The two spellings a class can arrive in — `attributes.class` from HTML, `className` from Jx. */
+const CLASS_KEYS = ["class", "className"] as const;
+
+/**
+ * Remove class names from `node` and every descendant, in place.
+ *
+ * @param {JxElement | string} node - A tree node; strings (text) pass through untouched.
+ * @returns {number} How many class names were removed, for the progress line.
+ */
+export function stripClasses(node: JxElement | string): number {
+  if (typeof node !== "object" || node === null) {
+    return 0;
+  }
+
+  let removed = 0;
+
+  if ("className" in node && node.className !== undefined) {
+    delete node.className;
+    removed += 1;
+  }
+
+  const { attributes } = node;
+  if (attributes && typeof attributes === "object") {
+    for (const key of CLASS_KEYS) {
+      if (key in attributes) {
+        delete (attributes as Record<string, unknown>)[key];
+        removed += 1;
+      }
+    }
+    /* An `attributes` map emptied by the strip is deleted rather than left as `{}`: the emitted JSON
+       is read by a person, and a bare `"attributes": {}` on every node is the same noise the classes
+       were. `htmlToJx` never writes an empty one either, so this restores its own invariant. */
+    if (Object.keys(attributes).length === 0) {
+      delete node.attributes;
+    }
+  }
+
+  if (Array.isArray(node.children)) {
+    for (const child of node.children) {
+      removed += stripClasses(child as JxElement | string);
+    }
+  }
+
+  return removed;
+}

package/src/style-capture.ts

@@ -6,7 +6,7 @@
  * UA-default baselines per unique tagName by rendering throwaway elements.
  */
 
-import type { Page } from "puppeteer-core";
+import type { ImportPage } from "./capture.ts";
 
 export interface CapturedStyle {
   /** Depth-first element-index path from <body>, e.g. [0, 2, 1]. */
@@ -163,7 +163,7 @@ const STYLE_ALLOWLIST: string[] = [
  * Run the style capture in a puppeteer page. Executes entirely in-browser via page.evaluate() — no
  * round-trips per element.
  */
-export async function captureStyles(page: Page): Promise<StyleCaptureResult> {
+export async function captureStyles(page: ImportPage): Promise<StyleCaptureResult> {
   return page.evaluate((allowlist: string[]) => {
     const elements: { path: number[]; tagName: string; styles: Record<string, string> }[] = [];
     const tagsSeen = new Set<string>();
@@ -337,7 +337,10 @@ export async function captureStyles(page: Page): Promise<StyleCaptureResult> {
  * Re-capture computed styles at a different viewport width (for $media extraction). Returns only
  * the per-element styles — no UA defaults or media queries (reuse from base).
  */
-export async function captureStylesAtWidth(page: Page, width: number): Promise<CapturedStyle[]> {
+export async function captureStylesAtWidth(
+  page: ImportPage,
+  width: number,
+): Promise<CapturedStyle[]> {
   await page.setViewport({ width, height: 900 });
   // Let layout reflow settle
   await page.evaluate(

package/src/verify.ts

@@ -7,10 +7,10 @@
  * during import.
  */
 
-import { join } from "node:path";
+import { extname, join } from "node:path";
 import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
 import type { Browser, Page } from "puppeteer-core";
-import { launchBrowser, closeBrowser } from "./capture.ts";
+import { launchBrowser, closeBrowser } from "./browser-local.ts";
 import { diffScreenshots } from "./screenshot-diff.ts";
 
 export interface PageRef {
@@ -30,8 +30,27 @@ export interface VerifyOptions {
   viewportHeight?: number;
   /** Where to write diff images and the report (default: <projectDir>/verify/). */
   reportDir?: string;
-  /** Pixelmatch threshold (default: 0.15). */
+  /**
+   * Pixelmatch's per-pixel COLOUR tolerance, 0..1 (default 0.15).
+   *
+   * It decides when two pixels count as the same colour, so it moves the score; it is not a
+   * pass/fail bar. `minFidelity` is the bar. Calling this one a "threshold" is what let a run
+   * scoring 8% exit 0 (issue #232).
+   */
   threshold?: number;
+  /**
+   * The average fidelity, 0..100, below which the run is a failure (default 0 — report only).
+   *
+   * Reported as `passed`; the caller decides what to do with it. `jx-import` exits non-zero.
+   */
+  minFidelity?: number;
+  /**
+   * Compare the whole scrollable page rather than the viewport (default true).
+   *
+   * It used to be the first 900px, which on a long marketing page is a small fraction of what was
+   * imported — most of the clone was never looked at.
+   */
+  fullPage?: boolean;
   /** Progress callback. */
   onProgress?: (msg: string) => void;
   /** Existing browser instance to reuse. */
@@ -47,6 +66,15 @@ export interface PageVerifyResult {
   diffImagePath: string;
   originalScreenshotPath: string;
   renderedScreenshotPath: string;
+  /**
+   * Console errors and uncaught exceptions the rendered page produced.
+   *
+   * A fidelity number says the clone looks wrong; these say why. The first import measured 404'd on
+   * 15 asset references and the verify pass reported a percentage and nothing else.
+   */
+  consoleErrors: string[];
+  /** Requests the rendered page made that failed or answered 4xx/5xx, as "<status> <url>". */
+  failedRequests: string[];
   error?: string;
 }
 
@@ -55,6 +83,18 @@ export interface VerifyResult {
   /** Average fidelity across all pages. */
   averageFidelity: number;
   reportDir: string;
+  /**
+   * Errors the compiler reported while building the project.
+   *
+   * Previously logged, written into report.json, and then dropped — `verifyProject` carried on and
+   * `importSite` never saw them, so a project that did not build still exited 0.
+   */
+  buildErrors: string[];
+  /**
+   * Whether this run met its bar: it built cleanly, every page rendered, and the average fidelity
+   * reached `minFidelity`. The single fact a caller needs to gate on.
+   */
+  passed: boolean;
 }
 
 /**
@@ -76,18 +116,59 @@ export function routeToUrlPath(route: string): string {
   return path;
 }
 
+/** What the rendered page said about itself while it loaded. */
+interface PageDiagnostics {
+  consoleErrors: string[];
+  failedRequests: string[];
+}
+
+/**
+ * Listen for the two things a fidelity percentage cannot tell you: what the page logged, and what
+ * it asked for and did not get.
+ *
+ * Attached before `goto`, because a 404 on an asset referenced in the HTML happens during the
+ * navigation. Registration is optional — a stand-in page that only screenshots is still usable, and
+ * diagnostics are then simply empty rather than a crash.
+ */
+function watchPage(page: Page): PageDiagnostics {
+  const diagnostics: PageDiagnostics = { consoleErrors: [], failedRequests: [] };
+  if (typeof (page as Partial<Page>).on !== "function") {
+    return diagnostics;
+  }
+  page.on("console", (msg) => {
+    if (msg.type() === "error") {
+      diagnostics.consoleErrors.push(msg.text());
+    }
+  });
+  // `pageerror` carries `Error | unknown`: a page can reject with anything at all, and a thrown
+  // String is still a page that broke.
+  page.on("pageerror", (error) => {
+    diagnostics.consoleErrors.push(error instanceof Error ? error.message : String(error));
+  });
+  page.on("requestfailed", (request) => {
+    diagnostics.failedRequests.push(`failed ${request.url()}`);
+  });
+  page.on("response", (response) => {
+    if (response.status() >= 400) {
+      diagnostics.failedRequests.push(`${response.status()} ${response.url()}`);
+    }
+  });
+  return diagnostics;
+}
+
 async function screenshotPage(
   page: Page,
   url: string,
   width: number,
   height: number,
+  fullPage: boolean,
 ): Promise<Buffer> {
   await page.setViewport({ width, height });
   await page.goto(url, { waitUntil: "networkidle0", timeout: 30_000 });
   await new Promise<void>((r) => {
     setTimeout(r, 500);
   });
-  const screenshot = await page.screenshot({ type: "png", fullPage: false });
+  const screenshot = await page.screenshot({ fullPage, type: "png" });
   return Buffer.from(screenshot);
 }
 
@@ -99,9 +180,10 @@ export async function captureReferenceScreenshot(
   page: Page,
   width = 1440,
   height = 900,
+  fullPage = true,
 ): Promise<Buffer> {
   await page.setViewport({ width, height });
-  const screenshot = await page.screenshot({ type: "png", fullPage: false });
+  const screenshot = await page.screenshot({ fullPage, type: "png" });
   return Buffer.from(screenshot);
 }
 
@@ -113,6 +195,39 @@ async function buildProject(projectDir: string): Promise<{ distDir: string; erro
   return { distDir, errors: result.errors };
 }
 
+/**
+ * What the verify server declares for each extension it can serve.
+ *
+ * `.html`, `.css` and `.js` were the whole map, so every SVG, PNG, WOFF2 and JSON went out as
+ * `application/octet-stream`. A browser will not render an `<img src>` served as a binary blob and
+ * will not use a font it was handed as one, which cost the clone fidelity points that were not the
+ * importer's fault at all (issue #232).
+ */
+const CONTENT_TYPES: Record<string, string> = {
+  ".avif": "image/avif",
+  ".css": "text/css",
+  ".gif": "image/gif",
+  ".htm": "text/html; charset=utf-8",
+  ".html": "text/html; charset=utf-8",
+  ".ico": "image/x-icon",
+  ".jpeg": "image/jpeg",
+  ".jpg": "image/jpeg",
+  ".js": "text/javascript; charset=utf-8",
+  ".json": "application/json; charset=utf-8",
+  ".map": "application/json; charset=utf-8",
+  ".mjs": "text/javascript; charset=utf-8",
+  ".otf": "font/otf",
+  ".png": "image/png",
+  ".svg": "image/svg+xml",
+  ".ttf": "font/ttf",
+  ".txt": "text/plain; charset=utf-8",
+  ".webmanifest": "application/manifest+json",
+  ".webp": "image/webp",
+  ".woff": "font/woff",
+  ".woff2": "font/woff2",
+  ".xml": "application/xml; charset=utf-8",
+};
+
 /**
  * Serve a directory over HTTP using Bun's built-in server. Returns the server instance and the base
  * URL.
@@ -134,13 +249,8 @@ export function serveDirectory(dir: string): {
       const filePath = join(dir, pathname);
       const file = Bun.file(filePath);
       if (await file.exists()) {
-        const contentType = filePath.endsWith(".html")
-          ? "text/html"
-          : filePath.endsWith(".css")
-            ? "text/css"
-            : filePath.endsWith(".js")
-              ? "application/javascript"
-              : "application/octet-stream";
+        const contentType =
+          CONTENT_TYPES[extname(filePath).toLowerCase()] ?? "application/octet-stream";
         return new Response(file, { headers: { "Content-Type": contentType } });
       }
       return new Response("Not Found", { status: 404 });
@@ -172,6 +282,8 @@ export async function verifyProject(opts: VerifyOptions): Promise<VerifyResult>
     viewportWidth = 1440,
     viewportHeight = 900,
     threshold = 0.15,
+    minFidelity = 0,
+    fullPage = true,
     onProgress = () => {},
   } = opts;
 
@@ -201,10 +313,14 @@ export async function verifyProject(opts: VerifyOptions): Promise<VerifyResult>
         diffImagePath: "",
         originalScreenshotPath: "",
         renderedScreenshotPath: "",
+        consoleErrors: [],
+        failedRequests: [],
         error: `Build failed: ${err.message}`,
       })),
       averageFidelity: 0,
       reportDir,
+      buildErrors: [`Build failed: ${err.message}`],
+      passed: false,
     };
   }
 
@@ -230,13 +346,23 @@ export async function verifyProject(opts: VerifyOptions): Promise<VerifyResult>
         const renderedUrl = `${baseUrl}${urlPath}`;
         onProgress(`    Screenshotting rendered: ${renderedUrl}`);
         const renderedPage = await browser.newPage();
+        // Listeners go on before the navigation — a 404 on an asset named in the HTML happens
+        // During it, and that is exactly the class of failure the score alone cannot explain.
+        const diagnostics = watchPage(renderedPage);
         const renderedPng = await screenshotPage(
           renderedPage,
           renderedUrl,
           viewportWidth,
           viewportHeight,
+          fullPage,
         );
         await renderedPage.close();
+        if (diagnostics.failedRequests.length > 0) {
+          onProgress(`    ${diagnostics.failedRequests.length} request(s) failed or 404'd`);
+        }
+        if (diagnostics.consoleErrors.length > 0) {
+          onProgress(`    ${diagnostics.consoleErrors.length} console error(s)`);
+        }
 
         // Diff
         onProgress("    Computing pixel diff...");
@@ -262,6 +388,8 @@ export async function verifyProject(opts: VerifyOptions): Promise<VerifyResult>
           diffImagePath: diffPath,
           originalScreenshotPath: origPath,
           renderedScreenshotPath: renderedPath,
+          consoleErrors: diagnostics.consoleErrors,
+          failedRequests: diagnostics.failedRequests,
         });
       } catch (error) {
         const err = error as Error;
@@ -275,6 +403,8 @@ export async function verifyProject(opts: VerifyOptions): Promise<VerifyResult>
           diffImagePath: "",
           originalScreenshotPath: "",
           renderedScreenshotPath: "",
+          consoleErrors: [],
+          failedRequests: [],
           error: err.message,
         });
       }
@@ -290,18 +420,40 @@ export async function verifyProject(opts: VerifyOptions): Promise<VerifyResult>
           ) / 100
         : 0;
 
+    /*
+     * A run passes only if all three hold. Any one of them alone can be silently wrong: a project
+     * that did not build still renders SOMETHING, a page that failed to screenshot is excluded
+     * from the average rather than dragging it down, and a page that rendered can still be a
+     * different site. Previously none of them could fail the run — an 8%-fidelity clone printed
+     * `Done!` and exited 0, which is worse than failing because it is only visible to somebody who
+     * opens the result and looks at it (issue #232).
+     */
+    const errored = results.filter((r) => r.error);
+    const passed =
+      (buildErrors ?? []).length === 0 && errored.length === 0 && averageFidelity >= minFidelity;
+
     const report = {
       timestamp: new Date().toISOString(),
       projectDir,
       viewport: { width: viewportWidth, height: viewportHeight },
-      threshold,
+      fullPage,
+      /** Pixelmatch's per-pixel colour tolerance — see `VerifyOptions.threshold`. */
+      colorTolerance: threshold,
+      minFidelity,
       averageFidelity,
+      passed,
       pages: results,
       buildErrors: buildErrors ?? [],
     };
     writeFileSync(join(reportDir, "report.json"), JSON.stringify(report, null, 2));
 
-    return { pages: results, averageFidelity, reportDir };
+    return {
+      pages: results,
+      averageFidelity,
+      reportDir,
+      buildErrors: buildErrors ?? [],
+      passed,
+    };
   } finally {
     void server.stop();
     if (!opts.browser) {