@jxsuite/import 0.40.0 → 0.40.2

package/src/asset-collect.ts

@@ -5,7 +5,7 @@
  * SVGs are kept as-is (no download needed). Returns absolute URLs for downloading.
  */
 
-import type { Page } from "puppeteer-core";
+import type { ImportPage } from "./capture.ts";
 import { SRCSET_SEPARATOR } from "./srcset.ts";
 
 export interface DiscoveredAsset {
@@ -46,7 +46,7 @@ export interface AssetCollectionResult {
  * Collect all asset URLs from the live page DOM and stylesheets. Runs entirely in-browser via
  * page.evaluate() — no per-element round-trips.
  */
-export async function collectAssets(page: Page): Promise<AssetCollectionResult> {
+export async function collectAssets(page: ImportPage): Promise<AssetCollectionResult> {
   /*
    * The separator crosses into the page as a SOURCE STRING: an evaluated body is serialised and
    * re-parsed in the browser, so it cannot close over a value declared out here.

package/src/asset-download.ts

@@ -5,9 +5,9 @@
  * subdirectories by type: images/, fonts/, icons/.
  */
 
-import { join, extname } from "node:path";
-import { mkdir } from "node:fs/promises";
+import { extname } from "node:path";
 import type { DiscoveredAsset } from "./asset-collect.ts";
+import type { ImportIo } from "./io.ts";
 
 export interface DownloadResult {
   /**
@@ -127,7 +127,7 @@ function isBlocked(url: string): boolean {
 
 export async function downloadAssets(
   assets: DiscoveredAsset[],
-  outDir: string,
+  io: ImportIo,
   sourceUrl?: string,
 ): Promise<DownloadResult> {
   const rewriteMap = new Map<string, string>();
@@ -143,12 +143,11 @@ export async function downloadAssets(
     }
   }
 
-  // Create subdirectories
-  const assetsDir = join(outDir, "public", "assets");
-  await mkdir(join(assetsDir, "images"), { recursive: true });
-  await mkdir(join(assetsDir, "fonts"), { recursive: true });
-  await mkdir(join(assetsDir, "icons"), { recursive: true });
-  await mkdir(join(assetsDir, "other"), { recursive: true });
+  // Create subdirectories (a sink with no directories declines — see ImportIo.mkdir)
+  const assetsDir = "public/assets";
+  for (const subdir of ["images", "fonts", "icons", "other"]) {
+    await io.mkdir?.(`${assetsDir}/${subdir}`);
+  }
 
   // Track filenames to avoid collisions
   const usedNames = new Map<string, number>();
@@ -172,7 +171,7 @@ export async function downloadAssets(
     }
     usedNames.set(nameKey, count + 1);
 
-    const destPath = join(assetsDir, subdir, filename);
+    const destPath = `${assetsDir}/${subdir}/${filename}`;
     // The site-absolute path the built asset is served from. `public/` is the compiler's static
     // ROOT — its contents land at dist/<path>, so the "public/" segment must not survive into a
     // Reference, and the leading slash must be there or a reference from /components/x.js or a
@@ -195,7 +194,7 @@ export async function downloadAssets(
 
       const buffer = await response.arrayBuffer();
       totalBytes += buffer.byteLength;
-      await Bun.write(destPath, buffer);
+      await io.write(destPath, new Uint8Array(buffer));
       rewriteMap.set(url, relativePath);
     } catch {
       failed.push(url);

package/src/asset-rewrite.ts

@@ -52,20 +52,34 @@ export function rewriteAssetUrls(
    * the importer would fetch one set of URLs and rewrite another — the attribute would be
    * reassembled around fragments nothing had downloaded.
    */
-  function rewriteSrcset(srcset: string): string {
-    return srcset
+  function rewriteSrcset(srcset: string): { srcset: string; collapsedTo: string | null } {
+    const locals = new Set<string>();
+    let unresolved = false;
+
+    const rewritten = srcset
       .split(SRCSET_SEPARATOR)
       .map((entry) => {
         const parts = entry.trim().split(/\s+/);
         const [url = ""] = parts;
-        const rewritten = rewriteUrl(url);
-        if (rewritten) {
+        const local = rewriteUrl(url);
+        if (local) {
           count += 1;
-          parts[0] = rewritten;
+          parts[0] = local;
+          locals.add(local);
+        } else {
+          unresolved = true;
         }
         return parts.join(" ");
       })
       .join(", ");
+
+    /* Every candidate now points at the ONE file the family kept, so the attribute has become a
+       list of the same path repeated with different width descriptors — which lies to the browser
+       about what it can choose. Collapse it, and let the compiler build a real `srcset` from the
+       original it owns. A candidate that failed to resolve means the family is only partly local,
+       so the attribute stays as it is rather than silently narrowing the set. */
+    const collapsedTo = !unresolved && locals.size === 1 ? [...locals][0]! : null;
+    return { collapsedTo, srcset: rewritten };
   }
 
   function rewriteCssUrls(value: string): string {
@@ -125,7 +139,17 @@ export function rewriteAssetUrls(
 
       // Srcset attribute
       if (typeof attrs.srcset === "string") {
-        attrs.srcset = rewriteSrcset(attrs.srcset);
+        const result = rewriteSrcset(attrs.srcset);
+        if (result.collapsedTo === null) {
+          attrs.srcset = result.srcset;
+        } else {
+          delete attrs.srcset;
+          /* `sizes` describes a layout that no longer exists once the candidate list is gone, and
+             the compiler honours an author `sizes` over its own container measurement — so a stale
+             one would misdescribe the imported layout rather than merely be redundant. */
+          delete attrs.sizes;
+          attrs.src = result.collapsedTo;
+        }
       }
     }
 

package/src/browser-local.ts

@@ -0,0 +1,67 @@
+/**
+ * Launching a real Chrome on this machine — the one module in the import that loads
+ * `puppeteer-core` as a VALUE.
+ *
+ * It is separate from `capture.ts` for a bundling reason rather than a tidiness one.
+ * `puppeteer-core` cannot run in workerd at all (it reaches for `child_process`, `net` and a binary
+ * on disk), and a Worker bundler follows value imports transitively — so a single `import { launch
+ * }` anywhere in `pipeline.ts`'s graph would pull the whole of it in and fail the deploy. Keeping
+ * the launch here, and having every phase take a browser it was HANDED, is what lets Jx Cloud run
+ * the same pipeline against a `@cloudflare/puppeteer` session.
+ */
+
+import { launch } from "puppeteer-core";
+import type { Browser } from "puppeteer-core";
+
+const DEFAULT_CHROME_PATHS = [
+  "google-chrome-stable",
+  "google-chrome",
+  "chromium-browser",
+  "chromium",
+];
+
+function findChrome(executablePath?: string): string {
+  if (executablePath) {
+    return executablePath;
+  }
+  const env = process.env.CHROME_PATH;
+  if (env) {
+    return env;
+  }
+
+  for (const name of DEFAULT_CHROME_PATHS) {
+    const path = Bun.which(name);
+    if (path) {
+      return path;
+    }
+  }
+  throw new Error(
+    "Could not find Chrome/Chromium. Set CHROME_PATH or install google-chrome-stable.",
+  );
+}
+
+let _browser: Browser | null = null;
+
+export interface LaunchOptions {
+  /** Explicit browser binary. Wins over CHROME_PATH and PATH discovery. */
+  executablePath?: string;
+}
+
+export async function launchBrowser(options?: LaunchOptions): Promise<Browser> {
+  if (_browser?.connected) {
+    return _browser;
+  }
+  _browser = await launch({
+    executablePath: findChrome(options?.executablePath),
+    headless: true,
+    args: ["--no-sandbox", "--disable-gpu"],
+  });
+  return _browser;
+}
+
+export async function closeBrowser(): Promise<void> {
+  if (_browser?.connected) {
+    await _browser.close();
+    _browser = null;
+  }
+}

package/src/capture.ts

@@ -1,5 +1,44 @@
-import { launch } from "puppeteer-core";
-import type { Browser, Page } from "puppeteer-core";
+/**
+ * Page capture, and the browser surface the whole import is written against.
+ *
+ * That surface is declared STRUCTURALLY here rather than imported from `puppeteer-core`, and the
+ * reason is bundling, not taste: `puppeteer-core` is unloadable in workerd, and a Worker bundler
+ * follows value imports transitively, so a single one anywhere in `pipeline.ts`'s graph would fail
+ * the deploy. Naming only `newPage` — and, on a page, the five calls the phases actually make —
+ * lets the same code drive a local Chrome (`browser-local.ts`) or a `@cloudflare/puppeteer`
+ * session, both of which provide exactly this and more.
+ */
+
+/**
+ * What `evaluate` hands the in-page function.
+ *
+ * A homomorphic mapped type rather than plain `Params`, because puppeteer's own `evaluate` declares
+ * its parameters through one (it unwraps `JSHandle`s there). Two generic signatures only relate
+ * when they are shaped alike, and with a bare tuple puppeteer's `Page` stops being assignable to
+ * `ImportPage` — which is the one thing this interface has to be true of.
+ */
+export type PageArgs<Params extends unknown[]> = { [K in keyof Params]: Params[K] };
+
+export interface ImportPage {
+  goto: (
+    url: string,
+    options?: {
+      waitUntil?: "load" | "domcontentloaded" | "networkidle0" | "networkidle2";
+      timeout?: number;
+    },
+  ) => Promise<unknown>;
+  setViewport: (viewport: { width: number; height: number }) => Promise<unknown>;
+  evaluate: <Params extends unknown[], R>(
+    fn: (...params: PageArgs<Params>) => R,
+    ...args: Params
+  ) => Promise<Awaited<R>>;
+  screenshot: (options?: { fullPage?: boolean; type?: "png" }) => Promise<Uint8Array>;
+  close: () => Promise<unknown>;
+}
+
+export interface ImportBrowser {
+  newPage: () => Promise<ImportPage>;
+}
 
 export interface CaptureResult {
   url: string;
@@ -7,61 +46,8 @@ export interface CaptureResult {
   bodyHtml: string;
   /** Discovered same-origin links for the crawler (Phase 3). */
   links: string[];
-  /** The puppeteer Page, kept open for style capture (Phase 1). Caller must close. */
-  page: Page;
-}
-
-const DEFAULT_CHROME_PATHS = [
-  "google-chrome-stable",
-  "google-chrome",
-  "chromium-browser",
-  "chromium",
-];
-
-function findChrome(executablePath?: string): string {
-  if (executablePath) {
-    return executablePath;
-  }
-  const env = process.env.CHROME_PATH;
-  if (env) {
-    return env;
-  }
-
-  for (const name of DEFAULT_CHROME_PATHS) {
-    const path = Bun.which(name);
-    if (path) {
-      return path;
-    }
-  }
-  throw new Error(
-    "Could not find Chrome/Chromium. Set CHROME_PATH or install google-chrome-stable.",
-  );
-}
-
-let _browser: Browser | null = null;
-
-export interface LaunchOptions {
-  /** Explicit browser binary. Wins over CHROME_PATH and PATH discovery. */
-  executablePath?: string;
-}
-
-export async function launchBrowser(options?: LaunchOptions): Promise<Browser> {
-  if (_browser?.connected) {
-    return _browser;
-  }
-  _browser = await launch({
-    executablePath: findChrome(options?.executablePath),
-    headless: true,
-    args: ["--no-sandbox", "--disable-gpu"],
-  });
-  return _browser;
-}
-
-export async function closeBrowser(): Promise<void> {
-  if (_browser?.connected) {
-    await _browser.close();
-    _browser = null;
-  }
+  /** The page, kept open for style capture (Phase 1). Caller must close. */
+  page: ImportPage;
 }
 
 export interface CaptureOptions {
@@ -73,7 +59,7 @@ export interface CaptureOptions {
  * Scroll the page to the bottom in steps to trigger lazy-loaded images and intersection-observer
  * content, then scroll back to top. Settles between steps to let content render.
  */
-async function scrollToRevealAll(page: Page): Promise<void> {
+async function scrollToRevealAll(page: ImportPage): Promise<void> {
   await page.evaluate(async () => {
     const delay = (ms: number) =>
       new Promise<void>((r) => {
@@ -111,15 +97,18 @@ async function scrollToRevealAll(page: Page): Promise<void> {
 /**
  * Capture a page's DOM. Returns the page object still open — the caller is responsible for closing
  * it (after style capture in Phase 1, or immediately).
+ *
+ * The browser is a parameter and has no default. It used to fall back to launching a local Chrome,
+ * and that fallback was the value import of `puppeteer-core` that kept this module — and everything
+ * that reaches it — out of a Worker.
  */
 export async function capturePage(
   url: string,
-  browser?: Browser,
+  browser: ImportBrowser,
   options?: CaptureOptions,
 ): Promise<CaptureResult> {
   const { scrollToBottom = true } = options ?? {};
-  const br = browser ?? (await launchBrowser());
-  const page: Page = await br.newPage();
+  const page = await browser.newPage();
   await page.setViewport({ width: 1440, height: 900 });
 
   await page.goto(url, { waitUntil: "networkidle0", timeout: 30_000 });

package/src/componentize.ts

@@ -185,6 +185,39 @@ function setNodeAt(root: JxElement, path: number[], replacement: JxElement): JxE
   return result;
 }
 
+/**
+ * Tags that must never become a component ROOT, because the element itself carries the behaviour.
+ *
+ * A component's template root takes the component's own tag name, so promoting one of these
+ * replaces it with a custom element and the browser semantics go with it: `<details>` stops being a
+ * disclosure, `<summary>` stops being its label, `<dialog>` stops being a dialog, and a table or
+ * list part is torn out of the parent structure that gives it meaning. The children of one of these
+ * are still fair game — only the root is refused.
+ *
+ * This matters most for the widgets `apply-accordions.ts` converts: rewriting an accordion into
+ * `<details name>` makes its rows structurally identical, which is exactly the shape componentize
+ * looks for, so without this guard the pass would hand back a working disclosure and componentize
+ * would immediately turn it into an inert custom element.
+ */
+const NEVER_A_COMPONENT_ROOT = new Set([
+  "details",
+  "summary",
+  "dialog",
+  "table",
+  "thead",
+  "tbody",
+  "tfoot",
+  "tr",
+  "td",
+  "th",
+  "li",
+  "dt",
+  "dd",
+  "option",
+  "optgroup",
+  "figcaption",
+]);
+
 function collectSubtrees(
   node: JxElement,
   pageRoute: string,
@@ -193,7 +226,8 @@ function collectSubtrees(
   out: SubtreeLocation[],
 ) {
   // Skip root node (path.length === 0) — don't componentize entire pages
-  if (path.length > 0 && nodeDepth(node) >= minDepth) {
+  const tag = String(node.tagName ?? "").toLowerCase();
+  if (path.length > 0 && nodeDepth(node) >= minDepth && !NEVER_A_COMPONENT_ROOT.has(tag)) {
     out.push({ pageRoute, path: [...path], node });
   }
   if (Array.isArray(node.children)) {

package/src/crawl.ts

@@ -1,5 +1,6 @@
-import { launchBrowser, capturePage } from "./capture.ts";
-import type { CaptureResult } from "./capture.ts";
+import { capturePage } from "./capture.ts";
+import type { CaptureResult, ImportBrowser } from "./capture.ts";
+import type { ImportIo } from "./io.ts";
 import { convertToJx } from "./to-jx.ts";
 import type { ToJxResult } from "./to-jx.ts";
 import { captureStyles } from "./style-capture.ts";
@@ -14,12 +15,16 @@ import type { Breakpoint, BreakpointPolicy } from "./breakpoint-plan.ts";
 import { applyStylesToTree } from "./apply-styles.ts";
 import { collectAssets } from "./asset-collect.ts";
 import { downloadAssets } from "./asset-download.ts";
+import { applyFamilyAliases, planImageFamilies } from "./image-family.ts";
 import { rewriteAssetUrls } from "./asset-rewrite.ts";
 import { applyTokens } from "./css-tokens.ts";
 
 export interface CrawlOptions {
   url: string;
-  outDir: string;
+  /** The browser to capture with. The crawl never launches one — see `browser-local.ts`. */
+  browser: ImportBrowser;
+  /** Where downloaded assets go. */
+  io: ImportIo;
   maxDepth: number;
   maxPages: number;
   maxNodesPerPage: number;
@@ -52,8 +57,8 @@ export interface CrawledPage {
   jx: ToJxResult;
   depth: number;
   links: string[];
-  /** Reference screenshot PNG buffer, present when captureScreenshots is true. */
-  screenshot?: Buffer | undefined;
+  /** Reference screenshot PNG bytes, present when captureScreenshots is true. */
+  screenshot?: Uint8Array | undefined;
 }
 
 export interface CrawlResult {
@@ -187,7 +192,8 @@ function isDisallowed(url: string, disallowedPaths: Set<string>): boolean {
 export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
   const {
     url,
-    outDir,
+    browser,
+    io,
     maxDepth,
     maxPages,
     maxNodesPerPage,
@@ -199,7 +205,6 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
   } = options;
 
   const { origin } = new URL(url);
-  const browser = await launchBrowser();
 
   // Fetch robots.txt
   let disallowedPaths = new Set<string>();
@@ -261,11 +266,9 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
         `  ⚠ Page exceeds node cap (${jx.nodeCount} > ${maxNodesPerPage}), skipping styles/assets`,
       );
       skippedByNodeCap.push(entry.url);
-      let screenshot: Buffer | undefined;
+      let screenshot: Uint8Array | undefined;
       if (options.captureScreenshots) {
-        screenshot = Buffer.from(
-          await capture.page.screenshot({ fullPage: fullPageScreenshots, type: "png" }),
-        );
+        screenshot = await capture.page.screenshot({ fullPage: fullPageScreenshots, type: "png" });
       }
       await capture.page.close();
 
@@ -325,7 +328,7 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
               mergedBreakpoints = {};
             }
             Object.assign(mergedBreakpoints, media.breakpoints);
-            applyStylesToTree(jx.document, diffed, media.deltas);
+            applyStylesToTree(jx.document, diffed, media.deltas, media.breakpoints);
           } else {
             applyStylesToTree(jx.document, diffed);
           }
@@ -363,7 +366,12 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
         }
 
         if (collected.assets.length > 0) {
-          const downloaded = await downloadAssets(collected.assets, outDir, entry.url);
+          /* One member per responsive family reaches the network; every dropped derivative is
+             aliased onto the file that WAS written, so a reference to any rung of the ladder still
+             resolves. The compiler regenerates the sizes from the original it now owns. */
+          const families = planImageFamilies(collected.assets);
+          const downloaded = await downloadAssets(families.keep, io, entry.url);
+          applyFamilyAliases(downloaded.rewriteMap, families.alias);
           if (downloaded.rewriteMap.size > 0) {
             rewriteAssetUrls(jx.document, downloaded.rewriteMap, entry.url);
             // Collect font rewrites (R2)
@@ -381,11 +389,9 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
       }
     }
 
-    let screenshot: Buffer | undefined;
+    let screenshot: Uint8Array | undefined;
     if (options.captureScreenshots) {
-      screenshot = Buffer.from(
-        await capture.page.screenshot({ fullPage: fullPageScreenshots, type: "png" }),
-      );
+      screenshot = await capture.page.screenshot({ fullPage: fullPageScreenshots, type: "png" });
     }
     await capture.page.close();
 

package/src/derived-geometry.ts

@@ -0,0 +1,115 @@
+/**
+ * Drop the `width` and `height` a browser MEASURED, keeping the ones a site actually authored.
+ *
+ * `getComputedStyle` returns used values, so `width` comes back as the pixels an element happens to
+ * occupy rather than the `100%` or `auto` that produced them. Written into a document as an
+ * unconditional style, that pins the layout: a full-width section captured at a 1440px viewport
+ * becomes `width: 1440px` and never fills anything else again, which is visible the moment a canvas
+ * is dragged wider than the width the import was taken at. On the reference corpus this was 814
+ * elements pinned at 1000px or more, and `width` and `height` were the two most common declarations
+ * in the entire project.
+ *
+ * `media-extract.ts` already removes these from BREAKPOINT deltas by sampling each band twice and
+ * keeping only what both samples agree on. That technique cannot rescue the base, for a reason
+ * worth writing down: a second sample must stay inside its own media band or it measures different
+ * rules, and the base band is bounded below by the site's widest `min-width` breakpoint. A
+ * container with `max-width: 1390px` measures 1390 at every width in that band, so it agrees with
+ * itself and survives — while being exactly the fluid element the reader will notice.
+ *
+ * So the base needs a different question, and the useful one is structural rather than numerical:
+ * **would this element have got that width on its own?** A block-level box in normal flow fills its
+ * containing block by definition, and a `max-width` or `flex-basis` that narrows it is captured
+ * separately and survives — so its measured `width` states a fact the layout already implies, and
+ * removing it restores the fluidity without losing the constraint. A replaced element, an
+ * out-of-flow box and a shrink-to-fit inline box are all sized by something the layout does NOT
+ * imply, so their measurements are kept.
+ */
+
+import type { JxElement } from "@jxsuite/schema/types";
+
+/** Elements sized by their own content or attributes rather than by their container. */
+const REPLACED = new Set([
+  "img",
+  "svg",
+  "video",
+  "canvas",
+  "iframe",
+  "object",
+  "embed",
+  "input",
+  "select",
+  "textarea",
+  "progress",
+  "meter",
+]);
+
+/** A box taken out of the flow is positioned deliberately, so its measurements may be intentional. */
+const OUT_OF_FLOW = new Set(["absolute", "fixed", "sticky"]);
+
+/** A bare pixel length, which is the only shape a measurement takes. */
+const PIXEL_LENGTH = /^-?[\d.]+px$/;
+
+function isMeasured(value: unknown): boolean {
+  return typeof value === "string" && PIXEL_LENGTH.test(value);
+}
+
+/**
+ * Whether this node holds anything that gives it a height of its own.
+ *
+ * The distinction matters only for `height`. A box with content is as tall as that content, and
+ * pinning it breaks the moment text reflows at a narrower width — the single most damaging way a
+ * clone can fail. An EMPTY box is different: a spacer's entire purpose is its height, and there is
+ * nothing else in the document that would reproduce it.
+ */
+function hasContent(node: JxElement): boolean {
+  if (typeof node.textContent === "string" && node.textContent.trim().length > 0) {
+    return true;
+  }
+  const children = Array.isArray(node.children) ? node.children : [];
+  return children.some((child) =>
+    typeof child === "string" ? child.trim().length > 0 : child !== undefined,
+  );
+}
+
+/**
+ * Remove measured geometry from a style object, in place.
+ *
+ * @param {JxElement} node - The element the style belongs to; its tag and content decide the answer
+ * @param {Record<string, unknown>} style - The style object to filter, mutated
+ * @param {Record<string, unknown>} [context] - Where `position` and `display` are read from, when
+ *   they are not in `style` itself. A breakpoint delta carries only what CHANGED, so it usually
+ *   states neither, and judging it on its own would strip an absolutely-positioned box's width on
+ *   the grounds that it looked like an in-flow one.
+ * @returns {number} How many declarations were removed
+ */
+export function dropDerivedGeometry(
+  node: JxElement,
+  style: Record<string, unknown>,
+  context: Record<string, unknown> = style,
+): number {
+  const tag = String(node.tagName ?? "").toLowerCase();
+  if (REPLACED.has(tag)) {
+    return 0;
+  }
+  const position = style["position"] ?? context["position"];
+  if (OUT_OF_FLOW.has(String(position ?? ""))) {
+    return 0;
+  }
+  /* A shrink-to-fit box is sized by its content in the inline direction, so its width is not
+     implied by its container the way a block-level box's is. */
+  const display = style["display"] ?? context["display"];
+  if (String(display ?? "").startsWith("inline")) {
+    return 0;
+  }
+
+  let dropped = 0;
+  if (isMeasured(style["width"])) {
+    delete style["width"];
+    dropped += 1;
+  }
+  if (isMeasured(style["height"]) && hasContent(node)) {
+    delete style["height"];
+    dropped += 1;
+  }
+  return dropped;
+}