@jxsuite/import 0.34.0

package/src/asset-collect.ts

@@ -0,0 +1,254 @@
+/**
+ * In-browser asset URL discovery — runs inside page.evaluate().
+ *
+ * Finds images, background images, fonts (from @font-face), favicons, and srcset entries. Inline
+ * SVGs are kept as-is (no download needed). Returns absolute URLs for downloading.
+ */
+
+import type { Page } from "puppeteer-core";
+
+export interface DiscoveredAsset {
+  /** Absolute URL of the asset. */
+  url: string;
+  /** How it was found — guides download priority and rewrite strategy. */
+  source:
+    | "img-src"
+    | "img-srcset"
+    | "source-srcset"
+    | "picture-source"
+    | "video-poster"
+    | "css-background"
+    | "css-url"
+    | "font-face"
+    | "favicon"
+    | "og-image";
+}
+
+export interface CapturedStylesheet {
+  /** The stylesheet's href (null for inline <style> blocks). */
+  href: string | null;
+  /** Full cssText of the sheet. Null if cross-origin and cssRules threw. */
+  cssText: string | null;
+  /** Individual @font-face rule texts extracted from the sheet. */
+  fontFaceRules: string[];
+}
+
+export interface AssetCollectionResult {
+  assets: DiscoveredAsset[];
+  /** Inline SVGs found (kept inline, not downloaded). */
+  inlineSvgCount: number;
+  /** Retained source stylesheets (F0 foundation — enables font emission, breakpoint reading, etc). */
+  stylesheets: CapturedStylesheet[];
+}
+
+/**
+ * Collect all asset URLs from the live page DOM and stylesheets. Runs entirely in-browser via
+ * page.evaluate() — no per-element round-trips.
+ */
+export async function collectAssets(page: Page): Promise<AssetCollectionResult> {
+  const result = await page.evaluate(() => {
+    const assets: { url: string; source: string }[] = [];
+    const seen = new Set<string>();
+    let inlineSvgCount = 0;
+
+    function add(url: string, source: string) {
+      if (!url || seen.has(url)) {
+        return;
+      }
+      try {
+        const abs = new URL(url, location.href).href;
+        // oxlint-disable-next-line no-script-url -- detection, not usage
+        if (abs.startsWith("data:") || abs.startsWith("blob:") || abs.startsWith("javascript:")) {
+          return;
+        }
+        if (seen.has(abs)) {
+          return;
+        }
+        seen.add(abs);
+        assets.push({ url: abs, source });
+      } catch {
+        // Invalid URL — skip
+      }
+    }
+
+    function parseSrcset(srcset: string): string[] {
+      return srcset
+        .split(",")
+        .map((entry) => entry.trim().split(/\s+/)[0] ?? "")
+        .filter(Boolean);
+    }
+
+    function extractCssUrls(cssText: string): string[] {
+      const urls: string[] = [];
+      const re = /url\(\s*(['"]?)(.+?)\1\s*\)/g;
+      let m;
+      while ((m = re.exec(cssText)) !== null) {
+        const u = m.at(2);
+        if (u && !u.startsWith("data:")) {
+          urls.push(u);
+        }
+      }
+      return urls;
+    }
+
+    // Img[src]
+    for (const el of document.querySelectorAll("img[src]")) {
+      add((el as HTMLImageElement).src, "img-src");
+    }
+
+    // Img[srcset] and source[srcset]
+    for (const el of document.querySelectorAll("img[srcset], source[srcset]")) {
+      const srcset = el.getAttribute("srcset") ?? "";
+      const source = el.tagName.toLowerCase() === "img" ? "img-srcset" : "source-srcset";
+      for (const u of parseSrcset(srcset)) {
+        add(u, source);
+      }
+    }
+
+    // Picture > source (type-based, not just srcset)
+    for (const el of document.querySelectorAll("picture > source[src]")) {
+      add((el as HTMLSourceElement).src, "picture-source");
+    }
+
+    // Video[poster]
+    for (const el of document.querySelectorAll("video[poster]")) {
+      add((el as HTMLVideoElement).poster, "video-poster");
+    }
+
+    // Inline style url() on any element
+    for (const el of document.querySelectorAll("[style]")) {
+      const style = el.getAttribute("style") ?? "";
+      for (const u of extractCssUrls(style)) {
+        add(u, "css-url");
+      }
+    }
+
+    // Computed background-image on all elements (catches CSS-applied bg images)
+    for (const el of document.querySelectorAll("*")) {
+      const cs = window.getComputedStyle(el);
+      const bg = cs.getPropertyValue("background-image");
+      if (bg && bg !== "none") {
+        for (const u of extractCssUrls(bg)) {
+          add(u, "css-background");
+        }
+      }
+    }
+
+    // @font-face from stylesheets + retain full source CSS (F0)
+    const stylesheets: { href: string | null; cssText: string | null; fontFaceRules: string[] }[] =
+      [];
+    for (const sheet of document.styleSheets) {
+      let rules: CSSRuleList | null = null;
+      let fullCssText: string | null = null;
+      const fontFaceRules: string[] = [];
+      const { href } = sheet;
+
+      try {
+        rules = sheet.cssRules;
+        // Build full cssText from all rules
+        const parts: string[] = [];
+        for (const rule of rules) {
+          parts.push(rule.cssText);
+          if (rule instanceof CSSFontFaceRule) {
+            fontFaceRules.push(rule.cssText);
+            const src = rule.style.getPropertyValue("src");
+            if (src) {
+              for (const u of extractCssUrls(src)) {
+                add(u, "font-face");
+              }
+            }
+          }
+        }
+        fullCssText = parts.join("\n");
+      } catch {
+        // Cross-origin sheet — cssRules inaccessible. Record href for browser-context refetch.
+      }
+
+      stylesheets.push({ href, cssText: fullCssText, fontFaceRules });
+    }
+
+    // Favicon and other link icons
+    for (const el of document.querySelectorAll(
+      'link[rel~="icon"], link[rel="shortcut icon"], link[rel="apple-touch-icon"]',
+    )) {
+      const { href } = el as HTMLLinkElement;
+      if (href) {
+        add(href, "favicon");
+      }
+    }
+
+    // Open Graph images
+    for (const el of document.querySelectorAll('meta[property="og:image"]')) {
+      const { content } = el as HTMLMetaElement;
+      if (content) {
+        add(content, "og-image");
+      }
+    }
+
+    // Count inline SVGs (kept inline, not downloaded)
+    inlineSvgCount = document.querySelectorAll("svg").length;
+
+    return { assets, inlineSvgCount, stylesheets } as {
+      assets: { url: string; source: string }[];
+      inlineSvgCount: number;
+      stylesheets: { href: string | null; cssText: string | null; fontFaceRules: string[] }[];
+    };
+  });
+
+  // R2: For cross-origin sheets we couldn't access (cssText=null, has href), fetch them
+  // In-browser to get @font-face rules (e.g. Google Fonts loaded via <link>)
+  const inaccessibleSheets = result.stylesheets.filter((s) => s.cssText === null && s.href);
+  if (inaccessibleSheets.length > 0) {
+    const fetched = await page.evaluate(
+      async (hrefs: string[]) => {
+        const results: { href: string; cssText: string; fontFaceRules: string[] }[] = [];
+        for (const href of hrefs) {
+          try {
+            const res = await fetch(href);
+            if (res.ok) {
+              const text = await res.text();
+              // Parse @font-face rules from the raw CSS text
+              const fontFaceRules: string[] = [];
+              const re = /@font-face\s*\{[^}]*\}/g;
+              let match;
+              while ((match = re.exec(text)) !== null) {
+                fontFaceRules.push(match[0]);
+              }
+              results.push({ href, cssText: text, fontFaceRules });
+            }
+          } catch {
+            // Skip unreachable sheets
+          }
+        }
+        return results;
+      },
+      inaccessibleSheets.map((s) => s.href!),
+    );
+
+    // Merge fetched data back and discover font URLs
+    for (const fetched_sheet of fetched) {
+      const existing = result.stylesheets.find((s) => s.href === fetched_sheet.href);
+      if (existing) {
+        existing.cssText = fetched_sheet.cssText;
+        existing.fontFaceRules = fetched_sheet.fontFaceRules;
+      }
+
+      // Extract font URLs from the fetched @font-face rules
+      const urlRe = /url\(\s*(['"]?)(.+?)\1\s*\)/g;
+      for (const rule of fetched_sheet.fontFaceRules) {
+        let m;
+        while ((m = urlRe.exec(rule)) !== null) {
+          const fontUrl = m.at(2);
+          if (fontUrl && !fontUrl.startsWith("data:")) {
+            const abs = new URL(fontUrl, fetched_sheet.href).href;
+            if (!result.assets.some((a) => a.url === abs)) {
+              result.assets.push({ url: abs, source: "font-face" });
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return result as AssetCollectionResult;
+}

package/src/asset-download.ts

@@ -0,0 +1,201 @@
+/**
+ * Download discovered assets to public/assets/ and build a URL rewrite map.
+ *
+ * Skips tracking/analytics domains. Dedupes by URL. Preserves file extensions. Organizes into
+ * subdirectories by type: images/, fonts/, icons/.
+ */
+
+import { join, extname } from "node:path";
+import { mkdir } from "node:fs/promises";
+import type { DiscoveredAsset } from "./asset-collect.ts";
+
+export interface DownloadResult {
+  /**
+   * Maps original absolute URL → relative path from project root (e.g.
+   * "public/assets/images/hero.jpg").
+   */
+  rewriteMap: Map<string, string>;
+  /** URLs that failed to download. */
+  failed: string[];
+  /** URLs skipped (tracking, analytics, etc.). */
+  skipped: string[];
+  /** Total bytes downloaded. */
+  totalBytes: number;
+}
+
+const BLOCKED_DOMAINS = new Set([
+  "www.google-analytics.com",
+  "google-analytics.com",
+  "googletagmanager.com",
+  "www.googletagmanager.com",
+  "analytics.google.com",
+  "stats.g.doubleclick.net",
+  "www.facebook.com",
+  "connect.facebook.net",
+  "px.ads.linkedin.com",
+  "bat.bing.com",
+  "snap.licdn.com",
+  "tr.snapchat.com",
+  "cdn.segment.com",
+  "api.segment.io",
+  "static.hotjar.com",
+  "script.hotjar.com",
+  "plausible.io",
+  "cdn.amplitude.com",
+]);
+
+const FONT_EXTENSIONS = new Set([".woff", ".woff2", ".ttf", ".otf", ".eot"]);
+const IMAGE_EXTENSIONS = new Set([
+  ".jpg",
+  ".jpeg",
+  ".png",
+  ".gif",
+  ".webp",
+  ".avif",
+  ".svg",
+  ".ico",
+  ".bmp",
+  ".tiff",
+]);
+
+function classifyAsset(
+  url: string,
+  source: DiscoveredAsset["source"],
+): "images" | "fonts" | "icons" | "other" {
+  if (source === "font-face") {
+    return "fonts";
+  }
+  if (source === "favicon") {
+    return "icons";
+  }
+
+  const [ext = ""] = extname(new URL(url).pathname).toLowerCase().split("?");
+  if (FONT_EXTENSIONS.has(ext)) {
+    return "fonts";
+  }
+  if (IMAGE_EXTENSIONS.has(ext)) {
+    return "images";
+  }
+  return "other";
+}
+
+function sanitizeFilename(url: string): string {
+  const parsed = new URL(url);
+  let { pathname } = parsed;
+
+  // Remove leading slash
+  if (pathname.startsWith("/")) {
+    pathname = pathname.slice(1);
+  }
+
+  // Use just the filename part, not the full path
+  const parts = pathname.split("/");
+  let filename = parts.at(-1) || "asset";
+
+  // Strip query params from extension
+  const qIdx = filename.indexOf("?");
+  if (qIdx !== -1) {
+    filename = filename.slice(0, qIdx);
+  }
+
+  // Sanitize: keep alphanumeric, hyphens, dots, underscores
+  filename = filename.replaceAll(/[^a-zA-Z0-9._-]/g, "_");
+
+  // Ensure it has an extension
+  if (!extname(filename)) {
+    filename += ".bin";
+  }
+
+  // Cap length
+  if (filename.length > 100) {
+    const ext = extname(filename);
+    filename = filename.slice(0, 96) + ext;
+  }
+
+  return filename;
+}
+
+function isBlocked(url: string): boolean {
+  try {
+    const { hostname } = new URL(url);
+    return BLOCKED_DOMAINS.has(hostname);
+  } catch {
+    return true;
+  }
+}
+
+export async function downloadAssets(
+  assets: DiscoveredAsset[],
+  outDir: string,
+  sourceUrl?: string,
+): Promise<DownloadResult> {
+  const rewriteMap = new Map<string, string>();
+  const failed: string[] = [];
+  const skipped: string[] = [];
+  let totalBytes = 0;
+
+  // Dedupe by URL
+  const unique = new Map<string, DiscoveredAsset>();
+  for (const asset of assets) {
+    if (!unique.has(asset.url)) {
+      unique.set(asset.url, asset);
+    }
+  }
+
+  // Create subdirectories
+  const assetsDir = join(outDir, "public", "assets");
+  await mkdir(join(assetsDir, "images"), { recursive: true });
+  await mkdir(join(assetsDir, "fonts"), { recursive: true });
+  await mkdir(join(assetsDir, "icons"), { recursive: true });
+  await mkdir(join(assetsDir, "other"), { recursive: true });
+
+  // Track filenames to avoid collisions
+  const usedNames = new Map<string, number>();
+
+  for (const [url, asset] of unique) {
+    if (isBlocked(url)) {
+      skipped.push(url);
+      continue;
+    }
+
+    const subdir = classifyAsset(url, asset.source);
+    let filename = sanitizeFilename(url);
+
+    // Dedupe filenames within subdirectory
+    const nameKey = `${subdir}/${filename}`;
+    const count = usedNames.get(nameKey) ?? 0;
+    if (count > 0) {
+      const ext = extname(filename);
+      const base = filename.slice(0, -ext.length || undefined);
+      filename = `${base}-${count}${ext}`;
+    }
+    usedNames.set(nameKey, count + 1);
+
+    const destPath = join(assetsDir, subdir, filename);
+    const relativePath = `public/assets/${subdir}/${filename}`;
+
+    try {
+      const headers: Record<string, string> = {
+        "User-Agent":
+          "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+      };
+      if (sourceUrl) {
+        headers["Referer"] = sourceUrl;
+      }
+      const response = await fetch(url, { headers, redirect: "follow" });
+      if (!response.ok) {
+        failed.push(url);
+        continue;
+      }
+
+      const buffer = await response.arrayBuffer();
+      totalBytes += buffer.byteLength;
+      await Bun.write(destPath, buffer);
+      rewriteMap.set(url, relativePath);
+    } catch {
+      failed.push(url);
+    }
+  }
+
+  return { rewriteMap, failed, skipped, totalBytes };
+}

package/src/asset-rewrite.ts

@@ -0,0 +1,143 @@
+/**
+ * Walk a Jx tree and rewrite absolute asset URLs to local relative paths using a rewrite map.
+ *
+ * Handles: - attributes.src, attributes.href, attributes.poster, attributes.srcset -
+ * style.backgroundImage (url() references) - $media nested style objects with backgroundImage
+ */
+
+import type { JxElement, JxStyle } from "@jxsuite/schema/types";
+
+/** Rewrite all asset URLs in a Jx tree in-place. Returns the count of rewrites performed. */
+export function rewriteAssetUrls(
+  root: JxElement,
+  rewriteMap: Map<string, string>,
+  sourceUrl?: string,
+): number {
+  let count = 0;
+
+  function rewriteUrl(url: string): string | null {
+    const rewritten = rewriteMap.get(url);
+    if (rewritten) {
+      return rewritten;
+    }
+
+    // Try without trailing slash
+    const trimmed = url.endsWith("/") ? url.slice(0, -1) : `${url}/`;
+    const alt = rewriteMap.get(trimmed);
+    if (alt) {
+      return alt;
+    }
+
+    // Resolve relative/protocol-relative URLs against the source page URL
+    if (sourceUrl) {
+      try {
+        const resolved = new URL(url, sourceUrl).href;
+        if (resolved !== url) {
+          const fromResolved = rewriteMap.get(resolved);
+          if (fromResolved) {
+            return fromResolved;
+          }
+        }
+      } catch {
+        // Invalid URL — skip
+      }
+    }
+
+    return null;
+  }
+
+  function rewriteSrcset(srcset: string): string {
+    return srcset
+      .split(",")
+      .map((entry) => {
+        const parts = entry.trim().split(/\s+/);
+        const [url = ""] = parts;
+        const rewritten = rewriteUrl(url);
+        if (rewritten) {
+          count += 1;
+          parts[0] = rewritten;
+        }
+        return parts.join(" ");
+      })
+      .join(", ");
+  }
+
+  function rewriteCssUrls(value: string): string {
+    return value.replaceAll(/url\(\s*(['"]?)(.+?)\1\s*\)/g, (_match, quote, url: string) => {
+      const rewritten = rewriteUrl(url);
+      if (rewritten) {
+        count += 1;
+        return `url(${quote}${rewritten}${quote})`;
+      }
+      return _match;
+    });
+  }
+
+  function rewriteStyleObject(style: JxStyle): void {
+    for (const [key, value] of Object.entries(style)) {
+      if (typeof value === "string" && value.includes("url(")) {
+        (style as Record<string, unknown>)[key] = rewriteCssUrls(value);
+      } else if (typeof value === "object" && value !== null && key.startsWith("@")) {
+        rewriteStyleObject(value as JxStyle);
+      }
+    }
+  }
+
+  function walk(node: JxElement): void {
+    const attrs = node.attributes as Record<string, unknown> | undefined;
+
+    if (attrs) {
+      // Src attribute (img, video, audio, source, etc.)
+      if (typeof attrs.src === "string") {
+        const rewritten = rewriteUrl(attrs.src);
+        if (rewritten) {
+          attrs.src = rewritten;
+          count += 1;
+        }
+      }
+
+      // Href on non-anchor elements (could be link icon, etc.)
+      if (typeof attrs.href === "string") {
+        const tag = (node.tagName ?? "").toLowerCase();
+        if (tag !== "a") {
+          const rewritten = rewriteUrl(attrs.href);
+          if (rewritten) {
+            attrs.href = rewritten;
+            count += 1;
+          }
+        }
+      }
+
+      // Poster attribute (video)
+      if (typeof attrs.poster === "string") {
+        const rewritten = rewriteUrl(attrs.poster);
+        if (rewritten) {
+          attrs.poster = rewritten;
+          count += 1;
+        }
+      }
+
+      // Srcset attribute
+      if (typeof attrs.srcset === "string") {
+        attrs.srcset = rewriteSrcset(attrs.srcset);
+      }
+    }
+
+    // Style object — background-image url() references
+    if (node.style) {
+      rewriteStyleObject(node.style);
+    }
+
+    // Recurse
+    if (Array.isArray(node.children)) {
+      for (const child of node.children) {
+        if (typeof child !== "string") {
+          walk(child);
+        }
+      }
+    }
+  }
+
+  walk(root);
+  return count;
+}