@jxsuite/import 0.34.0

package/src/crawl.ts

@@ -0,0 +1,391 @@
+import { launchBrowser, capturePage } from "./capture.ts";
+import type { CaptureResult } from "./capture.ts";
+import { convertToJx } from "./to-jx.ts";
+import type { ToJxResult } from "./to-jx.ts";
+import { captureStyles } from "./style-capture.ts";
+import { diffAllStyles, kebabToCamel } from "./style-diff.ts";
+import { extractMedia } from "./media-extract.ts";
+import { applyStylesToTree } from "./apply-styles.ts";
+import { collectAssets } from "./asset-collect.ts";
+import { downloadAssets } from "./asset-download.ts";
+import { rewriteAssetUrls } from "./asset-rewrite.ts";
+import { applyTokens } from "./css-tokens.ts";
+
+export interface CrawlOptions {
+  url: string;
+  outDir: string;
+  maxDepth: number;
+  maxPages: number;
+  maxNodesPerPage: number;
+  skipStyles: boolean;
+  skipAssets: boolean;
+  respectRobots: boolean;
+  /** Skip scroll-to-bottom before capture (default: false). */
+  noScroll?: boolean;
+  /** Capture a reference screenshot of each page before closing (for --verify). */
+  captureScreenshots?: boolean;
+  onProgress?: (msg: string) => void;
+  /** Abort the crawl between pages; the loop throws before capturing the next page. */
+  signal?: AbortSignal;
+}
+
+export interface CrawledPage {
+  url: string;
+  route: string;
+  title: string;
+  jx: ToJxResult;
+  depth: number;
+  links: string[];
+  /** Reference screenshot PNG buffer, present when captureScreenshots is true. */
+  screenshot?: Buffer | undefined;
+}
+
+export interface CrawlResult {
+  pages: CrawledPage[];
+  breakpoints: Record<string, string> | undefined;
+  skippedByRobots: string[];
+  skippedByNodeCap: string[];
+  /** Collected @font-face rules from all pages (R2). */
+  fontFaceRules: string[];
+  /** Font URL → local path rewrite map (R2). */
+  fontRewriteMap: Map<string, string>;
+  /** CSS custom property tokens merged across all pages (R5). */
+  styleTokens: Record<string, string> | undefined;
+}
+
+/**
+ * Normalize a URL for deduplication: strip hash, trailing slash, tracking params, sort remaining
+ * params.
+ */
+export function normalizeUrl(raw: string): string {
+  try {
+    const u = new URL(raw);
+    u.hash = "";
+
+    const TRACKING_PARAMS = new Set([
+      "utm_source",
+      "utm_medium",
+      "utm_campaign",
+      "utm_term",
+      "utm_content",
+      "ref",
+      "fbclid",
+      "gclid",
+      "mc_cid",
+      "mc_eid",
+    ]);
+    for (const p of TRACKING_PARAMS) {
+      u.searchParams.delete(p);
+    }
+
+    u.searchParams.sort();
+
+    let path = u.pathname;
+    if (path.length > 1 && path.endsWith("/")) {
+      path = path.slice(0, -1);
+    }
+    u.pathname = path;
+
+    return u.toString();
+  } catch {
+    return raw;
+  }
+}
+
+/**
+ * Convert a URL path to a Jx page file path under pages/. / → pages/index.json /about →
+ * pages/about.json /blog/post → pages/blog/post.json
+ */
+export function routeToFilePath(url: string, _baseOrigin?: string): string {
+  const u = new URL(url);
+  let path = u.pathname;
+
+  if (path.length > 1 && path.endsWith("/")) {
+    path = path.slice(0, -1);
+  }
+
+  if (path === "" || path === "/") {
+    return "pages/index.json";
+  }
+
+  // Remove leading slash, sanitize segments
+  const segments = path
+    .slice(1)
+    .split("/")
+    .map((s) => s.replaceAll(/[^a-zA-Z0-9._-]/g, "_"));
+
+  // If the last segment has an extension (like .html), strip it
+  const last = segments.at(-1)!;
+  const dotIdx = last.lastIndexOf(".");
+  if (dotIdx > 0) {
+    segments[segments.length - 1] = last.slice(0, dotIdx);
+  }
+
+  return `pages/${segments.join("/")}.json`;
+}
+
+/** Fetch and parse robots.txt for a given origin. Returns a set of disallowed path prefixes. */
+export async function fetchRobotsTxt(origin: string): Promise<Set<string>> {
+  const disallowed = new Set<string>();
+  try {
+    const res = await fetch(`${origin}/robots.txt`);
+    if (!res.ok) {
+      return disallowed;
+    }
+
+    const text = await res.text();
+    let inUserAgent = false;
+    for (const line of text.split("\n")) {
+      const trimmed = line.trim();
+      if (trimmed.startsWith("#") || trimmed === "") {
+        continue;
+      }
+
+      if (trimmed.toLowerCase().startsWith("user-agent:")) {
+        const agent = trimmed.slice("user-agent:".length).trim();
+        inUserAgent = agent === "*";
+      } else if (inUserAgent && trimmed.toLowerCase().startsWith("disallow:")) {
+        const path = trimmed.slice("disallow:".length).trim();
+        if (path) {
+          disallowed.add(path);
+        }
+      }
+    }
+  } catch {
+    // Robots.txt unavailable — allow everything
+  }
+  return disallowed;
+}
+
+function isDisallowed(url: string, disallowedPaths: Set<string>): boolean {
+  const path = new URL(url).pathname;
+  for (const prefix of disallowedPaths) {
+    if (path.startsWith(prefix)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/** BFS crawl a site starting from `url`. Captures each page, converts to Jx, applies styles/assets. */
+export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
+  const {
+    url,
+    outDir,
+    maxDepth,
+    maxPages,
+    maxNodesPerPage,
+    skipStyles,
+    skipAssets,
+    respectRobots,
+    onProgress = console.log,
+  } = options;
+
+  const { origin } = new URL(url);
+  const browser = await launchBrowser();
+
+  // Fetch robots.txt
+  let disallowedPaths = new Set<string>();
+  if (respectRobots) {
+    onProgress("  Fetching robots.txt...");
+    disallowedPaths = await fetchRobotsTxt(origin);
+    if (disallowedPaths.size > 0) {
+      onProgress(`  robots.txt: ${disallowedPaths.size} disallowed paths`);
+    }
+  }
+
+  const visited = new Set<string>();
+  const queue: { url: string; depth: number }[] = [{ url: normalizeUrl(url), depth: 0 }];
+  visited.add(normalizeUrl(url));
+
+  const pages: CrawledPage[] = [];
+  const skippedByRobots: string[] = [];
+  const skippedByNodeCap: string[] = [];
+  let mergedBreakpoints: Record<string, string> | undefined;
+  let mergedTokens: Record<string, string> | undefined;
+  const allFontFaceRules: string[] = [];
+  const fontRewriteMap = new Map<string, string>();
+  const seenFontRules = new Set<string>();
+
+  while (queue.length > 0 && pages.length < maxPages) {
+    if (options.signal?.aborted) {
+      throw new Error("Import aborted");
+    }
+    const entry = queue.shift()!;
+
+    if (respectRobots && isDisallowed(entry.url, disallowedPaths)) {
+      skippedByRobots.push(entry.url);
+      onProgress(`  Skipped (robots.txt): ${entry.url}`);
+      continue;
+    }
+
+    onProgress(
+      `  [${pages.length + 1}/${maxPages}] Capturing ${entry.url} (depth ${entry.depth})...`,
+    );
+
+    let capture: CaptureResult;
+    try {
+      capture = await capturePage(entry.url, browser, { scrollToBottom: !options.noScroll });
+    } catch (error) {
+      onProgress(
+        `  ⚠ Failed to capture ${entry.url}: ${error instanceof Error ? error.message : error}`,
+      );
+      continue;
+    }
+
+    const jx = convertToJx(capture.bodyHtml);
+    onProgress(`  Converted: ${jx.nodeCount} nodes`);
+
+    if (jx.nodeCount > maxNodesPerPage) {
+      onProgress(
+        `  ⚠ Page exceeds node cap (${jx.nodeCount} > ${maxNodesPerPage}), skipping styles/assets`,
+      );
+      skippedByNodeCap.push(entry.url);
+      let screenshot: Buffer | undefined;
+      if (options.captureScreenshots) {
+        screenshot = Buffer.from(await capture.page.screenshot({ type: "png" }));
+      }
+      await capture.page.close();
+
+      pages.push({
+        url: entry.url,
+        route: routeToFilePath(entry.url, origin),
+        title: capture.title,
+        jx,
+        depth: entry.depth,
+        links: capture.links,
+        screenshot,
+      });
+      continue;
+    }
+
+    // Style capture (Phase 1)
+    if (!skipStyles) {
+      try {
+        const styleResult = await captureStyles(capture.page);
+        const diffed = diffAllStyles(styleResult.elements, styleResult.uaDefaults);
+
+        // R5: Replace resolved values with var(--name) references
+        if (Object.keys(styleResult.customProperties).length > 0) {
+          const tokenResult = applyTokens(diffed, styleResult.customProperties);
+          if (tokenResult.replacements > 0) {
+            if (!mergedTokens) {
+              mergedTokens = {};
+            }
+            Object.assign(mergedTokens, tokenResult.tokens);
+          }
+        }
+
+        if (styleResult.mediaQueries.length > 0) {
+          const media = await extractMedia(
+            capture.page,
+            styleResult.elements,
+            styleResult.uaDefaults,
+            styleResult.mediaQueries,
+          );
+          const bpCount = Object.keys(media.breakpoints).length;
+          if (bpCount > 0) {
+            if (!mergedBreakpoints) {
+              mergedBreakpoints = {};
+            }
+            Object.assign(mergedBreakpoints, media.breakpoints);
+            applyStylesToTree(jx.document, diffed, media.deltas);
+          } else {
+            applyStylesToTree(jx.document, diffed);
+          }
+        } else {
+          applyStylesToTree(jx.document, diffed);
+        }
+
+        // Apply <html>/<body> computed styles to the root wrapper
+        if (Object.keys(styleResult.documentStyles).length > 0) {
+          if (!jx.document.style) {
+            jx.document.style = {};
+          }
+          for (const [prop, val] of Object.entries(styleResult.documentStyles)) {
+            jx.document.style[kebabToCamel(prop)] = val;
+          }
+        }
+      } catch (error) {
+        onProgress(`  ⚠ Style capture failed: ${error instanceof Error ? error.message : error}`);
+      }
+    }
+
+    // Asset download (Phase 2)
+    if (!skipAssets) {
+      try {
+        const collected = await collectAssets(capture.page);
+
+        // Accumulate @font-face rules (R2) — dedupe across pages
+        for (const sheet of collected.stylesheets) {
+          for (const rule of sheet.fontFaceRules) {
+            if (!seenFontRules.has(rule)) {
+              seenFontRules.add(rule);
+              allFontFaceRules.push(rule);
+            }
+          }
+        }
+
+        if (collected.assets.length > 0) {
+          const downloaded = await downloadAssets(collected.assets, outDir, entry.url);
+          if (downloaded.rewriteMap.size > 0) {
+            rewriteAssetUrls(jx.document, downloaded.rewriteMap, entry.url);
+            // Collect font rewrites (R2)
+            for (const [originalUrl, localPath] of downloaded.rewriteMap) {
+              if (localPath.includes("/fonts/") || /\.(woff2?|ttf|otf|eot)$/i.test(localPath)) {
+                fontRewriteMap.set(originalUrl, localPath);
+              }
+            }
+          }
+        }
+      } catch (error) {
+        onProgress(
+          `  ⚠ Asset collection failed: ${error instanceof Error ? error.message : error}`,
+        );
+      }
+    }
+
+    let screenshot: Buffer | undefined;
+    if (options.captureScreenshots) {
+      screenshot = Buffer.from(await capture.page.screenshot({ type: "png" }));
+    }
+    await capture.page.close();
+
+    const route = routeToFilePath(entry.url, origin);
+    pages.push({
+      url: entry.url,
+      route,
+      title: capture.title,
+      jx,
+      depth: entry.depth,
+      links: capture.links,
+      screenshot,
+    });
+
+    // Enqueue discovered links (BFS)
+    if (entry.depth < maxDepth) {
+      for (const link of capture.links) {
+        const normalized = normalizeUrl(link);
+        if (!normalized.startsWith(origin)) {
+          continue;
+        }
+        if (visited.has(normalized)) {
+          continue;
+        }
+
+        visited.add(normalized);
+        queue.push({ url: normalized, depth: entry.depth + 1 });
+      }
+    }
+  }
+
+  return {
+    pages,
+    breakpoints: mergedBreakpoints,
+    skippedByRobots,
+    skippedByNodeCap,
+    fontFaceRules: allFontFaceRules,
+    fontRewriteMap,
+    styleTokens: mergedTokens,
+  };
+}

package/src/css-tokens.ts

@@ -0,0 +1,156 @@
+/**
+ * Extract CSS custom properties from retained stylesheets and replace resolved computed values with
+ * var(--name) references. Hoists token declarations for project.json.$style.
+ */
+
+import type { DiffedStyle } from "./style-diff.ts";
+
+export interface TokenExtractionResult {
+  /**
+   * Custom property declarations to hoist into project.json.$style (e.g. { "--brand-blue":
+   * "#3b82f6" }).
+   */
+  tokens: Record<string, string>;
+  /** Number of value replacements made across all diffed styles. */
+  replacements: number;
+}
+
+/**
+ * Normalize a CSS color value for comparison. Handles common variations: - rgb(r, g, b) vs rgb(r g
+ * b) - Trailing semicolons/whitespace
+ */
+function normalizeValue(val: string): string {
+  return val
+    .trim()
+    .replace(/\s*;\s*$/, "")
+    .replaceAll(/,\s*/g, ", ")
+    .replaceAll(/\s+/g, " ");
+}
+
+/**
+ * Build a reverse map: resolved CSS value → var(--name). When multiple custom properties resolve to
+ * the same value, prefer shorter names (heuristic: shorter names are more likely to be semantic
+ * tokens).
+ */
+function buildValueToVarMap(props: Map<string, string>): Map<string, string> {
+  const valueToVar = new Map<string, string>();
+
+  // Sort by name length ascending so shorter (more semantic) names win ties
+  const sorted = [...props.entries()].toSorted((a, b) => a[0].length - b[0].length);
+
+  for (const [name, value] of sorted) {
+    const normalized = normalizeValue(value);
+    if (!valueToVar.has(normalized)) {
+      valueToVar.set(normalized, `var(${name})`);
+    }
+  }
+
+  return valueToVar;
+}
+
+const TOKEN_ELIGIBLE_PROPS = new Set([
+  "color",
+  "backgroundColor",
+  "borderColor",
+  "borderTopColor",
+  "borderRightColor",
+  "borderBottomColor",
+  "borderLeftColor",
+  "outlineColor",
+  "textDecorationColor",
+  "fill",
+  "stroke",
+  "caretColor",
+  "columnRuleColor",
+  "boxShadow",
+  "textShadow",
+  "background",
+  "backgroundImage",
+  "borderTop",
+  "borderRight",
+  "borderBottom",
+  "borderLeft",
+  "border",
+  "fontFamily",
+  "fontSize",
+  "fontWeight",
+  "lineHeight",
+  "letterSpacing",
+  "gap",
+  "rowGap",
+  "columnGap",
+  "borderRadius",
+  "borderTopLeftRadius",
+  "borderTopRightRadius",
+  "borderBottomLeftRadius",
+  "borderBottomRightRadius",
+  "padding",
+  "paddingTop",
+  "paddingRight",
+  "paddingBottom",
+  "paddingLeft",
+  "margin",
+  "marginTop",
+  "marginRight",
+  "marginBottom",
+  "marginLeft",
+  "maxWidth",
+  "width",
+]);
+
+/**
+ * Replace resolved computed values with var(--name) references in diffed styles. Also collects the
+ * set of custom properties actually used (for hoisting into project.json).
+ *
+ * Mutates the diffed styles in place for efficiency.
+ *
+ * @param customProperties - Map of CSS custom property name → resolved value, as extracted by
+ *   captureStyles() in-browser.
+ */
+export function applyTokens(
+  diffedStyles: DiffedStyle[],
+  customProperties: Record<string, string>,
+): TokenExtractionResult {
+  const props = new Map(Object.entries(customProperties));
+  if (props.size === 0) {
+    return { tokens: {}, replacements: 0 };
+  }
+
+  const valueToVar = buildValueToVarMap(props);
+  const usedTokens = new Map<string, string>();
+  let replacements = 0;
+
+  for (const el of diffedStyles) {
+    for (const [prop, value] of Object.entries(el.style)) {
+      if (!TOKEN_ELIGIBLE_PROPS.has(prop)) {
+        continue;
+      }
+      if (typeof value !== "string") {
+        continue;
+      }
+      // Already a var() reference
+      if (value.startsWith("var(")) {
+        continue;
+      }
+
+      const normalized = normalizeValue(value);
+      const varRef = valueToVar.get(normalized);
+      if (varRef) {
+        el.style[prop] = varRef;
+        replacements += 1;
+
+        // Track which token was used so we can hoist its declaration
+        const varName = varRef.slice(4, -1); // "var(--x)" → "--x"
+        usedTokens.set(varName, props.get(varName)!);
+      }
+    }
+  }
+
+  // Sort tokens alphabetically for stable output
+  const tokens: Record<string, string> = {};
+  for (const name of [...usedTokens.keys()].toSorted()) {
+    tokens[name] = usedTokens.get(name)!;
+  }
+
+  return { tokens, replacements };
+}