@jxsuite/import 0.34.0

package/src/capture.ts

@@ -0,0 +1,159 @@
+import { launch } from "puppeteer-core";
+import type { Browser, Page } from "puppeteer-core";
+
+export interface CaptureResult {
+  url: string;
+  title: string;
+  bodyHtml: string;
+  /** Discovered same-origin links for the crawler (Phase 3). */
+  links: string[];
+  /** The puppeteer Page, kept open for style capture (Phase 1). Caller must close. */
+  page: Page;
+}
+
+const DEFAULT_CHROME_PATHS = [
+  "google-chrome-stable",
+  "google-chrome",
+  "chromium-browser",
+  "chromium",
+];
+
+function findChrome(executablePath?: string): string {
+  if (executablePath) {
+    return executablePath;
+  }
+  const env = process.env.CHROME_PATH;
+  if (env) {
+    return env;
+  }
+
+  for (const name of DEFAULT_CHROME_PATHS) {
+    const path = Bun.which(name);
+    if (path) {
+      return path;
+    }
+  }
+  throw new Error(
+    "Could not find Chrome/Chromium. Set CHROME_PATH or install google-chrome-stable.",
+  );
+}
+
+let _browser: Browser | null = null;
+
+export interface LaunchOptions {
+  /** Explicit browser binary. Wins over CHROME_PATH and PATH discovery. */
+  executablePath?: string;
+}
+
+export async function launchBrowser(options?: LaunchOptions): Promise<Browser> {
+  if (_browser?.connected) {
+    return _browser;
+  }
+  _browser = await launch({
+    executablePath: findChrome(options?.executablePath),
+    headless: true,
+    args: ["--no-sandbox", "--disable-gpu"],
+  });
+  return _browser;
+}
+
+export async function closeBrowser(): Promise<void> {
+  if (_browser?.connected) {
+    await _browser.close();
+    _browser = null;
+  }
+}
+
+export interface CaptureOptions {
+  /** Scroll to bottom before capture to trigger lazy-loaded content (default: true). */
+  scrollToBottom?: boolean;
+}
+
+/**
+ * Scroll the page to the bottom in steps to trigger lazy-loaded images and intersection-observer
+ * content, then scroll back to top. Settles between steps to let content render.
+ */
+async function scrollToRevealAll(page: Page): Promise<void> {
+  await page.evaluate(async () => {
+    const delay = (ms: number) =>
+      new Promise<void>((r) => {
+        setTimeout(r, ms);
+      });
+    const scrollHeight = () => document.body.scrollHeight;
+    const viewportHeight = window.innerHeight;
+    let lastHeight = 0;
+    let currentPosition = 0;
+
+    // Scroll down in viewport-sized steps
+    while (currentPosition < scrollHeight()) {
+      currentPosition += viewportHeight;
+      window.scrollTo(0, currentPosition);
+      await delay(100);
+
+      // If the page grew (infinite scroll), keep going but cap at 20 iterations
+      if (scrollHeight() > lastHeight) {
+        lastHeight = scrollHeight();
+      }
+      if (currentPosition > viewportHeight * 20) {
+        break;
+      }
+    }
+
+    // Settle for lazy images that load on scroll
+    await delay(300);
+
+    // Scroll back to top for the reference screenshot
+    window.scrollTo(0, 0);
+    await delay(100);
+  });
+}
+
+/**
+ * Capture a page's DOM. Returns the page object still open — the caller is responsible for closing
+ * it (after style capture in Phase 1, or immediately).
+ */
+export async function capturePage(
+  url: string,
+  browser?: Browser,
+  options?: CaptureOptions,
+): Promise<CaptureResult> {
+  const { scrollToBottom = true } = options ?? {};
+  const br = browser ?? (await launchBrowser());
+  const page: Page = await br.newPage();
+  await page.setViewport({ width: 1440, height: 900 });
+
+  await page.goto(url, { waitUntil: "networkidle0", timeout: 30_000 });
+
+  // R1: Scroll to bottom to trigger lazy-loaded content before capture
+  if (scrollToBottom) {
+    await scrollToRevealAll(page);
+  }
+
+  const result = await page.evaluate(() => {
+    // Strip scripts and noscript before capture
+    for (const el of document.querySelectorAll("script, noscript")) {
+      el.remove();
+    }
+
+    const links: string[] = [];
+    const { origin } = location;
+    for (const a of document.querySelectorAll("a[href]")) {
+      try {
+        const { href } = new URL((a as HTMLAnchorElement).href, location.href);
+        if (href.startsWith(origin) && !href.includes("#")) {
+          links.push(href);
+        }
+      } catch {
+        // Skip invalid URLs
+      }
+    }
+
+    return {
+      title: document.title,
+      bodyHtml: document.body.innerHTML,
+      links: [...new Set(links)],
+    };
+  });
+
+  return { url, ...result, page };
+}

package/src/cli.ts

@@ -0,0 +1,153 @@
+#!/usr/bin/env bun
+
+import { resolve } from "node:path";
+import { homedir } from "node:os";
+import { importSite } from "./run.ts";
+
+function usage(): never {
+  console.log(`Usage: jx-import <url> [options]
+
+Clone a live website into a Jx project.
+
+Options:
+  --out <dir>              Output directory (default: ~/jx-imports/<hostname>)
+  --depth <n>              Max crawl depth (default: 2, 0 = single page)
+  --max-pages <n>          Max pages to capture (default: 25)
+  --max-nodes-per-page <n> Skip styles/assets for pages above this (default: 5000)
+  --no-styles              Skip CSS capture
+  --no-assets              Skip asset download
+  --no-crawl               Single page only (equivalent to --depth 0)
+  --no-scroll              Skip scroll-to-bottom (faster, may miss lazy content)
+  --no-robots              Ignore robots.txt
+  --no-components          Skip component extraction (Phase 4)
+  --min-instances <n>      Min recurring instances to extract a component (default: 2)
+  --min-depth <n>          Min subtree depth to consider for componentization (default: 2)
+  --ai-components          Use LLM to refine component names and props (Phase 4 AI pass)
+  --ai-model <model>       Model for AI componentization (default: gpt-4o-mini)
+  --verify                 After import, build and screenshot-diff vs original (Phase 5)
+  --verify-threshold <n>   Pixel diff threshold 0..1 (default: 0.15)
+
+Environment:
+  OPENAI_API_KEY           Required for --ai-components
+  OPENAI_BASE_URL          Custom API base URL (default: https://api.openai.com/v1)
+  CHROME_PATH              Explicit Chrome/Chromium binary
+
+Examples:
+  jx-import https://example.com
+  jx-import https://example.com --depth 1 --max-pages 10
+  jx-import https://example.com --no-crawl
+  jx-import https://example.com --out sites/my-clone --no-styles
+  jx-import https://example.com --verify
+  jx-import https://example.com --ai-components`);
+  process.exit(1);
+}
+
+function parseIntArg(args: string[], flag: string, fallback: number): number {
+  const idx = args.indexOf(flag);
+  if (idx === -1 || !args[idx + 1]) {
+    return fallback;
+  }
+  const n = Math.trunc(Number(args[idx + 1]));
+  return Number.isNaN(n) ? fallback : n;
+}
+
+const args = process.argv.slice(2);
+if (args.length === 0 || args.includes("--help") || args.includes("-h")) {
+  usage();
+}
+
+const [url] = args;
+if (!url || (!url.startsWith("http://") && !url.startsWith("https://"))) {
+  console.error("Error: URL must start with http:// or https://");
+  process.exit(1);
+}
+
+const noCrawl = args.includes("--no-crawl");
+const noComponents = args.includes("--no-components");
+const doAiComponents = args.includes("--ai-components");
+const aiModelIdx = args.indexOf("--ai-model");
+const aiModel = aiModelIdx !== -1 && args[aiModelIdx + 1] ? args[aiModelIdx + 1] : undefined;
+const doVerify = args.includes("--verify");
+const verifyThresholdIdx = args.indexOf("--verify-threshold");
+const verifyThreshold =
+  verifyThresholdIdx !== -1 && args[verifyThresholdIdx + 1]
+    ? Number(args[verifyThresholdIdx + 1]) || 0.15
+    : 0.15;
+
+const maxDepth = noCrawl ? 0 : parseIntArg(args, "--depth", 2);
+const maxPages = parseIntArg(args, "--max-pages", 25);
+const maxNodesPerPage = parseIntArg(args, "--max-nodes-per-page", 5000);
+const minInstances = parseIntArg(args, "--min-instances", 2);
+const minDepth = parseIntArg(args, "--min-depth", 2);
+
+let ai: false | { apiKey: string; baseUrl?: string | undefined; model?: string | undefined } =
+  false;
+if (doAiComponents) {
+  const apiKey = process.env.OPENAI_API_KEY;
+  if (!apiKey) {
+    console.error("Error: --ai-components requires OPENAI_API_KEY environment variable");
+    process.exit(1);
+  }
+  ai = { apiKey, baseUrl: process.env.OPENAI_BASE_URL, model: aiModel };
+}
+
+let outDir: string;
+const outIdx = args.indexOf("--out");
+const outArg = args[outIdx + 1];
+if (outIdx !== -1 && outArg) {
+  outDir = resolve(outArg);
+} else {
+  const hostname = new URL(url).hostname.replace(/^www\./, "");
+  outDir = resolve(homedir(), "jx-imports", hostname);
+}
+
+console.log(`Importing ${url} → ${outDir}`);
+if (maxDepth > 0) {
+  console.log(`  Crawl: depth=${maxDepth}, max-pages=${maxPages}, max-nodes=${maxNodesPerPage}\n`);
+} else {
+  console.log(`  Single page mode\n`);
+}
+
+// The interactive flow runs inside an async function rather than via top-level await so a test can
+// Dynamic-import this entry and await the same sequence (see create/index.ts for the rationale).
+async function main() {
+  try {
+    const result = await importSite(
+      {
+        url: url as string,
+        outDir,
+        maxDepth,
+        maxPages,
+        maxNodesPerPage,
+        styles: !args.includes("--no-styles"),
+        assets: !args.includes("--no-assets"),
+        scroll: !args.includes("--no-scroll"),
+        respectRobots: !args.includes("--no-robots"),
+        componentize: noComponents ? false : { minInstances, minDepth },
+        ai,
+        verify: doVerify ? { threshold: verifyThreshold } : false,
+      },
+      (e) => console.log(`  ${e.message}`),
+    );
+
+    console.log(`\n  Imported ${result.pages.length} page${result.pages.length === 1 ? "" : "s"}:`);
+    for (const page of result.pages) {
+      console.log(`    ${page.route} — "${page.title}" (${page.nodeCount} nodes)`);
+    }
+    if (result.verify) {
+      console.log(`  Average fidelity: ${result.verify.averageFidelity}%`);
+      console.log(`  Report: ${result.verify.reportDir}/report.json`);
+    }
+
+    console.log(`\nDone! Open in Studio:`);
+    console.log(
+      `  http://localhost:3000/packages/studio/index.html?project=${outDir}/project.json`,
+    );
+  } catch (error) {
+    console.error(`Error: ${error instanceof Error ? error.message : error}`);
+    process.exitCode = 1;
+  }
+}
+
+// oxlint-disable-next-line unicorn/prefer-top-level-await
+export const ready = main();

package/src/componentize.ts

@@ -0,0 +1,348 @@
+import type { JxElement } from "@jxsuite/schema/types";
+
+export interface ComponentizeOptions {
+  minInstances?: number;
+  minDepth?: number;
+}
+
+export interface ExtractedComponent {
+  $id: string;
+  tagName: string;
+  template: JxElement;
+  instanceCount: number;
+}
+
+export interface ComponentizeResult {
+  components: Map<string, ExtractedComponent>;
+  rewrittenPages: Map<string, JxElement>;
+}
+
+interface SubtreeLocation {
+  pageRoute: string;
+  path: number[];
+  node: JxElement;
+}
+
+function nodeDepth(node: JxElement | string): number {
+  if (typeof node === "string") {
+    return 0;
+  }
+  let max = 0;
+  if (Array.isArray(node.children)) {
+    for (const c of node.children) {
+      max = Math.max(max, nodeDepth(c as JxElement | string));
+    }
+  }
+  return 1 + max;
+}
+
+function structuralHash(node: JxElement | string): string {
+  if (typeof node === "string") {
+    return "#text";
+  }
+  const tag = node.tagName ?? "div";
+  const childHashes: string[] = [];
+  if (Array.isArray(node.children)) {
+    for (const child of node.children) {
+      childHashes.push(structuralHash(child as JxElement | string));
+    }
+  }
+  if (node.textContent) {
+    childHashes.push("#text");
+  }
+  return `<${tag}>${childHashes.join(",")}`;
+}
+
+function collectLeafValues(node: JxElement | string, prefix: string, out: Map<string, string>) {
+  if (typeof node === "string") {
+    out.set(`${prefix}.$text`, node);
+    return;
+  }
+  if (node.textContent) {
+    out.set(`${prefix}.textContent`, node.textContent as string);
+  }
+  if (node.attributes) {
+    for (const [k, v] of Object.entries(node.attributes)) {
+      if (typeof v === "string" || typeof v === "number" || typeof v === "boolean") {
+        out.set(`${prefix}.attr.${k}`, String(v));
+      }
+    }
+  }
+  if (Array.isArray(node.children)) {
+    for (let i = 0; i < node.children.length; i++) {
+      collectLeafValues(node.children[i] as JxElement | string, `${prefix}.${i}`, out);
+    }
+  }
+}
+
+function leafPathToStateName(leafPath: string): string {
+  const parts = leafPath.split(".").slice(1);
+  const meaningful: string[] = [];
+  for (const p of parts) {
+    if (p === "$text" || p === "textContent") {
+      meaningful.push("text");
+    } else if (p === "attr") {
+      continue;
+    } else if (/^\d+$/.test(p)) {
+      continue;
+    } else {
+      meaningful.push(p);
+    }
+  }
+  if (meaningful.length === 0) {
+    meaningful.push("value");
+  }
+  const raw = meaningful.join("_");
+  return raw.replaceAll(/[^a-zA-Z0-9_]/g, "");
+}
+
+function applyStateInterpolation(
+  node: JxElement | string,
+  prefix: string,
+  varyingPaths: Set<string>,
+  nameMap: Map<string, string>,
+): JxElement | string {
+  if (typeof node === "string") {
+    const key = `${prefix}.$text`;
+    if (varyingPaths.has(key)) {
+      return `\${state.${nameMap.get(key)}}`;
+    }
+    return node;
+  }
+
+  const result: JxElement = { ...node };
+
+  if (node.textContent) {
+    const key = `${prefix}.textContent`;
+    if (varyingPaths.has(key)) {
+      result.textContent = `\${state.${nameMap.get(key)}}`;
+    }
+  }
+
+  if (node.attributes) {
+    result.attributes = { ...node.attributes };
+    for (const [k] of Object.entries(node.attributes)) {
+      const key = `${prefix}.attr.${k}`;
+      if (varyingPaths.has(key)) {
+        result.attributes[k] = `\${state.${nameMap.get(key)}}`;
+      }
+    }
+  }
+
+  if (Array.isArray(node.children)) {
+    result.children = node.children.map((child, i) =>
+      applyStateInterpolation(child as JxElement | string, `${prefix}.${i}`, varyingPaths, nameMap),
+    ) as JxElement[];
+  }
+
+  return result;
+}
+
+function toPascalCase(str: string): string {
+  return str
+    .split(/[-_\s]+/)
+    .map((s) => s.charAt(0).toUpperCase() + s.slice(1).toLowerCase())
+    .join("");
+}
+
+function toKebabCase(base: string, index: number): string {
+  return `component-${base}-${index}`;
+}
+
+function setNodeAt(root: JxElement, path: number[], replacement: JxElement): JxElement {
+  if (path.length === 0) {
+    return replacement;
+  }
+
+  const result: JxElement = { ...root };
+  if (Array.isArray(root.children)) {
+    result.children = [...root.children] as JxElement[];
+    const head = path[0]!;
+    const rest = path.slice(1);
+    if (rest.length === 0) {
+      (result.children as JxElement[])[head] = replacement;
+    } else {
+      const child = result.children[head];
+      if (typeof child !== "string") {
+        (result.children as JxElement[])[head] = setNodeAt(child as JxElement, rest, replacement);
+      }
+    }
+  }
+  return result;
+}
+
+function collectSubtrees(
+  node: JxElement,
+  pageRoute: string,
+  path: number[],
+  minDepth: number,
+  out: SubtreeLocation[],
+) {
+  // Skip root node (path.length === 0) — don't componentize entire pages
+  if (path.length > 0 && nodeDepth(node) >= minDepth) {
+    out.push({ pageRoute, path: [...path], node });
+  }
+  if (Array.isArray(node.children)) {
+    for (let i = 0; i < node.children.length; i++) {
+      const child = node.children[i];
+      if (typeof child !== "string") {
+        collectSubtrees(child as JxElement, pageRoute, [...path, i], minDepth, out);
+      }
+    }
+  }
+}
+
+export function componentize(
+  pages: Map<string, JxElement>,
+  options: ComponentizeOptions = {},
+): ComponentizeResult {
+  const minInstances = options.minInstances ?? 2;
+  const minDepth = options.minDepth ?? 2;
+
+  const allSubtrees: SubtreeLocation[] = [];
+  for (const [route, doc] of pages) {
+    collectSubtrees(doc, route, [], minDepth, allSubtrees);
+  }
+
+  const byHash = new Map<string, SubtreeLocation[]>();
+  for (const loc of allSubtrees) {
+    const hash = structuralHash(loc.node);
+    if (!byHash.has(hash)) {
+      byHash.set(hash, []);
+    }
+    byHash.get(hash)!.push(loc);
+  }
+
+  const candidates = [...byHash.entries()]
+    .filter(([, locs]) => locs.length >= minInstances)
+    .toSorted((a, b) => {
+      const depthA = nodeDepth(a[1][0]!.node);
+      const depthB = nodeDepth(b[1][0]!.node);
+      if (depthB !== depthA) {
+        return depthB - depthA;
+      }
+      return b[1].length - a[1].length;
+    });
+
+  const components = new Map<string, ExtractedComponent>();
+  const claimed = new Set<string>();
+  const replacements: {
+    pageRoute: string;
+    path: number[];
+    tagName: string;
+    props: Record<string, string>;
+  }[] = [];
+
+  let compIndex = 0;
+  for (const [, locations] of candidates) {
+    const unclaimed = locations.filter((loc) => {
+      const locKey = `${loc.pageRoute}:${loc.path.join(".")}`;
+      for (const c of claimed) {
+        if (locKey.startsWith(c) || c.startsWith(locKey)) {
+          return false;
+        }
+      }
+      return true;
+    });
+
+    if (unclaimed.length < minInstances) {
+      continue;
+    }
+
+    const allLeaves: Map<string, string>[] = unclaimed.map((loc) => {
+      const leaves = new Map<string, string>();
+      collectLeafValues(loc.node, "root", leaves);
+      return leaves;
+    });
+
+    const allKeys = new Set<string>();
+    for (const m of allLeaves) {
+      for (const k of m.keys()) {
+        allKeys.add(k);
+      }
+    }
+
+    const varyingPaths = new Set<string>();
+    for (const key of allKeys) {
+      const values = allLeaves.map((m) => m.get(key) ?? "");
+      if (new Set(values).size > 1) {
+        varyingPaths.add(key);
+      }
+    }
+
+    const nameMap = new Map<string, string>();
+    const usedNames = new Set<string>();
+    for (const vp of varyingPaths) {
+      let name = leafPathToStateName(vp);
+      if (usedNames.has(name)) {
+        let suffix = 2;
+        while (usedNames.has(`${name}${suffix}`)) {
+          suffix += 1;
+        }
+        name = `${name}${suffix}`;
+      }
+      usedNames.add(name);
+      nameMap.set(vp, name);
+    }
+
+    const baseTag = unclaimed[0]!.node.tagName ?? "div";
+    const kebabName = toKebabCase(baseTag, compIndex);
+    const pascalName = toPascalCase(kebabName);
+    compIndex += 1;
+
+    const state: Record<string, string> = {};
+    for (const [leafPath, stateName] of nameMap) {
+      state[stateName] = allLeaves[0]!.get(leafPath) ?? "";
+    }
+
+    const template = applyStateInterpolation(
+      unclaimed[0]!.node,
+      "root",
+      varyingPaths,
+      nameMap,
+    ) as JxElement;
+
+    const component: ExtractedComponent = {
+      $id: pascalName,
+      tagName: kebabName,
+      template,
+      instanceCount: unclaimed.length,
+    };
+
+    const fileName = `${kebabName}.json`;
+    components.set(fileName, component);
+
+    for (const loc of unclaimed) {
+      const locKey = `${loc.pageRoute}:${loc.path.join(".")}`;
+      claimed.add(locKey);
+
+      const props: Record<string, string> = {};
+      const leaves = new Map<string, string>();
+      collectLeafValues(loc.node, "root", leaves);
+      for (const [leafPath, stateName] of nameMap) {
+        props[stateName] = leaves.get(leafPath) ?? "";
+      }
+
+      replacements.push({ pageRoute: loc.pageRoute, path: loc.path, tagName: kebabName, props });
+    }
+  }
+
+  const rewrittenPages = new Map<string, JxElement>();
+  for (const [route, doc] of pages) {
+    rewrittenPages.set(route, structuredClone(doc));
+  }
+
+  const sortedReplacements = replacements.toSorted((a, b) => b.path.length - a.path.length);
+
+  for (const rep of sortedReplacements) {
+    const page = rewrittenPages.get(rep.pageRoute)!;
+    const callSite: JxElement = { tagName: rep.tagName };
+    const hasNonDefaultProps = Object.keys(rep.props).length > 0;
+    if (hasNonDefaultProps) {
+      callSite.$props = rep.props;
+    }
+    rewrittenPages.set(rep.pageRoute, setNodeAt(page, rep.path, callSite));
+  }
+
+  return { components, rewrittenPages };
+}