@jxsuite/import 0.34.0

package/src/emit.ts

@@ -0,0 +1,231 @@
+import { join, dirname } from "node:path";
+import { mkdir } from "node:fs/promises";
+import type { JxElement } from "@jxsuite/schema/types";
+import { componentize } from "./componentize.ts";
+import type { ComponentizeOptions, ComponentizeResult } from "./componentize.ts";
+
+export interface EmitOptions {
+  outDir: string;
+  title: string;
+  document: JxElement;
+  sourceUrl: string;
+  /** $media breakpoints to seed into project.json (Phase 1). */
+  breakpoints?: Record<string, string>;
+  /** Phase 4: componentization options. Pass false to skip. */
+  componentizeOptions?: ComponentizeOptions | false;
+}
+
+export interface MultiEmitOptions {
+  outDir: string;
+  title: string;
+  sourceUrl: string;
+  /** Map of route path (e.g. "pages/index.json") → Jx document. */
+  pages: Map<string, JxElement>;
+  /** Optional layout to write to layouts/base.json. */
+  layout?: JxElement | undefined;
+  /** $media breakpoints to seed into project.json. */
+  breakpoints?: Record<string, string> | undefined;
+  /** Phase 4: componentization options. Pass false to skip. */
+  componentizeOptions?: ComponentizeOptions | false | undefined;
+  /** Pre-computed componentization result (from AI pass). Overrides componentizeOptions. */
+  precomputedComponents?: ComponentizeResult | undefined;
+  /** Font-face CSS rule texts to emit as public/assets/fonts.css (R2). */
+  fontFaceRules?: string[] | undefined;
+  /** URL rewrite map for fonts — maps original font URLs to local paths. */
+  fontRewriteMap?: Map<string, string> | undefined;
+  /** CSS custom property tokens to hoist into project.json.$style (R5). */
+  styleTokens?: Record<string, string> | undefined;
+}
+
+export async function emitProject({
+  outDir,
+  title,
+  document,
+  sourceUrl,
+  breakpoints,
+  componentizeOptions,
+}: EmitOptions): Promise<{ files: string[] }> {
+  return emitMultiPageProject({
+    outDir,
+    title,
+    sourceUrl,
+    pages: new Map([["pages/index.json", document]]),
+    breakpoints,
+    componentizeOptions,
+  });
+}
+
+export async function emitMultiPageProject({
+  outDir,
+  title,
+  sourceUrl,
+  pages,
+  layout,
+  breakpoints,
+  componentizeOptions,
+  precomputedComponents,
+  fontFaceRules,
+  fontRewriteMap,
+  styleTokens,
+}: MultiEmitOptions): Promise<{ files: string[] }> {
+  await mkdir(join(outDir, "pages"), { recursive: true });
+  await mkdir(join(outDir, "layouts"), { recursive: true });
+  await mkdir(join(outDir, "components"), { recursive: true });
+  await mkdir(join(outDir, "public"), { recursive: true });
+
+  const projectJson: Record<string, unknown> = {
+    name: title || "Imported Site",
+    title: title || "Imported Site",
+    description: `Imported from ${sourceUrl}`,
+    imports: {},
+    images: { optimize: false },
+  };
+
+  if (breakpoints && Object.keys(breakpoints).length > 0) {
+    projectJson.$media = breakpoints;
+  }
+
+  if (styleTokens && Object.keys(styleTokens).length > 0) {
+    projectJson.$style = styleTokens;
+  }
+
+  const files: string[] = [];
+
+  // Phase 4: componentize pages
+  let finalPages = pages;
+  const componentFiles = new Map<string, JxElement>();
+
+  const compResult =
+    precomputedComponents ??
+    (componentizeOptions !== false ? componentize(pages, componentizeOptions ?? {}) : null);
+
+  if (compResult && compResult.components.size > 0) {
+    finalPages = compResult.rewrittenPages;
+
+    for (const [fileName, comp] of compResult.components) {
+      const compDoc: JxElement = {
+        ...comp.template,
+        $id: comp.$id,
+        tagName: comp.tagName,
+      };
+
+      const state: Record<string, string> = {};
+      extractStateDefaults(comp.template, state);
+      if (Object.keys(state).length > 0) {
+        compDoc.state = state;
+      }
+
+      componentFiles.set(fileName, compDoc);
+    }
+  }
+
+  // R2: Emit @font-face CSS with URLs rewritten to local paths
+  if (fontFaceRules && fontFaceRules.length > 0) {
+    let fontCss = fontFaceRules.join("\n\n");
+    if (fontRewriteMap) {
+      for (const [originalUrl, localPath] of fontRewriteMap) {
+        // Rewrite absolute URLs in font-face rules to relative local paths
+        const relativePath = localPath.startsWith("public/")
+          ? localPath.slice("public/".length)
+          : localPath;
+        fontCss = fontCss.replaceAll(originalUrl, `/${relativePath}`);
+      }
+    }
+    const fontsDir = join(outDir, "public", "assets");
+    await mkdir(fontsDir, { recursive: true });
+    const fontsCssPath = join(fontsDir, "fonts.css");
+    await Bun.write(fontsCssPath, fontCss);
+    files.push(fontsCssPath);
+
+    // Add fonts.css to project head
+    if (!projectJson.$head) {
+      projectJson.$head = [];
+    }
+    (projectJson.$head as unknown[]).push({
+      tagName: "link",
+      attributes: { rel: "stylesheet", href: "/assets/fonts.css" },
+    });
+  }
+
+  const projectPath = join(outDir, "project.json");
+  await Bun.write(projectPath, `${JSON.stringify(projectJson, null, 2)}\n`);
+  files.push(projectPath);
+
+  // Write components
+  for (const [fileName, compDoc] of componentFiles) {
+    const compPath = join(outDir, "components", fileName);
+    await Bun.write(compPath, `${JSON.stringify(compDoc, null, 2)}\n`);
+    files.push(compPath);
+  }
+
+  // Build $elements refs for component registration
+  const elementRefs: JxElement[] =
+    componentFiles.size > 0
+      ? [...componentFiles.keys()].map(
+          (f) => ({ $ref: `../components/${f}` }) as unknown as JxElement,
+        )
+      : [];
+
+  // Write each page
+  for (const [route, doc] of finalPages) {
+    const pageDoc = { ...doc };
+    if (elementRefs.length > 0) {
+      pageDoc.$elements = [
+        ...elementRefs,
+        ...((pageDoc.$elements as JxElement[]) ?? []),
+      ] as JxElement[];
+    }
+    const pagePath = join(outDir, route);
+    await mkdir(dirname(pagePath), { recursive: true });
+    await Bun.write(pagePath, `${JSON.stringify(pageDoc, null, 2)}\n`);
+    files.push(pagePath);
+  }
+
+  // Write layout
+  const layoutPath = join(outDir, "layouts", "base.json");
+  const layoutDoc: JxElement = layout ?? {
+    tagName: "div",
+    children: [{ tagName: "slot" }],
+  };
+  if (elementRefs.length > 0 && layout) {
+    (layoutDoc as Record<string, unknown>).$elements = [
+      ...elementRefs,
+      ...(((layoutDoc as Record<string, unknown>).$elements as JxElement[]) ?? []),
+    ];
+  }
+  await Bun.write(layoutPath, `${JSON.stringify(layoutDoc, null, 2)}\n`);
+  files.push(layoutPath);
+
+  return { files };
+}
+
+function extractStateDefaults(node: JxElement | string, out: Record<string, string>) {
+  if (typeof node === "string") {
+    const key = node.match(/^\$\{state\.([^}]+)\}$/)?.[1];
+    if (key && !(key in out)) {
+      out[key] = "";
+    }
+    return;
+  }
+  if (typeof node.textContent === "string") {
+    const key = node.textContent.match(/^\$\{state\.([^}]+)\}$/)?.[1];
+    if (key && !(key in out)) {
+      out[key] = "";
+    }
+  }
+  if (node.attributes) {
+    for (const v of Object.values(node.attributes)) {
+      if (typeof v === "string") {
+        const key = v.match(/^\$\{state\.([^}]+)\}$/)?.[1];
+        if (key && !(key in out)) {
+          out[key] = "";
+        }
+      }
+    }
+  }
+  if (Array.isArray(node.children)) {
+    for (const child of node.children) {
+      extractStateDefaults(child as JxElement | string, out);
+    }
+  }
+}

package/src/index.ts

@@ -0,0 +1,53 @@
+export { capturePage, launchBrowser, closeBrowser } from "./capture.ts";
+export { importSite } from "./run.ts";
+export { convertToJx } from "./to-jx.ts";
+export { emitProject, emitMultiPageProject } from "./emit.ts";
+export { captureStyles, captureStylesAtWidth, STYLE_ALLOWLIST } from "./style-capture.ts";
+export { diffStyles, diffAllStyles, computeMediaDelta, kebabToCamel } from "./style-diff.ts";
+export { analyzeMediaQueries, extractMedia } from "./media-extract.ts";
+export { applyStylesToTree } from "./apply-styles.ts";
+export { collectAssets } from "./asset-collect.ts";
+export { downloadAssets } from "./asset-download.ts";
+export { rewriteAssetUrls } from "./asset-rewrite.ts";
+export { applyTokens } from "./css-tokens.ts";
+export { crawlSite, normalizeUrl, routeToFilePath, fetchRobotsTxt } from "./crawl.ts";
+export { detectLayout, hashSubtree, treesEqual } from "./layout-detect.ts";
+export { componentize } from "./componentize.ts";
+export { aiComponentize } from "./ai-componentize.ts";
+export { diffScreenshots } from "./screenshot-diff.ts";
+export {
+  verifyProject,
+  captureReferenceScreenshot,
+  serveDirectory,
+  routeToUrlPath,
+} from "./verify.ts";
+export type { CaptureResult, LaunchOptions } from "./capture.ts";
+export type {
+  ImportSiteOptions,
+  ImportProgressEvent,
+  ImportPhase,
+  ImportSiteResult,
+} from "./run.ts";
+export type { ToJxResult } from "./to-jx.ts";
+export type { EmitOptions, MultiEmitOptions } from "./emit.ts";
+export type { CapturedStyle, StyleCaptureResult } from "./style-capture.ts";
+export type { DiffedStyle } from "./style-diff.ts";
+export type { Breakpoint, MediaExtractionResult } from "./media-extract.ts";
+export type {
+  DiscoveredAsset,
+  AssetCollectionResult,
+  CapturedStylesheet,
+} from "./asset-collect.ts";
+export type { CaptureOptions } from "./capture.ts";
+export type { DownloadResult } from "./asset-download.ts";
+export type { TokenExtractionResult } from "./css-tokens.ts";
+export type { CrawlOptions, CrawledPage, CrawlResult } from "./crawl.ts";
+export type { LayoutResult } from "./layout-detect.ts";
+export type {
+  ComponentizeOptions,
+  ExtractedComponent,
+  ComponentizeResult,
+} from "./componentize.ts";
+export type { AiComponentizeOptions } from "./ai-componentize.ts";
+export type { DiffResult, DiffOptions } from "./screenshot-diff.ts";
+export type { VerifyOptions, PageRef, PageVerifyResult, VerifyResult } from "./verify.ts";

package/src/layout-detect.ts

@@ -0,0 +1,151 @@
+import type { JxElement } from "@jxsuite/schema/types";
+
+export interface LayoutResult {
+  /** The shared layout with a $slot placeholder where page-specific content goes. */
+  layout: JxElement;
+  /** Per-page trees with the shared header/footer removed. */
+  strippedPages: Map<string, JxElement>;
+}
+
+/**
+ * Hash a subtree by structure (tagName + child structure), ignoring text content and attribute
+ * values. Used to detect structurally identical subtrees across pages.
+ */
+export function hashSubtree(node: JxElement | string): string {
+  if (typeof node === "string") {
+    return `#text`;
+  }
+
+  const tag = node.tagName ?? "div";
+  const childHashes: string[] = [];
+
+  if (Array.isArray(node.children)) {
+    for (const child of node.children) {
+      childHashes.push(hashSubtree(child as JxElement | string));
+    }
+  }
+
+  if (node.textContent) {
+    childHashes.push("#text");
+  }
+
+  return `<${tag}>${childHashes.join(",")}`;
+}
+
+/**
+ * Compare two Jx trees for deep structural equality (tagName + children structure). Text content
+ * and attribute values are compared exactly — this is for detecting truly identical
+ * headers/footers, not fuzzy matches.
+ */
+export function treesEqual(a: JxElement | string, b: JxElement | string): boolean {
+  if (typeof a === "string" && typeof b === "string") {
+    return a === b;
+  }
+  if (typeof a === "string" || typeof b === "string") {
+    return false;
+  }
+
+  if ((a.tagName ?? "div") !== (b.tagName ?? "div")) {
+    return false;
+  }
+  if ((a.textContent ?? "") !== (b.textContent ?? "")) {
+    return false;
+  }
+
+  const aChildren = Array.isArray(a.children) ? a.children : [];
+  const bChildren = Array.isArray(b.children) ? b.children : [];
+
+  if (aChildren.length !== bChildren.length) {
+    return false;
+  }
+
+  for (let i = 0; i < aChildren.length; i++) {
+    if (!treesEqual(aChildren[i] as JxElement | string, bChildren[i] as JxElement | string)) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+/**
+ * Detect shared header/footer subtrees across multiple pages and extract them into a layout.
+ *
+ * Strategy: look at the top-level children of each page's root div. Children that appear
+ * identically (by deep structural+content equality) at the same position from the start (header) or
+ * end (footer) across ALL pages are candidates for the shared layout.
+ *
+ * Returns null if fewer than 2 pages or no shared subtrees found.
+ */
+export function detectLayout(pages: Map<string, JxElement>): LayoutResult | null {
+  if (pages.size < 2) {
+    return null;
+  }
+
+  const entries = [...pages.entries()];
+  const allRoots = entries.map(([, doc]) =>
+    Array.isArray(doc.children) ? (doc.children as (JxElement | string)[]) : [],
+  );
+  const firstRoot = allRoots[0]!;
+
+  // Find shared prefix (header elements)
+  let sharedPrefixLen = 0;
+  const minLen = Math.min(...allRoots.map((c) => c.length));
+
+  for (let i = 0; i < minLen; i++) {
+    const reference = firstRoot[i]!;
+    const allMatch = allRoots.every((children) => treesEqual(children[i]!, reference));
+    if (allMatch) {
+      sharedPrefixLen = i + 1;
+    } else {
+      break;
+    }
+  }
+
+  // Find shared suffix (footer elements), not overlapping with prefix
+  let sharedSuffixLen = 0;
+  for (let i = 0; i < minLen - sharedPrefixLen; i++) {
+    const reference = firstRoot[firstRoot.length - 1 - i]!;
+    const allMatch = allRoots.every((children) =>
+      treesEqual(children[children.length - 1 - i]!, reference),
+    );
+    if (allMatch) {
+      sharedSuffixLen = i + 1;
+    } else {
+      break;
+    }
+  }
+
+  if (sharedPrefixLen === 0 && sharedSuffixLen === 0) {
+    return null;
+  }
+
+  // Build layout: shared prefix + $slot + shared suffix
+  const headerChildren = firstRoot.slice(0, sharedPrefixLen);
+  const footerChildren =
+    sharedSuffixLen > 0 ? firstRoot.slice(firstRoot.length - sharedSuffixLen) : [];
+
+  const slot: JxElement = { tagName: "slot" };
+
+  const layout: JxElement = {
+    tagName: "div",
+    children: [...(headerChildren as JxElement[]), slot, ...(footerChildren as JxElement[])],
+  };
+
+  // Strip shared elements from each page
+  const strippedPages = new Map<string, JxElement>();
+  for (const [route, doc] of entries) {
+    const children = Array.isArray(doc.children) ? (doc.children as (JxElement | string)[]) : [];
+
+    const endIdx = sharedSuffixLen > 0 ? children.length - sharedSuffixLen : children.length;
+    const uniqueChildren = children.slice(sharedPrefixLen, endIdx);
+
+    strippedPages.set(route, {
+      ...doc,
+      children: uniqueChildren as JxElement[],
+      $layout: "layouts/base.json",
+    });
+  }
+
+  return { layout, strippedPages };
+}

package/src/media-extract.ts

@@ -0,0 +1,121 @@
+/**
+ * Extract @media breakpoints from captured CSS and orchestrate style re-capture at each breakpoint
+ * width to produce per-node $media style deltas.
+ */
+
+import type { Page } from "puppeteer-core";
+import type { CapturedStyle } from "./style-capture.ts";
+import { captureStylesAtWidth } from "./style-capture.ts";
+import { computeMediaDelta } from "./style-diff.ts";
+import type { DiffedStyle } from "./style-diff.ts";
+
+export interface Breakpoint {
+  /** Jx breakpoint name, e.g. "--md" */
+  name: string;
+  /** CSS media query string, e.g. "(max-width: 768px)" */
+  query: string;
+  /** The width in px to test this breakpoint at */
+  testWidth: number;
+}
+
+export interface MediaExtractionResult {
+  /** Breakpoints to add to project.json.$media */
+  breakpoints: Record<string, string>;
+  /** Per-breakpoint style deltas keyed by breakpoint name */
+  deltas: Record<string, DiffedStyle[]>;
+}
+
+/**
+ * Parse a media query string and extract a pixel width if it's a simple min-width or max-width
+ * query. Returns null for complex/non-size queries.
+ */
+function parseWidthQuery(query: string): { type: "min" | "max"; width: number } | null {
+  const minMatch = query.match(/^\s*\(\s*min-width\s*:\s*([\d.]+)px\s*\)\s*$/);
+  if (minMatch) {
+    return { type: "min", width: Number(minMatch[1]) };
+  }
+  const maxMatch = query.match(/^\s*\(\s*max-width\s*:\s*([\d.]+)px\s*\)\s*$/);
+  if (maxMatch) {
+    return { type: "max", width: Number(maxMatch[1]) };
+  }
+  return null;
+}
+
+/** Generate a Jx breakpoint name from a width, e.g. 768 → "--768" */
+function breakpointName(width: number): string {
+  return `--${width}`;
+}
+
+/**
+ * Analyze discovered @media queries and produce a set of testable breakpoints. Only picks up simple
+ * min-width / max-width queries — complex queries (orientation, hover, prefers-*) are logged and
+ * skipped.
+ */
+export function analyzeMediaQueries(queries: string[]): Breakpoint[] {
+  const seen = new Set<number>();
+  const breakpoints: Breakpoint[] = [];
+
+  for (const query of queries) {
+    const parsed = parseWidthQuery(query);
+    if (!parsed) {
+      continue;
+    }
+    if (seen.has(parsed.width)) {
+      continue;
+    }
+    seen.add(parsed.width);
+
+    // For max-width: test AT that width (it's the upper bound)
+    // For min-width: test AT that width (it's the lower bound — styles kick in here)
+    breakpoints.push({
+      name: breakpointName(parsed.width),
+      query,
+      testWidth: parsed.width,
+    });
+  }
+
+  // Sort by width ascending
+  breakpoints.sort((a, b) => a.testWidth - b.testWidth);
+  return breakpoints;
+}
+
+/**
+ * Extract $media style deltas by re-capturing at each breakpoint width.
+ *
+ * @param page - Puppeteer page (must still have the target page loaded)
+ * @param baseElements - Style capture from the base viewport (1440px)
+ * @param uaDefaults - UA-default baselines per tagName
+ * @param mediaQueries - @media queries discovered in the page's stylesheets
+ * @param originalWidth - The base viewport width to restore after extraction
+ */
+export async function extractMedia(
+  page: Page,
+  baseElements: CapturedStyle[],
+  uaDefaults: Record<string, Record<string, string>>,
+  mediaQueries: string[],
+  originalWidth = 1440,
+): Promise<MediaExtractionResult> {
+  const breakpoints = analyzeMediaQueries(mediaQueries);
+
+  if (breakpoints.length === 0) {
+    return { breakpoints: {}, deltas: {} };
+  }
+
+  const projectBreakpoints: Record<string, string> = {};
+  const allDeltas: Record<string, DiffedStyle[]> = {};
+
+  for (const bp of breakpoints) {
+    const bpElements = await captureStylesAtWidth(page, bp.testWidth);
+    const deltas = computeMediaDelta(baseElements, bpElements, uaDefaults);
+
+    if (deltas.length > 0) {
+      projectBreakpoints[bp.name] = bp.query;
+      allDeltas[bp.name] = deltas;
+    }
+  }
+
+  // Restore original viewport
+  await page.setViewport({ width: originalWidth, height: 900 });
+
+  return { breakpoints: projectBreakpoints, deltas: allDeltas };
+}