@jxsuite/import 0.39.8 → 0.40.1

package/src/emit.ts

@@ -1,22 +1,55 @@
-import { join, dirname } from "node:path";
-import { mkdir } from "node:fs/promises";
 import type { JxElement } from "@jxsuite/schema/types";
 import { componentize } from "./componentize.ts";
+import { stripClasses } from "./strip-classes.ts";
 import type { ComponentizeOptions, ComponentizeResult } from "./componentize.ts";
+import type { ImportIo } from "./io.ts";
+
+/**
+ * The `$media` key the base width lives under — a number of CSS pixels, not a query.
+ *
+ * Studio's canvas reads it to size the default artboard (`utils/canvas-media.ts`), and an import
+ * never wrote one, so every imported project's base canvas fell back to 320px while its narrowest
+ * real breakpoint was three times that. Mirrors `settings/contexts-section.ts`'s `BASE_KEY`.
+ */
+const BASE_MEDIA_KEY = "--";
+
+/** The width a `--<width>` breakpoint key names, for sorting. Non-numeric keys sort first. */
+function mediaKeyWidth(key: string): number {
+  const width = Number(key.replace(/^--/, ""));
+  return Number.isFinite(width) ? width : -1;
+}
+
+export interface EmitResult {
+  /** Project-relative, forward-slashed paths of every file written, in the order they were written. */
+  files: string[];
+  /** How many source-site class names were removed on the way out. */
+  classesStripped: number;
+  /**
+   * The exact `project.json` text that was written.
+   *
+   * Returned rather than left to be read back, because a caller with no filesystem cannot read it
+   * back at all: Jx Cloud commits the emitted file set to git and needs the configuration to name
+   * the project, and re-deriving it here would be a second writer on the same bytes.
+   */
+  projectJson: string;
+}
 
 export interface EmitOptions {
-  outDir: string;
+  io: ImportIo;
   title: string;
   document: JxElement;
   sourceUrl: string;
   /** $media breakpoints to seed into project.json (Phase 1). */
   breakpoints?: Record<string, string>;
+  /** The capture viewport width, written as the `$media` base entry. */
+  baseWidth?: number;
   /** Phase 4: componentization options. Pass false to skip. */
   componentizeOptions?: ComponentizeOptions | false;
 }
 
 export interface MultiEmitOptions {
-  outDir: string;
+  /** Where the emitted files go. See `io.ts` — a disk is one implementation, not the requirement. */
+  io: ImportIo;
   title: string;
   sourceUrl: string;
   /** Map of route path (e.g. "pages/index.json") → Jx document. */
@@ -25,6 +58,8 @@ export interface MultiEmitOptions {
   layout?: JxElement | undefined;
   /** $media breakpoints to seed into project.json. */
   breakpoints?: Record<string, string> | undefined;
+  /** The capture viewport width, written as the `$media` base entry. */
+  baseWidth?: number | undefined;
   /** Phase 4: componentization options. Pass false to skip. */
   componentizeOptions?: ComponentizeOptions | false | undefined;
   /** Pre-computed componentization result (from AI pass). Overrides componentizeOptions. */
@@ -38,55 +73,77 @@ export interface MultiEmitOptions {
 }
 
 export async function emitProject({
-  outDir,
+  io,
   title,
   document,
   sourceUrl,
   breakpoints,
+  baseWidth,
   componentizeOptions,
-}: EmitOptions): Promise<{ files: string[] }> {
+}: EmitOptions): Promise<EmitResult> {
   return emitMultiPageProject({
-    outDir,
+    io,
     title,
     sourceUrl,
     pages: new Map([["pages/index.json", document]]),
     breakpoints,
+    baseWidth,
     componentizeOptions,
   });
 }
 
 export async function emitMultiPageProject({
-  outDir,
+  io,
   title,
-  sourceUrl,
   pages,
   layout,
   breakpoints,
+  baseWidth,
   componentizeOptions,
   precomputedComponents,
   fontFaceRules,
   fontRewriteMap,
   styleTokens,
-}: MultiEmitOptions): Promise<{ files: string[] }> {
-  await mkdir(join(outDir, "pages"), { recursive: true });
-  await mkdir(join(outDir, "layouts"), { recursive: true });
-  await mkdir(join(outDir, "components"), { recursive: true });
-  await mkdir(join(outDir, "public"), { recursive: true });
+}: MultiEmitOptions): Promise<EmitResult> {
+  /* Seeded even when nothing lands in them, so a host that opened the project at the seed event
+     sees the shape it expects rather than a directory that grows one folder at a time. A sink with
+     no directories declines by not implementing it. */
+  for (const dir of ["pages", "layouts", "components", "public"]) {
+    await io.mkdir?.(dir);
+  }
 
+  /*
+   * Only keys the project schema declares (#228). `title` and `description` are page-level keys and
+   * `$style` is not a key at all, and all three made every imported project fail `jx validate` —
+   * and, because Studio's Contexts editor validates the WHOLE configuration before saving, made an
+   * imported project's base width uneditable, reporting `(root): must NOT have unevaluated
+   * properties` three times while naming nothing.
+   *
+   * The source URL has no schema-legal home and does not get one. `$head`'s description meta is the
+   * nearest candidate and the wrong one: it is the site's own user-facing description, and
+   * "Imported from …" is provenance, not a description of the site. The importer reports it instead.
+   */
   const projectJson: Record<string, unknown> = {
     name: title || "Imported Site",
-    title: title || "Imported Site",
-    description: `Imported from ${sourceUrl}`,
     imports: {},
     images: { optimize: false },
   };
 
+  if (baseWidth !== undefined) {
+    projectJson.$media = { [BASE_MEDIA_KEY]: `${Math.round(baseWidth)}px` };
+  }
   if (breakpoints && Object.keys(breakpoints).length > 0) {
-    projectJson.$media = breakpoints;
+    /* Ascending, and the base first. `$media` is a map, so its order IS the order Studio's Contexts
+       list and the pane's size switcher offer these in; a crawl merged them in page-visit order,
+       which is not an order at all. */
+    const sorted = Object.entries(breakpoints).toSorted(
+      ([a], [b]) => mediaKeyWidth(a) - mediaKeyWidth(b),
+    );
+    projectJson.$media = { ...(projectJson.$media as object), ...Object.fromEntries(sorted) };
   }
 
   if (styleTokens && Object.keys(styleTokens).length > 0) {
-    projectJson.$style = styleTokens;
+    projectJson.style = styleTokens;
   }
 
   const files: string[] = [];
@@ -123,18 +180,45 @@ export async function emitMultiPageProject({
   if (fontFaceRules && fontFaceRules.length > 0) {
     let fontCss = fontFaceRules.join("\n\n");
     if (fontRewriteMap) {
+      // A @font-face rule carries the url() form its AUTHOR wrote — usually root-relative
+      // ("/wp-content/.../lato.woff2"), sometimes protocol-relative. The rewrite map is keyed by the
+      // Absolute URL the downloader resolved, so matching on that alone rewrote nothing: every
+      // Downloaded font stayed unreferenced and the page fell back to system fonts.
+      //
+      // Replaced in ONE pass, longest form first. Replacing form by form re-scans text this loop
+      // Already rewrote, and a bare pathname ("/a.woff2") then matches inside the local path it
+      // Just produced ("/assets/fonts/a.woff2" to "/assets/fonts/assets/fonts/a.woff2").
+      const byForm = new Map<string, string>();
       for (const [originalUrl, localPath] of fontRewriteMap) {
-        // Rewrite absolute URLs in font-face rules to relative local paths
-        const relativePath = localPath.startsWith("public/")
+        const local = localPath.startsWith("public/")
           ? localPath.slice("public/".length)
           : localPath;
-        fontCss = fontCss.replaceAll(originalUrl, `/${relativePath}`);
+        const href = local.startsWith("/") ? local : `/${local}`;
+        const forms = new Set([originalUrl]);
+        try {
+          const u = new URL(originalUrl);
+          forms.add(`//${u.host}${u.pathname}${u.search}`);
+          forms.add(`${u.pathname}${u.search}`);
+        } catch {
+          // Already a relative reference — the literal form is all there is.
+        }
+        for (const form of forms) {
+          if (form) {
+            byForm.set(form, href);
+          }
+        }
+      }
+      const forms = [...byForm.keys()].toSorted((a, b) => b.length - a.length);
+      if (forms.length > 0) {
+        const pattern = new RegExp(
+          forms.map((f) => f.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`)).join("|"),
+          "g",
+        );
+        fontCss = fontCss.replaceAll(pattern, (m) => byForm.get(m) ?? m);
       }
     }
-    const fontsDir = join(outDir, "public", "assets");
-    await mkdir(fontsDir, { recursive: true });
-    const fontsCssPath = join(fontsDir, "fonts.css");
-    await Bun.write(fontsCssPath, fontCss);
+    const fontsCssPath = "public/assets/fonts.css";
+    await io.write(fontsCssPath, fontCss);
     files.push(fontsCssPath);
 
     // Add fonts.css to project head
@@ -147,14 +231,30 @@ export async function emitMultiPageProject({
     });
   }
 
-  const projectPath = join(outDir, "project.json");
-  await Bun.write(projectPath, `${JSON.stringify(projectJson, null, 2)}\n`);
-  files.push(projectPath);
+  const projectText = `${JSON.stringify(projectJson, null, 2)}\n`;
+  await io.write("project.json", projectText);
+  files.push("project.json");
+
+  /*
+   * Classes go here, at the last possible moment, and that placement is the whole design.
+   *
+   * `componentize` groups by structure and `ai-componentize` reads the template to choose a name —
+   * `product-card` over `component-div-0` — and the class names are the strongest signal either has.
+   * Stripping earlier would cost that; stripping later is impossible. So both passes see the source
+   * site's classes, and nothing downstream of this line does.
+   */
+  let classesStripped = 0;
+  for (const compDoc of componentFiles.values()) {
+    classesStripped += stripClasses(compDoc);
+  }
+  for (const doc of finalPages.values()) {
+    classesStripped += stripClasses(doc);
+  }
 
   // Write components
   for (const [fileName, compDoc] of componentFiles) {
-    const compPath = join(outDir, "components", fileName);
-    await Bun.write(compPath, `${JSON.stringify(compDoc, null, 2)}\n`);
+    const compPath = `components/${fileName}`;
+    await io.write(compPath, `${JSON.stringify(compDoc, null, 2)}\n`);
     files.push(compPath);
   }
 
@@ -175,14 +275,12 @@ export async function emitMultiPageProject({
         ...((pageDoc.$elements as JxElement[]) ?? []),
       ] as JxElement[];
     }
-    const pagePath = join(outDir, route);
-    await mkdir(dirname(pagePath), { recursive: true });
-    await Bun.write(pagePath, `${JSON.stringify(pageDoc, null, 2)}\n`);
-    files.push(pagePath);
+    await io.write(route, `${JSON.stringify(pageDoc, null, 2)}\n`);
+    files.push(route);
   }
 
   // Write layout
-  const layoutPath = join(outDir, "layouts", "base.json");
+  const layoutPath = "layouts/base.json";
   const layoutDoc: JxElement = layout ?? {
     tagName: "div",
     children: [{ tagName: "slot" }],
@@ -193,10 +291,11 @@ export async function emitMultiPageProject({
       ...(((layoutDoc as Record<string, unknown>).$elements as JxElement[]) ?? []),
     ];
   }
-  await Bun.write(layoutPath, `${JSON.stringify(layoutDoc, null, 2)}\n`);
+  classesStripped += stripClasses(layoutDoc);
+  await io.write(layoutPath, `${JSON.stringify(layoutDoc, null, 2)}\n`);
   files.push(layoutPath);
 
-  return { files };
+  return { classesStripped, files, projectJson: projectText };
 }
 
 function extractStateDefaults(node: JxElement | string, out: Record<string, string>) {

package/src/index.ts

@@ -1,10 +1,25 @@
-export { capturePage, launchBrowser, closeBrowser } from "./capture.ts";
+export { capturePage } from "./capture.ts";
+export { launchBrowser, closeBrowser } from "./browser-local.ts";
+export { createLocalIo } from "./io.ts";
+export { runImportPipeline } from "./pipeline.ts";
 export { importSite } from "./run.ts";
 export { convertToJx } from "./to-jx.ts";
 export { emitProject, emitMultiPageProject } from "./emit.ts";
 export { captureStyles, captureStylesAtWidth, STYLE_ALLOWLIST } from "./style-capture.ts";
 export { diffStyles, diffAllStyles, computeMediaDelta, kebabToCamel } from "./style-diff.ts";
-export { analyzeMediaQueries, extractMedia } from "./media-extract.ts";
+export { extractMedia } from "./media-extract.ts";
+export {
+  DEFAULT_BREAKPOINT_POLICY,
+  analyzeMediaQueries,
+  skippedWidthQueries,
+  MAX_BREAKPOINTS,
+  MAX_BREAKPOINT_WIDTH,
+  MIN_BREAKPOINT_WIDTH,
+  breakpointName,
+  parseWidthQuery,
+  planBreakpoints,
+} from "./breakpoint-plan.ts";
+export { stripClasses } from "./strip-classes.ts";
 export { applyStylesToTree } from "./apply-styles.ts";
 export { collectAssets } from "./asset-collect.ts";
 export { downloadAssets } from "./asset-download.ts";
@@ -21,18 +36,27 @@ export {
   serveDirectory,
   routeToUrlPath,
 } from "./verify.ts";
-export type { CaptureResult, LaunchOptions } from "./capture.ts";
+export type { CaptureResult, ImportBrowser, ImportPage } from "./capture.ts";
+export type { LaunchOptions } from "./browser-local.ts";
+export type { ImportIo } from "./io.ts";
 export type {
-  ImportSiteOptions,
+  ImportPipelineOptions,
+  ImportPipelineResult,
   ImportProgressEvent,
   ImportPhase,
-  ImportSiteResult,
-} from "./run.ts";
+} from "./pipeline.ts";
+export type { ImportSiteOptions, ImportSiteResult } from "./run.ts";
 export type { ToJxResult } from "./to-jx.ts";
-export type { EmitOptions, MultiEmitOptions } from "./emit.ts";
+export type { EmitOptions, EmitResult, MultiEmitOptions } from "./emit.ts";
 export type { CapturedStyle, StyleCaptureResult } from "./style-capture.ts";
 export type { DiffedStyle } from "./style-diff.ts";
-export type { Breakpoint, MediaExtractionResult } from "./media-extract.ts";
+export type { ExtractMediaOptions, MediaExtractionResult } from "./media-extract.ts";
+export type {
+  Breakpoint,
+  BreakpointPlanResult,
+  BreakpointPolicy,
+  BreakpointRounding,
+} from "./breakpoint-plan.ts";
 export type {
   DiscoveredAsset,
   AssetCollectionResult,

package/src/io.ts

@@ -0,0 +1,52 @@
+/**
+ * The sink every file an import produces goes through.
+ *
+ * The pipeline used to call `Bun.write(join(outDir, …))` at six sites, which made "run an import"
+ * and "have a writable filesystem" the same requirement. Jx Cloud is a Cloudflare Worker: it has
+ * neither `Bun` nor a disk, and commits the emitted project straight to git. So the destination is
+ * a parameter now — `createLocalIo` is the disk implementation and the only one that imports
+ * `node:fs`, and a Worker supplies its own without the pipeline knowing the difference.
+ */
+
+import { mkdir } from "node:fs/promises";
+import { dirname, join } from "node:path";
+
+export interface ImportIo {
+  /**
+   * Write one file.
+   *
+   * `relPath` is PROJECT-RELATIVE and forward-slashed — `"pages/blog/post.json"`, never an absolute
+   * path and never a backslash. A sink that has no filesystem uses it as a key verbatim; a sink
+   * that does joins it onto its own root.
+   */
+  write: (relPath: string, data: string | Uint8Array) => Promise<void>;
+  /**
+   * Create an empty directory, if directories are a thing this sink has.
+   *
+   * Optional because they usually are not: a git tree, an object store and a zip all record only
+   * the files, and an empty directory is unrepresentable in every one of them. The local import
+   * still seeds `pages/`, `layouts/`, `components/` and `public/` up front so a host can open the
+   * project the moment the run starts, which is what this exists for.
+   */
+  mkdir?: (relPath: string) => Promise<void>;
+}
+
+/** Split a project-relative path into segments, so a sink's own separator is the one used. */
+function toSegments(relPath: string): string[] {
+  return relPath.split("/").filter((segment) => segment !== "");
+}
+
+/** The disk sink: `outDir` plus the relative path, with parent directories created on demand. */
+export function createLocalIo(outDir: string): ImportIo {
+  const resolve = (relPath: string): string => join(outDir, ...toSegments(relPath));
+  return {
+    async write(relPath, data) {
+      const path = resolve(relPath);
+      await mkdir(dirname(path), { recursive: true });
+      await Bun.write(path, data);
+    },
+    async mkdir(relPath) {
+      await mkdir(resolve(relPath), { recursive: true });
+    },
+  };
+}

package/src/media-extract.ts

@@ -1,110 +1,86 @@
 /**
- * Extract @media breakpoints from captured CSS and orchestrate style re-capture at each breakpoint
- * width to produce per-node $media style deltas.
+ * Extract `@media` breakpoints from captured CSS and orchestrate style re-capture at each kept
+ * breakpoint width to produce per-node `$media` style deltas.
+ *
+ * The set of breakpoints a project keeps is decided in `breakpoint-plan.ts`, BEFORE anything is
+ * re-captured — each kept breakpoint costs a viewport change and a full style walk, so a limited
+ * import does less work rather than merely emitting less.
+ *
+ * A folded-away width is not sampled at all, and does not need to be. The computed style at a kept
+ * width is already the site's own cascade resolved AT that width, folded rules included; merging a
+ * separately-sampled delta on top of it could only disagree with the browser. The fold map exists
+ * so the import can SAY where a width went, not so its styles can be recovered.
  */
 
-import type { Page } from "puppeteer-core";
+import type { ImportPage } from "./capture.ts";
 import type { CapturedStyle } from "./style-capture.ts";
 import { captureStylesAtWidth } from "./style-capture.ts";
 import { computeMediaDelta } from "./style-diff.ts";
+import { analyzeMediaQueries, planBreakpoints } from "./breakpoint-plan.ts";
+import type { Breakpoint, BreakpointPolicy } from "./breakpoint-plan.ts";
 import type { DiffedStyle } from "./style-diff.ts";
 
-export interface Breakpoint {
-  /** Jx breakpoint name, e.g. "--md" */
-  name: string;
-  /** CSS media query string, e.g. "(max-width: 768px)" */
-  query: string;
-  /** The width in px to test this breakpoint at */
-  testWidth: number;
-}
+/* The query analysis is pure and lives with the planner; it is re-exported here because this is the
+   module a caller reaches for when it is thinking about media. */
+export { analyzeMediaQueries, skippedWidthQueries } from "./breakpoint-plan.ts";
+export type {
+  Breakpoint,
+  BreakpointPolicy,
+  BreakpointPlanResult,
+  BreakpointRounding,
+} from "./breakpoint-plan.ts";
 
 export interface MediaExtractionResult {
-  /** Breakpoints to add to project.json.$media */
+  /** Breakpoints to add to `project.json`'s `$media`. */
   breakpoints: Record<string, string>;
-  /** Per-breakpoint style deltas keyed by breakpoint name */
+  /** Per-breakpoint style deltas keyed by breakpoint name. */
   deltas: Record<string, DiffedStyle[]>;
 }
 
-/**
- * Parse a media query string and extract a pixel width if it's a simple min-width or max-width
- * query. Returns null for complex/non-size queries.
- */
-function parseWidthQuery(query: string): { type: "min" | "max"; width: number } | null {
-  const minMatch = query.match(/^\s*\(\s*min-width\s*:\s*([\d.]+)px\s*\)\s*$/);
-  if (minMatch) {
-    return { type: "min", width: Number(minMatch[1]) };
-  }
-  const maxMatch = query.match(/^\s*\(\s*max-width\s*:\s*([\d.]+)px\s*\)\s*$/);
-  if (maxMatch) {
-    return { type: "max", width: Number(maxMatch[1]) };
-  }
-  return null;
-}
-
-/** Generate a Jx breakpoint name from a width, e.g. 768 → "--768" */
-function breakpointName(width: number): string {
-  return `--${width}`;
-}
-
-/**
- * Analyze discovered @media queries and produce a set of testable breakpoints. Only picks up simple
- * min-width / max-width queries — complex queries (orientation, hover, prefers-*) are logged and
- * skipped.
- */
-export function analyzeMediaQueries(queries: string[]): Breakpoint[] {
-  const seen = new Set<number>();
-  const breakpoints: Breakpoint[] = [];
-
-  for (const query of queries) {
-    const parsed = parseWidthQuery(query);
-    if (!parsed) {
-      continue;
-    }
-    if (seen.has(parsed.width)) {
-      continue;
-    }
-    seen.add(parsed.width);
-
-    // For max-width: test AT that width (it's the upper bound)
-    // For min-width: test AT that width (it's the lower bound — styles kick in here)
-    breakpoints.push({
-      name: breakpointName(parsed.width),
-      query,
-      testWidth: parsed.width,
-    });
-  }
-
-  // Sort by width ascending
-  breakpoints.sort((a, b) => a.testWidth - b.testWidth);
-  return breakpoints;
+export interface ExtractMediaOptions {
+  /** The base viewport width to restore after extraction. */
+  originalWidth?: number;
+  /** Which of the discovered breakpoints to keep (default: the three-breakpoint limit). */
+  policy?: BreakpointPolicy | undefined;
+  /**
+   * An already-decided plan, used verbatim instead of planning from this page's own queries.
+   *
+   * A crawl needs this. Planning per page would let page 2 keep a width page 1 folded away, so the
+   * project's `$media` would be the UNION of several plans — the very thing a limit is for — and
+   * the breakpoint names a node carries would depend on which page it came from. One plan, decided
+   * on the first page that declares any width, is what keeps the names meaning one thing.
+   */
+  plan?: readonly Breakpoint[] | undefined;
 }
 
 /**
- * Extract $media style deltas by re-capturing at each breakpoint width.
+ * Extract `$media` style deltas by re-capturing at each KEPT breakpoint width.
  *
- * @param page - Puppeteer page (must still have the target page loaded)
- * @param baseElements - Style capture from the base viewport (1440px)
- * @param uaDefaults - UA-default baselines per tagName
- * @param mediaQueries - @media queries discovered in the page's stylesheets
- * @param originalWidth - The base viewport width to restore after extraction
+ * @param {ImportPage} page - Browser page (must still have the target page loaded)
+ * @param {CapturedStyle[]} baseElements - Style capture from the base viewport
+ * @param {Record<string, Record<string, string>>} uaDefaults - UA-default baselines per tagName
+ * @param {readonly string[]} mediaQueries - `@media` queries discovered in the page's stylesheets
+ * @param {ExtractMediaOptions} [options]
+ * @returns {Promise<MediaExtractionResult>}
  */
 export async function extractMedia(
-  page: Page,
+  page: ImportPage,
   baseElements: CapturedStyle[],
   uaDefaults: Record<string, Record<string, string>>,
-  mediaQueries: string[],
-  originalWidth = 1440,
+  mediaQueries: readonly string[],
+  options: ExtractMediaOptions = {},
 ): Promise<MediaExtractionResult> {
-  const breakpoints = analyzeMediaQueries(mediaQueries);
+  const { originalWidth = 1440, plan, policy } = options;
+  const keep = plan ?? planBreakpoints(analyzeMediaQueries(mediaQueries), policy).keep;
 
-  if (breakpoints.length === 0) {
+  if (keep.length === 0) {
     return { breakpoints: {}, deltas: {} };
   }
 
   const projectBreakpoints: Record<string, string> = {};
   const allDeltas: Record<string, DiffedStyle[]> = {};
 
-  for (const bp of breakpoints) {
+  for (const bp of keep) {
     const bpElements = await captureStylesAtWidth(page, bp.testWidth);
     const deltas = computeMediaDelta(baseElements, bpElements, uaDefaults);