@jxsuite/import 0.39.8 → 0.40.0

package/src/crawl.ts

@@ -5,6 +5,12 @@ import type { ToJxResult } from "./to-jx.ts";
 import { captureStyles } from "./style-capture.ts";
 import { diffAllStyles, kebabToCamel } from "./style-diff.ts";
 import { extractMedia } from "./media-extract.ts";
+import {
+  DEFAULT_BREAKPOINT_POLICY,
+  analyzeMediaQueries,
+  planBreakpoints,
+} from "./breakpoint-plan.ts";
+import type { Breakpoint, BreakpointPolicy } from "./breakpoint-plan.ts";
 import { applyStylesToTree } from "./apply-styles.ts";
 import { collectAssets } from "./asset-collect.ts";
 import { downloadAssets } from "./asset-download.ts";
@@ -24,6 +30,16 @@ export interface CrawlOptions {
   noScroll?: boolean;
   /** Capture a reference screenshot of each page before closing (for --verify). */
   captureScreenshots?: boolean;
+  /**
+   * Capture the whole scrollable page rather than the viewport (default true).
+   *
+   * It has to agree with what `verifyProject` screenshots on the other side. A viewport reference
+   * diffed against a full-page render is compared by padding the shorter image, so the two would
+   * disagree over everything below the fold before a single style was compared.
+   */
+  fullPageScreenshots?: boolean;
+  /** Which of the site's declared breakpoints the project keeps (see `breakpoint-plan.ts`). */
+  breakpoints?: BreakpointPolicy | undefined;
   onProgress?: (msg: string) => void;
   /** Abort the crawl between pages; the loop throws before capturing the next page. */
   signal?: AbortSignal;
@@ -178,6 +194,7 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
     skipStyles,
     skipAssets,
     respectRobots,
+    fullPageScreenshots = true,
     onProgress = console.log,
   } = options;
 
@@ -202,6 +219,9 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
   const skippedByRobots: string[] = [];
   const skippedByNodeCap: string[] = [];
   let mergedBreakpoints: Record<string, string> | undefined;
+  /* Decided on the first page that declares a width query, then reused. See ExtractMediaOptions.plan
+     — a per-page plan would union back up to the count the policy just refused. */
+  let breakpointPlan: Breakpoint[] | undefined;
   let mergedTokens: Record<string, string> | undefined;
   const allFontFaceRules: string[] = [];
   const fontRewriteMap = new Map<string, string>();
@@ -243,7 +263,9 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
       skippedByNodeCap.push(entry.url);
       let screenshot: Buffer | undefined;
       if (options.captureScreenshots) {
-        screenshot = Buffer.from(await capture.page.screenshot({ type: "png" }));
+        screenshot = Buffer.from(
+          await capture.page.screenshot({ fullPage: fullPageScreenshots, type: "png" }),
+        );
       }
       await capture.page.close();
 
@@ -277,11 +299,25 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
         }
 
         if (styleResult.mediaQueries.length > 0) {
+          if (!breakpointPlan) {
+            const declared = analyzeMediaQueries(styleResult.mediaQueries);
+            breakpointPlan = planBreakpoints(
+              declared,
+              options.breakpoints ?? DEFAULT_BREAKPOINT_POLICY,
+            ).keep;
+            if (breakpointPlan.length < declared.length) {
+              onProgress?.(
+                `  ${declared.length} breakpoints declared, keeping ` +
+                  `${breakpointPlan.length}: ${breakpointPlan.map((b) => b.name).join(", ")}\n`,
+              );
+            }
+          }
           const media = await extractMedia(
             capture.page,
             styleResult.elements,
             styleResult.uaDefaults,
             styleResult.mediaQueries,
+            { plan: breakpointPlan },
           );
           const bpCount = Object.keys(media.breakpoints).length;
           if (bpCount > 0) {
@@ -347,7 +383,9 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
 
     let screenshot: Buffer | undefined;
     if (options.captureScreenshots) {
-      screenshot = Buffer.from(await capture.page.screenshot({ type: "png" }));
+      screenshot = Buffer.from(
+        await capture.page.screenshot({ fullPage: fullPageScreenshots, type: "png" }),
+      );
     }
     await capture.page.close();
 

package/src/emit.ts

@@ -2,8 +2,31 @@ import { join, dirname } from "node:path";
 import { mkdir } from "node:fs/promises";
 import type { JxElement } from "@jxsuite/schema/types";
 import { componentize } from "./componentize.ts";
+import { stripClasses } from "./strip-classes.ts";
 import type { ComponentizeOptions, ComponentizeResult } from "./componentize.ts";
 
+/**
+ * The `$media` key the base width lives under — a number of CSS pixels, not a query.
+ *
+ * Studio's canvas reads it to size the default artboard (`utils/canvas-media.ts`), and an import
+ * never wrote one, so every imported project's base canvas fell back to 320px while its narrowest
+ * real breakpoint was three times that. Mirrors `settings/contexts-section.ts`'s `BASE_KEY`.
+ */
+const BASE_MEDIA_KEY = "--";
+
+/** The width a `--<width>` breakpoint key names, for sorting. Non-numeric keys sort first. */
+function mediaKeyWidth(key: string): number {
+  const width = Number(key.replace(/^--/, ""));
+  return Number.isFinite(width) ? width : -1;
+}
+
+export interface EmitResult {
+  /** Absolute paths of every file written. */
+  files: string[];
+  /** How many source-site class names were removed on the way out. */
+  classesStripped: number;
+}
+
 export interface EmitOptions {
   outDir: string;
   title: string;
@@ -11,6 +34,8 @@ export interface EmitOptions {
   sourceUrl: string;
   /** $media breakpoints to seed into project.json (Phase 1). */
   breakpoints?: Record<string, string>;
+  /** The capture viewport width, written as the `$media` base entry. */
+  baseWidth?: number;
   /** Phase 4: componentization options. Pass false to skip. */
   componentizeOptions?: ComponentizeOptions | false;
 }
@@ -25,6 +50,8 @@ export interface MultiEmitOptions {
   layout?: JxElement | undefined;
   /** $media breakpoints to seed into project.json. */
   breakpoints?: Record<string, string> | undefined;
+  /** The capture viewport width, written as the `$media` base entry. */
+  baseWidth?: number | undefined;
   /** Phase 4: componentization options. Pass false to skip. */
   componentizeOptions?: ComponentizeOptions | false | undefined;
   /** Pre-computed componentization result (from AI pass). Overrides componentizeOptions. */
@@ -43,14 +70,16 @@ export async function emitProject({
   document,
   sourceUrl,
   breakpoints,
+  baseWidth,
   componentizeOptions,
-}: EmitOptions): Promise<{ files: string[] }> {
+}: EmitOptions): Promise<EmitResult> {
   return emitMultiPageProject({
     outDir,
     title,
     sourceUrl,
     pages: new Map([["pages/index.json", document]]),
     breakpoints,
+    baseWidth,
     componentizeOptions,
   });
 }
@@ -58,35 +87,53 @@ export async function emitProject({
 export async function emitMultiPageProject({
   outDir,
   title,
-  sourceUrl,
   pages,
   layout,
   breakpoints,
+  baseWidth,
   componentizeOptions,
   precomputedComponents,
   fontFaceRules,
   fontRewriteMap,
   styleTokens,
-}: MultiEmitOptions): Promise<{ files: string[] }> {
+}: MultiEmitOptions): Promise<EmitResult> {
   await mkdir(join(outDir, "pages"), { recursive: true });
   await mkdir(join(outDir, "layouts"), { recursive: true });
   await mkdir(join(outDir, "components"), { recursive: true });
   await mkdir(join(outDir, "public"), { recursive: true });
 
+  /*
+   * Only keys the project schema declares (#228). `title` and `description` are page-level keys and
+   * `$style` is not a key at all, and all three made every imported project fail `jx validate` —
+   * and, because Studio's Contexts editor validates the WHOLE configuration before saving, made an
+   * imported project's base width uneditable, reporting `(root): must NOT have unevaluated
+   * properties` three times while naming nothing.
+   *
+   * The source URL has no schema-legal home and does not get one. `$head`'s description meta is the
+   * nearest candidate and the wrong one: it is the site's own user-facing description, and
+   * "Imported from …" is provenance, not a description of the site. The importer reports it instead.
+   */
   const projectJson: Record<string, unknown> = {
     name: title || "Imported Site",
-    title: title || "Imported Site",
-    description: `Imported from ${sourceUrl}`,
     imports: {},
     images: { optimize: false },
   };
 
+  if (baseWidth !== undefined) {
+    projectJson.$media = { [BASE_MEDIA_KEY]: `${Math.round(baseWidth)}px` };
+  }
   if (breakpoints && Object.keys(breakpoints).length > 0) {
-    projectJson.$media = breakpoints;
+    /* Ascending, and the base first. `$media` is a map, so its order IS the order Studio's Contexts
+       list and the pane's size switcher offer these in; a crawl merged them in page-visit order,
+       which is not an order at all. */
+    const sorted = Object.entries(breakpoints).toSorted(
+      ([a], [b]) => mediaKeyWidth(a) - mediaKeyWidth(b),
+    );
+    projectJson.$media = { ...(projectJson.$media as object), ...Object.fromEntries(sorted) };
   }
 
   if (styleTokens && Object.keys(styleTokens).length > 0) {
-    projectJson.$style = styleTokens;
+    projectJson.style = styleTokens;
   }
 
   const files: string[] = [];
@@ -123,12 +170,41 @@ export async function emitMultiPageProject({
   if (fontFaceRules && fontFaceRules.length > 0) {
     let fontCss = fontFaceRules.join("\n\n");
     if (fontRewriteMap) {
+      // A @font-face rule carries the url() form its AUTHOR wrote — usually root-relative
+      // ("/wp-content/.../lato.woff2"), sometimes protocol-relative. The rewrite map is keyed by the
+      // Absolute URL the downloader resolved, so matching on that alone rewrote nothing: every
+      // Downloaded font stayed unreferenced and the page fell back to system fonts.
+      //
+      // Replaced in ONE pass, longest form first. Replacing form by form re-scans text this loop
+      // Already rewrote, and a bare pathname ("/a.woff2") then matches inside the local path it
+      // Just produced ("/assets/fonts/a.woff2" to "/assets/fonts/assets/fonts/a.woff2").
+      const byForm = new Map<string, string>();
       for (const [originalUrl, localPath] of fontRewriteMap) {
-        // Rewrite absolute URLs in font-face rules to relative local paths
-        const relativePath = localPath.startsWith("public/")
+        const local = localPath.startsWith("public/")
           ? localPath.slice("public/".length)
           : localPath;
-        fontCss = fontCss.replaceAll(originalUrl, `/${relativePath}`);
+        const href = local.startsWith("/") ? local : `/${local}`;
+        const forms = new Set([originalUrl]);
+        try {
+          const u = new URL(originalUrl);
+          forms.add(`//${u.host}${u.pathname}${u.search}`);
+          forms.add(`${u.pathname}${u.search}`);
+        } catch {
+          // Already a relative reference — the literal form is all there is.
+        }
+        for (const form of forms) {
+          if (form) {
+            byForm.set(form, href);
+          }
+        }
+      }
+      const forms = [...byForm.keys()].toSorted((a, b) => b.length - a.length);
+      if (forms.length > 0) {
+        const pattern = new RegExp(
+          forms.map((f) => f.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`)).join("|"),
+          "g",
+        );
+        fontCss = fontCss.replaceAll(pattern, (m) => byForm.get(m) ?? m);
       }
     }
     const fontsDir = join(outDir, "public", "assets");
@@ -151,6 +227,22 @@ export async function emitMultiPageProject({
   await Bun.write(projectPath, `${JSON.stringify(projectJson, null, 2)}\n`);
   files.push(projectPath);
 
+  /*
+   * Classes go here, at the last possible moment, and that placement is the whole design.
+   *
+   * `componentize` groups by structure and `ai-componentize` reads the template to choose a name —
+   * `product-card` over `component-div-0` — and the class names are the strongest signal either has.
+   * Stripping earlier would cost that; stripping later is impossible. So both passes see the source
+   * site's classes, and nothing downstream of this line does.
+   */
+  let classesStripped = 0;
+  for (const compDoc of componentFiles.values()) {
+    classesStripped += stripClasses(compDoc);
+  }
+  for (const doc of finalPages.values()) {
+    classesStripped += stripClasses(doc);
+  }
+
   // Write components
   for (const [fileName, compDoc] of componentFiles) {
     const compPath = join(outDir, "components", fileName);
@@ -193,10 +285,11 @@ export async function emitMultiPageProject({
       ...(((layoutDoc as Record<string, unknown>).$elements as JxElement[]) ?? []),
     ];
   }
+  classesStripped += stripClasses(layoutDoc);
   await Bun.write(layoutPath, `${JSON.stringify(layoutDoc, null, 2)}\n`);
   files.push(layoutPath);
 
-  return { files };
+  return { classesStripped, files };
 }
 
 function extractStateDefaults(node: JxElement | string, out: Record<string, string>) {

package/src/index.ts

@@ -4,7 +4,19 @@ export { convertToJx } from "./to-jx.ts";
 export { emitProject, emitMultiPageProject } from "./emit.ts";
 export { captureStyles, captureStylesAtWidth, STYLE_ALLOWLIST } from "./style-capture.ts";
 export { diffStyles, diffAllStyles, computeMediaDelta, kebabToCamel } from "./style-diff.ts";
-export { analyzeMediaQueries, extractMedia } from "./media-extract.ts";
+export { extractMedia } from "./media-extract.ts";
+export {
+  DEFAULT_BREAKPOINT_POLICY,
+  analyzeMediaQueries,
+  skippedWidthQueries,
+  MAX_BREAKPOINTS,
+  MAX_BREAKPOINT_WIDTH,
+  MIN_BREAKPOINT_WIDTH,
+  breakpointName,
+  parseWidthQuery,
+  planBreakpoints,
+} from "./breakpoint-plan.ts";
+export { stripClasses } from "./strip-classes.ts";
 export { applyStylesToTree } from "./apply-styles.ts";
 export { collectAssets } from "./asset-collect.ts";
 export { downloadAssets } from "./asset-download.ts";
@@ -29,10 +41,16 @@ export type {
   ImportSiteResult,
 } from "./run.ts";
 export type { ToJxResult } from "./to-jx.ts";
-export type { EmitOptions, MultiEmitOptions } from "./emit.ts";
+export type { EmitOptions, EmitResult, MultiEmitOptions } from "./emit.ts";
 export type { CapturedStyle, StyleCaptureResult } from "./style-capture.ts";
 export type { DiffedStyle } from "./style-diff.ts";
-export type { Breakpoint, MediaExtractionResult } from "./media-extract.ts";
+export type { ExtractMediaOptions, MediaExtractionResult } from "./media-extract.ts";
+export type {
+  Breakpoint,
+  BreakpointPlanResult,
+  BreakpointPolicy,
+  BreakpointRounding,
+} from "./breakpoint-plan.ts";
 export type {
   DiscoveredAsset,
   AssetCollectionResult,

package/src/media-extract.ts

@@ -1,110 +1,86 @@
 /**
- * Extract @media breakpoints from captured CSS and orchestrate style re-capture at each breakpoint
- * width to produce per-node $media style deltas.
+ * Extract `@media` breakpoints from captured CSS and orchestrate style re-capture at each kept
+ * breakpoint width to produce per-node `$media` style deltas.
+ *
+ * The set of breakpoints a project keeps is decided in `breakpoint-plan.ts`, BEFORE anything is
+ * re-captured — each kept breakpoint costs a viewport change and a full style walk, so a limited
+ * import does less work rather than merely emitting less.
+ *
+ * A folded-away width is not sampled at all, and does not need to be. The computed style at a kept
+ * width is already the site's own cascade resolved AT that width, folded rules included; merging a
+ * separately-sampled delta on top of it could only disagree with the browser. The fold map exists
+ * so the import can SAY where a width went, not so its styles can be recovered.
  */
 
 import type { Page } from "puppeteer-core";
 import type { CapturedStyle } from "./style-capture.ts";
 import { captureStylesAtWidth } from "./style-capture.ts";
 import { computeMediaDelta } from "./style-diff.ts";
+import { analyzeMediaQueries, planBreakpoints } from "./breakpoint-plan.ts";
+import type { Breakpoint, BreakpointPolicy } from "./breakpoint-plan.ts";
 import type { DiffedStyle } from "./style-diff.ts";
 
-export interface Breakpoint {
-  /** Jx breakpoint name, e.g. "--md" */
-  name: string;
-  /** CSS media query string, e.g. "(max-width: 768px)" */
-  query: string;
-  /** The width in px to test this breakpoint at */
-  testWidth: number;
-}
+/* The query analysis is pure and lives with the planner; it is re-exported here because this is the
+   module a caller reaches for when it is thinking about media. */
+export { analyzeMediaQueries, skippedWidthQueries } from "./breakpoint-plan.ts";
+export type {
+  Breakpoint,
+  BreakpointPolicy,
+  BreakpointPlanResult,
+  BreakpointRounding,
+} from "./breakpoint-plan.ts";
 
 export interface MediaExtractionResult {
-  /** Breakpoints to add to project.json.$media */
+  /** Breakpoints to add to `project.json`'s `$media`. */
   breakpoints: Record<string, string>;
-  /** Per-breakpoint style deltas keyed by breakpoint name */
+  /** Per-breakpoint style deltas keyed by breakpoint name. */
   deltas: Record<string, DiffedStyle[]>;
 }
 
-/**
- * Parse a media query string and extract a pixel width if it's a simple min-width or max-width
- * query. Returns null for complex/non-size queries.
- */
-function parseWidthQuery(query: string): { type: "min" | "max"; width: number } | null {
-  const minMatch = query.match(/^\s*\(\s*min-width\s*:\s*([\d.]+)px\s*\)\s*$/);
-  if (minMatch) {
-    return { type: "min", width: Number(minMatch[1]) };
-  }
-  const maxMatch = query.match(/^\s*\(\s*max-width\s*:\s*([\d.]+)px\s*\)\s*$/);
-  if (maxMatch) {
-    return { type: "max", width: Number(maxMatch[1]) };
-  }
-  return null;
-}
-
-/** Generate a Jx breakpoint name from a width, e.g. 768 → "--768" */
-function breakpointName(width: number): string {
-  return `--${width}`;
-}
-
-/**
- * Analyze discovered @media queries and produce a set of testable breakpoints. Only picks up simple
- * min-width / max-width queries — complex queries (orientation, hover, prefers-*) are logged and
- * skipped.
- */
-export function analyzeMediaQueries(queries: string[]): Breakpoint[] {
-  const seen = new Set<number>();
-  const breakpoints: Breakpoint[] = [];
-
-  for (const query of queries) {
-    const parsed = parseWidthQuery(query);
-    if (!parsed) {
-      continue;
-    }
-    if (seen.has(parsed.width)) {
-      continue;
-    }
-    seen.add(parsed.width);
-
-    // For max-width: test AT that width (it's the upper bound)
-    // For min-width: test AT that width (it's the lower bound — styles kick in here)
-    breakpoints.push({
-      name: breakpointName(parsed.width),
-      query,
-      testWidth: parsed.width,
-    });
-  }
-
-  // Sort by width ascending
-  breakpoints.sort((a, b) => a.testWidth - b.testWidth);
-  return breakpoints;
+export interface ExtractMediaOptions {
+  /** The base viewport width to restore after extraction. */
+  originalWidth?: number;
+  /** Which of the discovered breakpoints to keep (default: the three-breakpoint limit). */
+  policy?: BreakpointPolicy | undefined;
+  /**
+   * An already-decided plan, used verbatim instead of planning from this page's own queries.
+   *
+   * A crawl needs this. Planning per page would let page 2 keep a width page 1 folded away, so the
+   * project's `$media` would be the UNION of several plans — the very thing a limit is for — and
+   * the breakpoint names a node carries would depend on which page it came from. One plan, decided
+   * on the first page that declares any width, is what keeps the names meaning one thing.
+   */
+  plan?: readonly Breakpoint[] | undefined;
 }
 
 /**
- * Extract $media style deltas by re-capturing at each breakpoint width.
+ * Extract `$media` style deltas by re-capturing at each KEPT breakpoint width.
  *
- * @param page - Puppeteer page (must still have the target page loaded)
- * @param baseElements - Style capture from the base viewport (1440px)
- * @param uaDefaults - UA-default baselines per tagName
- * @param mediaQueries - @media queries discovered in the page's stylesheets
- * @param originalWidth - The base viewport width to restore after extraction
+ * @param {Page} page - Puppeteer page (must still have the target page loaded)
+ * @param {CapturedStyle[]} baseElements - Style capture from the base viewport
+ * @param {Record<string, Record<string, string>>} uaDefaults - UA-default baselines per tagName
+ * @param {readonly string[]} mediaQueries - `@media` queries discovered in the page's stylesheets
+ * @param {ExtractMediaOptions} [options]
+ * @returns {Promise<MediaExtractionResult>}
  */
 export async function extractMedia(
   page: Page,
   baseElements: CapturedStyle[],
   uaDefaults: Record<string, Record<string, string>>,
-  mediaQueries: string[],
-  originalWidth = 1440,
+  mediaQueries: readonly string[],
+  options: ExtractMediaOptions = {},
 ): Promise<MediaExtractionResult> {
-  const breakpoints = analyzeMediaQueries(mediaQueries);
+  const { originalWidth = 1440, plan, policy } = options;
+  const keep = plan ?? planBreakpoints(analyzeMediaQueries(mediaQueries), policy).keep;
 
-  if (breakpoints.length === 0) {
+  if (keep.length === 0) {
     return { breakpoints: {}, deltas: {} };
   }
 
   const projectBreakpoints: Record<string, string> = {};
   const allDeltas: Record<string, DiffedStyle[]> = {};
 
-  for (const bp of breakpoints) {
+  for (const bp of keep) {
     const bpElements = await captureStylesAtWidth(page, bp.testWidth);
     const deltas = computeMediaDelta(baseElements, bpElements, uaDefaults);