@jxsuite/import 0.40.1 → 0.40.3

package/src/emit.ts

@@ -166,10 +166,16 @@ export async function emitMultiPageProject({
         tagName: comp.tagName,
       };
 
-      const state: Record<string, string> = {};
-      extractStateDefaults(comp.template, state);
+      /* Derived placeholders fill in for props the template interpolates but nobody declared;
+         a value the template DECLARED wins over them. This was an assignment over the spread, so
+         a component that carried its own `state` — the `active` index a tab group switches on —
+         lost it on the way out, and every interpolation referencing it went dead. */
+      const derived: Record<string, string> = {};
+      extractStateDefaults(comp.template, derived);
+      const declared = (comp.template.state ?? {}) as Record<string, unknown>;
+      const state = { ...derived, ...declared };
       if (Object.keys(state).length > 0) {
-        compDoc.state = state;
+        compDoc.state = state as Record<string, string>;
       }
 
       componentFiles.set(fileName, compDoc);
@@ -298,16 +304,29 @@ export async function emitMultiPageProject({
   return { classesStripped, files, projectJson: projectText };
 }
 
+/**
+ * The state key a whole-string `${state.x}` names, or null.
+ *
+ * The identifier check is the point. The capture used to be `([^}]+)`, so `${state.active !== 0}` —
+ * an expression, not a reference — minted a state entry literally called `active !== 0` beside the
+ * real one. Only a bare identifier is a key; anything else is an expression over state and has no
+ * default of its own to declare.
+ */
+function stateKey(text: string): string | undefined {
+  const key = text.match(/^\$\{state\.([^}]+)\}$/)?.[1];
+  return key !== undefined && /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key) ? key : undefined;
+}
+
 function extractStateDefaults(node: JxElement | string, out: Record<string, string>) {
   if (typeof node === "string") {
-    const key = node.match(/^\$\{state\.([^}]+)\}$/)?.[1];
+    const key = stateKey(node);
     if (key && !(key in out)) {
       out[key] = "";
     }
     return;
   }
   if (typeof node.textContent === "string") {
-    const key = node.textContent.match(/^\$\{state\.([^}]+)\}$/)?.[1];
+    const key = stateKey(node.textContent);
     if (key && !(key in out)) {
       out[key] = "";
     }
@@ -315,7 +334,7 @@ function extractStateDefaults(node: JxElement | string, out: Record<string, stri
   if (node.attributes) {
     for (const v of Object.values(node.attributes)) {
       if (typeof v === "string") {
-        const key = v.match(/^\$\{state\.([^}]+)\}$/)?.[1];
+        const key = stateKey(v);
         if (key && !(key in out)) {
           out[key] = "";
         }

package/src/image-family.ts

@@ -0,0 +1,227 @@
+/**
+ * Group an image's responsive derivatives back into one family, and pick the member to keep.
+ *
+ * A CMS publishes one photograph as a dozen files. WordPress alone emits `-300x200`, `-768x512`,
+ * `-1024x683`, `-1536x1025`, `-2048x1366` and `-scaled` beside the upload, and every one of them
+ * appears in the `srcset` of every `<img>` that uses it. The importer enqueued each candidate as an
+ * independent asset, so a clone of a 451-image site downloaded 2,446 files and 427 MB, and the
+ * emitted markup carried the whole derivative ladder forward.
+ *
+ * That is the wrong shape twice over. It is large, and it hands the compiler a `srcset` it did not
+ * build and cannot reason about — while `image-transform.ts` caps its own ladder at the source
+ * file's width, so an `<img src>` pointing at a `-300x200` derivative is a permanent quality
+ * ceiling rather than merely a wasted byte.
+ *
+ * So the importer keeps ONE member per family and lets the compiler regenerate sizes from it.
+ *
+ * Two rules make this safe on a real site:
+ *
+ * 1. **The key comes from the source URL, never from the written filename.** `sanitizeFilename`
+ *    truncates a long name to 96 characters, which cuts the `-WxH` marker off the end — five
+ *    resolutions of one image would land as five unrelated families.
+ * 2. **The winner is always a URL the page actually referenced.** 120 of the 451 families on the
+ *    reference corpus have no undecorated member at all, because WordPress serves only
+ *    `-scaled.jpg` once an upload passes its threshold. Synthesising the bare name and fetching it
+ *    would 404 on those, and a failed download leaves the remote URL in the markup — the clone then
+ *    serves images from the host it cloned. Taking the largest real candidate cannot fail that
+ *    way.
+ */
+
+import type { DiscoveredAsset } from "./asset-collect.ts";
+
+/** Sources whose URLs are responsive variants of one image. A CSS background is not one. */
+const FAMILY_SOURCES = new Set(["img-src", "img-srcset", "source-srcset", "picture-source"]);
+
+/** `name-1024x683.jpg` → the `1024x683` marker, and where it starts. */
+const DIMENSION_SUFFIX = /-(\d{1,5})x(\d{1,5})$/;
+
+/** WordPress's marker for "the original was too big, this is the working copy". */
+const SCALED_SUFFIX = /-scaled$/;
+
+interface ParsedName {
+  /** The stem with any `-WxH` and `-scaled` markers removed. */
+  stem: string;
+  /** Pixel area of the `-WxH` marker, or null when there is none. */
+  area: number | null;
+  /** Whether the name carried `-scaled`. */
+  scaled: boolean;
+}
+
+function splitExtension(pathname: string): { base: string; ext: string } {
+  const slash = pathname.lastIndexOf("/");
+  const dot = pathname.lastIndexOf(".");
+  return dot > slash
+    ? { base: pathname.slice(0, dot), ext: pathname.slice(dot) }
+    : { base: pathname, ext: "" };
+}
+
+/**
+ * Strip the derivative markers from a pathname's final segment.
+ *
+ * Order matters: WordPress writes `photo-scaled-768x512.jpg`, so the dimensions come off first and
+ * `-scaled` second. Doing it the other way leaves `-768x512` welded to the stem and splits the
+ * family.
+ */
+function parseName(pathname: string): ParsedName {
+  const { base } = splitExtension(pathname);
+  let stem = base;
+  let area: number | null = null;
+
+  const dimensions = stem.match(DIMENSION_SUFFIX);
+  if (dimensions) {
+    area = Number(dimensions[1]) * Number(dimensions[2]);
+    stem = stem.slice(0, dimensions.index);
+  }
+
+  const scaled = SCALED_SUFFIX.test(stem);
+  if (scaled) {
+    stem = stem.replace(SCALED_SUFFIX, "");
+  }
+
+  return { area, scaled, stem };
+}
+
+/**
+ * The identity two derivatives of one image share.
+ *
+ * Origin and directory are part of it — two sites can both serve `hero.jpg`, and WordPress's
+ * `/uploads/<year>/<month>/` layout means the same basename recurs legitimately. The query string
+ * is not: a cache-buster is not a different image. A URL that will not parse is its own family,
+ * which is the conservative answer (it is kept, not merged with something it may not match).
+ */
+export function familyKey(url: string): string {
+  let parsed: URL;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return url;
+  }
+  const { ext } = splitExtension(parsed.pathname);
+  const { stem } = parseName(parsed.pathname);
+  return `${parsed.origin}${stem}${ext.toLowerCase()}`;
+}
+
+/**
+ * Pick the member of a family to keep.
+ *
+ * Ranked: an undecorated name beats everything, because it is the upload itself; otherwise the
+ * largest `-WxH` wins, and a `-scaled` copy is preferred to a small explicit crop only when nothing
+ * else is left. Ties break on the URL string so a crawl is reproducible.
+ *
+ * @param {readonly string[]} candidates - Absolute URLs, all of one family, non-empty
+ * @returns {string}
+ */
+export function chooseOriginal(candidates: readonly string[]): string {
+  const ranked = candidates.map((url) => {
+    let pathname: string;
+    try {
+      ({ pathname } = new URL(url));
+    } catch {
+      pathname = url;
+    }
+    const { area, scaled } = parseName(pathname);
+    // Undecorated first, then scaled-only, then explicit crops largest-first.
+    const tier = area === null ? (scaled ? 1 : 0) : 2;
+    return { area: area ?? 0, tier, url };
+  });
+
+  ranked.sort((a, b) => {
+    if (a.tier !== b.tier) {
+      return a.tier - b.tier;
+    }
+    if (a.area !== b.area) {
+      return b.area - a.area;
+    }
+    return a.url < b.url ? -1 : a.url > b.url ? 1 : 0;
+  });
+
+  return ranked[0]!.url;
+}
+
+export interface ImageFamilyPlan {
+  /** The assets to actually download — one per image family, plus every non-family asset. */
+  keep: DiscoveredAsset[];
+  /**
+   * Dropped URL → the URL kept in its place.
+   *
+   * The caller replays this over the download's `rewriteMap` so a reference to any derivative still
+   * resolves, to the one file that was written.
+   */
+  alias: Map<string, string>;
+}
+
+/**
+ * Reduce a discovered asset list to one member per image family.
+ *
+ * Only `<img>`/`<source>` URLs take part. A CSS background, a font and a favicon are each kept as
+ * found: a background is authored at one size, and merging two of them on a coincidental name match
+ * would silently swap an image the stylesheet asked for.
+ *
+ * @param {readonly DiscoveredAsset[]} assets
+ * @returns {ImageFamilyPlan}
+ */
+export function planImageFamilies(assets: readonly DiscoveredAsset[]): ImageFamilyPlan {
+  const families = new Map<string, string[]>();
+  for (const asset of assets) {
+    if (!FAMILY_SOURCES.has(asset.source)) {
+      continue;
+    }
+    const key = familyKey(asset.url);
+    families.set(key, [...(families.get(key) ?? []), asset.url]);
+  }
+
+  const winners = new Map<string, string>();
+  for (const [key, urls] of families) {
+    winners.set(key, chooseOriginal(urls));
+  }
+
+  const alias = new Map<string, string>();
+  const keep: DiscoveredAsset[] = [];
+  const taken = new Set<string>();
+
+  for (const asset of assets) {
+    if (!FAMILY_SOURCES.has(asset.source)) {
+      keep.push(asset);
+      continue;
+    }
+    const winner = winners.get(familyKey(asset.url))!;
+    if (asset.url === winner) {
+      if (!taken.has(winner)) {
+        taken.add(winner);
+        keep.push(asset);
+      }
+      continue;
+    }
+    alias.set(asset.url, winner);
+  }
+
+  return { alias, keep };
+}
+
+/**
+ * Point every dropped derivative at the file its family actually wrote.
+ *
+ * The download only ever saw one member per family, so the map it returns has one entry per family.
+ * Markup still refers to whichever rung it was authored against, and an unresolved reference is not
+ * merely a broken image: `asset-rewrite.ts` leaves a URL it cannot resolve untouched, so the built
+ * clone would go on serving that image from the site it was cloned from.
+ *
+ * @param {Map<string, string>} rewriteMap - Download result, mutated in place
+ * @param {ReadonlyMap<string, string>} alias - Dropped URL to kept URL, from
+ *   {@link planImageFamilies}
+ * @returns {number} How many references were reconnected
+ */
+export function applyFamilyAliases(
+  rewriteMap: Map<string, string>,
+  alias: ReadonlyMap<string, string>,
+): number {
+  let reconnected = 0;
+  for (const [dropped, kept] of alias) {
+    const local = rewriteMap.get(kept);
+    if (local !== undefined) {
+      rewriteMap.set(dropped, local);
+      reconnected += 1;
+    }
+  }
+  return reconnected;
+}

package/src/media-extract.ts

@@ -16,7 +16,7 @@ import type { ImportPage } from "./capture.ts";
 import type { CapturedStyle } from "./style-capture.ts";
 import { captureStylesAtWidth } from "./style-capture.ts";
 import { computeMediaDelta } from "./style-diff.ts";
-import { analyzeMediaQueries, planBreakpoints } from "./breakpoint-plan.ts";
+import { analyzeMediaQueries, parseWidthQuery, planBreakpoints } from "./breakpoint-plan.ts";
 import type { Breakpoint, BreakpointPolicy } from "./breakpoint-plan.ts";
 import type { DiffedStyle } from "./style-diff.ts";
 
@@ -53,6 +53,66 @@ export interface ExtractMediaOptions {
   plan?: readonly Breakpoint[] | undefined;
 }
 
+/**
+ * How far inside its own band the corroborating sample sits. Wide enough that a fluid element
+ * measures differently, narrow enough to stay clear of the neighbouring breakpoint.
+ */
+const SECOND_SAMPLE_OFFSET = 40;
+
+/**
+ * A second viewport width INSIDE the same media band.
+ *
+ * The direction is the whole point: for `(max-width: N)` the band runs downward, so the second
+ * sample is narrower; for `(min-width: N)` it runs upward and the sample must be wider, or it
+ * leaves the band and measures the rules this breakpoint exists to distinguish itself from.
+ */
+export function secondSampleWidth(bp: Breakpoint): number {
+  const parsed = parseWidthQuery(bp.query);
+  const direction = parsed?.type === "min" ? 1 : -1;
+  return Math.max(1, bp.testWidth + direction * SECOND_SAMPLE_OFFSET);
+}
+
+/**
+ * Keep only the declarations two samples of the same band AGREE on.
+ *
+ * `getComputedStyle` returns used values, so `width` and `height` come back as the pixels the
+ * element happens to occupy - never the authored `100%` or `auto`. Diffing one sample against the
+ * base therefore reported a "change" for every fluid element at every breakpoint: two thirds of all
+ * responsive declarations in a real import were the viewport's own width written back as the
+ * element's, which does not merely waste space but PINS the layout and defeats the reflow the
+ * breakpoint existed for.
+ *
+ * Measuring twice inside the band separates the two cases without knowing which properties are
+ * geometry. An authored value is the same at both widths and survives; a resolved one moves with
+ * the viewport and is dropped. A property whose value legitimately differs across the band (a
+ * percentage-positioned element) is dropped too - that is the intended trade, since nothing in a
+ * computed-style capture can tell it from noise.
+ */
+export function agreedDeltas(
+  primary: readonly DiffedStyle[],
+  corroborating: readonly DiffedStyle[],
+): DiffedStyle[] {
+  const second = new Map(corroborating.map((d) => [d.path.join(","), d.style]));
+  const agreed: DiffedStyle[] = [];
+
+  for (const delta of primary) {
+    const other = second.get(delta.path.join(","));
+    if (!other) {
+      continue;
+    }
+    const style: Record<string, string | number> = {};
+    for (const [prop, value] of Object.entries(delta.style)) {
+      if (other[prop] === value) {
+        style[prop] = value;
+      }
+    }
+    if (Object.keys(style).length > 0) {
+      agreed.push({ path: delta.path, style });
+    }
+  }
+  return agreed;
+}
+
 /**
  * Extract `$media` style deltas by re-capturing at each KEPT breakpoint width.
  *
@@ -81,8 +141,17 @@ export async function extractMedia(
   const allDeltas: Record<string, DiffedStyle[]> = {};
 
   for (const bp of keep) {
-    const bpElements = await captureStylesAtWidth(page, bp.testWidth);
-    const deltas = computeMediaDelta(baseElements, bpElements, uaDefaults);
+    const primary = computeMediaDelta(
+      baseElements,
+      await captureStylesAtWidth(page, bp.testWidth),
+      uaDefaults,
+    );
+    const corroborating = computeMediaDelta(
+      baseElements,
+      await captureStylesAtWidth(page, secondSampleWidth(bp)),
+      uaDefaults,
+    );
+    const deltas = agreedDeltas(primary, corroborating);
 
     if (deltas.length > 0) {
       projectBreakpoints[bp.name] = bp.query;

package/src/pipeline.ts

@@ -31,10 +31,14 @@ import { diffAllStyles, kebabToCamel } from "./style-diff.ts";
 import { applyStylesToTree } from "./apply-styles.ts";
 import { collectAssets } from "./asset-collect.ts";
 import { downloadAssets } from "./asset-download.ts";
+import { applyFamilyAliases, planImageFamilies } from "./image-family.ts";
 import { rewriteAssetUrls } from "./asset-rewrite.ts";
 import { applyTokens } from "./css-tokens.ts";
 import { crawlSite } from "./crawl.ts";
 import { detectLayout } from "./layout-detect.ts";
+import { applyAccordions } from "./apply-accordions.ts";
+import { applyPopovers } from "./apply-popovers.ts";
+import { applyDisclosures } from "./apply-disclosures.ts";
 import type { ImportIo } from "./io.ts";
 import type { JxElement } from "@jxsuite/schema/types";
 import type { ComponentizeResult } from "./componentize.ts";
@@ -99,6 +103,7 @@ export type ImportPhase =
   | "crawl"
   | "styles"
   | "assets"
+  | "convert"
   | "layout"
   | "components"
   | "ai"
@@ -322,7 +327,7 @@ export async function runImportPipeline(
         if (bpCount > 0) {
           progress("styles", `${bpCount} breakpoints with style changes`);
           ({ breakpoints } = media);
-          applyStylesToTree(jx.document, diffed, media.deltas);
+          applyStylesToTree(jx.document, diffed, media.deltas, media.breakpoints);
         } else {
           progress("styles", "No responsive breakpoints with style changes");
           applyStylesToTree(jx.document, diffed);
@@ -368,7 +373,12 @@ export async function runImportPipeline(
 
       if (collected.assets.length > 0) {
         progress("assets", "Downloading assets...");
-        const downloaded = await downloadAssets(collected.assets, io, url);
+        /* One member per responsive family reaches the network; every dropped derivative is
+           aliased onto the file that WAS written, so a reference to any rung of the ladder still
+           resolves. The compiler regenerates the sizes from the original it now owns. */
+        const families = planImageFamilies(collected.assets);
+        const downloaded = await downloadAssets(families.keep, io, url);
+        applyFamilyAliases(downloaded.rewriteMap, families.alias);
         progress(
           "assets",
           `Downloaded ${downloaded.rewriteMap.size} assets (${formatBytes(downloaded.totalBytes)})`,
@@ -408,6 +418,31 @@ export async function runImportPipeline(
     await capture.page.close();
 
     throwIfAborted(signal);
+    /* Widgets become native markup BEFORE layout detection, which replaces the pages it extracts a
+       layout from - a pass after it would see the site's widgets in neither half. */
+    const singleConverted = applyAccordions(jx.document);
+    const singleDisclosures = applyDisclosures(jx.document);
+    if (singleDisclosures.converted > 0) {
+      progress(
+        "convert",
+        `Converted ${singleDisclosures.converted} disclosure${singleDisclosures.converted === 1 ? "" : "s"} to native <details>`,
+      );
+    }
+    const singleOverlays = applyPopovers(jx.document);
+    if (singleOverlays.converted > 0) {
+      progress(
+        "convert",
+        `Converted ${singleOverlays.converted} dropdown${singleOverlays.converted === 1 ? "" : "s"} to popovers`,
+      );
+    }
+    if (singleConverted.converted > 0) {
+      progress(
+        "convert",
+        `Converted ${singleConverted.converted} accordion${singleConverted.converted === 1 ? "" : "s"} ` +
+          `(${singleConverted.rows} rows) to native <details>`,
+      );
+    }
+
     const pageMap = new Map([["pages/index.json", jx.document]]);
     const precomputedComponents = await maybeAiComponentize(
       pageMap,
@@ -484,6 +519,39 @@ export async function runImportPipeline(
       pageMap.set(page.route, page.jx.document);
     }
 
+    /* Widgets become native markup BEFORE layout detection, which replaces the pages it extracts a
+       layout from - a pass after it would see the site's widgets in neither half. */
+    let convertedWidgets = 0;
+    let convertedRows = 0;
+    let convertedOverlays = 0;
+    let convertedDisclosures = 0;
+    for (const document of pageMap.values()) {
+      const converted = applyAccordions(document);
+      convertedWidgets += converted.converted;
+      convertedRows += converted.rows;
+      convertedDisclosures += applyDisclosures(document).converted;
+      convertedOverlays += applyPopovers(document).converted;
+    }
+    if (convertedDisclosures > 0) {
+      progress(
+        "convert",
+        `Converted ${convertedDisclosures} disclosure${convertedDisclosures === 1 ? "" : "s"} to native <details>`,
+      );
+    }
+    if (convertedOverlays > 0) {
+      progress(
+        "convert",
+        `Converted ${convertedOverlays} dropdown${convertedOverlays === 1 ? "" : "s"} to popovers`,
+      );
+    }
+    if (convertedWidgets > 0) {
+      progress(
+        "convert",
+        `Converted ${convertedWidgets} accordion${convertedWidgets === 1 ? "" : "s"} ` +
+          `(${convertedRows} rows) to native <details>`,
+      );
+    }
+
     // Layout detection
     let layout: JxElement | undefined;
     if (result.pages.length >= 2) {