@jxsuite/import 0.39.7 → 0.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -25,9 +25,35 @@ jx-import <url> [options]
25
25
  --ai-components Use an LLM to refine component names and props
26
26
  --ai-model <model> Model for AI componentization (default: gpt-4o-mini)
27
27
  --verify After import, build and screenshot-diff vs the original
28
- --verify-threshold <n> Pixel diff threshold 0..1 (default: 0.15)
28
+ --min-fidelity <n> Fail (exit 1) below this average fidelity 0..100 (default: 25)
29
+ --verify-threshold <n> Per-pixel colour tolerance 0..1 for the diff (default: 0.15)
30
+ --verify-viewport-only Diff only the first viewport instead of the whole page
29
31
  ```
30
32
 
33
+ ### What `--verify` can fail on
34
+
35
+ `--verify` builds the emitted project, serves it, screenshots every page and pixel-diffs each
36
+ against a reference captured during the import. The run exits non-zero when any of three things is
37
+ true: the project did not build cleanly, a page could not be rendered, or the average fidelity is
38
+ below `--min-fidelity`.
39
+
40
+ `--verify-threshold` is **not** that bar. It is pixelmatch's per-pixel colour tolerance: it decides
41
+ when two pixels count as the same colour, so it moves the score without ever deciding the outcome.
42
+ `--min-fidelity` is the bar, and it defaults to a deliberately low `25`, a floor for "this is not a
43
+ clone of anything" rather than a quality target. A faithful import of a complicated site still
44
+ lands well under 100 for reasons no importer can fix (a rotating hero, a font rendering a hair
45
+ differently).
46
+
47
+ `verify/report.json` carries the whole picture: per-page fidelity, the console errors and failed or
48
+ 404'd requests each rendered page produced, and any build errors. When a page scores badly, read
49
+ `failedRequests` first, because a percentage cannot tell you that fifteen images 404'd and that
50
+ list can.
51
+
52
+ **Comparing two runs.** The reference screenshot is captured fresh each time, so two runs of the
53
+ same URL are not directly comparable. On a site with a rotating hero the spread across identical
54
+ runs has been measured at around 4 points, which is wide enough to hide a small regression. Compare
55
+ a fidelity number against another run of the same code, not against one from a different session.
56
+
31
57
  Environment: `CHROME_PATH` (explicit browser binary; otherwise `google-chrome-stable`,
32
58
  `google-chrome`, `chromium-browser`, or `chromium` is discovered on PATH), `OPENAI_API_KEY` and
33
59
  `OPENAI_BASE_URL` for `--ai-components`.
@@ -52,7 +78,8 @@ const result = await importSite(
52
78
  ```
53
79
 
54
80
  Lower-level building blocks (capture, style diffing, crawling, layout detection, componentization,
55
- emit, verify) are exported from the package root — see `src/index.ts`.
81
+ emit, verify) are exported from the package root; `./capture`, `./to-jx`, `./emit`, `./run` and
82
+ `./verify` are also addressable as subpaths. See `src/index.ts`.
56
83
 
57
84
  Verification (`verify` option / `--verify`) additionally needs `@jxsuite/compiler` (an optional
58
85
  dependency) to build the emitted project before screenshot-diffing it.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@jxsuite/import",
3
- "version": "0.39.7",
3
+ "version": "0.40.0",
4
4
  "description": "Clone a live website into a Jx project",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -20,7 +20,9 @@
20
20
  "./capture": "./src/capture.ts",
21
21
  "./to-jx": "./src/to-jx.ts",
22
22
  "./emit": "./src/emit.ts",
23
- "./run": "./src/run.ts"
23
+ "./run": "./src/run.ts",
24
+ "./verify": "./src/verify.ts",
25
+ "./breakpoint-plan": "./src/breakpoint-plan.ts"
24
26
  },
25
27
  "publishConfig": {
26
28
  "provenance": true
@@ -31,8 +33,8 @@
31
33
  "import": "bun run src/cli.ts"
32
34
  },
33
35
  "dependencies": {
34
- "@jxsuite/markup": "^0.4.6",
35
- "@jxsuite/schema": "^1.8.1",
36
+ "@jxsuite/markup": "^0.4.8",
37
+ "@jxsuite/schema": "^2.0.0",
36
38
  "pixelmatch": "^7.2.0",
37
39
  "pngjs": "^7.0.0",
38
40
  "puppeteer-core": "^24.9.0"
@@ -42,6 +44,6 @@
42
44
  "happy-dom": "^20.10.6"
43
45
  },
44
46
  "optionalDependencies": {
45
- "@jxsuite/compiler": "^2.0.7"
47
+ "@jxsuite/compiler": "^3.0.0"
46
48
  }
47
49
  }
@@ -6,6 +6,7 @@
6
6
  */
7
7
 
8
8
  import type { Page } from "puppeteer-core";
9
+ import { SRCSET_SEPARATOR } from "./srcset.ts";
9
10
 
10
11
  export interface DiscoveredAsset {
11
12
  /** Absolute URL of the asset. */
@@ -46,7 +47,12 @@ export interface AssetCollectionResult {
46
47
  * page.evaluate() — no per-element round-trips.
47
48
  */
48
49
  export async function collectAssets(page: Page): Promise<AssetCollectionResult> {
49
- const result = await page.evaluate(() => {
50
+ /*
51
+ * The separator crosses into the page as a SOURCE STRING: an evaluated body is serialised and
52
+ * re-parsed in the browser, so it cannot close over a value declared out here.
53
+ */
54
+ const result = await page.evaluate((srcsetSeparator: string) => {
55
+ const separator = new RegExp(srcsetSeparator);
50
56
  const assets: { url: string; source: string }[] = [];
51
57
  const seen = new Set<string>();
52
58
  let inlineSvgCount = 0;
@@ -73,7 +79,7 @@ export async function collectAssets(page: Page): Promise<AssetCollectionResult>
73
79
 
74
80
  function parseSrcset(srcset: string): string[] {
75
81
  return srcset
76
- .split(",")
82
+ .split(separator)
77
83
  .map((entry) => entry.trim().split(/\s+/)[0] ?? "")
78
84
  .filter(Boolean);
79
85
  }
@@ -193,7 +199,7 @@ export async function collectAssets(page: Page): Promise<AssetCollectionResult>
193
199
  inlineSvgCount: number;
194
200
  stylesheets: { href: string | null; cssText: string | null; fontFaceRules: string[] }[];
195
201
  };
196
- });
202
+ }, SRCSET_SEPARATOR.source);
197
203
 
198
204
  // R2: For cross-origin sheets we couldn't access (cssText=null, has href), fetch them
199
205
  // In-browser to get @font-face rules (e.g. Google Fonts loaded via <link>)
@@ -11,8 +11,9 @@ import type { DiscoveredAsset } from "./asset-collect.ts";
11
11
 
12
12
  export interface DownloadResult {
13
13
  /**
14
- * Maps original absolute URL → relative path from project root (e.g.
15
- * "public/assets/images/hero.jpg").
14
+ * Maps the original absolute URL to the site-absolute path the built asset is served from (e.g.
15
+ * "/assets/images/hero.jpg") — NOT the path it is written to on disk, which keeps the public/
16
+ * prefix the compiler copies from.
16
17
  */
17
18
  rewriteMap: Map<string, string>;
18
19
  /** URLs that failed to download. */
@@ -172,7 +173,11 @@ export async function downloadAssets(
172
173
  usedNames.set(nameKey, count + 1);
173
174
 
174
175
  const destPath = join(assetsDir, subdir, filename);
175
- const relativePath = `public/assets/${subdir}/${filename}`;
176
+ // The site-absolute path the built asset is served from. `public/` is the compiler's static
177
+ // ROOT — its contents land at dist/<path>, so the "public/" segment must not survive into a
178
+ // Reference, and the leading slash must be there or a reference from /components/x.js or a
179
+ // Nested route resolves against the wrong base.
180
+ const relativePath = `/assets/${subdir}/${filename}`;
176
181
 
177
182
  try {
178
183
  const headers: Record<string, string> = {
@@ -6,6 +6,7 @@
6
6
  */
7
7
 
8
8
  import type { JxElement, JxStyle } from "@jxsuite/schema/types";
9
+ import { SRCSET_SEPARATOR } from "./srcset.ts";
9
10
 
10
11
  /** Rewrite all asset URLs in a Jx tree in-place. Returns the count of rewrites performed. */
11
12
  export function rewriteAssetUrls(
@@ -46,9 +47,14 @@ export function rewriteAssetUrls(
46
47
  return null;
47
48
  }
48
49
 
50
+ /*
51
+ * Split by the same rule the collector downloads by (`SRCSET_SEPARATOR`). If the two disagreed,
52
+ * the importer would fetch one set of URLs and rewrite another — the attribute would be
53
+ * reassembled around fragments nothing had downloaded.
54
+ */
49
55
  function rewriteSrcset(srcset: string): string {
50
56
  return srcset
51
- .split(",")
57
+ .split(SRCSET_SEPARATOR)
52
58
  .map((entry) => {
53
59
  const parts = entry.trim().split(/\s+/);
54
60
  const [url = ""] = parts;
@@ -0,0 +1,297 @@
1
+ /**
2
+ * Decide which of a site's breakpoints the imported project gets.
3
+ *
4
+ * A real site declares breakpoints the way it grew: one framework's four, a theme's three, a
5
+ * plugin's two, a hand-written pair for one component. Taken literally that is what the importer
6
+ * used to emit — nine `$media` entries, unsorted, named `--767` and `--1025`, each one a canvas
7
+ * size in Studio and a column in every style editor. Nobody authors against nine breakpoints, and
8
+ * nobody asked for these nine.
9
+ *
10
+ * So the widths a site DECLARES and the widths a project KEEPS are two different questions, and
11
+ * this module is the second one. It is pure — no browser, no page, no capture — because the
12
+ * decision has to be made BEFORE the re-capture pass: each kept breakpoint costs a viewport change
13
+ * and a full style walk, so planning first is what makes a limited import faster rather than merely
14
+ * tidier.
15
+ *
16
+ * Nothing is thrown away silently. Every discovered width that is not kept is FOLDED into the kept
17
+ * one nearest it under the caller's rounding rule, so the styles that width carried still reach the
18
+ * project — at the breakpoint the project actually has.
19
+ *
20
+ * @docs studio/design/breakpoints
21
+ */
22
+
23
+ /** One breakpoint: what to call it, what to emit, and the viewport width to sample it at. */
24
+ export interface Breakpoint {
25
+ /** Jx breakpoint name, e.g. `--768`. */
26
+ name: string;
27
+ /** The CSS media query to write into `project.json`'s `$media`, e.g. `(max-width: 768px)`. */
28
+ query: string;
29
+ /**
30
+ * The viewport width to re-capture styles at.
31
+ *
32
+ * Usually the query's own width, and deliberately NOT so under an explicit override: the styles
33
+ * flip at the width the SITE declared, so that is where they must be sampled, even when the
34
+ * project is going to call the breakpoint something else.
35
+ */
36
+ testWidth: number;
37
+ }
38
+
39
+ /** How a requested width is matched against the widths a site actually declares. */
40
+ export type BreakpointRounding = "nearest" | "down" | "up";
41
+
42
+ /**
43
+ * What to do with the widths a capture discovered.
44
+ *
45
+ * - `all` — keep every one, sorted. What the importer did before there was a choice.
46
+ * - `limit` — keep `count` of them, evenly spaced across the discovered range (3 → the narrowest, the
47
+ * middle one, and the widest). The default, because "too many breakpoints" is the complaint this
48
+ * exists to answer.
49
+ * - `explicit` — keep the widths the author named, each backed by the declared width nearest it.
50
+ */
51
+ export type BreakpointPolicy =
52
+ | { mode: "all" }
53
+ | { mode: "limit"; count: number; rounding: BreakpointRounding }
54
+ | { mode: "explicit"; widths: number[]; rounding: BreakpointRounding };
55
+
56
+ /** What a new import gets when nobody says otherwise. */
57
+ export const DEFAULT_BREAKPOINT_POLICY: BreakpointPolicy = {
58
+ count: 3,
59
+ mode: "limit",
60
+ rounding: "nearest",
61
+ };
62
+
63
+ /** The most breakpoints a project may be asked to keep. Past this, "limit" is not limiting. */
64
+ export const MAX_BREAKPOINTS = 12;
65
+
66
+ /** The viewport widths an override may name. Outside this a "width" is a typo, not a breakpoint. */
67
+ export const MIN_BREAKPOINT_WIDTH = 120;
68
+ export const MAX_BREAKPOINT_WIDTH = 4000;
69
+
70
+ export interface BreakpointPlanResult {
71
+ /** The breakpoints to capture and emit, ascending by `testWidth`. */
72
+ keep: Breakpoint[];
73
+ /**
74
+ * Discovered breakpoint name → the kept breakpoint name its style deltas merge into.
75
+ *
76
+ * A kept breakpoint maps to itself, so a caller can look up every discovered name without a
77
+ * fallback branch.
78
+ */
79
+ fold: Map<string, string>;
80
+ }
81
+
82
+ /**
83
+ * Parse a media query into a pixel width, or null when it is not a simple one-clause width query.
84
+ *
85
+ * Anchored and `px`-only on purpose: `(min-width: 48rem)` has no pixel width without a root font
86
+ * size, and `(min-width:768px) and (max-width:1024px)` names a band rather than a threshold. Both
87
+ * are real and both are refused here rather than guessed at — {@link skippedWidthQueries} is how the
88
+ * import reports what it could not read.
89
+ *
90
+ * @param {string} query - A media query's condition text
91
+ * @returns {{ type: "min" | "max"; width: number } | null}
92
+ */
93
+ export function parseWidthQuery(query: string): { type: "min" | "max"; width: number } | null {
94
+ const minMatch = query.match(/^\s*\(\s*min-width\s*:\s*([\d.]+)px\s*\)\s*$/);
95
+ if (minMatch) {
96
+ return { type: "min", width: Number(minMatch[1]) };
97
+ }
98
+ const maxMatch = query.match(/^\s*\(\s*max-width\s*:\s*([\d.]+)px\s*\)\s*$/);
99
+ if (maxMatch) {
100
+ return { type: "max", width: Number(maxMatch[1]) };
101
+ }
102
+ return null;
103
+ }
104
+
105
+ /** The Jx breakpoint name for a width, e.g. `768` → `--768`. */
106
+ export function breakpointName(width: number): string {
107
+ return `--${width}`;
108
+ }
109
+
110
+ /**
111
+ * Analyze discovered `@media` queries and produce a set of testable breakpoints. Only simple
112
+ * one-clause `min-width` / `max-width` queries in `px` are readable as a breakpoint;
113
+ * {@link skippedWidthQueries} names the rest, so an import can say what it could not use instead of
114
+ * dropping it in silence.
115
+ *
116
+ * @param {readonly string[]} queries
117
+ * @returns {Breakpoint[]} Ascending by width, deduplicated
118
+ */
119
+ export function analyzeMediaQueries(queries: readonly string[]): Breakpoint[] {
120
+ const seen = new Set<number>();
121
+ const breakpoints: Breakpoint[] = [];
122
+
123
+ for (const query of queries) {
124
+ const parsed = parseWidthQuery(query);
125
+ if (!parsed || seen.has(parsed.width)) {
126
+ continue;
127
+ }
128
+ seen.add(parsed.width);
129
+
130
+ // Both directions are tested AT the declared width: for max-width it is the upper bound of the
131
+ // Narrow rules, for min-width it is where the wide rules start applying.
132
+ breakpoints.push({ name: breakpointName(parsed.width), query, testWidth: parsed.width });
133
+ }
134
+
135
+ breakpoints.sort((a, b) => a.testWidth - b.testWidth);
136
+ return breakpoints;
137
+ }
138
+
139
+ /**
140
+ * The width-ish queries {@link analyzeMediaQueries} could not read — a compound condition, a `rem`
141
+ * threshold, a range syntax. A query with no width in it at all (`prefers-color-scheme`, `hover`,
142
+ * `print`) is not listed: it is not a breakpoint and its absence is not a loss.
143
+ *
144
+ * @param {readonly string[]} queries
145
+ * @returns {string[]}
146
+ */
147
+ export function skippedWidthQueries(queries: readonly string[]): string[] {
148
+ const skipped: string[] = [];
149
+ for (const query of queries) {
150
+ if (parseWidthQuery(query)) {
151
+ continue;
152
+ }
153
+ if (/\bwidth\b/i.test(query)) {
154
+ skipped.push(query);
155
+ }
156
+ }
157
+ return [...new Set(skipped)];
158
+ }
159
+
160
+ /**
161
+ * The candidate nearest `target` under `rounding`.
162
+ *
163
+ * Each rule falls back to the nearest end of the range rather than to nothing: a `down` request
164
+ * below every candidate still has an answer, and returning null there would silently drop a width
165
+ * the author explicitly asked for.
166
+ *
167
+ * @param {readonly number[]} candidates - Ascending, non-empty
168
+ * @param {number} target
169
+ * @param {BreakpointRounding} rounding
170
+ * @returns {number}
171
+ */
172
+ function pickNearest(
173
+ candidates: readonly number[],
174
+ target: number,
175
+ rounding: BreakpointRounding,
176
+ ): number {
177
+ if (rounding === "down") {
178
+ let best = candidates[0]!;
179
+ for (const width of candidates) {
180
+ if (width <= target) {
181
+ best = width;
182
+ }
183
+ }
184
+ return best;
185
+ }
186
+ if (rounding === "up") {
187
+ for (const width of candidates) {
188
+ if (width >= target) {
189
+ return width;
190
+ }
191
+ }
192
+ return candidates.at(-1)!;
193
+ }
194
+ // Nearest, ties to the narrower width — deterministic, and the narrower styles are the ones a
195
+ // Reader is more likely to have meant when two are equidistant.
196
+ let best = candidates[0]!;
197
+ let bestDelta = Math.abs(best - target);
198
+ for (const width of candidates) {
199
+ const delta = Math.abs(width - target);
200
+ if (delta < bestDelta) {
201
+ best = width;
202
+ bestDelta = delta;
203
+ }
204
+ }
205
+ return best;
206
+ }
207
+
208
+ /**
209
+ * `count` indices evenly spaced across `[0, length - 1]`, both ends included.
210
+ *
211
+ * A single breakpoint takes the MEDIAN rather than the narrowest: asked for one, an author wants
212
+ * the width the site's layout actually turns at, and the narrowest is usually a phone edge case.
213
+ */
214
+ function evenIndices(length: number, count: number): number[] {
215
+ if (count >= length) {
216
+ return [...Array.from({ length }).keys()];
217
+ }
218
+ if (count === 1) {
219
+ return [Math.floor((length - 1) / 2)];
220
+ }
221
+ const out: number[] = [];
222
+ for (let i = 0; i < count; i += 1) {
223
+ out.push(Math.round((i * (length - 1)) / (count - 1)));
224
+ }
225
+ return [...new Set(out)];
226
+ }
227
+
228
+ /**
229
+ * Choose the breakpoints an import keeps, and where the rest of them go.
230
+ *
231
+ * @param {readonly Breakpoint[]} discovered - Everything the capture found, in any order
232
+ * @param {BreakpointPolicy} policy
233
+ * @returns {BreakpointPlanResult}
234
+ */
235
+ export function planBreakpoints(
236
+ discovered: readonly Breakpoint[],
237
+ policy: BreakpointPolicy = DEFAULT_BREAKPOINT_POLICY,
238
+ ): BreakpointPlanResult {
239
+ const sorted = discovered.toSorted((a, b) => a.testWidth - b.testWidth);
240
+ if (sorted.length === 0) {
241
+ return { fold: new Map(), keep: [] };
242
+ }
243
+
244
+ const widths = sorted.map((bp) => bp.testWidth);
245
+ const byWidth = new Map(sorted.map((bp) => [bp.testWidth, bp]));
246
+
247
+ const keep: Breakpoint[] = [];
248
+ if (policy.mode === "all") {
249
+ keep.push(...sorted);
250
+ } else if (policy.mode === "limit") {
251
+ const count = Math.max(1, Math.min(MAX_BREAKPOINTS, Math.trunc(policy.count) || 1));
252
+ for (const index of evenIndices(sorted.length, count)) {
253
+ keep.push(sorted[index]!);
254
+ }
255
+ } else {
256
+ /* An override names the widths the PROJECT will have; each is backed by the declared width
257
+ nearest it, because that is where the site's own rules actually change. Two overrides that
258
+ land on the same declared width collapse into one — emitting the same styles twice under two
259
+ names would be two canvases showing the same thing. */
260
+ const requested = [...new Set(policy.widths.map((w) => Math.trunc(w)))]
261
+ .filter((w) => w >= MIN_BREAKPOINT_WIDTH && w <= MAX_BREAKPOINT_WIDTH)
262
+ .toSorted((a, b) => a - b)
263
+ .slice(0, MAX_BREAKPOINTS);
264
+ const claimed = new Set<number>();
265
+ for (const target of requested) {
266
+ const backing = pickNearest(widths, target, policy.rounding);
267
+ if (claimed.has(backing)) {
268
+ continue;
269
+ }
270
+ claimed.add(backing);
271
+ const source = byWidth.get(backing)!;
272
+ const parsed = parseWidthQuery(source.query);
273
+ keep.push({
274
+ name: breakpointName(target),
275
+ query: parsed ? `(${parsed.type}-width: ${target}px)` : source.query,
276
+ testWidth: backing,
277
+ });
278
+ }
279
+ keep.splice(0, keep.length, ...keep.toSorted((a, b) => a.testWidth - b.testWidth));
280
+ }
281
+
282
+ const fold = new Map<string, string>();
283
+ if (keep.length === 0) {
284
+ return { fold, keep };
285
+ }
286
+
287
+ const keptWidths = keep.map((bp) => bp.testWidth);
288
+ const keptByWidth = new Map(keep.map((bp) => [bp.testWidth, bp]));
289
+ const rounding = policy.mode === "all" ? "nearest" : policy.rounding;
290
+ for (const bp of sorted) {
291
+ const target =
292
+ keptByWidth.get(bp.testWidth) ??
293
+ keptByWidth.get(pickNearest(keptWidths, bp.testWidth, rounding))!;
294
+ fold.set(bp.name, target.name);
295
+ }
296
+ return { fold, keep };
297
+ }
package/src/cli.ts CHANGED
@@ -3,6 +3,13 @@
3
3
  import { resolve } from "node:path";
4
4
  import { homedir } from "node:os";
5
5
  import { importSite } from "./run.ts";
6
+ import {
7
+ DEFAULT_BREAKPOINT_POLICY,
8
+ MAX_BREAKPOINTS,
9
+ MAX_BREAKPOINT_WIDTH,
10
+ MIN_BREAKPOINT_WIDTH,
11
+ } from "./breakpoint-plan.ts";
12
+ import type { BreakpointPolicy, BreakpointRounding } from "./breakpoint-plan.ts";
6
13
 
7
14
  function usage(): never {
8
15
  console.log(`Usage: jx-import <url> [options]
@@ -14,6 +21,10 @@ Options:
14
21
  --depth <n> Max crawl depth (default: 2, 0 = single page)
15
22
  --max-pages <n> Max pages to capture (default: 25)
16
23
  --max-nodes-per-page <n> Skip styles/assets for pages above this (default: 5000)
24
+ --max-breakpoints <n> Keep at most n of the site's breakpoints (default: 3, 0 = keep all)
25
+ --breakpoints <list> Keep these widths instead, e.g. 640,1024,1440
26
+ --breakpoint-rounding <r> How a kept width matches a declared one: nearest|down|up
27
+ (default: nearest)
17
28
  --no-styles Skip CSS capture
18
29
  --no-assets Skip asset download
19
30
  --no-crawl Single page only (equivalent to --depth 0)
@@ -25,7 +36,10 @@ Options:
25
36
  --ai-components Use LLM to refine component names and props (Phase 4 AI pass)
26
37
  --ai-model <model> Model for AI componentization (default: gpt-4o-mini)
27
38
  --verify After import, build and screenshot-diff vs original (Phase 5)
28
- --verify-threshold <n> Pixel diff threshold 0..1 (default: 0.15)
39
+ --min-fidelity <n> Fail (exit 1) below this average fidelity 0..100 (default: 25)
40
+ --verify-threshold <n> Per-pixel colour tolerance 0..1 for the diff (default: 0.15).
41
+ Moves the score; it is NOT the pass bar — --min-fidelity is.
42
+ --verify-viewport-only Diff only the first viewport instead of the whole page
29
43
 
30
44
  Environment:
31
45
  OPENAI_API_KEY Required for --ai-components
@@ -37,6 +51,8 @@ Examples:
37
51
  jx-import https://example.com --depth 1 --max-pages 10
38
52
  jx-import https://example.com --no-crawl
39
53
  jx-import https://example.com --out sites/my-clone --no-styles
54
+ jx-import https://example.com --breakpoints 640,1024,1440
55
+ jx-import https://example.com --max-breakpoints 0
40
56
  jx-import https://example.com --verify
41
57
  jx-import https://example.com --ai-components`);
42
58
  process.exit(1);
@@ -73,7 +89,60 @@ const verifyThreshold =
73
89
  verifyThresholdIdx !== -1 && args[verifyThresholdIdx + 1]
74
90
  ? Number(args[verifyThresholdIdx + 1]) || 0.15
75
91
  : 0.15;
92
+ /*
93
+ * The floor below which the output is not a clone of anything.
94
+ *
95
+ * It is deliberately low rather than a quality bar: a real import of a complicated site lands well
96
+ * under 100 for reasons nobody can fix from here (a rotating hero, a font that renders a hair
97
+ * differently), and failing those would train people to pass `--min-fidelity 0` and stop looking.
98
+ * Under a quarter of pixels matching is a different question — that is the 8%-fidelity Wix clone
99
+ * that printed `Done!` and exited 0, which is worse than a failure because the only way to see it
100
+ * is to open the result (issue #232).
101
+ */
102
+ const minFidelity = parseIntArg(args, "--min-fidelity", 25);
103
+ const verifyFullPage = !args.includes("--verify-viewport-only");
76
104
 
105
+ /**
106
+ * The breakpoint policy from the flags.
107
+ *
108
+ * `--breakpoints` wins over `--max-breakpoints` because it is the more specific answer: naming the
109
+ * widths says everything a count could have.
110
+ */
111
+ function breakpointPolicy(): BreakpointPolicy {
112
+ const roundingIdx = args.indexOf("--breakpoint-rounding");
113
+ const roundingArg = roundingIdx === -1 ? "" : (args[roundingIdx + 1] ?? "");
114
+ const rounding: BreakpointRounding =
115
+ roundingArg === "down" || roundingArg === "up" ? roundingArg : "nearest";
116
+
117
+ const widthsIdx = args.indexOf("--breakpoints");
118
+ const widthsArg = widthsIdx === -1 ? "" : (args[widthsIdx + 1] ?? "");
119
+ if (widthsArg) {
120
+ const widths = widthsArg
121
+ .split(",")
122
+ .map((part) => Math.trunc(Number(part.trim())))
123
+ .filter((n) => Number.isFinite(n) && n >= MIN_BREAKPOINT_WIDTH && n <= MAX_BREAKPOINT_WIDTH);
124
+ if (widths.length === 0) {
125
+ console.error(
126
+ `Error: --breakpoints needs widths between ${MIN_BREAKPOINT_WIDTH} and ` +
127
+ `${MAX_BREAKPOINT_WIDTH}, e.g. --breakpoints 640,1024,1440`,
128
+ );
129
+ process.exit(1);
130
+ }
131
+ return { mode: "explicit", rounding, widths };
132
+ }
133
+
134
+ if (!args.includes("--max-breakpoints")) {
135
+ return DEFAULT_BREAKPOINT_POLICY;
136
+ }
137
+ const count = parseIntArg(args, "--max-breakpoints", 3);
138
+ // Zero is the only spelling of "keep all" a count can have, and it reads better than omitting the
139
+ // Flag when a script is choosing between the two.
140
+ return count <= 0
141
+ ? { mode: "all" }
142
+ : { count: Math.min(MAX_BREAKPOINTS, count), mode: "limit", rounding };
143
+ }
144
+
145
+ const breakpoints = breakpointPolicy();
77
146
  const maxDepth = noCrawl ? 0 : parseIntArg(args, "--depth", 2);
78
147
  const maxPages = parseIntArg(args, "--max-pages", 25);
79
148
  const maxNodesPerPage = parseIntArg(args, "--max-nodes-per-page", 5000);
@@ -116,6 +185,7 @@ async function main() {
116
185
  {
117
186
  url: url as string,
118
187
  outDir,
188
+ breakpoints,
119
189
  maxDepth,
120
190
  maxPages,
121
191
  maxNodesPerPage,
@@ -125,7 +195,9 @@ async function main() {
125
195
  respectRobots: !args.includes("--no-robots"),
126
196
  componentize: noComponents ? false : { minInstances, minDepth },
127
197
  ai,
128
- verify: doVerify ? { threshold: verifyThreshold } : false,
198
+ verify: doVerify
199
+ ? { threshold: verifyThreshold, minFidelity, fullPage: verifyFullPage }
200
+ : false,
129
201
  },
130
202
  (e) => console.log(` ${e.message}`),
131
203
  );
@@ -136,9 +208,41 @@ async function main() {
136
208
  }
137
209
  if (result.verify) {
138
210
  console.log(` Average fidelity: ${result.verify.averageFidelity}%`);
211
+ for (const page of result.verify.pages) {
212
+ const misses = [
213
+ page.failedRequests > 0 ? `${page.failedRequests} failed request(s)` : "",
214
+ page.consoleErrors > 0 ? `${page.consoleErrors} console error(s)` : "",
215
+ ].filter(Boolean);
216
+ if (misses.length > 0) {
217
+ console.log(` ${page.route}: ${misses.join(", ")}`);
218
+ }
219
+ }
139
220
  console.log(` Report: ${result.verify.reportDir}/report.json`);
140
221
  }
141
222
 
223
+ /*
224
+ * A verify that cannot fail is a report, not a gate. An import nobody opens and looks at is
225
+ * exactly the case the exit code exists for, so a build error or a fidelity under the floor
226
+ * ends the run non-zero and says which one it was.
227
+ */
228
+ if (result.verify && !result.verify.passed) {
229
+ for (const error of result.verify.buildErrors) {
230
+ console.error(`Build error: ${error}`);
231
+ }
232
+ console.error(
233
+ `\nVerification FAILED — the project is at ${outDir}, but it does not match the original.`,
234
+ );
235
+ if (result.verify.buildErrors.length === 0) {
236
+ console.error(
237
+ ` Average fidelity ${result.verify.averageFidelity}% is below the ` +
238
+ `${result.verify.minFidelity}% minimum. Pass --min-fidelity to change it.`,
239
+ );
240
+ }
241
+ console.error(` Report: ${result.verify.reportDir}/report.json`);
242
+ process.exitCode = 1;
243
+ return;
244
+ }
245
+
142
246
  console.log(`\nDone! Open in Studio:`);
143
247
  console.log(
144
248
  ` http://localhost:3000/packages/studio/index.html?project=${outDir}/project.json`,
@@ -54,6 +54,16 @@ function structuralHash(node: JxElement | string): string {
54
54
  return `<${tag}>${childHashes.join(",")}`;
55
55
  }
56
56
 
57
+ /**
58
+ * Attributes that may not become a component prop.
59
+ *
60
+ * A prop is something a person fills in per instance, and a class is not one: `emit.ts` strips the
61
+ * source site's classes on the way out (`strip-classes.ts`), so a class that happened to differ
62
+ * between two instances of the same block would otherwise mint a `${state.class}` prop whose value
63
+ * is then deleted — a prop with a name nobody chose, wired to nothing.
64
+ */
65
+ const PRESENTATION_ATTRS = new Set(["class", "className"]);
66
+
57
67
  function collectLeafValues(node: JxElement | string, prefix: string, out: Map<string, string>) {
58
68
  if (typeof node === "string") {
59
69
  out.set(`${prefix}.$text`, node);
@@ -64,6 +74,9 @@ function collectLeafValues(node: JxElement | string, prefix: string, out: Map<st
64
74
  }
65
75
  if (node.attributes) {
66
76
  for (const [k, v] of Object.entries(node.attributes)) {
77
+ if (PRESENTATION_ATTRS.has(k)) {
78
+ continue;
79
+ }
67
80
  if (typeof v === "string" || typeof v === "number" || typeof v === "boolean") {
68
81
  out.set(`${prefix}.attr.${k}`, String(v));
69
82
  }