@jxsuite/import 0.39.8 → 0.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -2
- package/package.json +7 -5
- package/src/asset-collect.ts +9 -3
- package/src/asset-download.ts +8 -3
- package/src/asset-rewrite.ts +7 -1
- package/src/breakpoint-plan.ts +297 -0
- package/src/cli.ts +106 -2
- package/src/componentize.ts +13 -0
- package/src/crawl.ts +40 -2
- package/src/emit.ts +104 -11
- package/src/index.ts +21 -3
- package/src/media-extract.ts +51 -75
- package/src/run.ts +227 -12
- package/src/srcset.ts +32 -0
- package/src/strip-classes.ts +73 -0
- package/src/verify.ts +165 -13
package/README.md
CHANGED
|
@@ -25,9 +25,35 @@ jx-import <url> [options]
|
|
|
25
25
|
--ai-components Use an LLM to refine component names and props
|
|
26
26
|
--ai-model <model> Model for AI componentization (default: gpt-4o-mini)
|
|
27
27
|
--verify After import, build and screenshot-diff vs the original
|
|
28
|
-
--
|
|
28
|
+
--min-fidelity <n> Fail (exit 1) below this average fidelity 0..100 (default: 25)
|
|
29
|
+
--verify-threshold <n> Per-pixel colour tolerance 0..1 for the diff (default: 0.15)
|
|
30
|
+
--verify-viewport-only Diff only the first viewport instead of the whole page
|
|
29
31
|
```
|
|
30
32
|
|
|
33
|
+
### What `--verify` can fail on
|
|
34
|
+
|
|
35
|
+
`--verify` builds the emitted project, serves it, screenshots every page and pixel-diffs each
|
|
36
|
+
against a reference captured during the import. The run exits non-zero when any of three things is
|
|
37
|
+
true: the project did not build cleanly, a page could not be rendered, or the average fidelity is
|
|
38
|
+
below `--min-fidelity`.
|
|
39
|
+
|
|
40
|
+
`--verify-threshold` is **not** that bar. It is pixelmatch's per-pixel colour tolerance: it decides
|
|
41
|
+
when two pixels count as the same colour, so it moves the score without ever deciding the outcome.
|
|
42
|
+
`--min-fidelity` is the bar, and it defaults to a deliberately low `25`, a floor for "this is not a
|
|
43
|
+
clone of anything" rather than a quality target. A faithful import of a complicated site still
|
|
44
|
+
lands well under 100 for reasons no importer can fix (a rotating hero, a font rendering a hair
|
|
45
|
+
differently).
|
|
46
|
+
|
|
47
|
+
`verify/report.json` carries the whole picture: per-page fidelity, the console errors and failed or
|
|
48
|
+
404'd requests each rendered page produced, and any build errors. When a page scores badly, read
|
|
49
|
+
`failedRequests` first, because a percentage cannot tell you that fifteen images 404'd and that
|
|
50
|
+
list can.
|
|
51
|
+
|
|
52
|
+
**Comparing two runs.** The reference screenshot is captured fresh each time, so two runs of the
|
|
53
|
+
same URL are not directly comparable. On a site with a rotating hero the spread across identical
|
|
54
|
+
runs has been measured at around 4 points, which is wide enough to hide a small regression. Compare
|
|
55
|
+
a fidelity number against another run of the same code, not against one from a different session.
|
|
56
|
+
|
|
31
57
|
Environment: `CHROME_PATH` (explicit browser binary; otherwise `google-chrome-stable`,
|
|
32
58
|
`google-chrome`, `chromium-browser`, or `chromium` is discovered on PATH), `OPENAI_API_KEY` and
|
|
33
59
|
`OPENAI_BASE_URL` for `--ai-components`.
|
|
@@ -52,7 +78,8 @@ const result = await importSite(
|
|
|
52
78
|
```
|
|
53
79
|
|
|
54
80
|
Lower-level building blocks (capture, style diffing, crawling, layout detection, componentization,
|
|
55
|
-
emit, verify) are exported from the package root
|
|
81
|
+
emit, verify) are exported from the package root; `./capture`, `./to-jx`, `./emit`, `./run` and
|
|
82
|
+
`./verify` are also addressable as subpaths. See `src/index.ts`.
|
|
56
83
|
|
|
57
84
|
Verification (`verify` option / `--verify`) additionally needs `@jxsuite/compiler` (an optional
|
|
58
85
|
dependency) to build the emitted project before screenshot-diffing it.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jxsuite/import",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.40.0",
|
|
4
4
|
"description": "Clone a live website into a Jx project",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -20,7 +20,9 @@
|
|
|
20
20
|
"./capture": "./src/capture.ts",
|
|
21
21
|
"./to-jx": "./src/to-jx.ts",
|
|
22
22
|
"./emit": "./src/emit.ts",
|
|
23
|
-
"./run": "./src/run.ts"
|
|
23
|
+
"./run": "./src/run.ts",
|
|
24
|
+
"./verify": "./src/verify.ts",
|
|
25
|
+
"./breakpoint-plan": "./src/breakpoint-plan.ts"
|
|
24
26
|
},
|
|
25
27
|
"publishConfig": {
|
|
26
28
|
"provenance": true
|
|
@@ -31,8 +33,8 @@
|
|
|
31
33
|
"import": "bun run src/cli.ts"
|
|
32
34
|
},
|
|
33
35
|
"dependencies": {
|
|
34
|
-
"@jxsuite/markup": "^0.4.
|
|
35
|
-
"@jxsuite/schema": "^
|
|
36
|
+
"@jxsuite/markup": "^0.4.8",
|
|
37
|
+
"@jxsuite/schema": "^2.0.0",
|
|
36
38
|
"pixelmatch": "^7.2.0",
|
|
37
39
|
"pngjs": "^7.0.0",
|
|
38
40
|
"puppeteer-core": "^24.9.0"
|
|
@@ -42,6 +44,6 @@
|
|
|
42
44
|
"happy-dom": "^20.10.6"
|
|
43
45
|
},
|
|
44
46
|
"optionalDependencies": {
|
|
45
|
-
"@jxsuite/compiler": "^
|
|
47
|
+
"@jxsuite/compiler": "^3.0.0"
|
|
46
48
|
}
|
|
47
49
|
}
|
package/src/asset-collect.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import type { Page } from "puppeteer-core";
|
|
9
|
+
import { SRCSET_SEPARATOR } from "./srcset.ts";
|
|
9
10
|
|
|
10
11
|
export interface DiscoveredAsset {
|
|
11
12
|
/** Absolute URL of the asset. */
|
|
@@ -46,7 +47,12 @@ export interface AssetCollectionResult {
|
|
|
46
47
|
* page.evaluate() — no per-element round-trips.
|
|
47
48
|
*/
|
|
48
49
|
export async function collectAssets(page: Page): Promise<AssetCollectionResult> {
|
|
49
|
-
|
|
50
|
+
/*
|
|
51
|
+
* The separator crosses into the page as a SOURCE STRING: an evaluated body is serialised and
|
|
52
|
+
* re-parsed in the browser, so it cannot close over a value declared out here.
|
|
53
|
+
*/
|
|
54
|
+
const result = await page.evaluate((srcsetSeparator: string) => {
|
|
55
|
+
const separator = new RegExp(srcsetSeparator);
|
|
50
56
|
const assets: { url: string; source: string }[] = [];
|
|
51
57
|
const seen = new Set<string>();
|
|
52
58
|
let inlineSvgCount = 0;
|
|
@@ -73,7 +79,7 @@ export async function collectAssets(page: Page): Promise<AssetCollectionResult>
|
|
|
73
79
|
|
|
74
80
|
function parseSrcset(srcset: string): string[] {
|
|
75
81
|
return srcset
|
|
76
|
-
.split(
|
|
82
|
+
.split(separator)
|
|
77
83
|
.map((entry) => entry.trim().split(/\s+/)[0] ?? "")
|
|
78
84
|
.filter(Boolean);
|
|
79
85
|
}
|
|
@@ -193,7 +199,7 @@ export async function collectAssets(page: Page): Promise<AssetCollectionResult>
|
|
|
193
199
|
inlineSvgCount: number;
|
|
194
200
|
stylesheets: { href: string | null; cssText: string | null; fontFaceRules: string[] }[];
|
|
195
201
|
};
|
|
196
|
-
});
|
|
202
|
+
}, SRCSET_SEPARATOR.source);
|
|
197
203
|
|
|
198
204
|
// R2: For cross-origin sheets we couldn't access (cssText=null, has href), fetch them
|
|
199
205
|
// In-browser to get @font-face rules (e.g. Google Fonts loaded via <link>)
|
package/src/asset-download.ts
CHANGED
|
@@ -11,8 +11,9 @@ import type { DiscoveredAsset } from "./asset-collect.ts";
|
|
|
11
11
|
|
|
12
12
|
export interface DownloadResult {
|
|
13
13
|
/**
|
|
14
|
-
* Maps original absolute URL
|
|
15
|
-
* "
|
|
14
|
+
* Maps the original absolute URL to the site-absolute path the built asset is served from (e.g.
|
|
15
|
+
* "/assets/images/hero.jpg") — NOT the path it is written to on disk, which keeps the public/
|
|
16
|
+
* prefix the compiler copies from.
|
|
16
17
|
*/
|
|
17
18
|
rewriteMap: Map<string, string>;
|
|
18
19
|
/** URLs that failed to download. */
|
|
@@ -172,7 +173,11 @@ export async function downloadAssets(
|
|
|
172
173
|
usedNames.set(nameKey, count + 1);
|
|
173
174
|
|
|
174
175
|
const destPath = join(assetsDir, subdir, filename);
|
|
175
|
-
|
|
176
|
+
// The site-absolute path the built asset is served from. `public/` is the compiler's static
|
|
177
|
+
// ROOT — its contents land at dist/<path>, so the "public/" segment must not survive into a
|
|
178
|
+
// Reference, and the leading slash must be there or a reference from /components/x.js or a
|
|
179
|
+
// Nested route resolves against the wrong base.
|
|
180
|
+
const relativePath = `/assets/${subdir}/${filename}`;
|
|
176
181
|
|
|
177
182
|
try {
|
|
178
183
|
const headers: Record<string, string> = {
|
package/src/asset-rewrite.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import type { JxElement, JxStyle } from "@jxsuite/schema/types";
|
|
9
|
+
import { SRCSET_SEPARATOR } from "./srcset.ts";
|
|
9
10
|
|
|
10
11
|
/** Rewrite all asset URLs in a Jx tree in-place. Returns the count of rewrites performed. */
|
|
11
12
|
export function rewriteAssetUrls(
|
|
@@ -46,9 +47,14 @@ export function rewriteAssetUrls(
|
|
|
46
47
|
return null;
|
|
47
48
|
}
|
|
48
49
|
|
|
50
|
+
/*
|
|
51
|
+
* Split by the same rule the collector downloads by (`SRCSET_SEPARATOR`). If the two disagreed,
|
|
52
|
+
* the importer would fetch one set of URLs and rewrite another — the attribute would be
|
|
53
|
+
* reassembled around fragments nothing had downloaded.
|
|
54
|
+
*/
|
|
49
55
|
function rewriteSrcset(srcset: string): string {
|
|
50
56
|
return srcset
|
|
51
|
-
.split(
|
|
57
|
+
.split(SRCSET_SEPARATOR)
|
|
52
58
|
.map((entry) => {
|
|
53
59
|
const parts = entry.trim().split(/\s+/);
|
|
54
60
|
const [url = ""] = parts;
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decide which of a site's breakpoints the imported project gets.
|
|
3
|
+
*
|
|
4
|
+
* A real site declares breakpoints the way it grew: one framework's four, a theme's three, a
|
|
5
|
+
* plugin's two, a hand-written pair for one component. Taken literally that is what the importer
|
|
6
|
+
* used to emit — nine `$media` entries, unsorted, named `--767` and `--1025`, each one a canvas
|
|
7
|
+
* size in Studio and a column in every style editor. Nobody authors against nine breakpoints, and
|
|
8
|
+
* nobody asked for these nine.
|
|
9
|
+
*
|
|
10
|
+
* So the widths a site DECLARES and the widths a project KEEPS are two different questions, and
|
|
11
|
+
* this module is the second one. It is pure — no browser, no page, no capture — because the
|
|
12
|
+
* decision has to be made BEFORE the re-capture pass: each kept breakpoint costs a viewport change
|
|
13
|
+
* and a full style walk, so planning first is what makes a limited import faster rather than merely
|
|
14
|
+
* tidier.
|
|
15
|
+
*
|
|
16
|
+
* Nothing is thrown away silently. Every discovered width that is not kept is FOLDED into the kept
|
|
17
|
+
* one nearest it under the caller's rounding rule, so the styles that width carried still reach the
|
|
18
|
+
* project — at the breakpoint the project actually has.
|
|
19
|
+
*
|
|
20
|
+
* @docs studio/design/breakpoints
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/** One breakpoint: what to call it, what to emit, and the viewport width to sample it at. */
|
|
24
|
+
export interface Breakpoint {
|
|
25
|
+
/** Jx breakpoint name, e.g. `--768`. */
|
|
26
|
+
name: string;
|
|
27
|
+
/** The CSS media query to write into `project.json`'s `$media`, e.g. `(max-width: 768px)`. */
|
|
28
|
+
query: string;
|
|
29
|
+
/**
|
|
30
|
+
* The viewport width to re-capture styles at.
|
|
31
|
+
*
|
|
32
|
+
* Usually the query's own width, and deliberately NOT so under an explicit override: the styles
|
|
33
|
+
* flip at the width the SITE declared, so that is where they must be sampled, even when the
|
|
34
|
+
* project is going to call the breakpoint something else.
|
|
35
|
+
*/
|
|
36
|
+
testWidth: number;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** How a requested width is matched against the widths a site actually declares. */
|
|
40
|
+
export type BreakpointRounding = "nearest" | "down" | "up";
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* What to do with the widths a capture discovered.
|
|
44
|
+
*
|
|
45
|
+
* - `all` — keep every one, sorted. What the importer did before there was a choice.
|
|
46
|
+
* - `limit` — keep `count` of them, evenly spaced across the discovered range (3 → the narrowest, the
|
|
47
|
+
* middle one, and the widest). The default, because "too many breakpoints" is the complaint this
|
|
48
|
+
* exists to answer.
|
|
49
|
+
* - `explicit` — keep the widths the author named, each backed by the declared width nearest it.
|
|
50
|
+
*/
|
|
51
|
+
export type BreakpointPolicy =
|
|
52
|
+
| { mode: "all" }
|
|
53
|
+
| { mode: "limit"; count: number; rounding: BreakpointRounding }
|
|
54
|
+
| { mode: "explicit"; widths: number[]; rounding: BreakpointRounding };
|
|
55
|
+
|
|
56
|
+
/** What a new import gets when nobody says otherwise. */
|
|
57
|
+
export const DEFAULT_BREAKPOINT_POLICY: BreakpointPolicy = {
|
|
58
|
+
count: 3,
|
|
59
|
+
mode: "limit",
|
|
60
|
+
rounding: "nearest",
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
/** The most breakpoints a project may be asked to keep. Past this, "limit" is not limiting. */
|
|
64
|
+
export const MAX_BREAKPOINTS = 12;
|
|
65
|
+
|
|
66
|
+
/** The viewport widths an override may name. Outside this a "width" is a typo, not a breakpoint. */
|
|
67
|
+
export const MIN_BREAKPOINT_WIDTH = 120;
|
|
68
|
+
export const MAX_BREAKPOINT_WIDTH = 4000;
|
|
69
|
+
|
|
70
|
+
export interface BreakpointPlanResult {
|
|
71
|
+
/** The breakpoints to capture and emit, ascending by `testWidth`. */
|
|
72
|
+
keep: Breakpoint[];
|
|
73
|
+
/**
|
|
74
|
+
* Discovered breakpoint name → the kept breakpoint name its style deltas merge into.
|
|
75
|
+
*
|
|
76
|
+
* A kept breakpoint maps to itself, so a caller can look up every discovered name without a
|
|
77
|
+
* fallback branch.
|
|
78
|
+
*/
|
|
79
|
+
fold: Map<string, string>;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Parse a media query into a pixel width, or null when it is not a simple one-clause width query.
|
|
84
|
+
*
|
|
85
|
+
* Anchored and `px`-only on purpose: `(min-width: 48rem)` has no pixel width without a root font
|
|
86
|
+
* size, and `(min-width:768px) and (max-width:1024px)` names a band rather than a threshold. Both
|
|
87
|
+
* are real and both are refused here rather than guessed at — {@link skippedWidthQueries} is how the
|
|
88
|
+
* import reports what it could not read.
|
|
89
|
+
*
|
|
90
|
+
* @param {string} query - A media query's condition text
|
|
91
|
+
* @returns {{ type: "min" | "max"; width: number } | null}
|
|
92
|
+
*/
|
|
93
|
+
export function parseWidthQuery(query: string): { type: "min" | "max"; width: number } | null {
|
|
94
|
+
const minMatch = query.match(/^\s*\(\s*min-width\s*:\s*([\d.]+)px\s*\)\s*$/);
|
|
95
|
+
if (minMatch) {
|
|
96
|
+
return { type: "min", width: Number(minMatch[1]) };
|
|
97
|
+
}
|
|
98
|
+
const maxMatch = query.match(/^\s*\(\s*max-width\s*:\s*([\d.]+)px\s*\)\s*$/);
|
|
99
|
+
if (maxMatch) {
|
|
100
|
+
return { type: "max", width: Number(maxMatch[1]) };
|
|
101
|
+
}
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** The Jx breakpoint name for a width, e.g. `768` → `--768`. */
|
|
106
|
+
export function breakpointName(width: number): string {
|
|
107
|
+
return `--${width}`;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Analyze discovered `@media` queries and produce a set of testable breakpoints. Only simple
|
|
112
|
+
* one-clause `min-width` / `max-width` queries in `px` are readable as a breakpoint;
|
|
113
|
+
* {@link skippedWidthQueries} names the rest, so an import can say what it could not use instead of
|
|
114
|
+
* dropping it in silence.
|
|
115
|
+
*
|
|
116
|
+
* @param {readonly string[]} queries
|
|
117
|
+
* @returns {Breakpoint[]} Ascending by width, deduplicated
|
|
118
|
+
*/
|
|
119
|
+
export function analyzeMediaQueries(queries: readonly string[]): Breakpoint[] {
|
|
120
|
+
const seen = new Set<number>();
|
|
121
|
+
const breakpoints: Breakpoint[] = [];
|
|
122
|
+
|
|
123
|
+
for (const query of queries) {
|
|
124
|
+
const parsed = parseWidthQuery(query);
|
|
125
|
+
if (!parsed || seen.has(parsed.width)) {
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
seen.add(parsed.width);
|
|
129
|
+
|
|
130
|
+
// Both directions are tested AT the declared width: for max-width it is the upper bound of the
|
|
131
|
+
// Narrow rules, for min-width it is where the wide rules start applying.
|
|
132
|
+
breakpoints.push({ name: breakpointName(parsed.width), query, testWidth: parsed.width });
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
breakpoints.sort((a, b) => a.testWidth - b.testWidth);
|
|
136
|
+
return breakpoints;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* The width-ish queries {@link analyzeMediaQueries} could not read — a compound condition, a `rem`
|
|
141
|
+
* threshold, a range syntax. A query with no width in it at all (`prefers-color-scheme`, `hover`,
|
|
142
|
+
* `print`) is not listed: it is not a breakpoint and its absence is not a loss.
|
|
143
|
+
*
|
|
144
|
+
* @param {readonly string[]} queries
|
|
145
|
+
* @returns {string[]}
|
|
146
|
+
*/
|
|
147
|
+
export function skippedWidthQueries(queries: readonly string[]): string[] {
|
|
148
|
+
const skipped: string[] = [];
|
|
149
|
+
for (const query of queries) {
|
|
150
|
+
if (parseWidthQuery(query)) {
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
if (/\bwidth\b/i.test(query)) {
|
|
154
|
+
skipped.push(query);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return [...new Set(skipped)];
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* The candidate nearest `target` under `rounding`.
|
|
162
|
+
*
|
|
163
|
+
* Each rule falls back to the nearest end of the range rather than to nothing: a `down` request
|
|
164
|
+
* below every candidate still has an answer, and returning null there would silently drop a width
|
|
165
|
+
* the author explicitly asked for.
|
|
166
|
+
*
|
|
167
|
+
* @param {readonly number[]} candidates - Ascending, non-empty
|
|
168
|
+
* @param {number} target
|
|
169
|
+
* @param {BreakpointRounding} rounding
|
|
170
|
+
* @returns {number}
|
|
171
|
+
*/
|
|
172
|
+
function pickNearest(
|
|
173
|
+
candidates: readonly number[],
|
|
174
|
+
target: number,
|
|
175
|
+
rounding: BreakpointRounding,
|
|
176
|
+
): number {
|
|
177
|
+
if (rounding === "down") {
|
|
178
|
+
let best = candidates[0]!;
|
|
179
|
+
for (const width of candidates) {
|
|
180
|
+
if (width <= target) {
|
|
181
|
+
best = width;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
return best;
|
|
185
|
+
}
|
|
186
|
+
if (rounding === "up") {
|
|
187
|
+
for (const width of candidates) {
|
|
188
|
+
if (width >= target) {
|
|
189
|
+
return width;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
return candidates.at(-1)!;
|
|
193
|
+
}
|
|
194
|
+
// Nearest, ties to the narrower width — deterministic, and the narrower styles are the ones a
|
|
195
|
+
// Reader is more likely to have meant when two are equidistant.
|
|
196
|
+
let best = candidates[0]!;
|
|
197
|
+
let bestDelta = Math.abs(best - target);
|
|
198
|
+
for (const width of candidates) {
|
|
199
|
+
const delta = Math.abs(width - target);
|
|
200
|
+
if (delta < bestDelta) {
|
|
201
|
+
best = width;
|
|
202
|
+
bestDelta = delta;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return best;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* `count` indices evenly spaced across `[0, length - 1]`, both ends included.
|
|
210
|
+
*
|
|
211
|
+
* A single breakpoint takes the MEDIAN rather than the narrowest: asked for one, an author wants
|
|
212
|
+
* the width the site's layout actually turns at, and the narrowest is usually a phone edge case.
|
|
213
|
+
*/
|
|
214
|
+
function evenIndices(length: number, count: number): number[] {
|
|
215
|
+
if (count >= length) {
|
|
216
|
+
return [...Array.from({ length }).keys()];
|
|
217
|
+
}
|
|
218
|
+
if (count === 1) {
|
|
219
|
+
return [Math.floor((length - 1) / 2)];
|
|
220
|
+
}
|
|
221
|
+
const out: number[] = [];
|
|
222
|
+
for (let i = 0; i < count; i += 1) {
|
|
223
|
+
out.push(Math.round((i * (length - 1)) / (count - 1)));
|
|
224
|
+
}
|
|
225
|
+
return [...new Set(out)];
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* Choose the breakpoints an import keeps, and where the rest of them go.
|
|
230
|
+
*
|
|
231
|
+
* @param {readonly Breakpoint[]} discovered - Everything the capture found, in any order
|
|
232
|
+
* @param {BreakpointPolicy} policy
|
|
233
|
+
* @returns {BreakpointPlanResult}
|
|
234
|
+
*/
|
|
235
|
+
export function planBreakpoints(
|
|
236
|
+
discovered: readonly Breakpoint[],
|
|
237
|
+
policy: BreakpointPolicy = DEFAULT_BREAKPOINT_POLICY,
|
|
238
|
+
): BreakpointPlanResult {
|
|
239
|
+
const sorted = discovered.toSorted((a, b) => a.testWidth - b.testWidth);
|
|
240
|
+
if (sorted.length === 0) {
|
|
241
|
+
return { fold: new Map(), keep: [] };
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const widths = sorted.map((bp) => bp.testWidth);
|
|
245
|
+
const byWidth = new Map(sorted.map((bp) => [bp.testWidth, bp]));
|
|
246
|
+
|
|
247
|
+
const keep: Breakpoint[] = [];
|
|
248
|
+
if (policy.mode === "all") {
|
|
249
|
+
keep.push(...sorted);
|
|
250
|
+
} else if (policy.mode === "limit") {
|
|
251
|
+
const count = Math.max(1, Math.min(MAX_BREAKPOINTS, Math.trunc(policy.count) || 1));
|
|
252
|
+
for (const index of evenIndices(sorted.length, count)) {
|
|
253
|
+
keep.push(sorted[index]!);
|
|
254
|
+
}
|
|
255
|
+
} else {
|
|
256
|
+
/* An override names the widths the PROJECT will have; each is backed by the declared width
|
|
257
|
+
nearest it, because that is where the site's own rules actually change. Two overrides that
|
|
258
|
+
land on the same declared width collapse into one — emitting the same styles twice under two
|
|
259
|
+
names would be two canvases showing the same thing. */
|
|
260
|
+
const requested = [...new Set(policy.widths.map((w) => Math.trunc(w)))]
|
|
261
|
+
.filter((w) => w >= MIN_BREAKPOINT_WIDTH && w <= MAX_BREAKPOINT_WIDTH)
|
|
262
|
+
.toSorted((a, b) => a - b)
|
|
263
|
+
.slice(0, MAX_BREAKPOINTS);
|
|
264
|
+
const claimed = new Set<number>();
|
|
265
|
+
for (const target of requested) {
|
|
266
|
+
const backing = pickNearest(widths, target, policy.rounding);
|
|
267
|
+
if (claimed.has(backing)) {
|
|
268
|
+
continue;
|
|
269
|
+
}
|
|
270
|
+
claimed.add(backing);
|
|
271
|
+
const source = byWidth.get(backing)!;
|
|
272
|
+
const parsed = parseWidthQuery(source.query);
|
|
273
|
+
keep.push({
|
|
274
|
+
name: breakpointName(target),
|
|
275
|
+
query: parsed ? `(${parsed.type}-width: ${target}px)` : source.query,
|
|
276
|
+
testWidth: backing,
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
keep.splice(0, keep.length, ...keep.toSorted((a, b) => a.testWidth - b.testWidth));
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
const fold = new Map<string, string>();
|
|
283
|
+
if (keep.length === 0) {
|
|
284
|
+
return { fold, keep };
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
const keptWidths = keep.map((bp) => bp.testWidth);
|
|
288
|
+
const keptByWidth = new Map(keep.map((bp) => [bp.testWidth, bp]));
|
|
289
|
+
const rounding = policy.mode === "all" ? "nearest" : policy.rounding;
|
|
290
|
+
for (const bp of sorted) {
|
|
291
|
+
const target =
|
|
292
|
+
keptByWidth.get(bp.testWidth) ??
|
|
293
|
+
keptByWidth.get(pickNearest(keptWidths, bp.testWidth, rounding))!;
|
|
294
|
+
fold.set(bp.name, target.name);
|
|
295
|
+
}
|
|
296
|
+
return { fold, keep };
|
|
297
|
+
}
|
package/src/cli.ts
CHANGED
|
@@ -3,6 +3,13 @@
|
|
|
3
3
|
import { resolve } from "node:path";
|
|
4
4
|
import { homedir } from "node:os";
|
|
5
5
|
import { importSite } from "./run.ts";
|
|
6
|
+
import {
|
|
7
|
+
DEFAULT_BREAKPOINT_POLICY,
|
|
8
|
+
MAX_BREAKPOINTS,
|
|
9
|
+
MAX_BREAKPOINT_WIDTH,
|
|
10
|
+
MIN_BREAKPOINT_WIDTH,
|
|
11
|
+
} from "./breakpoint-plan.ts";
|
|
12
|
+
import type { BreakpointPolicy, BreakpointRounding } from "./breakpoint-plan.ts";
|
|
6
13
|
|
|
7
14
|
function usage(): never {
|
|
8
15
|
console.log(`Usage: jx-import <url> [options]
|
|
@@ -14,6 +21,10 @@ Options:
|
|
|
14
21
|
--depth <n> Max crawl depth (default: 2, 0 = single page)
|
|
15
22
|
--max-pages <n> Max pages to capture (default: 25)
|
|
16
23
|
--max-nodes-per-page <n> Skip styles/assets for pages above this (default: 5000)
|
|
24
|
+
--max-breakpoints <n> Keep at most n of the site's breakpoints (default: 3, 0 = keep all)
|
|
25
|
+
--breakpoints <list> Keep these widths instead, e.g. 640,1024,1440
|
|
26
|
+
--breakpoint-rounding <r> How a kept width matches a declared one: nearest|down|up
|
|
27
|
+
(default: nearest)
|
|
17
28
|
--no-styles Skip CSS capture
|
|
18
29
|
--no-assets Skip asset download
|
|
19
30
|
--no-crawl Single page only (equivalent to --depth 0)
|
|
@@ -25,7 +36,10 @@ Options:
|
|
|
25
36
|
--ai-components Use LLM to refine component names and props (Phase 4 AI pass)
|
|
26
37
|
--ai-model <model> Model for AI componentization (default: gpt-4o-mini)
|
|
27
38
|
--verify After import, build and screenshot-diff vs original (Phase 5)
|
|
28
|
-
--
|
|
39
|
+
--min-fidelity <n> Fail (exit 1) below this average fidelity 0..100 (default: 25)
|
|
40
|
+
--verify-threshold <n> Per-pixel colour tolerance 0..1 for the diff (default: 0.15).
|
|
41
|
+
Moves the score; it is NOT the pass bar — --min-fidelity is.
|
|
42
|
+
--verify-viewport-only Diff only the first viewport instead of the whole page
|
|
29
43
|
|
|
30
44
|
Environment:
|
|
31
45
|
OPENAI_API_KEY Required for --ai-components
|
|
@@ -37,6 +51,8 @@ Examples:
|
|
|
37
51
|
jx-import https://example.com --depth 1 --max-pages 10
|
|
38
52
|
jx-import https://example.com --no-crawl
|
|
39
53
|
jx-import https://example.com --out sites/my-clone --no-styles
|
|
54
|
+
jx-import https://example.com --breakpoints 640,1024,1440
|
|
55
|
+
jx-import https://example.com --max-breakpoints 0
|
|
40
56
|
jx-import https://example.com --verify
|
|
41
57
|
jx-import https://example.com --ai-components`);
|
|
42
58
|
process.exit(1);
|
|
@@ -73,7 +89,60 @@ const verifyThreshold =
|
|
|
73
89
|
verifyThresholdIdx !== -1 && args[verifyThresholdIdx + 1]
|
|
74
90
|
? Number(args[verifyThresholdIdx + 1]) || 0.15
|
|
75
91
|
: 0.15;
|
|
92
|
+
/*
|
|
93
|
+
* The floor below which the output is not a clone of anything.
|
|
94
|
+
*
|
|
95
|
+
* It is deliberately low rather than a quality bar: a real import of a complicated site lands well
|
|
96
|
+
* under 100 for reasons nobody can fix from here (a rotating hero, a font that renders a hair
|
|
97
|
+
* differently), and failing those would train people to pass `--min-fidelity 0` and stop looking.
|
|
98
|
+
* Under a quarter of pixels matching is a different question — that is the 8%-fidelity Wix clone
|
|
99
|
+
* that printed `Done!` and exited 0, which is worse than a failure because the only way to see it
|
|
100
|
+
* is to open the result (issue #232).
|
|
101
|
+
*/
|
|
102
|
+
const minFidelity = parseIntArg(args, "--min-fidelity", 25);
|
|
103
|
+
const verifyFullPage = !args.includes("--verify-viewport-only");
|
|
76
104
|
|
|
105
|
+
/**
|
|
106
|
+
* The breakpoint policy from the flags.
|
|
107
|
+
*
|
|
108
|
+
* `--breakpoints` wins over `--max-breakpoints` because it is the more specific answer: naming the
|
|
109
|
+
* widths says everything a count could have.
|
|
110
|
+
*/
|
|
111
|
+
function breakpointPolicy(): BreakpointPolicy {
|
|
112
|
+
const roundingIdx = args.indexOf("--breakpoint-rounding");
|
|
113
|
+
const roundingArg = roundingIdx === -1 ? "" : (args[roundingIdx + 1] ?? "");
|
|
114
|
+
const rounding: BreakpointRounding =
|
|
115
|
+
roundingArg === "down" || roundingArg === "up" ? roundingArg : "nearest";
|
|
116
|
+
|
|
117
|
+
const widthsIdx = args.indexOf("--breakpoints");
|
|
118
|
+
const widthsArg = widthsIdx === -1 ? "" : (args[widthsIdx + 1] ?? "");
|
|
119
|
+
if (widthsArg) {
|
|
120
|
+
const widths = widthsArg
|
|
121
|
+
.split(",")
|
|
122
|
+
.map((part) => Math.trunc(Number(part.trim())))
|
|
123
|
+
.filter((n) => Number.isFinite(n) && n >= MIN_BREAKPOINT_WIDTH && n <= MAX_BREAKPOINT_WIDTH);
|
|
124
|
+
if (widths.length === 0) {
|
|
125
|
+
console.error(
|
|
126
|
+
`Error: --breakpoints needs widths between ${MIN_BREAKPOINT_WIDTH} and ` +
|
|
127
|
+
`${MAX_BREAKPOINT_WIDTH}, e.g. --breakpoints 640,1024,1440`,
|
|
128
|
+
);
|
|
129
|
+
process.exit(1);
|
|
130
|
+
}
|
|
131
|
+
return { mode: "explicit", rounding, widths };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if (!args.includes("--max-breakpoints")) {
|
|
135
|
+
return DEFAULT_BREAKPOINT_POLICY;
|
|
136
|
+
}
|
|
137
|
+
const count = parseIntArg(args, "--max-breakpoints", 3);
|
|
138
|
+
// Zero is the only spelling of "keep all" a count can have, and it reads better than omitting the
|
|
139
|
+
// Flag when a script is choosing between the two.
|
|
140
|
+
return count <= 0
|
|
141
|
+
? { mode: "all" }
|
|
142
|
+
: { count: Math.min(MAX_BREAKPOINTS, count), mode: "limit", rounding };
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const breakpoints = breakpointPolicy();
|
|
77
146
|
const maxDepth = noCrawl ? 0 : parseIntArg(args, "--depth", 2);
|
|
78
147
|
const maxPages = parseIntArg(args, "--max-pages", 25);
|
|
79
148
|
const maxNodesPerPage = parseIntArg(args, "--max-nodes-per-page", 5000);
|
|
@@ -116,6 +185,7 @@ async function main() {
|
|
|
116
185
|
{
|
|
117
186
|
url: url as string,
|
|
118
187
|
outDir,
|
|
188
|
+
breakpoints,
|
|
119
189
|
maxDepth,
|
|
120
190
|
maxPages,
|
|
121
191
|
maxNodesPerPage,
|
|
@@ -125,7 +195,9 @@ async function main() {
|
|
|
125
195
|
respectRobots: !args.includes("--no-robots"),
|
|
126
196
|
componentize: noComponents ? false : { minInstances, minDepth },
|
|
127
197
|
ai,
|
|
128
|
-
verify: doVerify
|
|
198
|
+
verify: doVerify
|
|
199
|
+
? { threshold: verifyThreshold, minFidelity, fullPage: verifyFullPage }
|
|
200
|
+
: false,
|
|
129
201
|
},
|
|
130
202
|
(e) => console.log(` ${e.message}`),
|
|
131
203
|
);
|
|
@@ -136,9 +208,41 @@ async function main() {
|
|
|
136
208
|
}
|
|
137
209
|
if (result.verify) {
|
|
138
210
|
console.log(` Average fidelity: ${result.verify.averageFidelity}%`);
|
|
211
|
+
for (const page of result.verify.pages) {
|
|
212
|
+
const misses = [
|
|
213
|
+
page.failedRequests > 0 ? `${page.failedRequests} failed request(s)` : "",
|
|
214
|
+
page.consoleErrors > 0 ? `${page.consoleErrors} console error(s)` : "",
|
|
215
|
+
].filter(Boolean);
|
|
216
|
+
if (misses.length > 0) {
|
|
217
|
+
console.log(` ${page.route}: ${misses.join(", ")}`);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
139
220
|
console.log(` Report: ${result.verify.reportDir}/report.json`);
|
|
140
221
|
}
|
|
141
222
|
|
|
223
|
+
/*
|
|
224
|
+
* A verify that cannot fail is a report, not a gate. An import nobody opens and looks at is
|
|
225
|
+
* exactly the case the exit code exists for, so a build error or a fidelity under the floor
|
|
226
|
+
* ends the run non-zero and says which one it was.
|
|
227
|
+
*/
|
|
228
|
+
if (result.verify && !result.verify.passed) {
|
|
229
|
+
for (const error of result.verify.buildErrors) {
|
|
230
|
+
console.error(`Build error: ${error}`);
|
|
231
|
+
}
|
|
232
|
+
console.error(
|
|
233
|
+
`\nVerification FAILED — the project is at ${outDir}, but it does not match the original.`,
|
|
234
|
+
);
|
|
235
|
+
if (result.verify.buildErrors.length === 0) {
|
|
236
|
+
console.error(
|
|
237
|
+
` Average fidelity ${result.verify.averageFidelity}% is below the ` +
|
|
238
|
+
`${result.verify.minFidelity}% minimum. Pass --min-fidelity to change it.`,
|
|
239
|
+
);
|
|
240
|
+
}
|
|
241
|
+
console.error(` Report: ${result.verify.reportDir}/report.json`);
|
|
242
|
+
process.exitCode = 1;
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
|
|
142
246
|
console.log(`\nDone! Open in Studio:`);
|
|
143
247
|
console.log(
|
|
144
248
|
` http://localhost:3000/packages/studio/index.html?project=${outDir}/project.json`,
|
package/src/componentize.ts
CHANGED
|
@@ -54,6 +54,16 @@ function structuralHash(node: JxElement | string): string {
|
|
|
54
54
|
return `<${tag}>${childHashes.join(",")}`;
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Attributes that may not become a component prop.
|
|
59
|
+
*
|
|
60
|
+
* A prop is something a person fills in per instance, and a class is not one: `emit.ts` strips the
|
|
61
|
+
* source site's classes on the way out (`strip-classes.ts`), so a class that happened to differ
|
|
62
|
+
* between two instances of the same block would otherwise mint a `${state.class}` prop whose value
|
|
63
|
+
* is then deleted — a prop with a name nobody chose, wired to nothing.
|
|
64
|
+
*/
|
|
65
|
+
const PRESENTATION_ATTRS = new Set(["class", "className"]);
|
|
66
|
+
|
|
57
67
|
function collectLeafValues(node: JxElement | string, prefix: string, out: Map<string, string>) {
|
|
58
68
|
if (typeof node === "string") {
|
|
59
69
|
out.set(`${prefix}.$text`, node);
|
|
@@ -64,6 +74,9 @@ function collectLeafValues(node: JxElement | string, prefix: string, out: Map<st
|
|
|
64
74
|
}
|
|
65
75
|
if (node.attributes) {
|
|
66
76
|
for (const [k, v] of Object.entries(node.attributes)) {
|
|
77
|
+
if (PRESENTATION_ATTRS.has(k)) {
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
67
80
|
if (typeof v === "string" || typeof v === "number" || typeof v === "boolean") {
|
|
68
81
|
out.set(`${prefix}.attr.${k}`, String(v));
|
|
69
82
|
}
|