@jxsuite/import 0.39.8 → 0.40.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -2
- package/package.json +9 -5
- package/src/ai-componentize.ts +12 -2
- package/src/asset-collect.ts +11 -5
- package/src/asset-download.ts +18 -14
- package/src/asset-rewrite.ts +7 -1
- package/src/breakpoint-plan.ts +297 -0
- package/src/browser-local.ts +67 -0
- package/src/capture.ts +50 -61
- package/src/cli.ts +106 -2
- package/src/componentize.ts +13 -0
- package/src/crawl.ts +50 -12
- package/src/emit.ts +136 -37
- package/src/index.ts +32 -8
- package/src/io.ts +52 -0
- package/src/media-extract.ts +53 -77
- package/src/pipeline.ts +605 -0
- package/src/run.ts +208 -403
- package/src/srcset.ts +32 -0
- package/src/strip-classes.ts +73 -0
- package/src/style-capture.ts +6 -3
- package/src/verify.ts +166 -14
package/src/run.ts
CHANGED
|
@@ -1,27 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* The local import — the orchestrator behind both the jx-import CLI and the Studio import endpoint.
|
|
3
|
+
*
|
|
4
|
+
* Everything portable moved to `pipeline.ts`. What is left is what only a machine with a disk and a
|
|
5
|
+
* Chrome can do: the empty-directory guard, the `project.json` seed, the local write sink, the
|
|
6
|
+
* browser launch, and the verify phase (which builds the project with `@jxsuite/compiler` and
|
|
7
|
+
* serves it over `Bun.serve` to screenshot it — a hosted platform never executes a project's own
|
|
8
|
+
* JavaScript, so verification does not follow the pipeline anywhere).
|
|
5
9
|
*/
|
|
6
10
|
|
|
7
11
|
import { existsSync, readdirSync } from "node:fs";
|
|
8
|
-
import { isAbsolute } from "node:path";
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import { applyStylesToTree } from "./apply-styles.ts";
|
|
16
|
-
import { collectAssets } from "./asset-collect.ts";
|
|
17
|
-
import { downloadAssets } from "./asset-download.ts";
|
|
18
|
-
import { rewriteAssetUrls } from "./asset-rewrite.ts";
|
|
19
|
-
import { applyTokens } from "./css-tokens.ts";
|
|
20
|
-
import { crawlSite } from "./crawl.ts";
|
|
21
|
-
import { detectLayout } from "./layout-detect.ts";
|
|
12
|
+
import { isAbsolute, join } from "node:path";
|
|
13
|
+
import { mkdir } from "node:fs/promises";
|
|
14
|
+
import { closeBrowser, launchBrowser } from "./browser-local.ts";
|
|
15
|
+
import { createLocalIo } from "./io.ts";
|
|
16
|
+
import { runImportPipeline } from "./pipeline.ts";
|
|
17
|
+
import type { BreakpointPolicy } from "./breakpoint-plan.ts";
|
|
18
|
+
import type { ImportPhase, ImportProgressEvent } from "./pipeline.ts";
|
|
22
19
|
import type { Browser } from "puppeteer-core";
|
|
23
|
-
|
|
24
|
-
|
|
20
|
+
|
|
21
|
+
export type { ImportPhase, ImportProgressEvent } from "./pipeline.ts";
|
|
25
22
|
|
|
26
23
|
export interface ImportSiteOptions {
|
|
27
24
|
/** The page to import; must be http(s). */
|
|
@@ -34,6 +31,14 @@ export interface ImportSiteOptions {
|
|
|
34
31
|
maxPages?: number;
|
|
35
32
|
/** Skip styles/assets for pages above this node count (default 5000). */
|
|
36
33
|
maxNodesPerPage?: number;
|
|
34
|
+
/**
|
|
35
|
+
* Which of the site's declared breakpoints the project keeps (default: three, evenly spaced).
|
|
36
|
+
*
|
|
37
|
+
* A real site declares as many breakpoints as it has accumulated frameworks; nine `$media`
|
|
38
|
+
* entries is nine canvas sizes in Studio and nine columns in every style editor, and nobody
|
|
39
|
+
* authors against that. See `breakpoint-plan.ts`.
|
|
40
|
+
*/
|
|
41
|
+
breakpoints?: BreakpointPolicy;
|
|
37
42
|
/** Capture computed styles (default true). */
|
|
38
43
|
styles?: boolean;
|
|
39
44
|
/** Download and rewrite assets (default true). */
|
|
@@ -46,87 +51,77 @@ export interface ImportSiteOptions {
|
|
|
46
51
|
componentize?: false | { minInstances?: number; minDepth?: number };
|
|
47
52
|
/** LLM refinement of component/prop names; false/undefined to skip. */
|
|
48
53
|
ai?: false | { apiKey: string; baseUrl?: string | undefined; model?: string | undefined };
|
|
49
|
-
/**
|
|
50
|
-
|
|
54
|
+
/**
|
|
55
|
+
* Build + screenshot-diff the emitted project against the original; false to skip.
|
|
56
|
+
*
|
|
57
|
+
* `threshold` is pixelmatch's per-pixel COLOUR tolerance, not a bar; `minFidelity` is the bar,
|
|
58
|
+
* and the run reports `passed` against it. `fullPage` compares the whole scrollable page (default
|
|
59
|
+
* true) rather than the first viewport.
|
|
60
|
+
*/
|
|
61
|
+
verify?: false | { threshold?: number; minFidelity?: number; fullPage?: boolean | undefined };
|
|
51
62
|
/** Explicit browser binary (wins over CHROME_PATH and PATH discovery). */
|
|
52
63
|
chromePath?: string;
|
|
53
64
|
/** Aborts between phases (and between crawled pages). */
|
|
54
65
|
signal?: AbortSignal;
|
|
55
66
|
}
|
|
56
67
|
|
|
57
|
-
export type ImportPhase =
|
|
58
|
-
| "launch"
|
|
59
|
-
| "capture"
|
|
60
|
-
| "crawl"
|
|
61
|
-
| "styles"
|
|
62
|
-
| "assets"
|
|
63
|
-
| "layout"
|
|
64
|
-
| "components"
|
|
65
|
-
| "ai"
|
|
66
|
-
| "emit"
|
|
67
|
-
| "verify"
|
|
68
|
-
| "done";
|
|
69
|
-
|
|
70
|
-
export interface ImportProgressEvent {
|
|
71
|
-
phase: ImportPhase;
|
|
72
|
-
message: string;
|
|
73
|
-
current?: number;
|
|
74
|
-
total?: number;
|
|
75
|
-
}
|
|
76
|
-
|
|
77
68
|
export interface ImportSiteResult {
|
|
78
69
|
outDir: string;
|
|
79
70
|
pages: { route: string; title: string; nodeCount: number }[];
|
|
80
71
|
fileCount: number;
|
|
81
|
-
verify: {
|
|
72
|
+
verify: {
|
|
73
|
+
averageFidelity: number;
|
|
74
|
+
reportDir: string;
|
|
75
|
+
/**
|
|
76
|
+
* Whether the run met its bar — it built cleanly, every page rendered, and the average reached
|
|
77
|
+
* `minFidelity`. Nothing about verify could fail before this existed: a clone scoring 8%
|
|
78
|
+
* finished exactly like one scoring 95% (issue #232).
|
|
79
|
+
*/
|
|
80
|
+
passed: boolean;
|
|
81
|
+
/** The bar `passed` was measured against. */
|
|
82
|
+
minFidelity: number;
|
|
83
|
+
/** Errors the compiler reported building the project — recorded, and now also enforced. */
|
|
84
|
+
buildErrors: string[];
|
|
85
|
+
/**
|
|
86
|
+
* Per page, because the average cannot name one.
|
|
87
|
+
*
|
|
88
|
+
* "Average fidelity 84%" is a fact nobody can act on; "the pricing page renders at 61%" is a
|
|
89
|
+
* decision — retry it, patch it by hand, or accept it. The verifier computed both and only the
|
|
90
|
+
* average was reported, so the actionable half was thrown away one level below the caller.
|
|
91
|
+
*
|
|
92
|
+
* The counts alongside it are what a percentage cannot say: a page that 404s on fifteen images
|
|
93
|
+
* scores badly, and only `failedRequests` says why.
|
|
94
|
+
*/
|
|
95
|
+
pages: {
|
|
96
|
+
route: string;
|
|
97
|
+
fidelity: number;
|
|
98
|
+
consoleErrors: number;
|
|
99
|
+
failedRequests: number;
|
|
100
|
+
error?: string;
|
|
101
|
+
}[];
|
|
102
|
+
} | null;
|
|
82
103
|
warnings: string[];
|
|
83
104
|
}
|
|
84
105
|
|
|
85
|
-
function throwIfAborted(signal?: AbortSignal): void {
|
|
86
|
-
if (signal?.aborted) {
|
|
87
|
-
throw new Error("Import aborted");
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
function formatBytes(bytes: number): string {
|
|
92
|
-
if (bytes < 1024) {
|
|
93
|
-
return `${bytes} B`;
|
|
94
|
-
}
|
|
95
|
-
if (bytes < 1024 * 1024) {
|
|
96
|
-
return `${(bytes / 1024).toFixed(1)} KB`;
|
|
97
|
-
}
|
|
98
|
-
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
99
|
-
}
|
|
100
|
-
|
|
101
106
|
/**
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
107
|
+
* What the CLI and the OSS server ask for when the caller named no model.
|
|
108
|
+
*
|
|
109
|
+
* It lives at the call site rather than inside `ai-componentize.ts`, because a default belongs to
|
|
110
|
+
* whoever knows the provider. This path talks to an OpenAI-compatible endpoint; a backend brokering
|
|
111
|
+
* Workers AI would 404 on this id, which is exactly why the module below it demands one.
|
|
105
112
|
*/
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
progress("components", "Running heuristic componentization...");
|
|
118
|
-
const heuristic = componentize(pageMap, componentizeOptions);
|
|
119
|
-
if (heuristic.components.size === 0) {
|
|
120
|
-
return undefined;
|
|
113
|
+
const DEFAULT_AI_MODEL = "gpt-4o-mini";
|
|
114
|
+
|
|
115
|
+
/** The one file the seed writes, and the only thing the empty-directory guard tolerates. */
|
|
116
|
+
const PROJECT_FILE = "project.json";
|
|
117
|
+
|
|
118
|
+
/** A placeholder project name from the URL, replaced by the page title at emit. */
|
|
119
|
+
function seedName(url: string): string {
|
|
120
|
+
try {
|
|
121
|
+
return new URL(url).hostname.replace(/^www\./, "") || "Imported Site";
|
|
122
|
+
} catch {
|
|
123
|
+
return "Imported Site";
|
|
121
124
|
}
|
|
122
|
-
progress("ai", `Found ${heuristic.components.size} component(s), refining with AI...`);
|
|
123
|
-
const refined = await aiComponentize(
|
|
124
|
-
heuristic,
|
|
125
|
-
{ apiKey: ai.apiKey, baseUrl: ai.baseUrl, model: ai.model },
|
|
126
|
-
(msg) => progress("ai", msg),
|
|
127
|
-
);
|
|
128
|
-
progress("ai", `AI refined ${refined.components.size} component(s)`);
|
|
129
|
-
return refined;
|
|
130
125
|
}
|
|
131
126
|
|
|
132
127
|
/**
|
|
@@ -139,22 +134,7 @@ export async function importSite(
|
|
|
139
134
|
options: ImportSiteOptions,
|
|
140
135
|
onProgress?: (e: ImportProgressEvent) => void,
|
|
141
136
|
): Promise<ImportSiteResult> {
|
|
142
|
-
const {
|
|
143
|
-
url,
|
|
144
|
-
outDir,
|
|
145
|
-
maxDepth = 2,
|
|
146
|
-
maxPages = 25,
|
|
147
|
-
maxNodesPerPage = 5000,
|
|
148
|
-
styles = true,
|
|
149
|
-
assets = true,
|
|
150
|
-
scroll = true,
|
|
151
|
-
respectRobots = true,
|
|
152
|
-
componentize: componentizeOptions = {},
|
|
153
|
-
ai = false,
|
|
154
|
-
verify = false,
|
|
155
|
-
chromePath,
|
|
156
|
-
signal,
|
|
157
|
-
} = options;
|
|
137
|
+
const { url, outDir, maxDepth = 2, ai = false, verify = false, chromePath, signal } = options;
|
|
158
138
|
|
|
159
139
|
if (!url.startsWith("http://") && !url.startsWith("https://")) {
|
|
160
140
|
throw new Error("URL must start with http:// or https://");
|
|
@@ -162,16 +142,16 @@ export async function importSite(
|
|
|
162
142
|
if (!isAbsolute(outDir)) {
|
|
163
143
|
throw new Error(`outDir must be an absolute path, got "${outDir}"`);
|
|
164
144
|
}
|
|
165
|
-
|
|
145
|
+
/* The guard admits the run's OWN seed and nothing else. It has to: the destination is created and
|
|
146
|
+
given a `project.json` before the browser launches, so a host can open the project and watch the
|
|
147
|
+
rest of the run arrive in its file tree — and a guard that could not tell that seed from a
|
|
148
|
+
stranger's files would refuse every import at its second statement. */
|
|
149
|
+
if (existsSync(outDir) && readdirSync(outDir).some((entry) => entry !== PROJECT_FILE)) {
|
|
166
150
|
throw new Error(`Directory "${outDir}" is not empty`);
|
|
167
151
|
}
|
|
168
152
|
|
|
169
|
-
const progress = (
|
|
170
|
-
phase
|
|
171
|
-
message: string,
|
|
172
|
-
counts?: { current?: number; total?: number },
|
|
173
|
-
) => {
|
|
174
|
-
onProgress?.({ phase, message, ...counts });
|
|
153
|
+
const progress = (phase: ImportPhase, message: string) => {
|
|
154
|
+
onProgress?.({ phase, message });
|
|
175
155
|
};
|
|
176
156
|
const warnings: string[] = [];
|
|
177
157
|
const warn = (phase: ImportPhase, message: string) => {
|
|
@@ -179,350 +159,175 @@ export async function importSite(
|
|
|
179
159
|
progress(phase, `⚠ ${message}`);
|
|
180
160
|
};
|
|
181
161
|
const verifyThreshold = verify === false ? 0.15 : (verify.threshold ?? 0.15);
|
|
162
|
+
const verifyMinFidelity = verify === false ? 0 : (verify.minFidelity ?? 0);
|
|
163
|
+
const verifyFullPage = verify === false ? true : (verify.fullPage ?? true);
|
|
182
164
|
|
|
183
165
|
try {
|
|
184
|
-
|
|
185
|
-
return await runSinglePage();
|
|
186
|
-
}
|
|
187
|
-
return await runCrawl();
|
|
188
|
-
} finally {
|
|
189
|
-
await closeBrowser();
|
|
190
|
-
}
|
|
166
|
+
await seedProject();
|
|
191
167
|
|
|
192
|
-
// ─── Single-page mode: Phase 0-2 pipeline, no crawl overhead ────────────────
|
|
193
|
-
async function runSinglePage(): Promise<ImportSiteResult> {
|
|
194
168
|
progress("launch", "Launching browser...");
|
|
195
169
|
const browser = await launchBrowser(
|
|
196
170
|
chromePath === undefined ? {} : { executablePath: chromePath },
|
|
197
171
|
);
|
|
198
172
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
let breakpoints: Record<string, string> | undefined;
|
|
215
|
-
let styleTokens: Record<string, string> | undefined;
|
|
216
|
-
|
|
217
|
-
if (styles) {
|
|
218
|
-
throwIfAborted(signal);
|
|
219
|
-
progress("styles", "Capturing computed styles...");
|
|
220
|
-
const styleResult = await captureStyles(capture.page);
|
|
221
|
-
progress(
|
|
222
|
-
"styles",
|
|
223
|
-
`Captured styles for ${styleResult.elements.length} elements, ${Object.keys(styleResult.uaDefaults).length} tag baselines`,
|
|
224
|
-
);
|
|
225
|
-
|
|
226
|
-
progress("styles", "Diffing against UA defaults...");
|
|
227
|
-
const diffed = diffAllStyles(styleResult.elements, styleResult.uaDefaults);
|
|
228
|
-
progress("styles", `${diffed.length} elements with non-default styles`);
|
|
229
|
-
|
|
230
|
-
// Replace resolved values with var(--name) references
|
|
231
|
-
const propCount = Object.keys(styleResult.customProperties).length;
|
|
232
|
-
if (propCount > 0) {
|
|
233
|
-
const tokenResult = applyTokens(diffed, styleResult.customProperties);
|
|
234
|
-
if (tokenResult.replacements > 0) {
|
|
235
|
-
styleTokens = tokenResult.tokens;
|
|
236
|
-
progress(
|
|
237
|
-
"styles",
|
|
238
|
-
`Extracted ${Object.keys(tokenResult.tokens).length} design tokens (${tokenResult.replacements} replacements)`,
|
|
239
|
-
);
|
|
240
|
-
} else {
|
|
241
|
-
progress(
|
|
242
|
-
"styles",
|
|
243
|
-
`${propCount} custom properties found, but none matched computed values`,
|
|
244
|
-
);
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
if (styleResult.mediaQueries.length > 0) {
|
|
249
|
-
progress(
|
|
250
|
-
"styles",
|
|
251
|
-
`Extracting @media breakpoints (${styleResult.mediaQueries.length} queries found)...`,
|
|
252
|
-
);
|
|
253
|
-
const media = await extractMedia(
|
|
254
|
-
capture.page,
|
|
255
|
-
styleResult.elements,
|
|
256
|
-
styleResult.uaDefaults,
|
|
257
|
-
styleResult.mediaQueries,
|
|
258
|
-
);
|
|
259
|
-
const bpCount = Object.keys(media.breakpoints).length;
|
|
260
|
-
if (bpCount > 0) {
|
|
261
|
-
progress("styles", `${bpCount} breakpoints with style changes`);
|
|
262
|
-
({ breakpoints } = media);
|
|
263
|
-
applyStylesToTree(jx.document, diffed, media.deltas);
|
|
264
|
-
} else {
|
|
265
|
-
progress("styles", "No responsive breakpoints with style changes");
|
|
266
|
-
applyStylesToTree(jx.document, diffed);
|
|
267
|
-
}
|
|
268
|
-
} else {
|
|
269
|
-
progress("styles", "No @media queries found");
|
|
270
|
-
applyStylesToTree(jx.document, diffed);
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
// Apply <html>/<body> computed styles to the root wrapper
|
|
274
|
-
if (Object.keys(styleResult.documentStyles).length > 0) {
|
|
275
|
-
if (!jx.document.style) {
|
|
276
|
-
jx.document.style = {};
|
|
277
|
-
}
|
|
278
|
-
for (const [prop, val] of Object.entries(styleResult.documentStyles)) {
|
|
279
|
-
const camel = kebabToCamel(prop);
|
|
280
|
-
jx.document.style[camel] = val;
|
|
281
|
-
}
|
|
282
|
-
progress(
|
|
173
|
+
const result = await runImportPipeline(
|
|
174
|
+
{
|
|
175
|
+
url,
|
|
176
|
+
browser,
|
|
177
|
+
io: createLocalIo(outDir),
|
|
178
|
+
maxDepth,
|
|
179
|
+
/* The model default is applied here so the pipeline can require one. See DEFAULT_AI_MODEL. */
|
|
180
|
+
ai: ai === false ? false : { ...ai, model: ai.model ?? DEFAULT_AI_MODEL },
|
|
181
|
+
referenceScreenshots: verify === false ? false : { fullPage: verifyFullPage },
|
|
182
|
+
...pick(options, [
|
|
183
|
+
"maxPages",
|
|
184
|
+
"maxNodesPerPage",
|
|
185
|
+
"breakpoints",
|
|
283
186
|
"styles",
|
|
284
|
-
`Applied ${Object.keys(styleResult.documentStyles).length} document-level styles to root`,
|
|
285
|
-
);
|
|
286
|
-
}
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
let fontFaceRules: string[] | undefined;
|
|
290
|
-
let fontRewriteMap: Map<string, string> | undefined;
|
|
291
|
-
|
|
292
|
-
if (assets) {
|
|
293
|
-
throwIfAborted(signal);
|
|
294
|
-
progress("assets", "Collecting asset URLs...");
|
|
295
|
-
const collected = await collectAssets(capture.page);
|
|
296
|
-
progress(
|
|
297
|
-
"assets",
|
|
298
|
-
`Found ${collected.assets.length} assets (${collected.inlineSvgCount} inline SVGs kept, ${collected.stylesheets.length} stylesheets retained)`,
|
|
299
|
-
);
|
|
300
|
-
|
|
301
|
-
const allFontRules = collected.stylesheets.flatMap((s) => s.fontFaceRules);
|
|
302
|
-
if (allFontRules.length > 0) {
|
|
303
|
-
fontFaceRules = allFontRules;
|
|
304
|
-
progress("assets", `Found ${allFontRules.length} @font-face rules`);
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
if (collected.assets.length > 0) {
|
|
308
|
-
progress("assets", "Downloading assets...");
|
|
309
|
-
const downloaded = await downloadAssets(collected.assets, outDir, url);
|
|
310
|
-
progress(
|
|
311
187
|
"assets",
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
if (fontFaceRules) {
|
|
322
|
-
fontRewriteMap = new Map<string, string>();
|
|
323
|
-
for (const [originalUrl, localPath] of downloaded.rewriteMap) {
|
|
324
|
-
if (localPath.includes("/fonts/") || /\.(woff2?|ttf|otf|eot)$/i.test(localPath)) {
|
|
325
|
-
fontRewriteMap.set(originalUrl, localPath);
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
|
|
330
|
-
progress("assets", "Rewriting asset URLs...");
|
|
331
|
-
const rewrites = rewriteAssetUrls(jx.document, downloaded.rewriteMap, url);
|
|
332
|
-
progress("assets", `Rewrote ${rewrites} URL references`);
|
|
333
|
-
}
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
// Capture the reference screenshot before closing the page (for verify)
|
|
337
|
-
let referenceScreenshot: Buffer | undefined;
|
|
338
|
-
if (verify !== false) {
|
|
339
|
-
progress("verify", "Capturing reference screenshot...");
|
|
340
|
-
const { captureReferenceScreenshot } = await import("./verify.ts");
|
|
341
|
-
referenceScreenshot = await captureReferenceScreenshot(capture.page);
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
await capture.page.close();
|
|
345
|
-
|
|
346
|
-
throwIfAborted(signal);
|
|
347
|
-
const pageMap = new Map([["pages/index.json", jx.document]]);
|
|
348
|
-
const precomputedComponents = await maybeAiComponentize(
|
|
349
|
-
pageMap,
|
|
350
|
-
componentizeOptions,
|
|
351
|
-
ai,
|
|
352
|
-
progress,
|
|
353
|
-
);
|
|
354
|
-
|
|
355
|
-
throwIfAborted(signal);
|
|
356
|
-
progress("emit", "Writing project...");
|
|
357
|
-
const { files } = await emitMultiPageProject({
|
|
358
|
-
outDir,
|
|
359
|
-
title: capture.title,
|
|
360
|
-
pages: pageMap,
|
|
361
|
-
sourceUrl: url,
|
|
362
|
-
breakpoints,
|
|
363
|
-
componentizeOptions: precomputedComponents ? false : componentizeOptions,
|
|
364
|
-
precomputedComponents,
|
|
365
|
-
fontFaceRules,
|
|
366
|
-
fontRewriteMap,
|
|
367
|
-
styleTokens,
|
|
368
|
-
});
|
|
369
|
-
progress("emit", `Wrote ${files.length} files`);
|
|
370
|
-
|
|
371
|
-
let verifySummary: ImportSiteResult["verify"] = null;
|
|
372
|
-
if (verify !== false && referenceScreenshot) {
|
|
373
|
-
verifySummary = await runVerify(
|
|
374
|
-
new Map([["pages/index.json", { sourceUrl: url, screenshot: referenceScreenshot }]]),
|
|
375
|
-
browser,
|
|
376
|
-
);
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
progress("done", `Imported ${url} → ${outDir}`);
|
|
380
|
-
return {
|
|
381
|
-
outDir,
|
|
382
|
-
pages: [{ route: "pages/index.json", title: capture.title, nodeCount: jx.nodeCount }],
|
|
383
|
-
fileCount: files.length,
|
|
384
|
-
verify: verifySummary,
|
|
385
|
-
warnings,
|
|
386
|
-
};
|
|
387
|
-
}
|
|
388
|
-
|
|
389
|
-
// ─── Multi-page crawl mode (Phase 3) ────────────────────────────────────────
|
|
390
|
-
async function runCrawl(): Promise<ImportSiteResult> {
|
|
391
|
-
progress("launch", "Launching browser...");
|
|
392
|
-
await launchBrowser(chromePath === undefined ? {} : { executablePath: chromePath });
|
|
393
|
-
|
|
394
|
-
throwIfAborted(signal);
|
|
395
|
-
const result = await crawlSite({
|
|
396
|
-
url,
|
|
397
|
-
outDir,
|
|
398
|
-
maxDepth,
|
|
399
|
-
maxPages,
|
|
400
|
-
maxNodesPerPage,
|
|
401
|
-
skipStyles: !styles,
|
|
402
|
-
skipAssets: !assets,
|
|
403
|
-
respectRobots,
|
|
404
|
-
noScroll: !scroll,
|
|
405
|
-
captureScreenshots: verify !== false,
|
|
406
|
-
onProgress: (msg) => progress("crawl", msg.trim()),
|
|
407
|
-
...(signal === undefined ? {} : { signal }),
|
|
408
|
-
});
|
|
409
|
-
|
|
410
|
-
progress(
|
|
411
|
-
"crawl",
|
|
412
|
-
`Crawled ${result.pages.length} page${result.pages.length === 1 ? "" : "s"}`,
|
|
413
|
-
{ current: result.pages.length, total: maxPages },
|
|
188
|
+
"scroll",
|
|
189
|
+
"respectRobots",
|
|
190
|
+
"componentize",
|
|
191
|
+
"signal",
|
|
192
|
+
]),
|
|
193
|
+
},
|
|
194
|
+
onProgress,
|
|
414
195
|
);
|
|
415
|
-
|
|
416
|
-
progress("crawl", `Skipped ${result.skippedByRobots.length} URLs (robots.txt)`);
|
|
417
|
-
}
|
|
418
|
-
if (result.skippedByNodeCap.length > 0) {
|
|
419
|
-
warn(
|
|
420
|
-
"crawl",
|
|
421
|
-
`${result.skippedByNodeCap.length} pages exceeded node cap (styles/assets skipped)`,
|
|
422
|
-
);
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
const pageMap = new Map<string, JxElement>();
|
|
426
|
-
for (const page of result.pages) {
|
|
427
|
-
pageMap.set(page.route, page.jx.document);
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
// Layout detection
|
|
431
|
-
let layout: JxElement | undefined;
|
|
432
|
-
if (result.pages.length >= 2) {
|
|
433
|
-
throwIfAborted(signal);
|
|
434
|
-
progress("layout", "Detecting shared layout (header/footer)...");
|
|
435
|
-
const layoutResult = detectLayout(pageMap);
|
|
436
|
-
if (layoutResult) {
|
|
437
|
-
({ layout } = layoutResult);
|
|
438
|
-
const sharedCount = (Array.isArray(layout.children) ? layout.children.length : 0) - 1; // Minus the slot
|
|
439
|
-
progress("layout", `Found shared layout with ${sharedCount} shared elements`);
|
|
440
|
-
for (const [route, stripped] of layoutResult.strippedPages) {
|
|
441
|
-
pageMap.set(route, stripped);
|
|
442
|
-
}
|
|
443
|
-
} else {
|
|
444
|
-
progress("layout", "No shared layout detected");
|
|
445
|
-
}
|
|
446
|
-
}
|
|
447
|
-
|
|
448
|
-
throwIfAborted(signal);
|
|
449
|
-
const precomputedComponents = await maybeAiComponentize(
|
|
450
|
-
pageMap,
|
|
451
|
-
componentizeOptions,
|
|
452
|
-
ai,
|
|
453
|
-
progress,
|
|
454
|
-
);
|
|
455
|
-
|
|
456
|
-
throwIfAborted(signal);
|
|
457
|
-
progress("emit", "Writing project...");
|
|
458
|
-
const title = result.pages[0]?.title || new URL(url).hostname;
|
|
459
|
-
const { files } = await emitMultiPageProject({
|
|
460
|
-
outDir,
|
|
461
|
-
title,
|
|
462
|
-
sourceUrl: url,
|
|
463
|
-
pages: pageMap,
|
|
464
|
-
layout,
|
|
465
|
-
breakpoints: result.breakpoints,
|
|
466
|
-
componentizeOptions: precomputedComponents ? false : componentizeOptions,
|
|
467
|
-
precomputedComponents,
|
|
468
|
-
fontFaceRules: result.fontFaceRules.length > 0 ? result.fontFaceRules : undefined,
|
|
469
|
-
fontRewriteMap: result.fontRewriteMap.size > 0 ? result.fontRewriteMap : undefined,
|
|
470
|
-
styleTokens: result.styleTokens,
|
|
471
|
-
});
|
|
472
|
-
progress("emit", `Wrote ${files.length} files`);
|
|
196
|
+
warnings.push(...result.warnings);
|
|
473
197
|
|
|
474
198
|
let verifySummary: ImportSiteResult["verify"] = null;
|
|
475
199
|
if (verify !== false) {
|
|
476
|
-
|
|
477
|
-
for (const page of result.pages) {
|
|
478
|
-
if (page.screenshot) {
|
|
479
|
-
verifyPages.set(page.route, { sourceUrl: page.url, screenshot: page.screenshot });
|
|
480
|
-
}
|
|
481
|
-
}
|
|
482
|
-
if (verifyPages.size === 0) {
|
|
200
|
+
if (result.references.size === 0) {
|
|
483
201
|
progress("verify", "No reference screenshots captured — skipping verify");
|
|
484
202
|
} else {
|
|
485
|
-
|
|
203
|
+
/* Only the single-page run hands the verifier its browser. A crawl's pages were captured
|
|
204
|
+
and closed one at a time, so by now the connection has outlived every page it opened and
|
|
205
|
+
`verifyProject` launching its own is the same cost either way. */
|
|
206
|
+
verifySummary = await runVerify(result.references, maxDepth === 0 ? browser : undefined);
|
|
486
207
|
}
|
|
487
208
|
}
|
|
488
209
|
|
|
489
210
|
progress("done", `Imported ${url} → ${outDir}`);
|
|
490
211
|
return {
|
|
491
212
|
outDir,
|
|
492
|
-
pages: result.pages
|
|
493
|
-
|
|
494
|
-
title: p.title,
|
|
495
|
-
nodeCount: p.jx.nodeCount,
|
|
496
|
-
})),
|
|
497
|
-
fileCount: files.length,
|
|
213
|
+
pages: result.pages,
|
|
214
|
+
fileCount: result.files.length,
|
|
498
215
|
verify: verifySummary,
|
|
499
216
|
warnings,
|
|
500
217
|
};
|
|
218
|
+
} finally {
|
|
219
|
+
await closeBrowser();
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Create the destination and give it a `project.json` valid enough to open.
|
|
224
|
+
*
|
|
225
|
+
* The emit phase rewrites this file completely, so nothing here is a guess at what the import
|
|
226
|
+
* will find — it is the minimum that makes the directory a PROJECT, so the caller can open it now
|
|
227
|
+
* instead of at the end. Everything a crawl writes then lands in a file tree somebody is
|
|
228
|
+
* watching.
|
|
229
|
+
*/
|
|
230
|
+
async function seedProject(): Promise<void> {
|
|
231
|
+
await mkdir(outDir, { recursive: true });
|
|
232
|
+
const seed = { name: seedName(url), imports: {}, images: { optimize: false } };
|
|
233
|
+
await Bun.write(join(outDir, PROJECT_FILE), `${JSON.stringify(seed, null, 2)}\n`);
|
|
234
|
+
onProgress?.({
|
|
235
|
+
message: `Created ${outDir}`,
|
|
236
|
+
phase: "seed",
|
|
237
|
+
root: outDir,
|
|
238
|
+
});
|
|
501
239
|
}
|
|
502
240
|
|
|
503
241
|
// ─── Phase 5: build + screenshot-diff against the original ─────────────────
|
|
504
242
|
async function runVerify(
|
|
505
|
-
|
|
243
|
+
references: Map<string, { sourceUrl: string; screenshot: Uint8Array }>,
|
|
506
244
|
browser?: Browser,
|
|
507
245
|
): Promise<ImportSiteResult["verify"]> {
|
|
508
246
|
throwIfAborted(signal);
|
|
509
247
|
progress("verify", "Verifying against the original...");
|
|
510
248
|
const { verifyProject } = await import("./verify.ts");
|
|
249
|
+
const verifyPages = new Map<string, { sourceUrl: string; screenshot: Buffer | string }>();
|
|
250
|
+
for (const [route, ref] of references) {
|
|
251
|
+
verifyPages.set(route, { sourceUrl: ref.sourceUrl, screenshot: Buffer.from(ref.screenshot) });
|
|
252
|
+
}
|
|
511
253
|
const verifyResult = await verifyProject({
|
|
512
254
|
projectDir: outDir,
|
|
513
255
|
pages: verifyPages,
|
|
514
256
|
threshold: verifyThreshold,
|
|
257
|
+
minFidelity: verifyMinFidelity,
|
|
258
|
+
fullPage: verifyFullPage,
|
|
515
259
|
onProgress: (msg) => progress("verify", msg),
|
|
516
260
|
...(browser === undefined ? {} : { browser }),
|
|
517
261
|
});
|
|
262
|
+
/*
|
|
263
|
+
* Build errors were logged inside the verifier and dropped there. They are warnings on the
|
|
264
|
+
* import as a whole: a project that did not compile is not a clone of anything, however the
|
|
265
|
+
* pixels happen to score.
|
|
266
|
+
*/
|
|
267
|
+
for (const error of verifyResult.buildErrors) {
|
|
268
|
+
warn("verify", `Build error: ${error}`);
|
|
269
|
+
}
|
|
518
270
|
for (const page of verifyResult.pages) {
|
|
519
271
|
const status = page.error ? `ERROR: ${page.error}` : `${page.fidelity}% fidelity`;
|
|
520
|
-
|
|
272
|
+
const misses =
|
|
273
|
+
page.failedRequests.length > 0 ? `, ${page.failedRequests.length} failed request(s)` : "";
|
|
274
|
+
progress("verify", `${page.route} — ${status}${misses}`);
|
|
521
275
|
}
|
|
522
276
|
progress("verify", `Average fidelity: ${verifyResult.averageFidelity}%`);
|
|
277
|
+
if (!verifyResult.passed) {
|
|
278
|
+
warn(
|
|
279
|
+
"verify",
|
|
280
|
+
`Verification failed: average fidelity ${verifyResult.averageFidelity}% ` +
|
|
281
|
+
`(minimum ${verifyMinFidelity}%). See ${verifyResult.reportDir}/report.json`,
|
|
282
|
+
);
|
|
283
|
+
}
|
|
523
284
|
return {
|
|
524
285
|
averageFidelity: verifyResult.averageFidelity,
|
|
525
286
|
reportDir: verifyResult.reportDir,
|
|
287
|
+
passed: verifyResult.passed,
|
|
288
|
+
minFidelity: verifyMinFidelity,
|
|
289
|
+
buildErrors: verifyResult.buildErrors,
|
|
290
|
+
pages: verifyResult.pages.map((page) => {
|
|
291
|
+
const entry: {
|
|
292
|
+
route: string;
|
|
293
|
+
fidelity: number;
|
|
294
|
+
consoleErrors: number;
|
|
295
|
+
failedRequests: number;
|
|
296
|
+
error?: string;
|
|
297
|
+
} = {
|
|
298
|
+
route: page.route,
|
|
299
|
+
fidelity: page.fidelity,
|
|
300
|
+
consoleErrors: page.consoleErrors.length,
|
|
301
|
+
failedRequests: page.failedRequests.length,
|
|
302
|
+
};
|
|
303
|
+
if (page.error !== undefined) {
|
|
304
|
+
entry.error = page.error;
|
|
305
|
+
}
|
|
306
|
+
return entry;
|
|
307
|
+
}),
|
|
526
308
|
};
|
|
527
309
|
}
|
|
528
310
|
}
|
|
311
|
+
|
|
312
|
+
function throwIfAborted(signal?: AbortSignal): void {
|
|
313
|
+
if (signal?.aborted) {
|
|
314
|
+
throw new Error("Import aborted");
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Forward the options this wrapper shares with the pipeline, and only the ones actually given.
|
|
320
|
+
*
|
|
321
|
+
* `exactOptionalPropertyTypes` is on, so `{ signal: undefined }` is a different thing from an
|
|
322
|
+
* absent `signal` and the pipeline would reject it. Copying present keys is also what keeps every
|
|
323
|
+
* default in ONE place — restating `maxPages = 25` here is how the CLI and the cloud drift apart.
|
|
324
|
+
*/
|
|
325
|
+
function pick<T extends object, K extends keyof T>(source: T, keys: readonly K[]): Pick<T, K> {
|
|
326
|
+
const out = {} as Pick<T, K>;
|
|
327
|
+
for (const key of keys) {
|
|
328
|
+
if (source[key] !== undefined) {
|
|
329
|
+
out[key] = source[key];
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
return out;
|
|
333
|
+
}
|