@jxsuite/import 0.40.0 → 0.40.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -4
- package/src/ai-componentize.ts +12 -2
- package/src/asset-collect.ts +2 -2
- package/src/asset-download.ts +10 -11
- package/src/browser-local.ts +67 -0
- package/src/capture.ts +50 -61
- package/src/crawl.ts +16 -16
- package/src/emit.ts +34 -28
- package/src/index.ts +11 -5
- package/src/io.ts +52 -0
- package/src/media-extract.ts +3 -3
- package/src/pipeline.ts +605 -0
- package/src/run.ts +93 -503
- package/src/style-capture.ts +6 -3
- package/src/verify.ts +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jxsuite/import",
|
|
3
|
-
"version": "0.40.
|
|
3
|
+
"version": "0.40.1",
|
|
4
4
|
"description": "Clone a live website into a Jx project",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -20,6 +20,8 @@
|
|
|
20
20
|
"./capture": "./src/capture.ts",
|
|
21
21
|
"./to-jx": "./src/to-jx.ts",
|
|
22
22
|
"./emit": "./src/emit.ts",
|
|
23
|
+
"./io": "./src/io.ts",
|
|
24
|
+
"./pipeline": "./src/pipeline.ts",
|
|
23
25
|
"./run": "./src/run.ts",
|
|
24
26
|
"./verify": "./src/verify.ts",
|
|
25
27
|
"./breakpoint-plan": "./src/breakpoint-plan.ts"
|
|
@@ -33,8 +35,8 @@
|
|
|
33
35
|
"import": "bun run src/cli.ts"
|
|
34
36
|
},
|
|
35
37
|
"dependencies": {
|
|
36
|
-
"@jxsuite/markup": "^0.4.
|
|
37
|
-
"@jxsuite/schema": "^2.
|
|
38
|
+
"@jxsuite/markup": "^0.4.9",
|
|
39
|
+
"@jxsuite/schema": "^2.1.0",
|
|
38
40
|
"pixelmatch": "^7.2.0",
|
|
39
41
|
"pngjs": "^7.0.0",
|
|
40
42
|
"puppeteer-core": "^24.9.0"
|
|
@@ -44,6 +46,6 @@
|
|
|
44
46
|
"happy-dom": "^20.10.6"
|
|
45
47
|
},
|
|
46
48
|
"optionalDependencies": {
|
|
47
|
-
"@jxsuite/compiler": "^3.
|
|
49
|
+
"@jxsuite/compiler": "^3.1.0"
|
|
48
50
|
}
|
|
49
51
|
}
|
package/src/ai-componentize.ts
CHANGED
|
@@ -15,7 +15,17 @@ import type { ExtractedComponent, ComponentizeResult } from "./componentize.ts";
|
|
|
15
15
|
export interface AiComponentizeOptions {
|
|
16
16
|
apiKey: string;
|
|
17
17
|
baseUrl?: string | undefined;
|
|
18
|
-
|
|
18
|
+
/**
|
|
19
|
+
* The model id, and it has no default on purpose.
|
|
20
|
+
*
|
|
21
|
+
* `"gpt-4o-mini"` used to be one, which was harmless while the only caller was the OSS CLI
|
|
22
|
+
* pointing at OpenAI and wrong the moment a backend brokered anything else: Workers AI answers a
|
|
23
|
+
* model it does not know with a 404, `callLlm` reads that as "the LLM failed", and every
|
|
24
|
+
* component silently keeps its heuristic `component-div-0` name. A required id makes the caller
|
|
25
|
+
* that knows which provider it is talking to say so. The CLI and the OSS server still default to
|
|
26
|
+
* `gpt-4o-mini` — at their own call site, in `run.ts`.
|
|
27
|
+
*/
|
|
28
|
+
model: string;
|
|
19
29
|
}
|
|
20
30
|
|
|
21
31
|
interface RenameResult {
|
|
@@ -58,7 +68,7 @@ Respond with ONLY the JSON object, no markdown fences or explanation.`;
|
|
|
58
68
|
|
|
59
69
|
async function callLlm(prompt: string, opts: AiComponentizeOptions): Promise<RenameResult | null> {
|
|
60
70
|
const baseUrl = opts.baseUrl || "https://api.openai.com/v1";
|
|
61
|
-
const model = opts
|
|
71
|
+
const { model } = opts;
|
|
62
72
|
|
|
63
73
|
const response = await fetch(`${baseUrl}/chat/completions`, {
|
|
64
74
|
method: "POST",
|
package/src/asset-collect.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* SVGs are kept as-is (no download needed). Returns absolute URLs for downloading.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import type {
|
|
8
|
+
import type { ImportPage } from "./capture.ts";
|
|
9
9
|
import { SRCSET_SEPARATOR } from "./srcset.ts";
|
|
10
10
|
|
|
11
11
|
export interface DiscoveredAsset {
|
|
@@ -46,7 +46,7 @@ export interface AssetCollectionResult {
|
|
|
46
46
|
* Collect all asset URLs from the live page DOM and stylesheets. Runs entirely in-browser via
|
|
47
47
|
* page.evaluate() — no per-element round-trips.
|
|
48
48
|
*/
|
|
49
|
-
export async function collectAssets(page:
|
|
49
|
+
export async function collectAssets(page: ImportPage): Promise<AssetCollectionResult> {
|
|
50
50
|
/*
|
|
51
51
|
* The separator crosses into the page as a SOURCE STRING: an evaluated body is serialised and
|
|
52
52
|
* re-parsed in the browser, so it cannot close over a value declared out here.
|
package/src/asset-download.ts
CHANGED
|
@@ -5,9 +5,9 @@
|
|
|
5
5
|
* subdirectories by type: images/, fonts/, icons/.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import {
|
|
9
|
-
import { mkdir } from "node:fs/promises";
|
|
8
|
+
import { extname } from "node:path";
|
|
10
9
|
import type { DiscoveredAsset } from "./asset-collect.ts";
|
|
10
|
+
import type { ImportIo } from "./io.ts";
|
|
11
11
|
|
|
12
12
|
export interface DownloadResult {
|
|
13
13
|
/**
|
|
@@ -127,7 +127,7 @@ function isBlocked(url: string): boolean {
|
|
|
127
127
|
|
|
128
128
|
export async function downloadAssets(
|
|
129
129
|
assets: DiscoveredAsset[],
|
|
130
|
-
|
|
130
|
+
io: ImportIo,
|
|
131
131
|
sourceUrl?: string,
|
|
132
132
|
): Promise<DownloadResult> {
|
|
133
133
|
const rewriteMap = new Map<string, string>();
|
|
@@ -143,12 +143,11 @@ export async function downloadAssets(
|
|
|
143
143
|
}
|
|
144
144
|
}
|
|
145
145
|
|
|
146
|
-
// Create subdirectories
|
|
147
|
-
const assetsDir =
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
await mkdir(join(assetsDir, "other"), { recursive: true });
|
|
146
|
+
// Create subdirectories (a sink with no directories declines — see ImportIo.mkdir)
|
|
147
|
+
const assetsDir = "public/assets";
|
|
148
|
+
for (const subdir of ["images", "fonts", "icons", "other"]) {
|
|
149
|
+
await io.mkdir?.(`${assetsDir}/${subdir}`);
|
|
150
|
+
}
|
|
152
151
|
|
|
153
152
|
// Track filenames to avoid collisions
|
|
154
153
|
const usedNames = new Map<string, number>();
|
|
@@ -172,7 +171,7 @@ export async function downloadAssets(
|
|
|
172
171
|
}
|
|
173
172
|
usedNames.set(nameKey, count + 1);
|
|
174
173
|
|
|
175
|
-
const destPath =
|
|
174
|
+
const destPath = `${assetsDir}/${subdir}/${filename}`;
|
|
176
175
|
// The site-absolute path the built asset is served from. `public/` is the compiler's static
|
|
177
176
|
// ROOT — its contents land at dist/<path>, so the "public/" segment must not survive into a
|
|
178
177
|
// Reference, and the leading slash must be there or a reference from /components/x.js or a
|
|
@@ -195,7 +194,7 @@ export async function downloadAssets(
|
|
|
195
194
|
|
|
196
195
|
const buffer = await response.arrayBuffer();
|
|
197
196
|
totalBytes += buffer.byteLength;
|
|
198
|
-
await
|
|
197
|
+
await io.write(destPath, new Uint8Array(buffer));
|
|
199
198
|
rewriteMap.set(url, relativePath);
|
|
200
199
|
} catch {
|
|
201
200
|
failed.push(url);
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Launching a real Chrome on this machine — the one module in the import that loads
|
|
3
|
+
* `puppeteer-core` as a VALUE.
|
|
4
|
+
*
|
|
5
|
+
* It is separate from `capture.ts` for a bundling reason rather than a tidiness one.
|
|
6
|
+
* `puppeteer-core` cannot run in workerd at all (it reaches for `child_process`, `net` and a binary
|
|
7
|
+
* on disk), and a Worker bundler follows value imports transitively — so a single `import { launch
|
|
8
|
+
* }` anywhere in `pipeline.ts`'s graph would pull the whole of it in and fail the deploy. Keeping
|
|
9
|
+
* the launch here, and having every phase take a browser it was HANDED, is what lets Jx Cloud run
|
|
10
|
+
* the same pipeline against a `@cloudflare/puppeteer` session.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { launch } from "puppeteer-core";
|
|
14
|
+
import type { Browser } from "puppeteer-core";
|
|
15
|
+
|
|
16
|
+
const DEFAULT_CHROME_PATHS = [
|
|
17
|
+
"google-chrome-stable",
|
|
18
|
+
"google-chrome",
|
|
19
|
+
"chromium-browser",
|
|
20
|
+
"chromium",
|
|
21
|
+
];
|
|
22
|
+
|
|
23
|
+
function findChrome(executablePath?: string): string {
|
|
24
|
+
if (executablePath) {
|
|
25
|
+
return executablePath;
|
|
26
|
+
}
|
|
27
|
+
const env = process.env.CHROME_PATH;
|
|
28
|
+
if (env) {
|
|
29
|
+
return env;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
for (const name of DEFAULT_CHROME_PATHS) {
|
|
33
|
+
const path = Bun.which(name);
|
|
34
|
+
if (path) {
|
|
35
|
+
return path;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
throw new Error(
|
|
39
|
+
"Could not find Chrome/Chromium. Set CHROME_PATH or install google-chrome-stable.",
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
let _browser: Browser | null = null;
|
|
44
|
+
|
|
45
|
+
export interface LaunchOptions {
|
|
46
|
+
/** Explicit browser binary. Wins over CHROME_PATH and PATH discovery. */
|
|
47
|
+
executablePath?: string;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export async function launchBrowser(options?: LaunchOptions): Promise<Browser> {
|
|
51
|
+
if (_browser?.connected) {
|
|
52
|
+
return _browser;
|
|
53
|
+
}
|
|
54
|
+
_browser = await launch({
|
|
55
|
+
executablePath: findChrome(options?.executablePath),
|
|
56
|
+
headless: true,
|
|
57
|
+
args: ["--no-sandbox", "--disable-gpu"],
|
|
58
|
+
});
|
|
59
|
+
return _browser;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export async function closeBrowser(): Promise<void> {
|
|
63
|
+
if (_browser?.connected) {
|
|
64
|
+
await _browser.close();
|
|
65
|
+
_browser = null;
|
|
66
|
+
}
|
|
67
|
+
}
|
package/src/capture.ts
CHANGED
|
@@ -1,5 +1,44 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Page capture, and the browser surface the whole import is written against.
|
|
3
|
+
*
|
|
4
|
+
* That surface is declared STRUCTURALLY here rather than imported from `puppeteer-core`, and the
|
|
5
|
+
* reason is bundling, not taste: `puppeteer-core` is unloadable in workerd, and a Worker bundler
|
|
6
|
+
* follows value imports transitively, so a single one anywhere in `pipeline.ts`'s graph would fail
|
|
7
|
+
* the deploy. Naming only `newPage` — and, on a page, the five calls the phases actually make —
|
|
8
|
+
* lets the same code drive a local Chrome (`browser-local.ts`) or a `@cloudflare/puppeteer`
|
|
9
|
+
* session, both of which provide exactly this and more.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* What `evaluate` hands the in-page function.
|
|
14
|
+
*
|
|
15
|
+
* A homomorphic mapped type rather than plain `Params`, because puppeteer's own `evaluate` declares
|
|
16
|
+
* its parameters through one (it unwraps `JSHandle`s there). Two generic signatures only relate
|
|
17
|
+
* when they are shaped alike, and with a bare tuple puppeteer's `Page` stops being assignable to
|
|
18
|
+
* `ImportPage` — which is the one thing this interface has to be true of.
|
|
19
|
+
*/
|
|
20
|
+
export type PageArgs<Params extends unknown[]> = { [K in keyof Params]: Params[K] };
|
|
21
|
+
|
|
22
|
+
export interface ImportPage {
|
|
23
|
+
goto: (
|
|
24
|
+
url: string,
|
|
25
|
+
options?: {
|
|
26
|
+
waitUntil?: "load" | "domcontentloaded" | "networkidle0" | "networkidle2";
|
|
27
|
+
timeout?: number;
|
|
28
|
+
},
|
|
29
|
+
) => Promise<unknown>;
|
|
30
|
+
setViewport: (viewport: { width: number; height: number }) => Promise<unknown>;
|
|
31
|
+
evaluate: <Params extends unknown[], R>(
|
|
32
|
+
fn: (...params: PageArgs<Params>) => R,
|
|
33
|
+
...args: Params
|
|
34
|
+
) => Promise<Awaited<R>>;
|
|
35
|
+
screenshot: (options?: { fullPage?: boolean; type?: "png" }) => Promise<Uint8Array>;
|
|
36
|
+
close: () => Promise<unknown>;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface ImportBrowser {
|
|
40
|
+
newPage: () => Promise<ImportPage>;
|
|
41
|
+
}
|
|
3
42
|
|
|
4
43
|
export interface CaptureResult {
|
|
5
44
|
url: string;
|
|
@@ -7,61 +46,8 @@ export interface CaptureResult {
|
|
|
7
46
|
bodyHtml: string;
|
|
8
47
|
/** Discovered same-origin links for the crawler (Phase 3). */
|
|
9
48
|
links: string[];
|
|
10
|
-
/** The
|
|
11
|
-
page:
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
const DEFAULT_CHROME_PATHS = [
|
|
15
|
-
"google-chrome-stable",
|
|
16
|
-
"google-chrome",
|
|
17
|
-
"chromium-browser",
|
|
18
|
-
"chromium",
|
|
19
|
-
];
|
|
20
|
-
|
|
21
|
-
function findChrome(executablePath?: string): string {
|
|
22
|
-
if (executablePath) {
|
|
23
|
-
return executablePath;
|
|
24
|
-
}
|
|
25
|
-
const env = process.env.CHROME_PATH;
|
|
26
|
-
if (env) {
|
|
27
|
-
return env;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
for (const name of DEFAULT_CHROME_PATHS) {
|
|
31
|
-
const path = Bun.which(name);
|
|
32
|
-
if (path) {
|
|
33
|
-
return path;
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
throw new Error(
|
|
37
|
-
"Could not find Chrome/Chromium. Set CHROME_PATH or install google-chrome-stable.",
|
|
38
|
-
);
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
let _browser: Browser | null = null;
|
|
42
|
-
|
|
43
|
-
export interface LaunchOptions {
|
|
44
|
-
/** Explicit browser binary. Wins over CHROME_PATH and PATH discovery. */
|
|
45
|
-
executablePath?: string;
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
export async function launchBrowser(options?: LaunchOptions): Promise<Browser> {
|
|
49
|
-
if (_browser?.connected) {
|
|
50
|
-
return _browser;
|
|
51
|
-
}
|
|
52
|
-
_browser = await launch({
|
|
53
|
-
executablePath: findChrome(options?.executablePath),
|
|
54
|
-
headless: true,
|
|
55
|
-
args: ["--no-sandbox", "--disable-gpu"],
|
|
56
|
-
});
|
|
57
|
-
return _browser;
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
export async function closeBrowser(): Promise<void> {
|
|
61
|
-
if (_browser?.connected) {
|
|
62
|
-
await _browser.close();
|
|
63
|
-
_browser = null;
|
|
64
|
-
}
|
|
49
|
+
/** The page, kept open for style capture (Phase 1). Caller must close. */
|
|
50
|
+
page: ImportPage;
|
|
65
51
|
}
|
|
66
52
|
|
|
67
53
|
export interface CaptureOptions {
|
|
@@ -73,7 +59,7 @@ export interface CaptureOptions {
|
|
|
73
59
|
* Scroll the page to the bottom in steps to trigger lazy-loaded images and intersection-observer
|
|
74
60
|
* content, then scroll back to top. Settles between steps to let content render.
|
|
75
61
|
*/
|
|
76
|
-
async function scrollToRevealAll(page:
|
|
62
|
+
async function scrollToRevealAll(page: ImportPage): Promise<void> {
|
|
77
63
|
await page.evaluate(async () => {
|
|
78
64
|
const delay = (ms: number) =>
|
|
79
65
|
new Promise<void>((r) => {
|
|
@@ -111,15 +97,18 @@ async function scrollToRevealAll(page: Page): Promise<void> {
|
|
|
111
97
|
/**
|
|
112
98
|
* Capture a page's DOM. Returns the page object still open — the caller is responsible for closing
|
|
113
99
|
* it (after style capture in Phase 1, or immediately).
|
|
100
|
+
*
|
|
101
|
+
* The browser is a parameter and has no default. It used to fall back to launching a local Chrome,
|
|
102
|
+
* and that fallback was the value import of `puppeteer-core` that kept this module — and everything
|
|
103
|
+
* that reaches it — out of a Worker.
|
|
114
104
|
*/
|
|
115
105
|
export async function capturePage(
|
|
116
106
|
url: string,
|
|
117
|
-
browser
|
|
107
|
+
browser: ImportBrowser,
|
|
118
108
|
options?: CaptureOptions,
|
|
119
109
|
): Promise<CaptureResult> {
|
|
120
110
|
const { scrollToBottom = true } = options ?? {};
|
|
121
|
-
const
|
|
122
|
-
const page: Page = await br.newPage();
|
|
111
|
+
const page = await browser.newPage();
|
|
123
112
|
await page.setViewport({ width: 1440, height: 900 });
|
|
124
113
|
|
|
125
114
|
await page.goto(url, { waitUntil: "networkidle0", timeout: 30_000 });
|
package/src/crawl.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import type { CaptureResult } from "./capture.ts";
|
|
1
|
+
import { capturePage } from "./capture.ts";
|
|
2
|
+
import type { CaptureResult, ImportBrowser } from "./capture.ts";
|
|
3
|
+
import type { ImportIo } from "./io.ts";
|
|
3
4
|
import { convertToJx } from "./to-jx.ts";
|
|
4
5
|
import type { ToJxResult } from "./to-jx.ts";
|
|
5
6
|
import { captureStyles } from "./style-capture.ts";
|
|
@@ -19,7 +20,10 @@ import { applyTokens } from "./css-tokens.ts";
|
|
|
19
20
|
|
|
20
21
|
export interface CrawlOptions {
|
|
21
22
|
url: string;
|
|
22
|
-
|
|
23
|
+
/** The browser to capture with. The crawl never launches one — see `browser-local.ts`. */
|
|
24
|
+
browser: ImportBrowser;
|
|
25
|
+
/** Where downloaded assets go. */
|
|
26
|
+
io: ImportIo;
|
|
23
27
|
maxDepth: number;
|
|
24
28
|
maxPages: number;
|
|
25
29
|
maxNodesPerPage: number;
|
|
@@ -52,8 +56,8 @@ export interface CrawledPage {
|
|
|
52
56
|
jx: ToJxResult;
|
|
53
57
|
depth: number;
|
|
54
58
|
links: string[];
|
|
55
|
-
/** Reference screenshot PNG
|
|
56
|
-
screenshot?:
|
|
59
|
+
/** Reference screenshot PNG bytes, present when captureScreenshots is true. */
|
|
60
|
+
screenshot?: Uint8Array | undefined;
|
|
57
61
|
}
|
|
58
62
|
|
|
59
63
|
export interface CrawlResult {
|
|
@@ -187,7 +191,8 @@ function isDisallowed(url: string, disallowedPaths: Set<string>): boolean {
|
|
|
187
191
|
export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
|
|
188
192
|
const {
|
|
189
193
|
url,
|
|
190
|
-
|
|
194
|
+
browser,
|
|
195
|
+
io,
|
|
191
196
|
maxDepth,
|
|
192
197
|
maxPages,
|
|
193
198
|
maxNodesPerPage,
|
|
@@ -199,7 +204,6 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
|
|
|
199
204
|
} = options;
|
|
200
205
|
|
|
201
206
|
const { origin } = new URL(url);
|
|
202
|
-
const browser = await launchBrowser();
|
|
203
207
|
|
|
204
208
|
// Fetch robots.txt
|
|
205
209
|
let disallowedPaths = new Set<string>();
|
|
@@ -261,11 +265,9 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
|
|
|
261
265
|
` ⚠ Page exceeds node cap (${jx.nodeCount} > ${maxNodesPerPage}), skipping styles/assets`,
|
|
262
266
|
);
|
|
263
267
|
skippedByNodeCap.push(entry.url);
|
|
264
|
-
let screenshot:
|
|
268
|
+
let screenshot: Uint8Array | undefined;
|
|
265
269
|
if (options.captureScreenshots) {
|
|
266
|
-
screenshot =
|
|
267
|
-
await capture.page.screenshot({ fullPage: fullPageScreenshots, type: "png" }),
|
|
268
|
-
);
|
|
270
|
+
screenshot = await capture.page.screenshot({ fullPage: fullPageScreenshots, type: "png" });
|
|
269
271
|
}
|
|
270
272
|
await capture.page.close();
|
|
271
273
|
|
|
@@ -363,7 +365,7 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
|
|
|
363
365
|
}
|
|
364
366
|
|
|
365
367
|
if (collected.assets.length > 0) {
|
|
366
|
-
const downloaded = await downloadAssets(collected.assets,
|
|
368
|
+
const downloaded = await downloadAssets(collected.assets, io, entry.url);
|
|
367
369
|
if (downloaded.rewriteMap.size > 0) {
|
|
368
370
|
rewriteAssetUrls(jx.document, downloaded.rewriteMap, entry.url);
|
|
369
371
|
// Collect font rewrites (R2)
|
|
@@ -381,11 +383,9 @@ export async function crawlSite(options: CrawlOptions): Promise<CrawlResult> {
|
|
|
381
383
|
}
|
|
382
384
|
}
|
|
383
385
|
|
|
384
|
-
let screenshot:
|
|
386
|
+
let screenshot: Uint8Array | undefined;
|
|
385
387
|
if (options.captureScreenshots) {
|
|
386
|
-
screenshot =
|
|
387
|
-
await capture.page.screenshot({ fullPage: fullPageScreenshots, type: "png" }),
|
|
388
|
-
);
|
|
388
|
+
screenshot = await capture.page.screenshot({ fullPage: fullPageScreenshots, type: "png" });
|
|
389
389
|
}
|
|
390
390
|
await capture.page.close();
|
|
391
391
|
|
package/src/emit.ts
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import { join, dirname } from "node:path";
|
|
2
|
-
import { mkdir } from "node:fs/promises";
|
|
3
1
|
import type { JxElement } from "@jxsuite/schema/types";
|
|
4
2
|
import { componentize } from "./componentize.ts";
|
|
5
3
|
import { stripClasses } from "./strip-classes.ts";
|
|
6
4
|
import type { ComponentizeOptions, ComponentizeResult } from "./componentize.ts";
|
|
5
|
+
import type { ImportIo } from "./io.ts";
|
|
7
6
|
|
|
8
7
|
/**
|
|
9
8
|
* The `$media` key the base width lives under — a number of CSS pixels, not a query.
|
|
@@ -21,14 +20,22 @@ function mediaKeyWidth(key: string): number {
|
|
|
21
20
|
}
|
|
22
21
|
|
|
23
22
|
export interface EmitResult {
|
|
24
|
-
/**
|
|
23
|
+
/** Project-relative, forward-slashed paths of every file written, in the order they were written. */
|
|
25
24
|
files: string[];
|
|
26
25
|
/** How many source-site class names were removed on the way out. */
|
|
27
26
|
classesStripped: number;
|
|
27
|
+
/**
|
|
28
|
+
* The exact `project.json` text that was written.
|
|
29
|
+
*
|
|
30
|
+
* Returned rather than left to be read back, because a caller with no filesystem cannot read it
|
|
31
|
+
* back at all: Jx Cloud commits the emitted file set to git and needs the configuration to name
|
|
32
|
+
* the project, and re-deriving it here would be a second writer on the same bytes.
|
|
33
|
+
*/
|
|
34
|
+
projectJson: string;
|
|
28
35
|
}
|
|
29
36
|
|
|
30
37
|
export interface EmitOptions {
|
|
31
|
-
|
|
38
|
+
io: ImportIo;
|
|
32
39
|
title: string;
|
|
33
40
|
document: JxElement;
|
|
34
41
|
sourceUrl: string;
|
|
@@ -41,7 +48,8 @@ export interface EmitOptions {
|
|
|
41
48
|
}
|
|
42
49
|
|
|
43
50
|
export interface MultiEmitOptions {
|
|
44
|
-
|
|
51
|
+
/** Where the emitted files go. See `io.ts` — a disk is one implementation, not the requirement. */
|
|
52
|
+
io: ImportIo;
|
|
45
53
|
title: string;
|
|
46
54
|
sourceUrl: string;
|
|
47
55
|
/** Map of route path (e.g. "pages/index.json") → Jx document. */
|
|
@@ -65,7 +73,7 @@ export interface MultiEmitOptions {
|
|
|
65
73
|
}
|
|
66
74
|
|
|
67
75
|
export async function emitProject({
|
|
68
|
-
|
|
76
|
+
io,
|
|
69
77
|
title,
|
|
70
78
|
document,
|
|
71
79
|
sourceUrl,
|
|
@@ -74,7 +82,7 @@ export async function emitProject({
|
|
|
74
82
|
componentizeOptions,
|
|
75
83
|
}: EmitOptions): Promise<EmitResult> {
|
|
76
84
|
return emitMultiPageProject({
|
|
77
|
-
|
|
85
|
+
io,
|
|
78
86
|
title,
|
|
79
87
|
sourceUrl,
|
|
80
88
|
pages: new Map([["pages/index.json", document]]),
|
|
@@ -85,7 +93,7 @@ export async function emitProject({
|
|
|
85
93
|
}
|
|
86
94
|
|
|
87
95
|
export async function emitMultiPageProject({
|
|
88
|
-
|
|
96
|
+
io,
|
|
89
97
|
title,
|
|
90
98
|
pages,
|
|
91
99
|
layout,
|
|
@@ -97,10 +105,12 @@ export async function emitMultiPageProject({
|
|
|
97
105
|
fontRewriteMap,
|
|
98
106
|
styleTokens,
|
|
99
107
|
}: MultiEmitOptions): Promise<EmitResult> {
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
108
|
+
/* Seeded even when nothing lands in them, so a host that opened the project at the seed event
|
|
109
|
+
sees the shape it expects rather than a directory that grows one folder at a time. A sink with
|
|
110
|
+
no directories declines by not implementing it. */
|
|
111
|
+
for (const dir of ["pages", "layouts", "components", "public"]) {
|
|
112
|
+
await io.mkdir?.(dir);
|
|
113
|
+
}
|
|
104
114
|
|
|
105
115
|
/*
|
|
106
116
|
* Only keys the project schema declares (#228). `title` and `description` are page-level keys and
|
|
@@ -207,10 +217,8 @@ export async function emitMultiPageProject({
|
|
|
207
217
|
fontCss = fontCss.replaceAll(pattern, (m) => byForm.get(m) ?? m);
|
|
208
218
|
}
|
|
209
219
|
}
|
|
210
|
-
const
|
|
211
|
-
await
|
|
212
|
-
const fontsCssPath = join(fontsDir, "fonts.css");
|
|
213
|
-
await Bun.write(fontsCssPath, fontCss);
|
|
220
|
+
const fontsCssPath = "public/assets/fonts.css";
|
|
221
|
+
await io.write(fontsCssPath, fontCss);
|
|
214
222
|
files.push(fontsCssPath);
|
|
215
223
|
|
|
216
224
|
// Add fonts.css to project head
|
|
@@ -223,9 +231,9 @@ export async function emitMultiPageProject({
|
|
|
223
231
|
});
|
|
224
232
|
}
|
|
225
233
|
|
|
226
|
-
const
|
|
227
|
-
await
|
|
228
|
-
files.push(
|
|
234
|
+
const projectText = `${JSON.stringify(projectJson, null, 2)}\n`;
|
|
235
|
+
await io.write("project.json", projectText);
|
|
236
|
+
files.push("project.json");
|
|
229
237
|
|
|
230
238
|
/*
|
|
231
239
|
* Classes go here, at the last possible moment, and that placement is the whole design.
|
|
@@ -245,8 +253,8 @@ export async function emitMultiPageProject({
|
|
|
245
253
|
|
|
246
254
|
// Write components
|
|
247
255
|
for (const [fileName, compDoc] of componentFiles) {
|
|
248
|
-
const compPath =
|
|
249
|
-
await
|
|
256
|
+
const compPath = `components/${fileName}`;
|
|
257
|
+
await io.write(compPath, `${JSON.stringify(compDoc, null, 2)}\n`);
|
|
250
258
|
files.push(compPath);
|
|
251
259
|
}
|
|
252
260
|
|
|
@@ -267,14 +275,12 @@ export async function emitMultiPageProject({
|
|
|
267
275
|
...((pageDoc.$elements as JxElement[]) ?? []),
|
|
268
276
|
] as JxElement[];
|
|
269
277
|
}
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
await Bun.write(pagePath, `${JSON.stringify(pageDoc, null, 2)}\n`);
|
|
273
|
-
files.push(pagePath);
|
|
278
|
+
await io.write(route, `${JSON.stringify(pageDoc, null, 2)}\n`);
|
|
279
|
+
files.push(route);
|
|
274
280
|
}
|
|
275
281
|
|
|
276
282
|
// Write layout
|
|
277
|
-
const layoutPath =
|
|
283
|
+
const layoutPath = "layouts/base.json";
|
|
278
284
|
const layoutDoc: JxElement = layout ?? {
|
|
279
285
|
tagName: "div",
|
|
280
286
|
children: [{ tagName: "slot" }],
|
|
@@ -286,10 +292,10 @@ export async function emitMultiPageProject({
|
|
|
286
292
|
];
|
|
287
293
|
}
|
|
288
294
|
classesStripped += stripClasses(layoutDoc);
|
|
289
|
-
await
|
|
295
|
+
await io.write(layoutPath, `${JSON.stringify(layoutDoc, null, 2)}\n`);
|
|
290
296
|
files.push(layoutPath);
|
|
291
297
|
|
|
292
|
-
return { classesStripped, files };
|
|
298
|
+
return { classesStripped, files, projectJson: projectText };
|
|
293
299
|
}
|
|
294
300
|
|
|
295
301
|
function extractStateDefaults(node: JxElement | string, out: Record<string, string>) {
|
package/src/index.ts
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
export { capturePage
|
|
1
|
+
export { capturePage } from "./capture.ts";
|
|
2
|
+
export { launchBrowser, closeBrowser } from "./browser-local.ts";
|
|
3
|
+
export { createLocalIo } from "./io.ts";
|
|
4
|
+
export { runImportPipeline } from "./pipeline.ts";
|
|
2
5
|
export { importSite } from "./run.ts";
|
|
3
6
|
export { convertToJx } from "./to-jx.ts";
|
|
4
7
|
export { emitProject, emitMultiPageProject } from "./emit.ts";
|
|
@@ -33,13 +36,16 @@ export {
|
|
|
33
36
|
serveDirectory,
|
|
34
37
|
routeToUrlPath,
|
|
35
38
|
} from "./verify.ts";
|
|
36
|
-
export type { CaptureResult,
|
|
39
|
+
export type { CaptureResult, ImportBrowser, ImportPage } from "./capture.ts";
|
|
40
|
+
export type { LaunchOptions } from "./browser-local.ts";
|
|
41
|
+
export type { ImportIo } from "./io.ts";
|
|
37
42
|
export type {
|
|
38
|
-
|
|
43
|
+
ImportPipelineOptions,
|
|
44
|
+
ImportPipelineResult,
|
|
39
45
|
ImportProgressEvent,
|
|
40
46
|
ImportPhase,
|
|
41
|
-
|
|
42
|
-
} from "./run.ts";
|
|
47
|
+
} from "./pipeline.ts";
|
|
48
|
+
export type { ImportSiteOptions, ImportSiteResult } from "./run.ts";
|
|
43
49
|
export type { ToJxResult } from "./to-jx.ts";
|
|
44
50
|
export type { EmitOptions, EmitResult, MultiEmitOptions } from "./emit.ts";
|
|
45
51
|
export type { CapturedStyle, StyleCaptureResult } from "./style-capture.ts";
|
package/src/io.ts
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The sink every file an import produces goes through.
|
|
3
|
+
*
|
|
4
|
+
* The pipeline used to call `Bun.write(join(outDir, …))` at six sites, which made "run an import"
|
|
5
|
+
* and "have a writable filesystem" the same requirement. Jx Cloud is a Cloudflare Worker: it has
|
|
6
|
+
* neither `Bun` nor a disk, and commits the emitted project straight to git. So the destination is
|
|
7
|
+
* a parameter now — `createLocalIo` is the disk implementation and the only one that imports
|
|
8
|
+
* `node:fs`, and a Worker supplies its own without the pipeline knowing the difference.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { mkdir } from "node:fs/promises";
|
|
12
|
+
import { dirname, join } from "node:path";
|
|
13
|
+
|
|
14
|
+
export interface ImportIo {
|
|
15
|
+
/**
|
|
16
|
+
* Write one file.
|
|
17
|
+
*
|
|
18
|
+
* `relPath` is PROJECT-RELATIVE and forward-slashed — `"pages/blog/post.json"`, never an absolute
|
|
19
|
+
* path and never a backslash. A sink that has no filesystem uses it as a key verbatim; a sink
|
|
20
|
+
* that does joins it onto its own root.
|
|
21
|
+
*/
|
|
22
|
+
write: (relPath: string, data: string | Uint8Array) => Promise<void>;
|
|
23
|
+
/**
|
|
24
|
+
* Create an empty directory, if directories are a thing this sink has.
|
|
25
|
+
*
|
|
26
|
+
* Optional because they usually are not: a git tree, an object store and a zip all record only
|
|
27
|
+
* the files, and an empty directory is unrepresentable in every one of them. The local import
|
|
28
|
+
* still seeds `pages/`, `layouts/`, `components/` and `public/` up front so a host can open the
|
|
29
|
+
* project the moment the run starts, which is what this exists for.
|
|
30
|
+
*/
|
|
31
|
+
mkdir?: (relPath: string) => Promise<void>;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Split a project-relative path into segments, so a sink's own separator is the one used. */
|
|
35
|
+
function toSegments(relPath: string): string[] {
|
|
36
|
+
return relPath.split("/").filter((segment) => segment !== "");
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** The disk sink: `outDir` plus the relative path, with parent directories created on demand. */
|
|
40
|
+
export function createLocalIo(outDir: string): ImportIo {
|
|
41
|
+
const resolve = (relPath: string): string => join(outDir, ...toSegments(relPath));
|
|
42
|
+
return {
|
|
43
|
+
async write(relPath, data) {
|
|
44
|
+
const path = resolve(relPath);
|
|
45
|
+
await mkdir(dirname(path), { recursive: true });
|
|
46
|
+
await Bun.write(path, data);
|
|
47
|
+
},
|
|
48
|
+
async mkdir(relPath) {
|
|
49
|
+
await mkdir(resolve(relPath), { recursive: true });
|
|
50
|
+
},
|
|
51
|
+
};
|
|
52
|
+
}
|