@trueclone/cli 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +145 -2
- package/bin/trueclone.mjs +146 -0
- package/package.json +62 -4
- package/scripts/capture.mjs +248 -0
- package/scripts/curves.mjs +254 -0
- package/scripts/design.mjs +491 -0
- package/scripts/detail.mjs +225 -0
- package/scripts/diff.mjs +88 -0
- package/scripts/hovers.mjs +116 -0
- package/scripts/interact.mjs +188 -0
- package/scripts/lib.mjs +423 -0
- package/scripts/motion.mjs +239 -0
- package/scripts/next.mjs +532 -0
- package/scripts/report.mjs +114 -0
- package/scripts/reveals.mjs +191 -0
- package/scripts/rules.mjs +109 -0
- package/scripts/score.mjs +82 -0
- package/scripts/seed.mjs +513 -0
- package/scripts/spec.mjs +78 -0
- package/scripts/states.mjs +173 -0
- package/scripts/textshape.mjs +384 -0
- package/scripts/tiles.mjs +25 -0
- package/scripts/widgets.mjs +169 -0
- package/skill/SKILL.md +158 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ayush Pratap Singh
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,146 @@
|
|
|
1
|
-
#
|
|
1
|
+
# trueclone
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
**A working twin of any website, from one URL, with scores that prove it.**
|
|
4
|
+
|
|
5
|
+
Give your coding agent a URL. trueclone captures the site, extracts its design system, guides the rebuild and measures the result against the original until it matches: layout, scroll states, motion curves, interactions, hover effects and computed styles, at phone, tablet and desktop widths. You get the clone as HTML and as a Next.js app, plus the design system as `DESIGN.md` and a shadcn `globals.css`.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npx @trueclone/cli init # once per project: installs trueclone, the skill for your agent, REA as an MCP server
|
|
9
|
+
# then tell Claude Code, Codex or Cursor:
|
|
10
|
+
# "clone https://example.com with trueclone"
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## What you get from one URL
|
|
14
|
+
|
|
15
|
+
| Output | What it is |
|
|
16
|
+
|---|---|
|
|
17
|
+
| `clone/` | The site in plain HTML, CSS and JS, interactions included (sliders, tabs, menus, canvas demos, scroll motion) |
|
|
18
|
+
| `next/` | The same site as a Next.js + Tailwind v4 + shadcn app, one component per section, scored against the original too |
|
|
19
|
+
| `DESIGN.md` | The design system in [Google's design.md format](https://github.com/google-labs-code/design.md), linted |
|
|
20
|
+
| `globals.css` | A shadcn theme (Tailwind v4, oklch) generated from the same tokens |
|
|
21
|
+
| `motion.md` | The motion recipe: every ScrollTrigger, tween, reveal and loop with its trigger, range, easing and timing |
|
|
22
|
+
| `REPORT.md` | Every score against its target, what to fix first, and what was not copied (server logic, tracking, each API call) |
|
|
23
|
+
|
|
24
|
+
## Proof
|
|
25
|
+
|
|
26
|
+
Three benchmarks with very different stacks, scored by the tool itself:
|
|
27
|
+
|
|
28
|
+
**[rea.tools](https://rea.tools/)**: static docs with a canvas game and interactive demos, exact mode (copied with the owner's permission).
|
|
29
|
+
|
|
30
|
+
| Check | Desktop | Tablet | Mobile |
|
|
31
|
+
|---|---|---|---|
|
|
32
|
+
| Layout, real pixels | 100% | 100% | 100% |
|
|
33
|
+
| Scroll states | 100% | 100% | 100% |
|
|
34
|
+
| Computed styles (≈280 points per width) | 100% | 100% | 100% |
|
|
35
|
+
| Hover effects | 100% | | |
|
|
36
|
+
| Interactions (copy, tabs, calculator, dinosaur game, language menu) | 5/5 | | |
|
|
37
|
+
|
|
38
|
+
**A Next.js + React + Framer Motion developer landing page** (12,000px, 11 scroll reveals with spring easing, a scroll-state header, marquees, view-mounted components), template mode, **seeded automatically**: no hand-written layout code, no original text or media in the clone.
|
|
39
|
+
|
|
40
|
+
| Check | Desktop | Tablet | Mobile |
|
|
41
|
+
|---|---|---|---|
|
|
42
|
+
| Layout, structure | 99.99% | 99.93% | 99.94% |
|
|
43
|
+
| Scroll states | 99.96% | 99.92% | 99.94% |
|
|
44
|
+
| Computed styles | 99.5% | 98.7% | 99.2% |
|
|
45
|
+
| Hover effects | 100% | | |
|
|
46
|
+
|
|
47
|
+
**A Webflow motorsport portfolio** built with GSAP, ScrollTrigger, SplitText, Lenis, Swiper and three.js (97 scroll triggers), template mode, built by hand before seeding existed:
|
|
48
|
+
|
|
49
|
+
| Check | Desktop | Tablet | Mobile |
|
|
50
|
+
|---|---|---|---|
|
|
51
|
+
| Layout, structure | 99.6% | 99.7% | 96.8% |
|
|
52
|
+
| Scroll states | 98.0% | 96.6% | 96.5% |
|
|
53
|
+
| Motion curves | 96.9% | 98.2% | 99.6% |
|
|
54
|
+
| Computed styles | 94% | 88.9% | 85.4% |
|
|
55
|
+
| Hover effects | 100% | | |
|
|
56
|
+
| Interactions (drag sliders, drag gallery, cursor follow, menu, skip) | 5/5 | | |
|
|
57
|
+
|
|
58
|
+
The hand-built portfolio isn't perfect yet, and its report says so: computed styles and mobile layout are below target (placeholder text in highlighted phrases, a scroll-linked colour fade, mobile spacing). The generated Next.js app scores within half a point of the HTML clone on every check. A full scoring run takes 1.5 to 9 minutes.
|
|
59
|
+
|
|
60
|
+
## How it works
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
URL ─► capture ─► understand ─► design system ─► build ─► score ─► report
|
|
64
|
+
3 widths motion DESIGN.md your 7 checks REPORT.md
|
|
65
|
+
HAR, HTML inventory globals.css agent in parallel
|
|
66
|
+
assets + REA tokens.css ▲ │
|
|
67
|
+
└─────┘ until every target passes
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
1. **Capture** (`trueclone capture`): full-page screenshots at 390, 768 and 1440px, sections, rendered HTML, assets, every network request.
|
|
71
|
+
2. **Understand** (`trueclone motion`, REA): reads the live animation runtime (GSAP timelines, ScrollTriggers, Lenis and Swiper settings, CSS animations, cursor-driven tweens) and, through REA, bundles, source maps, event listeners and API endpoints.
|
|
72
|
+
3. **Design system** (`trueclone design`): one canonical `design-tokens.json`, rendered to DESIGN.md, globals.css and the clone's tokens.
|
|
73
|
+
4. **Build**: `trueclone reveals` records the original's scroll reveals frame by frame and its scroll-driven class changes; `trueclone widgets` clicks every tab and menu and records each state; `trueclone seed` turns the script-free mirror into the clone (template mode: layout-measured lorem text and same-size media placeholders) and replays what was recorded. Layout is at 99%+ before anyone writes CSS; your agent writes the remaining behaviour, guided by the skill. `trueclone rules` shows the original's CSS for any element (like DevTools).
|
|
74
|
+
5. **Score** (`trueclone score`): layout pixel diff per section, scroll states replayed, motion curves sampled, interaction scenarios with change assertions, every hover, computed styles at matching points. Reference measurements are cached, so only the clone is re-measured.
|
|
75
|
+
6. **Report**: `REPORT.md` with the verdict, what to fix first, and what was not copied.
|
|
76
|
+
|
|
77
|
+
## Why not just REA?
|
|
78
|
+
|
|
79
|
+
trueclone **bundles** [REA](https://github.com/morluto/rea) (Reverse Engineer Anything) and uses it, but works without it too.
|
|
80
|
+
|
|
81
|
+
| | REA on its own | trueclone |
|
|
82
|
+
|---|---|---|
|
|
83
|
+
| What it is | A microscope: typed tools that let your agent look inside apps, binaries and pages | A factory with quality control: capture, design system, build guide, scores, report |
|
|
84
|
+
| Capture | Your Chrome over CDP, one page state | 3 widths, every scroll state, motion runtime, assets, network |
|
|
85
|
+
| Knows when the clone is right | No: the agent judges by eye | Yes: 7 measured checks with targets |
|
|
86
|
+
| Tells you what's wrong | No | "Footer column 2 is at x 189, the original's at 732", plus the original's CSS rule |
|
|
87
|
+
| Outputs | Whatever the agent writes | HTML clone, Next.js app, DESIGN.md, globals.css, motion recipe, REPORT.md |
|
|
88
|
+
|
|
89
|
+
REA answers *how does this work?* trueclone answers *is my copy identical, and where isn't it?* For the hard parts (a bundled physics rule, a shader, the API a component calls) the skill sends your agent to REA.
|
|
90
|
+
|
|
91
|
+
## Modes and guardrails
|
|
92
|
+
|
|
93
|
+
| Mode | For | Content |
|
|
94
|
+
|---|---|---|
|
|
95
|
+
| `template` (default) | Any site: learning, layout study, a starting template | Structure, motion and behaviour of the original; your own text, images and brand |
|
|
96
|
+
| `exact` | Sites you own or are authorised to copy, such as a Webflow, Framer or WordPress site moving to Next.js | The site's own text and media |
|
|
97
|
+
|
|
98
|
+
- Exact mode refuses to run without `--permission "<who authorised it>"` and records it in the capture and the report.
|
|
99
|
+
- No bot-check, CAPTCHA, login or paywall bypass. Tracking and analytics are never copied.
|
|
100
|
+
- Clones are `noindex`, forms submit nowhere, and everything stays in your local `sites/` folder.
|
|
101
|
+
|
|
102
|
+
## Commands
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
npx @trueclone/cli init # set up a project for your agent (then `npx trueclone ...`)
|
|
106
|
+
npx trueclone doctor # check Node, Chrome and REA
|
|
107
|
+
npx trueclone <url> [name] # capture + design-system draft
|
|
108
|
+
npx trueclone capture <url> [name] [--mode exact --permission "<owner>"]
|
|
109
|
+
npx trueclone motion <name> # motion inventory
|
|
110
|
+
npx trueclone design <name> # DESIGN.md + globals.css
|
|
111
|
+
npx trueclone reveals <name> # record scroll reveals and scroll class changes
|
|
112
|
+
npx trueclone widgets <name> # record every tab and menu state
|
|
113
|
+
npx trueclone seed <name> # clone/ from the mirror (template: placeholder text and media)
|
|
114
|
+
npx trueclone rules <name> "<selector>" # the original's CSS rules for an element
|
|
115
|
+
npx trueclone textshape <name> --write # template mode: placeholder text with the original's line breaks
|
|
116
|
+
npx trueclone score <name> [--quick] # every check, then REPORT.md
|
|
117
|
+
npx trueclone next <name> --verify # Next.js + Tailwind + shadcn app, scored
|
|
118
|
+
npx trueclone rea <args> # bundled REA
|
|
119
|
+
npx trueclone help # every step
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## What it can't do
|
|
123
|
+
|
|
124
|
+
- **Server-side behaviour** is invisible from outside: databases, accounts, payments, search. trueclone rebuilds the client and lists every API call the original makes in the report.
|
|
125
|
+
- A live site is never pixel-identical twice: rotating content, A/B tests and font hinting vary.
|
|
126
|
+
- One page at a time today. Multi-page crawling with shared components is next.
|
|
127
|
+
- Developed and tested on Windows 11 with Chrome. macOS and Linux should work but aren't tested yet.
|
|
128
|
+
|
|
129
|
+
## Requirements
|
|
130
|
+
|
|
131
|
+
Node 22.19+, 24.11+ or 26+ (the same as REA), and Google Chrome for captures (it plays the H.264/HLS video many sites use; Playwright's Chromium is the fallback). A coding agent: Claude Code, Codex, Cursor or any agent that reads `AGENTS.md`.
|
|
132
|
+
|
|
133
|
+
## Roadmap
|
|
134
|
+
|
|
135
|
+
- **Whole sites:** crawl every route, shared header and footer as components.
|
|
136
|
+
- **Data layer:** captured API responses turned into mock routes and fixtures, so the clone works offline.
|
|
137
|
+
- **Benchmarks:** React with Framer Motion, a Next.js site with source maps, a data-driven app; each scored with and without REA.
|
|
138
|
+
- **A free web page:** paste a URL, get its DESIGN.md and globals.css.
|
|
139
|
+
|
|
140
|
+
## Credits
|
|
141
|
+
|
|
142
|
+
[REA](https://github.com/morluto/rea) by morluto (MIT), [Playwright](https://playwright.dev), [pixelmatch](https://github.com/mapbox/pixelmatch), [@google/design.md](https://github.com/google-labs-code/design.md), [parse5](https://github.com/inikulin/parse5), and the [shadcn](https://ui.shadcn.com) theme format.
|
|
143
|
+
|
|
144
|
+
## License
|
|
145
|
+
|
|
146
|
+
MIT
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// trueclone CLI: one entry point for every step. Steps run in the current folder: everything about a site lives
|
|
3
|
+
// in ./sites/<name>/ (reference capture, clone, Next.js app, design system, scores).
|
|
4
|
+
//
|
|
5
|
+
// npx @trueclone/cli init set up this folder for a coding agent (skill, AGENTS.md, REA MCP, .gitignore) and
|
|
6
|
+
// install trueclone as a dev dependency, so `npx trueclone <step>` works from then on
|
|
7
|
+
// trueclone <url> [name] capture a site (template mode) and draft its design system: the agent's first step
|
|
8
|
+
// trueclone <step> <name> [...] run one step (see `trueclone help`)
|
|
9
|
+
// trueclone rea <args> REA (Reverse Engineer Anything), bundled: `trueclone rea mcp`, `trueclone rea doctor`
|
|
10
|
+
import { spawn } from 'node:child_process';
|
|
11
|
+
import fs from 'node:fs/promises';
|
|
12
|
+
import path from 'node:path';
|
|
13
|
+
import { createRequire } from 'node:module';
|
|
14
|
+
import { fileURLToPath } from 'node:url';
|
|
15
|
+
|
|
16
|
+
const PKG = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
17
|
+
const SCRIPTS = path.join(PKG, 'scripts');
|
|
18
|
+
const require = createRequire(import.meta.url);
|
|
19
|
+
const { name, version } = JSON.parse(await fs.readFile(path.join(PKG, 'package.json'), 'utf8'));
|
|
20
|
+
|
|
21
|
+
const STEPS = {
|
|
22
|
+
capture: ['capture.mjs', 'capture a live page: screenshots at 390/768/1440, sections, tokens, rendered HTML, assets'],
|
|
23
|
+
seed: ['seed.mjs', 'exact mode: start clone/ from the script-free mirror (markup, text, styles, media)'],
|
|
24
|
+
design: ['design.mjs', 'design-tokens.json -> DESIGN.md (linted), shadcn globals.css, clone/tokens.css'],
|
|
25
|
+
motion: ['motion.mjs', 'motion inventory from the live runtime (GSAP/ScrollTrigger, Lenis, Swiper, CSS animations)'],
|
|
26
|
+
reveals: ['reveals.mjs', 'record one-shot scroll reveals frame by frame, so seed can replay them (any library)'],
|
|
27
|
+
widgets: ['widgets.mjs', 'record every tab and menu state by clicking it, so seed can replay them (template mode)'],
|
|
28
|
+
states: ['states.mjs', 'live scroll states; --clone scores the rebuild step by step'],
|
|
29
|
+
curves: ['curves.mjs', 'motion score: sampled property curves, reference vs clone (motion-map.json)'],
|
|
30
|
+
interact: ['interact.mjs', 'interaction tests: click/hover/drag/key scenarios on both sides (interactions.json)'],
|
|
31
|
+
detail: ['detail.mjs', 'style diff at matching points: font, size, weight, line height, colour, background, radius'],
|
|
32
|
+
hovers: ['hovers.mjs', 'hover every interactive element on both sides and compare what changes'],
|
|
33
|
+
rules: ['rules.mjs', 'every CSS rule matching an element on the reference (+ ::before/::after, @media)'],
|
|
34
|
+
textshape: ['textshape.mjs', 'template mode: placeholder words with the reference word widths, so lines break the same'],
|
|
35
|
+
diff: ['diff.mjs', 'full-page pixel diff per viewport and section (--structure masks text and media)'],
|
|
36
|
+
spec: ['spec.mjs', 'measured boxes and styles of the live page (--clone for the rebuild)'],
|
|
37
|
+
tiles: ['tiles.mjs', 'cut a tall screenshot into readable tiles'],
|
|
38
|
+
next: ['next.mjs', 'Next.js + Tailwind v4 + shadcn app from the verified clone; --verify scores it'],
|
|
39
|
+
score: ['score.mjs', 'every score in parallel groups, time limit per step, writes REPORT.md'],
|
|
40
|
+
report: ['report.mjs', 'REPORT.md from the latest scores: every number, every gap, what cannot be copied'],
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
// npm/npx are .cmd files on Windows and need a shell; Node warns (DEP0190) when a shell gets separate arguments, so
|
|
44
|
+
// build one quoted command line there instead
|
|
45
|
+
const spawnCmd = (cmd, args, opts = {}) => (process.platform === 'win32'
|
|
46
|
+
? spawn([cmd, ...args].map((a) => (/[\s"]/.test(a) ? `"${a.replace(/"/g, '\\"')}"` : a)).join(' '), { ...opts, shell: true })
|
|
47
|
+
: spawn(cmd, args, opts));
|
|
48
|
+
const run = (file, args) => new Promise((resolve) => {
|
|
49
|
+
const child = spawn(process.execPath, [file, ...args], { stdio: 'inherit' });
|
|
50
|
+
child.on('close', (code) => resolve(code ?? 1));
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
function help() {
|
|
54
|
+
console.log(`trueclone ${version}: a working twin of a website, proven by scores
|
|
55
|
+
|
|
56
|
+
usage
|
|
57
|
+
npx @trueclone/cli init set up this folder for your coding agent (installs trueclone)
|
|
58
|
+
trueclone <url> [name] capture + design-system draft (template mode)
|
|
59
|
+
trueclone <step> <name> [options] run one step
|
|
60
|
+
trueclone rea <args> bundled REA (Reverse Engineer Anything)
|
|
61
|
+
trueclone doctor check Node, browsers and REA
|
|
62
|
+
|
|
63
|
+
steps
|
|
64
|
+
${Object.entries(STEPS).map(([k, [, d]]) => ` ${k.padEnd(10)} ${d}`).join('\n')}
|
|
65
|
+
|
|
66
|
+
Then ask your agent: "clone <url> with trueclone". The skill walks it through capture, build, scores and report.`);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
async function exists(file) { return fs.stat(file).then(() => true, () => false); }
|
|
70
|
+
|
|
71
|
+
async function init() {
|
|
72
|
+
const cwd = process.cwd();
|
|
73
|
+
// 1. the agent skill (Claude Code reads .claude/skills; other agents read AGENTS.md, which points at the same file)
|
|
74
|
+
const skillDir = path.join(cwd, '.claude', 'skills', 'trueclone');
|
|
75
|
+
await fs.mkdir(skillDir, { recursive: true });
|
|
76
|
+
await fs.copyFile(path.join(PKG, 'skill', 'SKILL.md'), path.join(skillDir, 'SKILL.md'));
|
|
77
|
+
// 2. AGENTS.md block for Codex, Cursor and other agents
|
|
78
|
+
const agentsFile = path.join(cwd, 'AGENTS.md');
|
|
79
|
+
const block = `<!-- trueclone:start -->\n## Cloning websites with trueclone\n\nTo clone or rebuild a website, read \`.claude/skills/trueclone/SKILL.md\` first and follow it step by step. Run every step with \`npx trueclone <step> <name>\` and finish only when \`npx trueclone score <name>\` passes the targets in the skill; \`sites/<name>/REPORT.md\` is the deliverable.\n<!-- trueclone:end -->\n`;
|
|
80
|
+
let agents = (await exists(agentsFile)) ? await fs.readFile(agentsFile, 'utf8') : '';
|
|
81
|
+
agents = agents.includes('<!-- trueclone:start -->') ? agents.replace(/<!-- trueclone:start -->[\s\S]*?<!-- trueclone:end -->\n?/, block) : `${agents}${agents && !agents.endsWith('\n\n') ? '\n' : ''}${block}`;
|
|
82
|
+
await fs.writeFile(agentsFile, agents);
|
|
83
|
+
// 3. REA as an MCP server for agents that speak MCP (Claude Code reads .mcp.json)
|
|
84
|
+
const mcpFile = path.join(cwd, '.mcp.json');
|
|
85
|
+
const mcp = (await exists(mcpFile)) ? JSON.parse(await fs.readFile(mcpFile, 'utf8')) : {};
|
|
86
|
+
mcp.mcpServers = { ...(mcp.mcpServers || {}), rea: { command: 'npx', args: ['-y', `${name}@${version}`, 'rea', 'mcp'] } };
|
|
87
|
+
await fs.writeFile(mcpFile, `${JSON.stringify(mcp, null, 2)}\n`);
|
|
88
|
+
// 4. captures, clones and browser profiles stay local
|
|
89
|
+
const ignoreFile = path.join(cwd, '.gitignore');
|
|
90
|
+
let ignore = (await exists(ignoreFile)) ? await fs.readFile(ignoreFile, 'utf8') : '';
|
|
91
|
+
const want = ['sites/', '.chrome-profile/', '*.har'].filter((line) => !ignore.split(/\r?\n/).includes(line));
|
|
92
|
+
if (want.length) await fs.writeFile(ignoreFile, `${ignore}${ignore && !ignore.endsWith('\n') ? '\n' : ''}${want.join('\n')}\n`);
|
|
93
|
+
// 5. the package itself as a dev dependency, pinned: afterwards `npx trueclone <step>` runs this version, offline
|
|
94
|
+
let installed = 'skipped (--no-install)';
|
|
95
|
+
if (!process.argv.includes('--no-install')) {
|
|
96
|
+
if (!(await exists(path.join(cwd, 'package.json')))) await fs.writeFile(path.join(cwd, 'package.json'), `${JSON.stringify({ name: path.basename(cwd).toLowerCase().replace(/[^a-z0-9-]+/g, '-'), private: true }, null, 2)}\n`);
|
|
97
|
+
const code = await new Promise((resolve) => {
|
|
98
|
+
const child = spawnCmd('npm', ['install', '--save-dev', '--save-exact', `${name}@${version}`], { stdio: 'inherit' });
|
|
99
|
+
child.on('close', (c) => resolve(c ?? 1));
|
|
100
|
+
});
|
|
101
|
+
installed = code === 0 ? `${name}@${version} (dev dependency: npx trueclone <step> now runs it)` : `failed: run npm install -D ${name}@${version} yourself`;
|
|
102
|
+
}
|
|
103
|
+
console.log(`trueclone set up in ${cwd}
|
|
104
|
+
.claude/skills/trueclone/SKILL.md the workflow your agent follows
|
|
105
|
+
AGENTS.md points Codex, Cursor and others at the skill
|
|
106
|
+
.mcp.json REA as an MCP server
|
|
107
|
+
.gitignore sites/ stays local
|
|
108
|
+
package.json ${installed}
|
|
109
|
+
next: npx trueclone doctor, then ask your agent "clone <url> with trueclone"`);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
async function doctor() {
|
|
113
|
+
let ok = true;
|
|
114
|
+
const [major, minor] = process.versions.node.split('.').map(Number);
|
|
115
|
+
const nodeOk = (major === 22 && minor >= 19) || (major === 24 && minor >= 11) || major >= 26;
|
|
116
|
+
console.log(`${nodeOk ? 'ok ' : 'FAIL'} Node ${process.versions.node} (needs 22.19+, 24.11+ or 26+)`);
|
|
117
|
+
ok &&= nodeOk;
|
|
118
|
+
const { chromium } = await import('playwright');
|
|
119
|
+
for (const [label, opts] of [['Chrome (captures; plays H.264/HLS video)', { channel: 'chrome' }], ['Playwright Chromium (fallback)', {}]]) {
|
|
120
|
+
try { const b = await chromium.launch(opts); await b.close(); console.log(`ok ${label}`); }
|
|
121
|
+
catch { console.log(`warn ${label} not found${opts.channel ? ': install Google Chrome' : ': npx playwright install chromium'}`); }
|
|
122
|
+
}
|
|
123
|
+
try { require.resolve('rea-agents/package.json'); console.log('ok REA bundled (trueclone rea doctor checks its decompilers)'); }
|
|
124
|
+
catch { console.log('warn REA not installed'); ok = false; }
|
|
125
|
+
process.exitCode = ok ? 0 : 1;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const [cmd, ...rest] = process.argv.slice(2);
|
|
129
|
+
if (!cmd || cmd === 'help' || cmd === '--help' || cmd === '-h') help();
|
|
130
|
+
else if (cmd === '--version' || cmd === '-v') console.log(version);
|
|
131
|
+
else if (cmd === 'init') await init();
|
|
132
|
+
else if (cmd === 'doctor') await doctor();
|
|
133
|
+
else if (cmd === 'rea') process.exitCode = await run(path.join(path.dirname(require.resolve('rea-agents/package.json')), 'scripts', 'rea.mjs'), rest);
|
|
134
|
+
else if (/^https?:\/\//.test(cmd)) {
|
|
135
|
+
// the first step for any site: capture it (template mode) and draft its design system
|
|
136
|
+
const name = rest.find((a) => !a.startsWith('--')) || new URL(cmd).hostname.replace(/^www\./, '').replace(/\.[a-z]+$/, '').replace(/[^a-z0-9]+/gi, '-');
|
|
137
|
+
let code = await run(path.join(SCRIPTS, 'capture.mjs'), [cmd, name, ...rest.filter((a) => a !== name)]);
|
|
138
|
+
if (code === 0) code = await run(path.join(SCRIPTS, 'design.mjs'), [name]);
|
|
139
|
+
if (code === 0) console.log(`\nnext: npx trueclone motion ${name}, then build sites/${name}/clone/ following .claude/skills/trueclone/SKILL.md`);
|
|
140
|
+
process.exitCode = code;
|
|
141
|
+
} else if (STEPS[cmd]) process.exitCode = await run(path.join(SCRIPTS, STEPS[cmd][0]), rest);
|
|
142
|
+
else {
|
|
143
|
+
console.error(`unknown command: ${cmd}\n`);
|
|
144
|
+
help();
|
|
145
|
+
process.exitCode = 1;
|
|
146
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,64 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@trueclone/cli",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"description": "A working twin of any website from its URL: HTML clone, Next.js + Tailwind + shadcn app, DESIGN.md and globals.css, proven by layout, motion, interaction and style scores. Bundles REA.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"bin": {
|
|
7
|
+
"trueclone": "bin/trueclone.mjs"
|
|
8
|
+
},
|
|
9
|
+
"files": [
|
|
10
|
+
"bin/",
|
|
11
|
+
"scripts/*.mjs",
|
|
12
|
+
"skill/",
|
|
13
|
+
"README.md",
|
|
14
|
+
"LICENSE"
|
|
15
|
+
],
|
|
16
|
+
"engines": {
|
|
17
|
+
"node": "^22.19.0 || ^24.11.0 || >=26.0.0"
|
|
18
|
+
},
|
|
19
|
+
"scripts": {
|
|
20
|
+
"trueclone": "node bin/trueclone.mjs",
|
|
21
|
+
"chrome": "powershell -ExecutionPolicy Bypass -File scripts/chrome-debug.ps1"
|
|
22
|
+
},
|
|
23
|
+
"dependencies": {
|
|
24
|
+
"@google/design.md": "^0.4.0",
|
|
25
|
+
"parse5": "^8.0.1",
|
|
26
|
+
"pixelmatch": "^7.2.0",
|
|
27
|
+
"playwright": "^1.64.0",
|
|
28
|
+
"pngjs": "^7.0.0",
|
|
29
|
+
"rea-agents": "^6.4.0"
|
|
30
|
+
},
|
|
31
|
+
"keywords": [
|
|
32
|
+
"clone",
|
|
33
|
+
"website",
|
|
34
|
+
"replicate",
|
|
35
|
+
"rebuild",
|
|
36
|
+
"migration",
|
|
37
|
+
"design-system",
|
|
38
|
+
"design.md",
|
|
39
|
+
"shadcn",
|
|
40
|
+
"nextjs",
|
|
41
|
+
"tailwind",
|
|
42
|
+
"playwright",
|
|
43
|
+
"pixel-diff",
|
|
44
|
+
"gsap",
|
|
45
|
+
"coding-agent",
|
|
46
|
+
"claude-code",
|
|
47
|
+
"mcp",
|
|
48
|
+
"reverse-engineering",
|
|
49
|
+
"rea"
|
|
50
|
+
],
|
|
51
|
+
"author": "Ayush Pratap Singh",
|
|
52
|
+
"license": "MIT",
|
|
53
|
+
"repository": {
|
|
54
|
+
"type": "git",
|
|
55
|
+
"url": "git+https://github.com/Ayuxh-Pratap/trueclone.git"
|
|
56
|
+
},
|
|
57
|
+
"homepage": "https://github.com/Ayuxh-Pratap/trueclone#readme",
|
|
58
|
+
"bugs": {
|
|
59
|
+
"url": "https://github.com/Ayuxh-Pratap/trueclone/issues"
|
|
60
|
+
},
|
|
61
|
+
"publishConfig": {
|
|
62
|
+
"access": "public"
|
|
63
|
+
}
|
|
64
|
+
}
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
// Capture a live page as the clone's reference: full-page screenshots, sections, design tokens,
|
|
2
|
+
// rendered HTML, CSS/image/font files and a HAR.
|
|
3
|
+
// usage: node scripts/capture.mjs <url> [name] [--hide ".cookie-banner,#popup"] [--cdp [endpoint]] [--mode exact --permission "<owner>"]
|
|
4
|
+
// node scripts/capture.mjs <url> <name> --mirror-only (rebuild reference/mirror from the saved capture)
|
|
5
|
+
import fs from 'node:fs/promises';
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
import crypto from 'node:crypto';
|
|
8
|
+
import { VIEWPORTS, parseArgs, siteName, openBrowser, newPage, settle, sectionsOf, mask } from './lib.mjs';
|
|
9
|
+
|
|
10
|
+
const { pos, flags } = parseArgs(process.argv.slice(2));
|
|
11
|
+
const [url, nameArg] = pos;
|
|
12
|
+
if (!url) {
|
|
13
|
+
console.error('usage: npm run capture -- <url> [name] [--hide "<css selectors>"] [--cdp]');
|
|
14
|
+
process.exit(1);
|
|
15
|
+
}
|
|
16
|
+
const name = nameArg || siteName(url);
|
|
17
|
+
const hide = typeof flags.hide === 'string' ? flags.hide.split(',').map((s) => s.trim()).filter(Boolean) : [];
|
|
18
|
+
// --mode template (default): assets are kept as reference only; the clone uses placeholder content.
|
|
19
|
+
// --mode exact: the clone may reuse the site's own text and media. Only for sites you own or are
|
|
20
|
+
// authorised to copy, so it refuses to run without --permission "<who authorised it>". Output stays in
|
|
21
|
+
// sites/ (gitignored), the mirror is noindex with every script removed, and nothing here deploys it.
|
|
22
|
+
const mode = flags.mode === 'exact' ? 'exact' : 'template';
|
|
23
|
+
const permission = typeof flags.permission === 'string' ? flags.permission : null;
|
|
24
|
+
if (mode === 'exact' && !permission) {
|
|
25
|
+
console.error('exact mode copies the site\'s own text and media. Pass --permission "<owner / who authorised this copy>".');
|
|
26
|
+
process.exit(1);
|
|
27
|
+
}
|
|
28
|
+
const root = path.resolve('sites', name);
|
|
29
|
+
const ref = path.join(root, 'reference');
|
|
30
|
+
const MAX_ASSET_BYTES = mode === 'exact' ? 200 * 1024 * 1024 : 15 * 1024 * 1024;
|
|
31
|
+
const SAVED_TYPES = new Set(['stylesheet', 'image', 'font', 'media']);
|
|
32
|
+
// Exact mode also keeps runtime-fetched media: 3D models, textures, video streams, animation files. Never scripts.
|
|
33
|
+
const EXTRA_MEDIA = /\.(glb|gltf|bin|hdr|exr|ktx2|basis|mp4|webm|mov|m3u8|ts|m4s|lottie|riv|json)(\?|$)/i;
|
|
34
|
+
const keepResponse = (type, url) => SAVED_TYPES.has(type) || (mode === 'exact' && ['fetch', 'xhr', 'other'].includes(type) && EXTRA_MEDIA.test(url));
|
|
35
|
+
|
|
36
|
+
// --mirror-only: rebuild reference/mirror from an existing capture (no new screenshots, caches stay valid)
|
|
37
|
+
if (flags['mirror-only']) {
|
|
38
|
+
const meta = JSON.parse(await fs.readFile(path.join(ref, 'meta.json'), 'utf8'));
|
|
39
|
+
const saved = JSON.parse(await fs.readFile(path.join(ref, 'assets', 'manifest.json'), 'utf8'));
|
|
40
|
+
await buildMirror(saved, meta.url, meta.mode === 'exact' ? `Exact mode, authorised by: ${meta.permission}.` : "Template mode: the site's own content, kept as a reference only.");
|
|
41
|
+
process.exit(0);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Only clear what capture owns; states/, motion.*, *.spec.json come from other scripts and stay.
|
|
45
|
+
await fs.rm(path.join(ref, 'assets'), { recursive: true, force: true });
|
|
46
|
+
await fs.rm(path.join(ref, 'mirror'), { recursive: true, force: true });
|
|
47
|
+
await fs.mkdir(path.join(ref, 'assets'), { recursive: true });
|
|
48
|
+
|
|
49
|
+
function tokensInPage() {
|
|
50
|
+
const count = (m, k) => m.set(k, (m.get(k) || 0) + 1);
|
|
51
|
+
const colors = new Map(), bgs = new Map(), fonts = new Map(), type = new Map(), radii = new Map(), shadows = new Map();
|
|
52
|
+
for (const el of [...document.querySelectorAll('body *')].slice(0, 8000)) {
|
|
53
|
+
const s = getComputedStyle(el);
|
|
54
|
+
if (s.display === 'none' || s.visibility === 'hidden') continue;
|
|
55
|
+
if ([...el.childNodes].some((n) => n.nodeType === 3 && n.textContent.trim())) {
|
|
56
|
+
count(colors, s.color);
|
|
57
|
+
count(fonts, s.fontFamily);
|
|
58
|
+
count(type, `${s.fontSize} / ${s.lineHeight} / ${s.fontWeight} / ${s.letterSpacing}`);
|
|
59
|
+
}
|
|
60
|
+
if (s.backgroundColor !== 'rgba(0, 0, 0, 0)') count(bgs, s.backgroundColor);
|
|
61
|
+
if (s.backgroundImage !== 'none') count(bgs, s.backgroundImage.slice(0, 200));
|
|
62
|
+
if (s.borderRadius !== '0px') count(radii, s.borderRadius);
|
|
63
|
+
if (s.boxShadow !== 'none') count(shadows, s.boxShadow);
|
|
64
|
+
}
|
|
65
|
+
const top = (m) => [...m].sort((a, b) => b[1] - a[1]).slice(0, 30).map(([value, uses]) => ({ value, uses }));
|
|
66
|
+
const cssVariables = {};
|
|
67
|
+
for (const sheet of document.styleSheets) {
|
|
68
|
+
try {
|
|
69
|
+
for (const rule of sheet.cssRules) {
|
|
70
|
+
if (rule.selectorText === ':root' || rule.selectorText === 'html') {
|
|
71
|
+
for (const p of rule.style) if (p.startsWith('--')) cssVariables[p] = rule.style.getPropertyValue(p).trim();
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
} catch { /* cross-origin sheet */ }
|
|
75
|
+
}
|
|
76
|
+
return {
|
|
77
|
+
title: document.title,
|
|
78
|
+
lang: document.documentElement.lang,
|
|
79
|
+
bodyBackground: getComputedStyle(document.body).backgroundColor,
|
|
80
|
+
textColors: top(colors),
|
|
81
|
+
backgrounds: top(bgs),
|
|
82
|
+
fontFamilies: top(fonts),
|
|
83
|
+
typeScale: top(type),
|
|
84
|
+
radii: top(radii),
|
|
85
|
+
shadows: top(shadows),
|
|
86
|
+
cssVariables,
|
|
87
|
+
fontsLoaded: [...document.fonts].filter((f) => f.status === 'loaded').map((f) => `${f.family} ${f.weight} ${f.style}`),
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const browser = await openBrowser(flags.cdp);
|
|
92
|
+
const manifest = [];
|
|
93
|
+
const summary = [];
|
|
94
|
+
const rootFontSize = {};
|
|
95
|
+
try {
|
|
96
|
+
for (const vp of VIEWPORTS) {
|
|
97
|
+
const desktop = vp.name === 'desktop';
|
|
98
|
+
const { context, page } = await newPage(browser, vp, desktop ? { recordHar: { path: path.join(ref, 'desktop.har'), content: 'omit' } } : {});
|
|
99
|
+
if (desktop) {
|
|
100
|
+
page.on('response', async (res) => {
|
|
101
|
+
const rawType = res.request().resourceType();
|
|
102
|
+
if (!keepResponse(rawType, res.url()) || res.status() >= 300) return;
|
|
103
|
+
const type = SAVED_TYPES.has(rawType) ? rawType : 'media';
|
|
104
|
+
try {
|
|
105
|
+
const body = await res.body();
|
|
106
|
+
if (body.length > MAX_ASSET_BYTES) return;
|
|
107
|
+
const u = new URL(res.url());
|
|
108
|
+
const base = (path.basename(u.pathname) || 'index').replace(/[^a-z0-9._-]/gi, '_').slice(-80);
|
|
109
|
+
const hash = crypto.createHash('sha1').update(res.url()).digest('hex').slice(0, 8);
|
|
110
|
+
const file = path.join('assets', type, `${hash}-${base}`);
|
|
111
|
+
await fs.mkdir(path.join(ref, 'assets', type), { recursive: true });
|
|
112
|
+
await fs.writeFile(path.join(ref, file), body);
|
|
113
|
+
manifest.push({ url: res.url(), type, file: file.replaceAll('\\', '/'), contentType: res.headers()['content-type'], bytes: body.length });
|
|
114
|
+
} catch { /* redirects, data: urls, aborted */ }
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
await page.goto(url, { waitUntil: 'load', timeout: 60000 });
|
|
118
|
+
// Components that mount only while in view (a lazy wordmark, an embed) are often unmounted again by the time
|
|
119
|
+
// the DOM is saved. Watch from load: an element that was empty, got content while scrolled into view and is empty
|
|
120
|
+
// again at snapshot time is saved with the fullest content it had. Things that start full and empty out (a
|
|
121
|
+
// loader) are left alone.
|
|
122
|
+
if (desktop) await page.evaluate(() => {
|
|
123
|
+
const count = (el) => el.getElementsByTagName('*').length;
|
|
124
|
+
const initial = new WeakMap();
|
|
125
|
+
for (const el of document.body.getElementsByTagName('*')) initial.set(el, count(el));
|
|
126
|
+
window.__tcRich = new Map();
|
|
127
|
+
new MutationObserver((records) => {
|
|
128
|
+
for (const r of records) {
|
|
129
|
+
for (let el = r.target, up = 0; el instanceof Element && el !== document.body && up < 8; el = el.parentElement, up++) {
|
|
130
|
+
const start = initial.has(el) ? initial.get(el) : 0;
|
|
131
|
+
const n = count(el);
|
|
132
|
+
if (start > 0 || n === 0) continue;
|
|
133
|
+
const best = window.__tcRich.get(el);
|
|
134
|
+
if (!best || n > best.n) window.__tcRich.set(el, { n, html: el.innerHTML });
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}).observe(document.body, { subtree: true, childList: true });
|
|
138
|
+
});
|
|
139
|
+
await settle(page, hide);
|
|
140
|
+
// A full-page screenshot brings the whole page into view at once, which mounts view-mounted components and starts
|
|
141
|
+
// their reveals: shoot until two shots agree, so the reference shows a settled page, not a fade caught at 0%.
|
|
142
|
+
let shot = await page.screenshot({ fullPage: true });
|
|
143
|
+
for (let tries = 0; tries < 3; tries++) {
|
|
144
|
+
await page.waitForTimeout(2000);
|
|
145
|
+
const again = await page.screenshot({ fullPage: true });
|
|
146
|
+
const same = again.equals(shot);
|
|
147
|
+
shot = again;
|
|
148
|
+
if (same) break;
|
|
149
|
+
}
|
|
150
|
+
await fs.writeFile(path.join(ref, `${vp.name}.png`), shot);
|
|
151
|
+
const sections = await sectionsOf(page);
|
|
152
|
+
await fs.writeFile(path.join(ref, `${vp.name}.sections.json`), JSON.stringify(sections, null, 2));
|
|
153
|
+
const height = await page.evaluate(() => document.documentElement.scrollHeight);
|
|
154
|
+
rootFontSize[vp.name] = await page.evaluate(() => getComputedStyle(document.documentElement).fontSize);
|
|
155
|
+
if (desktop) {
|
|
156
|
+
const html = await page.evaluate(() => {
|
|
157
|
+
// restore view-mounted content (see above): only elements that are empty again now
|
|
158
|
+
const restore = [];
|
|
159
|
+
let i = 0;
|
|
160
|
+
for (const [el, best] of window.__tcRich || []) {
|
|
161
|
+
if (!el.isConnected || el.getElementsByTagName('*').length > 0) continue;
|
|
162
|
+
el.setAttribute('data-tc-rich', String(i));
|
|
163
|
+
restore[i++] = best.html;
|
|
164
|
+
}
|
|
165
|
+
const doc = document.documentElement.cloneNode(true);
|
|
166
|
+
for (const el of document.querySelectorAll('[data-tc-rich]')) el.removeAttribute('data-tc-rich');
|
|
167
|
+
for (const el of doc.querySelectorAll('[data-tc-rich]')) {
|
|
168
|
+
el.innerHTML = restore[Number(el.getAttribute('data-tc-rich'))];
|
|
169
|
+
el.removeAttribute('data-tc-rich');
|
|
170
|
+
// marked for the layout diff: a full-page still can't show view-mounted content on either side, so its box
|
|
171
|
+
// is excluded there (the scroll-state check, with a real viewport, scores it)
|
|
172
|
+
el.setAttribute('data-tc-viewmount', '');
|
|
173
|
+
}
|
|
174
|
+
doc.querySelector('style[data-lab-freeze]')?.remove();
|
|
175
|
+
console.log(`tc-rich: restored ${restore.length} view-mounted element(s)`);
|
|
176
|
+
return '<!DOCTYPE html>\n' + doc.outerHTML;
|
|
177
|
+
});
|
|
178
|
+
await fs.writeFile(path.join(ref, 'rendered.html'), html);
|
|
179
|
+
await fs.writeFile(path.join(ref, 'tokens.json'), JSON.stringify(await page.evaluate(tokensInPage), null, 2));
|
|
180
|
+
}
|
|
181
|
+
await mask(page);
|
|
182
|
+
await page.screenshot({ path: path.join(ref, `${vp.name}.mask.png`), fullPage: true });
|
|
183
|
+
summary.push({ viewport: vp.name, width: vp.width, pageHeight: height, sections: sections.length });
|
|
184
|
+
await context.close(); // flushes the HAR
|
|
185
|
+
}
|
|
186
|
+
} finally {
|
|
187
|
+
if (!flags.cdp) await browser.close();
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
await fs.writeFile(path.join(ref, 'assets', 'manifest.json'), JSON.stringify(manifest, null, 2));
|
|
191
|
+
await fs.writeFile(path.join(ref, 'meta.json'), JSON.stringify({ url, name, mode, permission, hide, capturedAt: new Date().toISOString(), rootFontSize, viewports: summary }, null, 2));
|
|
192
|
+
await buildMirror(manifest, url, mode === 'exact' ? `Exact mode, authorised by: ${permission}.` : "Template mode: the site's own content, kept as a reference only.");
|
|
193
|
+
console.table(summary);
|
|
194
|
+
console.log(`${manifest.length} assets saved. Reference: ${path.relative(process.cwd(), ref)}, mirror in reference/mirror${mode === 'exact' ? ` (exact mode, authorised by: ${permission})` : ''}`);
|
|
195
|
+
|
|
196
|
+
// ---------- Static local mirror of the rendered page ----------
|
|
197
|
+
// Same DOM as rendered.html, asset URLs rewritten to the saved copies, every script and inline handler
|
|
198
|
+
// removed (we never reuse the site's code), forms disabled, noindex. Static by design: motion is rebuilt
|
|
199
|
+
// from the states/motion inventory, not replayed. Exact mode seeds the clone from it as is; template mode
|
|
200
|
+
// seeds it with every text and medium swapped for placeholders (seed.mjs).
|
|
201
|
+
async function buildMirror(manifest, url, note) {
|
|
202
|
+
const mirror = path.join(ref, 'mirror');
|
|
203
|
+
await fs.rm(mirror, { recursive: true, force: true });
|
|
204
|
+
await fs.mkdir(path.join(mirror, 'css'), { recursive: true });
|
|
205
|
+
const byUrl = new Map(manifest.map((m) => [m.url.split('#')[0], m]));
|
|
206
|
+
const abs = (u, base) => { try { return new URL(u.trim().replace(/^['"]|['"]$/g, '').replaceAll('&', '&'), base).href.split('#')[0]; } catch { return null; } };
|
|
207
|
+
const cssName = (m) => path.basename(m.file);
|
|
208
|
+
// where a saved asset lives, seen from a file inside `fromDir`
|
|
209
|
+
const local = (absUrl, fromDir) => {
|
|
210
|
+
const m = absUrl && byUrl.get(absUrl);
|
|
211
|
+
if (!m) return null;
|
|
212
|
+
if (m.type === 'stylesheet') return path.relative(fromDir, path.join(mirror, 'css', cssName(m))).replaceAll('\\', '/');
|
|
213
|
+
return path.relative(fromDir, path.join(ref, m.file)).replaceAll('\\', '/');
|
|
214
|
+
};
|
|
215
|
+
const rewriteCssUrls = (css, base, fromDir) => css
|
|
216
|
+
// url(#id) points inside the document (SVG gradients, filters, masks): never resolve it against the stylesheet
|
|
217
|
+
.replace(/url\(\s*(['"]?)([^'")]+)\1\s*\)/g, (all, q, u) => (u.startsWith('data:') || u.trim().startsWith('#') ? all : `url("${local(abs(u, base), fromDir) || abs(u, base) || u}")`))
|
|
218
|
+
.replace(/@import\s+(['"])([^'"]+)\1/g, (all, q, u) => `@import "${local(abs(u, base), fromDir) || u}"`);
|
|
219
|
+
|
|
220
|
+
// stylesheets: rewritten copies under mirror/css
|
|
221
|
+
for (const m of manifest.filter((x) => x.type === 'stylesheet')) {
|
|
222
|
+
const css = await fs.readFile(path.join(ref, m.file), 'utf8');
|
|
223
|
+
await fs.writeFile(path.join(mirror, 'css', cssName(m)), rewriteCssUrls(css, m.url, path.join(mirror, 'css')));
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
let html = await fs.readFile(path.join(ref, 'rendered.html'), 'utf8');
|
|
227
|
+
html = html
|
|
228
|
+
.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, '')
|
|
229
|
+
.replace(/<script\b[^>]*\/>/gi, '')
|
|
230
|
+
.replace(/<link\b[^>]*rel=["']?(?:modulepreload|preload)["']?[^>]*as=["']?script["']?[^>]*>/gi, '')
|
|
231
|
+
.replace(/\son[a-z]+\s*=\s*("[^"]*"|'[^']*')/gi, '')
|
|
232
|
+
.replace(/<form\b([^>]*?)\saction=("[^"]*"|'[^']*')/gi, '<form$1')
|
|
233
|
+
.replace(/<form\b/gi, '<form action="#"')
|
|
234
|
+
.replace(/\s(src|href|poster|data-src)=("|')([^"']+)\2/gi, (all, attr, q, u) => {
|
|
235
|
+
const l = local(abs(u, url), mirror);
|
|
236
|
+
return l ? ` ${attr}=${q}${l}${q}` : all;
|
|
237
|
+
})
|
|
238
|
+
.replace(/\s(srcset|data-srcset)=("|')([^"']+)\2/gi, (all, attr, q, list) => ` ${attr}=${q}${list.split(',').map((part) => {
|
|
239
|
+
const [u, ...rest] = part.trim().split(/\s+/);
|
|
240
|
+
return [local(abs(u, url), mirror) || u, ...rest].join(' ');
|
|
241
|
+
}).join(', ')}${q}`)
|
|
242
|
+
.replace(/<style\b([^>]*)>([\s\S]*?)<\/style>/gi, (all, attrs, css) => `<style${attrs}>${rewriteCssUrls(css, url, mirror)}</style>`)
|
|
243
|
+
.replace(/\sstyle=("|')([^"']*url\([^"']*)\1/gi, (all, q, css) => ` style=${q}${rewriteCssUrls(css, url, mirror)}${q}`)
|
|
244
|
+
.replace(/<head([^>]*)>/i, `<head$1>\n<!-- trueclone local mirror of ${url}. ${note.replace(/--/g, '-')} Local reference only: do not deploy. -->\n<meta name="robots" content="noindex,nofollow">`);
|
|
245
|
+
await fs.writeFile(path.join(mirror, 'index.html'), html);
|
|
246
|
+
const missing = [...html.matchAll(/\s(?:src|href)=["'](https?:[^"']+)["']/gi)].map((m) => m[1]).filter((u) => !/^https?:\/\/[^/]+\/?$/.test(u));
|
|
247
|
+
console.log(`mirror: reference/mirror/index.html (${manifest.length} local assets, ${missing.length} remote URLs left, e.g. links)`);
|
|
248
|
+
}
|