@trueclone/cli 0.0.0-stage → 0.2.0

package/scripts/widgets.mjs

@@ -0,0 +1,169 @@
+// Record the discrete states of a page's widgets (tabs, menus, accordions) so the clone can replay them. Many
+// component libraries render only the active panel, so the other tabs' content and closed menus' content aren't in
+// the captured page at all; the only faithful source is the original after each click.
+//
+// For every control (role="tab", or a button with aria-expanded) the reference page is clicked, and the region the
+// control changes is snapshotted: the smallest element containing both the control and the panel it points at
+// (aria-controls), falling back to its section. Controls are marked data-tc-trigger and regions data-tc-scope
+// before clicking, so the snapshots carry the marks and `trueclone seed` can wire the clone: a click on a trigger
+// swaps its region to the recorded state. Markup only, never the site's scripts; template mode gets placeholders.
+//
+// Not covered yet: popovers rendered in a portal at the end of <body> (comboboxes, tooltips); they are listed.
+//
+// usage: node scripts/widgets.mjs <name> [--viewport desktop]   -> sites/<name>/reference/widgets.<vp>.json
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { VIEWPORTS, parseArgs, openBrowser, newPage, waitStill } from './lib.mjs';
+
+const { pos, flags } = parseArgs(process.argv.slice(2));
+const [name] = pos;
+if (!name) {
+  console.error('usage: trueclone widgets <name> [--viewport desktop]');
+  process.exit(1);
+}
+const root = path.resolve('sites', name);
+const meta = JSON.parse(await fs.readFile(path.join(root, 'reference', 'meta.json'), 'utf8'));
+const vp = VIEWPORTS.find((v) => v.name === (flags.viewport || 'desktop'));
+
+// Marks every control and its region; returns the groups (a tablist, or a single expandable button).
+function markInPage() {
+  const SKIP = new Set(['SCRIPT', 'NOSCRIPT', 'TEMPLATE', 'STYLE', 'LINK', 'META']);
+  const pathOf = (el) => {
+    const parts = [];
+    for (let n = el; n && n !== document.body; n = n.parentElement) {
+      const sibs = [...n.parentElement.children].filter((c) => !SKIP.has(c.tagName));
+      parts.unshift(`${n.tagName.toLowerCase()}:${sibs.indexOf(n) + 1}`);
+    }
+    return parts.join('>');
+  };
+  const lca = (a, b) => { for (let n = a; n; n = n.parentElement) if (n.contains(b)) return n; return null; };
+  const visible = (el) => { const r = el.getBoundingClientRect(); return r.width > 2 && r.height > 2; };
+  const groups = [];
+  const portals = [];
+  // tabs, grouped by their tablist
+  const lists = new Set([...document.querySelectorAll('[role="tab"]')].filter(visible).map((t) => t.closest('[role="tablist"]') || t.parentElement));
+  for (const list of lists) {
+    const tabs = [...list.querySelectorAll('[role="tab"]')].filter(visible);
+    if (tabs.length < 2) continue;
+    const selected = tabs.find((t) => t.getAttribute('aria-selected') === 'true') || tabs[0];
+    const panel = selected.getAttribute('aria-controls') && document.getElementById(selected.getAttribute('aria-controls'));
+    let scope = panel ? lca(list, panel) : list.closest('section, header, footer, main > div') || list.parentElement;
+    if (!scope || scope === document.body) scope = list.closest('section, header, footer') || list.parentElement;
+    groups.push({ kind: 'tabs', scope, triggers: tabs, initial: tabs.indexOf(selected) });
+  }
+  // expandable buttons (menus, accordions, disclosures), not inside a tablist
+  for (const btn of [...document.querySelectorAll('button[aria-expanded], [role="button"][aria-expanded]')].filter(visible)) {
+    if (btn.closest('[role="tablist"]')) continue;
+    const popup = btn.getAttribute('aria-haspopup');
+    if (btn.getAttribute('role') === 'combobox' || popup === 'listbox' || popup === 'dialog') { portals.push(btn); continue; }
+    const scope = btn.closest('header, nav, section, footer, li') || btn.parentElement;
+    groups.push({ kind: 'expand', scope, triggers: [btn], initial: btn.getAttribute('aria-expanded') === 'true' ? 0 : -1 });
+  }
+  // an element can be the region of several widgets (all the menus of one nav): marks are space-separated lists
+  const addMark = (el, v) => { const cur = (el.getAttribute('data-tc-scope') || '').split(' ').filter(Boolean); if (!cur.includes(v)) cur.push(v); el.setAttribute('data-tc-scope', cur.join(' ')); };
+  groups.forEach((g, i) => {
+    addMark(g.scope, String(i));
+    g.triggers.forEach((t, j) => t.setAttribute('data-tc-trigger', `${i}:${j}`));
+  });
+  window.__tcGroups = groups;
+  return {
+    groups: groups.map((g, i) => ({ i, kind: g.kind, scopePath: pathOf(g.scope), triggers: g.triggers.map((t) => ({ path: pathOf(t), label: (t.getAttribute('aria-label') || t.textContent || '').trim().replace(/\s+/g, ' ').slice(0, 40) })), initial: g.initial })),
+    portals: portals.map((b) => (b.getAttribute('aria-label') || b.textContent || '').trim().slice(0, 40)),
+  };
+}
+
+// Frameworks re-render a region when its state changes, which drops the marks on anything inside it (a tab set
+// nested in another tab's panel). Marks are re-applied by path before every click and every snapshot.
+function remarkInPage(groups) {
+  const SKIP = new Set(['SCRIPT', 'NOSCRIPT', 'TEMPLATE', 'STYLE', 'LINK', 'META']);
+  const resolve = (p) => {
+    let el = document.body;
+    for (const part of p.split('>')) {
+      const [tag, n] = part.split(':');
+      el = [...el.children].filter((c) => !SKIP.has(c.tagName))[Number(n) - 1];
+      if (!el || el.tagName.toLowerCase() !== tag) return null;
+    }
+    return el;
+  };
+  for (const g of groups) {
+    const scope = resolve(g.scopePath);
+    if (scope) { const cur = (scope.getAttribute('data-tc-scope') || '').split(' ').filter(Boolean); if (!cur.includes(String(g.i))) cur.push(String(g.i)); scope.setAttribute('data-tc-scope', cur.join(' ')); }
+    g.triggers.forEach((t, j) => resolve(t.path)?.setAttribute('data-tc-trigger', `${g.i}:${j}`));
+  }
+}
+
+// Menus often render their panel outside the trigger's own container (a shared viewport after the list): once open,
+// the region becomes the smallest element holding both the trigger and the panel it controls.
+function widenScopeInPage([i, sel]) {
+  const SKIP = new Set(['SCRIPT', 'NOSCRIPT', 'TEMPLATE', 'STYLE', 'LINK', 'META']);
+  const pathOf = (el) => {
+    const parts = [];
+    for (let n = el; n && n !== document.body; n = n.parentElement) {
+      const sibs = [...n.parentElement.children].filter((c) => !SKIP.has(c.tagName));
+      parts.unshift(`${n.tagName.toLowerCase()}:${sibs.indexOf(n) + 1}`);
+    }
+    return parts.join('>');
+  };
+  const btn = document.querySelector(sel);
+  const panel = btn?.getAttribute('aria-controls') && document.getElementById(btn.getAttribute('aria-controls'));
+  if (!btn || !panel) return null;
+  let scope = btn;
+  while (scope && !scope.contains(panel)) scope = scope.parentElement;
+  if (!scope || scope === document.body) return null;
+  for (const old of document.querySelectorAll(`[data-tc-scope~="${i}"]`)) {
+    const rest = old.getAttribute('data-tc-scope').split(' ').filter((v) => v !== String(i));
+    if (rest.length) old.setAttribute('data-tc-scope', rest.join(' ')); else old.removeAttribute('data-tc-scope');
+  }
+  const cur = (scope.getAttribute('data-tc-scope') || '').split(' ').filter(Boolean);
+  cur.push(String(i));
+  scope.setAttribute('data-tc-scope', cur.join(' '));
+  return pathOf(scope);
+}
+
+const browser = await openBrowser(flags.cdp);
+let result;
+try {
+  const { page } = await newPage(browser, vp, { reducedMotion: 'reduce' });
+  await page.goto(meta.url, { waitUntil: 'load', timeout: 60000 });
+  await waitStill(page, { timeout: 15000, interval: 200 });
+  const found = await page.evaluate(markInPage);
+  console.log(`${found.groups.length} widget group(s): ${found.groups.filter((g) => g.kind === 'tabs').length} tab sets, ${found.groups.filter((g) => g.kind === 'expand').length} expandable${found.portals.length ? `; not recorded (portal popovers): ${found.portals.join(', ')}` : ''}`);
+  for (const g of found.groups) {
+    g.states = [];
+    const snapshot = async () => { await page.evaluate(remarkInPage, found.groups); return page.evaluate((i) => document.querySelector(`[data-tc-scope~="${i}"]`)?.innerHTML ?? null, g.i); };
+    for (let j = 0; j < g.triggers.length; j++) {
+      await page.evaluate(remarkInPage, found.groups);
+      const handle = await page.$(`[data-tc-trigger="${g.i}:${j}"]`);
+      if (!handle) { g.states.push(null); continue; }
+      await handle.scrollIntoViewIfNeeded().catch(() => {});
+      await handle.click({ timeout: 5000 }).catch(() => {});
+      await page.waitForTimeout(350);
+      await waitStill(page, { timeout: 2500 });
+      if (g.kind === 'expand') {
+        const wider = await page.evaluate(widenScopeInPage, [g.i, `[data-tc-trigger="${g.i}:${j}"]`]);
+        if (wider) g.scopePath = wider;
+      }
+      g.states.push(await snapshot());
+      if (g.kind === 'expand') { // the closed state, so the clone can close it again
+        await page.evaluate(remarkInPage, found.groups);
+        await page.click(`[data-tc-trigger="${g.i}:${j}"]`, { timeout: 5000 }).catch(() => {});
+        await page.waitForTimeout(350);
+        g.closed = await snapshot();
+      }
+    }
+    // back to the initial state before the next group (tabs: click the initially selected one)
+    if (g.kind === 'tabs' && g.initial >= 0) {
+      await page.evaluate(remarkInPage, found.groups);
+      await page.click(`[data-tc-trigger="${g.i}:${g.initial}"]`, { timeout: 5000 }).catch(() => {});
+      await page.waitForTimeout(350);
+    }
+    const changed = new Set(g.states.filter(Boolean)).size;
+    console.log(`  ${g.kind.padEnd(6)} ${g.triggers.map((t) => t.label).slice(0, 6).join(' | ')}${g.triggers.length > 6 ? ' ...' : ''}: ${changed} distinct state(s)`);
+  }
+  result = found;
+} finally {
+  if (!flags.cdp) await browser.close();
+}
+const out = path.join(root, 'reference', `widgets.${vp.name}.json`);
+await fs.writeFile(out, JSON.stringify({ url: meta.url, viewport: vp, at: new Date().toISOString(), ...result }, null, 2));
+console.log(`-> ${path.relative(process.cwd(), out)}`);

package/skill/SKILL.md

@@ -0,0 +1,158 @@
+---
+name: trueclone
+description: Build a working twin of a website from its URL - HTML/CSS/JS clone, Next.js + Tailwind + shadcn app, DESIGN.md and shadcn globals.css - and prove the match with layout, scroll-state, motion, interaction, hover and style scores at mobile, tablet and desktop widths. Use when the user gives a URL and asks to clone, replicate, rebuild or migrate a site or page, or to extract its design system.
+---
+
+# Build a true clone of a website
+
+Everything for a site lives in `sites/<name>/` (gitignored, stays local):
+- `clone/`: plain HTML/CSS/JS, the source of truth.
+- `next/`: the same site as a Next.js + Tailwind v4 + shadcn app, generated from `clone/`.
+- `design-tokens.json`: the **one canonical source** of the design system, rendered to `DESIGN.md` (Google design.md format, linted), `globals.css` (shadcn theme) and `clone/tokens.css`.
+- `REPORT.md`: every score against its target, what to fix first, what was not copied. This is the deliverable.
+- `notes.md`: your notes: stack, motion inventory, decisions, known gaps.
+
+Run every step as `npx trueclone <step> <name>` from the project folder (`npx trueclone help` lists them).
+
+## 0. Pick the mode
+
+| Mode | Use it when | Content in the clone |
+|---|---|---|
+| `template` (default) | Anyone else's site: learning, layout study, a starting template | Placeholder brand, text, images and logos; scored with `--structure` |
+| `exact` | The user owns the site or is authorised to copy it (a client migration, their own Webflow/Framer/WordPress site) | The site's own text and media |
+
+Exact mode refuses to run without `--permission "<who authorised it>"`, and the value is recorded in `meta.json` and in `REPORT.md`. Never use exact mode on a site the user doesn't own or have permission for. If permission is unclear, ask.
+
+## 1. Capture
+
+```powershell
+npx trueclone capture <url> [name] [--hide "<cookie banner / popup selectors>"] [--mode exact --permission "<owner>"]
+npx trueclone states <name>      # live scroll states + motion inventory (animations running)
+npx trueclone motion <name>      # animation runtime: ScrollTriggers, timelines, reveals, cursor tweens, loops, Lenis/Swiper, CSS
+npx trueclone spec <name>        # measured boxes and styles per viewport
+```
+
+`npx trueclone <url> [name]` is a shortcut for capture + design-system draft.
+
+`reference/` then holds:
+- full-page PNGs and their `*.mask.png` structure versions, `sections.json` and `rendered.html`;
+- `tokens.json` (colours, type, radii, CSS variables), `assets/` with `manifest.json`, and `desktop.har` (every request, including the data/API calls);
+- **`states/<vp>/NN.png`**: what a visitor sees at every half-viewport scroll step, after motion settles;
+- **`states/<vp>.summary.json`** (the motion inventory): `reveals`, `animated`, `parallax`, `pinned` and `fixed` elements;
+- **`motion.<vp>.md`**: the exact animation recipe when the site uses GSAP/ScrollTrigger, Lenis, Swiper or CSS animations: trigger, start/end, scrub, pin, properties and ranges, duration, easing, stagger. Rebuild these with the same public libraries and these numbers, in your own code;
+- in exact mode, `mirror/index.html`: a script-free local snapshot with assets rewritten to local copies.
+
+Read the motion inventory before building. Each entry is a behaviour the clone must have.
+
+If the page is blank or blocked, capture through your own Chrome over CDP (`chrome --remote-debugging-port=9222 --user-data-dir=.chrome-profile`, then `--cdp`). Never try to get past bot checks, logins or paywalls.
+
+## 2. Understand it with REA
+
+trueclone bundles REA (Reverse Engineer Anything). `npx @trueclone/cli init` registers it as an MCP server; without MCP, run `npx trueclone rea <command>`. With the page open in your CDP Chrome:
+- `inspect_web_page`: DOM, scripts, resources.
+- `analyze_web_bundle` with source maps: framework, vendors (GSAP, ScrollTrigger, Lenis, Swiper, Webflow IX2, three.js), routes and API endpoints.
+- `inspect_web_event_listeners`: hover, drag, wheel and click logic.
+
+Use REA when the capture can't tell you how something works: a behaviour's exact rule, bundled shader or physics code, the API a component calls. Write `notes.md`: stack, breakpoints, and the motion inventory turned into a list of behaviours (trigger, property, range).
+
+## 3. Design system
+
+```powershell
+npx trueclone design <name>              # drafts design-tokens.json if missing, renders DESIGN.md, globals.css, clone/tokens.css + lints
+npx trueclone design <name> --redraft    # fresh draft in design-tokens.draft.json (never overwrites your edits)
+```
+
+Review the draft: role names, the `notes` prose (overview, do's and don'ts), anything marked `origin: generated`. Re-run after every edit. `DESIGN.md` must lint with **0 errors**.
+
+## 4. Build (`clone/`)
+
+**Start every clone from the mirror** (both modes):
+
+```powershell
+npx trueclone reveals <name>   # records scroll reveals frame by frame + class changes caused by scrolling (any library)
+npx trueclone widgets <name>   # clicks every tab and menu and records each state (they often aren't in the page until clicked)
+npx trueclone seed <name>      # clone/ from the script-free mirror; replays what reveals and widgets recorded
+```
+
+- **Exact mode:** the site's own markup, text, stylesheets and media at local paths.
+- **Template mode:** the same markup and stylesheets, but every text becomes lorem words of the same laid-out width (lines break where the reference's break), every image, video, large SVG and iframe a same-size placeholder, CSS backgrounds a transparent one, every link `#`, brand metadata and tracking hints dropped. No text or media of the site ends up in `clone/`; fonts stay for local study (swap licensed fonts before reusing the result).
+- **Recorded behaviour comes with it:** one-shot scroll reveals replay their measured curve (delay, duration, easing, spring overshoot) with the Web Animations API, and scroll class changes (a header that gets its backdrop past 12px) toggle the same classes at the same point. Both live in `clone/trueclone-reveals.js`; don't rewrite them by hand.
+- **Recorded widgets come with it too** (template mode): every tab set and expandable menu, with each state the original showed after a click, placeholdered like the page. `clone/trueclone-widgets.js` swaps the region on click (menus in a header or nav also on hover). Popovers rendered in a portal (comboboxes, tooltips) are listed as not replayed: build those in `script.js`. In interaction scenarios, `selector@n` picks the n-th match in `at`, `click` and `expectChange`; mind hidden duplicates (a mobile menu button is often the first match).
+
+Then write `clone/script.js`: every remaining interaction in your own code, using the parameters you read from the reference (timings, physics constants, offsets, breakpoints). Never copy the site's scripts. JS-set initial states, such as a `<details>` closed on phones, are part of the layout, so check every viewport. Data the page fetches live (a star count, a feed) stays as captured; no requests to third parties.
+
+A seed is where layout ends up at 99%+ with zero hand-written CSS; your work is behaviour. Build by hand (below) only what the mirror can't carry.
+
+**Building parts by hand:**
+
+- `styles.css` starts with `@import url("tokens.css");` and uses its variables. Never hard-code a token value.
+- Use the site's breakpoints and root font-size behaviour (in `design-tokens.json`, under `layout`).
+- Match `sections.json` heights. In template mode write your own placeholder words, then run `npx trueclone textshape <name> --write` (desktop, then `--viewport tablet` and `mobile`) so lines break where the reference's break. Use `--list "<ref row>" "<clone row>"` for repeated lists (calendar rows, cards), and give every data row its own placeholder text.
+
+**Both modes:**
+
+- **Copy the reference's CSS mechanism, not its look.** Before writing CSS for an element, run `npx trueclone rules <name> "<selector>"` and port the matching rules (with their `@media`). Look-alikes drift in flex/grid: a "capsize" `::before/::after` text trim is not `text-box-trim`, `height: 14.5rem` is not `flex: 1`, and `width: 100%; max-width: 40rem` with default flex shrink is not `flex: 0 10 auto` or `width: min-content`.
+- **Positional selectors:** `:nth-of-type` counts siblings of the same tag only. A logo `<a>` before the column `<div>`s shifts every column by one, so use `:nth-child` or a class.
+- **Stacked media:** keep every layer (colour, grayscale or depth-map images, overlays). Never flatten them into one image.
+- **Canvas/WebGL:** a site can draw its hero or a game on a `<canvas>`. Rebuild the drawing in your own code with the reference's parameters (exact mode) or a placeholder scene. Never drop it silently.
+- **Motion:** implement every inventory entry: reveals, parallax speeds, pinned sections, fixed elements.
+- **Interactions:** sliders, tabs, hover states, cursor followers, menus, counters, demos. Scope query selectors to their own component: a hidden duplicate (a mobile-only list) must never feed a desktop counter.
+- **Sliders in flex/grid:** give the slider and every flex/grid ancestor `min-width: 0`, or Swiper and similar grow to fit their slides and run away. `states --clone` warns when the page height drifts more than 20%.
+- `<meta name="robots" content="noindex">`, forms submit nowhere, no tracking scripts.
+
+## 5. Verify
+
+```powershell
+npx trueclone score <name> --quick   # layout, desktop states, interactions: the loop while building (about a minute)
+npx trueclone score <name>           # everything, in parallel groups, then REPORT.md
+```
+
+The steps, if you need one alone:
+
+```powershell
+npx trueclone diff <name> [--structure]            # layout: full page, frozen (structure = text and media masked)
+npx trueclone states <name> --clone [--structure]  # scroll states: the reference's scroll positions replayed on the clone
+npx trueclone curves <name> --draft                # motion-map.draft.json from the inventory: fill in clone selectors, save as motion-map.json
+npx trueclone curves <name> [--viewport tablet]    # motion: property curves (scroll- and time-driven), reference vs clone
+npx trueclone interact <name> --draft              # interactions.draft.json: fill in TODOs, save as interactions.json
+npx trueclone interact <name>                      # interactions: click/hover/drag/key scenarios + expectChange (geometry, text, state attributes, canvas pixels)
+npx trueclone detail <name> --viewport all         # styles at matching points: font, size, weight, line height, colour, boxes
+npx trueclone hovers <name>                        # hover effects present on the reference but missing on the clone
+```
+
+Reference measurements are cached until the next capture, so only the clone is re-measured. Sites without scroll/time motion or without interactions simply skip those steps.
+
+Fix in this order: page height drift, the worst layout section, the worst state step, the lowest motion curve, any interaction marked FAIL, then detail and hovers. `REPORT.md` lists them under "Fix first". For a drift, compare the element's ancestor boxes on both sides until one differs, then `rules` that ancestor: the cause is usually one declaration. Fix the cause; never tune numbers to the score.
+
+**Done when:** layout ≥ 98% at every viewport, states within 5 points of layout, motion ≥ 95%, detail ≥ 95%, hovers ≥ 90%, and every interaction passes. Otherwise stop after 8 passes and report what's left and why.
+
+How detail measures, so you can trust it:
+- motion on (many originals ignore `prefers-reduced-motion`, and a clone that honours it lays out differently);
+- both sides measured the same way: a hit test at the same point and scroll position, so sticky and pinned content is compared in place and an identical page scores 100%;
+- through `pointer-events: none` overlays, and never on content the browser doesn't paint (closed `<details>`);
+- colours written by scroll-scrubbed JS show up as in-between values: that is a motion difference, so check the motion inventory.
+
+How layout and states measure:
+- the structure mask hides text everywhere, gradient text (`background-clip: text`) and `::before/::after` photo backgrounds included;
+- content that a component mounts only while in view (marked `data-tc-viewmount` by capture) can't appear in a full-page still on either side: layout paints its box the same on both and reports the count; the scroll-state check, with a real viewport, scores it;
+- endless animations (marquees, rotating borders) are frozen on their first frame on both sides before every state screenshot: their phase depends on load time, not on the clone.
+
+The live site can change after capture (a redeploy changes its CSS bundle name). When live-site tools (detail, rules) and screenshot scores (layout, states) disagree about a section, re-capture (`capture`, then `states <name>`); that also resets the cached reference measurements.
+
+## 6. Next.js (after the HTML clone passes)
+
+```powershell
+npx trueclone next <name> --verify   # generates sites/<name>/next, installs, builds, serves, and runs every score on it
+```
+
+The HTML clone stays the source of truth: change `clone/`, then regenerate. The app has one component per section, the motion script as a client component (GSAP, Lenis, Swiper and three from npm at the clone's versions), `app/globals.css` = shadcn theme + tokens + the clone's stylesheet + preflight parity, and `components.json` so `npx shadcn add <component>` works. It is done when its scores match the HTML clone's within a point.
+
+## 7. Report
+
+`npx trueclone score <name>` writes `REPORT.md` (or run `npx trueclone report <name>`): mode and permission, every score against its target, "Fix first", the motion found on the reference, and what was not copied (server behaviour, tracking, and each data/API call the reference makes). Hand it over with the clone.
+
+## Honest limits
+
+- Server-side behaviour is never visible from outside: databases, accounts, payments, search. The clone reproduces the client; data calls are listed in the report.
+- A live site is never pixel-identical twice: rotating content, A/B tests and font hinting vary.
+- Template clones of other people's sites stay local. Exact clones need recorded permission and also stay local unless the owner deploys them.