launchframe 0.1.8 → 0.1.9

package/README.md

@@ -54,13 +54,15 @@ output/<runId>/
 ├── run.json             ← full run metadata (sources, timing, status)
 ├── screenshots/         ← captured PNGs
 ├── raw/                 ← per-site raw token + SiteLayout JSON
-├── reference/           ← verbatim DOM + copy for AI (see below)
+├── reference/           ← verbatim DOM + **exact structure JSON** + copy for AI
 │   └── <host>/
-│       ├── page.html           ← full HTML after JavaScript
-│       ├── visible-text.txt    ← paste-friendly copy extraction
-│       ├── visible-text.json   ← structured headings / body / buttons
-│       ├── media.json          ← img + video URLs
-│       ├── meta.json           ← title, description, lang
+│       ├── page.html
+│       ├── dom-structure.json   ← canonical body tree (tags, attrs, text nodes)
+│       ├── structure-outline.txt  ← tag skeleton for quick scanning
+│       ├── visible-text.txt
+│       ├── visible-text.json
+│       ├── media.json
+│       ├── meta.json
 │       └── FOR_AI_REFERENCE.md
 └── mirror/
     └── <host>/
@@ -74,15 +76,15 @@ output/<runId>/
 ## Hand the output to your AI
 
 1. Run the command above so `output/<runId>/` exists.
-2. Attach **`reference/<host>/`** (`visible-text.txt`, `page.html`, `media.json`) so the model sees **exact copy and structure** from the crawl.
+2. Attach **`reference/<host>/`**, especially **`dom-structure.json`** (exact tree) and **`visible-text.*`**, plus **`page.html`** and **`media.json`** so the model sees **exact structure and copy** from the crawl.
 3. Pick the mirror folder: `output/<runId>/mirror/<host>/`.
 4. Either:
    - **Cursor:** `@`-attach `reference/<host>/`, `mirror/<host>/`, `FOR_AI.md`, and
      `tokens.json`, then ask the agent to port copy from `visible-text.txt` into
      `page.tsx` and wire media from `media.json`.
    - **Claude Code:** copy both folders into your project, then ask the same.
-5. The AI's authority order is **reference/visible-text.txt & page.html →
-   MIRROR_NOTES.md → page.tsx → tokens.json → tailwind.config.ts + globals.css**. It must:
+5. The AI's authority order is **`dom-structure.json` (nesting) → `structure-outline.txt` / `page.html` → `visible-text.*` →
+   MIRROR_NOTES.md → mirror `page.tsx` → tokens.json → tailwind.config.ts + globals.css**. It must:
    - Keep the section tree, grid composition, density, Motion, and Phosphor usage in `page.tsx`.
    - Map strings from `visible-text.txt` into the right `<TextSlot>` slots (or replace slots with plain JSX).
    - Use `media.json` for image/video `src` / `poster` (respect licensing; prefer your own assets).
@@ -191,22 +193,30 @@ npm run analyze        # Run section classifier on captured screenshots
 npm run formalize      # Validate the pattern-atlas/*.json files
 npm run evaluate       # Grade a generated page (coherence + a11y)
 npm run typecheck      # Project-wide TypeScript check
+npm run sync:agents    # Regenerate Copilot / Cline / Continue / Amazon Q stubs from AGENTS.md
 ```
 
+### AI agents in this repo
+
+- **`AGENTS.md`** (root) is the **single source of truth** for how agents should work here (extract handoff, structure fidelity, compliance). **`docs/research/INSPECTION_GUIDE.md`** is inlined into derived configs when you run `npm run sync:agents`.
+- **Cursor:** `.cursor/rules/project.mdc` points at `AGENTS.md`.
+- **Claude Code / Gemini CLI:** `CLAUDE.md` and `GEMINI.md` import `AGENTS.md`.
+- Edit `AGENTS.md`, then run `npm run sync:agents` (or `bash scripts/sync-agent-rules.sh`) to refresh `.github/copilot-instructions.md`, `.clinerules`, `.continue/rules/project.md`, and `.amazonq/rules/project.md`.
+
 ---
 
 ## What this is not
 
-- **Not a verbatim site downloader.** The crawler builds a typed
-  `SiteLayout` model from the rendered DOM — section tree, geometry,
-  computed style tokens, content kinds — and emits code generated from
-  that model. It does not save the source's HTML/CSS to disk.
-- **Not a content lift.** Heading text, body copy, logos, illustrations,
-  and product imagery become `<TextSlot>` / `<MediaSlot>` placeholders in
-  the mirror page. You fill them with your own copy and assets before
-  shipping.
-- **Not a component library replacement.** It sits *on top* of
-  shadcn/ui and produces theme files plus slot-driven page templates.
+- **Not the source site's frontend bundle.** The **layout mirror** is React
+  code emitted from a typed `SiteLayout` (section tree, composition,
+  density) — not a dump of the origin's original components or stylesheets.
+- **Not a substitute for legal clearance.** `reference/<host>/` may contain
+  serialized DOM and visible text for tooling **you** run on pages you are
+  allowed to analyze. You are responsible for trademarks, copy licenses, and
+  `robots.txt`/ToS compliance when using those artifacts.
+- **Not a component library replacement.** Launchframe sits *alongside*
+  shadcn/ui: theme files, reference dumps, and slot-driven mirror pages —
+  you integrate into your own app.
 
 ---
 
@@ -216,11 +226,10 @@ Launchframe is intended for layout research and design-system seeding
 against pages you have permission to analyze (your own products, sites
 where the operator has permission, or pages where structural analysis is
 permitted by `robots.txt`). The crawler respects `robots.txt` by default
-and rate-limits per domain. The output is generated code derived from a
-normalized typed model and slot placeholders — not a verbatim copy of
-the source's markup, copy, or assets. Operators are responsible for the
-content they paste into those slots and for honoring third-party
-trademarks, terms of service, and licenses.
+and rate-limits per domain. Output includes synthesized theme files, a
+typed **mirror** page scaffold, and (per capture) a **reference** bundle
+(DOM snapshot, visible text, media URLs) for AI-assisted reconstruction.
+Operators remain responsible for how they use copy, media, and branding.
 
 ---
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "launchframe",
-  "version": "0.1.8",
+  "version": "0.1.9",
   "description": "Point Launchframe at SaaS sites you admire and get back a drop-in shadcn/ui design system (tokens, Tailwind theme, CSS variables, AI handoff) you can build your own UI on top of.",
   "license": "MIT",
   "author": "Evan Gruhlkey",
@@ -46,6 +46,7 @@
     "analyze": "tsx packages/analysis/analyze-screenshot.ts",
     "formalize": "tsx packages/patterns/pattern-registry.ts",
     "evaluate": "tsx packages/evaluation/evaluate-page.ts",
+    "sync:agents": "node scripts/sync-agent-rules.mjs",
     "typecheck": "tsc -p tsconfig.json --noEmit",
     "format:check": "prettier --check ."
   },

package/packages/extract/emit.ts

@@ -304,7 +304,7 @@ function emitReport(system: DesignSystem, run: ExtractionRun): string {
       const lines = [`- ${c.url}`, `  - screenshot: \`${relativize(c.screenshotPath, run.outputDir)}\``];
       if (c.referenceDir) {
         lines.push(
-          `  - reference: \`${relativize(c.referenceDir, run.outputDir)}/\` — \`page.html\`, \`visible-text.*\`, \`media.json\`, \`FOR_AI_REFERENCE.md\``,
+          `  - reference: \`${relativize(c.referenceDir, run.outputDir)}/\` — \`page.html\`, \`dom-structure.json\`, \`structure-outline.txt\`, \`visible-text.*\`, \`media.json\`, \`FOR_AI_REFERENCE.md\``,
         );
       }
       if (c.mirrorDir) {
@@ -403,7 +403,7 @@ ${rampRows(system.dark)}
   shadows). It does not, by itself, define every layout detail of a source page.
 - **Per-site \`reference/\` and \`mirror/\` folders** (when emitted) are the
   recon inputs for **structure and copy**: map landmarks and section order from
-  \`page.html\`, pull strings from \`visible-text.json\` or \`visible-text.txt\`,
+  \`dom-structure.json\` (canonical nesting) + \`page.html\`, pull strings from \`visible-text.json\` or \`visible-text.txt\`,
   align media via \`media.json\`, and implement or refine UI starting from
   \`mirror/<host>/page.tsx\` (\`data-mirror-section\` markers match the crawl).
 - **Compliance:** Do not impersonate another company, ship their trademarks or
@@ -429,7 +429,7 @@ function emitForAi(system: DesignSystem, run: ExtractionRun): string {
     .map((c) => {
       const lines = [`### ${c.url}`];
       if (c.referenceDir) {
-        lines.push(`- **Reference:** \`${relativize(c.referenceDir, run.outputDir)}/\` — start with \`FOR_AI_REFERENCE.md\`, then \`page.html\`, \`visible-text.txt\` (or \`.json\`), \`media.json\`.`);
+        lines.push(`- **Reference:** \`${relativize(c.referenceDir, run.outputDir)}/\` — start with \`FOR_AI_REFERENCE.md\`, then \`dom-structure.json\` (exact tree), \`structure-outline.txt\`, \`page.html\`, \`visible-text.txt\` (or \`.json\`), \`media.json\`.`);
       }
       if (c.mirrorDir) {
         lines.push(`- **Mirror:** \`${relativize(c.mirrorDir, run.outputDir)}/page.tsx\` — section scaffold + \`data-mirror-section\`; read \`MIRROR_NOTES.md\`.`);
@@ -448,7 +448,7 @@ ${perHost || "_No reference/mirror paths on this run — token-only._"}
 
 Workflow (similar in spirit to **recon → specs → build** pipelines):
 
-1. **Recon:** Skim \`page.html\` for DOM landmarks (\`<header>\`, main columns, card grids, repeated list patterns). Cross-check section order with \`mirror/.../page.tsx\` (\`data-mirror-section\`).
+1. **Recon:** Use \`dom-structure.json\` for **exact nesting and sibling order**; use \`structure-outline.txt\` or \`page.html\` for skimming. Cross-check with \`mirror/.../page.tsx\` (\`data-mirror-section\`).
 2. **Wire copy + media:** Map headings, buttons, and blocks from \`visible-text.*\` into the matching mirror sections (or your components). Use \`media.json\` for asset URLs; replace with licensed or original assets when shipping.
 3. **Build:** Prefer editing **mirror \`page.tsx\`** inside the user's app (or port its structure into their file tree) rather than inventing a new section order from scratch. Apply **REPORT.md** / **tokens** for colors, type, spacing, radii — mirror CSS variables under \`.mirror-root\` should converge to the same semantic palette where possible.
 
@@ -474,7 +474,7 @@ In **Cursor**, \`@\` those paths explicitly.
 
 ## Authority order
 
-1. **Structural fidelity:** \`reference/<host>/page.html\` + \`visible-text.*\` + \`mirror/<host>/page.tsx\` — section order, composition patterns, and strings (when the user wants fidelity).
+1. **Structural fidelity:** \`reference/<host>/dom-structure.json\` + \`page.html\` + \`visible-text.*\` + \`mirror/<host>/page.tsx\` — exact DOM tree shape, then copy and typed scaffold.
 2. **Design tokens:** **REPORT.md** and **tokens.json** — typography scale, spacing, radii, colors, container width, notes.
 3. **Integration:** **tailwind.config.ts** + **globals.css** — merge into a Next.js + Tailwind + shadcn-style app.
 ${structureSection}
@@ -485,7 +485,7 @@ You must use the attached \`output/${system.runId}/\` folder.
 
 - Read REPORT.md and tokens.json before writing UI. Merge tailwind.config.ts and globals.css into my project (preserve my paths unless I say otherwise).
 - Style with semantic tokens: bg-background, text-foreground, text-muted-foreground, border-border, bg-primary, text-primary-foreground, bg-card, text-card-foreground, etc. Prefer these over ad-hoc hex; mirror pages may use --mirror-* variables until merged.
-- If reference/ and mirror/ exist for my source URL: treat them as mandatory context. Preserve crawled section order and layout patterns: align components to data-mirror-section and page.html landmarks. Wire copy from visible-text.* and media from media.json unless I say to rewrite for a different product.
+- If reference/ and mirror/ exist for my source URL: treat them as mandatory context. Preserve **exact DOM nesting and sibling order** from `dom-structure.json` (and cross-check `page.html`). Align components to `data-mirror-section` and the mirror scaffold. Wire copy from visible-text.* and media from media.json unless I say to rewrite for a different product.
 - If I am building a NEW product unrelated to the crawl: keep layout inspiration from mirror/reference but REPLACE product names, claims, and sensitive copy with my copy. Never impersonate another brand.
 
 My product / intent: [describe goal — faithful mirror of URL vs new product in same layout; tone and CTA]

package/packages/extract/extract.ts

@@ -20,7 +20,8 @@
  *   - Per-domain rate limit defaults to 15 req/min (`--rate <n>`).
  *   - The crawler extracts a structured representation (section tree,
  *     computed style tokens, content kinds) and writes a verbatim
- *     `reference/<host>/` bundle (HTML + visible text + media URLs) for AI.
+ *     `reference/<host>/` bundle (HTML, DOM tree JSON, outlines, visible text,
+ *     media index) for AI structure cloning.
  */
 
 import { mkdirSync, writeFileSync } from "node:fs";
@@ -104,7 +105,8 @@ function printHelp(): void {
       "  2. Captures a full-page screenshot and harvests computed design tokens",
       "     (colors, type, spacing, radius, shadow) → raw/<host>.tokens.json.",
       "  3. Writes a verbatim reference bundle → reference/<host>/ (page.html,",
-      "     visible-text.json/.txt, media.json, meta.json, FOR_AI_REFERENCE.md).",
+      "     dom-structure.json, structure-outline.txt, visible-text.json/.txt,",
+      "     media.json, meta.json, FOR_AI_REFERENCE.md).",
       "  4. Crawls the DOM into SiteLayout → raw/<host>.layout.json and emits",
       "     mirror/<host>/page.tsx (Framer Motion + Phosphor + image/video slots).",
       "",
@@ -237,7 +239,7 @@ async function captureOne(
 
     let referenceWritten: string[] = [];
     try {
-      referenceWritten = await emitPageReference(page, url, referenceDir);
+      referenceWritten = await emitPageReference(page, url, referenceDir, viewport);
     } catch (err) {
       console.warn(`  ! reference dump failed for ${url}: ${(err as Error).message}`);
     }

package/packages/extract/reference-dump.ts

@@ -2,11 +2,13 @@
  * Verbatim reference dump for AI / human review.
  *
  * Writes everything under `output/<runId>/reference/<host>/`:
- *   - page.html          — full document HTML after JS render (`page.content()`)
- *   - visible-text.json  — structured visible copy (headings, buttons, key blocks)
- *   - media.json         — img / video / source URLs and attributes
- *   - meta.json          — title, description, canonical, lang
- *   - FOR_AI_REFERENCE.md — how to use these files with an AI
+ *   - page.html              — full document HTML after JS render (`page.content()`)
+ *   - dom-structure.json    — exact `body` subtree: tag order, attributes, text nodes (JSON)
+ *   - structure-outline.txt  — indented tag skeleton (ids/classes/roles) for quick scanning
+ *   - visible-text.json     — structured visible copy (headings, buttons, key blocks)
+ *   - media.json            — img / video / source URLs and attributes
+ *   - meta.json             — title, description, canonical, lang
+ *   - FOR_AI_REFERENCE.md   — how to use these files with an AI
  */
 
 import { mkdirSync, writeFileSync } from "node:fs";
@@ -14,6 +16,33 @@ import { join } from "node:path";
 
 import type { Page } from "playwright";
 
+/** Element node in the serialized `body` tree. */
+export interface DomStructureElement {
+  tag: string;
+  attrs?: Record<string, string>;
+  children?: DomStructureChild[];
+}
+
+export interface DomStructureTextNode {
+  type: "text";
+  value: string;
+}
+
+export type DomStructureChild = DomStructureElement | DomStructureTextNode;
+
+export interface DomStructurePayload {
+  title: string | null;
+  lang: string | null;
+  stats: {
+    elementNodes: number;
+    omitted: { script: number; style: number; noscript: number; template: number };
+    truncated: boolean;
+    maxNodes: number;
+    maxDepth: number;
+  };
+  root: DomStructureElement | null;
+}
+
 export interface ReferenceSnapshot {
   url: string;
   capturedAt: string;
@@ -35,7 +64,12 @@ export interface ReferenceSnapshot {
   >;
 }
 
-export async function emitPageReference(page: Page, url: string, refDir: string): Promise<string[]> {
+export async function emitPageReference(
+  page: Page,
+  url: string,
+  refDir: string,
+  viewport?: { width: number; height: number },
+): Promise<string[]> {
   mkdirSync(refDir, { recursive: true });
   const written: string[] = [];
   const capturedAt = new Date().toISOString();
@@ -50,6 +84,21 @@ export async function emitPageReference(page: Page, url: string, refDir: string)
   writeFileSync(htmlPath, html, "utf8");
   written.push(htmlPath);
 
+  const domEval = (await page.evaluate(collectDomStructureInPage)) as DomStructurePayload;
+  const domMerged = {
+    url,
+    capturedAt,
+    viewport: viewport ?? null,
+    ...domEval,
+  };
+  const domPath = join(refDir, "dom-structure.json");
+  writeFileSync(domPath, JSON.stringify(domMerged, null, 2) + "\n", "utf8");
+  written.push(domPath);
+
+  const outlinePath = join(refDir, "structure-outline.txt");
+  writeFileSync(outlinePath, buildStructureOutline(domMerged.root, domMerged.stats) + "\n", "utf8");
+  written.push(outlinePath);
+
   const snapshot = (await page.evaluate(collectSnapshot)) as Omit<ReferenceSnapshot, "url" | "capturedAt">;
   const full: ReferenceSnapshot = {
     url,
@@ -102,11 +151,13 @@ function emitAiReadme(url: string, refDir: string): string {
     "| File | Purpose |",
     "| ---- | ------- |",
     "| `page.html` | Full serialized DOM after JavaScript ran in Chromium. Layout, copy, and structure match what crawled (not necessarily valid static HTML elsewhere). |",
+    "| `dom-structure.json` | **Exact body subtree (JSON):** same child order as the live DOM; every attribute on each element (long `src` / `href` / `style` values truncated); inline text as separate typed entries. Use this as the canonical structure map for React. |",
+    "| `structure-outline.txt` | Indented tag skeleton (ids/classes/roles only; no text) for quick navigation. |",
     "| `visible-text.json` | Exact visible strings: headings, buttons, links, and block text — good for **verbatim copy** when rewriting `page.tsx`. |",
     "| `media.json` | Every image / video / source URL from the DOM. Host your own assets or swap for placeholders; do not hotlink without permission. |",
     "| `meta.json` | Title, description, lang. |",
     "",
-    "**Workflow:** (1) Recon — skim \`page.html\` for landmarks and grids; (2) Wire — map \`visible-text.*\` + \`media.json\` into sections; (3) Build — prefer editing sibling \`../mirror/<host>/page.tsx\` (section order via \`data-mirror-section\`) instead of inventing layout from scratch. See the run folder’s **FOR_AI.md** for full authority order and compliance notes.",
+    "**Workflow:** (1) Recon — use **`dom-structure.json`** (or `structure-outline.txt` + `page.html`) for **exact tag order and nesting**; (2) Wire — map `visible-text.*` + `media.json` into that tree; (3) Build — implement `page.tsx` (start from `../mirror/<host>/page.tsx` if present) so component boundaries follow this structure. See the run folder’s **FOR_AI.md** for full authority order and compliance notes.",
     "",
     `Sibling folder \`../mirror/<host>/\` has a typed \`page.tsx\` with Framer Motion, Phosphor icons, and slots — wire copy from \`visible-text.json\` and media from \`media.json\` into that file.`,
     "",
@@ -115,6 +166,154 @@ function emitAiReadme(url: string, refDir: string): string {
   ].join("\n");
 }
 
+function buildStructureOutline(root: DomStructureElement | null, stats: DomStructurePayload["stats"]): string {
+  const MAX_LINES = 15_000;
+  const lines: string[] = [
+    "# structure-outline.txt — tag skeleton under <body>",
+    "# Text nodes omitted here; see dom-structure.json for interleaved copy.",
+    `# elementNodes=${stats.elementNodes} truncated=${stats.truncated} omitted=${JSON.stringify(stats.omitted)}`,
+    "",
+  ];
+  let n = 0;
+  function walkEl(node: DomStructureElement, depth: number) {
+    if (n >= MAX_LINES) return;
+    const a = node.attrs ?? {};
+    let suffix = "";
+    if (a.id) suffix += `#${String(a.id).replace(/\s+/g, "")}`;
+    if (a.class) {
+      const parts = String(a.class)
+        .split(/\s+/)
+        .filter(Boolean)
+        .slice(0, 6)
+        .map((c) => `.${c}`);
+      suffix += parts.join("");
+    }
+    if (a.role) suffix += `[role=${a.role}]`;
+    lines.push(`${"  ".repeat(depth)}${node.tag}${suffix}`);
+    n++;
+    for (const c of node.children ?? []) {
+      if (n >= MAX_LINES) return;
+      if ("type" in c && c.type === "text") continue;
+      walkEl(c as DomStructureElement, depth + 1);
+    }
+  }
+  if (root) walkEl(root, 0);
+  if (n >= MAX_LINES) lines.push("\n… outline truncated …");
+  return lines.join("\n");
+}
+
+/**
+ * Runs in browser context. Serializes `document.body` with exact child order; skips
+ * script/style/noscript/template (counts only).
+ */
+function collectDomStructureInPage(): DomStructurePayload {
+  const MAX_NODES = 40_000;
+  const MAX_DEPTH = 128;
+  const MAX_TEXT = 12_000;
+
+  let nodeCount = 0;
+  let truncated = false;
+  const omitted = { script: 0, style: 0, noscript: 0, template: 0 };
+
+  function trimAttr(name: string, v: string): string {
+    const big =
+      name === "src" ||
+      name === "href" ||
+      name === "srcset" ||
+      name === "style" ||
+      name === "content" ||
+      name.startsWith("data-");
+    const max = big ? 3500 : 2000;
+    if (v.length <= max) return v;
+    return v.slice(0, max) + "…";
+  }
+
+  function walk(el: Element, depth: number): DomStructureElement | null {
+    if (nodeCount >= MAX_NODES) {
+      truncated = true;
+      return null;
+    }
+    const tag = el.tagName.toLowerCase();
+    if (tag === "script") {
+      omitted.script++;
+      return null;
+    }
+    if (tag === "style") {
+      omitted.style++;
+      return null;
+    }
+    if (tag === "noscript") {
+      omitted.noscript++;
+      return null;
+    }
+    if (tag === "template") {
+      omitted.template++;
+      return null;
+    }
+    if (depth > MAX_DEPTH) return null;
+
+    nodeCount++;
+    const attrs: Record<string, string> = {};
+    for (let i = 0; i < el.attributes.length; i++) {
+      const a = el.attributes.item(i);
+      if (!a) continue;
+      attrs[a.name] = trimAttr(a.name, a.value);
+    }
+
+    const children: DomStructureChild[] = [];
+    for (const child of el.childNodes) {
+      if (nodeCount >= MAX_NODES) {
+        truncated = true;
+        break;
+      }
+      if (child.nodeType === 3) {
+        let t = (child.textContent ?? "").replace(/\s+/g, " ").trim();
+        if (!t) continue;
+        if (t.length > MAX_TEXT) t = t.slice(0, MAX_TEXT) + "…";
+        children.push({ type: "text", value: t });
+      } else if (child.nodeType === 1) {
+        const sub = walk(child as Element, depth + 1);
+        if (sub) children.push(sub);
+      }
+    }
+
+    const out: DomStructureElement = { tag };
+    if (Object.keys(attrs).length > 0) out.attrs = attrs;
+    if (children.length > 0) out.children = children;
+    return out;
+  }
+
+  const body = document.body;
+  if (!body) {
+    return {
+      title: document.title || null,
+      lang: document.documentElement.getAttribute("lang"),
+      stats: {
+        elementNodes: 0,
+        omitted,
+        truncated: false,
+        maxNodes: MAX_NODES,
+        maxDepth: MAX_DEPTH,
+      },
+      root: null,
+    };
+  }
+
+  const root = walk(body, 0);
+  return {
+    title: document.title || null,
+    lang: document.documentElement.getAttribute("lang"),
+    stats: {
+      elementNodes: nodeCount,
+      omitted,
+      truncated,
+      maxNodes: MAX_NODES,
+      maxDepth: MAX_DEPTH,
+    },
+    root,
+  };
+}
+
 /**
  * Runs in browser context.
  */