launchframe 0.1.5 → 0.1.7

package/packages/extract/emit.ts

@@ -9,8 +9,8 @@
  *   globals.css         — shadcn-compatible CSS variables (light + dark)
  *   theme-preview.tsx   — a self-contained React component that renders
  *                         every token so you can eyeball the system
- *   REPORT.md           — what was extracted, from where, with the
- *                         anti-clone disclaimer
+ *   REPORT.md           — what was extracted, from where, and how the
+ *                         output is meant to be used
  */
 
 import { mkdirSync, writeFileSync } from "node:fs";

package/packages/extract/extract.ts

@@ -4,16 +4,23 @@
  *   npm run extract -- https://site-a.com https://site-b.com https://site-c.com
  *
  * For each URL: open in Chromium, screenshot, harvest computed design
- * tokens via `browser-extract.ts`. After all sites: synthesize an
- * original shadcn-compatible design system and emit drop-in files.
+ * tokens via `browser-extract.ts`, and crawl the rendered DOM into a
+ * typed `SiteLayout` model via `dom-crawler.ts`. After all sites:
+ *   - Synthesize a drop-in shadcn-compatible design system from the
+ *     aggregated tokens.
+ *   - Emit a per-site **layout mirror**: a Next.js page that reconstructs
+ *     the source's section structure from typed primitives, with
+ *     `<TextSlot>` / `<MediaSlot>` placeholders for the user's copy and
+ *     brand assets.
  *
  * Output goes to `output/<runId>/`.
  *
- * Policy (from rules/anti-clone-policy.md):
- *   - Honor robots.txt by default.
- *   - Per-domain rate limit defaults to 15 req/min.
- *   - Store only PNG screenshots and harvested computed-style values.
- *   - Never persist HTML, JS, CSS, or third-party assets.
+ * Operational defaults (configurable via flags):
+ *   - Honor robots.txt unless `--no-robots` is passed.
+ *   - Per-domain rate limit defaults to 15 req/min (`--rate <n>`).
+ *   - The crawler extracts a structured representation (section tree,
+ *     computed style tokens, content kinds) and writes a verbatim
+ *     `reference/<host>/` bundle (HTML + visible text + media URLs) for AI.
  */
 
 import { mkdirSync, writeFileSync } from "node:fs";
@@ -23,9 +30,11 @@ import { fileURLToPath, pathToFileURL } from "node:url";
 import { chromium, type Browser } from "playwright";
 
 import { harvestTokens } from "./browser-extract.js";
+import { crawlLayout } from "./dom-crawler.js";
 import { emitAll } from "./emit.js";
+import { emitPageReference } from "./reference-dump.js";
 import { synthesize } from "./synthesize.js";
-import type { ExtractionRun, RawTokens, SiteCapture } from "./types.js";
+import type { ExtractionRun, RawTokens, SiteCapture, SiteLayout } from "./types.js";
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -90,9 +99,17 @@ function printHelp(): void {
       "Writes to ./output/<runId>/ in your current working directory unless",
       "you pass --out.",
       "",
-      "Captures each URL at a desktop viewport, harvests computed design",
-      "tokens (colors, type, spacing, radius, shadow), and synthesizes a",
-      "drop-in shadcn-compatible design system in output/<runId>/.",
+      "For each URL the CLI:",
+      "  1. Renders the page at a desktop viewport in headless Chromium.",
+      "  2. Captures a full-page screenshot and harvests computed design tokens",
+      "     (colors, type, spacing, radius, shadow) → raw/<host>.tokens.json.",
+      "  3. Writes a verbatim reference bundle → reference/<host>/ (page.html,",
+      "     visible-text.json/.txt, media.json, meta.json, FOR_AI_REFERENCE.md).",
+      "  4. Crawls the DOM into SiteLayout → raw/<host>.layout.json and emits",
+      "     mirror/<host>/page.tsx (Framer Motion + Phosphor + image/video slots).",
+      "",
+      "After every URL, a drop-in shadcn-compatible design system is",
+      "synthesized from the aggregated tokens and written to output/<runId>/.",
       "",
       "Options:",
       "  --out <dir>          Output directory (default: output/<runId>)",
@@ -178,11 +195,14 @@ async function captureOne(
   url: string,
   viewport: { width: number; height: number },
   outDir: string,
-): Promise<{ raw: RawTokens; capture: SiteCapture } | null> {
+): Promise<{ raw: RawTokens; layout: SiteLayout | null; capture: SiteCapture } | null> {
   const host = new URL(url).host;
   const stamp = `${host}.png`;
   const screenshotPath = join(outDir, "screenshots", stamp);
   const rawPath = join(outDir, "raw", `${host}.tokens.json`);
+  const layoutPath = join(outDir, "raw", `${host}.layout.json`);
+  const mirrorDir = join(outDir, "mirror", host);
+  const referenceDir = join(outDir, "reference", host);
 
   const ctx = await browser.newContext({
     userAgent: USER_AGENT,
@@ -215,18 +235,40 @@ async function captureOne(
     mkdirSync(dirname(rawPath), { recursive: true });
     writeFileSync(rawPath, JSON.stringify(raw, null, 2));
 
+    let referenceWritten: string[] = [];
+    try {
+      referenceWritten = await emitPageReference(page, url, referenceDir);
+    } catch (err) {
+      console.warn(`  ! reference dump failed for ${url}: ${(err as Error).message}`);
+    }
+
+    let layout: SiteLayout | null = null;
+    let mirrorWritten: string[] = [];
+    try {
+      layout = await crawlLayout(page, url, viewport);
+      mkdirSync(dirname(layoutPath), { recursive: true });
+      writeFileSync(layoutPath, JSON.stringify(layout, null, 2));
+      mirrorWritten = emitMirror(layout, mirrorDir);
+    } catch (err) {
+      console.warn(`  ! layout crawl failed for ${url}: ${(err as Error).message}`);
+    }
+
     const capture: SiteCapture = {
       url,
       host,
       capturedAt: raw.capturedAt,
       screenshotPath,
       rawTokensPath: rawPath,
+      ...(referenceWritten.length > 0 ? { referenceDir } : {}),
+      ...(layout ? { layoutPath } : {}),
+      ...(mirrorWritten.length > 0 ? { mirrorDir } : {}),
       status: "ok",
     };
-    return { raw, capture };
+    return { raw, layout, capture };
   } catch (err) {
     return {
       raw: emptyRaw(url, viewport),
+      layout: null,
       capture: {
         url,
         host,
@@ -303,7 +345,11 @@ async function main(): Promise<void> {
       captures.push(result.capture);
       if (result.capture.status === "ok") {
         rawList.push(result.raw);
-        console.log(`  ✓ ${url}`);
+        const tag = result.layout ? "mirror" : "tokens-only";
+        const sectionCount = result.layout?.sections.length ?? 0;
+        console.log(
+          `  ✓ ${url}  → ${tag}${result.layout ? ` (${sectionCount} sections)` : ""}`,
+        );
       } else {
         console.log(`  ✗ ${url}  ${result.capture.reason ?? ""}`);
       }
@@ -339,9 +385,32 @@ async function main(): Promise<void> {
   console.log("[extract] wrote:");
   for (const f of written) console.log(`  → ${f}`);
   console.log(`  → ${join(outDir, "run.json")}`);
+  const mirrorDirs = captures.filter((c) => c.mirrorDir).map((c) => c.mirrorDir!);
+  const referenceDirs = captures.filter((c) => c.referenceDir).map((c) => c.referenceDir!);
+  if (mirrorDirs.length > 0) {
+    console.log("");
+    console.log("[extract] layout mirrors:");
+    for (const d of mirrorDirs) console.log(`  → ${d}/page.tsx`);
+  }
+  if (referenceDirs.length > 0) {
+    console.log("");
+    console.log("[extract] AI reference (verbatim DOM + copy):");
+    for (const d of referenceDirs) console.log(`  → ${d}/FOR_AI_REFERENCE.md`);
+  }
   console.log("");
-  console.log(`[extract] done. Open ${join(outDir, "REPORT.md")} for the summary.`);
-  console.log(`[extract] Give your AI: ${join(outDir, "FOR_AI.md")}`);
+  console.log(`[extract] done. Open ${join(outDir, "REPORT.md")} for the design-system summary.`);
+  if (mirrorDirs.length > 0) {
+    console.log(
+      `[extract] each mirror folder ships a Next.js page.tsx + MIRROR_NOTES.md.`,
+    );
+    console.log(`[extract] fill the <TextSlot> / <MediaSlot> placeholders with your own content.`);
+  }
+  if (referenceDirs.length > 0) {
+    console.log(
+      `[extract] paste reference/<host>/visible-text.txt or page.html into your AI for exact structure + copy.`,
+    );
+  }
+  console.log(`[extract] AI handoff: ${join(outDir, "FOR_AI.md")}`);
 }
 
 function makeRunId(startedAt: string, name: string | undefined): string {