launchframe 0.1.13 → 0.2.1

package/packages/extract/reference-dump.ts

@@ -1,431 +0,0 @@
-/**
- * Verbatim reference dump for AI / human review.
- *
- * Writes everything under `output/<runId>/reference/<host>/`:
- *   - page.html              — full document HTML after JS render (`page.content()`)
- *   - dom-structure.json    — exact `body` subtree: tag order, attributes, text nodes (JSON)
- *   - structure-outline.txt  — indented tag skeleton (ids/classes/roles) for quick scanning
- *   - visible-text.json     — structured visible copy (headings, buttons, key blocks)
- *   - media.json            — img / video / source URLs and attributes
- *   - meta.json             — title, description, canonical, lang
- *   - FOR_AI_REFERENCE.md   — how to use these files with an AI
- */
-
-import { mkdirSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-
-import type { Page } from "playwright";
-
-/** Element node in the serialized `body` tree. */
-export interface DomStructureElement {
-  tag: string;
-  attrs?: Record<string, string>;
-  children?: DomStructureChild[];
-}
-
-export interface DomStructureTextNode {
-  type: "text";
-  value: string;
-}
-
-export type DomStructureChild = DomStructureElement | DomStructureTextNode;
-
-export interface DomStructurePayload {
-  title: string | null;
-  lang: string | null;
-  stats: {
-    elementNodes: number;
-    omitted: { script: number; style: number; noscript: number; template: number };
-    truncated: boolean;
-    maxNodes: number;
-    maxDepth: number;
-  };
-  root: DomStructureElement | null;
-}
-
-export interface ReferenceSnapshot {
-  url: string;
-  capturedAt: string;
-  title: string | null;
-  description: string | null;
-  canonical: string | null;
-  lang: string | null;
-  /** Flattened visible strings in DOM order (useful for grep / LLM context). */
-  visibleTextBlocks: Array<{
-    tag: string;
-    role: string | null;
-    text: string;
-  }>;
-  links: Array<{ href: string; text: string }>;
-  media: Array<
-    | { type: "img"; src: string; alt: string; width: number | null; height: number | null }
-    | { type: "video"; src: string | null; poster: string | null }
-    | { type: "source"; src: string; kind: string | null }
-  >;
-}
-
-export async function emitPageReference(
-  page: Page,
-  url: string,
-  refDir: string,
-  viewport?: { width: number; height: number },
-): Promise<string[]> {
-  mkdirSync(refDir, { recursive: true });
-  const written: string[] = [];
-  const capturedAt = new Date().toISOString();
-
-  await page.evaluate(() => {
-    const g = globalThis as unknown as { __name?: (fn: unknown) => unknown };
-    if (typeof g.__name === "undefined") g.__name = (fn: unknown) => fn;
-  });
-
-  const html = await page.content();
-  const htmlPath = join(refDir, "page.html");
-  writeFileSync(htmlPath, html, "utf8");
-  written.push(htmlPath);
-
-  const domEval = (await page.evaluate(collectDomStructureInPage)) as DomStructurePayload;
-  const domMerged = {
-    url,
-    capturedAt,
-    viewport: viewport ?? null,
-    ...domEval,
-  };
-  const domPath = join(refDir, "dom-structure.json");
-  writeFileSync(domPath, JSON.stringify(domMerged, null, 2) + "\n", "utf8");
-  written.push(domPath);
-
-  const outlinePath = join(refDir, "structure-outline.txt");
-  writeFileSync(outlinePath, buildStructureOutline(domMerged.root, domMerged.stats) + "\n", "utf8");
-  written.push(outlinePath);
-
-  const snapshot = (await page.evaluate(collectSnapshot)) as Omit<ReferenceSnapshot, "url" | "capturedAt">;
-  const full: ReferenceSnapshot = {
-    url,
-    capturedAt,
-    ...snapshot,
-  };
-
-  writeFileSync(join(refDir, "visible-text.json"), JSON.stringify(full, null, 2) + "\n", "utf8");
-  written.push(join(refDir, "visible-text.json"));
-
-  const txtLines = [
-    `# ${full.title ?? "Untitled"}`,
-    "",
-    ...full.visibleTextBlocks.map((b) => b.text),
-    "",
-    "--- links ---",
-    ...full.links.map((l) => `${l.text}\t${l.href}`),
-  ];
-  writeFileSync(join(refDir, "visible-text.txt"), txtLines.join("\n"), "utf8");
-  written.push(join(refDir, "visible-text.txt"));
-
-  const mediaOnly = { url, capturedAt, media: full.media };
-  writeFileSync(join(refDir, "media.json"), JSON.stringify(mediaOnly, null, 2) + "\n", "utf8");
-  written.push(join(refDir, "media.json"));
-
-  const meta = {
-    url,
-    capturedAt,
-    title: full.title,
-    description: full.description,
-    canonical: full.canonical,
-    lang: full.lang,
-  };
-  writeFileSync(join(refDir, "meta.json"), JSON.stringify(meta, null, 2) + "\n", "utf8");
-  written.push(join(refDir, "meta.json"));
-
-  writeFileSync(join(refDir, "FOR_AI_REFERENCE.md"), emitAiReadme(url, refDir), "utf8");
-  written.push(join(refDir, "FOR_AI_REFERENCE.md"));
-
-  return written;
-}
-
-function emitAiReadme(url: string, refDir: string): string {
-  const base = refDir.replace(/\\/g, "/");
-  return [
-    `# Reference capture — ${url}`,
-    "",
-    "Use these files when rebuilding the page in React / Next.js:",
-    "",
-    "| File | Purpose |",
-    "| ---- | ------- |",
-    "| `page.html` | Full serialized DOM after JavaScript ran in Chromium. Layout, copy, and structure match what crawled (not necessarily valid static HTML elsewhere). |",
-    "| `dom-structure.json` | **Exact body subtree (JSON):** same child order as the live DOM; every attribute on each element (long `src` / `href` / `style` values truncated); inline text as separate typed entries. Use this as the canonical structure map for React. |",
-    "| `structure-outline.txt` | Indented tag skeleton (ids/classes/roles only; no text) for quick navigation. |",
-    "| `visible-text.json` | Exact visible strings: headings, buttons, links, and block text — good for **verbatim copy** when rewriting `page.tsx`. |",
-    "| `media.json` | Every image / video / source URL from the DOM. Host your own assets or swap for placeholders; do not hotlink without permission. |",
-    "| `meta.json` | Title, description, lang. |",
-    "",
-    "**Workflow:** (1) Recon — use **`dom-structure.json`** (or `structure-outline.txt` + `page.html`) for **exact tag order and nesting**; (2) Wire — map `visible-text.*` + `media.json` into that tree; (3) Build — implement `page.tsx` (start from `../mirror/<host>/page.tsx` if present) so component boundaries follow this structure. See the run folder’s **FOR_AI.md** for full authority order and compliance notes.",
-    "",
-    `Sibling folder \`../mirror/<host>/\` has a typed \`page.tsx\` with Framer Motion, Phosphor icons, and slots — wire copy from \`visible-text.json\` and media from \`media.json\` into that file.`,
-    "",
-    `Captured path: \`${base}\``,
-    "",
-  ].join("\n");
-}
-
-function buildStructureOutline(root: DomStructureElement | null, stats: DomStructurePayload["stats"]): string {
-  const MAX_LINES = 15_000;
-  const lines: string[] = [
-    "# structure-outline.txt — tag skeleton under <body>",
-    "# Text nodes omitted here; see dom-structure.json for interleaved copy.",
-    `# elementNodes=${stats.elementNodes} truncated=${stats.truncated} omitted=${JSON.stringify(stats.omitted)}`,
-    "",
-  ];
-  let n = 0;
-  function walkEl(node: DomStructureElement, depth: number) {
-    if (n >= MAX_LINES) return;
-    const a = node.attrs ?? {};
-    let suffix = "";
-    if (a.id) suffix += `#${String(a.id).replace(/\s+/g, "")}`;
-    if (a.class) {
-      const parts = String(a.class)
-        .split(/\s+/)
-        .filter(Boolean)
-        .slice(0, 6)
-        .map((c) => `.${c}`);
-      suffix += parts.join("");
-    }
-    if (a.role) suffix += `[role=${a.role}]`;
-    lines.push(`${"  ".repeat(depth)}${node.tag}${suffix}`);
-    n++;
-    for (const c of node.children ?? []) {
-      if (n >= MAX_LINES) return;
-      if ("type" in c && c.type === "text") continue;
-      walkEl(c as DomStructureElement, depth + 1);
-    }
-  }
-  if (root) walkEl(root, 0);
-  if (n >= MAX_LINES) lines.push("\n… outline truncated …");
-  return lines.join("\n");
-}
-
-/**
- * Runs in browser context. Serializes `document.body` with exact child order; skips
- * script/style/noscript/template (counts only).
- */
-function collectDomStructureInPage(): DomStructurePayload {
-  const MAX_NODES = 40_000;
-  const MAX_DEPTH = 128;
-  const MAX_TEXT = 12_000;
-
-  let nodeCount = 0;
-  let truncated = false;
-  const omitted = { script: 0, style: 0, noscript: 0, template: 0 };
-
-  function trimAttr(name: string, v: string): string {
-    const big =
-      name === "src" ||
-      name === "href" ||
-      name === "srcset" ||
-      name === "style" ||
-      name === "content" ||
-      name.startsWith("data-");
-    const max = big ? 3500 : 2000;
-    if (v.length <= max) return v;
-    return v.slice(0, max) + "…";
-  }
-
-  function walk(el: Element, depth: number): DomStructureElement | null {
-    if (nodeCount >= MAX_NODES) {
-      truncated = true;
-      return null;
-    }
-    const tag = el.tagName.toLowerCase();
-    if (tag === "script") {
-      omitted.script++;
-      return null;
-    }
-    if (tag === "style") {
-      omitted.style++;
-      return null;
-    }
-    if (tag === "noscript") {
-      omitted.noscript++;
-      return null;
-    }
-    if (tag === "template") {
-      omitted.template++;
-      return null;
-    }
-    if (depth > MAX_DEPTH) return null;
-
-    nodeCount++;
-    const attrs: Record<string, string> = {};
-    for (let i = 0; i < el.attributes.length; i++) {
-      const a = el.attributes.item(i);
-      if (!a) continue;
-      attrs[a.name] = trimAttr(a.name, a.value);
-    }
-
-    const children: DomStructureChild[] = [];
-    for (const child of el.childNodes) {
-      if (nodeCount >= MAX_NODES) {
-        truncated = true;
-        break;
-      }
-      if (child.nodeType === 3) {
-        let t = (child.textContent ?? "").replace(/\s+/g, " ").trim();
-        if (!t) continue;
-        if (t.length > MAX_TEXT) t = t.slice(0, MAX_TEXT) + "…";
-        children.push({ type: "text", value: t });
-      } else if (child.nodeType === 1) {
-        const sub = walk(child as Element, depth + 1);
-        if (sub) children.push(sub);
-      }
-    }
-
-    const out: DomStructureElement = { tag };
-    if (Object.keys(attrs).length > 0) out.attrs = attrs;
-    if (children.length > 0) out.children = children;
-    return out;
-  }
-
-  const body = document.body;
-  if (!body) {
-    return {
-      title: document.title || null,
-      lang: document.documentElement.getAttribute("lang"),
-      stats: {
-        elementNodes: 0,
-        omitted,
-        truncated: false,
-        maxNodes: MAX_NODES,
-        maxDepth: MAX_DEPTH,
-      },
-      root: null,
-    };
-  }
-
-  const root = walk(body, 0);
-  return {
-    title: document.title || null,
-    lang: document.documentElement.getAttribute("lang"),
-    stats: {
-      elementNodes: nodeCount,
-      omitted,
-      truncated,
-      maxNodes: MAX_NODES,
-      maxDepth: MAX_DEPTH,
-    },
-    root,
-  };
-}
-
-/**
- * Runs in browser context.
- */
-function collectSnapshot(): Omit<ReferenceSnapshot, "url" | "capturedAt"> {
-  const title = document.title || null;
-  const descEl = document.querySelector('meta[name="description"]');
-  const description = descEl?.getAttribute("content")?.trim() || null;
-  const canonicalEl = document.querySelector('link[rel="canonical"]');
-  const canonical = canonicalEl?.getAttribute("href") || null;
-  const lang = document.documentElement.getAttribute("lang");
-
-  const visibleTextBlocks: Array<{ tag: string; role: string | null; text: string }> = [];
-  const pushBlock = (tag: string, el: HTMLElement, role: string | null) => {
-    const text = el.innerText?.trim().replace(/\s+/g, " ") ?? "";
-    if (text.length < 2 || text.length > 4000) return;
-    visibleTextBlocks.push({ tag, role, text });
-  };
-
-  for (const tag of ["H1", "H2", "H3", "H4", "H5", "H6"] as const) {
-    for (const el of Array.from(document.querySelectorAll(tag))) {
-      if (!(el instanceof HTMLElement)) continue;
-      const style = getComputedStyle(el);
-      if (style.visibility === "hidden" || style.display === "none") continue;
-      pushBlock(tag, el, el.getAttribute("role"));
-    }
-  }
-
-  for (const el of Array.from(document.querySelectorAll("p, li, blockquote, figcaption, label"))) {
-    if (!(el instanceof HTMLElement)) continue;
-    const style = getComputedStyle(el);
-    if (style.visibility === "hidden" || style.display === "none") continue;
-    pushBlock(el.tagName, el, el.getAttribute("role"));
-  }
-
-  for (const el of Array.from(document.querySelectorAll("button, [role='button']"))) {
-    if (!(el instanceof HTMLElement)) continue;
-    const style = getComputedStyle(el);
-    if (style.visibility === "hidden" || style.display === "none") continue;
-    pushBlock("BUTTON", el, el.getAttribute("role"));
-  }
-
-  for (const el of Array.from(document.querySelectorAll("span, div"))) {
-    if (!(el instanceof HTMLElement)) continue;
-    const role = el.getAttribute("role");
-    if (
-      role !== "heading" &&
-      !el.classList.contains("badge") &&
-      el.getAttribute("data-slot") === null
-    ) {
-      // Only capture labeled small UI chrome (badges, pills) via short text + uppercase heuristic
-      const style = getComputedStyle(el);
-      if (style.visibility === "hidden" || style.display === "none") continue;
-      const text = el.innerText?.trim().replace(/\s+/g, " ") ?? "";
-      if (text.length < 8 || text.length > 240) continue;
-      if (!/^[A-Z0-9\s&.,:]+$/.test(text)) continue; // ALL-CAPS-ish eyebrow labels
-      pushBlock(el.tagName, el, role);
-    }
-  }
-
-  const links: Array<{ href: string; text: string }> = [];
-  for (const a of Array.from(document.querySelectorAll("a[href]"))) {
-    const href = a.getAttribute("href") ?? "";
-    if (!href || href.startsWith("javascript:")) continue;
-    const text = (a.textContent ?? "").trim().replace(/\s+/g, " ");
-    if (!text) continue;
-    try {
-      const abs = new URL(href, document.baseURI).href;
-      links.push({ href: abs, text: text.slice(0, 500) });
-    } catch {
-      links.push({ href, text: text.slice(0, 500) });
-    }
-  }
-
-  const media: ReferenceSnapshot["media"] = [];
-  for (const img of Array.from(document.querySelectorAll("img"))) {
-    const src = img.currentSrc || img.src;
-    if (!src) continue;
-    media.push({
-      type: "img",
-      src,
-      alt: img.alt || "",
-      width: img.naturalWidth || null,
-      height: img.naturalHeight || null,
-    });
-  }
-  for (const video of Array.from(document.querySelectorAll("video"))) {
-    const poster = video.getAttribute("poster");
-    let src: string | null = null;
-    if (video.currentSrc) src = video.currentSrc;
-    else {
-      const s = video.querySelector("source[src]");
-      src = s?.getAttribute("src") ?? null;
-    }
-    media.push({ type: "video", src, poster: poster || null });
-  }
-  for (const source of Array.from(document.querySelectorAll("source[src]"))) {
-    const src = source.getAttribute("src");
-    if (!src) continue;
-    media.push({
-      type: "source",
-      src,
-      kind: source.getAttribute("type"),
-    });
-  }
-
-  return {
-    title,
-    description,
-    canonical,
-    lang: lang || null,
-    visibleTextBlocks,
-    links,
-    media,
-  };
-}