launchframe 0.2.0 → 0.2.2

package/packages/extract/dom-crawler.ts

@@ -1,521 +0,0 @@
-/**
- * DOM layout crawler.
- *
- * Runs inside the rendered page via Playwright's `page.evaluate`. Walks the
- * DOM, identifies top-level sections, classifies each section's role and
- * composition, and counts the content slots it contains. Returns a
- * `SiteLayout` structural model the emitter rebuilds into a Next.js page.
- *
- * What this records:
- *   - Section tree (geometry, role, composition, density)
- *   - Slot inventory per section: how many headings / body paragraphs /
- *     buttons / images / icons / logos / code blocks etc. it contains
- *   - Per-section style tokens: background, foreground, padding
- *   - Page-level tokens: fonts, primary surface colors, container width
- *
- * What this does NOT record:
- *   - Heading or body text content (slots are counts, not strings).
- *   - Raw HTML, CSS, or class names from the source.
- *   - Brand assets (logos, illustrations, product screenshots).
- *
- * The structural model is what the mirror emitter uses to reconstruct the
- * page's section grammar with `<TextSlot>` / `<MediaSlot>` placeholders.
- */
-
-import type { Page } from "playwright";
-
-import type {
-  Composition,
-  SectionLayout,
-  SectionRole,
-  SiteLayout,
-  SiteTokens,
-  SlotCount,
-  SlotKind,
-} from "./types.js";
-
-export async function crawlLayout(
-  page: Page,
-  url: string,
-  viewport: { width: number; height: number },
-): Promise<SiteLayout> {
-  await page.evaluate(() => {
-    const g = globalThis as unknown as { __name?: (fn: unknown) => unknown };
-    if (typeof g.__name === "undefined") g.__name = (fn: unknown) => fn;
-  });
-
-  const host = new URL(url).host;
-  const partial = await page.evaluate(crawlInPage);
-
-  return {
-    url,
-    host,
-    capturedAt: new Date().toISOString(),
-    viewport,
-    ...partial,
-  };
-}
-
-/**
- * Browser-context crawler. Dependency-free so Playwright can serialize it.
- * Returns the layout-bearing fields of `SiteLayout` (url/host/capturedAt
- * are added on the Node side).
- */
-function crawlInPage(): Pick<
-  SiteLayout,
-  "pageHeightPx" | "sections" | "tokens"
-> {
-  const VIEWPORT_W = window.innerWidth;
-  const PAGE_H = Math.max(
-    document.documentElement.scrollHeight,
-    document.body.scrollHeight,
-  );
-
-  /* ----- helpers ----- */
-
-  function toHex(rgb: string): string | null {
-    if (!rgb || rgb === "transparent") return null;
-    const m = rgb.match(/rgba?\(([^)]+)\)/);
-    if (!m) return null;
-    const parts = m[1]!.split(",").map((s) => s.trim());
-    const r = parseInt(parts[0]!, 10);
-    const g = parseInt(parts[1]!, 10);
-    const b = parseInt(parts[2]!, 10);
-    const a = parts[3] !== undefined ? parseFloat(parts[3]) : 1;
-    if (a < 0.05) return null;
-    if ([r, g, b].some((n) => Number.isNaN(n))) return null;
-    const h = (n: number) => n.toString(16).padStart(2, "0");
-    return `#${h(r)}${h(g)}${h(b)}`;
-  }
-
-  function isVisible(el: HTMLElement): boolean {
-    const style = getComputedStyle(el);
-    if (style.visibility === "hidden" || style.display === "none") return false;
-    if (parseFloat(style.opacity) < 0.05) return false;
-    const r = el.getBoundingClientRect();
-    return r.width > 0 && r.height > 0;
-  }
-
-  function directTextLength(el: Element): number {
-    let total = 0;
-    for (const child of Array.from(el.childNodes)) {
-      if (child.nodeType === 3) {
-        const text = (child.nodeValue ?? "").trim();
-        if (text) total += text.length;
-      }
-    }
-    return total;
-  }
-
-  function classifySlotForElement(el: HTMLElement): SlotKind | null {
-    const tag = el.tagName;
-    const style = getComputedStyle(el);
-    const fontSize = parseFloat(style.fontSize) || 16;
-    const fontWeight = parseInt(style.fontWeight, 10) || 400;
-
-    if (tag === "IMG" || tag === "PICTURE") {
-      // Heuristic: small square-ish images in a logo strip vs hero photos.
-      const r = el.getBoundingClientRect();
-      const ratio = r.width > 0 && r.height > 0 ? r.width / r.height : 0;
-      if (r.height > 0 && r.height < 56 && ratio < 6) return "logo-mono";
-      return "image";
-    }
-    if (tag === "SVG" || tag === "svg") {
-      const r = el.getBoundingClientRect();
-      if (r.width < 32 && r.height < 32) return "icon";
-      return "image";
-    }
-    if (tag === "VIDEO") return "video";
-    if (tag === "PRE" || tag === "CODE") return "code";
-    if (tag === "INPUT" || tag === "TEXTAREA" || tag === "SELECT") return "input";
-
-    if (tag === "BUTTON" || (tag === "A" && el.getAttribute("role") === "button")) {
-      // Decide primary vs secondary by background contrast vs the page.
-      const bg = toHex(style.backgroundColor);
-      if (bg && bg !== toHex(getComputedStyle(document.body).backgroundColor)) {
-        return "button-primary";
-      }
-      return "button-secondary";
-    }
-
-    if (tag === "A" && directTextLength(el) > 0) {
-      // Anchor that looks like a styled CTA (padded, bordered, or backgrounded).
-      const padX = parseFloat(style.paddingLeft) + parseFloat(style.paddingRight);
-      const hasBackdrop =
-        toHex(style.backgroundColor) !== null ||
-        parseFloat(style.borderTopWidth) > 0;
-      if (padX > 16 && hasBackdrop) {
-        return toHex(style.backgroundColor) ? "button-primary" : "button-secondary";
-      }
-    }
-
-    if (directTextLength(el) === 0) return null;
-
-    if (/^H[1-6]$/.test(tag)) {
-      const level = parseInt(tag.slice(1), 10);
-      if (level === 1) return "heading-1";
-      if (level === 2) return "heading-2";
-      return "heading-3";
-    }
-
-    if (tag === "LI") return "bullet";
-
-    if (fontSize >= 36 && fontWeight >= 600) return "heading-1";
-    if (fontSize >= 24 && fontWeight >= 500) return "heading-2";
-    if (fontSize >= 18 && fontWeight >= 500) return "heading-3";
-
-    if (fontSize <= 12 && /uppercase/i.test(style.textTransform)) return "eyebrow";
-    if (fontSize <= 13 && fontWeight >= 600) return "badge";
-
-    if (tag === "P" || tag === "SPAN" || tag === "DIV") return "body";
-
-    return null;
-  }
-
-  function countSlots(root: HTMLElement): SlotCount[] {
-    const buckets = new Map<SlotKind, number>();
-    const all = root.querySelectorAll<HTMLElement>("*");
-    for (const el of Array.from(all)) {
-      if (!isVisible(el)) continue;
-      const kind = classifySlotForElement(el);
-      if (kind) buckets.set(kind, (buckets.get(kind) ?? 0) + 1);
-    }
-    // De-dupe nested H1 spans etc: if an H1 contains spans, the outer H1 counts.
-    // We accept some over-count for body since it's coarse anyway; cap it.
-    if ((buckets.get("body") ?? 0) > 12) buckets.set("body", 12);
-    if ((buckets.get("bullet") ?? 0) > 24) buckets.set("bullet", 24);
-
-    const out: SlotCount[] = [];
-    for (const [kind, count] of buckets) out.push({ kind, count });
-    return out.sort((a, b) => a.kind.localeCompare(b.kind));
-  }
-
-  /* ----- section discovery ----- */
-
-  /**
-   * A top-level section is a block-level element that:
-   *  - is at least ~70% of the viewport wide,
-   *  - has a non-trivial height (>= 80 px),
-   *  - and is one of <header>, <footer>, <main>, <section>, <article>,
-   *    or a direct child of <body> / <main> that visually plays that role.
-   */
-  function findSections(): HTMLElement[] {
-    const candidates = new Set<HTMLElement>();
-    const tagSet = ["HEADER", "MAIN", "SECTION", "ARTICLE", "FOOTER", "NAV"];
-    for (const tag of tagSet) {
-      for (const el of Array.from(document.getElementsByTagName(tag))) {
-        candidates.add(el as HTMLElement);
-      }
-    }
-    // Add direct children of <body> and <main> as fallback.
-    const bodyKids = Array.from(document.body.children) as HTMLElement[];
-    for (const el of bodyKids) candidates.add(el);
-    const main = document.querySelector("main");
-    if (main) {
-      for (const el of Array.from(main.children) as HTMLElement[]) {
-        candidates.add(el);
-      }
-    }
-
-    const accepted: HTMLElement[] = [];
-    for (const el of candidates) {
-      if (!isVisible(el)) continue;
-      const r = el.getBoundingClientRect();
-      if (r.width < VIEWPORT_W * 0.7) continue;
-      if (r.height < 80) continue;
-      // Skip if this element is nested inside another already-accepted candidate.
-      // We'll do a final pass after sorting.
-      accepted.push(el);
-    }
-    // Sort by document y position.
-    accepted.sort((a, b) => {
-      const ay = a.getBoundingClientRect().top + window.scrollY;
-      const by = b.getBoundingClientRect().top + window.scrollY;
-      return ay - by;
-    });
-
-    // Drop any element fully contained in an earlier accepted one.
-    const final: HTMLElement[] = [];
-    for (const el of accepted) {
-      const inside = final.some((p) => p !== el && p.contains(el));
-      if (!inside) final.push(el);
-    }
-    return final;
-  }
-
-  /* ----- per-section classification ----- */
-
-  function classifyRole(
-    el: HTMLElement,
-    indexFromTop: number,
-    indexFromBottom: number,
-    slots: SlotCount[],
-  ): SectionRole {
-    const r = el.getBoundingClientRect();
-    const tag = el.tagName;
-    const count = (k: SlotKind) => slots.find((s) => s.kind === k)?.count ?? 0;
-
-    if (tag === "NAV") return "nav";
-    if (tag === "FOOTER" || (indexFromBottom === 0 && r.height < 600)) return "footer";
-    if (tag === "HEADER" && indexFromTop === 0 && r.height < 140) return "nav";
-
-    const headingCount = count("heading-1") + count("heading-2") + count("heading-3");
-    const buttons = count("button-primary") + count("button-secondary");
-    const images = count("image");
-    const logos = count("logo-mono");
-    const bullets = count("bullet");
-
-    // First in-document, has H1 + CTA → hero.
-    if (indexFromTop <= 1 && count("heading-1") >= 1 && buttons >= 1) return "hero";
-
-    // A wide, short band of small uniform images → logo strip.
-    if (logos >= 4 && headingCount <= 1 && r.height < r.width * 0.25) {
-      return "proof-logos";
-    }
-
-    // Pricing tells: 2–4 tall columns each with bullets and a button.
-    if (bullets >= 6 && buttons >= 2 && r.height > 360) return "pricing";
-
-    // Lots of headings (3+) of the same level + small bodies → feature grid.
-    if (count("heading-2") + count("heading-3") >= 3 && images <= 2) return "feature-grid";
-
-    // One heading, generous body, one media slot → deep dive.
-    if (headingCount >= 1 && images >= 1 && bullets <= 4 && r.height > 320) {
-      return "feature-deep-dive";
-    }
-
-    // Heading + 2 buttons, short height → conversion band near the bottom.
-    if (headingCount <= 2 && buttons >= 1 && r.height < 480 && indexFromBottom <= 2) {
-      return "conversion";
-    }
-
-    // Quote-shaped: short body strings, sometimes 3-up.
-    if (count("body") >= 3 && images === 0 && bullets === 0 && buttons === 0) {
-      return "proof-quotes";
-    }
-
-    return "other";
-  }
-
-  function classifyComposition(el: HTMLElement): Composition {
-    // Find the deepest descendant that uses CSS grid or flex with >1 row of cols.
-    const candidates = el.querySelectorAll<HTMLElement>("*");
-    let bestCols = 1;
-    let bestKind: "grid" | "flex" | "none" = "none";
-    let logoRowCols = 0;
-    for (const c of Array.from(candidates).slice(0, 400)) {
-      if (!isVisible(c)) continue;
-      const s = getComputedStyle(c);
-      if (s.display === "grid") {
-        const cols = s.gridTemplateColumns
-          .split(" ")
-          .filter((x) => x.trim().length > 0).length;
-        if (cols > bestCols) {
-          bestCols = cols;
-          bestKind = "grid";
-        }
-      } else if (s.display === "flex" && s.flexDirection.startsWith("row")) {
-        const kids = Array.from(c.children) as HTMLElement[];
-        const visibleKids = kids.filter(isVisible);
-        if (visibleKids.length > bestCols && visibleKids.length <= 12) {
-          bestCols = visibleKids.length;
-          bestKind = "flex";
-        }
-        if (visibleKids.length >= 4) {
-          const allSmall = visibleKids.every((k) => {
-            const kr = k.getBoundingClientRect();
-            return kr.height < 80 && kr.width < 200;
-          });
-          if (allSmall) logoRowCols = Math.max(logoRowCols, visibleKids.length);
-        }
-      }
-    }
-
-    if (logoRowCols >= 4) return "logo-row";
-    if (bestKind === "none" || bestCols <= 1) return "single-column";
-    if (bestCols === 2) return "split-2";
-    if (bestCols === 3) return "grid-3";
-    if (bestCols === 4) return "grid-4";
-    if (bestCols >= 5) return "list";
-    return "unknown";
-  }
-
-  function classifyDensity(el: HTMLElement, slots: SlotCount[]): "thin" | "balanced" | "dense" {
-    const total = slots.reduce((sum, s) => sum + s.count, 0);
-    const r = el.getBoundingClientRect();
-    const density = total / Math.max(1, r.height / 100);
-    if (density < 0.8) return "thin";
-    if (density > 2.4) return "dense";
-    return "balanced";
-  }
-
-  function extractSectionStyles(el: HTMLElement): SectionLayout["styles"] {
-    const s = getComputedStyle(el);
-    return {
-      backgroundHex: toHex(s.backgroundColor),
-      foregroundHex: toHex(s.color),
-      paddingTopPx: Math.round(parseFloat(s.paddingTop) || 0) || null,
-      paddingBottomPx: Math.round(parseFloat(s.paddingBottom) || 0) || null,
-    };
-  }
-
-  /* ----- page-level token extraction ----- */
-
-  function extractPageTokens(): SiteTokens {
-    const body = document.body;
-    const bodyStyle = getComputedStyle(body);
-    const bodyFontFamily = bodyStyle.fontFamily.split(",")[0]!.trim().replace(/^["']|["']$/g, "") || "system-ui";
-
-    let headingFontFamily = bodyFontFamily;
-    const h = document.querySelector("h1, h2, h3");
-    if (h) {
-      const hs = getComputedStyle(h);
-      headingFontFamily = hs.fontFamily.split(",")[0]!.trim().replace(/^["']|["']$/g, "") || bodyFontFamily;
-    }
-
-    const bg = toHex(bodyStyle.backgroundColor) ?? "#ffffff";
-    const fg = toHex(bodyStyle.color) ?? "#0a0a0a";
-
-    // Primary = the most-used non-text colored button background.
-    const buttonBgCounts = new Map<string, number>();
-    for (const b of Array.from(document.querySelectorAll<HTMLElement>("button, a, [role='button']"))) {
-      if (!isVisible(b)) continue;
-      const sb = toHex(getComputedStyle(b).backgroundColor);
-      if (!sb || sb === bg) continue;
-      buttonBgCounts.set(sb, (buttonBgCounts.get(sb) ?? 0) + 1);
-    }
-    let primary = fg;
-    let primaryCount = 0;
-    for (const [hex, count] of buttonBgCounts) {
-      if (count > primaryCount) {
-        primary = hex;
-        primaryCount = count;
-      }
-    }
-
-    // Muted = a frequent off-white / off-black surface color (non-page).
-    const surfaceCounts = new Map<string, number>();
-    for (const el of Array.from(document.querySelectorAll<HTMLElement>("body *"))) {
-      if (!isVisible(el)) continue;
-      const sb = toHex(getComputedStyle(el).backgroundColor);
-      if (!sb || sb === bg) continue;
-      surfaceCounts.set(sb, (surfaceCounts.get(sb) ?? 0) + 1);
-    }
-    let muted = bg;
-    let mutedCount = 0;
-    for (const [hex, count] of surfaceCounts) {
-      if (hex === primary) continue;
-      if (count > mutedCount) {
-        muted = hex;
-        mutedCount = count;
-      }
-    }
-
-    // Border = most-used border color across all elements with a border.
-    const borderCounts = new Map<string, number>();
-    for (const el of Array.from(document.querySelectorAll<HTMLElement>("body *"))) {
-      if (!isVisible(el)) continue;
-      const s = getComputedStyle(el);
-      if (parseFloat(s.borderTopWidth) <= 0) continue;
-      const bh = toHex(s.borderTopColor);
-      if (!bh) continue;
-      borderCounts.set(bh, (borderCounts.get(bh) ?? 0) + 1);
-    }
-    let border = "#e5e7eb";
-    let borderCount = 0;
-    for (const [hex, count] of borderCounts) {
-      if (count > borderCount) {
-        border = hex;
-        borderCount = count;
-      }
-    }
-
-    // Radius = the most-used non-zero corner radius.
-    const radiusCounts = new Map<number, number>();
-    for (const el of Array.from(document.querySelectorAll<HTMLElement>("body *"))) {
-      if (!isVisible(el)) continue;
-      const r = parseFloat(getComputedStyle(el).borderTopLeftRadius);
-      if (!(r > 0 && r < 64)) continue;
-      const k = Math.round(r);
-      radiusCounts.set(k, (radiusCounts.get(k) ?? 0) + 1);
-    }
-    let radius = 8;
-    let radiusCount = 0;
-    for (const [px, count] of radiusCounts) {
-      if (count > radiusCount) {
-        radius = px;
-        radiusCount = count;
-      }
-    }
-
-    // Container: widest layout block under 1600px.
-    let containerPx: number | null = null;
-    let containerArea = 0;
-    for (const el of Array.from(
-      document.querySelectorAll<HTMLElement>("main, section, header, footer, div"),
-    )) {
-      if (!isVisible(el)) continue;
-      const r = el.getBoundingClientRect();
-      if (r.width < 720 || r.width > 1600) continue;
-      if (r.height < 240) continue;
-      const area = r.width * r.height;
-      if (area > containerArea) {
-        containerArea = area;
-        containerPx = Math.round(r.width);
-      }
-    }
-
-    return {
-      bodyFontFamily,
-      headingFontFamily,
-      backgroundHex: bg,
-      foregroundHex: fg,
-      primaryHex: primary,
-      mutedHex: muted,
-      borderHex: border,
-      radiusPx: radius,
-      containerPx,
-    };
-  }
-
-  /* ----- main pass ----- */
-
-  const sectionEls = findSections();
-  const sections: SectionLayout[] = [];
-  for (let i = 0; i < sectionEls.length; i++) {
-    const el = sectionEls[i]!;
-    const r = el.getBoundingClientRect();
-    const top = r.top + window.scrollY;
-    const slots = countSlots(el);
-    const composition = classifyComposition(el);
-    const density = classifyDensity(el, slots);
-    const role = classifyRole(el, i, sectionEls.length - 1 - i, slots);
-    const styles = extractSectionStyles(el);
-
-    const notes: string[] = [];
-    if (slots.length === 0) notes.push("No content slots detected; rendering an empty wrapper.");
-    if (composition === "unknown") notes.push("Composition was ambiguous; fell back to single-column.");
-
-    sections.push({
-      id: `s${i + 1}`,
-      role,
-      composition,
-      density,
-      bbox: [
-        Math.max(0, Math.min(1, r.left / VIEWPORT_W)),
-        Math.max(0, Math.min(1, top / PAGE_H)),
-        Math.max(0, Math.min(1, r.width / VIEWPORT_W)),
-        Math.max(0, Math.min(1, r.height / PAGE_H)),
-      ],
-      slots,
-      styles,
-      notes,
-    });
-  }
-
-  return {
-    pageHeightPx: PAGE_H,
-    sections,
-    tokens: extractPageTokens(),
-  };
-}