launchframe 0.2.0 → 0.2.1

package/packages/extract/extract.ts

@@ -1,548 +0,0 @@
-/**
- * `extract` — the headline command.
- *
- *   npx launchframe@latest https://site-a.com "Your SaaS idea"
- *   npm run extract -- https://site-a.com "idea" https://site-b.com
- *
- * For each URL: open in Chromium, screenshot, harvest computed design
- * tokens via `browser-extract.ts`, and crawl the rendered DOM into a
- * typed `SiteLayout` model via `dom-crawler.ts`. After all sites:
- *   - Synthesize a drop-in shadcn-compatible design system from the
- *     aggregated tokens.
- *   - Emit a per-site **layout mirror**: a Next.js page that reconstructs
- *     the source's section structure from typed primitives, with
- *     `<TextSlot>` / `<MediaSlot>` placeholders for the user's copy and
- *     brand assets.
- *
- * Output goes to `output/<runId>/`.
- *
- * Operational defaults (configurable via flags):
- *   - Honor robots.txt unless `--no-robots` is passed.
- *   - Per-domain rate limit defaults to 15 req/min (`--rate <n>`).
- *   - The crawler extracts a structured representation (section tree,
- *     computed style tokens, content kinds) and writes a verbatim
- *     `reference/<host>/` bundle (HTML, DOM tree JSON, outlines, visible text,
- *     media index) for AI structure cloning.
- *   - Automated **design-references** pass: responsive + scroll sweep PNGs,
- *     header probe, interaction hints, optional media downloads (like the
- *     ai-website-cloner-template recon outputs).
- */
-
-import { mkdirSync, writeFileSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { fileURLToPath, pathToFileURL } from "node:url";
-
-import { chromium, type Browser } from "playwright";
-
-import { harvestTokens } from "./browser-extract.js";
-import { runAutomatedClonePass } from "./automated-clone-pass.js";
-import { crawlLayout } from "./dom-crawler.js";
-import { emitClonerResearch } from "./cloner-research-emit.js";
-import { emitAll } from "./emit.js";
-import { emitMirror } from "./mirror-emit.js";
-import { emitPageReference } from "./reference-dump.js";
-import { synthesize } from "./synthesize.js";
-import type { ExtractionRun, RawTokens, SiteCapture, SiteLayout } from "./types.js";
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
-/** Writes under the user's cwd so `npx launchframe` / `npx landingfram` from any folder works. */
-const DEFAULT_OUTPUT_ROOT = join(process.cwd(), "output");
-
-const USER_AGENT =
-  "landingfram/0.1 (+https://github.com/evangruhlkey/launchframe; design-token research; respects robots.txt)";
-
-interface CliArgs {
-  urls: string[];
-  outDir: string;
-  viewport: { width: number; height: number };
-  respectRobots: boolean;
-  rateLimitPerMinute: number;
-  runName?: string;
-  /** SaaS product narrative from `--idea`, a positional string, or LANDINGFRAM_SAAS_IDEA / LAUNCHFRAME_SAAS_IDEA. */
-  saasIdea?: string;
-  /** When true, skip HTTP downloads from reference/media.json */
-  skipAssetDownload: boolean;
-}
-
-function parseArgs(argv: string[]): CliArgs {
-  const urls: string[] = [];
-  const positional: string[] = [];
-  const args: CliArgs = {
-    urls,
-    outDir: "",
-    viewport: { width: 1440, height: 900 },
-    respectRobots: true,
-    rateLimitPerMinute: 15,
-    skipAssetDownload: false,
-  };
-
-  for (let i = 0; i < argv.length; i++) {
-    const a = argv[i]!;
-    if (a === "--idea") {
-      const v = argv[++i];
-      if (!v) {
-        console.error("--idea requires a value");
-        process.exit(2);
-      }
-      args.saasIdea = v;
-      continue;
-    }
-    if (a === "--no-download") {
-      args.skipAssetDownload = true;
-      continue;
-    }
-    if (a === "--out") {
-      args.outDir = argv[++i]!;
-      continue;
-    }
-    if (a === "--name") {
-      args.runName = argv[++i];
-      continue;
-    }
-    if (a === "--no-robots") {
-      args.respectRobots = false;
-      continue;
-    }
-    if (a === "--rate") {
-      args.rateLimitPerMinute = parseInt(argv[++i]!, 10);
-      continue;
-    }
-    if (a === "--width") {
-      args.viewport.width = parseInt(argv[++i]!, 10);
-      continue;
-    }
-    if (a === "--height") {
-      args.viewport.height = parseInt(argv[++i]!, 10);
-      continue;
-    }
-    if (a === "--help" || a === "-h") {
-      printHelp();
-      process.exit(0);
-    }
-    if (a.startsWith("http://") || a.startsWith("https://")) {
-      urls.push(a);
-      continue;
-    }
-    if (a.startsWith("--")) {
-      console.error(`Unknown flag: ${a}`);
-      process.exit(2);
-    }
-    positional.push(a);
-  }
-
-  if (positional.length > 1) {
-    console.error(
-      'Too many positional arguments. Quote your SaaS idea as one argument or use --idea "..."',
-    );
-    process.exit(2);
-  }
-  if (positional.length === 1) {
-    if (args.saasIdea) {
-      console.error("Use either --idea or one positional SaaS idea string, not both.");
-      process.exit(2);
-    }
-    args.saasIdea = positional[0];
-  }
-
-  const envIdea =
-    process.env.LANDINGFRAM_SAAS_IDEA?.trim() || process.env.LAUNCHFRAME_SAAS_IDEA?.trim();
-  if (envIdea && !args.saasIdea) {
-    args.saasIdea = envIdea;
-  }
-
-  if (urls.length === 0) {
-    printHelp();
-    process.exit(2);
-  }
-  return args;
-}
-
-function printHelp(): void {
-  console.log(
-    [
-      "Usage:",
-      "  npx launchframe@latest <url> [\"SaaS idea\"] [<url> ...] [options]",
-      "  npx launchframe@latest <url> --idea \"SaaS idea\" [options]",
-      "  npm run extract -- <url> [\"idea\"] [...] [options]   (from this repo)",
-      "",
-      "Writes to ./output/<runId>/ in your current working directory unless",
-      "you pass --out.",
-      "",
-      "For each URL the CLI:",
-      "  1. Renders the page at a desktop viewport in headless Chromium.",
-      "  2. Captures a full-page screenshot and harvests computed design tokens",
-      "     (colors, type, spacing, radius, shadow) → raw/<host>.tokens.json.",
-      "  3. Writes a verbatim reference bundle → reference/<host>/ (page.html,",
-      "     dom-structure.json, structure-outline.txt, visible-text.json/.txt,",
-      "     media.json, meta.json, FOR_AI_REFERENCE.md).",
-      "  4. Crawls the DOM into SiteLayout → raw/<host>.layout.json and emits",
-      "     mirror/<host>/page.tsx (Framer Motion + Phosphor + image/video slots).",
-      "",
-      "After every URL, a drop-in shadcn-compatible design system is",
-      "synthesized from the aggregated tokens and written to output/<runId>/.",
-      "",
-      "  Your SaaS idea is stored in run.json, FOR_AI.md, and docs/research/ for agents.",
-      "",
-      "Options:",
-      "  --idea <text>        SaaS product narrative (alternative to one positional string)",
-      "  --no-download        Skip downloading binaries listed in reference/*/media.json",
-      "  --out <dir>          Run directory (default: ./output/<runId>). Use e.g.",
-      "                       ./my-next-app/extraction-acme to mirror template-style",
-      "                       docs/ placement inside a consumer repo.",
-      "  --name <slug>        Human-friendly slug used in the runId",
-      "  --no-robots          Skip robots.txt check (not recommended)",
-      "  --rate <per-min>     Per-domain rate limit, default 15",
-      "  --width <px>         Viewport width, default 1440",
-      "  --height <px>        Viewport height, default 900",
-      "  --help               Show this help",
-    ].join("\n"),
-  );
-}
-
-/* -------------------------------------------------------------------------- */
-/*  robots.txt                                                                */
-/* -------------------------------------------------------------------------- */
-
-async function isAllowedByRobots(url: string): Promise<boolean> {
-  try {
-    const u = new URL(url);
-    const res = await fetch(`${u.origin}/robots.txt`, {
-      headers: { "User-Agent": USER_AGENT },
-      signal: AbortSignal.timeout(8_000),
-    });
-    if (!res.ok) return true;
-    const body = await res.text();
-    return checkRobots(body, u.pathname);
-  } catch {
-    return true;
-  }
-}
-
-function checkRobots(body: string, pathname: string): boolean {
-  const lines = body.split(/\r?\n/);
-  let inStarBlock = false;
-  const disallow: string[] = [];
-  const allow: string[] = [];
-  for (const raw of lines) {
-    const line = raw.split("#")[0]!.trim();
-    if (!line) continue;
-    const idx = line.indexOf(":");
-    if (idx < 0) continue;
-    const key = line.slice(0, idx).toLowerCase().trim();
-    const value = line.slice(idx + 1).trim();
-    if (key === "user-agent") inStarBlock = value === "*";
-    else if (inStarBlock && key === "disallow" && value) disallow.push(value);
-    else if (inStarBlock && key === "allow" && value) allow.push(value);
-  }
-  const len = (patterns: string[]) =>
-    patterns.reduce((m, p) => (pathname.startsWith(p) ? Math.max(m, p.length) : m), -1);
-  const a = len(allow);
-  const d = len(disallow);
-  if (d < 0) return true;
-  return a >= d;
-}
-
-/* -------------------------------------------------------------------------- */
-/*  Rate limiter                                                              */
-/* -------------------------------------------------------------------------- */
-
-class RateLimiter {
-  private readonly intervalMs: number;
-  private readonly lastByHost = new Map<string, number>();
-  constructor(perMinute: number) {
-    this.intervalMs = Math.ceil(60_000 / Math.max(1, perMinute));
-  }
-  async wait(host: string): Promise<void> {
-    const last = this.lastByHost.get(host) ?? 0;
-    const elapsed = Date.now() - last;
-    if (elapsed < this.intervalMs) {
-      await new Promise((r) => setTimeout(r, this.intervalMs - elapsed));
-    }
-    this.lastByHost.set(host, Date.now());
-  }
-}
-
-/* -------------------------------------------------------------------------- */
-/*  Pipeline                                                                  */
-/* -------------------------------------------------------------------------- */
-
-async function captureOne(
-  browser: Browser,
-  url: string,
-  viewport: { width: number; height: number },
-  outDir: string,
-  opts: { skipAssetDownload: boolean },
-): Promise<{ raw: RawTokens; layout: SiteLayout | null; capture: SiteCapture } | null> {
-  const host = new URL(url).host;
-  const stamp = `${host}.png`;
-  const screenshotPath = join(outDir, "screenshots", stamp);
-  const rawPath = join(outDir, "raw", `${host}.tokens.json`);
-  const layoutPath = join(outDir, "raw", `${host}.layout.json`);
-  const mirrorDir = join(outDir, "mirror", host);
-  const referenceDir = join(outDir, "reference", host);
-
-  const ctx = await browser.newContext({
-    userAgent: USER_AGENT,
-    viewport,
-    deviceScaleFactor: 2,
-    reducedMotion: "reduce",
-  });
-  const page = await ctx.newPage();
-  try {
-    const response = await page.goto(url, { waitUntil: "load", timeout: 60_000 });
-    if (!response || response.status() >= 400) {
-      throw new Error(`HTTP ${response?.status() ?? "unknown"}`);
-    }
-
-    await page.evaluate(() => {
-      const style = document.createElement("style");
-      style.textContent = `*, *::before, *::after {
-        animation: none !important;
-        transition: none !important;
-        scroll-behavior: auto !important;
-      }`;
-      document.head.appendChild(style);
-    });
-    await page.waitForTimeout(400);
-
-    mkdirSync(dirname(screenshotPath), { recursive: true });
-    await page.screenshot({ path: screenshotPath, fullPage: true, type: "png" });
-
-    const raw = await harvestTokens(page, url, viewport);
-    mkdirSync(dirname(rawPath), { recursive: true });
-    writeFileSync(rawPath, JSON.stringify(raw, null, 2));
-
-    let referenceWritten: string[] = [];
-    try {
-      referenceWritten = await emitPageReference(page, url, referenceDir, viewport);
-    } catch (err) {
-      console.warn(`  ! reference dump failed for ${url}: ${(err as Error).message}`);
-    }
-
-    let layout: SiteLayout | null = null;
-    let mirrorWritten: string[] = [];
-    try {
-      layout = await crawlLayout(page, url, viewport);
-      mkdirSync(dirname(layoutPath), { recursive: true });
-      writeFileSync(layoutPath, JSON.stringify(layout, null, 2));
-      mirrorWritten = emitMirror(layout, mirrorDir);
-    } catch (err) {
-      console.warn(`  ! layout crawl failed for ${url}: ${(err as Error).message}`);
-    }
-
-    try {
-      const n = (
-        await runAutomatedClonePass({
-          page,
-          host,
-          outDir,
-          primaryViewport: viewport,
-          referenceDir: referenceWritten.length > 0 ? referenceDir : undefined,
-          userAgent: USER_AGENT,
-          downloadAssets: !opts.skipAssetDownload,
-        })
-      ).length;
-      if (n > 0) console.log(`  · automated recon (${n} files under docs/design-references & downloads)`);
-    } catch (err) {
-      console.warn(`  ! automated recon failed for ${url}: ${(err as Error).message}`);
-    }
-
-    const capture: SiteCapture = {
-      url,
-      host,
-      capturedAt: raw.capturedAt,
-      screenshotPath,
-      rawTokensPath: rawPath,
-      ...(referenceWritten.length > 0 ? { referenceDir } : {}),
-      ...(layout ? { layoutPath } : {}),
-      ...(mirrorWritten.length > 0 ? { mirrorDir } : {}),
-      status: "ok",
-    };
-    return { raw, layout, capture };
-  } catch (err) {
-    return {
-      raw: emptyRaw(url, viewport),
-      layout: null,
-      capture: {
-        url,
-        host,
-        capturedAt: new Date().toISOString(),
-        screenshotPath: "",
-        status: "failed",
-        reason: (err as Error).message,
-      },
-    };
-  } finally {
-    await ctx.close();
-  }
-}
-
-function emptyRaw(url: string, viewport: { width: number; height: number }): RawTokens {
-  return {
-    url,
-    capturedAt: new Date().toISOString(),
-    viewport,
-    colors: [],
-    typography: [],
-    spacing: [],
-    radii: [],
-    shadows: [],
-    dominantContainerPx: null,
-  };
-}
-
-async function main(): Promise<void> {
-  const args = parseArgs(process.argv.slice(2));
-  const startedAt = new Date().toISOString();
-  const runId = makeRunId(startedAt, args.runName);
-  const outDir = args.outDir || join(DEFAULT_OUTPUT_ROOT, runId);
-
-  console.log(`[extract] runId=${runId}`);
-  console.log(`[extract] urls=${args.urls.length} viewport=${args.viewport.width}x${args.viewport.height}`);
-  console.log(`[extract] output=${outDir}`);
-  if (args.saasIdea) {
-    console.log(`[extract] saasIdea=${args.saasIdea.slice(0, 80)}${args.saasIdea.length > 80 ? "…" : ""}`);
-  }
-  console.log("");
-
-  mkdirSync(outDir, { recursive: true });
-
-  const limiter = new RateLimiter(args.rateLimitPerMinute);
-  const captures: SiteCapture[] = [];
-  const rawList: RawTokens[] = [];
-  const layoutsByHost = new Map<string, SiteLayout>();
-
-  let browser: Browser | null = null;
-  try {
-    browser = await chromium.launch();
-
-    for (const url of args.urls) {
-      const host = new URL(url).host;
-
-      if (args.respectRobots) {
-        const allowed = await isAllowedByRobots(url);
-        if (!allowed) {
-          console.log(`  ⊘ ${url}  skipped — robots.txt disallows`);
-          captures.push({
-            url,
-            host,
-            capturedAt: new Date().toISOString(),
-            screenshotPath: "",
-            status: "skipped",
-            reason: "robots.txt",
-          });
-          continue;
-        }
-      }
-
-      await limiter.wait(host);
-      const result = await captureOne(browser, url, args.viewport, outDir, {
-        skipAssetDownload: args.skipAssetDownload,
-      });
-      if (!result) continue;
-      captures.push(result.capture);
-      if (result.capture.status === "ok") {
-        rawList.push(result.raw);
-        if (result.layout) {
-          layoutsByHost.set(host, result.layout);
-        }
-        const tag = result.layout ? "mirror" : "tokens-only";
-        const sectionCount = result.layout?.sections.length ?? 0;
-        console.log(
-          `  ✓ ${url}  → ${tag}${result.layout ? ` (${sectionCount} sections)` : ""}`,
-        );
-      } else {
-        console.log(`  ✗ ${url}  ${result.capture.reason ?? ""}`);
-      }
-    }
-  } finally {
-    if (browser) await browser.close();
-  }
-
-  if (rawList.length === 0) {
-    console.error("[extract] no successful captures — nothing to synthesize.");
-    process.exit(1);
-  }
-
-  console.log("");
-  console.log(`[extract] synthesizing design system from ${rawList.length} site(s)...`);
-  const designSystem = synthesize(rawList, {
-    runId,
-    sources: rawList.map((r) => ({ url: r.url, capturedAt: r.capturedAt })),
-  });
-
-  const run: ExtractionRun = {
-    runId,
-    startedAt,
-    finishedAt: new Date().toISOString(),
-    outputDir: outDir,
-    captures,
-    designSystem,
-    ...(args.saasIdea ? { saasIdea: args.saasIdea } : {}),
-  };
-
-  const written = emitAll(designSystem, run);
-  const researchWritten = emitClonerResearch(run, layoutsByHost);
-  writeFileSync(join(outDir, "run.json"), JSON.stringify(run, null, 2));
-  console.log("");
-  console.log("[extract] wrote:");
-  for (const f of written) console.log(`  → ${f}`);
-  for (const f of researchWritten) console.log(`  → ${f}`);
-  console.log(`  → ${join(outDir, "run.json")}`);
-  const mirrorDirs = captures.filter((c) => c.mirrorDir).map((c) => c.mirrorDir!);
-  const referenceDirs = captures.filter((c) => c.referenceDir).map((c) => c.referenceDir!);
-  if (mirrorDirs.length > 0) {
-    console.log("");
-    console.log("[extract] layout mirrors:");
-    for (const d of mirrorDirs) console.log(`  → ${d}/page.tsx`);
-  }
-  if (referenceDirs.length > 0) {
-    console.log("");
-    console.log("[extract] AI reference (verbatim DOM + copy):");
-    for (const d of referenceDirs) console.log(`  → ${d}/FOR_AI_REFERENCE.md`);
-  }
-  console.log("");
-  console.log(`[extract] done. Open ${join(outDir, "REPORT.md")} for the design-system summary.`);
-  if (mirrorDirs.length > 0) {
-    console.log(
-      `[extract] each mirror folder ships a Next.js page.tsx + MIRROR_NOTES.md.`,
-    );
-    console.log(`[extract] fill the <TextSlot> / <MediaSlot> placeholders with your own content.`);
-  }
-  if (referenceDirs.length > 0) {
-    console.log(
-      `[extract] paste reference/<host>/visible-text.txt or page.html into your AI for exact structure + copy.`,
-    );
-  }
-  console.log(`[extract] AI handoff: ${join(outDir, "FOR_AI.md")}`);
-  console.log(`[extract] clone-style research: ${join(outDir, "docs", "research", "README.md")}`);
-  console.log(`[extract] design references + downloads: ${join(outDir, "docs", "design-references")} · downloaded_assets/`);
-}
-
-function makeRunId(startedAt: string, name: string | undefined): string {
-  const stamp = startedAt.replace(/[-:T]/g, "").slice(0, 14);
-  return name ? `${stamp}-${name}` : stamp;
-}
-
-if (isMainModule(import.meta.url)) {
-  main().catch((err) => {
-    console.error(err);
-    process.exit(1);
-  });
-}
-
-/**
- * Cross-platform entry-point check. On Windows, `process.argv[1]` is a
- * backslash path while `import.meta.url` is a proper file URL, so the
- * naive `file://${argv[1]}` template literal never matches and the
- * script silently exits. `pathToFileURL` produces the encoded URL form
- * on every platform.
- */
-function isMainModule(metaUrl: string): boolean {
-  const entry = process.argv[1];
-  if (!entry) return false;
-  return metaUrl === pathToFileURL(entry).href;
-}
-
-export { main };

package/packages/extract/host-slug.ts

@@ -1,5 +0,0 @@
-/** Filesystem-safe slug for a URL host (cross-package helpers). */
-export function hostSlug(host: string): string {
-  const s = host.replace(/[^a-z0-9.-]+/gi, "-").replace(/^-|-$/g, "").toLowerCase();
-  return s.length > 0 ? s : "site";
-}