launchframe 0.1.10 → 0.1.13

package/packages/extract/extract.ts

@@ -1,7 +1,8 @@
 /**
  * `extract` — the headline command.
  *
- *   npm run extract -- https://site-a.com https://site-b.com https://site-c.com
+ *   npx launchframe@latest https://site-a.com "Your SaaS idea"
+ *   npm run extract -- https://site-a.com "idea" https://site-b.com
  *
  * For each URL: open in Chromium, screenshot, harvest computed design
  * tokens via `browser-extract.ts`, and crawl the rendered DOM into a
@@ -22,6 +23,9 @@
  *     computed style tokens, content kinds) and writes a verbatim
  *     `reference/<host>/` bundle (HTML, DOM tree JSON, outlines, visible text,
  *     media index) for AI structure cloning.
+ *   - Automated **design-references** pass: responsive + scroll sweep PNGs,
+ *     header probe, interaction hints, optional media downloads (like the
+ *     ai-website-cloner-template recon outputs).
  */
 
 import { mkdirSync, writeFileSync } from "node:fs";
@@ -31,8 +35,11 @@ import { fileURLToPath, pathToFileURL } from "node:url";
 import { chromium, type Browser } from "playwright";
 
 import { harvestTokens } from "./browser-extract.js";
+import { runAutomatedClonePass } from "./automated-clone-pass.js";
 import { crawlLayout } from "./dom-crawler.js";
+import { emitClonerResearch } from "./cloner-research-emit.js";
 import { emitAll } from "./emit.js";
+import { emitMirror } from "./mirror-emit.js";
 import { emitPageReference } from "./reference-dump.js";
 import { synthesize } from "./synthesize.js";
 import type { ExtractionRun, RawTokens, SiteCapture, SiteLayout } from "./types.js";
@@ -52,38 +59,98 @@ interface CliArgs {
   respectRobots: boolean;
   rateLimitPerMinute: number;
   runName?: string;
+  /** SaaS product narrative from `--idea`, a positional string, or LAUNCHFRAME_SAAS_IDEA. */
+  saasIdea?: string;
+  /** When true, skip HTTP downloads from reference/media.json */
+  skipAssetDownload: boolean;
 }
 
 function parseArgs(argv: string[]): CliArgs {
+  const urls: string[] = [];
+  const positional: string[] = [];
   const args: CliArgs = {
-    urls: [],
+    urls,
     outDir: "",
     viewport: { width: 1440, height: 900 },
     respectRobots: true,
     rateLimitPerMinute: 15,
+    skipAssetDownload: false,
   };
+
   for (let i = 0; i < argv.length; i++) {
     const a = argv[i]!;
-    if (a === "--out") args.outDir = argv[++i]!;
-    else if (a === "--name") args.runName = argv[++i];
-    else if (a === "--no-robots") args.respectRobots = false;
-    else if (a === "--rate") args.rateLimitPerMinute = parseInt(argv[++i]!, 10);
-    else if (a === "--width") args.viewport.width = parseInt(argv[++i]!, 10);
-    else if (a === "--height") args.viewport.height = parseInt(argv[++i]!, 10);
-    else if (a === "--help" || a === "-h") {
+    if (a === "--idea") {
+      const v = argv[++i];
+      if (!v) {
+        console.error("--idea requires a value");
+        process.exit(2);
+      }
+      args.saasIdea = v;
+      continue;
+    }
+    if (a === "--no-download") {
+      args.skipAssetDownload = true;
+      continue;
+    }
+    if (a === "--out") {
+      args.outDir = argv[++i]!;
+      continue;
+    }
+    if (a === "--name") {
+      args.runName = argv[++i];
+      continue;
+    }
+    if (a === "--no-robots") {
+      args.respectRobots = false;
+      continue;
+    }
+    if (a === "--rate") {
+      args.rateLimitPerMinute = parseInt(argv[++i]!, 10);
+      continue;
+    }
+    if (a === "--width") {
+      args.viewport.width = parseInt(argv[++i]!, 10);
+      continue;
+    }
+    if (a === "--height") {
+      args.viewport.height = parseInt(argv[++i]!, 10);
+      continue;
+    }
+    if (a === "--help" || a === "-h") {
       printHelp();
       process.exit(0);
-    } else if (a.startsWith("http://") || a.startsWith("https://")) {
-      args.urls.push(a);
-    } else if (a.startsWith("--")) {
+    }
+    if (a.startsWith("http://") || a.startsWith("https://")) {
+      urls.push(a);
+      continue;
+    }
+    if (a.startsWith("--")) {
       console.error(`Unknown flag: ${a}`);
       process.exit(2);
-    } else {
-      console.error(`Unrecognized argument: ${a}`);
+    }
+    positional.push(a);
+  }
+
+  if (positional.length > 1) {
+    console.error(
+      'Too many positional arguments. Quote your SaaS idea as one argument or use --idea "..."',
+    );
+    process.exit(2);
+  }
+  if (positional.length === 1) {
+    if (args.saasIdea) {
+      console.error("Use either --idea or one positional SaaS idea string, not both.");
       process.exit(2);
     }
+    args.saasIdea = positional[0];
   }
-  if (args.urls.length === 0) {
+
+  const envIdea = process.env.LAUNCHFRAME_SAAS_IDEA?.trim();
+  if (envIdea && !args.saasIdea) {
+    args.saasIdea = envIdea;
+  }
+
+  if (urls.length === 0) {
     printHelp();
     process.exit(2);
   }
@@ -94,8 +161,9 @@ function printHelp(): void {
   console.log(
     [
       "Usage:",
-      "  npx launchframe <url> [<url> ...] [options]      (from any folder)",
-      "  npm run extract -- <url> [<url> ...] [options]   (from this repo)",
+      "  npx launchframe@latest <url> [\"SaaS idea\"] [<url> ...] [options]",
+      "  npx launchframe@latest <url> --idea \"SaaS idea\" [options]",
+      "  npm run extract -- <url> [\"idea\"] [...] [options]   (from this repo)",
       "",
       "Writes to ./output/<runId>/ in your current working directory unless",
       "you pass --out.",
@@ -113,8 +181,14 @@ function printHelp(): void {
       "After every URL, a drop-in shadcn-compatible design system is",
       "synthesized from the aggregated tokens and written to output/<runId>/.",
       "",
+      "  Your SaaS idea is stored in run.json, FOR_AI.md, and docs/research/ for agents.",
+      "",
       "Options:",
-      "  --out <dir>          Output directory (default: output/<runId>)",
+      "  --idea <text>        SaaS product narrative (alternative to one positional string)",
+      "  --no-download        Skip downloading binaries listed in reference/*/media.json",
+      "  --out <dir>          Run directory (default: ./output/<runId>). Use e.g.",
+      "                       ./my-next-app/extraction-acme to mirror template-style",
+      "                       docs/ placement inside a consumer repo.",
       "  --name <slug>        Human-friendly slug used in the runId",
       "  --no-robots          Skip robots.txt check (not recommended)",
       "  --rate <per-min>     Per-domain rate limit, default 15",
@@ -197,6 +271,7 @@ async function captureOne(
   url: string,
   viewport: { width: number; height: number },
   outDir: string,
+  opts: { skipAssetDownload: boolean },
 ): Promise<{ raw: RawTokens; layout: SiteLayout | null; capture: SiteCapture } | null> {
   const host = new URL(url).host;
   const stamp = `${host}.png`;
@@ -214,7 +289,7 @@ async function captureOne(
   });
   const page = await ctx.newPage();
   try {
-    const response = await page.goto(url, { waitUntil: "networkidle", timeout: 30_000 });
+    const response = await page.goto(url, { waitUntil: "load", timeout: 60_000 });
     if (!response || response.status() >= 400) {
       throw new Error(`HTTP ${response?.status() ?? "unknown"}`);
     }
@@ -255,6 +330,23 @@ async function captureOne(
       console.warn(`  ! layout crawl failed for ${url}: ${(err as Error).message}`);
     }
 
+    try {
+      const n = (
+        await runAutomatedClonePass({
+          page,
+          host,
+          outDir,
+          primaryViewport: viewport,
+          referenceDir: referenceWritten.length > 0 ? referenceDir : undefined,
+          userAgent: USER_AGENT,
+          downloadAssets: !opts.skipAssetDownload,
+        })
+      ).length;
+      if (n > 0) console.log(`  · automated recon (${n} files under docs/design-references & downloads)`);
+    } catch (err) {
+      console.warn(`  ! automated recon failed for ${url}: ${(err as Error).message}`);
+    }
+
     const capture: SiteCapture = {
       url,
       host,
@@ -276,7 +368,6 @@ async function captureOne(
         host,
         capturedAt: new Date().toISOString(),
         screenshotPath: "",
-        rawTokensPath: "",
         status: "failed",
         reason: (err as Error).message,
       },
@@ -309,6 +400,9 @@ async function main(): Promise<void> {
   console.log(`[extract] runId=${runId}`);
   console.log(`[extract] urls=${args.urls.length} viewport=${args.viewport.width}x${args.viewport.height}`);
   console.log(`[extract] output=${outDir}`);
+  if (args.saasIdea) {
+    console.log(`[extract] saasIdea=${args.saasIdea.slice(0, 80)}${args.saasIdea.length > 80 ? "…" : ""}`);
+  }
   console.log("");
 
   mkdirSync(outDir, { recursive: true });
@@ -316,6 +410,7 @@ async function main(): Promise<void> {
   const limiter = new RateLimiter(args.rateLimitPerMinute);
   const captures: SiteCapture[] = [];
   const rawList: RawTokens[] = [];
+  const layoutsByHost = new Map<string, SiteLayout>();
 
   let browser: Browser | null = null;
   try {
@@ -333,7 +428,6 @@ async function main(): Promise<void> {
             host,
             capturedAt: new Date().toISOString(),
             screenshotPath: "",
-            rawTokensPath: "",
             status: "skipped",
             reason: "robots.txt",
           });
@@ -342,11 +436,16 @@ async function main(): Promise<void> {
       }
 
       await limiter.wait(host);
-      const result = await captureOne(browser, url, args.viewport, outDir);
+      const result = await captureOne(browser, url, args.viewport, outDir, {
+        skipAssetDownload: args.skipAssetDownload,
+      });
       if (!result) continue;
       captures.push(result.capture);
       if (result.capture.status === "ok") {
         rawList.push(result.raw);
+        if (result.layout) {
+          layoutsByHost.set(host, result.layout);
+        }
         const tag = result.layout ? "mirror" : "tokens-only";
         const sectionCount = result.layout?.sections.length ?? 0;
         console.log(
@@ -379,13 +478,16 @@ async function main(): Promise<void> {
     outputDir: outDir,
     captures,
     designSystem,
+    ...(args.saasIdea ? { saasIdea: args.saasIdea } : {}),
   };
 
   const written = emitAll(designSystem, run);
+  const researchWritten = emitClonerResearch(run, layoutsByHost);
   writeFileSync(join(outDir, "run.json"), JSON.stringify(run, null, 2));
   console.log("");
   console.log("[extract] wrote:");
   for (const f of written) console.log(`  → ${f}`);
+  for (const f of researchWritten) console.log(`  → ${f}`);
   console.log(`  → ${join(outDir, "run.json")}`);
   const mirrorDirs = captures.filter((c) => c.mirrorDir).map((c) => c.mirrorDir!);
   const referenceDirs = captures.filter((c) => c.referenceDir).map((c) => c.referenceDir!);
@@ -413,6 +515,8 @@ async function main(): Promise<void> {
     );
   }
   console.log(`[extract] AI handoff: ${join(outDir, "FOR_AI.md")}`);
+  console.log(`[extract] clone-style research: ${join(outDir, "docs", "research", "README.md")}`);
+  console.log(`[extract] design references + downloads: ${join(outDir, "docs", "design-references")} · downloaded_assets/`);
 }
 
 function makeRunId(startedAt: string, name: string | undefined): string {

package/packages/extract/host-slug.ts

@@ -0,0 +1,5 @@
+/** Filesystem-safe slug for a URL host (cross-package helpers). */
+export function hostSlug(host: string): string {
+  const s = host.replace(/[^a-z0-9.-]+/gi, "-").replace(/^-|-$/g, "").toLowerCase();
+  return s.length > 0 ? s : "site";
+}

package/packages/extract/mirror-emit.ts

@@ -408,9 +408,12 @@ function emitGrid(s: SectionLayout, labelId: string): string {
   const cardHeading: SlotKind =
     (slots["heading-2"] ?? 0) >= cols ? "heading-2" : "heading-3";
 
+  const cardCount = Math.max(cols, Math.min(Math.max(slots["heading-3"] ?? 0, cols * 2), 12));
+  const cards = Array.from({ length: cardCount }, (_, i) => i);
+
   const grid = [
     `<Stagger as="ul" className=${JSON.stringify(`mt-12 grid gap-6 md:grid-cols-${cols}`)}>`,
-    ...cards.map((c) =>
+    ...cards.map(() =>
       [
         '  <StaggerItem as="li" className="flex flex-col gap-3 rounded-lg border border-border bg-card p-6">',
         '    <Sparkle className="size-10 text-[var(--mirror-primary)]" weight="duotone" aria-hidden />',

package/packages/extract/types.ts

@@ -293,6 +293,9 @@ export interface SiteCapture {
   host: string;
   capturedAt: string;
   screenshotPath: string;
+  rawTokensPath?: string;
+  /** Path to crawled layout JSON when DOM crawl succeeded. */
+  layoutPath?: string;
   /** Verbatim HTML + copy + media listing for AI reference. */
   referenceDir?: string;
   /** Path to the per-site mirror page directory, if emission succeeded. */
@@ -308,4 +311,6 @@ export interface ExtractionRun {
   outputDir: string;
   captures: SiteCapture[];
   designSystem: DesignSystem | null;
+  /** Product narrative from CLI/env — drives FOR_AI.md and docs/research/. */
+  saasIdea?: string;
 }