demo-dev 0.0.1-alpha.0

package/src/planner/prompt.ts

@@ -0,0 +1,331 @@
+/**
+ * Prompt-driven demo planner.
+ *
+ * Instead of starting from a git diff, this planner takes a natural language
+ * prompt ("show the Magic Inbox, filter by positive replies, open a thread")
+ * and generates a DemoPlan by:
+ *
+ *   1. Launching Playwright to explore the target site
+ *   2. Taking screenshots + collecting interactive elements from key pages
+ *   3. Sending the screenshots + element inventory + user prompt to an LLM
+ *   4. Parsing the LLM response into a validated DemoPlan
+ *
+ * This gives users a zero-config, one-shot way to create demos.
+ */
+
+import { mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import { chromium, type Page } from "playwright";
+import {
+  getContextOptionsWithSession,
+  resolveSessionConfig,
+} from "../browser/session.js";
+import type { ProjectConfig } from "../config/project.js";
+import { summarizeProjectHints } from "../config/project.js";
+import { requestAiJson } from "../ai/provider.js";
+import type { DemoPlan } from "../types.js";
+import { demoPlanSchema } from "./schema.js";
+
+// ---------------------------------------------------------------------------
+// Page exploration
+// ---------------------------------------------------------------------------
+
+interface PageSnapshot {
+  url: string;
+  title: string;
+  headings: string[];
+  textPreview: string;
+  interactiveElements: Array<{
+    tag: string;
+    role: string;
+    name: string;
+    text?: string;
+    label?: string;
+    placeholder?: string;
+    href?: string;
+    testId?: string;
+  }>;
+  navLinks: Array<{ text: string; href: string }>;
+}
+
+const EXPLORE_SCRIPT = `(() => {
+  function norm(v) { return (v||"").replace(/\\s+/g," ").trim().slice(0,140); }
+  function vis(el) {
+    const r = el.getBoundingClientRect(), s = getComputedStyle(el);
+    return r.width>0 && r.height>0 && s.visibility!=="hidden" && s.display!=="none" && s.opacity!=="0";
+  }
+  function role(el) {
+    const r = el.getAttribute("role"); if(r) return r;
+    const t = el.tagName.toLowerCase();
+    if(t==="a") return "link"; if(t==="button") return "button";
+    if(t==="select") return "combobox"; if(t==="textarea") return "textbox";
+    if(t==="input") { const tp=(el.type||"text").toLowerCase(); return ["submit","button"].includes(tp)?"button":"textbox"; }
+    return t;
+  }
+  function name(el) {
+    return norm(el.getAttribute("aria-label")) || norm(el.placeholder) || norm(el.textContent) || role(el);
+  }
+  const sels = "button,a[href],input,select,textarea,[role=button],[role=link],[role=tab],[role=menuitem],[data-testid]";
+  const elems = Array.from(document.querySelectorAll(sels)).filter(vis).slice(0,50).map(el => ({
+    tag: el.tagName.toLowerCase(), role: role(el), name: name(el),
+    text: norm(el.textContent)||undefined, label: undefined,
+    placeholder: norm(el.placeholder)||undefined,
+    href: el.href||undefined, testId: el.getAttribute("data-testid")||undefined,
+  }));
+  const headings = Array.from(document.querySelectorAll("h1,h2,h3")).map(h=>norm(h.textContent)).filter(Boolean).slice(0,10);
+  const text = norm(document.body?.innerText).slice(0,800);
+  const navLinks = Array.from(document.querySelectorAll("nav a[href], aside a[href], [role=navigation] a[href]"))
+    .filter(vis).slice(0,20).map(a => ({ text: norm(a.textContent), href: a.href }));
+  return { headings, textPreview: text, interactiveElements: elems, navLinks };
+})()`;
+
+const collectPageSnapshot = async (page: Page): Promise<PageSnapshot> => {
+  const data = await page.evaluate(EXPLORE_SCRIPT) as Omit<PageSnapshot, "url" | "title">;
+  return {
+    url: page.url(),
+    title: await page.title().catch(() => ""),
+    ...data,
+  };
+};
+
+// ---------------------------------------------------------------------------
+// Site exploration: visit landing + follow key nav links
+// ---------------------------------------------------------------------------
+
+interface SiteExploration {
+  pages: PageSnapshot[];
+  screenshotPaths: string[];
+}
+
+const exploreSite = async (
+  baseUrl: string,
+  outputDir: string,
+  projectConfig?: ProjectConfig,
+): Promise<SiteExploration> => {
+  const screenshotDir = join(outputDir, "exploration");
+  await mkdir(screenshotDir, { recursive: true });
+
+  const session = resolveSessionConfig(outputDir);
+  const browser = await chromium.launch({ headless: true });
+  const context = await browser.newContext(
+    await getContextOptionsWithSession(
+      { viewport: { width: 1440, height: 900 } },
+      session,
+    ),
+  );
+  const page = await context.newPage();
+
+  const pages: PageSnapshot[] = [];
+  const screenshotPaths: string[] = [];
+  const visited = new Set<string>();
+
+  // Determine which URLs to explore
+  const startUrls = ["/"];
+  if (projectConfig?.preferredRoutes) {
+    startUrls.push(...projectConfig.preferredRoutes);
+  }
+
+  try {
+    for (const route of startUrls) {
+      const fullUrl = new URL(route, baseUrl).toString();
+      const normalized = new URL(fullUrl).pathname;
+      if (visited.has(normalized)) continue;
+      visited.add(normalized);
+
+      try {
+        await page.goto(fullUrl, { waitUntil: "networkidle", timeout: 20000 });
+      } catch {
+        try {
+          await page.goto(fullUrl, { waitUntil: "domcontentloaded", timeout: 20000 });
+          await page.waitForTimeout(1500);
+        } catch {
+          continue;
+        }
+      }
+
+      const snapshot = await collectPageSnapshot(page);
+      pages.push(snapshot);
+
+      const ssPath = join(screenshotDir, `page-${pages.length}.png`);
+      await page.screenshot({ path: ssPath });
+      screenshotPaths.push(ssPath);
+
+      // Follow nav links we haven't visited (up to 4 total pages)
+      if (pages.length >= 4) break;
+
+      for (const link of snapshot.navLinks) {
+        if (pages.length >= 4) break;
+        try {
+          const linkPath = new URL(link.href).pathname;
+          if (visited.has(linkPath)) continue;
+          if (!link.href.startsWith(baseUrl)) continue;
+          visited.add(linkPath);
+
+          await page.goto(link.href, { waitUntil: "networkidle", timeout: 15000 });
+          const navSnapshot = await collectPageSnapshot(page);
+          pages.push(navSnapshot);
+
+          const navSsPath = join(screenshotDir, `page-${pages.length}.png`);
+          await page.screenshot({ path: navSsPath });
+          screenshotPaths.push(navSsPath);
+        } catch {
+          continue;
+        }
+      }
+    }
+  } finally {
+    await context.close();
+    await browser.close();
+  }
+
+  return { pages, screenshotPaths };
+};
+
+// ---------------------------------------------------------------------------
+// LLM prompt construction
+// ---------------------------------------------------------------------------
+
+const buildPromptPlannerPrompt = (
+  userPrompt: string,
+  exploration: SiteExploration,
+  projectConfig?: ProjectConfig,
+): string => {
+  const pageDescriptions = exploration.pages.map((p, i) => {
+    const elements = p.interactiveElements
+      .map((el) => {
+        const parts = [`${el.role}: "${el.name}"`];
+        if (el.href) parts.push(`→ ${el.href}`);
+        if (el.placeholder) parts.push(`(placeholder: "${el.placeholder}")`);
+        if (el.testId) parts.push(`[data-testid="${el.testId}"]`);
+        return `    ${parts.join(" ")}`;
+      })
+      .join("\n");
+
+    return [
+      `Page ${i + 1}: ${p.title}`,
+      `  URL: ${p.url}`,
+      `  Headings: ${p.headings.join(" | ")}`,
+      `  Content preview: ${p.textPreview.slice(0, 300)}`,
+      `  Interactive elements (${p.interactiveElements.length}):`,
+      elements,
+    ].join("\n");
+  });
+
+  return [
+    "You are a product demo director.",
+    "A user wants to create a demo video of their web app with a single prompt.",
+    "",
+    "USER PROMPT:",
+    `"${userPrompt}"`,
+    "",
+    "SITE EXPLORATION:",
+    "I visited the app and found these pages with their interactive elements:",
+    "",
+    ...pageDescriptions,
+    "",
+    "INSTRUCTIONS:",
+    "1. Create 3-6 scenes that tell the story the user described.",
+    "2. Each scene should have specific, executable actions (navigate, click, hover, fill, scroll, wait, press).",
+    "3. Use REAL element selectors from the exploration above. Prefer strategy: 'text' or 'role' with the exact name/text shown.",
+    "4. Write narration that sounds like a human giving a product tour — conversational, not robotic.",
+    "5. All URLs must be relative paths (e.g., /inbox not https://app.example.com/inbox).",
+    "6. Keep durationMs between 4000 and 9000 per scene.",
+    "7. Start with a navigate action in the first scene.",
+    "8. Add wait actions (300-800ms) between interactions for pacing.",
+    "9. Use scroll actions to reveal below-the-fold content when relevant.",
+    "",
+    "ACTION TARGET STRATEGIES (use these exact formats):",
+    '  { "strategy": "text", "value": "Button Text", "exact": false }  — match by visible text',
+    '  { "strategy": "role", "role": "button", "name": "Submit" }      — match by ARIA role + name',
+    '  { "strategy": "css", "value": ".my-class" }                     — CSS selector',
+    '  { "strategy": "placeholder", "value": "Search..." }             — match by placeholder',
+    '  { "strategy": "testId", "value": "my-test-id" }                 — match by data-testid',
+    "",
+    "Project hints:",
+    JSON.stringify(summarizeProjectHints(projectConfig ?? {}), null, 2),
+    "",
+    "EXACT JSON SCHEMA (follow this structure precisely):",
+    JSON.stringify({
+      title: "Demo Title",
+      summary: "One sentence summary",
+      branch: "prompt",
+      generatedAt: "2026-01-01T00:00:00.000Z",
+      scenes: [{
+        id: "scene-1",
+        title: "Scene Title",
+        goal: "What this scene demonstrates",
+        url: "/relative-path",
+        viewport: { width: 1440, height: 900 },
+        actions: [
+          { type: "navigate", url: "/relative-path" },
+          { type: "wait", ms: 500 },
+          { type: "click", target: { strategy: "text", value: "Button Text", exact: false } },
+          { type: "hover", target: { strategy: "role", role: "button", name: "Name" } },
+          { type: "scroll", y: 200 },
+          { type: "fill", target: { strategy: "placeholder", value: "Search..." }, value: "typed text" },
+        ],
+        narration: "Conversational narration for this scene.",
+        caption: "Short caption",
+        durationMs: 6000,
+        evidenceHints: [],
+      }],
+    }, null, 2),
+    "",
+    "Output a strict JSON DemoPlan following the schema above. No markdown, no explanation.",
+  ].join("\n");
+};
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+export const buildPromptPlan = async (options: {
+  prompt: string;
+  baseUrl: string;
+  outputDir: string;
+  projectConfig?: ProjectConfig;
+}): Promise<DemoPlan> => {
+  // Step 1: Explore the site
+  const exploration = await exploreSite(
+    options.baseUrl,
+    options.outputDir,
+    options.projectConfig,
+  );
+
+  if (exploration.pages.length === 0) {
+    throw new Error("Could not load any pages from " + options.baseUrl);
+  }
+
+  // Step 2: Ask LLM to generate a plan
+  const prompt = buildPromptPlannerPrompt(
+    options.prompt,
+    exploration,
+    options.projectConfig,
+  );
+
+  const plan = await requestAiJson({
+    system:
+      "You are a product demo director. You create demo plans from natural language prompts. " +
+      "You have been given a site exploration with real page content and interactive elements. " +
+      "Use ONLY elements that actually exist on the pages. Output strict JSON only.",
+    prompt,
+    schema: demoPlanSchema,
+    temperature: 0.3,
+  });
+
+  if (!plan) {
+    throw new Error(
+      "AI provider could not generate a plan. Set DEMO_OPENAI_API_KEY or use a local AI provider.",
+    );
+  }
+
+  return {
+    ...plan,
+    branch: "prompt",
+    generatedAt: new Date().toISOString(),
+    scenes: plan.scenes.map((scene) => ({
+      ...scene,
+      evidenceHints: scene.evidenceHints ?? [],
+    })),
+  };
+};

package/src/planner/refine.ts

@@ -0,0 +1,155 @@
+import type { DemoPlan, DemoScene, PageProbe, ProbeSnapshot, SceneAction } from "../types.js";
+import { requestAiJson } from "../ai/provider.js";
+import { demoPlanSchema } from "./schema.js";
+
+const ACTIONABLE_NAME_RE =
+  /get started|start|continue|next|learn more|try demo|demo|explore|open|view|create|new|save|submit|search/i;
+const NEGATIVE_NAME_RE = /delete|remove|logout|log out|sign out|cancel/i;
+
+const snapshotFor = (probe: PageProbe): ProbeSnapshot => (probe.followUp && !probe.followUp.error ? probe.followUp : probe.initial);
+
+const normalizeTargetAction = (scene: DemoScene, probe: PageProbe): SceneAction[] => {
+  const initial = probe.initial;
+  const active = snapshotFor(probe);
+  const actions: SceneAction[] = [{ type: "navigate", url: scene.url }];
+
+  if (initial.headings[0]) {
+    actions.push({ type: "waitForText", value: initial.headings[0], timeoutMs: 10000 });
+  } else {
+    actions.push({ type: "wait", ms: 1200 });
+  }
+
+  const searchField = initial.interactiveElements.find(
+    (element) =>
+      element.role === "textbox" &&
+      /search/i.test([element.name, element.label, element.placeholder].filter(Boolean).join(" ")),
+  );
+
+  if (searchField?.placeholder) {
+    actions.push({
+      type: "fill",
+      target: { strategy: "placeholder", value: searchField.placeholder },
+      value: "demo",
+    });
+    actions.push({ type: "press", key: "Enter" });
+    actions.push({ type: "wait", ms: 1200 });
+    if (active !== initial && active.headings[0] && active.headings[0] !== initial.headings[0]) {
+      actions.push({ type: "waitForText", value: active.headings[0], timeoutMs: 10000 });
+    }
+    return actions;
+  }
+
+  const primaryAction = initial.interactiveElements.find(
+    (element) =>
+      ["button", "link", "tab"].includes(element.role) &&
+      ACTIONABLE_NAME_RE.test(element.name) &&
+      !NEGATIVE_NAME_RE.test(element.name),
+  );
+
+  if (primaryAction) {
+    actions.push({
+      type: "click",
+      target: primaryAction.testId
+        ? { strategy: "testId", value: primaryAction.testId }
+        : { strategy: "role", role: primaryAction.role, name: primaryAction.name, exact: false },
+    });
+
+    if (probe.followUp?.resolvedUrl && probe.followUp.resolvedUrl !== initial.resolvedUrl) {
+      actions.push({ type: "waitForUrl", value: probe.followUp.resolvedUrl, timeoutMs: 10000 });
+    } else {
+      actions.push({ type: "wait", ms: 1200 });
+    }
+
+    if (probe.followUp?.headings[0] && probe.followUp.headings[0] !== initial.headings[0]) {
+      actions.push({ type: "waitForText", value: probe.followUp.headings[0], timeoutMs: 10000 });
+    }
+
+    return actions;
+  }
+
+  actions.push({ type: "scroll", y: 360 });
+  actions.push({ type: "wait", ms: 400 });
+  return actions;
+};
+
+const allowHeuristicFallback = process.env.DEMO_AI_MANDATORY === "false";
+
+const buildRefinementPrompt = (context: {
+  initialPlan: DemoPlan;
+  probes: PageProbe[];
+}) => {
+  return [
+    "You are a product demo director. You already have an initial scene plan and real browser probe output.",
+    "Task: revise the initial plan into a more executable browser plan.",
+    "Rules:",
+    "1. Only use headings, buttons, links, inputs, and placeholders that actually appear in the probes.",
+    "2. If a followUp probe exists, prefer turning it into a multi-step flow such as click -> waitForUrl -> waitForText.",
+    "3. Do not invent labels, buttons, or text that were not observed.",
+    "4. If a probe failed, keep the scene as a page-level presentation.",
+    "5. Prefer stable flows: navigate -> waitForText -> click/fill -> waitForUrl/waitForText.",
+    "6. If no reliable element exists, keep the scene simple instead of forcing a complex form flow.",
+    "7. Return full JSON matching the original plan shape.",
+    "",
+    "Initial plan:",
+    JSON.stringify(context.initialPlan, null, 2),
+    "",
+    "Probe results:",
+    JSON.stringify(context.probes, null, 2),
+  ].join("\n");
+};
+
+export const refineDemoPlan = async (options: {
+  initialPlan: DemoPlan;
+  probes: PageProbe[];
+}): Promise<DemoPlan> => {
+  try {
+    const refined = await requestAiJson({
+      system:
+        "You refine browser demo plans using only observed page probes. Prefer concise multi-step flows when a follow-up page state was observed. Keep outputs executable and JSON-only.",
+      prompt: buildRefinementPrompt(options),
+      schema: demoPlanSchema,
+      temperature: 0.2,
+    });
+
+    if (refined) {
+      return {
+        ...refined,
+        branch: options.initialPlan.branch,
+        generatedAt: new Date().toISOString(),
+        scenes: refined.scenes.map((scene) => ({
+          ...scene,
+          evidenceHints: scene.evidenceHints ?? [],
+        })),
+      };
+    }
+  } catch (error) {
+    if (!allowHeuristicFallback) {
+      throw error;
+    }
+    console.warn("LLM refinement failed, fallback to heuristic refinement", error);
+  }
+
+  if (!allowHeuristicFallback) {
+    throw new Error("AI provider did not return a refined plan. Set DEMO_AI_MANDATORY=false to allow heuristic fallback.");
+  }
+
+  const probeBySceneId = new Map(options.probes.map((probe) => [probe.sceneId, probe]));
+
+  return {
+    ...options.initialPlan,
+    summary: `${options.initialPlan.summary} Refined using real page probes.`,
+    generatedAt: new Date().toISOString(),
+    scenes: options.initialPlan.scenes.map((scene) => {
+      const probe = probeBySceneId.get(scene.id);
+      if (!probe || probe.initial.error) return scene;
+
+      const active = snapshotFor(probe);
+
+      return {
+        ...scene,
+        caption: active.pageTitle ? `${scene.title} · ${active.pageTitle}` : scene.caption,
+        actions: normalizeTargetAction(scene, probe),
+      };
+    }),
+  };
+};

package/src/planner/schema.ts

@@ -0,0 +1,62 @@
+import { z } from "zod";
+
+const actionTargetSchema = z.discriminatedUnion("strategy", [
+  z.object({ strategy: z.literal("label"), value: z.string(), exact: z.boolean().optional() }),
+  z.object({ strategy: z.literal("text"), value: z.string(), exact: z.boolean().optional() }),
+  z.object({ strategy: z.literal("placeholder"), value: z.string(), exact: z.boolean().optional() }),
+  z.object({ strategy: z.literal("testId"), value: z.string() }),
+  z.object({ strategy: z.literal("css"), value: z.string() }),
+  z.object({ strategy: z.literal("role"), role: z.string(), name: z.string().optional(), exact: z.boolean().optional() }),
+]);
+
+const focusRegionSchema = z.object({
+  x: z.number(),
+  y: z.number(),
+  width: z.number(),
+  height: z.number(),
+  label: z.string().optional(),
+});
+
+const sceneDirectionSchema = z.object({
+  shot: z.enum(["hero", "detail", "workflow"]),
+  focusRegion: focusRegionSchema.optional(),
+  accentColor: z.string().optional(),
+  cameraMove: z.enum(["push-in", "pan-left", "pan-right"]).optional(),
+});
+
+export const sceneActionSchema = z.discriminatedUnion("type", [
+  z.object({ type: z.literal("navigate"), url: z.string() }),
+  z.object({ type: z.literal("wait"), ms: z.number().int().min(0).max(20000) }),
+  z.object({ type: z.literal("scroll"), y: z.number().int().min(-5000).max(5000) }),
+  z.object({ type: z.literal("click"), target: actionTargetSchema }),
+  z.object({ type: z.literal("hover"), target: actionTargetSchema }),
+  z.object({ type: z.literal("fill"), target: actionTargetSchema, value: z.string() }),
+  z.object({ type: z.literal("press"), key: z.string() }),
+  z.object({ type: z.literal("select"), target: actionTargetSchema, value: z.string() }),
+  z.object({ type: z.literal("waitForText"), value: z.string(), exact: z.boolean().optional(), timeoutMs: z.number().int().min(0).max(20000).optional() }),
+  z.object({ type: z.literal("waitForUrl"), value: z.string(), timeoutMs: z.number().int().min(0).max(20000).optional() }),
+]);
+
+export const demoSceneSchema = z.object({
+  id: z.string(),
+  title: z.string(),
+  goal: z.string(),
+  url: z.string(),
+  viewport: z.object({ width: z.number().int().min(320).max(2400), height: z.number().int().min(320).max(2400) }),
+  actions: z.array(sceneActionSchema).min(1),
+  narration: z.string(),
+  caption: z.string(),
+  durationMs: z.number().int().min(1000).max(30000),
+  evidenceHints: z.array(z.string()).default([]),
+  direction: sceneDirectionSchema.optional(),
+});
+
+export const demoPlanSchema = z.object({
+  title: z.string(),
+  summary: z.string(),
+  branch: z.string(),
+  generatedAt: z.string(),
+  scenes: z.array(demoSceneSchema).min(1).max(6),
+});
+
+export type DemoPlanInput = z.infer<typeof demoPlanSchema>;

package/src/presentation/polish.ts

@@ -0,0 +1,84 @@
+import type { DemoPlan } from "../types.js";
+import { requestAiJson } from "../ai/provider.js";
+import { z } from "zod";
+
+const polishedPresentationSchema = z.object({
+  title: z.string(),
+  summary: z.string(),
+  scenes: z.array(
+    z.object({
+      id: z.string(),
+      title: z.string(),
+      caption: z.string(),
+      narration: z.string(),
+    }),
+  ),
+});
+
+const getPresentationConfig = () => ({
+  language: process.env.DEMO_SCRIPT_LANGUAGE ?? "en",
+  style: process.env.DEMO_PRESENTATION_STYLE ?? "launch",
+});
+
+export const polishPresentationCopy = async (plan: DemoPlan): Promise<DemoPlan> => {
+  const config = getPresentationConfig();
+
+  try {
+    const polished = await requestAiJson({
+      system:
+        "You are a world-class product launch copy director. Rewrite demo video copy to sound premium, concise, cinematic, and confident. Output strict JSON only.",
+      prompt: [
+        `Rewrite the following demo plan copy in ${config.language}.`,
+        `Style: ${config.style}. Think product launch / premium SaaS demo, not QA output.`,
+        "Rules:",
+        "1. Keep the same number of scenes and preserve every scene id exactly.",
+        "2. Rewrite only title, caption, narration, plus top-level title and summary.",
+        "3. Narration should sound like polished spoken demo copy.",
+        "4. Caption should be short, punchy, and on-screen friendly.",
+        "5. Title should be product-marketing quality, not generic.",
+        "6. Do not mention git diff, tests, heuristics, JSON, or implementation details unless the product truly shows them.",
+        "7. English should be natural and launch-video ready.",
+        JSON.stringify(
+          {
+            title: plan.title,
+            summary: plan.summary,
+            scenes: plan.scenes.map((scene) => ({
+              id: scene.id,
+              title: scene.title,
+              caption: scene.caption,
+              narration: scene.narration,
+              goal: scene.goal,
+            })),
+          },
+          null,
+          2,
+        ),
+      ].join("\n"),
+      schema: polishedPresentationSchema,
+      temperature: 0.7,
+    });
+
+    if (!polished) return plan;
+
+    const copyBySceneId = new Map(polished.scenes.map((scene) => [scene.id, scene]));
+
+    return {
+      ...plan,
+      title: polished.title,
+      summary: polished.summary,
+      scenes: plan.scenes.map((scene) => {
+        const copy = copyBySceneId.get(scene.id);
+        if (!copy) return scene;
+        return {
+          ...scene,
+          title: copy.title,
+          caption: copy.caption,
+          narration: copy.narration,
+        };
+      }),
+    };
+  } catch (error) {
+    console.warn("presentation polish failed, using original copy", error);
+    return plan;
+  }
+};