@demon-utils/playwright 0.1.5 → 0.1.7

package/src/review.test.ts

@@ -3,116 +3,314 @@ import { describe, test, expect } from "bun:test";
 import type { SpawnFn } from "./review.ts";
 import {
   buildReviewPrompt,
+  extractJson,
   invokeClaude,
-  parseReviewMetadata,
+  parseLlmResponse,
 } from "./review.ts";
 
+function makeValidResponse(overrides?: Record<string, unknown>) {
+  return {
+    demos: [{ file: "login.webm", summary: "Shows a login flow" }],
+    review: {
+      summary: "Good changes overall",
+      highlights: ["Clean implementation"],
+      verdict: "approve",
+      verdictReason: "No major issues found",
+      issues: [],
+    },
+    ...overrides,
+  };
+}
+
 describe("buildReviewPrompt", () => {
   test("includes all filenames in the prompt", () => {
-    const prompt = buildReviewPrompt(["login-flow.webm", "signup.webm"]);
-    expect(prompt).toContain("- login-flow.webm");
-    expect(prompt).toContain("- signup.webm");
+    const stepsMap = {
+      "login-flow.webm": [],
+      "signup.webm": [],
+    };
+    const prompt = buildReviewPrompt({ filenames: ["login-flow.webm", "signup.webm"], stepsMap });
+    expect(prompt).toContain("login-flow.webm");
+    expect(prompt).toContain("signup.webm");
   });
 
   test("includes schema instructions", () => {
-    const prompt = buildReviewPrompt(["demo.webm"]);
+    const stepsMap = { "demo.webm": [] };
+    const prompt = buildReviewPrompt({ filenames: ["demo.webm"], stepsMap });
     expect(prompt).toContain('"demos"');
     expect(prompt).toContain('"file"');
     expect(prompt).toContain('"summary"');
-    expect(prompt).toContain('"annotations"');
-    expect(prompt).toContain('"timestampSeconds"');
+  });
+
+  test("does not include annotations in schema", () => {
+    const stepsMap = { "demo.webm": [] };
+    const prompt = buildReviewPrompt({ filenames: ["demo.webm"], stepsMap });
+    expect(prompt).not.toContain('"annotations"');
+  });
+
+  test("includes step data in prompt", () => {
+    const stepsMap = {
+      "login-flow.webm": [
+        { text: "Navigate to login", timestampSeconds: 0.5 },
+        { text: "Enter credentials", timestampSeconds: 3.2 },
+      ],
+    };
+    const prompt = buildReviewPrompt({ filenames: ["login-flow.webm"], stepsMap });
+    expect(prompt).toContain("[0.5s] Navigate to login");
+    expect(prompt).toContain("[3.2s] Enter credentials");
+    expect(prompt).toContain("Recorded steps:");
+  });
+
+  test("shows (no steps recorded) when steps array is empty", () => {
+    const stepsMap = { "demo.webm": [] };
+    const prompt = buildReviewPrompt({ filenames: ["demo.webm"], stepsMap });
+    expect(prompt).toContain("(no steps recorded)");
+  });
+
+  test("shows (no steps recorded) when video not in stepsMap", () => {
+    const stepsMap = { "other.webm": [] };
+    const prompt = buildReviewPrompt({ filenames: ["demo.webm"], stepsMap });
+    expect(prompt).toContain("(no steps recorded)");
+  });
+
+  test("includes git diff when provided", () => {
+    const prompt = buildReviewPrompt({
+      filenames: ["demo.webm"],
+      stepsMap: { "demo.webm": [] },
+      gitDiff: "diff --git a/file.ts\n+added line",
+    });
+    expect(prompt).toContain("## Git Diff");
+    expect(prompt).toContain("+added line");
+  });
+
+  test("does not include git diff section when not provided", () => {
+    const prompt = buildReviewPrompt({
+      filenames: ["demo.webm"],
+      stepsMap: { "demo.webm": [] },
+    });
+    expect(prompt).not.toContain("## Git Diff");
+  });
+
+  test("includes guidelines when provided", () => {
+    const prompt = buildReviewPrompt({
+      filenames: ["demo.webm"],
+      stepsMap: { "demo.webm": [] },
+      guidelines: ["# CLAUDE.md\nUse dependency injection"],
+    });
+    expect(prompt).toContain("## Coding Guidelines");
+    expect(prompt).toContain("Use dependency injection");
+  });
+
+  test("does not include guidelines section when empty", () => {
+    const prompt = buildReviewPrompt({
+      filenames: ["demo.webm"],
+      stepsMap: { "demo.webm": [] },
+      guidelines: [],
+    });
+    expect(prompt).not.toContain("## Coding Guidelines");
+  });
+
+  test("truncates git diff at 50k characters", () => {
+    const longDiff = "x".repeat(60_000);
+    const prompt = buildReviewPrompt({
+      filenames: ["demo.webm"],
+      stepsMap: { "demo.webm": [] },
+      gitDiff: longDiff,
+    });
+    expect(prompt).toContain("diff truncated at 50k characters");
+    expect(prompt).not.toContain("x".repeat(60_000));
+  });
+
+  test("includes review schema in prompt", () => {
+    const prompt = buildReviewPrompt({
+      filenames: ["demo.webm"],
+      stepsMap: { "demo.webm": [] },
+    });
+    expect(prompt).toContain('"review"');
+    expect(prompt).toContain('"verdict"');
+    expect(prompt).toContain('"issues"');
+    expect(prompt).toContain('"highlights"');
   });
 });
 
-describe("parseReviewMetadata", () => {
-  const validInput = JSON.stringify({
-    demos: [
-      {
-        file: "login.webm",
-        summary: "Shows a login flow",
-        annotations: [{ timestampSeconds: 0, text: "Start" }],
-      },
-    ],
+describe("extractJson", () => {
+  test("returns raw string when it is valid JSON", () => {
+    const json = '{"key": "value"}';
+    expect(extractJson(json)).toBe(json);
+  });
+
+  test("strips preamble text before JSON object", () => {
+    const raw = 'Here is the review.\n\n{"demos": []}';
+    expect(extractJson(raw)).toBe('{"demos": []}');
+  });
+
+  test("strips trailing text after JSON object", () => {
+    const raw = '{"demos": []}\n\nHope this helps!';
+    expect(extractJson(raw)).toBe('{"demos": []}');
+  });
+
+  test("strips both preamble and trailing text", () => {
+    const raw = 'Let me analyze this.\n{"demos": []}\nDone.';
+    expect(extractJson(raw)).toBe('{"demos": []}');
+  });
+
+  test("handles nested braces correctly", () => {
+    const json = '{"a": {"b": "c"}}';
+    const raw = `Some preamble\n${json}\nsome trailing`;
+    expect(extractJson(raw)).toBe(json);
+  });
+
+  test("throws when no JSON object found", () => {
+    expect(() => extractJson("no json here at all")).toThrow("No JSON object found");
   });
 
+  test("throws when only opening brace", () => {
+    expect(() => extractJson("just { nothing")).toThrow("No JSON object found");
+  });
+});
+
+describe("parseLlmResponse", () => {
   test("parses valid input correctly", () => {
-    const result = parseReviewMetadata(validInput);
+    const result = parseLlmResponse(JSON.stringify(makeValidResponse()));
     expect(result.demos).toHaveLength(1);
     expect(result.demos[0]!.file).toBe("login.webm");
     expect(result.demos[0]!.summary).toBe("Shows a login flow");
-    expect(result.demos[0]!.annotations).toHaveLength(1);
-    expect(result.demos[0]!.annotations[0]!.timestampSeconds).toBe(0);
-    expect(result.demos[0]!.annotations[0]!.text).toBe("Start");
+    expect(result.review.verdict).toBe("approve");
+    expect(result.review.summary).toBe("Good changes overall");
+    expect(result.review.highlights).toEqual(["Clean implementation"]);
+    expect(result.review.issues).toEqual([]);
+  });
+
+  test("parses response with issues", () => {
+    const input = makeValidResponse({
+      review: {
+        summary: "Needs work",
+        highlights: ["Good tests"],
+        verdict: "request_changes",
+        verdictReason: "Major bug found",
+        issues: [
+          { severity: "major", description: "Null pointer dereference" },
+          { severity: "minor", description: "Missing error message" },
+          { severity: "nit", description: "Inconsistent naming" },
+        ],
+      },
+    });
+    const result = parseLlmResponse(JSON.stringify(input));
+    expect(result.review.issues).toHaveLength(3);
+    expect(result.review.issues[0]!.severity).toBe("major");
+    expect(result.review.verdict).toBe("request_changes");
+  });
+
+  test("parses response with preamble text before JSON", () => {
+    const json = JSON.stringify(makeValidResponse());
+    const raw = `Let me review the changes.\n\n${json}`;
+    const result = parseLlmResponse(raw);
+    expect(result.demos).toHaveLength(1);
+    expect(result.review.verdict).toBe("approve");
   });
 
-  test("throws on invalid JSON", () => {
-    expect(() => parseReviewMetadata("not json")).toThrow("Invalid JSON");
+  test("parses response with trailing text after JSON", () => {
+    const json = JSON.stringify(makeValidResponse());
+    const raw = `${json}\n\nHope this helps!`;
+    const result = parseLlmResponse(raw);
+    expect(result.demos).toHaveLength(1);
+  });
+
+  test("throws when no JSON object found", () => {
+    expect(() => parseLlmResponse("not json at all")).toThrow("No JSON object found");
   });
 
   test("throws when demos array is missing", () => {
-    expect(() => parseReviewMetadata("{}")).toThrow("Missing 'demos' array");
+    expect(() => parseLlmResponse("{}")).toThrow("Missing 'demos' array");
   });
 
   test("throws when demos is not an array", () => {
-    expect(() => parseReviewMetadata('{"demos": "nope"}')).toThrow(
+    expect(() => parseLlmResponse('{"demos": "nope"}')).toThrow(
       "'demos' must be an array",
     );
   });
 
   test("throws when demo is missing file", () => {
     expect(() =>
-      parseReviewMetadata(
-        JSON.stringify({ demos: [{ summary: "x", annotations: [] }] }),
+      parseLlmResponse(
+        JSON.stringify({ demos: [{ summary: "x" }], review: makeValidResponse().review }),
       ),
     ).toThrow("'file' string");
   });
 
   test("throws when demo is missing summary", () => {
     expect(() =>
-      parseReviewMetadata(
-        JSON.stringify({ demos: [{ file: "x", annotations: [] }] }),
+      parseLlmResponse(
+        JSON.stringify({ demos: [{ file: "x" }], review: makeValidResponse().review }),
       ),
     ).toThrow("'summary' string");
   });
 
-  test("throws when annotations is missing", () => {
+  test("throws when review object is missing", () => {
     expect(() =>
-      parseReviewMetadata(
+      parseLlmResponse(
         JSON.stringify({ demos: [{ file: "x", summary: "s" }] }),
       ),
-    ).toThrow("'annotations' array");
+    ).toThrow("Missing 'review' object");
   });
 
-  test("throws when annotation has wrong timestampSeconds type", () => {
-    expect(() =>
-      parseReviewMetadata(
-        JSON.stringify({
-          demos: [
-            {
-              file: "x",
-              summary: "s",
-              annotations: [{ timestampSeconds: "0", text: "t" }],
-            },
-          ],
-        }),
-      ),
-    ).toThrow("'timestampSeconds' number");
+  test("throws when review.verdict is invalid", () => {
+    const input = makeValidResponse({
+      review: { ...makeValidResponse().review, verdict: "maybe" },
+    });
+    expect(() => parseLlmResponse(JSON.stringify(input))).toThrow(
+      "review.verdict must be 'approve' or 'request_changes'",
+    );
   });
 
-  test("throws when annotation has wrong text type", () => {
-    expect(() =>
-      parseReviewMetadata(
-        JSON.stringify({
-          demos: [
-            {
-              file: "x",
-              summary: "s",
-              annotations: [{ timestampSeconds: 0, text: 123 }],
-            },
-          ],
-        }),
-      ),
-    ).toThrow("'text' string");
+  test("throws when review.highlights is empty", () => {
+    const input = makeValidResponse({
+      review: { ...makeValidResponse().review, highlights: [] },
+    });
+    expect(() => parseLlmResponse(JSON.stringify(input))).toThrow(
+      "review.highlights must not be empty",
+    );
+  });
+
+  test("throws when review.summary is missing", () => {
+    const input = makeValidResponse({
+      review: { ...makeValidResponse().review, summary: 42 },
+    });
+    expect(() => parseLlmResponse(JSON.stringify(input))).toThrow(
+      "review.summary must be a string",
+    );
+  });
+
+  test("throws when review.verdictReason is missing", () => {
+    const input = makeValidResponse({
+      review: { ...makeValidResponse().review, verdictReason: null },
+    });
+    expect(() => parseLlmResponse(JSON.stringify(input))).toThrow(
+      "review.verdictReason must be a string",
+    );
+  });
+
+  test("throws when issue has invalid severity", () => {
+    const input = makeValidResponse({
+      review: {
+        ...makeValidResponse().review,
+        issues: [{ severity: "critical", description: "bad" }],
+      },
+    });
+    expect(() => parseLlmResponse(JSON.stringify(input))).toThrow(
+      "Each issue severity must be 'major', 'minor', or 'nit'",
+    );
+  });
+
+  test("throws when issue is missing description", () => {
+    const input = makeValidResponse({
+      review: {
+        ...makeValidResponse().review,
+        issues: [{ severity: "major" }],
+      },
+    });
+    expect(() => parseLlmResponse(JSON.stringify(input))).toThrow(
+      "Each issue must have a 'description' string",
+    );
   });
 });
 

package/src/review.ts

@@ -1,5 +1,3 @@
-import type { ReviewMetadata } from "./review-types.ts";
-
 export type SpawnFn = (
   cmd: string[],
 ) => { exitCode: Promise<number>; stdout: ReadableStream<Uint8Array> };
@@ -9,33 +7,84 @@ export interface InvokeClaudeOptions {
   spawn?: SpawnFn;
 }
 
-export function buildReviewPrompt(filenames: string[]): string {
-  const fileList = filenames.map((f) => `- ${f}`).join("\n");
+const GIT_DIFF_MAX_CHARS = 50_000;
+
+export interface BuildReviewPromptOptions {
+  filenames: string[];
+  stepsMap: Record<string, Array<{ text: string; timestampSeconds: number }>>;
+  gitDiff?: string;
+  guidelines?: string[];
+}
+
+export function buildReviewPrompt(options: BuildReviewPromptOptions): string {
+  const { filenames, stepsMap, gitDiff, guidelines } = options;
 
-  return `You are given the following .webm demo video filenames:
+  const demoEntries = filenames.map((f) => {
+    const steps = stepsMap[f] ?? [];
+    const stepLines = steps
+      .map((s) => `- [${s.timestampSeconds}s] ${s.text}`)
+      .join("\n");
+    return `Video: ${f}\nRecorded steps:\n${stepLines || "(no steps recorded)"}`;
+  });
 
-${fileList}
+  const sections: string[] = [];
 
-Based on the filenames, generate a JSON object matching this exact schema:
+  if (guidelines && guidelines.length > 0) {
+    sections.push(`## Coding Guidelines\n\n${guidelines.join("\n\n")}`);
+  }
+
+  if (gitDiff) {
+    let diff = gitDiff;
+    if (diff.length > GIT_DIFF_MAX_CHARS) {
+      diff = diff.slice(0, GIT_DIFF_MAX_CHARS) + "\n\n... (diff truncated at 50k characters)";
+    }
+    sections.push(`## Git Diff\n\n\`\`\`diff\n${diff}\n\`\`\``);
+  }
+
+  sections.push(`## Demo Recordings\n\n${demoEntries.join("\n\n")}`);
+
+  return `You are a code reviewer. You are given a git diff, coding guidelines, and demo recordings that show the feature in action.
+
+${sections.join("\n\n")}
+
+## Task
+
+Review the code changes and demo recordings. Generate a JSON object matching this exact schema:
 
 {
   "demos": [
     {
       "file": "<filename>",
-      "summary": "<a short sentence describing what the demo likely shows>",
-      "annotations": [
-        { "timestampSeconds": <number>, "text": "<annotation text>" }
-      ]
+      "summary": "<a meaningful sentence describing what this demo showcases based on the steps>"
     }
-  ]
+  ],
+  "review": {
+    "summary": "<2-3 sentence overview of the changes>",
+    "highlights": ["<positive aspect 1>", "<positive aspect 2>"],
+    "verdict": "approve" | "request_changes",
+    "verdictReason": "<one sentence justifying the verdict>",
+    "issues": [
+      {
+        "severity": "major" | "minor" | "nit",
+        "description": "<what the issue is and how to fix it>"
+      }
+    ]
+  }
 }
 
 Rules:
 - Return ONLY the JSON object, no markdown fences or extra text.
 - Include one entry in "demos" for each filename, in the same order.
-- Infer the summary and annotations from the filename.
-- Each demo should have at least one annotation starting at timestampSeconds 0.
-- "file" must exactly match the provided filename.`;
+- "file" must exactly match the provided filename.
+- "verdict" must be exactly "approve" or "request_changes".
+- Use "request_changes" if there are any "major" issues.
+- "severity" must be exactly "major", "minor", or "nit".
+- "major": bugs, security issues, broken functionality, guideline violations.
+- "minor": code quality, readability, missing edge cases.
+- "nit": style, naming, trivial improvements.
+- "highlights" must have at least one entry.
+- "issues" can be an empty array if there are no issues.
+- Verify that demo steps demonstrate the acceptance criteria being met.`;
 }
 
 export async function invokeClaude(
@@ -68,10 +117,46 @@ export async function invokeClaude(
   return output.trim();
 }
 
-export function parseReviewMetadata(raw: string): ReviewMetadata {
+import type { IssueSeverity, ReviewVerdict } from "./review-types.ts";
+
+export interface LlmReviewResponse {
+  demos: Array<{ file: string; summary: string }>;
+  review: {
+    summary: string;
+    highlights: string[];
+    verdict: ReviewVerdict;
+    verdictReason: string;
+    issues: Array<{ severity: IssueSeverity; description: string }>;
+  };
+}
+
+const VALID_VERDICTS: ReadonlySet<string> = new Set(["approve", "request_changes"]);
+const VALID_SEVERITIES: ReadonlySet<string> = new Set(["major", "minor", "nit"]);
+
+export function extractJson(raw: string): string {
+  // Try raw string first
+  try {
+    JSON.parse(raw);
+    return raw;
+  } catch {
+    // look for first { and last }
+  }
+
+  const start = raw.indexOf("{");
+  const end = raw.lastIndexOf("}");
+  if (start === -1 || end === -1 || end <= start) {
+    throw new Error(`No JSON object found in LLM response: ${raw.slice(0, 200)}`);
+  }
+
+  return raw.slice(start, end + 1);
+}
+
+export function parseLlmResponse(raw: string): LlmReviewResponse {
+  const jsonStr = extractJson(raw);
+
   let parsed: unknown;
   try {
-    parsed = JSON.parse(raw);
+    parsed = JSON.parse(jsonStr);
   } catch {
     throw new Error(`Invalid JSON from LLM: ${raw.slice(0, 200)}`);
   }
@@ -97,39 +182,76 @@ export function parseReviewMetadata(raw: string): ReviewMetadata {
     if (typeof d["summary"] !== "string") {
       throw new Error("Each demo must have a 'summary' string");
     }
-    if (!Array.isArray(d["annotations"])) {
-      throw new Error("Each demo must have an 'annotations' array");
+  }
+
+  if (typeof obj["review"] !== "object" || obj["review"] === null) {
+    throw new Error("Missing 'review' object in response");
+  }
+
+  const review = obj["review"] as Record<string, unknown>;
+
+  if (typeof review["summary"] !== "string") {
+    throw new Error("review.summary must be a string");
+  }
+
+  if (!Array.isArray(review["highlights"])) {
+    throw new Error("review.highlights must be an array");
+  }
+  if (review["highlights"].length === 0) {
+    throw new Error("review.highlights must not be empty");
+  }
+  for (const h of review["highlights"]) {
+    if (typeof h !== "string") {
+      throw new Error("Each highlight must be a string");
     }
+  }
 
-    for (const ann of d["annotations"] as unknown[]) {
-      if (typeof ann !== "object" || ann === null) {
-        throw new Error("Each annotation must be an object");
-      }
-      const a = ann as Record<string, unknown>;
-      if (typeof a["timestampSeconds"] !== "number") {
-        throw new Error("Each annotation must have a 'timestampSeconds' number");
-      }
-      if (typeof a["text"] !== "string") {
-        throw new Error("Each annotation must have a 'text' string");
-      }
+  if (typeof review["verdict"] !== "string" || !VALID_VERDICTS.has(review["verdict"])) {
+    throw new Error("review.verdict must be 'approve' or 'request_changes'");
+  }
+
+  if (typeof review["verdictReason"] !== "string") {
+    throw new Error("review.verdictReason must be a string");
+  }
+
+  if (!Array.isArray(review["issues"])) {
+    throw new Error("review.issues must be an array");
+  }
+
+  for (const issue of review["issues"] as unknown[]) {
+    if (typeof issue !== "object" || issue === null) {
+      throw new Error("Each issue must be an object");
+    }
+    const i = issue as Record<string, unknown>;
+    if (typeof i["severity"] !== "string" || !VALID_SEVERITIES.has(i["severity"])) {
+      throw new Error("Each issue severity must be 'major', 'minor', or 'nit'");
+    }
+    if (typeof i["description"] !== "string") {
+      throw new Error("Each issue must have a 'description' string");
     }
   }
 
-  return parsed as ReviewMetadata;
+  return parsed as LlmReviewResponse;
 }
 
+import { spawn } from "node:child_process";
+import { Readable } from "node:stream";
+
 function defaultSpawn(
   cmd: string[],
 ): { exitCode: Promise<number>; stdout: ReadableStream<Uint8Array> } {
   const [command, ...args] = cmd;
-  const proc = Bun.spawn([command!, ...args], {
-    stdout: "pipe",
-    stderr: "pipe",
+  const proc = spawn(command!, args, {
+    stdio: ["ignore", "pipe", "pipe"],
   });
-  return {
-    exitCode: proc.exited,
-    stdout: proc.stdout as unknown as ReadableStream<Uint8Array>,
-  };
+
+  const exitCode = new Promise<number>((resolve) => {
+    proc.on("close", (code) => resolve(code ?? 1));
+  });
+
+  const stdout = Readable.toWeb(proc.stdout!) as unknown as ReadableStream<Uint8Array>;
+
+  return { exitCode, stdout };
 }
 
 function concatUint8Arrays(arrays: Uint8Array[]): Uint8Array {