@capagents/forgeqe 0.0.0-stage → 1.0.0

package/src/code/review.ts

@@ -0,0 +1,202 @@
+import { readFileSync, statSync } from "node:fs"
+import path from "node:path"
+
+import type { CodeEvent, CodeReview, Usage } from "../types.ts"
+import { privateEnvNames } from "../childEnv.ts"
+import type { AppConfig } from "../config.ts"
+import type { Handoff } from "./handoff.ts"
+import { openSession, type AgentSpec, type OpencodeSession } from "./runtime.ts"
+import type { AgentReport, ReviewIssue } from "./prompt.ts"
+import { loadSkills } from "../skills/registry.ts"
+import type { Price } from "../usage.ts"
+import { prompt } from "../prompts.ts"
+import { domainNotes } from "../domain.ts"
+
+const OUTPUT_CHARS = 4_000
+/** Changed files are shown in the prompt up to these sizes; the reviewer reads the rest itself. */
+const FILE_CHARS = 12_000
+const FILES_CHARS = 40_000
+
+export const reviewAgentPrompt = () => prompt("test-reviewer")
+
+export const REVIEW_AGENT: AgentSpec = {
+  name: "forgeqe-review",
+  description: "Checks a test ForgeQE's coding agent wrote against the goal and the test rules, without changing anything",
+  get prompt() {
+    return reviewAgentPrompt()
+  },
+  permission: {
+    edit: "deny",
+    bash: { "*": "deny", "git diff*": "allow", "git status*": "allow", "git log*": "allow", "git show*": "allow", "ls*": "allow" },
+    webfetch: "deny",
+    task: "deny",
+  },
+  skills: "scribe",
+  // A review reads the changed files it is given and a few around them; it never needs a long turn.
+  steps: 25,
+}
+
+export const REVIEW_SCHEMA = {
+  type: "object",
+  properties: {
+    approved: { type: "boolean", description: "true only when the test does what the goal says and keeps to the rules" },
+    summary: { type: "string", description: "One or two sentences on the verdict" },
+    issues: {
+      type: "array",
+      items: {
+        type: "object",
+        properties: {
+          file: { type: "string", description: "Path relative to the repository root" },
+          problem: { type: "string", description: "What is wrong" },
+          fix: { type: "string", description: "What to change" },
+          kind: { type: "string", enum: ["correctness", "standards"], description: "correctness: it does not test what the goal says, or not reliably. standards: it works, but not in the repository's way" },
+        },
+        required: ["problem", "kind"],
+      },
+    },
+  },
+  required: ["approved", "summary", "issues"],
+} as const
+
+type ReviewReport = { approved?: boolean; summary?: string; issues?: ReviewIssue[] }
+
+export type ReviewInput = {
+  handoff: Handoff
+  report: AgentReport
+  output: string
+  browsers: string[]
+  seconds: number
+  /** Paths this job changed, from git; absent outside git. */
+  changes?: string[]
+  /** Uncommitted paths the user had before the job. */
+  before?: string[]
+  /** This reviewer's earlier verdict on this job, when the agent has since changed the test to answer it. */
+  earlier?: CodeReview
+}
+
+const LIST_MAX = 60
+const list = (paths: string[]) => (paths.length > LIST_MAX ? `${paths.slice(0, LIST_MAX).join(", ")}, and ${paths.length - LIST_MAX} more` : paths.join(", "))
+
+/** A review of one passing attempt; `error` when the reviewer gave no verdict. */
+export type ReviewOutcome = { review?: CodeReview; error?: string }
+
+/** `price`: the reviewer's model price (its own profile with code.review_profile); unset when unknown. */
+export type Reviewer = { review(input: ReviewInput): Promise<ReviewOutcome>; close(): Promise<void>; usage: Usage; price?: Price }
+
+/** The changed files' contents for the prompt, each up to FILE_CHARS and all up to FILES_CHARS. */
+export function changedFiles(repo: string, paths: string[]): string[] {
+  const lines: string[] = []
+  let used = 0
+  for (const rel of paths) {
+    let text: string
+    try {
+      const file = path.join(repo, rel)
+      if (!statSync(file).isFile()) continue
+      text = readFileSync(file, "utf8")
+    } catch {
+      lines.push(`### ${rel}`, "(deleted)", "")
+      continue
+    }
+    if (used >= FILES_CHARS) {
+      lines.push(`### ${rel}`, "(not shown here; read it)", "")
+      continue
+    }
+    const room = Math.min(FILE_CHARS, FILES_CHARS - used)
+    const shown = text.length > room ? `${text.slice(0, room)}\n… (the rest is not shown here; read the file)` : text
+    used += shown.length
+    lines.push(`### ${rel}`, "```", shown, "```", "")
+  }
+  return lines
+}
+
+export function reviewPrompt(input: ReviewInput, repo?: string): string {
+  const { handoff, report } = input
+  const out = input.output.length > OUTPUT_CHARS ? `…${input.output.slice(-OUTPUT_CHARS)}` : input.output
+  const changed = input.changes || report.changed || []
+  const checks = handoff.markdown.split("\n## Checks the browser agent made\n")[1]?.split("\n## ")[0]?.trim()
+  return [
+    `Review the test the coding agent wrote for this goal: ${handoff.goal}`,
+    `The handoff it worked from: ${handoff.file}`,
+    ...(checks ? ["", "The browser agent's checks, from the handoff:", checks] : []),
+    "",
+    `Test file: ${report.test_file}`,
+    `Command: ${report.command}`,
+    input.changes
+      ? `Files this job changed (git, against the working tree before the job): ${list(input.changes) || "(none)"}`
+      : `Files it reports changing: ${(report.changed || []).join(", ") || "(none reported)"}`,
+    input.before?.length ? `Already modified or untracked before the job, the user's work and not under review: ${list(input.before)}` : "",
+    report.notes ? `Its notes: ${report.notes}` : "",
+    "",
+    `ForgeQE ran the command: it passed in ${input.seconds}s with ${input.browsers.length ? input.browsers.join(", ") : `a browser on another machine (${report.remote_browser || "remote"})`}. End of the output:`,
+    "```",
+    out,
+    "```",
+    ...(repo && changed.length ? ["", "The changed files as they are now:", "", ...changedFiles(repo, changed)] : []),
+    ...(input.earlier
+      ? [
+          "",
+          "This is a follow-up review. Your earlier review raised these issues, and the coding agent has changed the test to answer them:",
+          ...input.earlier.issues.map((i, n) => `${n + 1}. [${i.kind || "correctness"}] ${i.file ? `${i.file}: ` : ""}${i.problem}`),
+          "Check each one: fixed, or still there. Raise a new correctness issue only when this change introduced it (it broke or weakened what the test checks); anything else you notice now is a standards issue at most. The test case and the handoff decide what the test must do: do not ask for more than they say.",
+        ]
+      : []),
+  ]
+    .filter((l, i, all) => l !== "" || all[i - 1] !== "")
+    .join("\n")
+}
+
+/** A read-only Scribe session, opened on the first review and kept for the next attempts. */
+export function openReviewer(opts: {
+  config: AppConfig
+  repo: string
+  readDirs: string[]
+  signal?: AbortSignal
+  onEvent?: (e: CodeEvent) => void
+  open?: (o: Parameters<typeof openSession>[0]) => Promise<OpencodeSession>
+}): Reviewer {
+  let session: OpencodeSession<ReviewReport> | null = null
+  const usage: Usage = { input: 0, output: 0 }
+  // code.review_profile: a cheaper llms.json profile for reading and judging; else the run's profile (or code.model).
+  const named = opts.config.code.reviewProfile ? opts.config.profiles[opts.config.code.reviewProfile] : undefined
+  const reviewProfile = named || opts.config.llm
+  const price = named ? named.price : opts.config.code.model ? undefined : opts.config.llm.price
+  const emit = opts.onEvent || (() => {})
+  return {
+    usage,
+    price,
+    async review(input) {
+      if (!session) {
+        const open = (opts.open || openSession) as (o: Parameters<typeof openSession>[0]) => Promise<OpencodeSession<ReviewReport>>
+        session = await open({
+          repo: opts.repo,
+          profile: reviewProfile,
+          domain: domainNotes(opts.config),
+          model: reviewProfile === opts.config.llm ? opts.config.code.model : undefined,
+          bin: opts.config.code.opencode,
+          readDirs: opts.readDirs,
+          hide: privateEnvNames(opts.config),
+          role: "reviewer",
+          signal: opts.signal,
+          onEvent: (e) => {
+            if (e.kind === "tool" || e.kind === "say") emit({ ...e, text: `review: ${e.text}` })
+            else if (e.kind === "info") emit(e)
+          },
+          agent: REVIEW_AGENT,
+          skills: loadSkills(opts.config.skills, opts.repo).skills,
+          schema: REVIEW_SCHEMA as unknown as Record<string, unknown>,
+          title: "ForgeQE review",
+        })
+      }
+      let reply = await session.prompt(reviewPrompt(input, opts.repo))
+      if (!reply.report && !reply.error) reply = await session.prompt("Give the structured result: approved, summary, and issues.")
+      Object.assign(usage, session.usage)
+      const r = reply.report
+      if (!r || typeof r.approved !== "boolean") return { error: reply.error || "the reviewer gave no verdict" }
+      const issues = (r.issues || []).filter((i) => i?.problem).map((i) => ({ ...i, ...(i.kind === "standards" || i.kind === "correctness" ? {} : { kind: "correctness" as const }) }))
+      return { review: { approved: r.approved, summary: String(r.summary || ""), issues } }
+    },
+    async close() {
+      if (session) await session.close()
+    },
+  }
+}