@capagents/forgeqe 0.0.0-stage → 1.0.0

package/src/evals/domain.ts

@@ -0,0 +1,338 @@
+import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import os from "node:os"
+import path from "node:path"
+
+import YAML from "yaml"
+
+import { openBrain, type BrainOptions } from "../brain/index.ts"
+import { openSession, type OpencodeSession } from "../code/runtime.ts"
+import type { AppConfig, DomainConfig } from "../config.ts"
+import { prompt } from "../prompts.ts"
+import type { CodeEvent, Usage } from "../types.ts"
+
+/**
+ * Whether ForgeQE's agent brings domain and regulatory knowledge to test design: each case is a realistic story whose
+ * acceptance criteria leave the industry's risks and the regulations' obligations unstated. ForgeQE's own agent writes
+ * test cases for it, once with the domain declared (domain: and a compliance file, as a team would set them) and once
+ * without; a separate judge session scores both against what a domain expert expects them to cover, and lists every
+ * control or article id they cite, so ids not in the team's compliance file show up as invented.
+ */
+
+export const DOMAIN_CASES = path.resolve(import.meta.dir, "..", "..", "evals", "domain")
+
+export type Source = "story" | "domain" | "regulation"
+export type Expectation = { id: string; source: Source; covers: string }
+export type DomainCase = {
+  id: string
+  title: string
+  domain: DomainConfig
+  /** The team's compliance file for this product (markdown), when it has one. */
+  compliance?: string
+  story: { title: string; description: string; acceptance: string }
+  expect: Expectation[]
+}
+export type Variant = "declared" | "undeclared"
+export const VARIANTS: Variant[] = ["declared", "undeclared"]
+const SOURCES: Source[] = ["story", "domain", "regulation"]
+
+/** Every case file in a folder (or one file), checked so a broken case fails before any model is called. */
+export function loadDomainCases(where: string): DomainCase[] {
+  const files = where.endsWith(".yaml") || where.endsWith(".yml") ? [where] : readdirSync(where).filter((f) => /\.ya?ml$/.test(f)).sort().map((f) => path.join(where, f))
+  if (!files.length) throw new Error(`No evaluation cases (.yaml) in ${where}.`)
+  const cases = files.map((file) => {
+    const raw = YAML.parse(readFileSync(file, "utf8")) as Partial<DomainCase>
+    const where = path.basename(file)
+    if (!raw?.id || !raw.title || !raw.story?.title || !raw.domain?.industry) throw new Error(`${where}: id, title, domain.industry, and story.title are required.`)
+    if (!Array.isArray(raw.expect) || !raw.expect.length) throw new Error(`${where}: expect needs at least one expectation.`)
+    for (const e of raw.expect) {
+      if (!e?.id || !e.covers || !SOURCES.includes(e.source as Source)) throw new Error(`${where}: each expectation needs id, covers, and source (${SOURCES.join(", ")}).`)
+    }
+    const ids = raw.expect.map((e) => e.id)
+    if (new Set(ids).size !== ids.length) throw new Error(`${where}: expectation ids must be unique.`)
+    return {
+      id: String(raw.id),
+      title: String(raw.title),
+      domain: { industry: String(raw.domain.industry), regulations: (raw.domain.regulations || []).map(String), ...(raw.domain.notes ? { notes: String(raw.domain.notes) } : {}) },
+      ...(raw.compliance ? { compliance: String(raw.compliance) } : {}),
+      story: { title: String(raw.story.title), description: String(raw.story.description || ""), acceptance: String(raw.story.acceptance || "") },
+      expect: raw.expect.map((e) => ({ id: String(e.id), source: e.source as Source, covers: String(e.covers) })),
+    }
+  })
+  const dup = cases.map((c) => c.id).filter((id, i, all) => all.indexOf(id) !== i)
+  if (dup.length) throw new Error(`Duplicate case ids: ${[...new Set(dup)].join(", ")}.`)
+  return cases
+}
+
+export function storyRequest(c: DomainCase): string {
+  return [
+    "Write the test cases for this story, as a test analyst would before testing starts. Do not save them anywhere, run anything, or open a browser: reply with the test cases only, each with a title, steps, and expected results.",
+    "",
+    `# ${c.story.title}`,
+    "",
+    c.story.description,
+    "",
+    "## Acceptance criteria",
+    "",
+    c.story.acceptance,
+  ].join("\n")
+}
+
+/** The config the agent runs with: no systems to save into, and the case's domain only when declared. */
+export function caseConfig(base: AppConfig, c: DomainCase, variant: Variant, knowledgeDir: string): AppConfig {
+  return {
+    ...base,
+    mcp: undefined,
+    sources: { ...base.sources, ado: undefined, jira: undefined, testrail: undefined, zephyr: undefined, qtest: undefined, confluence: undefined, bugs: undefined, writeBack: false, saveCases: false, xray: { ...base.sources.xray, url: "" } },
+    knowledge: { dir: knowledgeDir, browserMaxChars: 0 },
+    domain: variant === "declared" ? c.domain : undefined,
+  }
+}
+
+export const JUDGE_SCHEMA = {
+  type: "object",
+  properties: {
+    items: {
+      type: "array",
+      items: {
+        type: "object",
+        properties: { id: { type: "string" }, covered: { type: "boolean" }, evidence: { type: "string" } },
+        required: ["id", "covered", "evidence"],
+      },
+    },
+    cited_ids: { type: "array", items: { type: "string" } },
+  },
+  required: ["items", "cited_ids"],
+} as const
+
+export type Judgement = { items: { id: string; covered: boolean; evidence: string }[]; cited_ids: string[] }
+
+export function judgeRequest(c: DomainCase, variant: Variant, testCases: string): string {
+  return [
+    `# Story: ${c.story.title}`,
+    c.story.description,
+    "",
+    "## Acceptance criteria",
+    c.story.acceptance,
+    "",
+    variant === "declared"
+      ? `## Declared domain\n${c.domain.industry}; regulations in scope: ${c.domain.regulations.join(", ") || "none named"}.`
+      : "## Declared domain\nNone was declared.",
+    "",
+    variant === "declared" && c.compliance ? `## The team's compliance file\n${c.compliance.trim()}` : "## The team's compliance file\nThere is none.",
+    "",
+    "## Expectations",
+    ...c.expect.map((e) => `- ${e.id}: ${e.covers}`),
+    "",
+    "## The test cases to judge",
+    testCases.trim() || "(the agent wrote none)",
+  ].join("\n")
+}
+
+export type Coverage = Record<Source, { covered: number; total: number }>
+
+export type CaseRun = {
+  case: string
+  variant: Variant
+  repeat: number
+  testCases: string
+  items: { id: string; source: Source; covered: boolean; evidence: string }[]
+  coverage: Coverage
+  /** Ids the test cases cite that the team's compliance file does not list. */
+  invented: string[]
+  cited: string[]
+  error?: string
+}
+
+export function score(c: DomainCase, variant: Variant, j: Judgement): Pick<CaseRun, "items" | "coverage" | "invented" | "cited"> {
+  const items = c.expect.map((e) => {
+    const got = j.items.find((i) => i.id === e.id)
+    return { id: e.id, source: e.source, covered: Boolean(got?.covered), evidence: got?.evidence || "not judged" }
+  })
+  const coverage = Object.fromEntries(
+    SOURCES.map((s) => {
+      const of = items.filter((i) => i.source === s)
+      return [s, { covered: of.filter((i) => i.covered).length, total: of.length }]
+    }),
+  ) as Coverage
+  const allowed = variant === "declared" ? c.compliance || "" : ""
+  const cited = [...new Set(j.cited_ids.map((x) => x.trim()).filter(Boolean))]
+  return { items, coverage, cited, invented: cited.filter((id) => !allowed.includes(id)) }
+}
+
+type Opener<R> = (o: Parameters<typeof openSession>[0]) => Promise<OpencodeSession<R>>
+
+export type EvalOptions = {
+  config: AppConfig
+  cases: DomainCase[]
+  variants?: Variant[]
+  repeat?: number
+  /** The judge's config (another profile); default: the agent's. */
+  judgeConfig?: AppConfig
+  signal?: AbortSignal
+  onProgress?: (text: string) => void
+  onEvent?: (e: CodeEvent) => void
+  /** Replace the agent and the judge sessions (tests). */
+  openAgent?: BrainOptions["open"]
+  openJudge?: Opener<Judgement>
+}
+
+export type EvalReport = {
+  at: string
+  model: string
+  judge: string
+  runs: CaseRun[]
+  /** Per variant, over every run: coverage by source, and invented ids. */
+  summary: Record<Variant, { coverage: Coverage; invented: number; errors: number }>
+  usage: Usage
+}
+
+const JUDGE_AGENT = {
+  name: "forgeqe-judge",
+  description: "Scores test cases against what a domain expert expects them to cover",
+  get prompt() {
+    return prompt("eval-judge")
+  },
+  permission: { edit: "deny", bash: "deny", webfetch: "deny" },
+}
+
+function addUsage(to: Usage, u: Usage) {
+  to.input += u.input || 0
+  to.output += u.output || 0
+  if (u.cached) to.cached = (to.cached || 0) + u.cached
+}
+
+export async function runDomainEval(opts: EvalOptions): Promise<EvalReport> {
+  const variants = opts.variants?.length ? opts.variants : VARIANTS
+  const repeat = Math.max(1, opts.repeat || 1)
+  const judgeConfig = opts.judgeConfig || opts.config
+  const usage: Usage = { input: 0, output: 0 }
+  const runs: CaseRun[] = []
+  let model = ""
+  let judgeModel = ""
+  const total = opts.cases.length * variants.length * repeat
+  for (const c of opts.cases) {
+    for (const variant of variants) {
+      for (let r = 1; r <= repeat; r++) {
+        if (opts.signal?.aborted) break
+        opts.onProgress?.(`[${runs.length + 1}/${total}] ${c.id} · ${variant}${repeat > 1 ? ` · run ${r}` : ""}`)
+        const dir = mkdtempSync(path.join(os.tmpdir(), "wp-eval-"))
+        const knowledge = path.join(dir, "knowledge")
+        mkdirSync(knowledge)
+        if (variant === "declared" && c.compliance) writeFileSync(path.join(knowledge, "compliance.md"), `# Compliance\n\n${c.compliance.trim()}\n`)
+        const config = caseConfig(opts.config, c, variant, knowledge)
+        let testCases = ""
+        try {
+          const brain = await openBrain({ config, cwd: dir, context: () => ({ session: "once", writeTests: false }), signal: opts.signal, onEvent: opts.onEvent, open: opts.openAgent, external: { servers: [], skipped: [] } })
+          try {
+            model = brain.model
+            const answer = await brain.ask(storyRequest(c))
+            if (answer.outcome === "failed" || answer.outcome === "stopped") throw new Error(answer.error || `the agent ${answer.outcome}`)
+            testCases = answer.reply
+          } finally {
+            addUsage(usage, brain.usage)
+            await brain.close()
+          }
+          const open = (opts.openJudge || openSession) as Opener<Judgement>
+          const judge = await open({
+            repo: dir,
+            profile: judgeConfig.llm,
+            // The judge is given the domain as data and must not apply the domain rule itself.
+            domain: null,
+            role: "judge",
+            model: judgeConfig.code.model,
+            bin: judgeConfig.code.opencode,
+            readDirs: [dir],
+            agent: JUDGE_AGENT,
+            schema: JUDGE_SCHEMA as unknown as Record<string, unknown>,
+            signal: opts.signal,
+            title: "ForgeQE evaluation judge",
+          })
+          try {
+            judgeModel = judge.model
+            let out = await judge.prompt(judgeRequest(c, variant, testCases))
+            if (!out.report && !out.error) out = await judge.prompt("Give the structured result: items and cited_ids.")
+            if (!out.report) throw new Error(`the judge gave no result${out.error ? `: ${out.error}` : ""}`)
+            runs.push({ case: c.id, variant, repeat: r, testCases, ...score(c, variant, out.report) })
+          } finally {
+            addUsage(usage, judge.usage)
+            await judge.close()
+          }
+        } catch (err) {
+          const empty = score(c, variant, { items: [], cited_ids: [] })
+          runs.push({ case: c.id, variant, repeat: r, testCases, ...empty, error: err instanceof Error ? err.message : String(err) })
+        } finally {
+          rmSync(dir, { recursive: true, force: true })
+        }
+      }
+    }
+  }
+  return { at: new Date().toISOString(), model, judge: judgeModel, runs, summary: summarise(runs, variants), usage }
+}
+
+export function summarise(runs: CaseRun[], variants: Variant[] = VARIANTS): EvalReport["summary"] {
+  const out = {} as EvalReport["summary"]
+  for (const v of variants) {
+    const of = runs.filter((r) => r.variant === v)
+    const coverage = Object.fromEntries(
+      SOURCES.map((s) => [s, of.reduce((a, r) => ({ covered: a.covered + r.coverage[s].covered, total: a.total + r.coverage[s].total }), { covered: 0, total: 0 })]),
+    ) as Coverage
+    out[v] = { coverage, invented: of.reduce((n, r) => n + r.invented.length, 0), errors: of.filter((r) => r.error).length }
+  }
+  return out
+}
+
+const pct = (x: { covered: number; total: number }) => (x.total ? `${Math.round((100 * x.covered) / x.total)}%` : "–")
+
+/** The report as Markdown: the comparison first, then each case's misses. */
+export function reportMarkdown(r: EvalReport): string {
+  const variants = Object.keys(r.summary) as Variant[]
+  const lines = [
+    `# Domain evaluation`,
+    "",
+    `${r.at} · agent ${r.model || "?"} · judge ${r.judge || "?"} · ${r.runs.length} runs · ${r.usage.input + r.usage.output} tokens`,
+    "",
+    `| | ${variants.join(" | ")} |`,
+    `|---|${variants.map(() => "---").join("|")}|`,
+    ...SOURCES.map((s) => `| ${s} coverage | ${variants.map((v) => pct(r.summary[v].coverage[s])).join(" | ")} |`),
+    `| invented ids | ${variants.map((v) => r.summary[v].invented).join(" | ")} |`,
+    `| errors | ${variants.map((v) => r.summary[v].errors).join(" | ")} |`,
+    "",
+  ]
+  for (const run of r.runs) {
+    lines.push(`## ${run.case} · ${run.variant}${run.repeat > 1 ? ` · run ${run.repeat}` : ""}`)
+    if (run.error) lines.push("", `Error: ${run.error}`)
+    lines.push("", SOURCES.map((s) => `${s} ${run.coverage[s].covered}/${run.coverage[s].total}`).join(" · "))
+    const missed = run.items.filter((i) => !i.covered)
+    if (missed.length) lines.push("", "Missed:", ...missed.map((i) => `- ${i.id} (${i.source}): ${i.evidence}`))
+    if (run.invented.length) lines.push("", `Invented ids: ${run.invented.join(", ")}`)
+    lines.push("")
+  }
+  return lines.join("\n")
+}
+
+/** Writes report.json and report.md under <reports>/evals/domain/<time>/ and returns the folder. */
+export function saveReport(r: EvalReport, outDir: string): string {
+  const dir = path.join(outDir, "evals", "domain", r.at.replace(/[:.]/g, "-"))
+  mkdirSync(dir, { recursive: true })
+  writeFileSync(path.join(dir, "report.json"), JSON.stringify(r, null, 2))
+  writeFileSync(path.join(dir, "report.md"), reportMarkdown(r))
+  return dir
+}
+
+/**
+ * The gate for CI: the declared variant covers at least `min` of the domain and regulation expectations, cites no
+ * invented ids, and had no errors. Null when it passes, else why not.
+ */
+export function gate(r: EvalReport, min: number): string | null {
+  const d = r.summary.declared
+  if (!d) return "the declared variant did not run"
+  const covered = d.coverage.domain.covered + d.coverage.regulation.covered
+  const total = d.coverage.domain.total + d.coverage.regulation.total
+  const share = total ? covered / total : 1
+  const why = [
+    share < min ? `domain and regulation coverage ${Math.round(share * 100)}% is under ${Math.round(min * 100)}%` : "",
+    d.invented ? `${d.invented} invented id(s)` : "",
+    d.errors ? `${d.errors} run(s) failed` : "",
+  ].filter(Boolean)
+  return why.length ? why.join("; ") : null
+}

package/src/explore/charters.ts

@@ -0,0 +1,151 @@
+import type { AppConfig } from "../config.ts"
+import { privateEnvNames } from "../childEnv.ts"
+import type { CodeEvent, Usage } from "../types.ts"
+import { openSession, type AgentSpec, type OpencodeSession } from "../code/runtime.ts"
+import type { Story } from "./story.ts"
+import { domainNotes } from "../domain.ts"
+import { knowledgeBlock, readKnowledge, relevantKnowledge } from "../knowledge.ts"
+import { prompt } from "../prompts.ts"
+
+/**
+ * Exploratory charters: what to try on a story's feature beyond its scripted test cases. Each charter has a mission (the
+ * risk it probes), concrete steps, and an expected result for each step (the oracle: from the acceptance criteria, or
+ * what any well-behaved application does, like refusing an invalid email with a message). The browser agent then runs
+ * each charter and judges each expected result on the live page.
+ */
+
+export const CATEGORIES = ["variation", "edge_case", "invalid_input", "boundary", "role", "state", "negative"] as const
+export type Category = (typeof CATEGORIES)[number]
+
+export type Charter = {
+  id: string
+  title: string
+  category: Category
+  mission: string
+  /** The saved login to run as (a role), when the charter is about what a role may do. */
+  role?: string
+  /** The acceptance criterion it probes, when it probes one. */
+  criterion?: string
+  steps: { action: string; data?: string; expected: string }[]
+  /** Creates, changes, or deletes data in the application. */
+  changesData?: boolean
+}
+
+export const charterAgentPrompt = () => prompt("explore-charters")
+
+export const CHARTER_AGENT: AgentSpec = {
+  name: "forgeqe-explore-plan",
+  description: "Plans exploratory charters for a user story beyond its scripted test cases",
+  get prompt() {
+    return charterAgentPrompt()
+  },
+  permission: { edit: "deny", bash: "deny", webfetch: "deny", task: "deny", "live_*": "deny" },
+}
+
+export const CHARTER_SCHEMA = {
+  type: "object",
+  properties: {
+    charters: {
+      type: "array",
+      items: {
+        type: "object",
+        properties: {
+          title: { type: "string" },
+          category: { type: "string", enum: [...CATEGORIES] },
+          mission: { type: "string", description: "The risk this charter probes, in one sentence" },
+          role: { type: "string", description: "Only for role charters: the login name to run as, from the list" },
+          criterion: { type: "string", description: "The acceptance criterion it probes, quoted, when there is one" },
+          steps: {
+            type: "array",
+            items: { type: "object", properties: { action: { type: "string" }, data: { type: "string" }, expected: { type: "string" } }, required: ["action", "expected"] },
+          },
+          changes_data: { type: "boolean" },
+        },
+        required: ["title", "category", "mission", "steps"],
+      },
+    },
+    notes: { type: "string" },
+  },
+  required: ["charters"],
+} as const
+
+type CharterReport = { charters: (Omit<Charter, "id" | "changesData"> & { changes_data?: boolean })[]; notes?: string }
+
+const clip = (t: string, n: number) => (t.length > n ? `${t.slice(0, n - 1)}…` : t)
+
+export function charterPrompt(story: Story, opts: { count: number; roles: string[]; url?: string; knowledge?: string }): string {
+  const lines = [`# Story ${story.id === "story" ? "" : `${story.id}: `}${story.title}`]
+  if (story.url) lines.push(story.url)
+  if (story.description) lines.push("", clip(story.description, 3000))
+  if (story.acceptance) lines.push("", "## Acceptance criteria", "", clip(story.acceptance, 3000))
+  for (const p of story.parents) lines.push("", `## ${p.type} ${p.id}: ${p.title}`, clip(p.description || "", 800))
+  if (opts.knowledge) lines.push("", opts.knowledge)
+  if (story.passages.length) lines.push("", "## From the requirements search", "", ...story.passages.map((p) => `- ${clip(p, 600)}`))
+  if (story.scripted.length) {
+    lines.push("", "## Scripted test cases (already covered: go beyond them)")
+    for (const tc of story.scripted) lines.push("", `${tc.id}: ${tc.title}`, ...tc.steps.slice(0, 12).map((s, i) => `  ${i + 1}. ${s.action}${s.expected ? ` → ${s.expected}` : ""}`))
+  } else lines.push("", "No scripted test cases are linked to this story yet.")
+  lines.push(
+    "",
+    "## The session",
+    "",
+    `Start page: ${opts.url || "(not given: the browser agent finds the feature from the story)"}`,
+    opts.roles.length ? `Roles (saved logins the browser agent can sign in as): ${opts.roles.join(", ")}. Include role charters for the differences between them.` : "No roles: the browser agent uses the default sign-in. Plan no role charters.",
+    "",
+    `Plan ${opts.count} exploratory charters.`,
+  )
+  return lines.join("\n")
+}
+
+export async function planCharters(opts: {
+  config: AppConfig
+  story: Story
+  count: number
+  roles: string[]
+  url?: string
+  cwd: string
+  signal?: AbortSignal
+  onEvent?: (e: CodeEvent) => void
+  open?: (o: Parameters<typeof openSession>[0]) => Promise<OpencodeSession<CharterReport>>
+}): Promise<{ charters: Charter[]; notes?: string; usage: Usage; model: string }> {
+  const open = (opts.open || openSession) as (o: Parameters<typeof openSession>[0]) => Promise<OpencodeSession<CharterReport>>
+  const session = await open({
+    repo: opts.cwd,
+    profile: opts.config.llm,
+    domain: domainNotes(opts.config),
+    model: opts.config.code.model,
+    bin: opts.config.code.opencode,
+    readDirs: [opts.cwd],
+    hide: privateEnvNames(opts.config),
+    signal: opts.signal,
+    onEvent: opts.onEvent,
+    agent: CHARTER_AGENT,
+    schema: CHARTER_SCHEMA as unknown as Record<string, unknown>,
+    title: "ForgeQE exploratory charters",
+  })
+  try {
+    const k = opts.config.knowledge
+    const knowledge = k ? knowledgeBlock(relevantKnowledge(readKnowledge(k.dir), `${opts.story.title}\n${opts.story.description}\n${opts.story.acceptance}\n${opts.roles.join(" ")}`, 6000)) : ""
+    let reply = await session.prompt(charterPrompt(opts.story, { ...opts, knowledge }))
+    if (!reply.report && !reply.error && !opts.signal?.aborted) reply = await session.prompt("Give the structured result: charters.")
+    if (!reply.report?.charters?.length) throw new Error(`No charters were planned${reply.error ? `: ${reply.error}` : "."}`)
+    const roles = new Set(opts.roles)
+    const charters = reply.report.charters
+      .filter((c) => c.steps?.length)
+      .slice(0, opts.count)
+      .map((c, i): Charter => ({
+        id: `C${i + 1}`,
+        title: c.title,
+        category: (CATEGORIES as readonly string[]).includes(c.category) ? c.category : "variation",
+        mission: c.mission,
+        // A role that is not a saved login cannot be signed in as: the charter runs with the default sign-in.
+        ...(c.role && roles.has(c.role) ? { role: c.role } : {}),
+        ...(c.criterion ? { criterion: c.criterion } : {}),
+        steps: c.steps.slice(0, 8).map((s) => ({ action: s.action, ...(s.data ? { data: s.data } : {}), expected: s.expected })),
+        ...(c.changes_data ? { changesData: true } : {}),
+      }))
+    return { charters, ...(reply.report.notes ? { notes: reply.report.notes } : {}), usage: { ...session.usage }, model: session.model }
+  } finally {
+    await session.close()
+  }
+}