@capagents/forgeqe 1.2.1 → 1.3.1

package/src/mcp/packtools.ts

@@ -0,0 +1,187 @@
+import { realpathSync, writeFileSync } from "node:fs"
+import path from "node:path"
+
+import { listPacks, loadPack, makePack, packCard, packMarkdown, packRunCard, runPack, type PackSources } from "../pack/index.ts"
+import { packsDir } from "../pack/pack.ts"
+import { runPackOnPipeline } from "../pack/pipeline.ts"
+import type { McpTool, ToolContext } from "./tools.ts"
+
+const clip = (s: string | undefined, n: number) => {
+  const t = (s || "").trim()
+  return t.length <= n ? t : `…${t.slice(-(n - 1))}`
+}
+
+function repoOf(ctx: ToolContext, given?: unknown): string {
+  const p = given ? path.resolve(ctx.cwd, String(given)) : ctx.cwd
+  try {
+    return realpathSync(p)
+  } catch {
+    return p
+  }
+}
+
+const ITEM_SCHEMA = {
+  type: "object",
+  properties: {
+    source: { type: "string", description: "The tool it came from: jira, github, azure_devops, confluence, linear…" },
+    id: { type: "string" },
+    title: { type: "string" },
+    text: { type: "string", description: "What changes, in its words: description, acceptance criteria, release note" },
+    url: { type: "string" },
+    type: { type: "string", description: "Story, Bug, Feature, Task…" },
+    kind: { type: "string", enum: ["work_item", "pr", "commit", "note"] },
+    files: { type: "array", items: { type: "string" }, description: "Files it changed, when known" },
+    links: { type: "array", items: { type: "string" }, description: "Work items it refers to: ado:123, jira:PAY-12, github:45" },
+    test_cases: { type: "array", items: { type: "string" }, description: "Test cases that test it (Azure DevOps ids, or ADO-123)" },
+  },
+  required: ["title"],
+}
+
+export const PACK_TOOLS: McpTool[] = [
+  {
+    name: "build_regression_pack",
+    description:
+      "Build a regression pack: the repository's tests that a release's changes can break, each with why it is in (it automates a test case of a changed work item; the change is in the test code it uses; the work item's words match what it does; it works in a risk area the release touches; it is a smoke test), and the changes no test covers, each with the closest test and what it lacks. Changes come from any mix of: an Azure DevOps sprint, area, epic, or work item ids (with their linked pull requests' files); the commits between two refs of a repository; pull requests by number or merged in the last days; and items you read from any other connected tool (Jira, GitHub issues, a release note), passed in items. Saves the pack by name in .forgeqe/packs and shows a card. Nothing runs. Takes seconds.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        name: { type: "string", description: "The pack's name, e.g. release-1.5 (default: release-<date>)" },
+        repo: { type: "string", description: "The test repository (default: the working directory)" },
+        sprint: { type: "string", description: "Azure DevOps sprint name of the team, or current" },
+        iteration: { type: "string", description: "Azure DevOps iteration path and everything under it: a release such as Release 2026.09, or Project\\Release 2026.09\\Sprint 13" },
+        area: { type: "string", description: "Azure DevOps area path" },
+        parent: { type: "number", description: "An Azure DevOps epic or feature: everything under it" },
+        work_items: { type: "array", items: { type: "number" }, description: "Azure DevOps work item ids" },
+        project: { type: "string", description: "Azure DevOps project (default: sources.azure_devops.project)" },
+        range_repo: { type: "string", description: "Repository for from/to (the product's, or the tests' own); default: the test repository" },
+        from: { type: "string", description: "Commits after this ref: a tag, branch, or commit (e.g. release/1.4)" },
+        to: { type: "string", description: "Commits up to this ref (default HEAD)" },
+        prs: { type: "array", items: { type: "string" }, description: "Pull request numbers or URLs" },
+        merged_days: { type: "number", description: "Pull requests merged in the last N days" },
+        pr_repo: { type: "string", description: "Repository the pull requests are in (default: the test repository)" },
+        items: { type: "array", items: ITEM_SCHEMA, description: "Changes you read from other connected tools, as they are" },
+        risk: { type: "boolean", description: "Add tests for risk areas the release touches (default true)" },
+        smoke: { type: "boolean", description: "Add smoke-tagged tests (default true)" },
+      },
+    },
+    async run(args, ctx) {
+      const repo = repoOf(ctx, args.repo)
+      const sources: PackSources = {
+        ...(args.sprint || args.iteration || args.area || args.parent || (Array.isArray(args.work_items) && args.work_items.length)
+          ? { ado: { ...(args.sprint ? { sprint: String(args.sprint) } : {}), ...(args.iteration ? { iteration: String(args.iteration) } : {}), ...(args.area ? { area: String(args.area) } : {}), ...(args.parent ? { parent: Number(args.parent) } : {}), ...(Array.isArray(args.work_items) ? { ids: args.work_items.map(Number).filter(Boolean) } : {}), ...(args.project ? { project: String(args.project) } : {}) } }
+          : {}),
+        ...(args.from ? { range: { repo: repoOf(ctx, args.range_repo || args.repo), from: String(args.from), ...(args.to ? { to: String(args.to) } : {}) } } : {}),
+        ...(Array.isArray(args.prs) && args.prs.length ? { prs: args.prs.map(String) } : {}),
+        ...(Number(args.merged_days) > 0 ? { mergedDays: Number(args.merged_days) } : {}),
+        ...(args.pr_repo ? { prRepo: repoOf(ctx, args.pr_repo) } : {}),
+        ...(Array.isArray(args.items) && args.items.length ? { items: args.items } : {}),
+      }
+      const pack = await makePack({ config: ctx.config, repo, name: args.name ? String(args.name) : undefined, sources, risk: args.risk !== false, smoke: args.smoke !== false })
+      const md = path.join(packsDir(repo), `${pack.name}.md`)
+      writeFileSync(md, `${packMarkdown(pack)}\n`)
+      ctx.emit?.({ type: "card", card: packCard(pack) })
+      return {
+        name: pack.name,
+        tests: pack.tests.map((t) => ({ test: t.test, file: t.file, why: [...new Set(t.reasons.map((r) => r.text))], ...(t.flaky ? { flaky: t.flaky } : {}) })),
+        changes: pack.items.length,
+        from: pack.scope,
+        not_covered: pack.uncovered,
+        risk_areas: pack.risk,
+        notes: pack.notes.length ? pack.notes : undefined,
+        file: md,
+        next_step: [
+          pack.tests.length ? `Offer to run it: run_regression_pack with name ${pack.name} and on local, testmu, or pipeline.` : "",
+          pack.uncovered.length ? "Name the changes no test covers. For one with a closest test, offer to extend that test (browser_run with existing_test and existing_mode extend), so the next pack covers it." : "",
+        ]
+          .filter(Boolean)
+          .join(" "),
+      }
+    },
+  },
+  {
+    name: "run_regression_pack",
+    description:
+      "Run a saved regression pack after the user confirms: on local (this machine's browsers), testmu (TestMu AI's cloud browsers: the repository's tests connect to them through FORGEQE_PW_WS_ENDPOINT), or pipeline (a CI pipeline runs it). Reads each test's result, keeps the run with the pack, and says what changed since its last run. Where ForgeQE cannot ask (an MCP client), pass confirmed: true only after the user agreed. Takes minutes.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        name: { type: "string", description: "The pack's name" },
+        on: { type: "string", enum: ["local", "testmu", "pipeline"], description: "Where to run it (default local)" },
+        pipeline: { type: "string", enum: ["github", "azure_devops"], description: "With on pipeline: which one (default: packs.pipeline.default in forgeqe.yaml)" },
+        browsers: { type: "string", enum: ["local", "testmu"], description: "With on pipeline: the runner's own browser, or TestMu AI's (default local)" },
+        repo: { type: "string", description: "The test repository (default: the working directory)" },
+        confirmed: { type: "boolean", description: "Only where ForgeQE cannot ask: the user agreed to this run" },
+      },
+      required: ["name"],
+    },
+    async run(args, ctx) {
+      const repo = repoOf(ctx, args.repo)
+      const pack = loadPack(repo, String(args.name))
+      const on = args.on === "testmu" || args.on === "pipeline" ? args.on : "local"
+      const browsers = args.browsers === "testmu" ? "testmu" : "local"
+      const where = on === "testmu" ? "on TestMu AI" : on === "pipeline" ? `in the ${args.pipeline === "azure_devops" ? "Azure DevOps" : args.pipeline === "github" ? "GitHub Actions" : "configured"} pipeline${browsers === "testmu" ? " on TestMu AI" : ""}` : "on this machine"
+      const what = `${pack.tests.length} test${pack.tests.length === 1 ? "" : "s"}`
+      if (ctx.ask) {
+        const choice = await ctx.ask({
+          title: `Run pack ${pack.name} (${what}) ${where}?`,
+          lines: pack.tests.slice(0, 20).map((t) => `  • ${t.test}`).concat(pack.tests.length > 20 ? [`  and ${pack.tests.length - 20} more`] : []),
+          options: [
+            { id: "run", label: `Run ${what} ${where}` },
+            { id: "no", label: "Don't run" },
+          ],
+        })
+        if (choice !== "run") return { ran: false, reason: "The user chose not to run it." }
+      } else if (args.confirmed !== true) {
+        return { ran: false, needs_confirmation: true, tests: pack.tests.map((t) => t.test), on, next_step: "Show the user the pack and where it runs; call run_regression_pack again with confirmed: true only after they say yes." }
+      }
+      const { run, record, notes, pack: after } =
+        on === "pipeline"
+          ? await runPackOnPipeline({
+              config: ctx.config,
+              repo,
+              name: pack.name,
+              ...(args.pipeline === "github" || args.pipeline === "azure_devops" ? { kind: args.pipeline } : {}),
+              target: browsers,
+              signal: ctx.signal,
+              onProgress: (text) => ctx.emit?.({ type: "agent", event: { type: "code", kind: "info", text } }),
+            })
+          : await runPack({ config: ctx.config, repo, name: pack.name, on, signal: ctx.signal, runTest: ctx.runTest })
+      ctx.emit?.({ type: "card", card: packRunCard(after, run, record, notes) })
+      const failed = run.tests.filter((t) => t.status === "failed")
+      return {
+        ran: true,
+        on: record.target,
+        status: run.status,
+        passed: run.counts.passed,
+        failed: run.counts.failed,
+        skipped: run.counts.skipped,
+        seconds: run.seconds,
+        failures: failed.length ? failed.map((t) => ({ name: t.name, message: t.message })) : undefined,
+        ...(run.flaky?.length ? { flaky: run.flaky } : {}),
+        previous: after.runs.at(-2) || undefined,
+        output: run.status === "passed" ? undefined : clip(run.output, 3000),
+        folder: run.dir,
+        ...(record.url ? { pipeline_run: record.url } : {}),
+        notes: notes.length ? notes : undefined,
+        next_step: failed.length ? "Name the failing tests with their messages; offer heal_tests for a test the app outgrew, or file the defects (file_bug)." : undefined,
+      }
+    },
+  },
+  {
+    name: "list_regression_packs",
+    description: "The saved regression packs of a repository: name, when built, from what, how many tests and gaps, and their last run. Nothing runs.",
+    inputSchema: { type: "object", properties: { repo: { type: "string" } } },
+    async run(args, ctx) {
+      return {
+        packs: listPacks(repoOf(ctx, args.repo)).map((p) => ({
+          name: p.name,
+          built: p.createdAt.slice(0, 16).replace("T", " "),
+          from: p.scope,
+          tests: p.tests.length,
+          not_covered: p.uncovered.length,
+          last_run: p.runs.at(-1) ? { at: p.runs.at(-1)!.at.slice(0, 16).replace("T", " "), on: p.runs.at(-1)!.target, status: p.runs.at(-1)!.status, failed: p.runs.at(-1)!.counts.failed } : undefined,
+        })),
+      }
+    },
+  },
+]

package/src/mcp/routing.ts

@@ -0,0 +1,145 @@
+import { CAPABILITIES, type Capability } from "../capabilities/registry.ts"
+import { prompt } from "../prompts.ts"
+import type { McpTool } from "./tools.ts"
+
+/**
+ * What ForgeQE's agent carries on every model call, and what it gets only when a task needs it.
+ *
+ * Every model call sends the system prompt and the tool definitions again. Tools stay listed in full: their
+ * descriptions are how the agent chooses a tool and plans the task, and their arguments are needed for any call, so a
+ * tool listed in one line cost an extra model call each time (measured: more tokens, not fewer). The playbook is
+ * what moves: the sections of brain.md that a capability owns (`playbook`) leave the system prompt, an index of them
+ * stays, and the first result of one of the capability's tools in a session (or of a tool in `alsoAfter`) carries the
+ * section. The `playbook` tool reads one before that. A section no capability owns stays in the system prompt, so a
+ * section renamed in a team's own copy of brain.md is never lost.
+ */
+
+/** Rough tokens of a text: the budgets and `forgeqe context` use it, not billing. */
+export const tokens = (text: string) => Math.ceil(text.length / 4)
+
+const norm = (s: string) => s.trim().toLowerCase()
+
+/** The capability each tool belongs to. */
+export function capabilityOfTool(name: string, caps: Capability[] = CAPABILITIES): Capability | undefined {
+  return caps.find((c) => c.tools.includes(name))
+}
+
+// ---------------------------------------------------------------------------------------------
+// The playbook
+// ---------------------------------------------------------------------------------------------
+
+export type Playbook = { heading: string; when: string; capability: string; tools: string[]; text: string }
+
+/** brain.md split: the system prompt without the owned sections (with an index of them), and those sections. */
+export type BrainParts = { core: string; playbooks: Playbook[] }
+
+const SECTION = /^###\s+(.+?)\s*$/
+
+/** Split the agent's prompt by the capabilities' playbook sections. Unowned sections stay in `core`. */
+export function splitBrain(text: string, caps: Capability[] = CAPABILITIES): BrainParts {
+  const owned = new Map(caps.flatMap((c) => (c.playbook || []).map((p) => [norm(p.heading), { ...p, capability: c.id, tools: c.tools }] as const)))
+  const lines = text.split("\n")
+  const keep: string[] = []
+  const playbooks: Playbook[] = []
+  let current: { entry: Omit<Playbook, "text">; lines: string[] } | null = null
+  const close = () => {
+    if (current) playbooks.push({ ...current.entry, text: current.lines.join("\n").trim() })
+    current = null
+  }
+  for (const line of lines) {
+    const h = SECTION.exec(line)
+    if (h || /^##\s/.test(line)) {
+      close()
+      const entry = h ? owned.get(norm(h[1])) : undefined
+      if (entry) {
+        current = { entry: { heading: h![1].trim(), when: entry.when, capability: entry.capability, tools: entry.tools }, lines: [line] }
+        continue
+      }
+    }
+    if (current) current.lines.push(line)
+    else keep.push(line)
+  }
+  close()
+  let core = keep.join("\n").replace(/\n{3,}/g, "\n\n")
+  if (playbooks.length) {
+    const index = [
+      "",
+      "These playbooks are not in this prompt. The first result of their tools carries each one: obey it. To read one before you start, call playbook with its name.",
+      ...playbooks.map((p) => `- "${p.heading}": ${p.when}. Tools: ${p.tools.join(", ")}.`),
+      "",
+    ].join("\n")
+    core = /^## Playbooks\s*$/m.test(core) ? core.replace(/^## Playbooks\s*$/m, (m) => `${m}\n${index}`) : `${core.trimEnd()}\n\n## Playbooks\n${index}`
+  }
+  return { core: core.trim(), playbooks }
+}
+
+/** ForgeQE's agent's prompt as it is sent: the core, with the index of the playbooks it gets on demand. */
+export const brainParts = (): BrainParts => splitBrain(prompt("brain"))
+
+// ---------------------------------------------------------------------------------------------
+// One session of ForgeQE's agent
+// ---------------------------------------------------------------------------------------------
+
+/**
+ * Routing for one agent session: which playbooks it was given, and what goes with a result. The playbook text is read
+ * once, when the session starts, like the system prompt.
+ */
+export function createRouter(tools: McpTool[], opts: { caps?: Capability[]; brain?: string } = {}) {
+  const caps = opts.caps || CAPABILITIES
+  const parts = splitBrain(opts.brain ?? prompt("brain"), caps)
+  const given = new Set<string>()
+  const names = new Set(tools.map((t) => t.name))
+
+  const playbooksFor = (tool: string) => parts.playbooks.filter((p) => p.tools.includes(tool) || caps.find((c) => c.id === p.capability)?.alsoAfter?.includes(tool))
+  const give = (books: Playbook[]) => {
+    const fresh = books.filter((b) => !given.has(b.heading))
+    for (const b of fresh) given.add(b.heading)
+    return fresh
+  }
+  const block = (books: Playbook[]) => (books.length ? `\n\nPlaybook (obey it for this task):\n\n${books.map((b) => b.text).join("\n\n")}` : "")
+
+  return {
+    parts,
+    /** What goes after an error of a tool: the playbooks of its capability this session has not had yet. */
+    after(name: string): string {
+      return block(give(playbooksFor(name)))
+    },
+    /**
+     * A result with the playbooks it should carry: in a `playbook` field of a JSON object, so the result stays JSON for
+     * whatever reads it; appended to any other text.
+     */
+    attach(name: string, text: string): string {
+      const books = give(playbooksFor(name))
+      if (!books.length) return text
+      const playbook = `Obey it for this task:\n\n${books.map((b) => b.text).join("\n\n")}`
+      try {
+        const parsed = JSON.parse(text)
+        if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return JSON.stringify({ ...parsed, playbook }, null, 2)
+      } catch {}
+      return text + block(books)
+    },
+    /** The playbook tool: playbooks by name, by capability id, or of a tool's capability. */
+    help(asked: string[]): string {
+      const books: Playbook[] = []
+      const unknown: string[] = []
+      for (const raw of asked) {
+        const exact = norm(raw)
+        // The client shows ForgeQE's tools with a forgeqe_ prefix; forgeqe_setup is that tool's own name.
+        const tool = names.has(exact) ? exact : names.has(exact.replace(/^forgeqe_/, "")) ? exact.replace(/^forgeqe_/, "") : ""
+        const found = tool ? playbooksFor(tool) : parts.playbooks.filter((p) => norm(p.heading) === exact || p.capability === exact)
+        if (found.length) books.push(...found)
+        else unknown.push(raw)
+      }
+      const unique = [...new Map(books.map((b) => [b.heading, b])).values()]
+      for (const b of unique) given.add(b.heading)
+      return [
+        ...(unique.length ? ["Playbook (obey it for this task):", "", ...unique.map((b) => b.text)] : []),
+        ...(unknown.length ? ["", `No playbook for: ${unknown.join(", ")}. Playbooks: ${parts.playbooks.map((p) => `"${p.heading}"`).join(", ")}. A tool not under one of them has its rules in its own description and in the prompt.`] : []),
+      ]
+        .join("\n")
+        .trim()
+    },
+  }
+}
+
+export type Router = ReturnType<typeof createRouter>

package/src/mcp/server.ts

@@ -2,6 +2,7 @@ import { createHash, randomBytes, timingSafeEqual } from "node:crypto"
 
 import { EvidenceLog } from "../report/evidence.ts"
 import { logEvent } from "../sessionLog.ts"
+import type { Router } from "./routing.ts"
 import type { McpTool, ToolContext, ToolEvent } from "./tools.ts"
 
 type Json = Record<string, any>
@@ -21,6 +22,8 @@ export type McpServerOptions = {
   /** Read on every call, so the shell's current profile and headed setting apply. */
   context: () => ToolContext
   instructions?: () => string
+  /** ForgeQE's agent: playbooks given with the first result of their tools. Other clients get results as they are. */
+  router?: Router
   version?: string
   progressMs?: number
 }
@@ -41,7 +44,7 @@ export function createMcpServer(opts: McpServerOptions) {
   async function callTool(params: Json, id: string | number, send: Send, outer?: AbortSignal): Promise<Json> {
     const tool = opts.tools.find((t) => t.name === params.name)
     if (!tool) return { content: [{ type: "text", text: `Unknown tool ${params.name}` }], isError: true }
-    const base = { evidence, ...opts.context() }
+    const base = { evidence, ...opts.context(), ...(opts.router ? { router: opts.router } : {}) }
     const controller = new AbortController()
     const abort = () => controller.abort()
     base.signal?.addEventListener("abort", abort, { once: true })
@@ -91,9 +94,10 @@ export function createMcpServer(opts: McpServerOptions) {
       said(true)
       const text = typeof out === "string" ? out : JSON.stringify(out, null, 2)
       base.evidence.record(tool.name, args, text)
-      return finish({ content: [{ type: "text", text }] })
+      return finish({ content: [{ type: "text", text: opts.router ? opts.router.attach(tool.name, text) : text }] })
     } catch (err) {
-      return finish({ content: [{ type: "text", text: controller.signal.aborted ? "Stopped by the user." : (err as Error).message || String(err) }], isError: true })
+      const message = controller.signal.aborted ? "Stopped by the user." : (err as Error).message || String(err)
+      return finish({ content: [{ type: "text", text: message + (controller.signal.aborted ? "" : opts.router?.after(tool.name) || "") }], isError: true })
     } finally {
       if (timer) clearInterval(timer)
       running.delete(id)