@capagents/forgeqe 1.1.1 → 1.1.3

package/src/evals/prompts.ts

@@ -0,0 +1,288 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"
+import path from "node:path"
+
+import YAML from "yaml"
+
+import type { AppConfig } from "../config.ts"
+import { configuredSystems } from "../capabilities/status.ts"
+import { steScore } from "../ste.ts"
+import type { CodeEvent } from "../types.ts"
+
+/**
+ * Evaluation-driven prompting: each prompt that can run on its own has cases (evals/prompts/<prompt>/*.yaml in
+ * ForgeQE, and .forgeqe/evals/prompts/<prompt>/ for a project's own), each an input and checks that are decided by
+ * code, not by a model: a field matches or is empty, the agent used or did not use a tool, the text keeps ASD-STE100.
+ * `forgeqe prompts eval` runs them, several times when asked, and compares the pass rate with the run before, so a
+ * change to a prompt is measured, not guessed.
+ */
+
+export const BUILTIN_CASES = path.resolve(import.meta.dir, "..", "..", "evals", "prompts")
+
+/** One check. Exactly one of the test keys is set besides `id` (and `field` for the field tests). */
+export type Check = {
+  id: string
+  field?: string
+  matches?: string
+  not_matches?: string
+  empty?: boolean
+  min_items?: number
+  max_items?: number
+  /** Least share of sentences that keep ASD-STE100, of `field` (default: the output text). */
+  ste_min?: number
+  /** At least one tool call matches one of these names (`*` matches any text). */
+  tools_any?: string[]
+  /** No tool call matches any of these. */
+  tools_none?: string[]
+  /** The first call that matches the first list comes before any call that matches the second. */
+  tools_before?: [string[], string[]]
+}
+
+export type PromptCase = {
+  id: string
+  title: string
+  prompt: string
+  input: string
+  /** Systems the case reads: it is skipped when one is not set up. */
+  needs: string[]
+  /** brain: the ForgeQE tools that may run; others answer that they are not available (default: those that only read). */
+  allow?: string[]
+  checks: Check[]
+  file: string
+}
+
+export type Observation = { fields: Record<string, unknown>; text: string; tools: string[]; tokens: number; ms: number }
+export type CheckResult = { id: string; pass: boolean; detail: string }
+export type RunResult = { pass: boolean; checks: CheckResult[]; tokens: number; ms: number; error?: string; /** The start of what came out, for a run that failed. */ seen?: string }
+export type CaseResult = { prompt: string; case: string; title: string; runs: RunResult[]; skipped?: string }
+
+/** The ForgeQE tools ForgeQE's agent may call in an evaluation: each only reads. */
+export const READ_ONLY_TOOLS = ["capabilities", "find_test_cases", "check_test_cases", "read_confluence", "run_details", "query_api", "list_sprints", "page_memory", "flaky_tests", "quality_report", "quality_trends", "forgeqe_setup"]
+
+const TESTS = ["matches", "not_matches", "empty", "min_items", "max_items", "ste_min", "tools_any", "tools_none", "tools_before"] as const
+
+/** Every case in a folder of `<prompt>/<case>.yaml`, checked so a broken case fails before any model is called. */
+export function loadPromptCases(dirs: string[], only?: { prompt?: string; case?: string }): PromptCase[] {
+  const out: PromptCase[] = []
+  for (const dir of dirs) {
+    if (!existsSync(dir)) continue
+    for (const prompt of readdirSync(dir).sort()) {
+      if (only?.prompt && prompt !== only.prompt) continue
+      const folder = path.join(dir, prompt)
+      let files: string[] = []
+      try {
+        files = readdirSync(folder).filter((f) => /\.ya?ml$/.test(f)).sort()
+      } catch {
+        continue
+      }
+      for (const f of files) {
+        const file = path.join(folder, f)
+        const c = parseCase(YAML.parse(readFileSync(file, "utf8")), prompt, file)
+        if (only?.case && c.id !== only.case) continue
+        // A project's case replaces ForgeQE's case of the same id.
+        const at = out.findIndex((x) => x.prompt === c.prompt && x.id === c.id)
+        if (at >= 0) out.splice(at, 1)
+        out.push(c)
+      }
+    }
+  }
+  return out
+}
+
+export function parseCase(raw: any, prompt: string, file: string): PromptCase {
+  const where = `${path.basename(path.dirname(file))}/${path.basename(file)}`
+  if (!raw?.id || !raw.input) throw new Error(`${where}: id and input are required.`)
+  if (!Array.isArray(raw.checks) || !raw.checks.length) throw new Error(`${where}: checks needs at least one check.`)
+  const checks: Check[] = raw.checks.map((c: any, i: number) => {
+    if (!c?.id) throw new Error(`${where}: check ${i + 1} needs an id.`)
+    const tests = TESTS.filter((t) => c[t] !== undefined)
+    if (tests.length !== 1) throw new Error(`${where}: check ${c.id} needs exactly one of ${TESTS.join(", ")}.`)
+    if (["matches", "not_matches", "empty", "min_items", "max_items"].includes(tests[0]!) && !c.field) throw new Error(`${where}: check ${c.id} needs field.`)
+    for (const k of ["matches", "not_matches"]) if (c[k] !== undefined) new RegExp(String(c[k]), "i")
+    return c as Check
+  })
+  const ids = checks.map((c) => c.id)
+  if (new Set(ids).size !== ids.length) throw new Error(`${where}: check ids must be unique.`)
+  return {
+    id: String(raw.id),
+    title: String(raw.title || raw.id),
+    prompt: String(raw.prompt || prompt),
+    input: String(raw.input),
+    needs: (raw.needs || []).map(String),
+    ...(raw.allow ? { allow: raw.allow.map(String) } : {}),
+    checks,
+    file,
+  }
+}
+
+const glob = (pattern: string) => new RegExp(`^${pattern.split("*").map((p) => p.replace(/[.+?^${}()|[\]\\]/g, "\\$&")).join(".*")}$`, "i")
+const matchesAny = (tool: string, patterns: string[]) => patterns.some((p) => glob(p).test(tool))
+const asText = (v: unknown): string => (v === undefined || v === null ? "" : typeof v === "string" ? v : Array.isArray(v) ? v.map(asText).join("\n") : JSON.stringify(v))
+
+/** Each check decided against what one run did. */
+export function judge(checks: Check[], o: Observation): CheckResult[] {
+  return checks.map((c): CheckResult => {
+    const value = c.field ? o.fields[c.field] : o.text
+    const text = asText(value)
+    const items = Array.isArray(value) ? value.length : text.trim() ? 1 : 0
+    if (c.matches !== undefined) {
+      const pass = new RegExp(c.matches, "i").test(text)
+      return { id: c.id, pass, detail: pass ? `${c.field} matches /${c.matches}/` : `${c.field} does not match /${c.matches}/: ${clip(text)}` }
+    }
+    if (c.not_matches !== undefined) {
+      const m = new RegExp(c.not_matches, "i").exec(text)
+      return { id: c.id, pass: !m, detail: m ? `${c.field} has "${m[0]}": ${clip(text)}` : `${c.field} does not match /${c.not_matches}/` }
+    }
+    if (c.empty !== undefined) {
+      const pass = (items === 0) === c.empty
+      return { id: c.id, pass, detail: pass ? `${c.field} is ${c.empty ? "empty" : "not empty"}` : `${c.field} is ${c.empty ? `not empty: ${clip(text)}` : "empty"}` }
+    }
+    if (c.min_items !== undefined) return { id: c.id, pass: items >= c.min_items, detail: `${c.field} has ${items} (at least ${c.min_items})` }
+    if (c.max_items !== undefined) return { id: c.id, pass: items <= c.max_items, detail: `${c.field} has ${items} (at most ${c.max_items})` }
+    if (c.ste_min !== undefined) {
+      const score = steScore(text).score
+      return { id: c.id, pass: score >= c.ste_min, detail: `ASD-STE100 ${Math.round(score * 100)}% (at least ${Math.round(c.ste_min * 100)}%)` }
+    }
+    if (c.tools_any) {
+      const hit = o.tools.find((t) => matchesAny(t, c.tools_any!))
+      return { id: c.id, pass: Boolean(hit), detail: hit ? `used ${hit}` : `used none of ${c.tools_any.join(", ")} (used: ${clip(o.tools.join(", ") || "no tools")})` }
+    }
+    if (c.tools_none) {
+      const hit = o.tools.filter((t) => matchesAny(t, c.tools_none!))
+      return { id: c.id, pass: !hit.length, detail: hit.length ? `used ${[...new Set(hit)].join(", ")}` : `used none of ${c.tools_none.join(", ")}` }
+    }
+    const [first, then] = c.tools_before!
+    const a = o.tools.findIndex((t) => matchesAny(t, first))
+    const b = o.tools.findIndex((t) => matchesAny(t, then))
+    const pass = a >= 0 && (b < 0 || a < b)
+    return { id: c.id, pass, detail: a < 0 ? `used none of ${first.join(", ")}` : b >= 0 && b < a ? `used ${o.tools[b]} before ${o.tools[a]}` : `used ${o.tools[a]} first` }
+  })
+}
+
+const clip = (s: string, n = 120) => {
+  const t = s.replace(/\s+/g, " ").trim()
+  return t.length <= n ? t : `${t.slice(0, n - 1)}…`
+}
+
+/** Why a case cannot run with this config, or null. */
+export function skipReason(c: PromptCase, config: AppConfig): string | null {
+  if (config.llm.provider === "none" || config.llm.provider === "mock") return "no live model"
+  const systems = configuredSystems(config)
+  const missing = c.needs.filter((n) => !systems.has(n as never))
+  return missing.length ? `needs ${missing.join(", ")}` : null
+}
+
+export type Runner = (c: PromptCase, config: AppConfig, cwd: string) => Promise<Observation>
+
+/** Tokens a session used, from its last usage event (the runtime reports running totals). */
+function usageMeter() {
+  let tokens = 0
+  return {
+    onEvent: (e: CodeEvent) => {
+      if (e.kind === "usage") tokens = e.usage.input + e.usage.output
+    },
+    get tokens() {
+      return tokens
+    },
+  }
+}
+
+/** How each prompt that can run on its own is run for a case. */
+export const RUNNERS: Record<string, Runner> = {
+  async refine(c, config, cwd) {
+    const { refine } = await import("../refine.ts")
+    const meter = usageMeter()
+    const started = Date.now()
+    const r = await refine({ config, cwd, text: c.input, recent: [], onEvent: meter.onEvent })
+    const fields: Record<string, unknown> = { ...(r.spec || {}), refined: r.refined, missing: r.missing, assumed: r.assumed, grounded: r.grounded.map((g) => g.fact) }
+    return { fields, text: r.refined, tools: [], tokens: meter.tokens, ms: Date.now() - started }
+  },
+
+  async brain(c, config, cwd) {
+    const { openBrain } = await import("../brain/index.ts")
+    const tools: string[] = []
+    let tokens = 0
+    const allow = new Set(c.allow || READ_ONLY_TOOLS)
+    const started = Date.now()
+    const brain = await openBrain({
+      config,
+      cwd,
+      context: () => ({ session: "shell" as const, observe: (tool: string) => tools.push(`forgeqe_${tool}`), allow: (tool: string) => allow.has(tool) }),
+      onEvent: (e) => {
+        if (e.kind === "usage") tokens = e.usage.input + e.usage.output
+        // ForgeQE's own tools are hidden from these events; observe records them.
+        if (e.kind === "tool" && e.tool && !e.tool.startsWith("forgeqe_")) tools.push(e.tool)
+      },
+    })
+    try {
+      const a = await brain.ask(c.input)
+      return { fields: { reply: a.reply, outcome: a.outcome, next: (a.next || []).map((n) => n.label) }, text: a.reply, tools, tokens, ms: Date.now() - started }
+    } finally {
+      await brain.close()
+    }
+  },
+
+  async "explore-charters"(c, config, cwd) {
+    const { planCharters } = await import("../explore/charters.ts")
+    const meter = usageMeter()
+    const started = Date.now()
+    const [title, ...rest] = c.input.split("\n")
+    const acceptance = rest.join("\n")
+    const story = { id: "story", system: "text" as const, title: title!.trim(), description: "", acceptance, parents: [], scripted: [], passages: [], errors: [] }
+    const r = await planCharters({ config, story, count: 5, roles: [], cwd, onEvent: meter.onEvent })
+    const fields = { charters: r.charters, titles: r.charters.map((x) => x.title), categories: r.charters.map((x) => x.category), steps: r.charters.flatMap((x) => x.steps.map((s) => `${s.action} → ${s.expected}`)) }
+    // Each mission is a paragraph of its own, ended as a sentence, so writing checks score each one.
+    const text = r.charters.map((x) => (/[.!?]$/.test(x.mission.trim()) ? x.mission.trim() : `${x.mission.trim()}.`)).join("\n\n")
+    return { fields, text, tools: [], tokens: meter.tokens, ms: Date.now() - started }
+  },
+}
+
+export async function runCase(c: PromptCase, config: AppConfig, cwd: string, runs: number): Promise<CaseResult> {
+  const base = { prompt: c.prompt, case: c.id, title: c.title }
+  const why = skipReason(c, config)
+  if (why) return { ...base, runs: [], skipped: why }
+  const runner = RUNNERS[c.prompt]
+  if (!runner) return { ...base, runs: [], skipped: `no runner for ${c.prompt} yet` }
+  const out: RunResult[] = []
+  for (let i = 0; i < runs; i++) {
+    const started = Date.now()
+    try {
+      const o = await runner(c, config, cwd)
+      const checks = judge(c.checks, o)
+      const pass = checks.every((x) => x.pass)
+      out.push({ pass, checks, tokens: o.tokens, ms: o.ms, ...(pass ? {} : { seen: clip(o.text, 300) }) })
+    } catch (err) {
+      out.push({ pass: false, checks: [], tokens: 0, ms: Date.now() - started, error: (err as Error).message })
+    }
+  }
+  return { ...base, runs: out }
+}
+
+/** Where results are kept, one file per `forgeqe prompts eval`. */
+export const resultsDir = (config: AppConfig) => path.join(config.export.outDir, "evals", "prompts")
+
+export function saveResults(config: AppConfig, results: CaseResult[], at = new Date()): string {
+  const dir = resultsDir(config)
+  mkdirSync(dir, { recursive: true })
+  const file = path.join(dir, `${at.toISOString().replace(/[:.]/g, "-")}.json`)
+  writeFileSync(file, JSON.stringify({ at: at.toISOString(), results }, null, 2))
+  return file
+}
+
+/** The pass rate of each case in the latest earlier results that ran it: "prompt/case" → [passed, runs]. */
+export function previousRates(config: AppConfig, except?: string): Map<string, [number, number]> {
+  const dir = resultsDir(config)
+  const out = new Map<string, [number, number]>()
+  if (!existsSync(dir)) return out
+  const files = readdirSync(dir).filter((f) => f.endsWith(".json")).sort().reverse()
+  for (const f of files) {
+    if (except && path.join(dir, f) === except) continue
+    try {
+      const { results } = JSON.parse(readFileSync(path.join(dir, f), "utf8")) as { results: CaseResult[] }
+      for (const r of results) {
+        const key = `${r.prompt}/${r.case}`
+        if (!out.has(key) && r.runs.length) out.set(key, [r.runs.filter((x) => x.pass).length, r.runs.length])
+      }
+    } catch {}
+  }
+  return out
+}

package/src/heal/loop.ts

@@ -7,7 +7,7 @@ import { addUsage, costOf, meterSince, meterSnapshot, type Cost } from "../usage
 import { openSession, type OpencodeSession } from "../code/runtime.ts"
 import { runTestCommand, type TestRun } from "../code/testrun.ts"
 import { changedFiles, ownPaths, protectPatterns, restoreRemoved, snapshot } from "../code/worktree.ts"
-import type { AppConfig } from "../config.ts"
+import type { AppConfig, BrowserProvider } from "../config.ts"
 import { MCP_NAME, TOOL_TIMEOUT_MS, localToolServer } from "../mcp/server.ts"
 import { TOOLS, environmentNotes, type ToolContext, type ToolEvent } from "../mcp/tools.ts"
 import { runDirectory } from "../run.ts"
@@ -79,6 +79,7 @@ export type HealOptions = {
   /** The repo's test command; default code.test_command, else the agent finds it. */
   command?: string
   outDir?: string
+  browserProvider?: BrowserProvider
   headed?: boolean
   maxSteps?: number
   url?: string
@@ -149,6 +150,7 @@ export async function runHeal(opts: HealOptions): Promise<HealResult> {
     config: runsConfig,
     cwd: repo,
     headed: opts.headed,
+    browserProvider: opts.browserProvider,
     maxSteps: opts.maxSteps,
     url: opts.url,
     signal: opts.signal,

package/src/init.ts

@@ -64,6 +64,24 @@ export function initWorkspace(
   // Prompts, skills, and product knowledge templates, to read and change: .forgeqe/ in a project, ~/.forgeqe for the user.
   const root = home ? forgeqeHome() : projectDir(directory)
   const shown = (p: string) => (home ? p : path.relative(directory, p) || p)
+  for (const [srcName, destName] of [
+    ["browserstack.example.yaml", "browserstack.yaml"],
+    ["testmu.example.yaml", "testmu.yaml"],
+  ] as const) {
+    const src = path.join(ROOT, srcName)
+    const dest = path.join(root, destName)
+    if (!existsSync(src)) {
+      notes.push(`Template missing: ${src}`)
+      continue
+    }
+    if (existsSync(dest) && !opts.force) {
+      notes.push(`Kept existing ${shown(dest)}`)
+      continue
+    }
+    mkdirSync(path.dirname(dest), { recursive: true })
+    writeFileSync(dest, readFileSync(src, "utf8"), "utf8")
+    created.push(dest)
+  }
   for (const { what, src, dest, keys } of [
     { what: "prompts", src: PROMPTS_DIR, dest: path.join(root, "prompts"), keys: entries(PROMPTS_DIR, (e) => e.endsWith(".md")) },
     { what: "skills", src: PREBUILT_DIR, dest: path.join(root, "skills"), keys: entries(PREBUILT_DIR, (e) => existsSync(path.join(PREBUILT_DIR, e, "SKILL.md"))) },

package/src/mcp/server.ts

@@ -42,6 +42,14 @@ export function createMcpServer(opts: McpServerOptions) {
     const timer = token !== undefined ? setInterval(() => send({ jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: token, progress: ++tick, message: `${tool.name} running` } }), opts.progressMs || PROGRESS_EVERY_MS) : null
     const started = Date.now()
     logEvent("tool.call", { tool: tool.name, arguments: params.arguments || {} })
+    base.observe?.(tool.name, params.arguments || {})
+    if (base.allow && !base.allow(tool.name)) {
+      running.delete(id)
+      if (timer) clearInterval(timer)
+      base.signal?.removeEventListener("abort", abort)
+      outer?.removeEventListener("abort", abort)
+      return { content: [{ type: "text", text: `${tool.name} is not available in this evaluation. Say what you would do with it, and stop.` }], isError: true }
+    }
     const finish = (result: Json) => {
       logEvent("tool.result", { tool: tool.name, ms: Date.now() - started, error: Boolean(result.isError), text: result.content?.[0]?.text })
       return result

package/src/mcp/tools.ts

@@ -15,7 +15,7 @@ import { buildHandoff } from "../code/handoff.ts"
 import { latestRunDir, runCodeAgent } from "../code/index.ts"
 import type { OpencodeSession } from "../code/runtime.ts"
 import { runTestCommand } from "../code/testrun.ts"
-import type { AppConfig } from "../config.ts"
+import { browserProvider, type AppConfig } from "../config.ts"
 import { clearMemory, memoryFilesFor, memoryFlows, memoryOverview, memoryShow } from "../memory.ts"
 import { executeRun, runDirectory } from "../run.ts"
 import { counts } from "../suite/report.ts"
@@ -123,6 +123,7 @@ export type ToolContext = {
   /** The repository or folder the agent works in. */
   cwd: string
   headed?: boolean
+  browserProvider?: AppConfig["browser"]["provider"]
   maxSteps?: number
   replay?: ReplayMode
   memory?: boolean
@@ -130,6 +131,10 @@ export type ToolContext = {
   url?: string
   signal?: AbortSignal
   emit?: (e: ToolEvent) => void
+  /** Sees every call before it runs, with its arguments: evaluations record which tools the agent chose. */
+  observe?: (tool: string, args: Record<string, unknown>) => void
+  /** Whether a tool may run; a refused call answers that it is not available here (evaluations refuse tools that act). */
+  allow?: (tool: string) => boolean
   /** How long this process lives: the shell, one `forgeqe run`, or `forgeqe mcp`. Background monitors need shell or server. */
   session?: "shell" | "once" | "server"
   /** Passed browser runs become tests in cwd (the shell and `forgeqe run` without --no-code). */
@@ -252,7 +257,7 @@ export const TOOLS: McpTool[] = [
   {
     name: "browser_run",
     description:
-      "Carry out a goal in a real browser with ForgeQE's browser agent and verify the outcome. The agent picks the site from the goal unless start_url is given. Returns pass/fail, the agent's summary, the steps, and the run folder (run.json, workflow.yaml with ranked locators, trace.md, steps and screenshots, report.html). Takes minutes.",
+      "Carry out a goal in a real browser with ForgeQE's browser agent and verify the outcome. The browser comes from forgeqe.yaml: local Chrome or Chromium by default, or BrowserStack or TestMu AI/LambdaTest when browser.provider selects one. The agent picks the site from the goal unless start_url is given. Returns pass/fail, the agent's summary, the steps, and the run folder (run.json, workflow.yaml with ranked locators, trace.md, steps and screenshots, report.html). Takes minutes.",
     inputSchema: {
       type: "object",
       properties: {
@@ -262,6 +267,12 @@ export const TOOLS: McpTool[] = [
             "The final instructions for the browser agent: what to do and what to check, in plain language, with every value it needs. Only the decided task; never your own questions, alternatives, or reasoning about it.",
         },
         start_url: { type: "string", description: "Page to open first, only when the user named one" },
+        browser_provider: {
+          type: "string",
+          enum: ["local", "browserstack", "testmu"],
+          description:
+            "Override the browser provider for this run when the user asked for one specifically. local uses the machine's own browser; browserstack and testmu use their configured YAML or browser.remote settings.",
+        },
         resume_from: {
           type: "string",
           description:
@@ -283,6 +294,7 @@ export const TOOLS: McpTool[] = [
       const from = args.resume_from ? String(args.resume_from) : ""
       const resumeFrom = from ? (existsSync(path.resolve(ctx.cwd, from)) ? path.resolve(ctx.cwd, from) : from) : undefined
       const url = args.start_url ? String(args.start_url) : ctx.url
+      const selectedBrowser = args.browser_provider ? browserProvider(args.browser_provider) : ctx.browserProvider
       const runDir = runDirectory(ctx.config.export.outDir)
       const emit = ctx.emit || (() => {})
       const t0 = Date.now()
@@ -291,6 +303,7 @@ export const TOOLS: McpTool[] = [
         const { result, files } = await (ctx.execute || executeRun)({
           config: ctx.config,
           goal,
+          browserProvider: selectedBrowser,
           url,
           headed: ctx.headed,
           maxSteps: ctx.maxSteps,
@@ -406,13 +419,19 @@ export const TOOLS: McpTool[] = [
   {
     name: "run_test_cases",
     description:
-      "Run test cases in real browsers: each case is one browser goal whose steps are done in order and whose expected results are checked, with a verdict for every step. Writes report.html (steps, screenshots, timings) and junit.xml, posts results back to Azure DevOps, Jira, Xray, TestRail, Zephyr Scale, or qTest unless write_back is false, and files or updates bugs for failures when bugs are set up. In a session that writes tests, each passed case is handed to ForgeQE's coding agent as soon as its browser run ends, and its test is written and run before the next case starts. Returns each case's result, step verdicts, run folder, and test (file, command, status, or why none was written). Takes minutes.",
+      "Run test cases in real browsers: each case is one browser goal whose steps are done in order and whose expected results are checked, with a verdict for every step. The browser comes from forgeqe.yaml: local Chrome or Chromium by default, or BrowserStack or TestMu AI/LambdaTest when browser.provider selects one. Writes report.html (steps, screenshots, timings) and junit.xml, posts results back to Azure DevOps, Jira, Xray, TestRail, Zephyr Scale, or qTest unless write_back is false, and files or updates bugs for failures when bugs are set up. In a session that writes tests, each passed case is handed to ForgeQE's coding agent as soon as its browser run ends, and its test is written and run before the next case starts. Returns each case's result, step verdicts, run folder, and test (file, command, status, or why none was written). Takes minutes.",
     inputSchema: {
       type: "object",
       properties: {
         sources: SOURCES_SCHEMA,
         parallel: { type: "integer", minimum: 1, maximum: 8, description: "Browsers at once (default from config)" },
         write_back: { type: "boolean", description: "Post results to the source system (default from config)" },
+        browser_provider: {
+          type: "string",
+          enum: ["local", "browserstack", "testmu"],
+          description:
+            "Override the browser provider for every case in this suite when the user asked for one specifically.",
+        },
         skills: SKILLS_SCHEMA,
       },
       required: ["sources"],
@@ -425,6 +444,7 @@ export const TOOLS: McpTool[] = [
       if (!cases.length) return { ran: false, reason: "No case can run as written (suite.case_check: block).", check: checkResult(checks) }
       if (!cases.length) throw new Error("No test cases found.")
       const parallel = Math.min(Number(args.parallel) || ctx.config.suite.parallel, cases.length)
+      const selectedBrowser = args.browser_provider ? browserProvider(args.browser_provider) : ctx.browserProvider
       const emit = ctx.emit || (() => {})
       emit({ type: "suite-start", cases, parallel })
       // In a session that writes tests, each passed case goes to the coding agent right after its own browser run,
@@ -451,6 +471,7 @@ export const TOOLS: McpTool[] = [
         config: ctx.writeTests ? { ...ctx.config, suite: { ...ctx.config.suite, code: true } } : ctx.config,
         cases,
         ...(ctx.writeTests ? { repo: ctx.cwd, coder } : {}),
+        browserProvider: selectedBrowser,
         parallel,
         writeBack: typeof args.write_back === "boolean" ? args.write_back : undefined,
         headed: ctx.headed,
@@ -761,7 +782,7 @@ function knowledgeNotes(cfg: AppConfig): string {
     : `No product knowledge yet: the team can describe the product, roles, rules, and test data in ${cfg.knowledge.dir} (forgeqe init adds templates).`
 }
 
-export function environmentNotes(ctx: Pick<ToolContext, "config" | "cwd" | "url">): string {
+export function environmentNotes(ctx: Pick<ToolContext, "config" | "cwd" | "url" | "browserProvider">): string {
   const { config: cfg } = ctx
   const s = cfg.sources
   const systems = [
@@ -798,13 +819,17 @@ export function environmentNotes(ctx: Pick<ToolContext, "config" | "cwd" | "url"
   const monitor = m.checks.length || channels.length
     ? `Monitoring (monitor: in forgeqe.yaml): ${m.checks.length ? `checks ${m.checks.join(" ")}` : "no checks yet"}, every ${Math.round(m.everyMs / 60_000)} min, alerts to ${channels.length ? channels.join(" and ") : "nobody yet"}.`
     : ""
+  const browser = cfg.browser || {}
   const browserSkills = skillsFor(loadSkills(cfg.skills, ctx.cwd), "browser")
+  const browserProvider = browser.provider || "local"
+  const remoteProviders = (["browserstack", "testmu"] as const).filter((name) => browser.remotes?.[name])
   const skills = browserSkills.length
     ? [`Browser skills (pass the ones a goal needs as skills to browser_run or run_test_cases):`, ...catalogLines(browserSkills)].join("\n")
     : ""
   return [
     `Working directory: ${ctx.cwd}`,
     `Browser runs are saved under ${cfg.export.outDir}; page memory is in ${cfg.memory.dir}.`,
+    `Browser provider default: ${browserProvider}.${remoteProviders.length ? ` Also configured: ${remoteProviders.join(", ")}. Pass browser_provider to browser_run or run_test_cases when the user asks for one of them.` : ""}`,
     `Test case systems configured: ${systems.length ? systems.join("; ") : "none (files and text still work)"}.`,
     s.confluence ? `Confluence: ${s.confluence.url}${s.confluence.space ? `, space ${s.confluence.space}` : ""} (read_confluence).` : "",
     `Sprints (list_sprints, find_tests sprint): ${[s.ado ? "Azure DevOps iterations" : "", s.jira ? "Jira scrum boards" : ""].filter(Boolean).join(" and ") || "none configured; ask the user for dates"}.`,
@@ -827,6 +852,7 @@ export function environmentNotes(ctx: Pick<ToolContext, "config" | "cwd" | "url"
     `Browser model profile: ${cfg.activeProfile} (${cfg.llm.provider}/${cfg.llm.deployment || cfg.llm.model}).`,
     keys.length ? `Set in ForgeQE's environment (names only, usable with set_ci_secret): ${keys.join(", ")}.` : "",
     ctx.url ? `The user pinned this start URL for browser runs: ${ctx.url}` : "",
+    ctx.browserProvider ? `The user pinned this browser provider for runs in this session: ${ctx.browserProvider}.` : "",
     skills,
     domainNotes(cfg),
     knowledgeNotes(cfg),

package/src/monitor/run.ts

@@ -3,7 +3,7 @@ import path from "node:path"
 
 import type { Card } from "../card.ts"
 import { loadCases, type TestCase } from "../cases/index.ts"
-import type { AppConfig } from "../config.ts"
+import type { AppConfig, BrowserProvider } from "../config.ts"
 import { counts, suiteUsage } from "../suite/report.ts"
 import { runSuite, type SuiteEvent, type SuiteOptions, type SuiteResult } from "../suite/run.ts"
 import { formatDuration, formatTokens } from "../tui/theme.ts"
@@ -28,6 +28,7 @@ export type MonitorOptions = {
   config: AppConfig
   /** Test case sources; default monitor.checks. */
   checks: string[]
+  browserProvider?: BrowserProvider
   headed?: boolean
   url?: string
   /** Browser skills for every check (empty: none); unset: the engine picks once per check for the monitor's lifetime. */
@@ -67,6 +68,7 @@ export async function monitorRound(opts: MonitorOptions): Promise<RoundResult> {
     suite = await runSuite({
       config: { ...config, suite: { ...config.suite, outDir: m.outDir, code: false } },
       cases,
+      browserProvider: opts.browserProvider,
       parallel: Math.min(m.parallel, cases.length),
       writeBack: m.writeBack,
       replay: m.replay,

package/src/prompts.ts

@@ -35,6 +35,8 @@ export type PromptInfo = {
   name: string
   description: string
   usedBy: string
+  /** What kind of prompt it is, which decides the standard it is checked against (src/promptstyle.ts). */
+  kind: string
   source: PromptSource | string
   file: string
   /** An edited copy whose built-in prompt changed since the copy was made. */
@@ -103,7 +105,7 @@ export function listPrompts(): PromptInfo[] {
       const name = f.replace(/\.md$/, "")
       const { meta } = parse(readFileSync(path.join(PROMPTS_DIR, f), "utf8"))
       const where = promptFile(name)
-      return { name, description: meta.description || "", usedBy: meta.used_by || "", source: where.source, file: where.file, outdated: where.outdated }
+      return { name, description: meta.description || "", usedBy: meta.used_by || "", kind: meta.kind || "", source: where.source, file: where.file, outdated: where.outdated }
     })
 }
 
@@ -114,12 +116,13 @@ export function promptWarnings(): string[] {
     .map((p) => `Your prompt ${p.name} (${p.file}) is based on an older version of ForgeQE's; forgeqe prompts diff ${p.name} shows what changed.`)
 }
 
-/** The browser engine's prompts (browser-*.md), resolved, for the engine to use instead of its own copies. */
+/** The browser engine's prompts (browser-*.md and conventions.md), resolved, for the engine to use instead of its own copies. */
 export function enginePrompts(): Record<string, string> {
   const out: Record<string, string> = {}
   for (const f of readdirSync(PROMPTS_DIR)) {
     const name = f.replace(/\.md$/, "")
-    if (!f.endsWith(".md") || !name.startsWith("browser-") || name === "browser-extra") continue
+    // The browser agent's prompts, and the conventions that state how their rules read.
+    if (!f.endsWith(".md") || !(name.startsWith("browser-") || name === "conventions") || name === "browser-extra") continue
     out[name] = prompt(name)
   }
   return out