@capagents/forgeqe 1.1.1 → 1.1.2

package/src/evals/prompts.ts

@@ -0,0 +1,288 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"
+import path from "node:path"
+
+import YAML from "yaml"
+
+import type { AppConfig } from "../config.ts"
+import { configuredSystems } from "../capabilities/status.ts"
+import { steScore } from "../ste.ts"
+import type { CodeEvent } from "../types.ts"
+
+/**
+ * Evaluation-driven prompting: each prompt that can run on its own has cases (evals/prompts/<prompt>/*.yaml in
+ * ForgeQE, and .forgeqe/evals/prompts/<prompt>/ for a project's own), each an input and checks that are decided by
+ * code, not by a model: a field matches or is empty, the agent used or did not use a tool, the text keeps ASD-STE100.
+ * `forgeqe prompts eval` runs them, several times when asked, and compares the pass rate with the run before, so a
+ * change to a prompt is measured, not guessed.
+ */
+
+export const BUILTIN_CASES = path.resolve(import.meta.dir, "..", "..", "evals", "prompts")
+
+/** One check. Exactly one of the test keys is set besides `id` (and `field` for the field tests). */
+export type Check = {
+  id: string
+  field?: string
+  matches?: string
+  not_matches?: string
+  empty?: boolean
+  min_items?: number
+  max_items?: number
+  /** Least share of sentences that keep ASD-STE100, of `field` (default: the output text). */
+  ste_min?: number
+  /** At least one tool call matches one of these names (`*` matches any text). */
+  tools_any?: string[]
+  /** No tool call matches any of these. */
+  tools_none?: string[]
+  /** The first call that matches the first list comes before any call that matches the second. */
+  tools_before?: [string[], string[]]
+}
+
+export type PromptCase = {
+  id: string
+  title: string
+  prompt: string
+  input: string
+  /** Systems the case reads: it is skipped when one is not set up. */
+  needs: string[]
+  /** brain: the ForgeQE tools that may run; others answer that they are not available (default: those that only read). */
+  allow?: string[]
+  checks: Check[]
+  file: string
+}
+
+export type Observation = { fields: Record<string, unknown>; text: string; tools: string[]; tokens: number; ms: number }
+export type CheckResult = { id: string; pass: boolean; detail: string }
+export type RunResult = { pass: boolean; checks: CheckResult[]; tokens: number; ms: number; error?: string; /** The start of what came out, for a run that failed. */ seen?: string }
+export type CaseResult = { prompt: string; case: string; title: string; runs: RunResult[]; skipped?: string }
+
+/** The ForgeQE tools ForgeQE's agent may call in an evaluation: each only reads. */
+export const READ_ONLY_TOOLS = ["capabilities", "find_test_cases", "check_test_cases", "read_confluence", "run_details", "query_api", "list_sprints", "page_memory", "flaky_tests", "quality_report", "quality_trends", "forgeqe_setup"]
+
+const TESTS = ["matches", "not_matches", "empty", "min_items", "max_items", "ste_min", "tools_any", "tools_none", "tools_before"] as const
+
+/** Every case in a folder of `<prompt>/<case>.yaml`, checked so a broken case fails before any model is called. */
+export function loadPromptCases(dirs: string[], only?: { prompt?: string; case?: string }): PromptCase[] {
+  const out: PromptCase[] = []
+  for (const dir of dirs) {
+    if (!existsSync(dir)) continue
+    for (const prompt of readdirSync(dir).sort()) {
+      if (only?.prompt && prompt !== only.prompt) continue
+      const folder = path.join(dir, prompt)
+      let files: string[] = []
+      try {
+        files = readdirSync(folder).filter((f) => /\.ya?ml$/.test(f)).sort()
+      } catch {
+        continue
+      }
+      for (const f of files) {
+        const file = path.join(folder, f)
+        const c = parseCase(YAML.parse(readFileSync(file, "utf8")), prompt, file)
+        if (only?.case && c.id !== only.case) continue
+        // A project's case replaces ForgeQE's case of the same id.
+        const at = out.findIndex((x) => x.prompt === c.prompt && x.id === c.id)
+        if (at >= 0) out.splice(at, 1)
+        out.push(c)
+      }
+    }
+  }
+  return out
+}
+
+export function parseCase(raw: any, prompt: string, file: string): PromptCase {
+  const where = `${path.basename(path.dirname(file))}/${path.basename(file)}`
+  if (!raw?.id || !raw.input) throw new Error(`${where}: id and input are required.`)
+  if (!Array.isArray(raw.checks) || !raw.checks.length) throw new Error(`${where}: checks needs at least one check.`)
+  const checks: Check[] = raw.checks.map((c: any, i: number) => {
+    if (!c?.id) throw new Error(`${where}: check ${i + 1} needs an id.`)
+    const tests = TESTS.filter((t) => c[t] !== undefined)
+    if (tests.length !== 1) throw new Error(`${where}: check ${c.id} needs exactly one of ${TESTS.join(", ")}.`)
+    if (["matches", "not_matches", "empty", "min_items", "max_items"].includes(tests[0]!) && !c.field) throw new Error(`${where}: check ${c.id} needs field.`)
+    for (const k of ["matches", "not_matches"]) if (c[k] !== undefined) new RegExp(String(c[k]), "i")
+    return c as Check
+  })
+  const ids = checks.map((c) => c.id)
+  if (new Set(ids).size !== ids.length) throw new Error(`${where}: check ids must be unique.`)
+  return {
+    id: String(raw.id),
+    title: String(raw.title || raw.id),
+    prompt: String(raw.prompt || prompt),
+    input: String(raw.input),
+    needs: (raw.needs || []).map(String),
+    ...(raw.allow ? { allow: raw.allow.map(String) } : {}),
+    checks,
+    file,
+  }
+}
+
+const glob = (pattern: string) => new RegExp(`^${pattern.split("*").map((p) => p.replace(/[.+?^${}()|[\]\\]/g, "\\$&")).join(".*")}$`, "i")
+const matchesAny = (tool: string, patterns: string[]) => patterns.some((p) => glob(p).test(tool))
+const asText = (v: unknown): string => (v === undefined || v === null ? "" : typeof v === "string" ? v : Array.isArray(v) ? v.map(asText).join("\n") : JSON.stringify(v))
+
+/** Each check decided against what one run did. */
+export function judge(checks: Check[], o: Observation): CheckResult[] {
+  return checks.map((c): CheckResult => {
+    const value = c.field ? o.fields[c.field] : o.text
+    const text = asText(value)
+    const items = Array.isArray(value) ? value.length : text.trim() ? 1 : 0
+    if (c.matches !== undefined) {
+      const pass = new RegExp(c.matches, "i").test(text)
+      return { id: c.id, pass, detail: pass ? `${c.field} matches /${c.matches}/` : `${c.field} does not match /${c.matches}/: ${clip(text)}` }
+    }
+    if (c.not_matches !== undefined) {
+      const m = new RegExp(c.not_matches, "i").exec(text)
+      return { id: c.id, pass: !m, detail: m ? `${c.field} has "${m[0]}": ${clip(text)}` : `${c.field} does not match /${c.not_matches}/` }
+    }
+    if (c.empty !== undefined) {
+      const pass = (items === 0) === c.empty
+      return { id: c.id, pass, detail: pass ? `${c.field} is ${c.empty ? "empty" : "not empty"}` : `${c.field} is ${c.empty ? `not empty: ${clip(text)}` : "empty"}` }
+    }
+    if (c.min_items !== undefined) return { id: c.id, pass: items >= c.min_items, detail: `${c.field} has ${items} (at least ${c.min_items})` }
+    if (c.max_items !== undefined) return { id: c.id, pass: items <= c.max_items, detail: `${c.field} has ${items} (at most ${c.max_items})` }
+    if (c.ste_min !== undefined) {
+      const score = steScore(text).score
+      return { id: c.id, pass: score >= c.ste_min, detail: `ASD-STE100 ${Math.round(score * 100)}% (at least ${Math.round(c.ste_min * 100)}%)` }
+    }
+    if (c.tools_any) {
+      const hit = o.tools.find((t) => matchesAny(t, c.tools_any!))
+      return { id: c.id, pass: Boolean(hit), detail: hit ? `used ${hit}` : `used none of ${c.tools_any.join(", ")} (used: ${clip(o.tools.join(", ") || "no tools")})` }
+    }
+    if (c.tools_none) {
+      const hit = o.tools.filter((t) => matchesAny(t, c.tools_none!))
+      return { id: c.id, pass: !hit.length, detail: hit.length ? `used ${[...new Set(hit)].join(", ")}` : `used none of ${c.tools_none.join(", ")}` }
+    }
+    const [first, then] = c.tools_before!
+    const a = o.tools.findIndex((t) => matchesAny(t, first))
+    const b = o.tools.findIndex((t) => matchesAny(t, then))
+    const pass = a >= 0 && (b < 0 || a < b)
+    return { id: c.id, pass, detail: a < 0 ? `used none of ${first.join(", ")}` : b >= 0 && b < a ? `used ${o.tools[b]} before ${o.tools[a]}` : `used ${o.tools[a]} first` }
+  })
+}
+
+const clip = (s: string, n = 120) => {
+  const t = s.replace(/\s+/g, " ").trim()
+  return t.length <= n ? t : `${t.slice(0, n - 1)}…`
+}
+
+/** Why a case cannot run with this config, or null. */
+export function skipReason(c: PromptCase, config: AppConfig): string | null {
+  if (config.llm.provider === "none" || config.llm.provider === "mock") return "no live model"
+  const systems = configuredSystems(config)
+  const missing = c.needs.filter((n) => !systems.has(n as never))
+  return missing.length ? `needs ${missing.join(", ")}` : null
+}
+
+export type Runner = (c: PromptCase, config: AppConfig, cwd: string) => Promise<Observation>
+
+/** Tokens a session used, from its last usage event (the runtime reports running totals). */
+function usageMeter() {
+  let tokens = 0
+  return {
+    onEvent: (e: CodeEvent) => {
+      if (e.kind === "usage") tokens = e.usage.input + e.usage.output
+    },
+    get tokens() {
+      return tokens
+    },
+  }
+}
+
+/** How each prompt that can run on its own is run for a case. */
+export const RUNNERS: Record<string, Runner> = {
+  async refine(c, config, cwd) {
+    const { refine } = await import("../refine.ts")
+    const meter = usageMeter()
+    const started = Date.now()
+    const r = await refine({ config, cwd, text: c.input, recent: [], onEvent: meter.onEvent })
+    const fields: Record<string, unknown> = { ...(r.spec || {}), refined: r.refined, missing: r.missing, assumed: r.assumed, grounded: r.grounded.map((g) => g.fact) }
+    return { fields, text: r.refined, tools: [], tokens: meter.tokens, ms: Date.now() - started }
+  },
+
+  async brain(c, config, cwd) {
+    const { openBrain } = await import("../brain/index.ts")
+    const tools: string[] = []
+    let tokens = 0
+    const allow = new Set(c.allow || READ_ONLY_TOOLS)
+    const started = Date.now()
+    const brain = await openBrain({
+      config,
+      cwd,
+      context: () => ({ session: "shell" as const, observe: (tool: string) => tools.push(`forgeqe_${tool}`), allow: (tool: string) => allow.has(tool) }),
+      onEvent: (e) => {
+        if (e.kind === "usage") tokens = e.usage.input + e.usage.output
+        // ForgeQE's own tools are hidden from these events; observe records them.
+        if (e.kind === "tool" && e.tool && !e.tool.startsWith("forgeqe_")) tools.push(e.tool)
+      },
+    })
+    try {
+      const a = await brain.ask(c.input)
+      return { fields: { reply: a.reply, outcome: a.outcome, next: (a.next || []).map((n) => n.label) }, text: a.reply, tools, tokens, ms: Date.now() - started }
+    } finally {
+      await brain.close()
+    }
+  },
+
+  async "explore-charters"(c, config, cwd) {
+    const { planCharters } = await import("../explore/charters.ts")
+    const meter = usageMeter()
+    const started = Date.now()
+    const [title, ...rest] = c.input.split("\n")
+    const acceptance = rest.join("\n")
+    const story = { id: "story", system: "text" as const, title: title!.trim(), description: "", acceptance, parents: [], scripted: [], passages: [], errors: [] }
+    const r = await planCharters({ config, story, count: 5, roles: [], cwd, onEvent: meter.onEvent })
+    const fields = { charters: r.charters, titles: r.charters.map((x) => x.title), categories: r.charters.map((x) => x.category), steps: r.charters.flatMap((x) => x.steps.map((s) => `${s.action} → ${s.expected}`)) }
+    // Each mission is a paragraph of its own, ended as a sentence, so writing checks score each one.
+    const text = r.charters.map((x) => (/[.!?]$/.test(x.mission.trim()) ? x.mission.trim() : `${x.mission.trim()}.`)).join("\n\n")
+    return { fields, text, tools: [], tokens: meter.tokens, ms: Date.now() - started }
+  },
+}
+
+export async function runCase(c: PromptCase, config: AppConfig, cwd: string, runs: number): Promise<CaseResult> {
+  const base = { prompt: c.prompt, case: c.id, title: c.title }
+  const why = skipReason(c, config)
+  if (why) return { ...base, runs: [], skipped: why }
+  const runner = RUNNERS[c.prompt]
+  if (!runner) return { ...base, runs: [], skipped: `no runner for ${c.prompt} yet` }
+  const out: RunResult[] = []
+  for (let i = 0; i < runs; i++) {
+    const started = Date.now()
+    try {
+      const o = await runner(c, config, cwd)
+      const checks = judge(c.checks, o)
+      const pass = checks.every((x) => x.pass)
+      out.push({ pass, checks, tokens: o.tokens, ms: o.ms, ...(pass ? {} : { seen: clip(o.text, 300) }) })
+    } catch (err) {
+      out.push({ pass: false, checks: [], tokens: 0, ms: Date.now() - started, error: (err as Error).message })
+    }
+  }
+  return { ...base, runs: out }
+}
+
+/** Where results are kept, one file per `forgeqe prompts eval`. */
+export const resultsDir = (config: AppConfig) => path.join(config.export.outDir, "evals", "prompts")
+
+export function saveResults(config: AppConfig, results: CaseResult[], at = new Date()): string {
+  const dir = resultsDir(config)
+  mkdirSync(dir, { recursive: true })
+  const file = path.join(dir, `${at.toISOString().replace(/[:.]/g, "-")}.json`)
+  writeFileSync(file, JSON.stringify({ at: at.toISOString(), results }, null, 2))
+  return file
+}
+
+/** The pass rate of each case in the latest earlier results that ran it: "prompt/case" → [passed, runs]. */
+export function previousRates(config: AppConfig, except?: string): Map<string, [number, number]> {
+  const dir = resultsDir(config)
+  const out = new Map<string, [number, number]>()
+  if (!existsSync(dir)) return out
+  const files = readdirSync(dir).filter((f) => f.endsWith(".json")).sort().reverse()
+  for (const f of files) {
+    if (except && path.join(dir, f) === except) continue
+    try {
+      const { results } = JSON.parse(readFileSync(path.join(dir, f), "utf8")) as { results: CaseResult[] }
+      for (const r of results) {
+        const key = `${r.prompt}/${r.case}`
+        if (!out.has(key) && r.runs.length) out.set(key, [r.runs.filter((x) => x.pass).length, r.runs.length])
+      }
+    } catch {}
+  }
+  return out
+}

package/src/mcp/server.ts

@@ -42,6 +42,14 @@ export function createMcpServer(opts: McpServerOptions) {
     const timer = token !== undefined ? setInterval(() => send({ jsonrpc: "2.0", method: "notifications/progress", params: { progressToken: token, progress: ++tick, message: `${tool.name} running` } }), opts.progressMs || PROGRESS_EVERY_MS) : null
     const started = Date.now()
     logEvent("tool.call", { tool: tool.name, arguments: params.arguments || {} })
+    base.observe?.(tool.name, params.arguments || {})
+    if (base.allow && !base.allow(tool.name)) {
+      running.delete(id)
+      if (timer) clearInterval(timer)
+      base.signal?.removeEventListener("abort", abort)
+      outer?.removeEventListener("abort", abort)
+      return { content: [{ type: "text", text: `${tool.name} is not available in this evaluation. Say what you would do with it, and stop.` }], isError: true }
+    }
     const finish = (result: Json) => {
       logEvent("tool.result", { tool: tool.name, ms: Date.now() - started, error: Boolean(result.isError), text: result.content?.[0]?.text })
       return result

package/src/mcp/tools.ts

@@ -130,6 +130,10 @@ export type ToolContext = {
   url?: string
   signal?: AbortSignal
   emit?: (e: ToolEvent) => void
+  /** Sees every call before it runs, with its arguments: evaluations record which tools the agent chose. */
+  observe?: (tool: string, args: Record<string, unknown>) => void
+  /** Whether a tool may run; a refused call answers that it is not available here (evaluations refuse tools that act). */
+  allow?: (tool: string) => boolean
   /** How long this process lives: the shell, one `forgeqe run`, or `forgeqe mcp`. Background monitors need shell or server. */
   session?: "shell" | "once" | "server"
   /** Passed browser runs become tests in cwd (the shell and `forgeqe run` without --no-code). */

package/src/prompts.ts

@@ -35,6 +35,8 @@ export type PromptInfo = {
   name: string
   description: string
   usedBy: string
+  /** What kind of prompt it is, which decides the standard it is checked against (src/promptstyle.ts). */
+  kind: string
   source: PromptSource | string
   file: string
   /** An edited copy whose built-in prompt changed since the copy was made. */
@@ -103,7 +105,7 @@ export function listPrompts(): PromptInfo[] {
       const name = f.replace(/\.md$/, "")
       const { meta } = parse(readFileSync(path.join(PROMPTS_DIR, f), "utf8"))
       const where = promptFile(name)
-      return { name, description: meta.description || "", usedBy: meta.used_by || "", source: where.source, file: where.file, outdated: where.outdated }
+      return { name, description: meta.description || "", usedBy: meta.used_by || "", kind: meta.kind || "", source: where.source, file: where.file, outdated: where.outdated }
     })
 }
 
@@ -114,12 +116,13 @@ export function promptWarnings(): string[] {
     .map((p) => `Your prompt ${p.name} (${p.file}) is based on an older version of ForgeQE's; forgeqe prompts diff ${p.name} shows what changed.`)
 }
 
-/** The browser engine's prompts (browser-*.md), resolved, for the engine to use instead of its own copies. */
+/** The browser engine's prompts (browser-*.md and conventions.md), resolved, for the engine to use instead of its own copies. */
 export function enginePrompts(): Record<string, string> {
   const out: Record<string, string> = {}
   for (const f of readdirSync(PROMPTS_DIR)) {
     const name = f.replace(/\.md$/, "")
-    if (!f.endsWith(".md") || !name.startsWith("browser-") || name === "browser-extra") continue
+    // The browser agent's prompts, and the conventions that state how their rules read.
+    if (!f.endsWith(".md") || !(name.startsWith("browser-") || name === "conventions") || name === "browser-extra") continue
     out[name] = prompt(name)
   }
   return out

package/src/promptstyle.ts

@@ -0,0 +1,159 @@
+import { steParagraphs, steScore, STE_TARGET, RFC_LEAD } from "./ste.ts"
+
+/**
+ * The prompting standard every ForgeQE prompt follows, and what of it can be checked in the text. Each technique is
+ * used where it fits the kind of prompt (frontmatter `kind`):
+ *
+ * | kind       | what it is                                     | structure                                                   | few-shot | execution contract |
+ * |------------|------------------------------------------------|-------------------------------------------------------------|----------|--------------------|
+ * | agent      | works with tools over many steps               | Role, Objective, Execution contract, Rules, Output          | optional | required           |
+ * | structured | one answer in a fixed shape (plans, verdicts)  | Role, Objective, Inputs, Rules, Output, Examples            | required | no                 |
+ * | fragment   | rules included in other prompts                | Rules                                                       | no       | no                 |
+ * | user       | the user's own text (browser-extra)            | not checked                                                 | no       | no                 |
+ *
+ * Every kind but user is written in ASD-STE100 (src/ste.ts) and states its rules with the RFC 2119 key words: MUST and
+ * MUST NOT for absolute rules, SHOULD and SHOULD NOT for strong defaults, MAY for a free choice. A lower-case "must",
+ * "should", or "shall" is ambiguous, so it is flagged. Evaluation cases (evals/prompts/<name>/) measure how reliably
+ * a prompt does its task; `forgeqe prompts eval` runs them.
+ */
+
+export type PromptKind = "agent" | "structured" | "fragment" | "user"
+export const PROMPT_KINDS: PromptKind[] = ["agent", "structured", "fragment", "user"]
+
+/** The `##` sections each kind has, in this order. Other sections may come between them. */
+export const SECTIONS: Record<PromptKind, string[]> = {
+  agent: ["Role", "Objective", "Execution contract", "Rules", "Output"],
+  structured: ["Role", "Objective", "Inputs", "Rules", "Output", "Examples"],
+  fragment: [],
+  user: [],
+}
+
+/** The parts of an execution contract, each a bold label or a `###` heading inside it. */
+export const CONTRACT_PARTS = ["Preconditions", "Tools", "Procedure", "Stop when", "On failure", "Evidence"]
+
+/** Fewest worked examples a structured prompt gives. */
+export const MIN_EXAMPLES = 2
+
+/** Share of rule lines (list items under a Rules section) that carry an RFC 2119 key word. */
+export const RFC_TARGET = 0.8
+
+const KEYWORD = /\b(MUST NOT|MUST|SHOULD NOT|SHOULD|MAY|SHALL NOT|SHALL|REQUIRED|RECOMMENDED|OPTIONAL)\b/
+/** Lower-case words that read like a rule but say nothing of its strength. */
+const AMBIGUOUS = /\b(must|should|shall)\b/
+
+export type StyleIssue = { rule: "ste" | "rfc" | "ambiguous" | "section" | "order" | "examples" | "contract" | "kind"; detail: string }
+
+export type StyleReport = {
+  kind: PromptKind
+  ste: number
+  /** Rule lines with a key word, of all rule lines; 1 when the prompt has none. */
+  rfc: number
+  ruleLines: number
+  examples: number
+  issues: StyleIssue[]
+}
+
+/** The `##` headings of a prompt, in order, without code blocks. */
+export function headings(markdown: string, level = 2): string[] {
+  const out: string[] = []
+  let fence = false
+  const mark = `${"#".repeat(level)} `
+  for (const line of markdown.split("\n")) {
+    if (/^\s*(```|~~~)/.test(line)) fence = !fence
+    else if (!fence && line.startsWith(mark)) out.push(line.slice(mark.length).trim())
+  }
+  return out
+}
+
+/** The text under a `##` heading, up to the next `##` heading. */
+export function section(markdown: string, name: string): string {
+  const lines = markdown.split("\n")
+  const at = lines.findIndex((l) => l.trim().toLowerCase() === `## ${name}`.toLowerCase())
+  if (at < 0) return ""
+  const end = lines.findIndex((l, i) => i > at && /^## /.test(l))
+  return lines.slice(at + 1, end < 0 ? undefined : end).join("\n")
+}
+
+/** Text with code, quotes, and placeholders taken out: a quoted example is not a rule. */
+const prose = (text: string) =>
+  text
+    .replace(/```[\s\S]*?```/g, "")
+    .replace(/`[^`]*`/g, "")
+    .replace(/"[^"\n]*"/g, "")
+    .replace(/“[^”\n]*”/g, "")
+    .replace(/\{\{[^}]*\}\}/g, "")
+
+/**
+ * The rules of the Rules sections (and of a fragment's whole text). A list item is one rule, with what is nested under
+ * it as its details; only an item without a key word that opens a list ("Write replies so that they read quickly:")
+ * hands its nested items on, each a rule of its own.
+ */
+export function ruleLines(markdown: string, kind: PromptKind): string[] {
+  const parts = kind === "fragment" ? [markdown] : headings(markdown).filter((h) => /rules/i.test(h)).map((h) => section(markdown, h))
+  const items: string[] = []
+  for (const part of parts) {
+    // The indent of the items that count as rules at each depth; deeper items under a rule are its details.
+    let ruleIndent = -1
+    for (const line of prose(part).split("\n")) {
+      const m = /^(\s*)(?:[-*+]|\d+[.)])\s+(.*)$/.exec(line)
+      if (!m) continue
+      const indent = m[1]!.length
+      const text = m[2]!.trim()
+      if (!text) continue
+      if (ruleIndent >= 0 && indent > ruleIndent) continue
+      // A numbered step is a MUST by the conventions, unless it says SHOULD or MAY.
+      const numbered = /^\s*\d+[.)]/.test(line)
+      const opensList = /:\s*$/.test(text) && !KEYWORD.test(text) && !numbered
+      if (opensList) {
+        ruleIndent = -1
+        continue
+      }
+      items.push(numbered && !KEYWORD.test(text) ? `MUST ${text}` : text)
+      ruleIndent = indent
+    }
+  }
+  return items
+}
+
+export function checkStyle(markdown: string, kind: PromptKind): StyleReport {
+  const issues: StyleIssue[] = []
+  if (kind === "user") return { kind, ste: 1, rfc: 1, ruleLines: 0, examples: 0, issues }
+  const ste = steScore(markdown).score
+  if (ste < STE_TARGET) issues.push({ rule: "ste", detail: `${Math.round(ste * 100)}% of sentences keep ASD-STE100; the target is ${Math.round(STE_TARGET * 100)}%` })
+
+  const rules = ruleLines(markdown, kind)
+  const keyed = rules.filter((r) => KEYWORD.test(r)).length
+  const rfc = rules.length ? keyed / rules.length : 1
+  if (rules.length && rfc < RFC_TARGET) issues.push({ rule: "rfc", detail: `${keyed} of ${rules.length} rules use an RFC 2119 key word; the target is ${Math.round(RFC_TARGET * 100)}%` })
+  for (const sentence of steParagraphs(prose(markdown)).flat()) {
+    const m = AMBIGUOUS.exec(sentence)
+    if (m) issues.push({ rule: "ambiguous", detail: `"${m[0]}" in: ${sentence.slice(0, 100)} — use ${m[0].toUpperCase()} (RFC 2119), or say it as a fact` })
+  }
+
+  const have = headings(markdown)
+  const need = SECTIONS[kind]
+  for (const name of need) if (!have.some((h) => h.toLowerCase() === name.toLowerCase())) issues.push({ rule: "section", detail: `no "## ${name}" section` })
+  const order = need.map((n) => have.findIndex((h) => h.toLowerCase() === n.toLowerCase())).filter((i) => i >= 0)
+  if (order.some((v, i) => i > 0 && v < order[i - 1]!)) issues.push({ rule: "order", detail: `sections out of order; use ${need.join(", ")}` })
+
+  const examples = headings(markdown, 3).filter((h) => /^example\b/i.test(h)).length
+  if (kind === "structured" && examples < MIN_EXAMPLES) issues.push({ rule: "examples", detail: `${examples} worked examples ("### Example …"); a structured prompt gives ${MIN_EXAMPLES} or more` })
+
+  if (kind === "agent") {
+    const contract = section(markdown, "Execution contract")
+    for (const part of CONTRACT_PARTS) {
+      const label = new RegExp(`(\\*\\*${part}\\*\\*|^###\\s+${part}\\b)`, "im")
+      if (contract && !label.test(contract)) issues.push({ rule: "contract", detail: `the execution contract has no "${part}" part` })
+    }
+  }
+  return { kind, ste, rfc, ruleLines: rules.length, examples, issues }
+}
+
+export function promptKind(value: string | undefined): PromptKind | undefined {
+  return PROMPT_KINDS.find((k) => k === value)
+}
+
+/** Whether a sentence states a rule with an RFC 2119 key word ("You MUST …", "… MUST NOT …"). */
+export function isRule(sentence: string): boolean {
+  return RFC_LEAD.test(sentence.trim()) || KEYWORD.test(sentence)
+}

package/src/run.ts

@@ -216,6 +216,7 @@ export async function executeRun(opts: ExecuteOptions): Promise<{ result: RunRes
       headed: Boolean(opts.headed) || !opts.config.browser.headless,
       maxSteps,
       allowedHosts: allowed,
+      browser: { provider: opts.config.browser.provider, remote: opts.config.browser.remote },
       extendSystemMessage: extend || undefined,
       memory,
       env,

package/src/ste.ts

@@ -91,8 +91,13 @@ export function countWords(text: string): number {
   return (collapse(text).match(/[A-Za-z0-9][\w'’./-]*/g) || []).length
 }
 
+/** "You MUST", "The agent SHOULD NOT": the RFC 2119 key word that makes a sentence a rule, with its subject. */
+export const RFC_LEAD = /^(([Yy]ou|[Tt]he agent|[Tt]he reply|[Ee]ach \w+|[Ee]very \w+)\s+)?(MUST NOT|MUST|SHOULD NOT|SHOULD|MAY|SHALL NOT|SHALL)\s+/
+
 export function isInstruction(sentence: string): boolean {
-  const s = collapse(sentence).replace(/^[\s>*_-]+/, "").replace(LEAD, "")
+  const lead = collapse(sentence).replace(/^[\s>*_-]+/, "").replace(LEAD, "")
+  // "You MUST run the tests." is an instruction: the verb after the key word decides, as without it.
+  const s = RFC_LEAD.test(lead) ? lead.replace(RFC_LEAD, "") : lead
   const first = (s.match(/^[A-Za-z']+/)?.[0] || "").toLowerCase()
   return VERBS.has(first)
 }
@@ -151,7 +156,8 @@ function splitSentences(text: string): string[] {
 }
 
 export function checkSentence(text: string): SteSentence {
-  const words = countWords(text)
+  // The subject and RFC 2119 key word of a rule ("You MUST NOT") do not count toward its length.
+  const words = countWords(text) - countWords(collapse(text).replace(/^[\s>*_-]+/, "").replace(LEAD, "").match(RFC_LEAD)?.[0] || "")
   const instruction = isInstruction(text)
   const issues: SteSentence["issues"] = []
   const limit = instruction ? 20 : 25