@capagents/forgeqe 1.1.1 → 1.1.3

package/src/promptstyle.ts

@@ -0,0 +1,159 @@
+import { steParagraphs, steScore, STE_TARGET, RFC_LEAD } from "./ste.ts"
+
+/**
+ * The prompting standard every ForgeQE prompt follows, and what of it can be checked in the text. Each technique is
+ * used where it fits the kind of prompt (frontmatter `kind`):
+ *
+ * | kind       | what it is                                     | structure                                                   | few-shot | execution contract |
+ * |------------|------------------------------------------------|-------------------------------------------------------------|----------|--------------------|
+ * | agent      | works with tools over many steps               | Role, Objective, Execution contract, Rules, Output          | optional | required           |
+ * | structured | one answer in a fixed shape (plans, verdicts)  | Role, Objective, Inputs, Rules, Output, Examples            | required | no                 |
+ * | fragment   | rules included in other prompts                | Rules                                                       | no       | no                 |
+ * | user       | the user's own text (browser-extra)            | not checked                                                 | no       | no                 |
+ *
+ * Every kind but user is written in ASD-STE100 (src/ste.ts) and states its rules with the RFC 2119 key words: MUST and
+ * MUST NOT for absolute rules, SHOULD and SHOULD NOT for strong defaults, MAY for a free choice. A lower-case "must",
+ * "should", or "shall" is ambiguous, so it is flagged. Evaluation cases (evals/prompts/<name>/) measure how reliably
+ * a prompt does its task; `forgeqe prompts eval` runs them.
+ */
+
+export type PromptKind = "agent" | "structured" | "fragment" | "user"
+export const PROMPT_KINDS: PromptKind[] = ["agent", "structured", "fragment", "user"]
+
+/** The `##` sections each kind has, in this order. Other sections may come between them. */
+export const SECTIONS: Record<PromptKind, string[]> = {
+  agent: ["Role", "Objective", "Execution contract", "Rules", "Output"],
+  structured: ["Role", "Objective", "Inputs", "Rules", "Output", "Examples"],
+  fragment: [],
+  user: [],
+}
+
+/** The parts of an execution contract, each a bold label or a `###` heading inside it. */
+export const CONTRACT_PARTS = ["Preconditions", "Tools", "Procedure", "Stop when", "On failure", "Evidence"]
+
+/** Fewest worked examples a structured prompt gives. */
+export const MIN_EXAMPLES = 2
+
+/** Share of rule lines (list items under a Rules section) that carry an RFC 2119 key word. */
+export const RFC_TARGET = 0.8
+
+const KEYWORD = /\b(MUST NOT|MUST|SHOULD NOT|SHOULD|MAY|SHALL NOT|SHALL|REQUIRED|RECOMMENDED|OPTIONAL)\b/
+/** Lower-case words that read like a rule but say nothing of its strength. */
+const AMBIGUOUS = /\b(must|should|shall)\b/
+
+export type StyleIssue = { rule: "ste" | "rfc" | "ambiguous" | "section" | "order" | "examples" | "contract" | "kind"; detail: string }
+
+export type StyleReport = {
+  kind: PromptKind
+  ste: number
+  /** Rule lines with a key word, of all rule lines; 1 when the prompt has none. */
+  rfc: number
+  ruleLines: number
+  examples: number
+  issues: StyleIssue[]
+}
+
+/** The `##` headings of a prompt, in order, without code blocks. */
+export function headings(markdown: string, level = 2): string[] {
+  const out: string[] = []
+  let fence = false
+  const mark = `${"#".repeat(level)} `
+  for (const line of markdown.split("\n")) {
+    if (/^\s*(```|~~~)/.test(line)) fence = !fence
+    else if (!fence && line.startsWith(mark)) out.push(line.slice(mark.length).trim())
+  }
+  return out
+}
+
+/** The text under a `##` heading, up to the next `##` heading. */
+export function section(markdown: string, name: string): string {
+  const lines = markdown.split("\n")
+  const at = lines.findIndex((l) => l.trim().toLowerCase() === `## ${name}`.toLowerCase())
+  if (at < 0) return ""
+  const end = lines.findIndex((l, i) => i > at && /^## /.test(l))
+  return lines.slice(at + 1, end < 0 ? undefined : end).join("\n")
+}
+
+/** Text with code, quotes, and placeholders taken out: a quoted example is not a rule. */
+const prose = (text: string) =>
+  text
+    .replace(/```[\s\S]*?```/g, "")
+    .replace(/`[^`]*`/g, "")
+    .replace(/"[^"\n]*"/g, "")
+    .replace(/“[^”\n]*”/g, "")
+    .replace(/\{\{[^}]*\}\}/g, "")
+
+/**
+ * The rules of the Rules sections (and of a fragment's whole text). A list item is one rule, with what is nested under
+ * it as its details; only an item without a key word that opens a list ("Write replies so that they read quickly:")
+ * hands its nested items on, each a rule of its own.
+ */
+export function ruleLines(markdown: string, kind: PromptKind): string[] {
+  const parts = kind === "fragment" ? [markdown] : headings(markdown).filter((h) => /rules/i.test(h)).map((h) => section(markdown, h))
+  const items: string[] = []
+  for (const part of parts) {
+    // The indent of the items that count as rules at each depth; deeper items under a rule are its details.
+    let ruleIndent = -1
+    for (const line of prose(part).split("\n")) {
+      const m = /^(\s*)(?:[-*+]|\d+[.)])\s+(.*)$/.exec(line)
+      if (!m) continue
+      const indent = m[1]!.length
+      const text = m[2]!.trim()
+      if (!text) continue
+      if (ruleIndent >= 0 && indent > ruleIndent) continue
+      // A numbered step is a MUST by the conventions, unless it says SHOULD or MAY.
+      const numbered = /^\s*\d+[.)]/.test(line)
+      const opensList = /:\s*$/.test(text) && !KEYWORD.test(text) && !numbered
+      if (opensList) {
+        ruleIndent = -1
+        continue
+      }
+      items.push(numbered && !KEYWORD.test(text) ? `MUST ${text}` : text)
+      ruleIndent = indent
+    }
+  }
+  return items
+}
+
+export function checkStyle(markdown: string, kind: PromptKind): StyleReport {
+  const issues: StyleIssue[] = []
+  if (kind === "user") return { kind, ste: 1, rfc: 1, ruleLines: 0, examples: 0, issues }
+  const ste = steScore(markdown).score
+  if (ste < STE_TARGET) issues.push({ rule: "ste", detail: `${Math.round(ste * 100)}% of sentences keep ASD-STE100; the target is ${Math.round(STE_TARGET * 100)}%` })
+
+  const rules = ruleLines(markdown, kind)
+  const keyed = rules.filter((r) => KEYWORD.test(r)).length
+  const rfc = rules.length ? keyed / rules.length : 1
+  if (rules.length && rfc < RFC_TARGET) issues.push({ rule: "rfc", detail: `${keyed} of ${rules.length} rules use an RFC 2119 key word; the target is ${Math.round(RFC_TARGET * 100)}%` })
+  for (const sentence of steParagraphs(prose(markdown)).flat()) {
+    const m = AMBIGUOUS.exec(sentence)
+    if (m) issues.push({ rule: "ambiguous", detail: `"${m[0]}" in: ${sentence.slice(0, 100)} — use ${m[0].toUpperCase()} (RFC 2119), or say it as a fact` })
+  }
+
+  const have = headings(markdown)
+  const need = SECTIONS[kind]
+  for (const name of need) if (!have.some((h) => h.toLowerCase() === name.toLowerCase())) issues.push({ rule: "section", detail: `no "## ${name}" section` })
+  const order = need.map((n) => have.findIndex((h) => h.toLowerCase() === n.toLowerCase())).filter((i) => i >= 0)
+  if (order.some((v, i) => i > 0 && v < order[i - 1]!)) issues.push({ rule: "order", detail: `sections out of order; use ${need.join(", ")}` })
+
+  const examples = headings(markdown, 3).filter((h) => /^example\b/i.test(h)).length
+  if (kind === "structured" && examples < MIN_EXAMPLES) issues.push({ rule: "examples", detail: `${examples} worked examples ("### Example …"); a structured prompt gives ${MIN_EXAMPLES} or more` })
+
+  if (kind === "agent") {
+    const contract = section(markdown, "Execution contract")
+    for (const part of CONTRACT_PARTS) {
+      const label = new RegExp(`(\\*\\*${part}\\*\\*|^###\\s+${part}\\b)`, "im")
+      if (contract && !label.test(contract)) issues.push({ rule: "contract", detail: `the execution contract has no "${part}" part` })
+    }
+  }
+  return { kind, ste, rfc, ruleLines: rules.length, examples, issues }
+}
+
+export function promptKind(value: string | undefined): PromptKind | undefined {
+  return PROMPT_KINDS.find((k) => k === value)
+}
+
+/** Whether a sentence states a rule with an RFC 2119 key word ("You MUST …", "… MUST NOT …"). */
+export function isRule(sentence: string): boolean {
+  return RFC_LEAD.test(sentence.trim()) || KEYWORD.test(sentence)
+}

package/src/run.ts

@@ -5,7 +5,7 @@ import path from "node:path"
 import { assertGoalUrlsAllowed, hostAllowed, normalizeHttpUrl } from "./guard.ts"
 import { missingKey } from "./llm.ts"
 import type { AgentEvent, ReplayMode, RunResult } from "./types.ts"
-import type { AppConfig } from "./config.ts"
+import { browserConfigFor, type AppConfig, type BrowserProvider } from "./config.ts"
 import { runEngine } from "./engine/bridge.ts"
 import { MOCK_ENGINE } from "./engine/mock.ts"
 import { ensureEngine } from "./engine/setup.ts"
@@ -23,6 +23,7 @@ import { knowledgeBlock, readKnowledge, relevantKnowledge } from "./knowledge.ts
 export type ExecuteOptions = {
   config: AppConfig
   goal: string
+  browserProvider?: BrowserProvider
   url?: string
   headed?: boolean
   maxSteps?: number
@@ -148,6 +149,7 @@ export async function executeRun(opts: ExecuteOptions): Promise<{ result: RunRes
 
   const dir = opts.outDir || runDirectory(opts.config.export.outDir)
   const maxSteps = opts.maxSteps || opts.config.browser.maxSteps
+  const browser = browserConfigFor(opts.config, opts.browserProvider)
   const t0 = Date.now()
   const withReport = (out: { result: RunResult; files: string[] }) => {
     out.result.cost = costOf(out.result.usage, opts.config.llm.price)
@@ -216,6 +218,7 @@ export async function executeRun(opts: ExecuteOptions): Promise<{ result: RunRes
       headed: Boolean(opts.headed) || !opts.config.browser.headless,
       maxSteps,
       allowedHosts: allowed,
+      browser,
       extendSystemMessage: extend || undefined,
       memory,
       env,

package/src/ste.ts

@@ -91,8 +91,13 @@ export function countWords(text: string): number {
   return (collapse(text).match(/[A-Za-z0-9][\w'’./-]*/g) || []).length
 }
 
+/** "You MUST", "The agent SHOULD NOT": the RFC 2119 key word that makes a sentence a rule, with its subject. */
+export const RFC_LEAD = /^(([Yy]ou|[Tt]he agent|[Tt]he reply|[Ee]ach \w+|[Ee]very \w+)\s+)?(MUST NOT|MUST|SHOULD NOT|SHOULD|MAY|SHALL NOT|SHALL)\s+/
+
 export function isInstruction(sentence: string): boolean {
-  const s = collapse(sentence).replace(/^[\s>*_-]+/, "").replace(LEAD, "")
+  const lead = collapse(sentence).replace(/^[\s>*_-]+/, "").replace(LEAD, "")
+  // "You MUST run the tests." is an instruction: the verb after the key word decides, as without it.
+  const s = RFC_LEAD.test(lead) ? lead.replace(RFC_LEAD, "") : lead
   const first = (s.match(/^[A-Za-z']+/)?.[0] || "").toLowerCase()
   return VERBS.has(first)
 }
@@ -151,7 +156,8 @@ function splitSentences(text: string): string[] {
 }
 
 export function checkSentence(text: string): SteSentence {
-  const words = countWords(text)
+  // The subject and RFC 2119 key word of a rule ("You MUST NOT") do not count toward its length.
+  const words = countWords(text) - countWords(collapse(text).replace(/^[\s>*_-]+/, "").replace(LEAD, "").match(RFC_LEAD)?.[0] || "")
   const instruction = isInstruction(text)
   const issues: SteSentence["issues"] = []
   const limit = instruction ? 20 : 25

package/src/suite/run.ts

@@ -6,7 +6,7 @@ import { costOf } from "../cases/types.ts"
 import { addUsage, NO_USAGE } from "../usage.ts"
 import type { AgentEvent, CodeResult, ReplayMode, RunResult } from "../types.ts"
 import { codeSkipReason, runCodeAgent } from "../code/index.ts"
-import type { AppConfig } from "../config.ts"
+import type { AppConfig, BrowserProvider } from "../config.ts"
 import { fileBugs } from "../cases/bugs.ts"
 import { classifyFailure, FAILURE_LABEL } from "../cases/failure.ts"
 import { associateAdoAutomation, readMappings } from "../code/traceability.ts"
@@ -43,6 +43,7 @@ export type SuiteOptions = {
   cases: TestCase[]
   /** Repo for the coding agent; unset: browser runs only. */
   repo?: string
+  browserProvider?: BrowserProvider
   parallel?: number
   outDir?: string
   writeBack?: boolean
@@ -104,6 +105,7 @@ export async function runSuite(opts: SuiteOptions): Promise<SuiteResult> {
         const { result } = await execute({
           config,
           goal: goalFor(tc),
+          browserProvider: opts.browserProvider,
           url: tc.startUrl || opts.url,
           headed: opts.headed,
           maxSteps: opts.maxSteps,

package/testmu.example.yaml

@@ -0,0 +1,14 @@
+provider: testmu
+remote:
+  browser: pw-chromium
+  browser_version: latest
+  platform: Windows 11
+  username_env: LT_USERNAME
+  access_key_env: LT_ACCESS_KEY
+  project: ForgeQE
+  build: nightly
+  name: checkout smoke
+  local: false
+  extra:
+    "lt:options":
+      console: true