@capagents/forgeqe 1.2.0 → 1.3.0

package/src/code/runtime.ts

@@ -9,7 +9,8 @@ import { entraGateway } from "../entra.ts"
 import { missingKey, type LlmProfile } from "../llm.ts"
 import type { CodeEvent, FileAction, PlanItem, Usage } from "../types.ts"
 import { envRefs, withoutNames } from "../childEnv.ts"
-import { addUsage, recordUsage, type Role } from "../usage.ts"
+import { recordCall } from "../tokenomics/ledger.ts"
+import { addUsage, type Role } from "../usage.ts"
 import { linkSkills, type Skill, type SkillAgent } from "../skills/registry.ts"
 import { EXE, githubDigest, installTool, isMusl, toolPath, type Log } from "../tools.ts"
 import { RESULT_SCHEMA, testAgentPrompt, type AgentReport } from "./prompt.ts"
@@ -648,6 +649,14 @@ export async function openSession<R = AgentReport>(opts: {
   const said = new Set<string>()
   /** Tool calls already said to have started, by part id. */
   const startedCalls = new Set<string>()
+  /** Per model call (assistant message): when it started and the tools it asked for, for the usage ledger. */
+  const steps = new Map<string, { started: number; tools: string[]; answered?: number }>()
+  const finished = new Set<string>()
+  const stepOf = (messageID: string) => {
+    let st = steps.get(messageID)
+    if (!st) steps.set(messageID, (st = { started: Date.now(), tools: [] }))
+    return st
+  }
   let idle: (() => void) | null = null
   let sessionError = ""
   // The turn's result comes from events: in agent runtime 1.18 listing a session's messages fails once a prompt used `format`.
@@ -668,6 +677,13 @@ export async function openSession<R = AgentReport>(opts: {
         break
       case "message.part.updated": {
         const part = p.part
+        if (part?.type === "step-start" && part.messageID) stepOf(part.messageID).started = Date.now()
+        if (part?.type === "tool" && part.messageID && part.tool) {
+          const st = stepOf(part.messageID)
+          if (!st.tools.includes(part.tool)) st.tools.push(part.tool)
+          // The model's part of the step ends when it hands over to a tool; the tool's own time is not the model's.
+          st.answered ??= Date.now()
+        }
         // A call is said as soon as it starts, so a long one (an API query, a test run) is visible while it runs.
         if (part?.type === "tool" && part.state?.status === "running" && part.id && !startedCalls.has(part.id) && !QUIET_TOOLS.has(part.tool) && !opts.hideTool?.(part.tool)) {
           startedCalls.add(part.id)
@@ -705,7 +721,8 @@ export async function openSession<R = AgentReport>(opts: {
             turn.text.push(text)
             emit({ type: "code", kind: "say", text: clip(text, 400) })
           }
-        } else if (part?.type === "step-finish" && part.tokens) {
+        } else if (part?.type === "step-finish" && part.tokens && !(part.id && finished.has(part.id))) {
+          if (part.id) finished.add(part.id)
           const t = part.tokens
           const step: Usage = {
             input: (t.input || 0) + (t.cache?.read || 0) + (t.cache?.write || 0),
@@ -714,7 +731,22 @@ export async function openSession<R = AgentReport>(opts: {
             ...(t.cache?.write ? { cacheWrite: t.cache.write } : {}),
           }
           Object.assign(usage, addUsage(usage, step))
-          recordUsage(role, step, price)
+          const st = part.messageID ? steps.get(part.messageID) : undefined
+          if (part.messageID) steps.delete(part.messageID)
+          recordCall({
+            agent: role,
+            usage: step,
+            price,
+            provider: model.providerID,
+            model: model.modelID,
+            session: session.id,
+            ...(opts.title ? { task: opts.title } : {}),
+            ...(st ? { ms: (st.answered ?? Date.now()) - st.started, tools: st.tools } : {}),
+            ...(t.reasoning ? { reasoning: t.reasoning } : {}),
+            outcome: part.reason === "error" ? "error" : "ok",
+            // With code.model the runtime prices the call itself; the profile's price is unknown then.
+            ...(typeof part.cost === "number" && part.cost > 0 ? { knownCost: { amount: part.cost, currency: "USD" } } : {}),
+          })
           emit({ type: "code", kind: "usage", usage: { ...usage }, context: step.input })
         }
         break

package/src/code/shortlist.ts

@@ -0,0 +1,309 @@
+import path from "node:path"
+
+import type { MemoryElement, MemoryPage } from "../memory.ts"
+import { allMembers, elementRefs, membersLocating, qualified, testsReachingMembers, type CodeGraph, type Located } from "./graph/graph.ts"
+
+/**
+ * Which of a repository's tests a request is about, before any browser run and without a model: hundreds of tests
+ * cannot all be shown to ForgeQE's agent, so they are ranked first and the closest few are shown with what each does.
+ *
+ * Three signals, each weighted by how rare it is among the tests (a link every test clicks says little):
+ * - elements: the request names elements page memory knows ("Select Skills", "Email"); the page-object members that
+ *   locate them, and the tests that reach those members, are found through the code graph, as coverage does after a run.
+ * - pages: the request names a page ("the Practice tab", a URL path); tests that reach that page's elements.
+ * - names: the request shares words with a test's class, method, case ids, or categories; a case id is decisive.
+ */
+
+/** A test of the repository, as the shortlist shows it. */
+export type TestEntry = {
+  /** Class.Method, or the test's title for a top-level test. */
+  test: string
+  file: string
+  line: number
+  lines: number
+  /** Other tests in the same class: a test can be added next to them. */
+  siblings: number
+  /** Case ids and categories from its attributes or decorators. */
+  tags: string[]
+  /** What it calls, in order, without assertions and framework plumbing. */
+  steps: string[]
+}
+
+export type Shortlisted = TestEntry & {
+  score: number
+  /** Elements the request names that this test reaches, and the ones it does not. */
+  covers: string[]
+  lacks: string[]
+  why: string[]
+}
+
+export type Shortlist = { candidates: Shortlisted[]; elements: string[]; unlocated: string[]; pages: string[]; tests: number; ms: number }
+
+/** A test this close is worth showing. */
+export const SHORTLIST_FROM = 0.2
+const MAX_ELEMENTS = 40
+/** How much the element signal counts when the request names a single known element. */
+const ONE_ELEMENT = 0.4
+const MAX_PAGE_ELEMENTS = 150
+const MAX_STEPS = 14
+
+const STOP = new Set(
+  "a an and the of in on to with within only each every any all then this that is are be as it its if for from by or not when also into page step steps test tests flow verify verifying check checks expected appears appear present remains accepted go open navigate click fill type enter select see should can must sure make use using value values field fields realistic data async com org net www http https html".split(
+    " ",
+  ),
+)
+
+/** Words of an identifier or phrase, endings off: FillSkillsSearchAsync → fill, skill, search. */
+function words(s: string): string[] {
+  return s
+    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+    .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter((w) => w.length > 1 && !STOP.has(w))
+    .map((w) => w.replace(/(ing|ed|es|s)$/, ""))
+    .filter((w) => w.length > 1)
+}
+
+const norm = (s: string) => ` ${s.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim()} `
+
+/** Assertions, waits, and plumbing: not steps of the journey. */
+const NOISE = /^(ConfigureAwait|Expect|expect|That|AreEqual|IsTrue|IsFalse|IsNotNull|Equal|True|False|assert\w*|Assert\w*|to[A-Z]\w*|To[A-Z]\w*|Not\w*|not|should\w*|Should\w*|log|Log\w*|WriteLine|WaitFor\w*|waitFor\w*|Delay|sleep|Sleep|StepAsync|Step|step|Run|GetAwaiter|GetResult|Result|toString|ToString|Format|Contains|Trim|Lower\w*|Upper\w*)$/
+const NOISE_RECEIVER = /^(Assertions|Assert|ClassicAssert|expect|console|Console|Task|string|String|Math|this\.Page|Page|page|TestContext|Logger|logger|log)$/
+
+/** Case ids (WEB-014, ADO-2238) and categories from a test's declaration. */
+function tagsOf(signature: string): string[] {
+  const ids = [...signature.matchAll(/\b([A-Z][A-Z0-9]+-\d+)\b/g)].map((m) => m[1]!)
+  const cats = [...signature.matchAll(/(?:Category|Tag|Trait|mark\.)\s*\(?\s*"?([A-Za-z][\w-]*)"?/g)].map((m) => m[1]!).filter((c) => !/^(Test|TestCase)$/.test(c))
+  const at = [...signature.matchAll(/@([a-z][\w-]+)/g)].map((m) => `@${m[1]}`)
+  return [...new Set([...ids, ...cats, ...at])]
+}
+
+function entryOf(at: Located, siblings: number): TestEntry {
+  const steps: string[] = []
+  // In source order: the parser gives calls in the order it meets them in the tree, not as the test runs them.
+  const calls = at.member.calls.map((c, i) => ({ c, i })).sort((a, b) => a.c.line - b.c.line || a.i - b.i).map(({ c }) => c)
+  for (const c of calls) {
+    if (NOISE.test(c.name) || NOISE_RECEIVER.test(c.receiver)) continue
+    const step = c.receiver && !/^(this|self|base|super)$/.test(c.receiver) ? `${c.receiver}.${c.name}` : c.name
+    if (steps[steps.length - 1] !== step) steps.push(step)
+  }
+  return {
+    test: qualified(at),
+    file: at.file,
+    line: at.member.line,
+    lines: at.member.endLine - at.member.line + 1,
+    siblings,
+    tags: tagsOf(at.member.signature),
+    steps: steps.length > MAX_STEPS ? [...steps.slice(0, MAX_STEPS), `… ${steps.length - MAX_STEPS} more`] : steps,
+  }
+}
+
+/** Every test in the graph, with its entry. */
+export function indexTests(g: CodeGraph): { at: Located; entry: TestEntry }[] {
+  const tests = allMembers(g).filter((l) => l.member.kind === "test")
+  const perClass = new Map<string, number>()
+  for (const t of tests) perClass.set(`${t.file}#${t.cls?.name || ""}`, (perClass.get(`${t.file}#${t.cls?.name || ""}`) || 0) + 1)
+  return tests.map((at) => ({ at, entry: entryOf(at, (perClass.get(`${at.file}#${at.cls?.name || ""}`) || 1) - 1) }))
+}
+
+/** The host a request names, if any. */
+function hostOf(request: string, url?: string): string {
+  const m = /https?:\/\/(?:www\.)?([^/\s,)]+)/i.exec(url || request)
+  return (m?.[1] || "").toLowerCase().replace(/[.:;!?]+$/, "")
+}
+
+const hostMatches = (page: MemoryPage, host: string) => !host || page.host.replace(/^www\./, "") === host
+
+/** Hosts page memory knows whose name the request says without a URL: "Open Wikipedia" is en.wikipedia.org. */
+function hostsNamed(pages: MemoryPage[], request: string): Set<string> {
+  const text = norm(request)
+  const hosts = new Set<string>()
+  for (const p of pages) {
+    const labels = p.host.replace(/^www\./, "").split(".").slice(0, -1).filter((l) => l.length >= 4 && !/^(www|com|org|net|app|en|de|fr)$/.test(l))
+    if (labels.some((l) => text.includes(` ${l} `) || text.includes(` ${l.replace(/-/g, " ")} `))) hosts.add(p.host.replace(/^www\./, ""))
+  }
+  return hosts
+}
+
+/** Only locators that name one element: a page's other elements located by CSS or text match code of any page. */
+const STABLE = new Set(["testid", "id", "role", "label", "placeholder", "name"])
+
+/** The label a person would use for an element: its accessible name, text, or placeholder. */
+function labelOf(e: MemoryElement): string {
+  const a = e.attributes || {}
+  const label = (e.name || a["aria-label"] || a.placeholder || e.text || "").replace(/\s+/g, " ").trim()
+  return label.length >= 3 && label.length <= 60 ? label : ""
+}
+
+/** The pages a request names: by its URL path, or a word of the page's path or title ("the Practice tab"). */
+function pagesNamed(pages: MemoryPage[], request: string, url?: string): MemoryPage[] {
+  const text = norm(request)
+  // The site's own name is not a page's: "Wikipedia" names the site, not every page titled "… - Wikipedia".
+  const site = new Set(pages.flatMap((p) => p.host.split(".")))
+  const paths = [url, ...(request.match(/https?:\/\/\S+/g) || [])].filter(Boolean).map((u) => {
+    try {
+      return new URL(String(u).replace(/[.,;)]+$/, "")).pathname.replace(/\/+$/, "") || "/"
+    } catch {
+      return ""
+    }
+  })
+  return pages.filter((p) => {
+    if (p.path !== "/" && paths.includes(p.path.replace(/\/+$/, ""))) return true
+    const named = [...p.path.split(/[/._-]+/), ...(p.title || "").split(/[^A-Za-z0-9]+/)].map((w) => w.toLowerCase()).filter((w) => w.length >= 4 && !site.has(w) && !STOP.has(w) && !/^(html?|index|aspx|php|www|com|main|home)$/.test(w))
+    return named.some((w) => text.includes(` ${w} `))
+  })
+}
+
+/** Tests reached through the members that locate any of these elements. */
+function testsThrough(g: CodeGraph, elements: MemoryElement[], stableOnly = false): Set<string> {
+  return reachOf(g, elements, stableOnly).tests
+}
+
+/** The page-object members that locate these elements, and the tests that reach them. */
+function reachOf(g: CodeGraph, elements: MemoryElement[], stableOnly = false): { members: string[]; tests: Set<string> } {
+  const refsOf = (e: MemoryElement) => elementRefs(e).filter((r) => !stableOnly || (STABLE.has(r.kind) && Boolean(r.value)))
+  const members = elements.flatMap((e) => membersLocating(g, refsOf(e)).filter((m) => m.how === "exact" || (!stableOnly && m.how !== "param")).map((m) => m.at))
+  const seen = new Set<string>()
+  const unique = members.filter((m) => {
+    const k = `${m.file}|${qualified(m)}`
+    return seen.has(k) ? false : (seen.add(k), true)
+  })
+  return { members: unique.map(qualified), tests: new Set(testsReachingMembers(g, unique).map((t) => `${t.file}#${qualified(t)}`)) }
+}
+
+/** The repository's tests closest to a request, best first, with what each covers of the elements it names. */
+export function shortlistTests(opts: { graph: CodeGraph; pages: MemoryPage[]; request: string; url?: string; max?: number }): Shortlist {
+  const started = performance.now()
+  const { graph, request } = opts
+  const index = indexTests(graph)
+  const n = index.length
+  const done = (candidates: Shortlisted[], extra: Partial<Shortlist> = {}): Shortlist => ({ candidates, elements: [], unlocated: [], pages: [], tests: n, ms: Math.round(performance.now() - started), ...extra })
+  if (!n) return done([])
+  const host = hostOf(request, opts.url)
+  const hosts = host ? null : hostsNamed(opts.pages, request)
+  const scope = opts.pages.filter((p) => (hosts?.size ? hosts.has(p.host.replace(/^www\./, "")) : hostMatches(p, host)))
+  const idf = (reach: number) => Math.log(1 + n / (1 + reach))
+
+  // Pages the request names, and the tests that reach their elements.
+  const pages = pagesNamed(scope, request, opts.url)
+  // Each element of those pages, weighted by how few tests reach it: a generic "Search" button other sites' page
+  // objects also locate says little; the elements only this page's tests use say the most.
+  const pageElements = pages.flatMap((p) => Object.values(p.elements || {}).filter((e) => elementRefs(e).some((r) => STABLE.has(r.kind))).slice(0, MAX_PAGE_ELEMENTS))
+  const pageWeight = new Map<string, number>()
+  for (const e of pageElements) {
+    const reach = testsThrough(graph, [e], true)
+    for (const t of reach) pageWeight.set(t, (pageWeight.get(t) || 0) + idf(reach.size))
+  }
+  const pageTop = Math.max(0, ...pageWeight.values())
+
+  // Elements the request names, longest labels first ("Select Skills" before "Skills"). On the pages it names when it
+  // names any ("Email" is on many pages), else anywhere on the site; every element with that label counts.
+  const text = norm(request)
+  const collect = (from: MemoryPage[]) => {
+    const named = new Map<string, MemoryElement[]>()
+    for (const page of from)
+      for (const e of Object.values(page.elements || {})) {
+        const key = norm(labelOf(e))
+        if (key.trim() && key.trim().split(" ").some((w) => !STOP.has(w)) && text.includes(key)) named.set(key, [...(named.get(key) || []), e])
+      }
+    return named
+  }
+  const onPages = pages.length ? collect(pages) : new Map<string, MemoryElement[]>()
+  const named = onPages.size ? onPages : collect(scope)
+  const labels = [...named.keys()].sort((a, b) => b.length - a.length).slice(0, MAX_ELEMENTS)
+  // A shorter label inside a longer one the request names ("skills" in "select skills") is the same mention.
+  const kept = labels.filter((l, i) => !labels.slice(0, i).some((longer) => longer.includes(l)))
+  const elementReach = kept.map((key) => {
+    const r = reachOf(graph, named.get(key)!)
+    return { label: labelOf(named.get(key)![0]!), reach: r.tests, members: r.members }
+  })
+  // Located: a page-object member finds it, whether or not a test uses it yet (then every test lacks it).
+  const located = elementReach.filter((e) => e.members.length)
+  const unlocated = elementReach.filter((e) => !e.members.length).map((e) => e.label)
+  const elementWeight = located.map((e) => idf(e.reach.size))
+  const elementTotal = elementWeight.reduce((a, b) => a + b, 0)
+
+  // Words: a request word in few test names says more than one in many.
+  const reqWords = new Set(words(request))
+  const nameWords = index.map(({ at, entry }) => new Set([...words(at.cls?.name || ""), ...words(at.member.name), ...entry.tags.flatMap(words)]))
+  const df = new Map<string, number>()
+  for (const ws of nameWords) for (const w of ws) df.set(w, (df.get(w) || 0) + 1)
+  const reqWeight = [...reqWords].reduce((sum, w) => sum + idf(df.get(w) || 0) ** 2, 0)
+  const reqIds = new Set((request.match(/\b[A-Z][A-Z0-9]+-\d+\b/g) || []).map((s) => s.toUpperCase()))
+
+  const scored: Shortlisted[] = index.map(({ at, entry }, i) => {
+    const key = `${at.file}#${qualified(at)}`
+    const why: string[] = []
+    const covers = located.filter((e) => e.reach.has(key)).map((e) => e.label)
+    const lacks = located.filter((e) => !e.reach.has(key)).map((e) => e.label)
+    const elem = elementTotal ? located.reduce((s, e, j) => s + (e.reach.has(key) ? elementWeight[j]! : 0), 0) / elementTotal : 0
+    if (covers.length) why.push(`reaches ${covers.length} of the ${located.length} elements the request names`)
+    const page = pageTop ? (pageWeight.get(key) || 0) / pageTop : 0
+    if (page >= 0.5) why.push(`works on ${pages.map((p) => p.path).join(", ")}`)
+    const ws = nameWords[i]!
+    // Cosine of the two word sets, each word weighted by how rare it is among test names.
+    let shared = 0
+    let own = 0
+    for (const w of ws) {
+      const weight = idf(df.get(w) || 0) ** 2
+      own += weight
+      if (reqWords.has(w)) shared += weight
+    }
+    // How much of the test's name the request says, with the cosine so a long name is not lost against a short one.
+    const name = own && reqWeight ? 0.6 * (shared / own) + 0.4 * (shared / Math.sqrt(own * reqWeight)) : 0
+    if (name >= 0.3) why.push(`its name shares "${[...ws].filter((w) => reqWords.has(w)).join(", ")}" with the request`)
+    const caseHit = entry.tags.some((t) => reqIds.has(t.toUpperCase()))
+    if (caseHit) why.push(`tagged ${entry.tags.filter((t) => reqIds.has(t.toUpperCase())).join(", ")}, which the request names`)
+    // Without elements named, pages and names carry the score.
+    // One element the request names may be a generic word ("use the language search"); two or more are its journey.
+    const strength = located.length >= 2 ? 1 : located.length === 1 ? ONE_ELEMENT : 0
+    const score = strength * (0.45 * elem + 0.2 * page + 0.35 * name) + (1 - strength) * (0.35 * page + 0.65 * name) + (caseHit ? 1 : 0)
+    return { ...entry, score: Number(score.toFixed(3)), covers, lacks, why }
+  })
+  const candidates = scored
+    .filter((c) => c.score >= SHORTLIST_FROM)
+    .sort((a, b) => b.score - a.score || b.covers.length - a.covers.length || a.lines - b.lines)
+    .slice(0, opts.max ?? 6)
+  return done(candidates, { elements: located.map((e) => e.label), unlocated, pages: pages.map((p) => `${p.host}${p.path}`) })
+}
+
+/**
+ * The command that runs one test, built the way the repository already runs one: from a command an earlier job used
+ * (its filter swapped for this test), else the framework's usual form. Null when neither fits.
+ */
+export function oneTestCommand(repo: string, t: Pick<TestEntry, "test" | "file">, known: string[], files: string[]): string | null {
+  const [cls, method] = t.test.includes(".") ? [t.test.slice(0, t.test.lastIndexOf(".")), t.test.slice(t.test.lastIndexOf(".") + 1)] : ["", t.test]
+  const ext = path.extname(t.file)
+  if (ext === ".cs") {
+    const filter = known.find((c) => /--filter\s+"?FullyQualifiedName~/.test(c))
+    if (filter) return filter.replace(/(FullyQualifiedName~)[^"\s]+/, `$1${cls}.${method}`)
+    // The test project nearest the file.
+    const projects = files.filter((f) => f.endsWith(".csproj")).filter((p) => t.file.startsWith(path.dirname(p) + "/")).sort((a, b) => b.length - a.length)
+    return projects[0] ? `dotnet test ${projects[0]} --filter "FullyQualifiedName~${cls}.${method}"` : null
+  }
+  if ([".ts", ".js", ".mjs", ".tsx"].includes(ext) && files.some((f) => /(^|\/)playwright\.config\.(ts|js|mjs|cjs)$/.test(f))) return `npx playwright test ${t.file} -g "${method.replace(/"/g, '\\"')}"`
+  if (ext === ".py") return `pytest ${t.file}::${cls ? `${cls}::` : ""}${method}`
+  if (ext === ".java") {
+    if (files.some((f) => /(^|\/)pom\.xml$/.test(f))) return `mvn test -Dtest="${cls}#${method}"`
+    if (files.some((f) => /(^|\/)build\.gradle(\.kts)?$/.test(f))) return `./gradlew test --tests "*${cls}.${method}"`
+  }
+  void repo
+  return null
+}
+
+/**
+ * The tests that work on the pages of these areas (/checkout, /api/cart/:id): they reach elements page memory knows on
+ * those pages, through stable locators. Keyed file#Class.Method, with the areas each one reaches.
+ */
+export function testsOnAreas(graph: CodeGraph, pages: MemoryPage[], areas: string[]): Map<string, string[]> {
+  const out = new Map<string, string[]>()
+  for (const area of areas) {
+    const pattern = new RegExp(`^${area.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\:[^/]+|:[^/]+/g, "[^/]+")}/?$`, "i")
+    const on = pages.filter((p) => pattern.test(p.path))
+    if (!on.length) continue
+    const reach = testsThrough(graph, on.flatMap((p) => Object.values(p.elements || {}).filter((e) => elementRefs(e).some((r) => STABLE.has(r.kind))).slice(0, MAX_PAGE_ELEMENTS)), true)
+    for (const t of reach) out.set(t, [...(out.get(t) || []), area])
+  }
+  return out
+}

package/src/code/testrun.ts

@@ -142,6 +142,8 @@ export async function runTestCommand(opts: {
   save?: string
   /** Variables the test must not see: ForgeQE's own secrets (privateEnvNames). */
   hide?: string[]
+  /** Variables the test gets on top of ForgeQE's own (a remote browser's endpoint): never put in the command line. */
+  env?: Record<string, string>
 }): Promise<TestRun> {
   const started = Date.now()
   const windows = process.platform === "win32"
@@ -149,7 +151,7 @@ export async function runTestCommand(opts: {
   const proc = Bun.spawn(argv, {
     cwd: opts.cwd,
     windowsVerbatimArguments: windows,
-    env: { ...withoutNames(process.env, opts.hide || []), ...sessionLoginEnv(), CI: process.env.CI || "1", PLAYWRIGHT_HTML_OPEN: "never", FORCE_COLOR: "0", NO_COLOR: "1" },
+    env: { ...withoutNames(process.env, opts.hide || []), ...sessionLoginEnv(), CI: process.env.CI || "1", PLAYWRIGHT_HTML_OPEN: "never", FORCE_COLOR: "0", NO_COLOR: "1", ...(opts.env || {}) },
     stdin: "ignore",
     stdout: "pipe",
     stderr: "pipe",

package/src/code/traceability.ts

@@ -33,6 +33,8 @@ export type Mapping = {
   bug?: { label: string; url?: string }
   /** The needs-attention folder, for an unfinished test. */
   attention?: string
+  /** The case's steps when the test was written (stepsHash): when they change, the next run updates the test. */
+  stepsHash?: string
   updatedAt: string
 }
 

package/src/commands.ts

@@ -14,6 +14,8 @@ export type ShellCommand =
   | { type: "init" }
   /** Sign in to a system (or save a secret by name), forget one, or show which are signed in. */
   | { type: "auth"; action: "status" | "login" | "logout"; system?: string }
+  /** Run a passed run again as recorded, without ForgeQE's agent: the last one, a run folder, or a goal. */
+  | { type: "rerun"; target: string }
   | { type: "text"; text: string }
   | { type: "unknown"; raw: string }
 
@@ -30,6 +32,7 @@ export const HELP = [
   "  Add a GitHub Actions workflow that runs tests/cases.yaml on every PR, and make it green",
   "",
   "  /stop              stop what is running (or press esc)",
+  "  /rerun [last|run]  run a passed run again as recorded, without ForgeQE's agent (last, a run folder, or its goal)",
   "  /headed on|off     show the browser (live profiles)",
   "  /details [on|off]  show everything the agents do (reasoning, every file read, engine logs); off: just progress",
   "  /profile <name>    switch LLM profile (or press tab)",
@@ -46,7 +49,7 @@ export const HELP = [
   "Select text with the mouse to copy it.",
 ].join("\n")
 
-const COMMANDS = new Set(["help", "?", "exit", "quit", "q", "clear", "stop", "headed", "details", "verbose", "profile", "p", "logs", "skills", "features", "auth", "signin", "init"])
+const COMMANDS = new Set(["help", "?", "exit", "quit", "q", "clear", "stop", "headed", "details", "verbose", "profile", "p", "logs", "skills", "features", "auth", "signin", "init", "rerun"])
 
 /** Session controls; every other line, including one that starts with a path, goes to the agent as text. */
 export function parseCommand(raw: string): ShellCommand {
@@ -80,6 +83,8 @@ export function parseCommand(raw: string): ShellCommand {
       return { type: "features" }
     case "init":
       return { type: "init" }
+    case "rerun":
+      return { type: "rerun", target: arg || "last" }
     case "auth":
     case "signin": {
       const [first, ...rest] = arg.split(/\s+/).filter(Boolean)

package/src/config.ts

@@ -15,6 +15,7 @@ import { systemLocale } from "./locale.ts"
 import { currencyCode, currencyForLocale } from "./money.ts"
 import { listPrice, pricesDate, type DeploymentType, type ListPrice } from "./prices.ts"
 import { ragConfig, type RagConfig } from "./cases/rag.ts"
+import { budgetConfig, type BudgetConfig } from "./tokenomics/budget.ts"
 
 export type BrowserProvider = "local" | "browserstack" | "testmu"
 export type RemoteBrowserProvider = Exclude<BrowserProvider, "local">
@@ -72,6 +73,9 @@ export type MemoryConfig = {
   replay: ReplayMode
 }
 
+/** registry: in forgeqe.yaml (docs/COST_ARCHITECTURE.md). on by default; off: every request takes the agent's path. */
+export type RegistryConfig = { mode: "off" | "shadow" | "on" }
+
 function runtimeBin(value: unknown, base: string): string | undefined {
   if (!value) return undefined
   const s = String(value)
@@ -373,6 +377,8 @@ export type AppConfig = {
   defects: DefectsConfig
   export: { outDir: string; format: string }
   memory: MemoryConfig
+  /** Unset: on. */
+  registry?: RegistryConfig
   code: CodeConfig
   sources: SourcesConfig
   suite: SuiteConfig
@@ -388,6 +394,8 @@ export type AppConfig = {
   rag?: RagConfig
   /** Flaky tests: retries in one run, and what to do with tests that keep flip-flopping (see src/code/history.ts). */
   flaky?: FlakyConfig
+  /** Regression packs: the pipelines that run one (packs.pipeline in forgeqe.yaml). */
+  packs?: PacksConfig
   /** Thresholds of the release-readiness verdict (release: in forgeqe.yaml). */
   release?: ReleaseThresholds
   /** API and contract tests (api: in forgeqe.yaml): see src/api/. */
@@ -400,6 +408,8 @@ export type AppConfig = {
   domain?: DomainConfig
   /** Teams and email notifications for suites, release readiness, and evaluations (notify: in forgeqe.yaml): see src/notify/. */
   notify?: NotifyConfig
+  /** Model spending budgets per run, day and month (budget: in forgeqe.yaml); they only warn. See src/tokenomics/budget.ts. */
+  budget?: BudgetConfig
 }
 
 export type DomainConfig = {
@@ -427,6 +437,13 @@ export function replayMode(raw: unknown): ReplayMode {
   return "verify"
 }
 
+export function registryModeOf(raw: unknown): RegistryConfig["mode"] {
+  const mode = String(raw ?? "on").toLowerCase()
+  if (mode === "off" || mode === "shadow" || mode === "on") return mode
+  if (raw === false) return "off"
+  throw new Error(`Unknown registry mode "${String(raw)}". Use off, shadow or on.`)
+}
+
 const MOCK_PROFILE: LlmProfile = { provider: "mock", model: "mock", temperature: 0 }
 const NO_PROFILE: LlmProfile = { provider: "none", model: "" }
 
@@ -868,6 +885,7 @@ export function loadConfig(opts?: { configPath?: string; profile?: string }): Ap
           : path.join(forgeqeHome(), "memory"),
       replay: replayMode(memory.replay),
     },
+    registry: { mode: registryModeOf(asRecord(yaml.registry).mode) },
     code: {
       repo: code.repo ? path.resolve(base, String(code.repo)) : undefined,
       maxAttempts: Number(code.max_attempts || 3),
@@ -888,7 +906,9 @@ export function loadConfig(opts?: { configPath?: string; profile?: string }): Ap
     sources: sourcesConfig(sources),
     ...(ragConfig(yaml.rag) ? { rag: ragConfig(yaml.rag) } : {}),
     flaky: flakyConfig(asRecord(yaml.flaky)),
+    packs: packsConfig(asRecord(yaml.packs)),
     release: releaseThresholds(yaml.release),
+    ...(budgetConfig(yaml.budget) ? { budget: budgetConfig(yaml.budget) } : {}),
     api: apiConfig(asRecord(yaml.api)),
     bi: biConfig(asRecord(yaml.bi), base, outDir, configPath),
     quality: withBaselineDir(qualityConfig(yaml.quality), base, configPath),
@@ -1100,6 +1120,32 @@ function githubConfig(raw: unknown): SourcesConfig["github"] {
   }
 }
 
+/** Where `forgeqe pack run --on pipeline` runs a pack: a GitHub Actions workflow, an Azure DevOps pipeline, or both. */
+export type PacksConfig = {
+  pipeline: {
+    default?: "github" | "azure_devops"
+    github?: { repo: string; workflow: string; ref: string }
+    azureDevops?: { project: string; pipelineId: number; ref: string }
+    /** Longest wait for a pipeline run, in minutes. */
+    timeoutMinutes: number
+  }
+}
+
+function packsConfig(p: Record<string, unknown>): PacksConfig {
+  const pipe = asRecord(p.pipeline)
+  const gh = asRecord(pipe.github)
+  const ado = asRecord(pipe.azure_devops)
+  const def = String(pipe.default || "")
+  return {
+    pipeline: {
+      ...(def === "github" || def === "azure_devops" ? { default: def } : {}),
+      ...(gh.repo ? { github: { repo: String(gh.repo), workflow: String(gh.workflow || "forgeqe-pack.yml"), ref: String(gh.ref || "main") } } : {}),
+      ...(ado.project && Number(ado.pipeline_id) > 0 ? { azureDevops: { project: String(ado.project), pipelineId: Number(ado.pipeline_id), ref: String(ado.ref || "refs/heads/main") } } : {}),
+      timeoutMinutes: Number(pipe.timeout_minutes) > 0 ? Number(pipe.timeout_minutes) : 60,
+    },
+  }
+}
+
 function flakyConfig(f: Record<string, unknown>): FlakyConfig {
   const q = String(f.quarantine ?? "suggest")
   return {