@capagents/forgeqe 1.2.1 → 1.3.0

package/src/report/usage.ts

@@ -0,0 +1,113 @@
+import { existsSync, readFileSync } from "node:fs"
+import path from "node:path"
+
+import type { BudgetConfig } from "../tokenomics/budget.ts"
+import { readLedger, sessionCalls, type CallRecord } from "../tokenomics/ledger.ts"
+
+/**
+ * What a report says about model usage: the report's own calls, one by one, and the last days of the ledger
+ * summed by day and by what the Breakdown page groups by, so a report shows this run and the history around it.
+ * Counts, names and times only: the ledger has no prompt or response text to carry.
+ */
+
+export type UsageCall = Omit<CallRecord, "v" | "proc">
+
+/** One day's calls with the same agent, job, purpose, model, user, machine and trigger, summed. */
+export type UsageRollup = {
+  day: string
+  agent: string
+  job: string
+  purpose: string
+  model: string
+  user: string
+  machine: string
+  trigger: string
+  calls: number
+  input: number
+  cached: number
+  cacheWrite: number
+  output: number
+  /** Cost of the priced calls, in `currency`; calls in another currency are left out of the sum. */
+  amount: number
+  currency?: string
+  unpriced: number
+  errors: number
+}
+
+export type UsageData = {
+  calls: UsageCall[]
+  history: UsageRollup[]
+  historyDays: number
+  budget?: BudgetConfig
+}
+
+export const HISTORY_DAYS = 30
+
+const strip = ({ v: _v, proc: _p, ...c }: CallRecord): UsageCall => c
+
+/** usage.json in a run folder: its calls, or none when it was not written (a run before the ledger). */
+export function readRunUsage(runDir?: string): CallRecord[] {
+  const file = runDir ? path.join(runDir, "usage.json") : ""
+  if (!file || !existsSync(file)) return []
+  try {
+    const calls = (JSON.parse(readFileSync(file, "utf8")) as { calls?: CallRecord[] }).calls
+    return Array.isArray(calls) ? calls : []
+  } catch {
+    return []
+  }
+}
+
+/** The ledger's last `days` days, summed per day and grouping; most recent day first. */
+export function usageHistory(days = HISTORY_DAYS, opts: { dir?: string; now?: Date } = {}): UsageRollup[] {
+  const now = opts.now || new Date()
+  const since = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() - (days - 1)))
+  const sums = new Map<string, UsageRollup>()
+  const currencies = new Map<string, number>()
+  const calls = readLedger({ since, dir: opts.dir })
+  for (const c of calls) if (c.currency) currencies.set(c.currency, (currencies.get(c.currency) || 0) + 1)
+  // One currency is summed: the one most calls were priced in.
+  const currency = [...currencies.entries()].sort((a, b) => b[1] - a[1])[0]?.[0]
+  for (const c of calls) {
+    const day = c.ts.slice(0, 10)
+    const key = [day, c.agent, c.job, c.purpose, c.model, c.user, c.machine, c.trigger].join("\u0000")
+    const r =
+      sums.get(key) ||
+      ({ day, agent: c.agent, job: c.job, purpose: c.purpose, model: c.model, user: c.user, machine: c.machine, trigger: c.trigger, calls: 0, input: 0, cached: 0, cacheWrite: 0, output: 0, amount: 0, ...(currency ? { currency } : {}), unpriced: 0, errors: 0 } as UsageRollup)
+    r.calls++
+    r.input += c.input
+    r.cached += c.cached || 0
+    r.cacheWrite += c.cacheWrite || 0
+    r.output += c.output
+    if (c.amount !== undefined && c.currency === currency) r.amount += c.amount
+    else r.unpriced++
+    if (c.outcome === "error") r.errors++
+    sums.set(key, r)
+  }
+  return [...sums.values()].sort((a, b) => b.day.localeCompare(a.day) || b.amount - a.amount)
+}
+
+/**
+ * A report's usage. A run: the calls of its folder. A suite: the calls of each case's run folder, and the calls this
+ * process made while the suite ran (ForgeQE's agent, Scribe, heals), each once. Undefined when there are none.
+ */
+export function reportUsage(p: { kind: "suite" | "run"; dir: string; startedAt: string; runDirs: string[]; budget?: BudgetConfig }, opts: { historyDays?: number; dir?: string } = {}): UsageData | undefined {
+  const seen = new Set<string>()
+  const calls: CallRecord[] = []
+  const add = (c: CallRecord) => {
+    if (!c?.id || seen.has(c.id)) return
+    seen.add(c.id)
+    calls.push(c)
+  }
+  const dirs = p.kind === "run" ? [p.dir, ...p.runDirs] : p.runDirs
+  for (const d of new Set(dirs)) for (const c of readRunUsage(d)) add(c)
+  if (p.kind === "run") for (const c of sessionCalls((c) => c.runId === path.basename(p.dir))) add(c)
+  if (p.kind === "suite") for (const c of sessionCalls((c) => c.ts >= p.startedAt)) add(c)
+  const history = usageHistory(opts.historyDays ?? HISTORY_DAYS, { dir: opts.dir })
+  if (!calls.length && !history.length) return undefined
+  return {
+    calls: calls.sort((a, b) => a.ts.localeCompare(b.ts)).map(strip),
+    history,
+    historyDays: opts.historyDays ?? HISTORY_DAYS,
+    ...(p.budget ? { budget: p.budget } : {}),
+  }
+}

package/src/run.ts

@@ -13,6 +13,7 @@ import { ensureEngine } from "./engine/setup.ts"
 import { reportEnv, writeRunReport } from "./report/html.ts"
 import { logEvent } from "./sessionLog.ts"
 import { costOf } from "./usage.ts"
+import { writeRunUsage } from "./tokenomics/ledger.ts"
 import { allLogins, redactSecrets } from "./sessionLogins.ts"
 import { browserSkills, rememberChoice } from "./skills/select.ts"
 import { axeTags, DEFAULT_QUALITY, emptyReport, evaluateQuality, readA11y } from "./quality/budgets.ts"
@@ -20,6 +21,7 @@ import { historyFile, recordQuality } from "./quality/history.ts"
 import { compareVisual, engineVisual } from "./quality/visual.ts"
 import { enginePrompts, prompt } from "./prompts.ts"
 import { knowledgeBlock, readKnowledge, relevantKnowledge } from "./knowledge.ts"
+import { registryMode } from "./registry/index.ts"
 
 export type ExecuteOptions = {
   config: AppConfig
@@ -39,6 +41,8 @@ export type ExecuteOptions = {
   memory?: boolean
   /** Steps in the test case this goal came from; the agent then reports a verdict for each. */
   caseSteps?: number
+  /** Each case step's stepKey: after the case is edited, the engine replays the steps before the first one that changed. */
+  caseStepKeys?: string[]
   /** A failed or cut-off run's folder (or its name under the runs folder): replay the steps that worked there first. */
   resumeFrom?: string
   /** Browser skills the caller chose (empty: none); unset: the engine picks the ones the goal needs. */
@@ -134,6 +138,16 @@ function shortPage(url: string): string {
   }
 }
 
+/**
+ * Page memory for the engine, or undefined to run without it. registry.mode on (the default): a replay may also be
+ * passed by its recorded checks, with no model (docs/COST_ARCHITECTURE.md); off or shadow: as it always was.
+ */
+export function engineMemory(opts: Pick<ExecuteOptions, "config" | "replay" | "memory">): { dir: string; replay: ReplayMode; checks_first?: boolean } | undefined {
+  const mem = opts.config.memory
+  if (!mem.enabled || opts.memory === false) return undefined
+  return { dir: mem.dir, replay: opts.replay || mem.replay, ...(registryMode(opts.config) === "on" ? { checks_first: true } : {}) }
+}
+
 export async function executeRun(opts: ExecuteOptions): Promise<{ result: RunResult; files: string[] }> {
   const goal = redactSecrets(opts.goal.trim())
   if (!goal) throw new Error("A goal is required. Pass --goal or type one in the shell.")
@@ -166,6 +180,12 @@ export async function executeRun(opts: ExecuteOptions): Promise<{ result: RunRes
     } catch (err) {
       opts.onEvent?.({ type: "log", level: "warn", text: `API steps not saved as a scenario: ${(err as Error).message}` })
     }
+    try {
+      const usage = writeRunUsage(dir)
+      if (usage) out.files.push(usage)
+    } catch (err) {
+      opts.onEvent?.({ type: "log", level: "warn", text: `usage.json not written: ${(err as Error).message}` })
+    }
     logEvent("run.done", { out_dir: dir, status: out.result.status, summary: out.result.summary, usage: out.result.usage, memory: out.result.memory, ...(out.result.skills ? { skills: out.result.skills } : {}) })
     try {
       out.files.push(writeRunReport(dir, out.result, reportEnv(opts.config, { url: startUrl, headed: opts.headed, browser }), t0))
@@ -213,8 +233,7 @@ export async function executeRun(opts: ExecuteOptions): Promise<{ result: RunRes
     BROWSER_USE_USE_JUDGE: opts.config.browser.judge ? "true" : "false",
   }
   for (const [k, v] of Object.entries(tuning)) if (!process.env[k]) env[k] = v
-  const mem = opts.config.memory
-  const memory = mem.enabled && opts.memory !== false ? { dir: mem.dir, replay: opts.replay || mem.replay } : undefined
+  const memory = engineMemory(opts)
   const resume = opts.resumeFrom ? loadResume(opts.config.export.outDir, opts.resumeFrom) : undefined
   if (resume && !memory) throw new Error("resume_from replays remembered steps, so it needs page memory; it is off for this run.")
   const q = opts.config.quality || DEFAULT_QUALITY
@@ -241,6 +260,7 @@ export async function executeRun(opts: ExecuteOptions): Promise<{ result: RunRes
       env,
       logins: allLogins(opts.config),
       caseSteps: opts.caseSteps,
+      ...(opts.caseRef?.caseId && opts.caseStepKeys?.length ? { caseId: opts.caseRef.caseId, caseStepKeys: opts.caseStepKeys } : {}),
       screenshots: opts.config.report.screenshots,
       accessibility: opts.config.report.accessibility,
       network: opts.config.api?.capture !== false,

package/src/suite/run.ts

@@ -2,7 +2,7 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs"
 import path from "node:path"
 
 import { goalFor, writeBack as postResults, type CaseResult, type TestCase, type WriteBack } from "../cases/index.ts"
-import { costOf, type Cost } from "../cases/types.ts"
+import { costOf, stepKey, stepsHash, type Cost } from "../cases/types.ts"
 import { addUsage, NO_USAGE } from "../usage.ts"
 import type { AgentEvent, CodeResult, ReplayMode, RunResult, Usage } from "../types.ts"
 import { codeSkipReason, runCodeAgent } from "../code/index.ts"
@@ -15,7 +15,7 @@ import { reproPack, writeReproFiles } from "../cases/repro.ts"
 import { commentAnalyses } from "../cases/bugs.ts"
 import { caseOutcomes, defectHistoryFile, failedInARow, noteHistory, readDefectHistory, recordRun, writeDefectHistory } from "../cases/defectHistory.ts"
 import { classifyFailure, FAILURE_LABEL } from "../cases/failure.ts"
-import { associateAdoAutomation, readMappings } from "../code/traceability.ts"
+import { associateAdoAutomation, readMappings, recordMapping } from "../code/traceability.ts"
 import type { ExistingTest } from "../code/prompt.ts"
 import { runTestCommand } from "../code/testrun.ts"
 import { privateEnvNames } from "../childEnv.ts"
@@ -160,6 +160,7 @@ export async function runSuite(opts: SuiteOptions): Promise<SuiteResult> {
           replay: opts.replay,
           memory: opts.memory,
           caseSteps: tc.steps.length,
+          caseStepKeys: original.steps.map(stepKey),
           outDir,
           skills: opts.skills,
           repo: opts.skillsRepo || opts.repo,
@@ -264,16 +265,20 @@ export async function runSuite(opts: SuiteOptions): Promise<SuiteResult> {
       const run = await (opts.runTest || runTestCommand)({ command: mapped.command, cwd: opts.repo!, timeoutSeconds: config.code.testTimeoutSeconds, signal: opts.signal, hide: privateEnvNames(config), save: path.join(runDir, "existing-test") })
       const defectNow = result.failure?.type === "functional_defect"
       // Reused as it is: a normal test that passes on a case that passes, or a known-defect test (an expected failure)
-      // on a case that still shows the defect.
+      // on a case that still shows the defect, as long as the case's steps are the ones the test was written from.
       const fits = mapped.status === "known_defect" ? defectNow : !defectNow
-      if (run.ok && fits) {
+      const hash = tc.steps.length ? stepsHash(tc.steps) : undefined
+      const changed = Boolean(mapped.stepsHash && hash && mapped.stepsHash !== hash)
+      if (run.ok && fits && !changed) {
+        // A mapping from before steps were fingerprinted gets this case's: a later change to the steps then shows.
+        if (hash && !mapped.stepsHash) recordMapping(opts.repo!, { ...mapped, stepsHash: hash })
         emit({ type: "case-event", index, case: tc, event: { type: "code", kind: "test", ok: true, attempt: 0, text: `${mapped.testFile} passes as it is: no new test needed` } })
         result.code = { status: "passed", testFile: mapped.testFile, command: mapped.command, attempts: 0, existing: "reused", ...(mapped.testName ? { testName: mapped.testName } : {}), ...(mapped.status === "known_defect" ? { knownDefect: true } : {}) }
         emit({ type: "code-done", index, case: tc, result })
         return
       }
       if (opts.signal?.aborted) return
-      const why =
+      const reason =
         mapped.status === "known_defect" && !defectNow
           ? `It is marked as an expected failure for a known defect${mapped.bug ? ` (${mapped.bug.label})` : ""}, and the browser run now passes: the defect looks fixed. Remove the expected-failure marker and keep the assertions.`
           : defectNow && run.ok
@@ -281,6 +286,8 @@ export async function runSuite(opts: SuiteOptions): Promise<SuiteResult> {
             : defectNow
               ? "It fails because of the functional defect this run found (below): keep its assertions and mark it as an expected failure naming the defect."
               : "It fails now: the application or the page changed since it was written."
+      const stale = "The test case's steps changed since this test was written (for example after a review): make the test do the case's steps and check its expected results as they are now, keeping the parts that still fit."
+      const why = !changed ? reason : run.ok && fits ? stale : `${stale} ${reason}`
       existingTest = { testFile: mapped.testFile, command: mapped.command, output: run.output, why }
     }
     emit({ type: "code-start", index, case: tc })

package/src/summary.ts

@@ -62,6 +62,7 @@ export function memoryLine(m?: MemoryInfo): string {
     parts.push(m.replayed === m.replayTotal ? `replayed all ${m.replayed} steps` : `replayed ${m.replayed} of ${m.replayTotal} steps`)
     parts.push(plural(m.llmSteps, "LLM step"))
   }
+  if (m.checked) parts.push(m.checksHeld === m.checked ? `${plural(m.checked, "recorded check")} held` : `${m.checksHeld || 0} of ${plural(m.checked, "recorded check")} held`)
   if (m.elements) parts.push(`${plural(m.elements, "element")} on ${plural(m.pages, "page")}${m.newElements ? ` (${m.newElements} new)` : ""}`)
   if (m.flow) parts.push(m.flow === "saved" ? "flow saved for replay" : "flow updated")
   return parts.join(" · ")

package/src/tokenomics/budget.ts

@@ -0,0 +1,172 @@
+import { closeSync, existsSync, openSync, readSync, statSync } from "node:fs"
+import path from "node:path"
+
+import { formatCost } from "../usage.ts"
+import { onCall, usageDir, type CallRecord } from "./ledger.ts"
+
+/**
+ * Spending budgets (budget: in forgeqe.yaml), per run, day and month, in the price currency. They only warn, once
+ * each at 80%, 90% and 100%, and never stop a run. Spend is read from the ledger file, which every ForgeQE process
+ * on this machine appends to: a browser run the agent starts through the MCP server runs in another process, and
+ * its calls count all the same. Days and months are UTC, as the ledger's timestamps are.
+ */
+
+export type BudgetConfig = { run?: number; day?: number; month?: number; currency?: string }
+export type BudgetPeriod = "run" | "day" | "month"
+
+export type BudgetWarning = { period: BudgetPeriod; threshold: number; spent: number; limit: number; currency: string; text: string }
+
+export const BUDGET_THRESHOLDS = [80, 90, 100] as const
+
+/** How often the ledger is read again for calls other processes made. */
+const POLL_MS = 5_000
+
+/** budget: in forgeqe.yaml; amounts that are not positive numbers are left out. */
+export function budgetConfig(raw: unknown): BudgetConfig | undefined {
+  const r = raw && typeof raw === "object" ? (raw as Record<string, unknown>) : null
+  if (!r) return undefined
+  const amount = (v: unknown) => (typeof v === "number" || (typeof v === "string" && v.trim() !== "") ? (Number(v) > 0 ? Number(v) : undefined) : undefined)
+  const out: BudgetConfig = {}
+  for (const k of ["run", "day", "month"] as const) {
+    const a = amount(r[k])
+    if (a !== undefined) out[k] = a
+  }
+  if (typeof r.currency === "string" && /^[A-Za-z]{3}$/.test(r.currency.trim())) out.currency = r.currency.trim().toUpperCase()
+  return out.run || out.day || out.month ? out : undefined
+}
+
+export type BudgetStatus = { period: BudgetPeriod; spent: number; limit: number; currency: string; percent: number }
+
+/**
+ * Watch spend against `cfg` and call `warn` the first time each period passes each threshold. Returns the
+ * unsubscribe and a way to read where each budget stands.
+ */
+export function watchBudgets(cfg: BudgetConfig, warn: (w: BudgetWarning) => void, opts: { dir?: string; now?: () => Date; pollMs?: number } = {}) {
+  const now = opts.now || (() => new Date())
+  const dir = () => opts.dir || usageDir()
+  const warned = new Set<string>()
+  const seen = new Set<string>()
+  let currency = cfg.currency
+  let month = ""
+  let offset = 0
+  let monthAmount = 0
+  const days = new Map<string, number>()
+  const runs = new Map<string, number>()
+  const touched = new Set<string>()
+
+  const add = (c: CallRecord) => {
+    if (!c?.id || seen.has(c.id) || c.amount === undefined || !c.ts?.startsWith(month)) return
+    seen.add(c.id)
+    currency ||= cfg.currency || c.currency
+    if (c.currency !== currency) return
+    monthAmount += c.amount
+    days.set(c.ts.slice(0, 10), (days.get(c.ts.slice(0, 10)) || 0) + c.amount)
+    if (c.runId) {
+      runs.set(c.runId, (runs.get(c.runId) || 0) + c.amount)
+      touched.add(c.runId)
+    }
+  }
+
+  /** Read what was appended to this month's file since the last read; a new month starts over. */
+  const sync = () => {
+    const m = now().toISOString().slice(0, 7)
+    if (m !== month) {
+      month = m
+      offset = 0
+      monthAmount = 0
+      days.clear()
+      runs.clear()
+      seen.clear()
+    }
+    const file = path.join(dir(), `${month}.jsonl`)
+    if (!existsSync(file)) return
+    try {
+      const size = statSync(file).size
+      if (size < offset) offset = 0
+      if (size === offset) return
+      const fd = openSync(file, "r")
+      const buf = Buffer.alloc(size - offset)
+      try {
+        readSync(fd, buf, 0, buf.length, offset)
+      } finally {
+        closeSync(fd)
+      }
+      // Only whole lines: a line another process is still writing is read next time.
+      const text = buf.toString("utf8")
+      const end = text.lastIndexOf("\n")
+      if (end < 0) return
+      offset += Buffer.byteLength(text.slice(0, end + 1))
+      for (const line of text.slice(0, end).split("\n")) {
+        try {
+          add(JSON.parse(line) as CallRecord)
+        } catch {}
+      }
+    } catch {}
+  }
+
+  const check = () => {
+    const day = now().toISOString().slice(0, 10)
+    const targets: { period: BudgetPeriod; key: string; spent: number }[] = []
+    if (cfg.day) targets.push({ period: "day", key: day, spent: days.get(day) || 0 })
+    if (cfg.month) targets.push({ period: "month", key: month, spent: monthAmount })
+    if (cfg.run) for (const id of touched) targets.push({ period: "run", key: id, spent: runs.get(id) || 0 })
+    touched.clear()
+    for (const { period, key, spent } of targets) {
+      const limit = cfg[period]!
+      // Only the highest threshold newly passed is said: a jump from 70% to 105% is one warning, not three.
+      const passed = BUDGET_THRESHOLDS.filter((t) => spent >= (limit * t) / 100 && !warned.has(`${period}:${key}:${t}`))
+      if (!passed.length) continue
+      for (const t of passed) warned.add(`${period}:${key}:${t}`)
+      const threshold = passed[passed.length - 1]!
+      const cur = currency || "USD"
+      warn({ period, threshold, spent, limit, currency: cur, text: warningText(period, threshold, spent, limit, cur) })
+    }
+  }
+
+  // Spend so far this month is counted, but only what passes a threshold from now on is said.
+  sync()
+  for (const [period, key, spent] of [["day", now().toISOString().slice(0, 10), days.get(now().toISOString().slice(0, 10)) || 0], ["month", month, monthAmount]] as const) {
+    const limit = cfg[period]
+    if (limit) for (const t of BUDGET_THRESHOLDS) if (spent >= (limit * t) / 100) warned.add(`${period}:${key}:${t}`)
+  }
+  touched.clear()
+
+  const stopListening = onCall((c) => {
+    sync()
+    add(c) // the ledger is off (FORGEQE_USAGE=off) or the line was not written: count it all the same
+    check()
+  })
+  const timer = setInterval(() => {
+    sync()
+    check()
+  }, opts.pollMs ?? POLL_MS)
+  timer.unref?.()
+
+  const stop = () => {
+    stopListening()
+    clearInterval(timer)
+  }
+  const status = (runId?: string): BudgetStatus[] => {
+    sync()
+    const day = now().toISOString().slice(0, 10)
+    const out: BudgetStatus[] = []
+    const push = (period: BudgetPeriod, spent: number) => {
+      const limit = cfg[period]
+      if (limit) out.push({ period, spent, limit, currency: currency || "USD", percent: Math.round((spent / limit) * 100) })
+    }
+    if (runId) push("run", runs.get(runId) || 0)
+    push("day", days.get(day) || 0)
+    push("month", monthAmount)
+    return out
+  }
+  return { stop, status }
+}
+
+const WHOSE: Record<BudgetPeriod, string> = { run: "this run's", day: "today's", month: "this month's" }
+
+/** "Budget: this month's model spend is at $245.10, 82% of the $300.00 month budget (…). Nothing is stopped." */
+export function warningText(period: BudgetPeriod, threshold: number, spent: number, limit: number, currency: string): string {
+  const money = (amount: number) => formatCost({ amount, currency })
+  const verb = threshold >= 100 ? "has reached" : "is at"
+  return `Budget: ${WHOSE[period]} model spend ${verb} ${money(spent)}, ${Math.round((spent / limit) * 100)}% of the ${money(limit)} ${period} budget (budget.${period} in forgeqe.yaml). Nothing is stopped.`
+}