@capagents/forgeqe 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/HOW_TO_USE.md +54 -3
  2. package/README.md +1 -1
  3. package/docs/COST_ARCHITECTURE.md +280 -0
  4. package/docs/feature-test-prompts.md +95 -0
  5. package/docs/user-guide.html +66 -8
  6. package/forgeqe.example.yaml +15 -0
  7. package/package.json +1 -1
  8. package/prompts/brain.md +36 -4
  9. package/prompts/browser-api.md +13 -0
  10. package/prompts/browser-capture.md +2 -2
  11. package/prompts/browser-checks.md +2 -0
  12. package/py/forgeqe_engine/api_step.py +130 -0
  13. package/py/forgeqe_engine/capture.py +53 -2
  14. package/py/forgeqe_engine/checks.py +8 -2
  15. package/py/forgeqe_engine/llm.py +33 -5
  16. package/py/forgeqe_engine/memory.py +70 -2
  17. package/py/forgeqe_engine/observe.py +123 -0
  18. package/py/forgeqe_engine/replay.py +55 -2
  19. package/py/forgeqe_engine/service.py +177 -5
  20. package/src/api/fromSteps.ts +111 -0
  21. package/src/capabilities/registry.ts +19 -5
  22. package/src/cases/ado.ts +1 -1
  23. package/src/cases/journey.ts +61 -0
  24. package/src/cases/quality.ts +3 -1
  25. package/src/cases/readiness.ts +35 -6
  26. package/src/cases/save.ts +241 -48
  27. package/src/cases/types.ts +11 -0
  28. package/src/cli/help.ts +8 -0
  29. package/src/cli/index.ts +64 -1
  30. package/src/code/coverage.ts +22 -3
  31. package/src/code/decisions.ts +73 -0
  32. package/src/code/find.ts +3 -1
  33. package/src/code/graph/graph.ts +33 -2
  34. package/src/code/handoff.ts +20 -0
  35. package/src/code/loop.ts +27 -1
  36. package/src/code/notes.ts +48 -1
  37. package/src/code/prompt.ts +31 -9
  38. package/src/code/protect.ts +103 -0
  39. package/src/code/replay.ts +33 -0
  40. package/src/code/review.ts +2 -0
  41. package/src/code/runtime.ts +35 -3
  42. package/src/code/shortlist.ts +309 -0
  43. package/src/code/testrun.ts +3 -1
  44. package/src/code/traceability.ts +2 -0
  45. package/src/commands.ts +6 -1
  46. package/src/config.ts +46 -0
  47. package/src/engine/bridge.ts +47 -5
  48. package/src/mcp/casetools.ts +107 -11
  49. package/src/mcp/commands.ts +74 -20
  50. package/src/mcp/packtools.ts +187 -0
  51. package/src/mcp/tools.ts +238 -29
  52. package/src/pack/collect.ts +251 -0
  53. package/src/pack/index.ts +134 -0
  54. package/src/pack/init.ts +45 -0
  55. package/src/pack/pack.ts +327 -0
  56. package/src/pack/pipeline.ts +373 -0
  57. package/src/pack/run.ts +155 -0
  58. package/src/registry/db.ts +249 -0
  59. package/src/registry/index.ts +72 -0
  60. package/src/registry/resolve.ts +57 -0
  61. package/src/registry/tier0.ts +81 -0
  62. package/src/report/data.ts +7 -0
  63. package/src/report/html.ts +14 -0
  64. package/src/report/template.html +52 -48
  65. package/src/report/usage.ts +113 -0
  66. package/src/run.ts +32 -2
  67. package/src/suite/report.ts +22 -4
  68. package/src/suite/run.ts +13 -5
  69. package/src/summary.ts +6 -0
  70. package/src/tokenomics/budget.ts +172 -0
  71. package/src/tokenomics/ledger.ts +287 -0
  72. package/src/tui/App.tsx +90 -2
  73. package/src/tui/inspector.tsx +40 -35
  74. package/src/tui/transcript.tsx +5 -4
  75. package/src/types.ts +11 -2
@@ -1,10 +1,12 @@
1
1
  import { spawn } from "node:child_process"
2
2
  import { createInterface } from "node:readline"
3
+ import path from "node:path"
3
4
 
4
5
  import type { LlmProfile } from "../llm.ts"
5
- import type { AgentEvent, CaseReport, MemoryInfo, ReplayMode, RunResult } from "../types.ts"
6
+ import type { AgentEvent, CaseReport, MemoryInfo, ReplayMode, RunResult, Usage } from "../types.ts"
6
7
  import { browserProviderEnvNames, loginEnvNames } from "../childEnv.ts"
7
- import { recordUsage } from "../usage.ts"
8
+ import { recordCall, withUsageScope } from "../tokenomics/ledger.ts"
9
+ import { addUsage } from "../usage.ts"
8
10
  import type { BrowserProvider, BrowserRemoteConfig, LoginConfig } from "../config.ts"
9
11
  import { debugLogging, logEvent } from "../sessionLog.ts"
10
12
  import { engineEnv } from "./setup.ts"
@@ -20,14 +22,20 @@ export type EngineRequest = {
20
22
  allowedHosts: string[]
21
23
  browser?: { provider: BrowserProvider; remote?: BrowserRemoteConfig }
22
24
  extendSystemMessage?: string
23
- /** Page memory folder and replay mode; omit to run without memory. */
24
- memory?: { dir: string; replay: ReplayMode }
25
+ /**
26
+ * Page memory folder and replay mode; omit to run without memory. checks_first (registry.mode on,
27
+ * docs/COST_ARCHITECTURE.md): read-only steps are left out of a replay, and recorded checks can pass it.
28
+ */
29
+ memory?: { dir: string; replay: ReplayMode; checks_first?: boolean }
25
30
  /** BROWSER_USE_* tuning for the engine process. */
26
31
  env?: Record<string, string>
27
32
  /** Saved logins: the engine reads values from the environment by name, or from `values` (logins remembered for this session). */
28
33
  logins?: (LoginConfig & { values?: { username?: string; password?: string; totp?: string } })[]
29
34
  /** Steps in the test case this goal came from; the agent reports a verdict for each. */
30
35
  caseSteps?: number
36
+ /** The test case and a key per step: a run of an edited case replays the recorded steps before the first change. */
37
+ caseId?: string
38
+ caseStepKeys?: string[]
31
39
  screenshots?: "all" | "failures" | "off"
32
40
  accessibility?: boolean
33
41
  /** Record the application's API calls (network.json). */
@@ -73,6 +81,9 @@ type EngineMemory = {
73
81
  new_elements?: number
74
82
  flow?: "saved" | "updated" | null
75
83
  resume?: { steps?: number; failed_at?: number; reason?: string; lines?: string[]; file?: string }
84
+ checked?: number
85
+ checks_held?: number
86
+ skipped_reads?: number
76
87
  }
77
88
 
78
89
  type EngineResult = {
@@ -108,6 +119,8 @@ type EngineEvent =
108
119
  }
109
120
  | { type: "result"; step?: number; items?: { ok?: boolean; done?: boolean; text?: string }[] }
110
121
  | { type: "usage"; input?: number; output?: number; cached?: number }
122
+ /** One model call of the browser agent: its time and tokens (prompt tokens include cached ones). */
123
+ | { type: "call"; ms?: number; ok?: boolean; input?: number; output?: number; cached?: number; cache_write?: number }
111
124
  | { type: "done"; result?: EngineResult }
112
125
 
113
126
  function toMemoryInfo(m: EngineMemory | undefined): MemoryInfo | undefined {
@@ -123,6 +136,8 @@ function toMemoryInfo(m: EngineMemory | undefined): MemoryInfo | undefined {
123
136
  elements: Number(m.elements || 0),
124
137
  newElements: Number(m.new_elements || 0),
125
138
  flow: m.flow || null,
139
+ ...(m.checked ? { checked: Number(m.checked), checksHeld: Number(m.checks_held || 0) } : {}),
140
+ ...(m.skipped_reads ? { skippedReads: Number(m.skipped_reads) } : {}),
126
141
  ...(m.resume?.steps
127
142
  ? {
128
143
  resume: {
@@ -222,6 +237,11 @@ export async function runEngine(opts: {
222
237
  graceMs?: { stop: number; kill: number }
223
238
  }): Promise<{ result: RunResult; files: string[] }> {
224
239
  const { request: req, onEvent } = opts
240
+ // Each model call goes to the usage ledger under this run (its folder's name) and case; what the per-call events
241
+ // missed (an engine that sends only a total) is recorded as one unattributed call when the run ends.
242
+ const scope = { runId: path.basename(req.outDir), ...(req.caseId ? { caseId: req.caseId } : {}) }
243
+ const model = req.llm.deployment || req.llm.model
244
+ const callTotal: Usage = { input: 0, output: 0 }
225
245
  const [bin, ...args] = opts.command || [opts.python, "-m", "forgeqe_engine", "run"]
226
246
  const child = spawn(bin!, args, {
227
247
  env: engineEnv(debugLogging() ? { ...req.env, BROWSER_USE_LOGGING_LEVEL: "debug" } : req.env, [
@@ -256,6 +276,7 @@ export async function runEngine(opts: {
256
276
  memory: req.memory || null,
257
277
  logins: loginPayload(req.logins),
258
278
  case_steps: req.caseSteps || 0,
279
+ ...(req.caseId && req.caseStepKeys?.length ? { case_id: req.caseId, case_step_keys: req.caseStepKeys } : {}),
259
280
  screenshots: req.screenshots || "all",
260
281
  accessibility: req.accessibility !== false,
261
282
  network: req.network !== false,
@@ -335,6 +356,19 @@ export async function runEngine(opts: {
335
356
  case "usage":
336
357
  onEvent?.({ type: "usage", usage: { input: Number(ev.input || 0), output: Number(ev.output || 0), ...(ev.cached ? { cached: Number(ev.cached) } : {}) } })
337
358
  return
359
+ case "call": {
360
+ const usage: Usage = {
361
+ input: Number(ev.input || 0),
362
+ output: Number(ev.output || 0),
363
+ ...(ev.cached ? { cached: Number(ev.cached) } : {}),
364
+ ...(ev.cache_write ? { cacheWrite: Number(ev.cache_write) } : {}),
365
+ }
366
+ Object.assign(callTotal, addUsage(callTotal, usage))
367
+ withUsageScope(scope, () =>
368
+ recordCall({ agent: "browser agent", usage, price: req.llm.price, provider: req.llm.provider, model, purpose: "browser", ms: Number(ev.ms || 0), outcome: ev.ok === false ? "error" : "ok" }),
369
+ )
370
+ return
371
+ }
338
372
  case "done":
339
373
  done = ev.result || {}
340
374
  setTimeout(() => child.kill(), 10_000).unref()
@@ -382,7 +416,15 @@ export async function runEngine(opts: {
382
416
  }
383
417
  const finished: EngineResult = done
384
418
  const result = toRunResult(req.goal, req.startUrl || "", finished)
385
- recordUsage("browser agent", result.usage, req.llm.price)
419
+ const rest: Usage = {
420
+ input: Math.max(0, result.usage.input - callTotal.input),
421
+ output: Math.max(0, result.usage.output - callTotal.output),
422
+ ...((result.usage.cached || 0) > (callTotal.cached || 0) ? { cached: (result.usage.cached || 0) - (callTotal.cached || 0) } : {}),
423
+ }
424
+ if (rest.cached) rest.cached = Math.min(rest.cached, rest.input)
425
+ withUsageScope(scope, () =>
426
+ recordCall({ agent: "browser agent", usage: rest, price: req.llm.price, provider: req.llm.provider, model, purpose: callTotal.input || callTotal.output ? "unattributed" : "browser", outcome: finished.cancelled ? "cancelled" : "ok" }),
427
+ )
386
428
  onEvent?.({ type: "usage", usage: result.usage })
387
429
  onEvent?.({ type: "done", result })
388
430
  return { result, files: finished.files || [] }
@@ -6,7 +6,8 @@ import type { TestCase } from "../cases/types.ts"
6
6
  import type { AppConfig } from "../config.ts"
7
7
  import { readPages } from "../memory.ts"
8
8
  import { DEFAULT_THRESHOLDS, releaseReadiness, saveReadiness, type Readiness } from "../cases/readiness.ts"
9
- import { updateAdoCase } from "../cases/save.ts"
9
+ import { jiraCase, storyCases, updateAdoCase, updateJiraCase, type NewCase } from "../cases/save.ts"
10
+ import { planJourneys, type JourneyCase } from "../cases/journey.ts"
10
11
  import { writeAdoWikiPage } from "../cases/wiki.ts"
11
12
  import type { McpTool, ToolContext } from "./tools.ts"
12
13
  import { notifiedLine, notify, readinessNotice } from "../notify/events.ts"
@@ -106,11 +107,11 @@ export const CASE_TOOLS: McpTool[] = [
106
107
  {
107
108
  name: "update_test_case",
108
109
  description:
109
- "Rewrite an Azure DevOps Test Case's steps (and optionally its title) after the user agrees: ForgeQE shows the current and the new steps and asks (Update / Don't update); the reason goes into the work item's history, and the old steps stay there. Use it for a case run_test_cases classified as test case needs updating (write the steps from what the run found the application does: run_details has every step and element), or a case check_test_cases found unclear. Where ForgeQE cannot ask (an MCP client), pass confirmed: true only after the user agreed to these exact steps.",
110
+ "Rewrite a test case's steps (and optionally its title) in place after the user agrees: an Azure DevOps Test Case (the reason goes into its history, the old steps stay there) or a Jira test issue (the steps in its description are replaced, the text above them kept, and the reason added as a comment). ForgeQE shows the current and the new steps and asks (Update / Don't update). Use it for a case run_test_cases classified as test case needs updating (write the steps from what the run found the application does: run_details has every step and element), a case check_test_cases found unclear, a change the user asks for after reviewing a case, and a case save_test_cases returned as differs or similar. The case's automated test, when it has one, is brought up to date on its next run. Where ForgeQE cannot ask (an MCP client), pass confirmed: true only after the user agreed to these exact steps.",
110
111
  inputSchema: {
111
112
  type: "object",
112
113
  properties: {
113
- id: { type: "string", description: "The test case: ADO-2236, 2236, or its link" },
114
+ id: { type: "string", description: "The test case: ADO-2236, 2236, or its link in Azure DevOps; SHOP-31 in Jira" },
114
115
  title: { type: "string", description: "A new title, only when the old one no longer fits" },
115
116
  steps: {
116
117
  type: "array",
@@ -123,17 +124,23 @@ export const CASE_TOOLS: McpTool[] = [
123
124
  required: ["id", "steps", "reason"],
124
125
  },
125
126
  async run(args, ctx) {
126
- const id = Number(/(\d+)\s*$/.exec(String(args.id).replace(/[/?#].*$/, ""))?.[1] || /edit\/(\d+)/.exec(String(args.id))?.[1])
127
- if (!id) throw new Error(`"${args.id}" is not an Azure DevOps test case id.`)
128
- if (!ctx.config.sources.ado) throw new Error("Azure DevOps is not set up: updating test cases works for Azure DevOps Test Cases.")
127
+ const raw = String(args.id).trim()
128
+ const jiraKey = /^(?!ADO-)[A-Z][A-Z0-9_]*-\d+$/i.test(raw) ? raw.toUpperCase() : /\/browse\/([A-Z][A-Z0-9_]*-\d+)/i.exec(raw)?.[1]?.toUpperCase()
129
+ const id = jiraKey ? 0 : Number(/(\d+)\s*$/.exec(raw.replace(/[/?#].*$/, ""))?.[1] || /edit\/(\d+)/.exec(raw)?.[1])
130
+ if (!jiraKey && !id) throw new Error(`"${args.id}" is not an Azure DevOps test case id or a Jira issue key.`)
131
+ if (jiraKey && !ctx.config.sources.jira) throw new Error("Jira is not set up: add sources.jira to forgeqe.yaml.")
132
+ if (!jiraKey && !ctx.config.sources.ado) throw new Error("Azure DevOps is not set up: add sources.azure_devops to forgeqe.yaml.")
129
133
  const steps = (args.steps as { action: string; expected?: string }[]).filter((s) => s?.action?.trim()).map((s) => ({ action: s.action.trim(), ...(s.expected?.trim() ? { expected: s.expected.trim() } : {}) }))
130
134
  if (!steps.length) throw new Error("steps is empty.")
131
- const [current] = await casesFrom([{ source: "azure_devops", ids: [String(id)] } as CaseSelector], ctx.config.sources, ctx.cwd)
135
+ const current: { steps: NewCase["steps"]; url?: string } | undefined = jiraKey
136
+ ? await jiraCase(ctx.config.sources, jiraKey)
137
+ : (await casesFrom([{ source: "azure_devops", ids: [String(id)] } as CaseSelector], ctx.config.sources, ctx.cwd))[0]
138
+ const label = jiraKey || String(id)
132
139
  const show = (list: { action: string; expected?: string }[]) => list.map((s, i) => ` ${i + 1}. ${s.action}${s.expected ? ` → ${s.expected}` : ""}`)
133
140
  const lines = ["Now:", ...show(current?.steps || []), "", "New:", ...show(steps), "", `Why: ${args.reason}`]
134
141
  if (ctx.ask) {
135
142
  const choice = await ctx.ask({
136
- title: `Update test case ${id}${args.title ? ` and rename it "${args.title}"` : ""}?`,
143
+ title: `Update test case ${label}${args.title ? ` and rename it "${args.title}"` : ""}?`,
137
144
  detail: current?.url,
138
145
  lines,
139
146
  options: [
@@ -145,9 +152,98 @@ export const CASE_TOOLS: McpTool[] = [
145
152
  } else if (args.confirmed !== true) {
146
153
  return { updated: false, needs_confirmation: true, current: current?.steps, proposed: steps, next_step: "Show the user the current and new steps; call update_test_case again with confirmed: true only after they agree." }
147
154
  }
148
- const url = await updateAdoCase(ctx.config.sources, id, { title: args.title ? String(args.title) : undefined, steps, reason: String(args.reason) })
149
- ctx.emit?.({ type: "card", card: { status: "ok", title: `Test case ${id} updated`, stats: `${steps.length} steps`, body: [...show(steps), "", `Why: ${args.reason}`, url] } })
150
- return { updated: true, url, steps: steps.length, next_step: "Offer to run it again with run_test_cases." }
155
+ const change = { title: args.title ? String(args.title) : undefined, steps, reason: String(args.reason) }
156
+ const url = jiraKey ? await updateJiraCase(ctx.config.sources, jiraKey, change) : await updateAdoCase(ctx.config.sources, id, change)
157
+ ctx.emit?.({ type: "card", card: { status: "ok", title: `Test case ${label} updated`, stats: `${steps.length} steps`, body: [...show(steps), "", `Why: ${args.reason}`, url] } })
158
+ return { updated: true, url, steps: steps.length, next_step: "Offer to run it again with run_test_cases; its automated test, when it has one, is updated to the new steps in that run." }
159
+ },
160
+ },
161
+ {
162
+ name: "story_test_cases",
163
+ description:
164
+ "The test cases already linked to an Azure DevOps or Jira story, with their steps and expected results. Call it before writing test cases for a story, so you write only the cases its acceptance criteria still need, and propose updates (update_test_case) to linked cases that no longer fit, instead of writing them again. Reads only.",
165
+ inputSchema: {
166
+ type: "object",
167
+ properties: {
168
+ system: { type: "string", enum: ["azure_devops", "jira"], description: "Where the story is" },
169
+ story: { type: "string", description: "The story's work item id (Azure DevOps) or issue key (Jira)" },
170
+ },
171
+ required: ["system", "story"],
172
+ },
173
+ async run(args, ctx) {
174
+ const story = String(args.story || "").trim()
175
+ if (!story) throw new Error("story is required.")
176
+ const found = await storyCases(ctx.config.sources, args.system === "jira" ? "jira" : "azure_devops", story)
177
+ ctx.emit?.({
178
+ type: "card",
179
+ card: {
180
+ status: "info",
181
+ title: `Test cases linked to ${found.story.id}${found.story.title ? ` ${found.story.title}` : ""}`,
182
+ stats: `${found.cases.length} linked`,
183
+ body: found.cases.length ? found.cases.map((c) => `${c.id} ${c.title} (${c.steps.length} steps)`) : ["None yet."],
184
+ },
185
+ })
186
+ return {
187
+ story: found.story,
188
+ cases: found.cases.map((c) => ({ id: c.id, title: c.title, url: c.url, steps: c.steps.map((st, i) => `${i + 1}. ${st.action}${st.expected ? ` → ${st.expected}` : ""}`) })),
189
+ next_step: found.cases.length
190
+ ? "Compare these with the story's acceptance criteria. Write only the cases for criteria and rules no linked case covers, and save only those. For a linked case that no longer fits the story, propose its new steps with update_test_case. Tell the user which criteria the linked cases already cover."
191
+ : "No case is linked yet: write the cases from the story.",
192
+ }
193
+ },
194
+ },
195
+ {
196
+ name: "e2e_journey",
197
+ description:
198
+ "Plan end-to-end journeys from test cases that already exist, instead of writing them again: from the cases linked to a story (system and story) or any find_test_cases sources. Groups the cases that start the same way (same start page and first steps, such as signing in), keeps those shared steps once, and lays out each case's remaining steps as a section of one journey, with which case and steps each section comes from. Flags cases that look like negative paths, which may end the session. Writes nothing.",
199
+ inputSchema: {
200
+ type: "object",
201
+ properties: {
202
+ system: { type: "string", enum: ["azure_devops", "jira"], description: "With story: the story's system" },
203
+ story: { type: "string", description: "A story whose linked cases to use" },
204
+ sources: { type: "array", description: "Or the same sources as find_test_cases", items: { type: "object" } },
205
+ },
206
+ },
207
+ async run(args, ctx) {
208
+ let cases: JourneyCase[]
209
+ let story: { id: string; title: string; url: string } | undefined
210
+ if (args.story) {
211
+ const found = await storyCases(ctx.config.sources, args.system === "jira" ? "jira" : "azure_devops", String(args.story).trim())
212
+ story = found.story
213
+ cases = found.cases.map((c) => ({ id: c.id, title: c.title, url: c.url, steps: c.steps }))
214
+ } else if (Array.isArray(args.sources) && args.sources.length) {
215
+ cases = (await casesFrom(args.sources as CaseSelector[], ctx.config.sources, ctx.cwd)).map((c) => ({ id: c.id, title: c.title, url: c.url, startUrl: c.startUrl, steps: c.steps }))
216
+ } else throw new Error("Give a story (with system), or sources.")
217
+ const plan = planJourneys(cases)
218
+ ctx.emit?.({
219
+ type: "card",
220
+ card: {
221
+ status: plan.journeys.length ? "ok" : "info",
222
+ title: `End-to-end journeys${story ? ` for ${story.id}` : ""}`,
223
+ stats: `${plan.journeys.length} journeys from ${cases.length} cases · ${plan.stepsSaved} repeated steps saved`,
224
+ body: [
225
+ ...plan.journeys.flatMap((j, n) => [
226
+ `Journey ${n + 1}: ${j.shared.length} shared steps, then ${j.sections.length} cases`,
227
+ ...j.sections.map((sec) => ` ${sec.negative ? "▲" : "·"} ${sec.caseId} ${sec.title} (steps ${sec.from}-${sec.to})`),
228
+ ]),
229
+ ...(plan.alone.length ? ["", `On their own: ${plan.alone.map((c) => c.id).join(", ")}`] : []),
230
+ ],
231
+ },
232
+ })
233
+ return {
234
+ ...(story ? { story } : {}),
235
+ journeys: plan.journeys.map((j) => ({
236
+ start_url: j.startUrl,
237
+ shared_steps: j.shared.length,
238
+ steps: j.steps.map((st, i) => `${i + 1}. ${st.action}${st.expected ? ` → ${st.expected}` : ""}`),
239
+ sections: j.sections.map((sec) => ({ case: sec.caseId, title: sec.title, steps: `${sec.from}-${sec.to}`, ...(sec.negative ? { negative: true } : {}) })),
240
+ repeated_steps_saved: j.saved,
241
+ })),
242
+ on_their_own: plan.alone.map((c) => ({ case: c.id, title: c.title })),
243
+ next_step: plan.journeys.length
244
+ ? "Order the sections so each leaves the application in the state the next one needs. A negative section (an error, a refusal, signing out, deleting shared data) goes last or stays its own case. Show the user each journey with the cases it covers. When they agree, save each as one case with save_test_cases, titled \"E2E: …\", with a description naming the cases it covers; keep the original cases. Run a journey instead of its separate cases only when the user asks for an end-to-end run."
245
+ : "The cases share no first steps, so a journey saves nothing: offer to run them as they are.",
246
+ }
151
247
  },
152
248
  },
153
249
  {
@@ -1,6 +1,7 @@
1
- import { coverageLines, EXTEND_FROM, measureCoverage, type Candidate } from "../code/coverage.ts"
1
+ import { coverageLines, EXTEND_FROM, measureCoverage, settle, type Candidate } from "../code/coverage.ts"
2
2
  import { buildGraph } from "../code/graph/graph.ts"
3
- import { EXISTING_TEST_FILE } from "../code/notes.ts"
3
+ import { EXISTING_TEST_FILE, EXTEND_TEST_FILE } from "../code/notes.ts"
4
+ import { recordDecision, type DecisionRecord } from "../code/decisions.ts"
4
5
  import type { ExistingTest, TestTarget } from "../code/prompt.ts"
5
6
  import { existsSync, readFileSync } from "node:fs"
6
7
  import path from "node:path"
@@ -33,19 +34,54 @@ const clip = (s: string | undefined, n: number) => {
33
34
  /** Marks a browser_run that was a task (a lookup in a tool), not a flow to automate; write_test refuses such runs. */
34
35
 
35
36
  /**
36
- * Reuse, extend, or new, from what the repository's tests already cover of the run (code graph and page memory).
37
- * The clear cases go ahead and say why; where the evidence fits more than one way, the user picks, seeing each
38
- * option's effect. A mode the user gave (through ForgeQE's agent) is followed without asking.
37
+ * Reuse, extend, add a test beside one, or new, from what the repository's tests already cover of the run (code graph
38
+ * and page memory). The clear cases go ahead and say why; where the evidence fits more than one way, the user picks,
39
+ * seeing each option's effect, and with no one to ask a fixed policy decides (settle). A mode the user gave is followed
40
+ * as it is; one ForgeQE's agent chose before the browser run is followed too, but a test is not made overlong.
39
41
  */
40
42
  export async function chooseTarget(runDir: string, repo: string, args: Record<string, any>, ctx: ToolContext): Promise<{ target?: TestTarget } | { cancelled: true }> {
41
- const asked = typeof args.mode === "string" && args.mode !== "auto" ? (args.mode as "new" | "extend" | "reuse" | "update") : null
42
- if (asked === "new") return {}
43
+ const asked = typeof args.mode === "string" && args.mode !== "auto" ? (args.mode as "new" | "extend" | "reuse" | "update" | "sibling") : null
44
+ // ForgeQE's agent chose it before the browser run (args.why); otherwise the user named it.
45
+ const byAgent = Boolean(args.why)
46
+ const goal = ((): string => {
47
+ try {
48
+ return (JSON.parse(readFileSync(path.join(runDir, "run.json"), "utf8")) as { goal?: string }).goal || ""
49
+ } catch {
50
+ return ""
51
+ }
52
+ })()
53
+ const note = (by: NonNullable<DecisionRecord["by"]>, mode: NonNullable<DecisionRecord["mode"]>, c: Candidate | undefined, why: string) =>
54
+ recordDecision(repo, { goal, stage: "chosen", where: "write_test", by, mode, ...(c ? { test: c.test, file: c.file } : {}), why, run_dir: runDir })
55
+ if (asked === "new") {
56
+ note(byAgent ? "agent" : "user", "new", undefined, "asked for a new test")
57
+ return {}
58
+ }
43
59
  const graph = await buildGraph(repo).catch(() => null)
44
60
  if (!graph) return {}
45
61
  const cov = measureCoverage(runDir, graph, ctx.config.memory.dir)
46
- const named = args.test_file ? cov.candidates.find((c) => c.file === String(args.test_file).replace(/^\.\//, "")) : undefined
62
+ const file = args.test_file ? String(args.test_file).replace(/^\.\//, "") : ""
63
+ // A test named but not measured (its page objects do not locate this run's elements) is still the one to work from.
64
+ const measurable = cov.items.flatMap((it, i) => (it.members.length ? [i] : []))
65
+ // In a file of many tests, the one named (Class.Method, or its method name) when there is one.
66
+ const testName = args.test_name ? String(args.test_name) : ""
67
+ const sameTest = (c: Candidate) => !testName || c.test === testName || c.test.endsWith(`.${testName}`)
68
+ const lineCount = (test: string) => graph.files.find((f) => f.file === file)?.classes.flatMap((k) => k.members.map((m) => ({ k, m }))).find(({ k, m }) => `${k.name}.${m.name}` === test || m.name === test)
69
+ const unmeasured = (): Candidate => {
70
+ const test = testName || path.basename(file).replace(/\.[^.]+$/, "")
71
+ const at = lineCount(test)
72
+ return { test: at ? `${at.k.name}.${at.m.name}` : test, file, line: at?.m.line || 1, lines: at ? at.m.endLine - at.m.line + 1 : undefined, covered: [], missing: measurable, share: 0 }
73
+ }
74
+ const named = file ? cov.candidates.find((c) => c.file === file && sameTest(c)) || unmeasured() : undefined
47
75
  let decision = cov.decision
48
- if (asked) decision = { mode: asked, ask: false, target: named || cov.candidates[0], why: "the user asked for it" }
76
+ if (asked) decision = { mode: asked, ask: false, target: named || cov.candidates[0], why: args.why ? String(args.why) : "the user asked for it" }
77
+ // Not the user's own choice, and no one to ask, or the agent's: ForgeQE's policy settles it.
78
+ // Who decided: the user, ForgeQE's agent, the coverage measure, or ForgeQE's policy when it changed or settled it.
79
+ let by: NonNullable<DecisionRecord["by"]> = asked ? (byAgent ? "agent" : "user") : "coverage"
80
+ if ((!asked || byAgent) && (byAgent || !decision.ask || !ctx.ask)) {
81
+ const settled = settle(decision)
82
+ if (settled.mode !== decision.mode || decision.ask) by = "policy"
83
+ decision = settled
84
+ }
49
85
  const lines = coverageLines(cov)
50
86
  const toTarget = (mode: TestTarget["mode"], c: Candidate): TestTarget => ({
51
87
  mode,
@@ -54,7 +90,7 @@ export async function chooseTarget(runDir: string, repo: string, args: Record<st
54
90
  line: c.line,
55
91
  covered: c.covered.map((i) => cov.items[i]!.label),
56
92
  missing: c.missing.map((i) => cov.items[i]!.label),
57
- why: `it covers ${c.covered.length} of ${c.covered.length + c.missing.length} steps and checks this run located`,
93
+ why: c.covered.length ? `it covers ${c.covered.length} of ${c.covered.length + c.missing.length} steps and checks this run located` : decision.why,
58
94
  })
59
95
  const say = (title: string, why: string) =>
60
96
  ctx.emit?.({ type: "card", card: { status: "info", title, stats: why, body: lines.length ? ["What the repository's tests already cover of this run:", ...lines] : [] } })
@@ -62,7 +98,7 @@ export async function chooseTarget(runDir: string, repo: string, args: Record<st
62
98
  const options = cov.candidates.filter((c) => c.share >= EXTEND_FROM).slice(0, 2).flatMap((c, n) => {
63
99
  const name = `${path.basename(c.file)}`
64
100
  return c.missing.length
65
- ? [{ id: `extend:${n}`, label: `Extend ${name} (adds ${c.missing.length})` }]
101
+ ? [{ id: `extend:${n}`, label: `Extend ${name} (adds ${c.missing.length})` }, ...(n === 0 ? [{ id: `sibling:${n}`, label: `Add a test beside ${c.test.split(".").pop()} in ${name}` }] : [])]
66
102
  : [{ id: `reuse:${n}`, label: `Reuse ${name} as it is` }]
67
103
  })
68
104
  const answer = await ctx.ask({
@@ -72,18 +108,26 @@ export async function chooseTarget(runDir: string, repo: string, args: Record<st
72
108
  options: [...options, { id: "new", label: "Write a new test" }, { id: "cancel", label: "Cancel" }],
73
109
  })
74
110
  if (!answer || answer === "cancel") return { cancelled: true }
75
- if (answer === "new") return {}
76
- const [mode, n] = answer.split(":") as ["reuse" | "extend", string]
111
+ if (answer === "new") {
112
+ note("user", "new", undefined, `${decision.why}; the user chose a new test`)
113
+ say("Writing a new test", "the user chose it")
114
+ return {}
115
+ }
116
+ const [mode, n] = answer.split(":") as ["reuse" | "extend" | "sibling", string]
77
117
  const c = cov.candidates.filter((x) => x.share >= EXTEND_FROM)[Number(n)]!
78
- say(mode === "reuse" ? `Reusing ${c.test}` : `Extending ${c.test}`, "the user chose it")
118
+ note("user", mode, c, `${decision.why}; the user chose it`)
119
+ say(mode === "reuse" ? `Reusing ${c.test}` : mode === "sibling" ? `Adding a test beside ${c.test}` : `Extending ${c.test}`, "the user chose it")
79
120
  return { target: toTarget(mode, c) }
80
121
  }
81
122
  if (decision.mode === "new" || !decision.target) {
123
+ note(by, "new", undefined, decision.why)
82
124
  if (cov.candidates.length) say("Writing a new test", decision.why)
83
125
  return {}
84
126
  }
85
- const mode: TestTarget["mode"] = decision.mode === "reuse" ? "reuse" : decision.mode === "update" ? "update" : "extend"
86
- say(mode === "reuse" ? `Reusing ${decision.target.test}` : mode === "update" ? `Updating ${decision.target.test}` : `Extending ${decision.target.test}`, decision.why)
127
+ const mode: TestTarget["mode"] = decision.mode === "reuse" || decision.mode === "update" || decision.mode === "sibling" ? decision.mode : "extend"
128
+ const verb = { reuse: "Reusing", update: "Updating", sibling: "Adding a test beside", extend: "Extending" }[mode]
129
+ note(by, mode, decision.target, decision.why)
130
+ say(`${verb} ${decision.target.test}`, decision.why)
87
131
  return { target: toTarget(mode, decision.target) }
88
132
  }
89
133
 
@@ -158,11 +202,12 @@ export const COMMAND_TOOLS: McpTool[] = [
158
202
  repo: { type: "string", description: "Repository to write the test in, relative to the working directory (default: the working directory)" },
159
203
  mode: {
160
204
  type: "string",
161
- enum: ["auto", "new", "extend", "reuse", "update"],
205
+ enum: ["auto", "new", "extend", "reuse", "update", "sibling"],
162
206
  description:
163
- "auto (default): ForgeQE measures which existing tests already cover the run, then reuses one that covers it all, extends one that covers most, or writes a new test, and asks the user when that is not clear. new, extend, reuse, update: only when the user said which (\"extend the login test\", \"write a separate test\", \"update it\" after a reused test failed).",
207
+ "auto (default): ForgeQE measures which existing tests already cover the run, then reuses one that covers it all, extends one that covers most (or adds a test beside it, in its class, when it is already long), or writes a new test that reuses the repository's code, and asks the user when that is not clear. new, extend, reuse, update, sibling: only when the user said which (\"extend the login test\", \"add a separate test next to it\", \"write a separate test\", \"update it\" after a reused test failed).",
164
208
  },
165
- test_file: { type: "string", description: "With mode extend, reuse, or update: the existing test file (from the user, or from an earlier write_test result)" },
209
+ test_file: { type: "string", description: "With mode extend, reuse, update, or sibling: the existing test file (from the user, or from an earlier write_test result)" },
210
+ test_name: { type: "string", description: "With test_file: the test in it (Class.Method or the method name), when the file has more than one" },
166
211
  },
167
212
  },
168
213
  async run(args, ctx) {
@@ -177,8 +222,16 @@ export const COMMAND_TOOLS: McpTool[] = [
177
222
  const goal = (readJson(path.join(runDir, "run.json")) as { goal?: string } | null)?.goal || ""
178
223
  // browser_run found an earlier test for this flow and it failed: Scribe updates it rather than adding another.
179
224
  const existingTest = readJson(path.join(runDir, EXISTING_TEST_FILE)) as ExistingTest | null
225
+ // ForgeQE's agent chose, before the browser ran, to extend an earlier test: that test, unless the user says otherwise.
226
+ const extend = readJson(path.join(runDir, EXTEND_TEST_FILE)) as { testFile?: string; test?: string; mode?: "extend" | "sibling" } | null
227
+ // The agent repeating that choice to write_test (its mode and file) is still the agent's choice, not the user's.
228
+ const sameChoice = extend?.testFile && args.mode === (extend.mode || "extend") && (!args.test_file || String(args.test_file).replace(/^\.\//, "") === extend.testFile)
229
+ const how =
230
+ extend?.testFile && (!args.mode || args.mode === "auto" || sameChoice)
231
+ ? { ...args, mode: extend.mode || "extend", test_file: extend.testFile, ...(extend.test ? { test_name: extend.test } : {}), why: "chosen before the browser run, from the existing tests closest to this flow" }
232
+ : args
180
233
  // Otherwise: what the repository's tests already cover of this run decides reuse, extend, or new.
181
- const chosen = existingTest?.testFile ? { target: undefined } : await chooseTarget(runDir, repo, args, ctx)
234
+ const chosen = existingTest?.testFile ? { target: undefined } : await chooseTarget(runDir, repo, how, ctx)
182
235
  if ("cancelled" in chosen) return { status: "cancelled", notes: "The user cancelled before Scribe started; nothing was changed." }
183
236
  const target = chosen.target
184
237
  ctx.emit?.({ type: "code-start", runDir, repo, goal })
@@ -200,6 +253,7 @@ export const COMMAND_TOOLS: McpTool[] = [
200
253
  throw err
201
254
  }
202
255
  ctx.emit?.({ type: "code-done", result: r, ms: Date.now() - t0 })
256
+ recordDecision(repo, { goal, stage: "outcome", where: "write_test", status: r.status, ...(r.existing ? { existing: r.existing } : {}), ...(r.testFile ? { file: r.testFile } : {}), ...(target ? { mode: target.mode, test: target.test } : existingTest ? { mode: "update" } : { mode: "new" }), ...(r.breaches ? { breaches: r.breaches } : {}), run_dir: runDir })
203
257
  ctx.emit?.({ type: "card", card: codeCard(buildCodeSummary(r, Date.now() - t0)) })
204
258
  return {
205
259
  status: r.status,