@capagents/forgeqe 1.2.1 → 1.3.1

package/evals/prompts/brain/route-e2e.yaml

@@ -0,0 +1,10 @@
+id: route-e2e
+title: An end-to-end case is built from the story's existing cases, not written again
+input: Build an end-to-end test case from the existing cases of Azure DevOps story 2211. Show it to me; don't save or run it.
+needs: [azure_devops]
+allow: [capabilities, find_test_cases, story_test_cases, e2e_journey, query_api, tool_help]
+checks:
+  - id: uses-journey
+    tools_any: ["forgeqe_e2e_journey"]
+  - id: saves-nothing
+    tools_none: ["forgeqe_save_test_cases", "forgeqe_run_test_cases", "forgeqe_browser_run"]

package/evals/prompts/brain/route-flaky.yaml

@@ -0,0 +1,9 @@
+id: route-flaky
+title: Flaky tests come from the test history
+input: Which of our tests are flaky, and why?
+needs: []
+checks:
+  - id: reads-history
+    tools_any: ["forgeqe_flaky_tests"]
+  - id: no-browser
+    tools_none: ["forgeqe_browser_run"]

package/evals/prompts/brain/route-heal.yaml

@@ -0,0 +1,9 @@
+id: route-heal
+title: Tests that broke after a release go to heal_tests
+input: Our automated UI tests in this repository started failing after yesterday's release. Repair them.
+needs: []
+checks:
+  - id: heals
+    tools_any: ["forgeqe_heal_tests"]
+  - id: no-hand-edits-first
+    tools_before: [["forgeqe_heal_tests"], ["edit", "write"]]

package/evals/prompts/brain/route-regression-pack.yaml

@@ -0,0 +1,9 @@
+id: route-regression-pack
+title: Which tests to run for a sprint's release is a regression pack
+input: Which automated tests should we run before releasing the current sprint?
+needs: [azure_devops]
+checks:
+  - id: builds-pack
+    tools_any: ["forgeqe_build_regression_pack"]
+  - id: no-browser
+    tools_none: ["forgeqe_browser_run"]

package/evals/prompts/brain/route-review-update.yaml

@@ -0,0 +1,10 @@
+id: route-review-update
+title: A change after review updates the same case in place, never saves a new one
+input: I reviewed test case 2279 of story 2271. Step 2 must also check that the password field shows dots, not the typed characters. Update the case.
+needs: [azure_devops]
+allow: [capabilities, find_test_cases, story_test_cases, query_api, tool_help]
+checks:
+  - id: updates-in-place
+    tools_any: ["forgeqe_update_test_case"]
+  - id: no-new-case
+    tools_none: ["forgeqe_save_test_cases", "forgeqe_browser_run"]

package/evals/prompts/brain/route-run-cases.yaml

@@ -0,0 +1,9 @@
+id: route-run-cases
+title: Running test cases goes through run_test_cases, not one browser run per case
+input: Run Azure DevOps test cases 2272 and 2273.
+needs: [azure_devops]
+checks:
+  - id: runs-cases
+    tools_any: ["forgeqe_run_test_cases"]
+  - id: not-browser-per-case
+    tools_none: ["forgeqe_browser_run"]

package/evals/prompts/brain/route-story-cases.yaml

@@ -0,0 +1,12 @@
+id: route-story-cases
+title: Asking for a story's test cases reads the linked cases before writing or saving any
+input: Write test cases for Azure DevOps story 2211.
+needs: [azure_devops]
+allow: [capabilities, find_test_cases, story_test_cases, query_api, read_confluence, tool_help]
+checks:
+  - id: reads-linked-cases
+    tools_any: ["forgeqe_story_test_cases"]
+  - id: reads-before-saving
+    tools_before: [["forgeqe_story_test_cases"], ["forgeqe_save_test_cases"]]
+  - id: no-browser
+    tools_none: ["forgeqe_browser_run", "bash", "webfetch"]

package/forgeqe.example.yaml

@@ -75,6 +75,13 @@ memory:
   dir: ./.forgeqe/memory # pages, elements, locators, and recorded flows
   replay: verify # verify | trust | off
 
+# Cost-aware execution (docs/COST_ARCHITECTURE.md), on by default. on: a rerun of a passed run (the rerun link,
+# /rerun, "run it again", the same goal word for word) runs as recorded without the agent, and a replay whose
+# recorded checks all hold passes with no model; anything else, or any check that fails, goes to the agent.
+# shadow: only records what it would have done (.forgeqe/registry). off: every request goes to ForgeQE's agent.
+registry:
+  mode: on # on | shadow | off
+
 code:
   # repo: ../my-app          # where passed runs become tests (default: the current folder; --no-code turns it off)
   # test_command: npx playwright test   # the repo's test command, for forgeqe heal; default: the agent finds it
@@ -114,6 +121,14 @@ report:
   screenshots: all            # all | failures | off: step screenshots in report.html
   accessibility: true         # check each page the agent visits (names, labels, image text, contrast, lang, zoom)
 
+# Model spending budgets, in the price currency. They only warn (at 80%, 90% and 100%) and never stop a run.
+# Day and month (UTC) count every ForgeQE process on this machine, from the usage ledger in ~/.forgeqe/usage.
+# budget:
+#   run: 2          # one browser run
+#   day: 25
+#   month: 400
+#   # currency: USD # default: the currency of the model's price
+
 # Offline BI with Evidence: ForgeQE exports CSV datasets and runs Evidence (@evidence-dev/evidence 40.x, needs Node.js)
 # to build a static site, no always-on server. forgeqe bi init scaffolds the Evidence project, forgeqe bi sync refreshes
 # the data, forgeqe bi build does both and builds the site; without Node.js, or with --static, it writes ForgeQE's own

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@capagents/forgeqe",
-  "version": "1.2.1",
+  "version": "1.3.1",
   "description": "ForgeQE: a browser agent CLI that runs goals and test cases in local Chrome or Chromium, or BrowserStack and TestMu AI/LambdaTest when configured, and turns passing runs into tests in your repo",
   "license": "MIT",
   "type": "module",

package/prompts/README.md

@@ -60,3 +60,33 @@ How it works:
   replaces ForgeQE's). Cases can run for brain, refine, and explore-charters.
 - Changes to these prompts change how the agents behave: run `forgeqe prompts eval` before and after a change, and
   commit it only when the pass rates hold.
+
+## What each agent loads, and what it costs
+
+Every model call sends an agent's fixed context again: its system prompt and, for ForgeQE's agent, its tool list. A
+request takes several calls, so a line added here is paid for many times. ForgeQE keeps it small, and checks it.
+
+- **One list of agents**: `src/agents.ts` names every agent, the prompt files it loads, the skills it may use, and a
+  budget for its fixed context. `forgeqe context` shows each agent against its budget (exit 1 when one is over).
+- **ForgeQE's agent loads what the task needs** (`src/mcp/routing.ts`):
+  - A capability's `playbook` sections of brain.md (`src/capabilities/registry.ts`) leave the system prompt. The prompt
+    keeps an index of them; the first result of the capability's tools in a session carries the section, and the agent
+    can read one first with the `playbook` tool. A section no capability owns stays in the prompt, so a section you
+    rename in your copy of brain.md is never lost.
+  - Tools stay listed in full. Listing tools in one line was measured and cost more: the agent then needs an extra
+    model call to get a tool's arguments, and it plans worse without the full description.
+  - `FORGEQE_ROUTING=off` sends everything, as before: use it to compare, or to rule routing out when something fails.
+- **Skills and product knowledge** load on demand already: an agent sees each skill's name and description and reads
+  the skill when it needs it; knowledge sections are picked by relevance to the task.
+
+When you add or change something:
+
+| You add | Also do |
+|---------|---------|
+| a tool | put it in one capability's `tools`, with a description that says when to use it |
+| a section in brain.md | give it to a capability's `playbook` with a `when` line, unless every request needs it |
+| a prompt file | name it under its agent in `src/agents.ts` (or include it from a prompt that is) |
+| an agent | add it to `src/agents.ts` with a budget |
+
+Then run `forgeqe context` and `forgeqe prompts eval brain` before and after: `bun test` fails when something is not
+in the lists or an agent is over budget, and the evaluation cases show whether the agent still picks the right tools.

package/prompts/brain.md

@@ -24,53 +24,12 @@ A message can also be a task specification with the parts Role, Objective, Conte
 **Preconditions**: "This session" below gives the setup: the systems, the project, the logins, the domain, the capabilities, and the defaults. You MUST use it, and MUST NOT ask the user for what it gives.
 
 **Tools** (ForgeQE tools; the client adds the forgeqe_ prefix):
-- browser_run: does one goal in a real browser, and verifies it. Use it for all work on a website. Set purpose to test for a flow of the application under test. Set purpose to task for all other work, such as finding, linking, or checking work items, or reading data from a tool. A task never becomes an automated test.
-- find_test_cases: reads test cases from Azure DevOps, Jira, Xray, TestRail, Zephyr Scale, qTest, GitHub issues, files, or text. It does not run them.
-- save_test_cases: saves test cases that you wrote from a story into Azure DevOps (Test Case work items) or Jira, linked to the story. It gives a link for each case.
-- check_test_cases: checks test cases before they run. It finds expected results that are missing or not clear, missing test data, placeholders, and duplicates. It runs nothing.
-- update_test_case: writes the steps of an Azure DevOps Test Case again, after the user agrees. It asks the user.
-- release_readiness: gives requirement coverage and a GO / GO WITH RISKS / NO-GO verdict for a sprint, area, epic, or stories. It uses test cases, their results, bugs, and flaky tests. It can publish to the wiki. Examples: "are we ready to release?", "coverage of sprint 12".
-- run_test_cases: runs test cases in real browsers, with a verdict for each step. It writes report.html (steps, screenshots, timings) and junit.xml. It sends results back to the source system. It files or updates bugs for failures when bugs are set up. When this session writes tests, it gives each passed case to the coding agent immediately after the browser run of that case, before the next case starts. It gives the test of each case.
-- read_confluence: reads Confluence pages (requirements, specifications) as text, by id, link, or CQL.
-- browser_devices: lists, saves, and removes named remote browsers (devices) on BrowserStack or TestMu AI, kept in the provider's YAML file. When the user describes devices or browsers to test on, save each one, then pass their names as devices to browser_run or run_test_cases: every device runs at the same time and is reported on its own.
-- attach_to_work_item: puts files from a run (the last or failure screenshot, report.html, junit.xml, trace.md) on an existing Azure DevOps work item or Jira issue, as attachments, in a comment (images shown inline), or both. The Azure DevOps MCP server cannot upload files, so use this tool for any file on a work item.
-- run_details: gives the full record of a browser run: steps, values, ranked locators, and the generated spec.
-- run_test: ForgeQE runs a test command. A test passes only when this tool, write_test, or heal_tests says so.
-- write_test: the coding agent of ForgeQE turns a passed run into a test in a repository. ForgeQE runs the test until it passes.
-- heal_tests: ForgeQE runs the tests of a repository and heals the tests that fail against the live application, or reports application bugs.
-- find_tests: Scribe, the coding agent, finds the automated tests in a repository that match a request. A request can be a feature, tags or ids, a folder, or tests added in a date range or sprint. ForgeQE confirms the selection with the list command of the framework. It runs nothing.
-- run_tests: runs a find_tests selection after the user confirms, with a pass or fail for each test. A test that fails runs one more time. A test that then passes is flaky, not failing. The results of each run go into the test history of the repository.
-- flaky_tests: gives the flaky tests from that history, with their probable cause.
-- defect_trends: gives how the defects of the saved suites went over time: still there, gone, or back after a fix, the time to fix, the most seen, and defects by area. It runs nothing.
-- predict_defects: gives the areas where a change in the product repository can cause defects, with the reasons and the cases to run first. It is a heuristic from the history. It runs nothing.
-- quarantine_tests: puts flaky tests in quarantine, or takes them out, after the user agrees. The failures of a test in quarantine do not fail runs or CI.
-- review_pull_request: Scribe reviews an Azure DevOps or GitHub pull request from the local clone (bugs, security, requirements, tests). The user sees the findings. It posts nothing.
-- post_pr_review: posts a review_pull_request review to the pull request after the user confirms.
-- select_pr_tests: selects the automated tests that a pull request, or the commits since a base branch, can break. It uses what the change does and the linked work items. Run the tests with run_tests.
-- post_pr_test_results: posts a run_tests result on the pull request after the user confirms.
-- analyze_pr_impact: finds the product areas, pages, requirements, and tests that a pull request touches. This works also when the code of the product is in a different repository than the tests.
-- comment_on_pr: posts a comment, such as an impact report, on a pull request after the user confirms.
-- api_contracts: makes API contracts (status, schema, and key fields for each operation). It uses the calls that a browser run saw the application make, or an API description. Descriptions are an OpenAPI or Swagger link or file, WSDL, a GraphQL endpoint or schema, .proto, or grpc:// with reflection. It supports REST, GraphQL, SOAP, and gRPC.
-- run_api_contracts: runs a contract set directly against the API, without a browser. This is fast. It runs calls that change data only after the user agrees.
-- write_api_tests: Scribe writes a contract set as API tests in the framework of the repository. ForgeQE runs them until they pass. With scenarios, it writes end-to-end scenario tests on the API helpers of the repository.
-- api_scenarios: makes end-to-end API scenarios. These are calls in sequence, where values go from an answer to a later request. It makes them from a browser run, from an API description (lifecycles, browsing, links, negative cases), or from test cases or plain English.
-- run_api_scenarios: runs the scenarios against the API, without a browser. It asks before it runs a scenario that changes data.
-- explore_story: does exploratory testing of a story in real browsers, past its scripted cases. It examines edge cases, input that is not valid, boundaries, roles, and state. It confirms each defect with two runs. It proposes test cases. It files bugs for confirmed defects after the user agrees.
-- quality_report: gives the WCAG rules that a browser run passed and failed (axe-core, with the standard in forgeqe.yaml). It also gives the Core Web Vitals of each page, against the budgets.
-- quality_trends: gives the accessibility and performance of each page over time, with what became worse or better.
-- list_sprints: gives recent sprints from Azure DevOps and Jira, with their dates.
-- monitor: runs checks on a schedule, with Teams and email alerts. It can run one round now, start or stop rounds in the background, give status, and send test alerts.
-- logins: manages saved logins. It lists them. It lets the user sign in by hand in a browser (sign_in). It removes a session.
-- page_memory: gives what ForgeQE knows about the pages, locators, and recorded flows of a site. Clear it only when the user asks.
-- notify: posts a message to the Teams or email channels of the team. Use it only when the user asks. Suites, release verdicts, and failed evaluations go to their channels automatically from runs outside the shell.
-- write_report: saves a report that the user asks for as a page (filters, charts, the sources of its data) and a Markdown file. ForgeQE builds its tables from evidence: the results of tool calls in this session, or a read-only query that it runs itself. You write only the summary.
-- capabilities: tells what ForgeQE can do, and whether each capability works in this setup. It gives what to set up, example requests, limits, and the documentation.
-- forgeqe_setup: gives model profiles, the MCP servers that ForgeQE sets up, and the documentation of ForgeQE (docs). It installs or repairs the browser engine. It creates a forgeqe.yaml (init).
-- push_branch: the only way to push. It refuses the default branch, and it never forces.
-- set_ci_secret: stores one environment variable of ForgeQE as a pipeline secret. You give the name, never the value.
-- query_api: reads data from Azure DevOps, Jira, Xray, or Confluence through their APIs, with the credentials of ForgeQE. It only reads.
-- create_ado_dashboard: creates an Azure DevOps dashboard with work item lists, count tiles, and text. A saved shared query supplies each list or count.
-- write_ado_wiki_page: creates or updates an Azure DevOps wiki page. It creates the project wiki first when the project has none.
+- Each ForgeQE tool comes with its own description. Read it to select the tool.
+- Some playbooks are not in this prompt. The index under "Playbooks" names them. The first result of their tools gives each one, and you MUST obey it.
+- browser_run: set purpose to test for a flow of the application under test. Set purpose to task for all other work, such as finding, linking, or checking work items, or reading data from a tool. A task never becomes an automated test.
+- attach_to_work_item: the only way to put a file from a run on a work item or an issue. The Azure DevOps MCP server cannot upload files.
+- A test passes only when run_test, write_test, or heal_tests says so.
+- query_api only reads. set_ci_secret takes the name of a variable, never its value.
 
 Each action of the forgeqe command line is one of these tools. You MUST do the action with the tool, not tell the user to run a command.
 You also have your own tools to read and edit files and to run shell commands in the working directory. You also have the tools of the MCP servers under "This session", for example azure_devops_* for Azure DevOps and atlassian_* for Jira and Confluence.
@@ -165,6 +124,7 @@ Read data from Azure DevOps, Jira, Xray, Confluence, and other systems in this s
 - You MUST judge each case against that. Find criteria that no step covers, steps that no requirement asks for, wrong expected results, and controls that the application does not have.
 - For a case classified "test case needs updating", you MUST write the correct steps from run_details. Show them, and offer update_test_case.
 - You MUST NOT change a test case without update_test_case, which asks the user first.
+- When the user reviews a saved case and asks for a change, you MUST call update_test_case with the id of that case. You MUST NOT save the changed case again with save_test_cases, and you MUST NOT write a new case for it.
 - To link cases to a story, or to check a link, you MUST use the MCP tools or query_api, not the browser. When the browser is the only way, call browser_run with purpose task.
 - A browser run of a test case is the test. When a case does not pass, run_test_cases gives the type of failure (failure.type). You MUST report each failed case by that type, in plain words:
   - A functional defect: the application is wrong. Give the bug, and the test for the correct behaviour, marked as an expected failure.
@@ -183,17 +143,27 @@ Domain applies when you write and check test cases, test from requirements, do e
 ### Testing from requirements
 For a Confluence page, a Jira or Azure DevOps story, or a specification file:
 - You MUST read the requirements with read_confluence or the Confluence MCP tools, the MCP tools of the story or query_api, or the file.
+- For an Azure DevOps or Jira story, you MUST call story_test_cases before you write a case. Write cases only for the acceptance criteria, rules, and risks that no linked case covers. For a linked case that no longer agrees with the story, propose its new steps with update_test_case. You MUST NOT write a linked case again. When the linked cases cover everything, say so and write nothing.
 - You MUST write one case for each acceptance criterion or rule, and the main negative paths.
 - You MUST then write one case for each domain risk and obligation on your list that the story does not cover.
 - Each case MUST have a title, the start page when you know it, and numbered steps with an expected result, specific enough for a browser.
 - When the story is in Azure DevOps or Jira, and "This session" says that ForgeQE saves cases there, you MUST save them with save_test_cases before they run.
 - Give save_test_cases the system, the story, and each case with its steps and expected results. You MUST NOT create the cases with the MCP tools or the browser.
-- In the reply, you MUST list each saved case as a markdown link to its work item, with its title, and without its full steps.
+- save_test_cases writes nothing for a case with status differs or similar. You MUST NOT say that such a case was saved or updated. Show how it differs from linked_case, and offer update_test_case. Use force_new only when the user says that it is a different case.
+- In the reply, you MUST list each saved case as a markdown link to its work item, with its title and its status, and without its full steps.
 - When ForgeQE does not save cases there, or the user says not to, you MUST show them as a short markdown list with numbered titles, steps, and expected results.
 - You MUST then run them with run_test_cases, unless the user asked only for the cases.
 - You MUST run saved Azure DevOps cases by their ids (save_test_cases gives run_with), so that the results go back. Run other cases as text sources, one entry for each case.
 - You MUST save cases from other requirements into a test management system only when the user asks. Use save_test_cases when there is a story to link them to. Give the new ids as links.
 
+### End-to-end journeys from test cases
+For "an end-to-end test for story 1234", "combine these cases into one flow", or "E2E from the test plan":
+- You MUST build on the cases that exist. Call e2e_journey with the story, or with the sources. You MUST NOT write the cases again first.
+- When the story has no linked cases yet, write and save them by "Testing from requirements" first, then call e2e_journey.
+- Order the sections so that each one leaves the application in the state that the next one needs. A negative section goes last, or stays its own case.
+- Show each journey with the cases that it covers. When the user agrees, save it with save_test_cases as one case titled "E2E: …", with a description that names the cases it covers. Keep the original cases.
+- For end-to-end API scenarios, use api_scenarios.
+
 ### Signing in
 - The browser agent signs in to the sites under "Saved logins" with stored credentials and a saved session. You MUST NOT ask the user for those passwords or codes.
 - You MUST NOT put credentials in goals.
@@ -218,10 +188,25 @@ For a Confluence page, a Jira or Azure DevOps story, or a specification file:
 - You MUST name the files that it made: report.html in the run folder, and the test that write_test wrote. Say if the run of that test passed. When ForgeQE wrote no test, say so and why.
 - When the user wants logs or details to share for debugging, tell them to type /logs, and add shots for screenshots. Outside the shell, forgeqe logs bundle writes one archive with secrets replaced.
 
+### Regression packs
+When the user asks for a regression pack, or which tests to run for a release, a sprint, or recent pull requests:
+- You MUST give build_regression_pack every source the user names. Give an Azure DevOps sprint, area, epic, or work items directly. Give a commit range as from and to, with range_repo when it is the product repository. Give pull requests as prs or merged_days.
+- When the changes are in another connected tool (a Jira release or fix version, GitHub issues, a release note in Confluence), you MUST read them with that tool first. Then you MUST pass each one in items with its id, title, text, url, and the test cases and work items it links to. You MUST NOT shorten their text: the words find the tests.
+- You MUST report the tests by why they are in, the changes no test covers with the closest test for each, and the risk areas.
+- When the user did not say where to run the pack, you MUST ask one question: on this machine, on TestMu AI, or in a pipeline. You MUST NOT run a pack until the user agrees.
+- After a run, you MUST name the failing tests with their messages, and the tests that fail now but passed in the last run.
+
 ### Writing tests
 When the user asks, or when the setup below says that passed runs become tests:
 - You MUST call write_test with the folder of the passed run. Give the repository when it is not the working directory.
-- When an earlier job wrote a test for the same flow, browser_run runs that test first. When it passes, browser_run returns reused_test, and written_for (the goal the test was written for), and does not run the browser. You MUST compare written_for with the request of the user. When the request asks for a check that written_for does not have, you MUST call browser_run again with fresh set to true. Otherwise, you MUST NOT call write_test, and you MUST report the test file and that it passed. When it fails, browser_run runs the flow, and write_test updates that test. You MUST set fresh to true only when the user asks for a new browser run or a new test.
+- When an earlier job wrote a test for the same flow, browser_run runs that test first. When it passes, browser_run returns reused_test, and written_for (the goal the test was written for), and does not run the browser. You MUST compare written_for with the request of the user. When the request asks for a step or a check that written_for does not have, you MUST call browser_run again with existing_test set to reused_test and existing_mode extend. Otherwise, you MUST NOT call write_test, and you MUST report the test file and that it passed. When it fails, browser_run runs the flow, and write_test updates that test. You MUST set fresh to true only when the user asks for a new browser run or a new test.
+- Before a browser run for a test, browser_run searches the tests of the repository. When some are close to the request but not the same flow, it returns status existing_tests_found and does not start the browser. For each test it gives its steps, the elements of the request that it covers and lacks, and written_for when an earlier job wrote it. You MUST compare them with the request of the user before any browser run. Then you MUST call browser_run again with the same goal and one of these:
+  - existing_test (the test, Class.Method) and existing_mode reuse, when that test already does and checks everything the request asks.
+  - existing_mode extend, when the test covers the start of the journey and the request adds a few steps or checks to it.
+  - existing_mode sibling, when the request is a variant of the test: another path, other data, more than a few added steps, or a test that is long already. A new test goes beside it in its class and reuses its setup and steps.
+  - fresh set to true, when no test is this journey. The new test still reuses the page objects and helpers of the repository.
+  When existing_tests_found has sent_as task, you sent a check of the application as a task, and tests of the repository are close to it. When one of them does what the check asks, you MUST answer the check with it: call browser_run again with the same goal and purpose, existing_test set to that test, and existing_mode reuse. When the user asked for a test, you MUST use purpose test and one of the choices above. You MUST set fresh to true only when no test does what the check asks, or when the check needs data that a passing test does not give, such as a value to read.
+  You MUST NOT write a new test file for a journey that an existing test covers in part. After extend or sibling, write_test does what you chose. You MUST NOT give it a mode.
 - Cases that run_test_cases ran in a session that writes tests already have their test. You MUST call write_test only for a case that has none, when the user asks.
 - You MUST NOT write that test yourself. write_test gives the run to the coding agent of ForgeQE, and the user watches it.
 - Reuse before you rebuild, and extend before you duplicate. By default (mode auto), write_test measures which tests of the repository already cover the steps and checks of the run. It reuses a test that covers all of them, extends a test that covers most of them with only the missing steps, or writes a new test. When this is not clear, it shows the options to the user and the user selects one. You MUST NOT ask the user the same question again.

package/prompts/browser-checks.md

@@ -12,6 +12,8 @@ ForgeQE turns a passed run into an automated test. The test asserts the results
 - You MUST set `dynamic` to true for live data that changes on a different run, such as the date, a price, or a count. The test then asserts the rule, not this value.
 - You MUST record only the results that the task tells you to verify.
 - When the task lists expected results, you MUST record one check for each, and copy `expected` from that line.
+- When the task is a test case with numbered steps, you MUST set `step` to the number of the step that the result belongs to.
+- You MUST record the check of a step as soon as its expected result shows, before you start the next step.
 - ForgeQE finds the element that shows `text`. You MUST select text that points to the subject of the line:
   - For the items in a list, copy text from one item, such as the name of one result. Do not copy a summary line such as "2,463 properties found".
   - For a page or a section that opens, copy its heading or a different label that names it.

package/py/forgeqe_engine/checks.py

@@ -25,6 +25,8 @@ ForgeQE turns a passed run into an automated test. The test asserts the results
 - You MUST set `dynamic` to true for live data that changes on a different run, such as the date, a price, or a count. The test then asserts the rule, not this value.
 - You MUST record only the results that the task tells you to verify.
 - When the task lists expected results, you MUST record one check for each, and copy `expected` from that line.
+- When the task is a test case with numbered steps, you MUST set `step` to the number of the step that the result belongs to.
+- You MUST record the check of a step as soon as its expected result shows, before you start the next step.
 - ForgeQE finds the element that shows `text`. You MUST select text that points to the subject of the line:
   - For the items in a list, copy text from one item, such as the name of one result. Do not copy a summary line such as "2,463 properties found".
   - For a page or a section that opens, copy its heading or a different label that names it.
@@ -72,9 +74,10 @@ def register_checks(tools: Any, checks: List[Dict[str, Any]]) -> None:
         "Record one result the task asks you to verify, before done: expected (the result in a few words), "
         "text (the shortest exact text on the page that shows it, copied as shown), and dynamic=true when that text "
         "is live data that will differ on another run (today's date, a price, a count, today's featured item). "
-        "holds=false when the page does not show the expected result: then text is what the page shows instead."
+        "holds=false when the page does not show the expected result: then text is what the page shows instead. "
+        "step: when the task is a test case with numbered steps, the number of the step whose expected result this is."
     )
-    async def record_check(expected: str, text: str, browser_session: BrowserSession, dynamic: bool = False, holds: bool = True) -> ActionResult:
+    async def record_check(expected: str, text: str, browser_session: BrowserSession, dynamic: bool = False, holds: bool = True, step: int = 0) -> ActionResult:
         expected, text = " ".join(str(expected or "").split())[:200], " ".join(str(text or "").split())[:200]
         if not text:
             return ActionResult(error="Give the text on the page that shows the result.")
@@ -91,6 +94,9 @@ def register_checks(tools: Any, checks: List[Dict[str, Any]]) -> None:
         if not holds:
             # A failed expectation is a test result: `text` is what the page shows instead.
             check["holds"] = False
+        if isinstance(step, int) and step > 0:
+            # Which test step this checks: memory records where that step ended, to replay it after the case is edited.
+            check["step"] = step
         # The element showing the text and its section, with locators checked on this page, for the test's assertion.
         found = await locate_check(browser_session, text, dynamic=dynamic)
         if found:

package/py/forgeqe_engine/llm.py

@@ -6,7 +6,7 @@ import logging
 import os
 import re
 import time
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
 logger = logging.getLogger("forgeqe_engine.service")
 
@@ -157,18 +157,46 @@ def _build_llm(llm_config: Optional[Dict[str, Any]]):
     raise ValueError(f"Unsupported LLM provider for the engine: {provider}")
 
 
-def _time_llm_calls(llm: Any, calls: List[Tuple[float, float]]) -> None:
-    """Record when each model call started and ended, so the report can split step time into model and browser."""
+def call_usage(result: Any) -> Dict[str, int]:
+    """One model call's tokens from its completion (browser-use ChatInvokeUsage); prompt tokens include cached ones."""
+    u = getattr(result, "usage", None)
+    if u is None:
+        return {}
+    out = {"input": int(getattr(u, "prompt_tokens", 0) or 0), "output": int(getattr(u, "completion_tokens", 0) or 0)}
+    cached = int(getattr(u, "prompt_cached_tokens", 0) or 0)
+    written = int(getattr(u, "prompt_cache_creation_tokens", 0) or 0)
+    if cached:
+        out["cached"] = cached
+    if written:
+        out["cache_write"] = written
+    return out
+
+
+def _time_llm_calls(llm: Any, calls: List[Tuple[float, float]], on_call: Optional[Callable[[Dict[str, Any]], None]] = None) -> None:
+    """
+    Record when each model call started and ended, so the report can split step time into model and browser;
+    on_call also gets each call's time and tokens (counts only, never the prompt), for ForgeQE's usage ledger.
+    """
     inner = getattr(llm, "ainvoke", None)
     if inner is None:
         return
 
     async def timed(*args: Any, **kwargs: Any) -> Any:
         started = time.time()
+        result: Any = None
+        ok = False
         try:
-            return await inner(*args, **kwargs)
+            result = await inner(*args, **kwargs)
+            ok = True
+            return result
         finally:
-            calls.append((started, time.time()))
+            ended = time.time()
+            calls.append((started, ended))
+            if on_call is not None:
+                try:
+                    on_call({"ms": int((ended - started) * 1000), "ok": ok, **call_usage(result)})
+                except Exception:
+                    logger.debug("model call usage not reported", exc_info=True)
 
     try:
         object.__setattr__(llm, "ainvoke", timed)

package/py/forgeqe_engine/memory.py

@@ -5,6 +5,7 @@ Layout under the memory directory:
 
   pages/<host>/<page>.json   every interactive element seen on that page, each with ranked locators
   flows/<goal>-<hash>.json   the actions of the last successful run of a goal, replayable without the LLM
+  cases/<hash>.json          for a test case: which flow it last passed with, so an edited case replays its unchanged steps
 
 A locator's confidence (1-99 %) multiplies four factors: how stable its kind is (test id > id >
 role+name > ... > xpath), how unique it was on the page, how consistently it reappeared on later
@@ -650,22 +651,44 @@ class MemoryStore:
         replayed: int = 0,
         llm_steps: int = 0,
         resume: Optional[Dict[str, Any]] = None,
+        case: Optional[Dict[str, Any]] = None,
+        checks: Optional[List[Dict[str, Any]]] = None,
     ) -> Optional[str]:
         """
         Update the goal's flow after a run. Only successful runs replace its steps. Returns "saved" or "updated".
 
         A failed run with `resume` (the steps that worked before it went wrong) keeps them under "resume", so the
         next run of this task replays them and the agent picks up where the failed run went wrong.
+
+        `case` ({id, keys, ends}) is the test case the goal came from: a key per case step, and for each the number
+        of flow steps done by the end of it (None where unknown). A successful run records it and points the case's
+        index at this flow, so a run of the case after an edit replays the steps before the first one that changed.
         """
         if status == "stopped":
             return None
         with _locked(self.root):
             return self._save_flow_result(goal, start_url, status=status, steps=steps, summary=summary,
-                                          final_url=final_url, replayed=replayed, llm_steps=llm_steps, resume=resume)
+                                          final_url=final_url, replayed=replayed, llm_steps=llm_steps, resume=resume, case=case,
+                                          checks=checks)
+
+    def case_file(self, case_id: str) -> Path:
+        return self.root / "cases" / f"{_sha(case_id, 16)}.json"
+
+    def load_case_flow(self, case_id: str) -> Optional[Dict[str, Any]]:
+        """The flow a test case last passed with, when it recorded where each case step ended."""
+        index = _read_json(self.case_file(case_id)) if case_id else None
+        name = str((index or {}).get("flow") or "")
+        if not name or "/" in name or "\\" in name:
+            return None
+        flow = _read_json(self.root / "flows" / name)
+        if not flow or flow.get("case_id") != case_id or not flow.get("successes") or not isinstance(flow.get("steps"), list):
+            return None
+        return flow
 
     def _save_flow_result(self, goal: str, start_url: Optional[str], *, status: str, steps: List[Dict[str, Any]],
                           summary: str, final_url: str, replayed: int, llm_steps: int,
-                          resume: Optional[Dict[str, Any]] = None) -> Optional[str]:
+                          resume: Optional[Dict[str, Any]] = None, case: Optional[Dict[str, Any]] = None,
+                          checks: Optional[List[Dict[str, Any]]] = None) -> Optional[str]:
         file = self.flow_file(goal, start_url)
         flow = self.load_flow(goal, start_url)
         created = flow is None
@@ -689,6 +712,18 @@ class MemoryStore:
             flow["llm_steps"] = llm_steps
             flow["replayed_steps"] = replayed
             flow.pop("resume", None)
+            # checks: what the run checked, with the flow step each came after (observe.py); None keeps the flow's.
+            # New steps from the agent with no checks given leave none, since old positions would not fit them.
+            if checks is not None:
+                flow["checks"] = checks
+            elif llm_steps and flow.get("checks"):
+                flow.pop("checks", None)
+            if case and case.get("id") and case.get("keys"):
+                kept = len(flow["steps"])
+                flow["case_id"] = str(case["id"])
+                flow["case_step_keys"] = [str(k) for k in case["keys"]]
+                flow["case_step_ends"] = [e if isinstance(e, int) and 0 < e <= kept else None for e in (case.get("ends") or [])][: len(case["keys"])]
+                _write_json(self.case_file(str(case["id"])), {"case_id": str(case["id"]), "flow": file.name, "updated": now})
         elif status != "complete":
             flow["failures"] = int(flow.get("failures") or 0) + 1
         if keep_resume and resume:
@@ -702,6 +737,28 @@ class MemoryStore:
             return None
         return "saved" if created else "updated"
 
+    @staticmethod
+    def case_prefix(flow: Dict[str, Any], keys: List[str]) -> Tuple[int, int]:
+        """
+        How much of a case's earlier flow still fits its steps now: (case steps, flow steps). The case steps are the
+        leading ones whose keys did not change, up to the last of them where the flow recorded its end. (0, 0) when none.
+
+        An end is trusted only when the next step that recorded one ended later: when the agent did the actions of two
+        steps before recording either check, both ends are the same, and replaying up to the first would also redo
+        the second step's actions, which may be the step that changed.
+        """
+        old = [str(k) for k in (flow.get("case_step_keys") or [])]
+        ends = [e if isinstance(e, int) and 0 < e <= len(flow.get("steps") or []) else None for e in (flow.get("case_step_ends") or [])]
+        same = 0
+        while same < min(len(old), len(keys)) and old[same] == keys[same]:
+            same += 1
+        for n in range(same, 0, -1):
+            end = ends[n - 1] if n - 1 < len(ends) else None
+            later = next((e for e in ends[n:] if e is not None), None)
+            if end is not None and (later is None or later > end):
+                return n, end
+        return 0, 0
+
     def recent_flows(self, host: str, *, exclude: Optional[Path] = None, limit: int = 2) -> List[Dict[str, Any]]:
         folder = self.root / "flows"
         if not folder.is_dir():