@capagents/forgeqe 1.3.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HOW_TO_USE.md +1 -1
- package/docs/user-guide.html +1 -1
- package/evals/prompts/brain/route-api-scenarios.yaml +9 -0
- package/evals/prompts/brain/route-e2e.yaml +10 -0
- package/evals/prompts/brain/route-flaky.yaml +9 -0
- package/evals/prompts/brain/route-heal.yaml +9 -0
- package/evals/prompts/brain/route-regression-pack.yaml +9 -0
- package/evals/prompts/brain/route-review-update.yaml +10 -0
- package/evals/prompts/brain/route-run-cases.yaml +9 -0
- package/evals/prompts/brain/route-story-cases.yaml +12 -0
- package/package.json +1 -1
- package/prompts/README.md +30 -0
- package/prompts/brain.md +6 -52
- package/src/agents.ts +52 -0
- package/src/brain/index.ts +8 -2
- package/src/capabilities/registry.ts +29 -1
- package/src/cli/help.ts +1 -0
- package/src/cli/index.ts +7 -0
- package/src/context.ts +75 -0
- package/src/mcp/captools.ts +19 -0
- package/src/mcp/routing.ts +145 -0
- package/src/mcp/server.ts +7 -3
- package/src/mcp/tools.ts +3 -0
package/HOW_TO_USE.md
CHANGED
|
@@ -71,7 +71,7 @@ Everything ForgeQE can do, by area. ForgeQE's agent answers "can you…?" questi
|
|
|
71
71
|
| Release | **Reports you can share**: Writes up what ForgeQE's agent found, such as open defects across projects, release status, or pipeline health, as a page with filters (project, state, priority, assignee, …), charts, and a sortable table, plus Markdown and CSV. Every table is built from evidence: the results of the tool calls the agent made, or a read-only query ForgeQE runs itself, listed on the page's Sources with each raw result kept beside the report. | `write_report` | [docs](#reports-from-the-agent) |
|
|
72
72
|
| Operations | **CI pipelines**: Writes a GitHub Actions or Azure Pipelines workflow that runs ForgeQE, stores the secrets it needs, pushes a branch, and makes the pipeline green. | `push_branch`, `set_ci_secret` | [docs](#8-pipelines) |
|
|
73
73
|
| Systems | **Work with Azure DevOps, Jira, GitHub, and Confluence**: Reads anything in the configured systems (work items, issues, queries, boards, pages), makes changes the user asks for, puts run evidence (screenshots, reports) on work items and issues as attachments or comments, and builds Azure DevOps dashboards and wiki pages. | `query_api`, `read_confluence`, `create_ado_dashboard`, `write_ado_wiki_page`, `attach_to_work_item` | [docs](README.md#azure-devops-jira-and-other-systems) |
|
|
74
|
-
| Setup | **Setup, skills, and documentation**: Shows the model profiles and MCP servers, installs or repairs the browser engine, creates a forgeqe.yaml, manages skills, and answers questions about ForgeQE from its documentation and this list. | `forgeqe_setup`, `capabilities` | [docs](#2-scaffold-config) |
|
|
74
|
+
| Setup | **Setup, skills, and documentation**: Shows the model profiles and MCP servers, installs or repairs the browser engine, creates a forgeqe.yaml, manages skills, and answers questions about ForgeQE from its documentation and this list. | `forgeqe_setup`, `capabilities`, `playbook` | [docs](#2-scaffold-config) |
|
|
75
75
|
|
|
76
76
|
Not supported: Native mobile apps (iOS, Android) and desktop applications (instead: web applications in local Chrome or Chromium, or configured remote web grids such as BrowserStack and TestMu AI/LambdaTest); Load, stress, or soak testing with many simultaneous users (instead: per-page Core Web Vitals with budgets); Penetration testing or dynamic security scanning (instead: security findings in pull request reviews).
|
|
77
77
|
<!-- capabilities:end -->
|
package/docs/user-guide.html
CHANGED
|
@@ -2124,7 +2124,7 @@ mcp:
|
|
|
2124
2124
|
<div class="facts">
|
|
2125
2125
|
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>What can you do?</li><li>Which model am I using?</li><li>How do I set up email alerts?</li></ul></div>
|
|
2126
2126
|
<div class="fact"><span class="mono">What it needs</span><p class="muted">Nothing beyond ForgeQE itself.</p></div>
|
|
2127
|
-
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>forgeqe_setup</code> <code>capabilities</code></p></div>
|
|
2127
|
+
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>forgeqe_setup</code> <code>capabilities</code> <code>playbook</code></p></div>
|
|
2128
2128
|
</div>
|
|
2129
2129
|
<div class="detail"><span class="mono">In detail</span>
|
|
2130
2130
|
<p class="see">See <a href="#doc-how_to_use-2-scaffold-config">2. Scaffold config</a>.</p>
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
id: route-api-scenarios
|
|
2
|
+
title: End-to-end API scenarios come from api_scenarios, without a browser
|
|
3
|
+
input: Make end-to-end API scenarios for the pet store API described at https://petstore3.swagger.io/api/v3/openapi.json.
|
|
4
|
+
needs: []
|
|
5
|
+
checks:
|
|
6
|
+
- id: uses-api-scenarios
|
|
7
|
+
tools_any: ["forgeqe_api_scenarios"]
|
|
8
|
+
- id: no-browser-or-scripts
|
|
9
|
+
tools_none: ["forgeqe_browser_run", "bash"]
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
id: route-e2e
|
|
2
|
+
title: An end-to-end case is built from the story's existing cases, not written again
|
|
3
|
+
input: Build an end-to-end test case from the existing cases of Azure DevOps story 2211. Show it to me; don't save or run it.
|
|
4
|
+
needs: [azure_devops]
|
|
5
|
+
allow: [capabilities, find_test_cases, story_test_cases, e2e_journey, query_api, tool_help]
|
|
6
|
+
checks:
|
|
7
|
+
- id: uses-journey
|
|
8
|
+
tools_any: ["forgeqe_e2e_journey"]
|
|
9
|
+
- id: saves-nothing
|
|
10
|
+
tools_none: ["forgeqe_save_test_cases", "forgeqe_run_test_cases", "forgeqe_browser_run"]
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
id: route-heal
|
|
2
|
+
title: Tests that broke after a release go to heal_tests
|
|
3
|
+
input: Our automated UI tests in this repository started failing after yesterday's release. Repair them.
|
|
4
|
+
needs: []
|
|
5
|
+
checks:
|
|
6
|
+
- id: heals
|
|
7
|
+
tools_any: ["forgeqe_heal_tests"]
|
|
8
|
+
- id: no-hand-edits-first
|
|
9
|
+
tools_before: [["forgeqe_heal_tests"], ["edit", "write"]]
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
id: route-regression-pack
|
|
2
|
+
title: Which tests to run for a sprint's release is a regression pack
|
|
3
|
+
input: Which automated tests should we run before releasing the current sprint?
|
|
4
|
+
needs: [azure_devops]
|
|
5
|
+
checks:
|
|
6
|
+
- id: builds-pack
|
|
7
|
+
tools_any: ["forgeqe_build_regression_pack"]
|
|
8
|
+
- id: no-browser
|
|
9
|
+
tools_none: ["forgeqe_browser_run"]
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
id: route-review-update
|
|
2
|
+
title: A change after review updates the same case in place, never saves a new one
|
|
3
|
+
input: I reviewed test case 2279 of story 2271. Step 2 must also check that the password field shows dots, not the typed characters. Update the case.
|
|
4
|
+
needs: [azure_devops]
|
|
5
|
+
allow: [capabilities, find_test_cases, story_test_cases, query_api, tool_help]
|
|
6
|
+
checks:
|
|
7
|
+
- id: updates-in-place
|
|
8
|
+
tools_any: ["forgeqe_update_test_case"]
|
|
9
|
+
- id: no-new-case
|
|
10
|
+
tools_none: ["forgeqe_save_test_cases", "forgeqe_browser_run"]
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
id: route-run-cases
|
|
2
|
+
title: Running test cases goes through run_test_cases, not one browser run per case
|
|
3
|
+
input: Run Azure DevOps test cases 2272 and 2273.
|
|
4
|
+
needs: [azure_devops]
|
|
5
|
+
checks:
|
|
6
|
+
- id: runs-cases
|
|
7
|
+
tools_any: ["forgeqe_run_test_cases"]
|
|
8
|
+
- id: not-browser-per-case
|
|
9
|
+
tools_none: ["forgeqe_browser_run"]
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
id: route-story-cases
|
|
2
|
+
title: Asking for a story's test cases reads the linked cases before writing or saving any
|
|
3
|
+
input: Write test cases for Azure DevOps story 2211.
|
|
4
|
+
needs: [azure_devops]
|
|
5
|
+
allow: [capabilities, find_test_cases, story_test_cases, query_api, read_confluence, tool_help]
|
|
6
|
+
checks:
|
|
7
|
+
- id: reads-linked-cases
|
|
8
|
+
tools_any: ["forgeqe_story_test_cases"]
|
|
9
|
+
- id: reads-before-saving
|
|
10
|
+
tools_before: [["forgeqe_story_test_cases"], ["forgeqe_save_test_cases"]]
|
|
11
|
+
- id: no-browser
|
|
12
|
+
tools_none: ["forgeqe_browser_run", "bash", "webfetch"]
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@capagents/forgeqe",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.1",
|
|
4
4
|
"description": "ForgeQE: a browser agent CLI that runs goals and test cases in local Chrome or Chromium, or BrowserStack and TestMu AI/LambdaTest when configured, and turns passing runs into tests in your repo",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
package/prompts/README.md
CHANGED
|
@@ -60,3 +60,33 @@ How it works:
|
|
|
60
60
|
replaces ForgeQE's). Cases can run for brain, refine, and explore-charters.
|
|
61
61
|
- Changes to these prompts change how the agents behave: run `forgeqe prompts eval` before and after a change, and
|
|
62
62
|
commit it only when the pass rates hold.
|
|
63
|
+
|
|
64
|
+
## What each agent loads, and what it costs
|
|
65
|
+
|
|
66
|
+
Every model call sends an agent's fixed context again: its system prompt and, for ForgeQE's agent, its tool list. A
|
|
67
|
+
request takes several calls, so a line added here is paid for many times. ForgeQE keeps it small, and checks it.
|
|
68
|
+
|
|
69
|
+
- **One list of agents**: `src/agents.ts` names every agent, the prompt files it loads, the skills it may use, and a
|
|
70
|
+
budget for its fixed context. `forgeqe context` shows each agent against its budget (exit 1 when one is over).
|
|
71
|
+
- **ForgeQE's agent loads what the task needs** (`src/mcp/routing.ts`):
|
|
72
|
+
- A capability's `playbook` sections of brain.md (`src/capabilities/registry.ts`) leave the system prompt. The prompt
|
|
73
|
+
keeps an index of them; the first result of the capability's tools in a session carries the section, and the agent
|
|
74
|
+
can read one first with the `playbook` tool. A section no capability owns stays in the prompt, so a section you
|
|
75
|
+
rename in your copy of brain.md is never lost.
|
|
76
|
+
- Tools stay listed in full. Listing tools in one line was measured and cost more: the agent then needs an extra
|
|
77
|
+
model call to get a tool's arguments, and it plans worse without the full description.
|
|
78
|
+
- `FORGEQE_ROUTING=off` sends everything, as before: use it to compare, or to rule routing out when something fails.
|
|
79
|
+
- **Skills and product knowledge** load on demand already: an agent sees each skill's name and description and reads
|
|
80
|
+
the skill when it needs it; knowledge sections are picked by relevance to the task.
|
|
81
|
+
|
|
82
|
+
When you add or change something:
|
|
83
|
+
|
|
84
|
+
| You add | Also do |
|
|
85
|
+
|---------|---------|
|
|
86
|
+
| a tool | put it in one capability's `tools`, with a description that says when to use it |
|
|
87
|
+
| a section in brain.md | give it to a capability's `playbook` with a `when` line, unless every request needs it |
|
|
88
|
+
| a prompt file | name it under its agent in `src/agents.ts` (or include it from a prompt that is) |
|
|
89
|
+
| an agent | add it to `src/agents.ts` with a budget |
|
|
90
|
+
|
|
91
|
+
Then run `forgeqe context` and `forgeqe prompts eval brain` before and after: `bun test` fails when something is not
|
|
92
|
+
in the lists or an agent is over budget, and the evaluation cases show whether the agent still picks the right tools.
|
package/prompts/brain.md
CHANGED
|
@@ -24,58 +24,12 @@ A message can also be a task specification with the parts Role, Objective, Conte
|
|
|
24
24
|
**Preconditions**: "This session" below gives the setup: the systems, the project, the logins, the domain, the capabilities, and the defaults. You MUST use it, and MUST NOT ask the user for what it gives.
|
|
25
25
|
|
|
26
26
|
**Tools** (ForgeQE tools; the client adds the forgeqe_ prefix):
|
|
27
|
-
-
|
|
28
|
-
-
|
|
29
|
-
-
|
|
30
|
-
-
|
|
31
|
-
-
|
|
32
|
-
-
|
|
33
|
-
- e2e_journey: plans end-to-end journeys from test cases that exist (the cases of a story, or any sources). It keeps the steps they share once, and gives each case as a section. It changes nothing.
|
|
34
|
-
- release_readiness: gives requirement coverage and a GO / GO WITH RISKS / NO-GO verdict for a sprint, area, epic, or stories. It uses test cases, their results, bugs, and flaky tests. It can publish to the wiki. Examples: "are we ready to release?", "coverage of sprint 12".
|
|
35
|
-
- run_test_cases: runs test cases in real browsers, with a verdict for each step. It writes report.html (steps, screenshots, timings) and junit.xml. It sends results back to the source system. It files or updates bugs for failures when bugs are set up. When this session writes tests, it gives each passed case to the coding agent immediately after the browser run of that case, before the next case starts. It gives the test of each case.
|
|
36
|
-
- read_confluence: reads Confluence pages (requirements, specifications) as text, by id, link, or CQL.
|
|
37
|
-
- browser_devices: lists, saves, and removes named remote browsers (devices) on BrowserStack or TestMu AI, kept in the provider's YAML file. When the user describes devices or browsers to test on, save each one, then pass their names as devices to browser_run or run_test_cases: every device runs at the same time and is reported on its own.
|
|
38
|
-
- attach_to_work_item: puts files from a run (the last or failure screenshot, report.html, junit.xml, trace.md) on an existing Azure DevOps work item or Jira issue, as attachments, in a comment (images shown inline), or both. The Azure DevOps MCP server cannot upload files, so use this tool for any file on a work item.
|
|
39
|
-
- run_details: gives the full record of a browser run: steps, values, ranked locators, and the generated spec.
|
|
40
|
-
- run_test: ForgeQE runs a test command. A test passes only when this tool, write_test, or heal_tests says so.
|
|
41
|
-
- write_test: the coding agent of ForgeQE turns a passed run into a test in a repository. ForgeQE runs the test until it passes.
|
|
42
|
-
- heal_tests: ForgeQE runs the tests of a repository and heals the tests that fail against the live application, or reports application bugs.
|
|
43
|
-
- find_tests: Scribe, the coding agent, finds the automated tests in a repository that match a request. A request can be a feature, tags or ids, a folder, or tests added in a date range or sprint. ForgeQE confirms the selection with the list command of the framework. It runs nothing.
|
|
44
|
-
- run_tests: runs a find_tests selection after the user confirms, with a pass or fail for each test. A test that fails runs one more time. A test that then passes is flaky, not failing. The results of each run go into the test history of the repository.
|
|
45
|
-
- flaky_tests: gives the flaky tests from that history, with their probable cause.
|
|
46
|
-
- defect_trends: gives how the defects of the saved suites went over time: still there, gone, or back after a fix, the time to fix, the most seen, and defects by area. It runs nothing.
|
|
47
|
-
- predict_defects: gives the areas where a change in the product repository can cause defects, with the reasons and the cases to run first. It is a heuristic from the history. It runs nothing.
|
|
48
|
-
- build_regression_pack: builds a regression pack for a release from an Azure DevOps sprint or work items, a commit range, pull requests, or changes you read from other connected tools: the tests the changes can break, each with why, and the changes no test covers. It runs nothing.
|
|
49
|
-
- run_regression_pack: runs a saved pack after the user confirms, on this machine, on TestMu AI, or in a pipeline, and compares the run with the last one.
|
|
50
|
-
- list_regression_packs: gives the saved packs and their last runs.
|
|
51
|
-
- quarantine_tests: puts flaky tests in quarantine, or takes them out, after the user agrees. The failures of a test in quarantine do not fail runs or CI.
|
|
52
|
-
- review_pull_request: Scribe reviews an Azure DevOps or GitHub pull request from the local clone (bugs, security, requirements, tests). The user sees the findings. It posts nothing.
|
|
53
|
-
- post_pr_review: posts a review_pull_request review to the pull request after the user confirms.
|
|
54
|
-
- select_pr_tests: selects the automated tests that a pull request, or the commits since a base branch, can break. It uses what the change does and the linked work items. Run the tests with run_tests.
|
|
55
|
-
- post_pr_test_results: posts a run_tests result on the pull request after the user confirms.
|
|
56
|
-
- analyze_pr_impact: finds the product areas, pages, requirements, and tests that a pull request touches. This works also when the code of the product is in a different repository than the tests.
|
|
57
|
-
- comment_on_pr: posts a comment, such as an impact report, on a pull request after the user confirms.
|
|
58
|
-
- api_contracts: makes API contracts (status, schema, and key fields for each operation). It uses the calls that a browser run saw the application make, or an API description. Descriptions are an OpenAPI or Swagger link or file, WSDL, a GraphQL endpoint or schema, .proto, or grpc:// with reflection. It supports REST, GraphQL, SOAP, and gRPC.
|
|
59
|
-
- run_api_contracts: runs a contract set directly against the API, without a browser. This is fast. It runs calls that change data only after the user agrees.
|
|
60
|
-
- write_api_tests: Scribe writes a contract set as API tests in the framework of the repository. ForgeQE runs them until they pass. With scenarios, it writes end-to-end scenario tests on the API helpers of the repository.
|
|
61
|
-
- api_scenarios: makes end-to-end API scenarios. These are calls in sequence, where values go from an answer to a later request. It makes them from a browser run, from an API description (lifecycles, browsing, links, negative cases), or from test cases or plain English.
|
|
62
|
-
- run_api_scenarios: runs the scenarios against the API, without a browser. It asks before it runs a scenario that changes data.
|
|
63
|
-
- explore_story: does exploratory testing of a story in real browsers, past its scripted cases. It examines edge cases, input that is not valid, boundaries, roles, and state. It confirms each defect with two runs. It proposes test cases. It files bugs for confirmed defects after the user agrees.
|
|
64
|
-
- quality_report: gives the WCAG rules that a browser run passed and failed (axe-core, with the standard in forgeqe.yaml). It also gives the Core Web Vitals of each page, against the budgets.
|
|
65
|
-
- quality_trends: gives the accessibility and performance of each page over time, with what became worse or better.
|
|
66
|
-
- list_sprints: gives recent sprints from Azure DevOps and Jira, with their dates.
|
|
67
|
-
- monitor: runs checks on a schedule, with Teams and email alerts. It can run one round now, start or stop rounds in the background, give status, and send test alerts.
|
|
68
|
-
- logins: manages saved logins. It lists them. It lets the user sign in by hand in a browser (sign_in). It removes a session.
|
|
69
|
-
- page_memory: gives what ForgeQE knows about the pages, locators, and recorded flows of a site. Clear it only when the user asks.
|
|
70
|
-
- notify: posts a message to the Teams or email channels of the team. Use it only when the user asks. Suites, release verdicts, and failed evaluations go to their channels automatically from runs outside the shell.
|
|
71
|
-
- write_report: saves a report that the user asks for as a page (filters, charts, the sources of its data) and a Markdown file. ForgeQE builds its tables from evidence: the results of tool calls in this session, or a read-only query that it runs itself. You write only the summary.
|
|
72
|
-
- capabilities: tells what ForgeQE can do, and whether each capability works in this setup. It gives what to set up, example requests, limits, and the documentation.
|
|
73
|
-
- forgeqe_setup: gives model profiles, the MCP servers that ForgeQE sets up, and the documentation of ForgeQE (docs). It installs or repairs the browser engine. It creates a forgeqe.yaml (init).
|
|
74
|
-
- push_branch: the only way to push. It refuses the default branch, and it never forces.
|
|
75
|
-
- set_ci_secret: stores one environment variable of ForgeQE as a pipeline secret. You give the name, never the value.
|
|
76
|
-
- query_api: reads data from Azure DevOps, Jira, Xray, or Confluence through their APIs, with the credentials of ForgeQE. It only reads.
|
|
77
|
-
- create_ado_dashboard: creates an Azure DevOps dashboard with work item lists, count tiles, and text. A saved shared query supplies each list or count.
|
|
78
|
-
- write_ado_wiki_page: creates or updates an Azure DevOps wiki page. It creates the project wiki first when the project has none.
|
|
27
|
+
- Each ForgeQE tool comes with its own description. Read it to select the tool.
|
|
28
|
+
- Some playbooks are not in this prompt. The index under "Playbooks" names them. The first result of their tools gives each one, and you MUST obey it.
|
|
29
|
+
- browser_run: set purpose to test for a flow of the application under test. Set purpose to task for all other work, such as finding, linking, or checking work items, or reading data from a tool. A task never becomes an automated test.
|
|
30
|
+
- attach_to_work_item: the only way to put a file from a run on a work item or an issue. The Azure DevOps MCP server cannot upload files.
|
|
31
|
+
- A test passes only when run_test, write_test, or heal_tests says so.
|
|
32
|
+
- query_api only reads. set_ci_secret takes the name of a variable, never its value.
|
|
79
33
|
|
|
80
34
|
Each action of the forgeqe command line is one of these tools. You MUST do the action with the tool, not tell the user to run a command.
|
|
81
35
|
You also have your own tools to read and edit files and to run shell commands in the working directory. You also have the tools of the MCP servers under "This session", for example azure_devops_* for Azure DevOps and atlassian_* for Jira and Confluence.
|
package/src/agents.ts
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { SkillAgent } from "./skills/registry.ts"
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Every agent ForgeQE runs, what it loads on every model call, and how much that may be.
|
|
5
|
+
*
|
|
6
|
+
* This is the one list to change when an agent, a prompt, or a skill audience is added: tests/agents.test.ts checks it
|
|
7
|
+
* against the code (every prompt file is loaded by an agent, every agent the code defines is here) and checks each
|
|
8
|
+
* agent's fixed context against its budget, so the cost of a change shows in the build and in `forgeqe context`.
|
|
9
|
+
*/
|
|
10
|
+
export type AgentEntry = {
|
|
11
|
+
/** The agent runtime's name for it (AgentSpec name), or "engine" for the browser agent. */
|
|
12
|
+
name: string
|
|
13
|
+
title: string
|
|
14
|
+
/** agent-runtime: a session with tools; engine: the Python browser agent. */
|
|
15
|
+
kind: "agent-runtime" | "engine"
|
|
16
|
+
/** The prompt files it loads (prompts/<name>.md); the fragments they include count too. */
|
|
17
|
+
prompts: string[]
|
|
18
|
+
/** Skills it may load (by their `agents` audience); loaded on demand, so they are not in the fixed context. */
|
|
19
|
+
skills?: SkillAgent
|
|
20
|
+
/** Where the code defines it. */
|
|
21
|
+
defined: string
|
|
22
|
+
/** Most tokens its fixed context (system prompt, and for ForgeQE's agent its tool list) may take. */
|
|
23
|
+
budget: number
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export const AGENTS: AgentEntry[] = [
|
|
27
|
+
{ name: "forgeqe", title: "ForgeQE's agent (shell, forgeqe run, forgeqe mcp)", kind: "agent-runtime", prompts: ["brain"], skills: "forgeqe", defined: "src/brain/index.ts", budget: 26000 },
|
|
28
|
+
{ name: "forgeqe-test", title: "Scribe: writes a test from a verified run", kind: "agent-runtime", prompts: ["test-agent"], skills: "scribe", defined: "src/code/runtime.ts", budget: 4900 },
|
|
29
|
+
{ name: "forgeqe-read", title: "Scribe, reading the repository before the run arrives", kind: "agent-runtime", prompts: ["test-agent"], skills: "scribe", defined: "src/code/runtime.ts", budget: 4900 },
|
|
30
|
+
{ name: "forgeqe-review", title: "Reviewer of Scribe's tests", kind: "agent-runtime", prompts: ["test-reviewer"], skills: "scribe", defined: "src/code/review.ts", budget: 4900 },
|
|
31
|
+
{ name: "forgeqe-heal", title: "Scribe healing tests the application outgrew", kind: "agent-runtime", prompts: ["heal"], skills: "scribe", defined: "src/heal/loop.ts", budget: 4200 },
|
|
32
|
+
{ name: "forgeqe-find", title: "Finds the repository tests a request means", kind: "agent-runtime", prompts: ["find-tests"], skills: "scribe", defined: "src/code/find.ts", budget: 1700 },
|
|
33
|
+
{ name: "forgeqe-pr-review", title: "Reviews pull requests", kind: "agent-runtime", prompts: ["pr-review"], skills: "scribe", defined: "src/code/prreview.ts", budget: 1700 },
|
|
34
|
+
{ name: "forgeqe-api", title: "Writes API and contract tests", kind: "agent-runtime", prompts: ["api-tests"], skills: "scribe", defined: "src/api/write.ts", budget: 2450 },
|
|
35
|
+
{ name: "forgeqe-api-plan", title: "Plans end-to-end API scenarios", kind: "agent-runtime", prompts: ["api-scenarios"], defined: "src/api/plan.ts", budget: 1600 },
|
|
36
|
+
{ name: "forgeqe-explore-plan", title: "Plans exploratory charters", kind: "agent-runtime", prompts: ["explore-charters"], defined: "src/explore/charters.ts", budget: 1650 },
|
|
37
|
+
{ name: "forgeqe-defect-rca", title: "Finds the root cause of a defect", kind: "agent-runtime", prompts: ["defect-rca"], defined: "src/cases/rca.ts", budget: 950 },
|
|
38
|
+
{ name: "forgeqe-refine", title: "Refine: turns a request into a task specification", kind: "agent-runtime", prompts: ["refine"], defined: "src/refine.ts", budget: 1450 },
|
|
39
|
+
{ name: "forgeqe-judge", title: "Judges domain evaluations", kind: "agent-runtime", prompts: ["eval-judge"], defined: "src/evals/domain.ts", budget: 750 },
|
|
40
|
+
{
|
|
41
|
+
name: "engine",
|
|
42
|
+
title: "The browser agent (every browser run and test case)",
|
|
43
|
+
kind: "engine",
|
|
44
|
+
prompts: ["browser-api", "browser-capture", "browser-case-report", "browser-checks", "browser-efficiency", "browser-error-recovery", "browser-extra", "browser-goal-verification", "browser-skill-select", "conventions"],
|
|
45
|
+
skills: "browser",
|
|
46
|
+
defined: "py/forgeqe_engine/service.py",
|
|
47
|
+
budget: 2700,
|
|
48
|
+
},
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
/** Prompt files that are documentation, not loaded by any agent. */
|
|
52
|
+
export const NOT_LOADED = ["README"]
|
package/src/brain/index.ts
CHANGED
|
@@ -8,6 +8,7 @@ import type { AppConfig } from "../config.ts"
|
|
|
8
8
|
import { externalMcp, externalNotes, type ExternalPlan } from "../mcp/external.ts"
|
|
9
9
|
import { MCP_NAME, TOOL_TIMEOUT_MS, localToolServer } from "../mcp/server.ts"
|
|
10
10
|
import { TOOLS, environmentNotes, type ToolContext } from "../mcp/tools.ts"
|
|
11
|
+
import { createRouter } from "../mcp/routing.ts"
|
|
11
12
|
import { brainPrompt, BRAIN_SCHEMA, cleanFollowUps, type BrainReport, type FollowUp } from "./prompt.ts"
|
|
12
13
|
import { redactSecrets } from "../sessionLogins.ts"
|
|
13
14
|
import { logEvent } from "../sessionLog.ts"
|
|
@@ -115,11 +116,16 @@ export function carryOver(turns: { user: string; reply: string }[], text: string
|
|
|
115
116
|
].join("\n")
|
|
116
117
|
}
|
|
117
118
|
|
|
119
|
+
/** Routing (src/mcp/routing.ts) is on unless FORGEQE_ROUTING=off, kept for comparing and for finding a routing fault. */
|
|
120
|
+
export const routingOn = (env: Record<string, string | undefined> = process.env) => !/^(off|0|false|no)$/i.test(env.FORGEQE_ROUTING || "")
|
|
121
|
+
|
|
118
122
|
export async function openBrain(opts: BrainOptions): Promise<Brain> {
|
|
119
123
|
const { config } = opts
|
|
120
124
|
const cwd = realpathOr(opts.cwd)
|
|
121
125
|
const context = (): ToolContext => ({ ...opts.context(), config, cwd })
|
|
122
|
-
|
|
126
|
+
// Tools listed in one line and playbooks on demand keep every model call small; FORGEQE_ROUTING=off sends it all.
|
|
127
|
+
const router = routingOn() ? createRouter(TOOLS) : undefined
|
|
128
|
+
const tools = localToolServer({ tools: TOOLS, context, router, instructions: () => environmentNotes(context()), timeoutMs: TOOL_TIMEOUT_MS })
|
|
123
129
|
const external = opts.external || externalMcp(config, Bun.which, process.env, { cwd })
|
|
124
130
|
const notes = [environmentNotes(context()), ...externalNotes(external), ...(opts.notes || [])].filter(Boolean).join("\n")
|
|
125
131
|
const readDirs = [...new Set([config.export.outDir, config.memory.dir, cwd].flatMap((d) => [path.resolve(d), realpathOr(d)]))]
|
|
@@ -147,7 +153,7 @@ export async function openBrain(opts: BrainOptions): Promise<Brain> {
|
|
|
147
153
|
agent: {
|
|
148
154
|
name: BRAIN_AGENT,
|
|
149
155
|
description: "ForgeQE: browser runs, test cases, tests, and CI pipelines",
|
|
150
|
-
prompt: `${brainPrompt()}\n\nThis session:\n${notes}`,
|
|
156
|
+
prompt: `${router ? router.parts.core : brainPrompt()}\n\nThis session:\n${notes}`,
|
|
151
157
|
skills: "forgeqe",
|
|
152
158
|
// Systems are read through MCP or query_api, never by scripts in the shell that handle tokens themselves.
|
|
153
159
|
permission: { bash: agentBashPermission(privateEnvNames(config)) },
|
|
@@ -53,6 +53,13 @@ export type Capability = {
|
|
|
53
53
|
/** What it does not do, so the agent never promises it. */
|
|
54
54
|
limits?: string[]
|
|
55
55
|
docs: DocRef[]
|
|
56
|
+
/**
|
|
57
|
+
* The sections of brain.md (### headings) this capability owns: they leave the system prompt and come on demand, with
|
|
58
|
+
* `when` in the index of playbooks that stays in it. A section no capability owns stays in the system prompt.
|
|
59
|
+
*/
|
|
60
|
+
playbook?: { heading: string; when: string }[]
|
|
61
|
+
/** Tools of other capabilities whose first result also carries this capability's playbook. */
|
|
62
|
+
alsoAfter?: string[]
|
|
56
63
|
}
|
|
57
64
|
|
|
58
65
|
export const CAPABILITIES: Capability[] = [
|
|
@@ -127,6 +134,10 @@ export const CAPABILITIES: Capability[] = [
|
|
|
127
134
|
examples: ["Write test cases for story 1234 and save them", "Build an end-to-end journey from story 1234's cases"],
|
|
128
135
|
requires: [{ kind: "system", anyOf: ["azure_devops", "jira"] }],
|
|
129
136
|
docs: [{ doc: "HOW_TO_USE.md", heading: "Where cases come from" }],
|
|
137
|
+
playbook: [
|
|
138
|
+
{ heading: "Testing from requirements", when: "write test cases from a story, a Confluence page, or a specification" },
|
|
139
|
+
{ heading: "End-to-end journeys from test cases", when: "an end-to-end case or flow from test cases that exist" },
|
|
140
|
+
],
|
|
130
141
|
},
|
|
131
142
|
{
|
|
132
143
|
id: "domain-compliance",
|
|
@@ -150,6 +161,7 @@ export const CAPABILITIES: Capability[] = [
|
|
|
150
161
|
examples: ["Explore story 1234 and find bugs", "Test this feature beyond the test cases"],
|
|
151
162
|
requires: [{ kind: "live_model" }, { kind: "system", anyOf: ["azure_devops", "jira"], optional: true, why: "to read the story and file bugs" }],
|
|
152
163
|
docs: [{ doc: "HOW_TO_USE.md", heading: "Exploratory testing from a story" }],
|
|
164
|
+
playbook: [{ heading: "Exploratory testing", when: "explore a story or feature past its scripted cases, or find bugs in it" }],
|
|
153
165
|
},
|
|
154
166
|
{
|
|
155
167
|
id: "write-tests",
|
|
@@ -161,6 +173,8 @@ export const CAPABILITIES: Capability[] = [
|
|
|
161
173
|
examples: ["Write a Playwright test for that run in ../app"],
|
|
162
174
|
requires: [{ kind: "coding_agent" }, { kind: "input", what: "a repository to write the test in (default: the working directory)" }],
|
|
163
175
|
docs: [{ doc: "HOW_TO_USE.md", heading: "6. Tests" }],
|
|
176
|
+
playbook: [{ heading: "Writing tests", when: "write an automated test from a run, or a session where passed runs become tests" }],
|
|
177
|
+
alsoAfter: ["browser_run", "run_test_cases"],
|
|
164
178
|
},
|
|
165
179
|
{
|
|
166
180
|
id: "heal-tests",
|
|
@@ -171,6 +185,7 @@ export const CAPABILITIES: Capability[] = [
|
|
|
171
185
|
examples: ["Heal the failing tests in ../app"],
|
|
172
186
|
requires: [{ kind: "coding_agent" }, { kind: "input", what: "a repository with tests" }],
|
|
173
187
|
docs: [{ doc: "HOW_TO_USE.md", heading: "Fixing broken tests" }],
|
|
188
|
+
playbook: [{ heading: "Repairing broken tests", when: "automated tests that fail, broke after a release, or need healing" }],
|
|
174
189
|
},
|
|
175
190
|
{
|
|
176
191
|
id: "find-run-tests",
|
|
@@ -185,6 +200,7 @@ export const CAPABILITIES: Capability[] = [
|
|
|
185
200
|
{ kind: "system", anyOf: ["azure_devops", "jira"], optional: true, why: "to pick tests by sprint" },
|
|
186
201
|
],
|
|
187
202
|
docs: [{ doc: "HOW_TO_USE.md", heading: "Finding and running existing tests" }],
|
|
203
|
+
playbook: [{ heading: "Repository tests", when: "find, list, or run the automated tests of the repository" }],
|
|
188
204
|
},
|
|
189
205
|
{
|
|
190
206
|
id: "regression-packs",
|
|
@@ -198,6 +214,7 @@ export const CAPABILITIES: Capability[] = [
|
|
|
198
214
|
{ kind: "system", anyOf: ["azure_devops", "jira"], optional: true, why: "to read a sprint's or release's work items" },
|
|
199
215
|
],
|
|
200
216
|
docs: [{ doc: "HOW_TO_USE.md", heading: "Regression packs for a release" }],
|
|
217
|
+
playbook: [{ heading: "Regression packs", when: "a regression pack, or which tests to run for a release, sprint, or pull requests" }],
|
|
201
218
|
},
|
|
202
219
|
{
|
|
203
220
|
id: "flaky-tests",
|
|
@@ -222,6 +239,8 @@ export const CAPABILITIES: Capability[] = [
|
|
|
222
239
|
{ kind: "either", of: [{ kind: "system", anyOf: ["azure_devops"], why: "for Azure DevOps pull requests" }, { kind: "command", command: "gh", why: "for GitHub pull requests" }] },
|
|
223
240
|
],
|
|
224
241
|
docs: [{ doc: "HOW_TO_USE.md", heading: "Reviewing pull requests" }],
|
|
242
|
+
playbook: [{ heading: "Pull requests", when: "review a pull request, its impact, or the tests it can break" }],
|
|
243
|
+
alsoAfter: ["analyze_pr_impact", "select_pr_tests", "post_pr_test_results", "comment_on_pr"],
|
|
225
244
|
},
|
|
226
245
|
{
|
|
227
246
|
id: "pr-impact",
|
|
@@ -254,6 +273,7 @@ export const CAPABILITIES: Capability[] = [
|
|
|
254
273
|
{ doc: "HOW_TO_USE.md", heading: "API and contract tests" },
|
|
255
274
|
{ doc: "HOW_TO_USE.md", heading: "End-to-end API scenarios" },
|
|
256
275
|
],
|
|
276
|
+
playbook: [{ heading: "API testing", when: "API contracts, API tests, or end-to-end API scenarios (REST, GraphQL, SOAP, gRPC)" }],
|
|
257
277
|
},
|
|
258
278
|
{
|
|
259
279
|
id: "accessibility-performance",
|
|
@@ -265,6 +285,8 @@ export const CAPABILITIES: Capability[] = [
|
|
|
265
285
|
requires: [{ kind: "input", what: "a browser run (from browser_run or run_test_cases); new runs need a live model" }],
|
|
266
286
|
limits: ["Per-page Core Web Vitals, not load testing under many users."],
|
|
267
287
|
docs: [{ doc: "HOW_TO_USE.md", heading: "Accessibility and performance budgets" }],
|
|
288
|
+
playbook: [{ heading: "Accessibility, performance, and visual changes", when: "accessibility, Core Web Vitals, performance budgets, or visual changes of pages" }],
|
|
289
|
+
alsoAfter: ["visual_compare", "visual_approve"],
|
|
268
290
|
},
|
|
269
291
|
{
|
|
270
292
|
id: "visual-comparison",
|
|
@@ -313,6 +335,7 @@ export const CAPABILITIES: Capability[] = [
|
|
|
313
335
|
examples: ["Check the login flow every 15 minutes and alert Teams"],
|
|
314
336
|
requires: [{ kind: "live_model" }, { kind: "monitor_alerts", optional: true }],
|
|
315
337
|
docs: [{ doc: "HOW_TO_USE.md", heading: "9. Monitoring" }],
|
|
338
|
+
playbook: [{ heading: "Monitoring", when: "checks on a schedule, with alerts" }],
|
|
316
339
|
},
|
|
317
340
|
{
|
|
318
341
|
id: "notifications",
|
|
@@ -354,6 +377,7 @@ export const CAPABILITIES: Capability[] = [
|
|
|
354
377
|
{ kind: "either", of: [{ kind: "command", command: "gh", why: "for GitHub Actions" }, { kind: "command", command: "az", why: "for Azure Pipelines" }] },
|
|
355
378
|
],
|
|
356
379
|
docs: [{ doc: "HOW_TO_USE.md", heading: "8. Pipelines" }],
|
|
380
|
+
playbook: [{ heading: "CI pipelines", when: "a GitHub Actions or Azure DevOps pipeline that runs ForgeQE or the tests" }],
|
|
357
381
|
},
|
|
358
382
|
{
|
|
359
383
|
id: "systems",
|
|
@@ -370,13 +394,17 @@ export const CAPABILITIES: Capability[] = [
|
|
|
370
394
|
],
|
|
371
395
|
requires: [{ kind: "system", anyOf: ["azure_devops", "jira", "xray", "confluence"] }],
|
|
372
396
|
docs: [{ doc: "README.md", heading: "Azure DevOps, Jira, and other systems" }],
|
|
397
|
+
playbook: [
|
|
398
|
+
{ heading: "Releases and pipelines in Azure DevOps", when: "releases, deployments, environments, or pipeline runs in Azure DevOps" },
|
|
399
|
+
{ heading: "Dashboards and wiki pages in Azure DevOps", when: "an Azure DevOps dashboard or wiki page" },
|
|
400
|
+
],
|
|
373
401
|
},
|
|
374
402
|
{
|
|
375
403
|
id: "setup-help",
|
|
376
404
|
area: "Setup",
|
|
377
405
|
title: "Setup, skills, and documentation",
|
|
378
406
|
summary: "Shows the model profiles and MCP servers, installs or repairs the browser engine, creates a forgeqe.yaml, manages skills, and answers questions about ForgeQE from its documentation and this list.",
|
|
379
|
-
tools: ["forgeqe_setup", "capabilities"],
|
|
407
|
+
tools: ["forgeqe_setup", "capabilities", "playbook"],
|
|
380
408
|
examples: ["What can you do?", "Which model am I using?", "How do I set up email alerts?"],
|
|
381
409
|
requires: [],
|
|
382
410
|
docs: [
|
package/src/cli/help.ts
CHANGED
|
@@ -12,6 +12,7 @@ Usage:
|
|
|
12
12
|
forgeqe setup [--force]
|
|
13
13
|
forgeqe models [--config forgeqe.yaml]
|
|
14
14
|
forgeqe features [--config forgeqe.yaml] what ForgeQE can do, and what each capability still needs here
|
|
15
|
+
forgeqe context what each agent loads on every model call, against its budget (exits 1 over budget)
|
|
15
16
|
forgeqe docs [--guide|--open] [--copy DIR] the guides and the architecture page (--guide: the user guide in the browser; --open: the architecture page)
|
|
16
17
|
forgeqe start [--profile name] [--url URL] [--goal TEXT] [--headed] [--replay MODE] [--no-memory]
|
|
17
18
|
[--repo DIR] [--no-code] [--browser-provider local|browserstack|testmu] [--devices a,b]
|
package/src/cli/index.ts
CHANGED
|
@@ -171,6 +171,13 @@ export async function main(argv: string[]): Promise<number> {
|
|
|
171
171
|
return cmdModels(flags)
|
|
172
172
|
case "docs":
|
|
173
173
|
return cmdDocs(flags)
|
|
174
|
+
case "context": {
|
|
175
|
+
// What each agent carries on every model call, against its budget in src/agents.ts.
|
|
176
|
+
const { agentContexts, contextReport } = await import("../context.ts")
|
|
177
|
+
const rows = agentContexts()
|
|
178
|
+
console.log(contextReport(rows).join("\n"))
|
|
179
|
+
return rows.some((r) => r.over) ? 1 : 0
|
|
180
|
+
}
|
|
174
181
|
case "features":
|
|
175
182
|
// What ForgeQE can do, and whether each capability works with this config and machine.
|
|
176
183
|
console.log(featuresText(loadConfig(configOpts(flags))))
|
package/src/context.ts
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { AGENTS, type AgentEntry } from "./agents.ts"
|
|
2
|
+
import { createRouter, tokens } from "./mcp/routing.ts"
|
|
3
|
+
import { TOOLS } from "./mcp/tools.ts"
|
|
4
|
+
import { prompt } from "./prompts.ts"
|
|
5
|
+
|
|
6
|
+
/** One agent's fixed context: what it carries on every model call. */
|
|
7
|
+
export type AgentContext = {
|
|
8
|
+
agent: AgentEntry
|
|
9
|
+
/** The system prompt as sent (ForgeQE's agent: the core, without its on-demand playbooks). */
|
|
10
|
+
prompt: number
|
|
11
|
+
/** ForgeQE's agent only: its tool list as sent. */
|
|
12
|
+
tools?: number
|
|
13
|
+
total: number
|
|
14
|
+
over: boolean
|
|
15
|
+
/** ForgeQE's agent only: what it gets on demand, and what the fixed context would be with everything always on. */
|
|
16
|
+
onDemand?: { playbooks: number; playbookTokens: number; everything: number }
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const expanded = (name: string) => {
|
|
20
|
+
try {
|
|
21
|
+
return prompt(name)
|
|
22
|
+
} catch {
|
|
23
|
+
return ""
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** The fixed context of each agent, from the prompts and tools as they are now (a team's own prompt copies included). */
|
|
28
|
+
export function agentContexts(): AgentContext[] {
|
|
29
|
+
return AGENTS.map((agent) => {
|
|
30
|
+
if (agent.name !== "forgeqe") {
|
|
31
|
+
const p = agent.prompts.reduce((sum, n) => sum + tokens(expanded(n)), 0)
|
|
32
|
+
return { agent, prompt: p, total: p, over: p > agent.budget }
|
|
33
|
+
}
|
|
34
|
+
const router = createRouter(TOOLS)
|
|
35
|
+
const p = tokens(router.parts.core)
|
|
36
|
+
const t = tokens(JSON.stringify(TOOLS.map((x) => ({ name: x.name, description: x.description, inputSchema: x.inputSchema }))))
|
|
37
|
+
const full = tokens(expanded("brain")) + t
|
|
38
|
+
return {
|
|
39
|
+
agent,
|
|
40
|
+
prompt: p,
|
|
41
|
+
tools: t,
|
|
42
|
+
total: p + t,
|
|
43
|
+
over: p + t > agent.budget,
|
|
44
|
+
onDemand: {
|
|
45
|
+
playbooks: router.parts.playbooks.length,
|
|
46
|
+
playbookTokens: router.parts.playbooks.reduce((s, b) => s + tokens(b.text), 0),
|
|
47
|
+
everything: full,
|
|
48
|
+
},
|
|
49
|
+
}
|
|
50
|
+
})
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const k = (n: number) => (n >= 1000 ? `${(n / 1000).toFixed(1)}K` : String(n))
|
|
54
|
+
|
|
55
|
+
/** `forgeqe context`: each agent's fixed context against its budget, and what ForgeQE's agent loads on demand. */
|
|
56
|
+
export function contextReport(rows = agentContexts()): string[] {
|
|
57
|
+
const out = ["Fixed context of each agent: sent again on every model call (~tokens).", ""]
|
|
58
|
+
const w = Math.max(...rows.map((r) => r.agent.name.length))
|
|
59
|
+
for (const r of rows) {
|
|
60
|
+
const parts = r.tools !== undefined ? `prompt ${k(r.prompt)} + tools ${k(r.tools)}` : `prompt ${k(r.prompt)}`
|
|
61
|
+
out.push(`${r.over ? "✗" : "✓"} ${r.agent.name.padEnd(w)} ${k(r.total).padStart(6)} of ${k(r.agent.budget).padStart(6)} ${parts}`)
|
|
62
|
+
}
|
|
63
|
+
const f = rows.find((r) => r.onDemand)
|
|
64
|
+
if (f?.onDemand) {
|
|
65
|
+
const d = f.onDemand
|
|
66
|
+
out.push(
|
|
67
|
+
"",
|
|
68
|
+
`ForgeQE's agent loads on demand: ${d.playbooks} playbooks (${k(d.playbookTokens)}), each with the first result of its tools or through the playbook tool.`,
|
|
69
|
+
`With everything always loaded it would carry ${k(d.everything)} on every call; it carries ${k(f.total)}.`,
|
|
70
|
+
)
|
|
71
|
+
}
|
|
72
|
+
const over = rows.filter((r) => r.over)
|
|
73
|
+
if (over.length) out.push("", `Over budget: ${over.map((r) => r.agent.name).join(", ")}. Shorten what they load, or raise the budget in src/agents.ts with the reason.`)
|
|
74
|
+
return out
|
|
75
|
+
}
|
package/src/mcp/captools.ts
CHANGED
|
@@ -22,4 +22,23 @@ export const CAPABILITY_TOOLS: McpTool[] = [
|
|
|
22
22
|
return { capabilities: ids.map((id) => capabilityDetail(capability(id)!, ctx.config)) }
|
|
23
23
|
},
|
|
24
24
|
},
|
|
25
|
+
{
|
|
26
|
+
name: "playbook",
|
|
27
|
+
description:
|
|
28
|
+
"Read a playbook that is not in your prompt (the index under Playbooks names them), by its name, its capability id, or one of its tools. The first result of a playbook's tools carries it anyway: read it first only when you need its rules before that call. Nothing runs.",
|
|
29
|
+
inputSchema: {
|
|
30
|
+
type: "object",
|
|
31
|
+
properties: {
|
|
32
|
+
names: { type: "array", items: { type: "string" }, description: "Playbook names (e.g. Testing from requirements), capability ids, or tool names" },
|
|
33
|
+
},
|
|
34
|
+
required: ["names"],
|
|
35
|
+
},
|
|
36
|
+
async run(args, ctx) {
|
|
37
|
+
const names = Array.isArray(args.names) ? args.names.map(String).filter(Boolean) : []
|
|
38
|
+
if (!names.length) throw new Error("names is empty.")
|
|
39
|
+
if (ctx.router) return ctx.router.help(names)
|
|
40
|
+
const [{ TOOLS }, { createRouter }] = await Promise.all([import("./tools.ts"), import("./routing.ts")])
|
|
41
|
+
return createRouter(TOOLS).help(names)
|
|
42
|
+
},
|
|
43
|
+
},
|
|
25
44
|
]
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { CAPABILITIES, type Capability } from "../capabilities/registry.ts"
|
|
2
|
+
import { prompt } from "../prompts.ts"
|
|
3
|
+
import type { McpTool } from "./tools.ts"
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* What ForgeQE's agent carries on every model call, and what it gets only when a task needs it.
|
|
7
|
+
*
|
|
8
|
+
* Every model call sends the system prompt and the tool definitions again. Tools stay listed in full: their
|
|
9
|
+
* descriptions are how the agent chooses a tool and plans the task, and their arguments are needed for any call, so a
|
|
10
|
+
* tool listed in one line cost an extra model call each time (measured: more tokens, not fewer). The playbook is
|
|
11
|
+
* what moves: the sections of brain.md that a capability owns (`playbook`) leave the system prompt, an index of them
|
|
12
|
+
* stays, and the first result of one of the capability's tools in a session (or of a tool in `alsoAfter`) carries the
|
|
13
|
+
* section. The `playbook` tool reads one before that. A section no capability owns stays in the system prompt, so a
|
|
14
|
+
* section renamed in a team's own copy of brain.md is never lost.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/** Rough tokens of a text: the budgets and `forgeqe context` use it, not billing. */
|
|
18
|
+
export const tokens = (text: string) => Math.ceil(text.length / 4)
|
|
19
|
+
|
|
20
|
+
const norm = (s: string) => s.trim().toLowerCase()
|
|
21
|
+
|
|
22
|
+
/** The capability each tool belongs to. */
|
|
23
|
+
export function capabilityOfTool(name: string, caps: Capability[] = CAPABILITIES): Capability | undefined {
|
|
24
|
+
return caps.find((c) => c.tools.includes(name))
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// ---------------------------------------------------------------------------------------------
|
|
28
|
+
// The playbook
|
|
29
|
+
// ---------------------------------------------------------------------------------------------
|
|
30
|
+
|
|
31
|
+
export type Playbook = { heading: string; when: string; capability: string; tools: string[]; text: string }
|
|
32
|
+
|
|
33
|
+
/** brain.md split: the system prompt without the owned sections (with an index of them), and those sections. */
|
|
34
|
+
export type BrainParts = { core: string; playbooks: Playbook[] }
|
|
35
|
+
|
|
36
|
+
const SECTION = /^###\s+(.+?)\s*$/
|
|
37
|
+
|
|
38
|
+
/** Split the agent's prompt by the capabilities' playbook sections. Unowned sections stay in `core`. */
|
|
39
|
+
export function splitBrain(text: string, caps: Capability[] = CAPABILITIES): BrainParts {
|
|
40
|
+
const owned = new Map(caps.flatMap((c) => (c.playbook || []).map((p) => [norm(p.heading), { ...p, capability: c.id, tools: c.tools }] as const)))
|
|
41
|
+
const lines = text.split("\n")
|
|
42
|
+
const keep: string[] = []
|
|
43
|
+
const playbooks: Playbook[] = []
|
|
44
|
+
let current: { entry: Omit<Playbook, "text">; lines: string[] } | null = null
|
|
45
|
+
const close = () => {
|
|
46
|
+
if (current) playbooks.push({ ...current.entry, text: current.lines.join("\n").trim() })
|
|
47
|
+
current = null
|
|
48
|
+
}
|
|
49
|
+
for (const line of lines) {
|
|
50
|
+
const h = SECTION.exec(line)
|
|
51
|
+
if (h || /^##\s/.test(line)) {
|
|
52
|
+
close()
|
|
53
|
+
const entry = h ? owned.get(norm(h[1])) : undefined
|
|
54
|
+
if (entry) {
|
|
55
|
+
current = { entry: { heading: h![1].trim(), when: entry.when, capability: entry.capability, tools: entry.tools }, lines: [line] }
|
|
56
|
+
continue
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
if (current) current.lines.push(line)
|
|
60
|
+
else keep.push(line)
|
|
61
|
+
}
|
|
62
|
+
close()
|
|
63
|
+
let core = keep.join("\n").replace(/\n{3,}/g, "\n\n")
|
|
64
|
+
if (playbooks.length) {
|
|
65
|
+
const index = [
|
|
66
|
+
"",
|
|
67
|
+
"These playbooks are not in this prompt. The first result of their tools carries each one: obey it. To read one before you start, call playbook with its name.",
|
|
68
|
+
...playbooks.map((p) => `- "${p.heading}": ${p.when}. Tools: ${p.tools.join(", ")}.`),
|
|
69
|
+
"",
|
|
70
|
+
].join("\n")
|
|
71
|
+
core = /^## Playbooks\s*$/m.test(core) ? core.replace(/^## Playbooks\s*$/m, (m) => `${m}\n${index}`) : `${core.trimEnd()}\n\n## Playbooks\n${index}`
|
|
72
|
+
}
|
|
73
|
+
return { core: core.trim(), playbooks }
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** ForgeQE's agent's prompt as it is sent: the core, with the index of the playbooks it gets on demand. */
|
|
77
|
+
export const brainParts = (): BrainParts => splitBrain(prompt("brain"))
|
|
78
|
+
|
|
79
|
+
// ---------------------------------------------------------------------------------------------
|
|
80
|
+
// One session of ForgeQE's agent
|
|
81
|
+
// ---------------------------------------------------------------------------------------------
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Routing for one agent session: which playbooks it was given, and what goes with a result. The playbook text is read
|
|
85
|
+
* once, when the session starts, like the system prompt.
|
|
86
|
+
*/
|
|
87
|
+
export function createRouter(tools: McpTool[], opts: { caps?: Capability[]; brain?: string } = {}) {
|
|
88
|
+
const caps = opts.caps || CAPABILITIES
|
|
89
|
+
const parts = splitBrain(opts.brain ?? prompt("brain"), caps)
|
|
90
|
+
const given = new Set<string>()
|
|
91
|
+
const names = new Set(tools.map((t) => t.name))
|
|
92
|
+
|
|
93
|
+
const playbooksFor = (tool: string) => parts.playbooks.filter((p) => p.tools.includes(tool) || caps.find((c) => c.id === p.capability)?.alsoAfter?.includes(tool))
|
|
94
|
+
const give = (books: Playbook[]) => {
|
|
95
|
+
const fresh = books.filter((b) => !given.has(b.heading))
|
|
96
|
+
for (const b of fresh) given.add(b.heading)
|
|
97
|
+
return fresh
|
|
98
|
+
}
|
|
99
|
+
const block = (books: Playbook[]) => (books.length ? `\n\nPlaybook (obey it for this task):\n\n${books.map((b) => b.text).join("\n\n")}` : "")
|
|
100
|
+
|
|
101
|
+
return {
|
|
102
|
+
parts,
|
|
103
|
+
/** What goes after an error of a tool: the playbooks of its capability this session has not had yet. */
|
|
104
|
+
after(name: string): string {
|
|
105
|
+
return block(give(playbooksFor(name)))
|
|
106
|
+
},
|
|
107
|
+
/**
|
|
108
|
+
* A result with the playbooks it should carry: in a `playbook` field of a JSON object, so the result stays JSON for
|
|
109
|
+
* whatever reads it; appended to any other text.
|
|
110
|
+
*/
|
|
111
|
+
attach(name: string, text: string): string {
|
|
112
|
+
const books = give(playbooksFor(name))
|
|
113
|
+
if (!books.length) return text
|
|
114
|
+
const playbook = `Obey it for this task:\n\n${books.map((b) => b.text).join("\n\n")}`
|
|
115
|
+
try {
|
|
116
|
+
const parsed = JSON.parse(text)
|
|
117
|
+
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return JSON.stringify({ ...parsed, playbook }, null, 2)
|
|
118
|
+
} catch {}
|
|
119
|
+
return text + block(books)
|
|
120
|
+
},
|
|
121
|
+
/** The playbook tool: playbooks by name, by capability id, or of a tool's capability. */
|
|
122
|
+
help(asked: string[]): string {
|
|
123
|
+
const books: Playbook[] = []
|
|
124
|
+
const unknown: string[] = []
|
|
125
|
+
for (const raw of asked) {
|
|
126
|
+
const exact = norm(raw)
|
|
127
|
+
// The client shows ForgeQE's tools with a forgeqe_ prefix; forgeqe_setup is that tool's own name.
|
|
128
|
+
const tool = names.has(exact) ? exact : names.has(exact.replace(/^forgeqe_/, "")) ? exact.replace(/^forgeqe_/, "") : ""
|
|
129
|
+
const found = tool ? playbooksFor(tool) : parts.playbooks.filter((p) => norm(p.heading) === exact || p.capability === exact)
|
|
130
|
+
if (found.length) books.push(...found)
|
|
131
|
+
else unknown.push(raw)
|
|
132
|
+
}
|
|
133
|
+
const unique = [...new Map(books.map((b) => [b.heading, b])).values()]
|
|
134
|
+
for (const b of unique) given.add(b.heading)
|
|
135
|
+
return [
|
|
136
|
+
...(unique.length ? ["Playbook (obey it for this task):", "", ...unique.map((b) => b.text)] : []),
|
|
137
|
+
...(unknown.length ? ["", `No playbook for: ${unknown.join(", ")}. Playbooks: ${parts.playbooks.map((p) => `"${p.heading}"`).join(", ")}. A tool not under one of them has its rules in its own description and in the prompt.`] : []),
|
|
138
|
+
]
|
|
139
|
+
.join("\n")
|
|
140
|
+
.trim()
|
|
141
|
+
},
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export type Router = ReturnType<typeof createRouter>
|
package/src/mcp/server.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { createHash, randomBytes, timingSafeEqual } from "node:crypto"
|
|
|
2
2
|
|
|
3
3
|
import { EvidenceLog } from "../report/evidence.ts"
|
|
4
4
|
import { logEvent } from "../sessionLog.ts"
|
|
5
|
+
import type { Router } from "./routing.ts"
|
|
5
6
|
import type { McpTool, ToolContext, ToolEvent } from "./tools.ts"
|
|
6
7
|
|
|
7
8
|
type Json = Record<string, any>
|
|
@@ -21,6 +22,8 @@ export type McpServerOptions = {
|
|
|
21
22
|
/** Read on every call, so the shell's current profile and headed setting apply. */
|
|
22
23
|
context: () => ToolContext
|
|
23
24
|
instructions?: () => string
|
|
25
|
+
/** ForgeQE's agent: playbooks given with the first result of their tools. Other clients get results as they are. */
|
|
26
|
+
router?: Router
|
|
24
27
|
version?: string
|
|
25
28
|
progressMs?: number
|
|
26
29
|
}
|
|
@@ -41,7 +44,7 @@ export function createMcpServer(opts: McpServerOptions) {
|
|
|
41
44
|
async function callTool(params: Json, id: string | number, send: Send, outer?: AbortSignal): Promise<Json> {
|
|
42
45
|
const tool = opts.tools.find((t) => t.name === params.name)
|
|
43
46
|
if (!tool) return { content: [{ type: "text", text: `Unknown tool ${params.name}` }], isError: true }
|
|
44
|
-
const base = { evidence, ...opts.context() }
|
|
47
|
+
const base = { evidence, ...opts.context(), ...(opts.router ? { router: opts.router } : {}) }
|
|
45
48
|
const controller = new AbortController()
|
|
46
49
|
const abort = () => controller.abort()
|
|
47
50
|
base.signal?.addEventListener("abort", abort, { once: true })
|
|
@@ -91,9 +94,10 @@ export function createMcpServer(opts: McpServerOptions) {
|
|
|
91
94
|
said(true)
|
|
92
95
|
const text = typeof out === "string" ? out : JSON.stringify(out, null, 2)
|
|
93
96
|
base.evidence.record(tool.name, args, text)
|
|
94
|
-
return finish({ content: [{ type: "text", text }] })
|
|
97
|
+
return finish({ content: [{ type: "text", text: opts.router ? opts.router.attach(tool.name, text) : text }] })
|
|
95
98
|
} catch (err) {
|
|
96
|
-
|
|
99
|
+
const message = controller.signal.aborted ? "Stopped by the user." : (err as Error).message || String(err)
|
|
100
|
+
return finish({ content: [{ type: "text", text: message + (controller.signal.aborted ? "" : opts.router?.after(tool.name) || "") }], isError: true })
|
|
97
101
|
} finally {
|
|
98
102
|
if (timer) clearInterval(timer)
|
|
99
103
|
running.delete(id)
|
package/src/mcp/tools.ts
CHANGED
|
@@ -36,6 +36,7 @@ import { API_TOOLS } from "./apitools.ts"
|
|
|
36
36
|
import { EXPLORE_TOOLS } from "./exploretools.ts"
|
|
37
37
|
import { QUALITY_TOOLS } from "./qualitytools.ts"
|
|
38
38
|
import { CAPABILITY_TOOLS } from "./captools.ts"
|
|
39
|
+
import type { Router } from "./routing.ts"
|
|
39
40
|
import { NOTIFY_TOOLS } from "./notifytools.ts"
|
|
40
41
|
import { REPORT_TOOLS } from "./reporttools.ts"
|
|
41
42
|
import { DEVICE_TOOLS } from "./devicetools.ts"
|
|
@@ -140,6 +141,8 @@ export function queryLine(args: Record<string, any>): string {
|
|
|
140
141
|
|
|
141
142
|
export type ToolContext = {
|
|
142
143
|
config: AppConfig
|
|
144
|
+
/** ForgeQE's agent's session: which tools and playbooks it gets on demand (src/mcp/routing.ts). */
|
|
145
|
+
router?: Router
|
|
143
146
|
/** The repository or folder the agent works in. */
|
|
144
147
|
cwd: string
|
|
145
148
|
headed?: boolean
|