@capagents/forgeqe 1.3.2 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HOW_TO_USE.md +373 -9
- package/README.md +55 -4
- package/docs/ARCHITECTURE.md +62 -1
- package/docs/CONTAINERS.md +227 -0
- package/docs/COST_ARCHITECTURE.md +32 -0
- package/docs/DESKTOP-CI.md +87 -0
- package/docs/DESKTOP.md +387 -0
- package/docs/MOBILE.md +228 -0
- package/docs/feature-test-prompts.md +25 -0
- package/docs/security-data-flow.html +440 -3
- package/docs/user-guide.html +797 -415
- package/env.example.yaml +67 -0
- package/forgeqe.example.yaml +56 -1
- package/llms.example.json +7 -0
- package/package.json +4 -1
- package/prompts/brain.md +31 -0
- package/prompts/find-tests.md +1 -0
- package/py/forgeqe_desktop/__init__.py +1 -0
- package/py/forgeqe_desktop/__main__.py +3 -0
- package/py/forgeqe_desktop/apps.py +80 -0
- package/py/forgeqe_desktop/checks.py +168 -0
- package/py/forgeqe_desktop/cli.py +90 -0
- package/py/forgeqe_desktop/handhooks.py +293 -0
- package/py/forgeqe_desktop/handrecord.py +239 -0
- package/py/forgeqe_desktop/heal.py +107 -0
- package/py/forgeqe_desktop/models.py +208 -0
- package/py/forgeqe_desktop/pywinauto_driver.py +217 -0
- package/py/forgeqe_desktop/record.py +150 -0
- package/py/forgeqe_desktop/replay.py +345 -0
- package/py/forgeqe_desktop/service.py +356 -0
- package/py/forgeqe_desktop/session.py +60 -0
- package/py/forgeqe_desktop/ufo_bridge.py +86 -0
- package/py/forgeqe_desktop/ufo_runner.py +305 -0
- package/py/forgeqe_desktop/visual.py +112 -0
- package/py/forgeqe_desktop/workflow.py +338 -0
- package/py/forgeqe_engine/llm.py +54 -4
- package/py/forgeqe_engine/model_params.py +169 -0
- package/py/forgeqe_mobile/__init__.py +1 -0
- package/py/forgeqe_mobile/__main__.py +3 -0
- package/py/forgeqe_mobile/appium_driver.py +342 -0
- package/py/forgeqe_mobile/apps.py +84 -0
- package/py/forgeqe_mobile/cli.py +70 -0
- package/py/forgeqe_mobile/explore.py +203 -0
- package/py/forgeqe_mobile/hyperexecute.py +261 -0
- package/py/forgeqe_mobile/maestro.py +321 -0
- package/py/forgeqe_mobile/service.py +332 -0
- package/skills/testcontainers/SKILL.md +42 -0
- package/src/agents.ts +11 -2
- package/src/api/plan.ts +2 -0
- package/src/api/write.ts +2 -0
- package/src/auth/copilot.ts +130 -0
- package/src/auth/signin.ts +54 -1
- package/src/brain/index.ts +26 -4
- package/src/capabilities/guide.ts +2 -0
- package/src/capabilities/registry.ts +110 -2
- package/src/capabilities/status.ts +15 -0
- package/src/cases/rca.ts +2 -0
- package/src/childEnv.ts +4 -1
- package/src/cli/args.ts +3 -2
- package/src/cli/help.ts +24 -3
- package/src/cli/index.ts +47 -3
- package/src/code/find.ts +26 -0
- package/src/code/loop.ts +4 -1
- package/src/code/prompt.ts +2 -1
- package/src/code/prreview.ts +2 -0
- package/src/code/review.ts +6 -2
- package/src/code/runtime.ts +34 -3
- package/src/commands.ts +6 -1
- package/src/config.ts +159 -3
- package/src/containers/command.ts +68 -0
- package/src/containers/compose.ts +173 -0
- package/src/containers/config.ts +283 -0
- package/src/containers/mock.ts +156 -0
- package/src/containers/run.ts +362 -0
- package/src/containers/tests.ts +80 -0
- package/src/context.ts +33 -6
- package/src/copilot.ts +155 -0
- package/src/desktop/bridge.ts +229 -0
- package/src/desktop/command.ts +350 -0
- package/src/desktop/edit.ts +217 -0
- package/src/desktop/junit.ts +35 -0
- package/src/desktop/report.ts +155 -0
- package/src/desktop/run.ts +252 -0
- package/src/desktop/setup.ts +134 -0
- package/src/engine/bridge.ts +12 -5
- package/src/engine/setup.ts +3 -3
- package/src/environments.ts +566 -0
- package/src/evals/domain.ts +1 -0
- package/src/explore/charters.ts +2 -0
- package/src/guardrails.ts +145 -0
- package/src/heal/loop.ts +4 -1
- package/src/heal/prompt.ts +2 -1
- package/src/init.ts +5 -2
- package/src/llm.ts +23 -2
- package/src/mcp/attachtools.ts +16 -4
- package/src/mcp/captools.ts +27 -0
- package/src/mcp/containertools.ts +212 -0
- package/src/mcp/desktoptools.ts +334 -0
- package/src/mcp/mobiletools.ts +132 -0
- package/src/mcp/reporttools.ts +17 -2
- package/src/mcp/repotests.ts +22 -3
- package/src/mcp/routing.ts +11 -5
- package/src/mcp/scope.ts +210 -0
- package/src/mcp/server.ts +80 -5
- package/src/mcp/tools.ts +38 -1
- package/src/mobile/command.ts +245 -0
- package/src/mobile/run.ts +226 -0
- package/src/pack/pipeline.ts +44 -2
- package/src/refine.ts +3 -0
- package/src/report/html.ts +3 -0
- package/src/report/template.html +48 -48
- package/src/run.ts +7 -3
- package/src/tokenomics/ledger.ts +3 -1
- package/src/tui/App.tsx +23 -0
- package/src/usage.ts +1 -1
package/docs/user-guide.html
CHANGED
|
@@ -322,20 +322,22 @@ a:hover{text-decoration-color:var(--guide-accent)}
|
|
|
322
322
|
<ol class="chapters">
|
|
323
323
|
<li><a href="#area-browser"><span class="n">04</span><span class="t">Browser</span><span class="d">Goals carried out in a real browser, and the logins they use</span></a></li>
|
|
324
324
|
<li><a href="#area-test-cases"><span class="n">05</span><span class="t">Test cases</span><span class="d">Run, check, write, and explore manual test cases</span></a></li>
|
|
325
|
-
<li><a href="#area-
|
|
326
|
-
<li><a href="#area-
|
|
327
|
-
<li><a href="#area-
|
|
328
|
-
<li><a href="#area-
|
|
329
|
-
<li><a href="#area-
|
|
330
|
-
<li><a href="#area-
|
|
331
|
-
<li><a href="#area-
|
|
332
|
-
<li><a href="#area-
|
|
325
|
+
<li><a href="#area-setup"><span class="n">06</span><span class="t">Setup</span><span class="d">Configuration, skills, and help</span></a></li>
|
|
326
|
+
<li><a href="#area-desktop"><span class="n">07</span><span class="t">Desktop</span><span class="d">Windows thick-client apps: explore a goal, record by hand, replay the saved workflow</span></a></li>
|
|
327
|
+
<li><a href="#area-mobile"><span class="n">08</span><span class="t">Mobile</span><span class="d">Android and iOS apps through Appium or Maestro: explore a goal, replay, run the team's flows</span></a></li>
|
|
328
|
+
<li><a href="#area-operations"><span class="n">09</span><span class="t">Operations</span><span class="d">Monitoring and CI pipelines</span></a></li>
|
|
329
|
+
<li><a href="#area-automated-tests"><span class="n">10</span><span class="t">Automated tests</span><span class="d">Write, heal, find, run, and stabilise tests in your repository</span></a></li>
|
|
330
|
+
<li><a href="#area-pull-requests"><span class="n">11</span><span class="t">Pull requests</span><span class="d">Reviews, impact, and the tests a change can break</span></a></li>
|
|
331
|
+
<li><a href="#area-api"><span class="n">12</span><span class="t">API</span><span class="d">Contracts and end-to-end scenarios, without a browser</span></a></li>
|
|
332
|
+
<li><a href="#area-quality"><span class="n">13</span><span class="t">Quality</span><span class="d">Accessibility, page performance, and visual changes</span></a></li>
|
|
333
|
+
<li><a href="#area-release"><span class="n">14</span><span class="t">Release</span><span class="d">Requirement coverage and a release verdict</span></a></li>
|
|
334
|
+
<li><a href="#area-systems"><span class="n">15</span><span class="t">Systems</span><span class="d">Azure DevOps, Jira, and Confluence</span></a></li>
|
|
333
335
|
</ol>
|
|
334
336
|
</section>
|
|
335
337
|
<section class="rail-group">
|
|
336
338
|
<h2 class="rail-title mono">Reference</h2>
|
|
337
339
|
<ol class="chapters">
|
|
338
|
-
<li><a href="#reference"><span class="n">
|
|
340
|
+
<li><a href="#reference"><span class="n">16</span><span class="t">Reference</span><span class="d">MCP, tuning, logs, environment variables, and every command</span></a></li>
|
|
339
341
|
</ol>
|
|
340
342
|
</section>
|
|
341
343
|
</div>
|
|
@@ -378,14 +380,16 @@ forgeqe docs --copy ./docs # copy the guide, the architecture page, and the re
|
|
|
378
380
|
<h3 class="band-title">Every feature</h3>
|
|
379
381
|
<div class="area"><a class="area-name" href="#area-browser"><span class="mono">Browser</span></a><ul class="tiles"><li><a class="tile" href="#cap-browser-goals"><b>Run any goal in a real browser</b><span>Go to booking.com and search hotels in Mumbai for next weekend</span></a></li><li><a class="tile" href="#cap-logins"><b>Saved logins for the application under test</b><span>Sign in to the admin site and keep the session</span></a></li></ul></div>
|
|
380
382
|
<div class="area"><a class="area-name" href="#area-test-cases"><span class="mono">Test cases</span></a><ul class="tiles"><li><a class="tile" href="#cap-run-test-cases"><b>Run manual test cases in browsers</b><span>Run SHOP-12 and SHOP-14 from Jira, two browsers at once</span></a></li><li><a class="tile" href="#cap-test-case-quality"><b>Check and improve test cases</b><span>Are the cases in test plan 12 good enough to run?</span></a></li><li><a class="tile" href="#cap-write-test-cases"><b>Write test cases from a story</b><span>Write test cases for story 1234 and save them</span></a></li><li><a class="tile" href="#cap-domain-compliance"><b>Domain and regulatory test design</b><span>Write test cases for story 1234, covering our DORA and PSD2 obligations</span></a></li><li><a class="tile" href="#cap-exploratory-testing"><b>Exploratory testing of a story</b><span>Explore story 1234 and find bugs</span></a></li></ul></div>
|
|
383
|
+
<div class="area"><a class="area-name" href="#area-setup"><span class="mono">Setup</span></a><ul class="tiles"><li><a class="tile" href="#cap-test-apps"><b>Test environments and their apps</b><span>Place an order in the shop, then check its invoice in Billing, in UAT</span></a></li><li><a class="tile" href="#cap-setup-help"><b>Setup, skills, and documentation</b><span>What can you do?</span></a></li></ul></div>
|
|
384
|
+
<div class="area"><a class="area-name" href="#area-desktop"><span class="mono">Desktop</span></a><ul class="tiles"><li><a class="tile" href="#cap-desktop-apps"><b>Automate Windows desktop apps</b><span>Create an invoice for Contoso in the Billing app and save it as a test</span></a></li></ul></div>
|
|
385
|
+
<div class="area"><a class="area-name" href="#area-mobile"><span class="mono">Mobile</span></a><ul class="tiles"><li><a class="tile" href="#cap-mobile-apps"><b>Automate Android and iOS apps</b><span>Sign in to the courier app and accept the first delivery, and save it as a test</span></a></li></ul></div>
|
|
386
|
+
<div class="area"><a class="area-name" href="#area-operations"><span class="mono">Operations</span></a><ul class="tiles"><li><a class="tile" href="#cap-containers"><b>Services and mocks in Docker or Podman</b><span>Start the database and the queue in Docker</span></a></li><li><a class="tile" href="#cap-monitoring"><b>Scheduled monitoring with alerts</b><span>Check the login flow every 15 minutes and alert Teams</span></a></li><li><a class="tile" href="#cap-notifications"><b>Teams and email notifications</b><span>Post the results to the QA channel</span></a></li><li><a class="tile" href="#cap-pipelines"><b>CI pipelines</b><span>Add a GitHub Actions workflow that runs tests/cases.yaml on every PR, and make it green</span></a></li></ul></div>
|
|
381
387
|
<div class="area"><a class="area-name" href="#area-automated-tests"><span class="mono">Automated tests</span></a><ul class="tiles"><li><a class="tile" href="#cap-write-tests"><b>Turn a passed run into an automated test</b><span>Write a Playwright test for that run in ../app</span></a></li><li><a class="tile" href="#cap-heal-tests"><b>Heal failing tests</b><span>Heal the failing tests in ../app</span></a></li><li><a class="tile" href="#cap-find-run-tests"><b>Find and run existing tests</b><span>Run the checkout tests</span></a></li><li><a class="tile" href="#cap-regression-packs"><b>Regression packs for a release</b><span>Build a regression pack for sprint 12 and run it on TestMu</span></a></li><li><a class="tile" href="#cap-flaky-tests"><b>Flaky test detection and quarantine</b><span>Which tests are flaky?</span></a></li></ul></div>
|
|
382
388
|
<div class="area"><a class="area-name" href="#area-pull-requests"><span class="mono">Pull requests</span></a><ul class="tiles"><li><a class="tile" href="#cap-pr-review"><b>Review pull requests</b><span>Review PR 42 and focus on error handling</span></a></li><li><a class="tile" href="#cap-pr-impact"><b>Pull request impact and test selection</b><span>What does PR 42 impact?</span></a></li></ul></div>
|
|
383
389
|
<div class="area"><a class="area-name" href="#area-api"><span class="mono">API</span></a><ul class="tiles"><li><a class="tile" href="#cap-api-testing"><b>API and contract testing</b><span>Turn the last run's API calls into tests</span></a></li></ul></div>
|
|
384
390
|
<div class="area"><a class="area-name" href="#area-quality"><span class="mono">Quality</span></a><ul class="tiles"><li><a class="tile" href="#cap-accessibility-performance"><b>Accessibility and page performance</b><span>Is the checkout page accessible?</span></a></li><li><a class="tile" href="#cap-visual-comparison"><b>Visual comparison with baselines</b><span>What changed visually since the last run?</span></a></li><li><a class="tile" href="#cap-defect-intelligence"><b>Defect intelligence: trends and prediction</b><span>Which defects keep coming back?</span></a></li></ul></div>
|
|
385
391
|
<div class="area"><a class="area-name" href="#area-release"><span class="mono">Release</span></a><ul class="tiles"><li><a class="tile" href="#cap-release-readiness"><b>Requirement coverage and release readiness</b><span>Are we ready to release?</span></a></li><li><a class="tile" href="#cap-reports"><b>Reports you can share</b><span>Create a release report for all projects</span></a></li></ul></div>
|
|
386
|
-
<div class="area"><a class="area-name" href="#area-operations"><span class="mono">Operations</span></a><ul class="tiles"><li><a class="tile" href="#cap-monitoring"><b>Scheduled monitoring with alerts</b><span>Check the login flow every 15 minutes and alert Teams</span></a></li><li><a class="tile" href="#cap-notifications"><b>Teams and email notifications</b><span>Post the results to the QA channel</span></a></li><li><a class="tile" href="#cap-pipelines"><b>CI pipelines</b><span>Add a GitHub Actions workflow that runs tests/cases.yaml on every PR, and make it green</span></a></li></ul></div>
|
|
387
392
|
<div class="area"><a class="area-name" href="#area-systems"><span class="mono">Systems</span></a><ul class="tiles"><li><a class="tile" href="#cap-systems"><b>Work with Azure DevOps, Jira, GitHub, and Confluence</b><span>Show the open bugs in this sprint</span></a></li></ul></div>
|
|
388
|
-
<div class="area"><a class="area-name" href="#area-setup"><span class="mono">Setup</span></a><ul class="tiles"><li><a class="tile" href="#cap-setup-help"><b>Setup, skills, and documentation</b><span>What can you do?</span></a></li></ul></div>
|
|
389
393
|
<h3 class="band-title">Not supported</h3>
|
|
390
394
|
<ul class="plainlist"><li><b>Native mobile apps (iOS, Android) and desktop applications</b>. Instead: web applications in local Chrome or Chromium, or configured remote web grids such as BrowserStack and TestMu AI/LambdaTest.</li><li><b>Load, stress, or soak testing with many simultaneous users</b>. Instead: per-page Core Web Vitals with budgets.</li><li><b>Penetration testing or dynamic security scanning</b>. Instead: security findings in pull request reviews.</li></ul>
|
|
391
395
|
</section>
|
|
@@ -401,7 +405,7 @@ forgeqe docs --copy ./docs # copy the guide, the architecture page, and the re
|
|
|
401
405
|
<tbody>
|
|
402
406
|
<tr><td><strong>npm (Node.js 18+) or pip (Python 3.9+)</strong></td><td>To install <code>@capagents/forgeqe</code> or <code>forgeqe-cli</code>; either one gives the same <code>forgeqe</code> command</td></tr>
|
|
403
407
|
<tr><td><strong>A terminal</strong></td><td>macOS Terminal, iTerm2, VS Code or Cursor's terminal, Windows Terminal</td></tr>
|
|
404
|
-
<tr><td><strong>A model</strong></td><td>A profile in <code>llms.json</code> with a working key, or Microsoft Entra ID sign-in for Azure (Azure OpenAI, OpenAI, an OpenAI-compatible gateway, or Ollama)</td></tr>
|
|
408
|
+
<tr><td><strong>A model</strong></td><td>A profile in <code>llms.json</code> with a working key, or Microsoft Entra ID sign-in for Azure (Azure OpenAI, OpenAI, an OpenAI-compatible gateway, or Ollama), or a GitHub Copilot seat (see <a href="../#github-copilot-seats-as-the-model">GitHub Copilot seats as the model</a>)</td></tr>
|
|
405
409
|
<tr><td><strong>Internet access on first use</strong></td><td>ForgeQE downloads Bun, the agent runtime, uv, Python, and the engine into <code>~/.forgeqe</code> once</td></tr>
|
|
406
410
|
</tbody>
|
|
407
411
|
</table></div>
|
|
@@ -434,6 +438,11 @@ export WP_AZURE_API_KEY=... # the key your llms.json profile names
|
|
|
434
438
|
forgeqe models # check the profiles
|
|
435
439
|
forgeqe start # the shell
|
|
436
440
|
</code></pre>
|
|
441
|
+
<p>With GitHub Copilot licences instead of a model key, sign in rather than exporting one, and set <code>llm.active: copilot</code> in <code>forgeqe.yaml</code> (or <code>"default": "copilot"</code> in <code>llms.json</code>):</p>
|
|
442
|
+
<pre><code class="language-bash">forgeqe auth login copilot # ForgeQE's agent and Scribe on your Copilot seat
|
|
443
|
+
forgeqe auth login github # the browser engine, through GitHub Models
|
|
444
|
+
forgeqe models # the copilot profile is active
|
|
445
|
+
</code></pre>
|
|
437
446
|
<p>In the shell, type a request and press Enter:</p>
|
|
438
447
|
<pre><code>Go to booking.com and search for hotels in Mumbai for next weekend
|
|
439
448
|
Run SHOP-12 and SHOP-14 from Jira, two browsers at once
|
|
@@ -507,6 +516,61 @@ bun src/index.ts init --global --profile azure-gpt4o
|
|
|
507
516
|
</table></div>
|
|
508
517
|
<p>Give that identity the <strong>Cognitive Services OpenAI User</strong> role on the resource. Tokens are renewed before they expire, so long suites and coding sessions keep running. The browser engine signs in itself; the coding agent's runtime only takes a key, so ForgeQE gives it a gateway on <code>127.0.0.1</code> that accepts a one-time random key and passes each request on with a current token. No key is stored anywhere.</p>
|
|
509
518
|
<p><code>auth: entra</code> also works on an <code>openai_compatible</code> profile, for a gateway such as API Management that checks Entra ID tokens: set <code>token_scope</code> to the scope it expects (for example <code>api://<app id>/.default</code>; the default is <code>https://cognitiveservices.azure.com/.default</code>). The engine needs the <code>azure-identity</code> package, which <code>forgeqe setup</code> installs (and adds to environments set up before it was needed); with <code>FORGEQE_PYTHON</code>, install it there yourself.</p>
|
|
519
|
+
</div></section>
|
|
520
|
+
<section class="doc sub" id="doc-how_to_use-github-copilot-seats-as-the-model"><h4>GitHub Copilot seats as the model</h4><div class="prose"><p>With GitHub Copilot licences (Pro, Pro+, Business or Enterprise), a <code>github_copilot</code> profile runs ForgeQE's agent and Scribe on the Copilot seat. They work through the agent runtime's own GitHub Copilot sign-in, <a href="https://github.blog/changelog/2026-01-16-github-copilot-now-supports-opencode/" rel="noopener">which GitHub supports</a>. Copilot gives other tools no model API of their own, so the browser engine and the desktop agent use <a href="https://docs.github.com/en/github-models" rel="noopener">GitHub Models</a> instead, with your GitHub sign-in:</p>
|
|
521
|
+
<pre><code class="language-json">"copilot": {
|
|
522
|
+
"provider": "github_copilot",
|
|
523
|
+
"model": "claude-sonnet-4.6",
|
|
524
|
+
"engine_model": "openai/gpt-4.1",
|
|
525
|
+
"github_org": "acme"
|
|
526
|
+
}
|
|
527
|
+
</code></pre>
|
|
528
|
+
<div class="tbl"><table>
|
|
529
|
+
<thead>
|
|
530
|
+
<tr><th>Field</th><th>What it is</th></tr>
|
|
531
|
+
</thead>
|
|
532
|
+
<tbody>
|
|
533
|
+
<tr><td><code>model</code></td><td>The Copilot model for ForgeQE's agent and Scribe, as Copilot names it (<code>claude-sonnet-4.6</code>, <code>gpt-5.4</code>). The sign-in lists the ones your seat has.</td></tr>
|
|
534
|
+
<tr><td><code>engine_model</code></td><td>The GitHub Models model for the browser engine and the desktop agent, as <code>publisher/model</code>. It must take images. Default <code>openai/gpt-4.1</code>.</td></tr>
|
|
535
|
+
<tr><td><code>github_org</code></td><td>Optional. Bills the engine's GitHub Models calls to this organization and applies its policies, rather than your personal free allowance.</td></tr>
|
|
536
|
+
<tr><td><code>api_key_env</code></td><td>Optional. The variable holding the GitHub Models token; default <code>GITHUB_TOKEN</code>, else the GitHub CLI's sign-in.</td></tr>
|
|
537
|
+
</tbody>
|
|
538
|
+
</table></div>
|
|
539
|
+
<p>Sign in once per machine:</p>
|
|
540
|
+
<pre><code class="language-bash">forgeqe auth login copilot # Copilot, for ForgeQE's agent and Scribe (GitHub.com or GitHub Enterprise, e.g. acme.ghe.com)
|
|
541
|
+
forgeqe auth login github # the GitHub CLI, whose sign-in the browser engine uses for GitHub Models
|
|
542
|
+
</code></pre>
|
|
543
|
+
<p><code>forgeqe auth login copilot</code> shows a one-time code and opens GitHub's device page; authorize with the account that has the seat. The sign-in goes to your system's credential store like ForgeQE's other secrets. The agent runtime gets it in its environment for each session, never in a file, and the agent's shell commands never see it. A Copilot sign-in made with <code>opencode auth login</code> also works. In CI, set <code>GITHUB_TOKEN</code> to a token with the <code>models:read</code> permission for the engine.</p>
|
|
544
|
+
<p><code>forgeqe auth</code> shows the sign-in on its <code>Copilot</code> line (<code>saved sign-in as dana</code>, or <code>agent runtime's own sign-in</code> for one made with <code>opencode auth login</code>); <code>forgeqe auth logout copilot</code> forgets it.</p>
|
|
545
|
+
<p>Which part uses what:</p>
|
|
546
|
+
<div class="tbl"><table>
|
|
547
|
+
<thead>
|
|
548
|
+
<tr><th>Part</th><th>Model</th><th>Signs in with</th></tr>
|
|
549
|
+
</thead>
|
|
550
|
+
<tbody>
|
|
551
|
+
<tr><td>ForgeQE's agent (the shell, <code>forgeqe run</code>)</td><td><code>model</code>, on the Copilot seat</td><td><code>forgeqe auth login copilot</code></td></tr>
|
|
552
|
+
<tr><td>Scribe (writing, fixing and reviewing tests)</td><td><code>model</code>, on the Copilot seat</td><td><code>forgeqe auth login copilot</code></td></tr>
|
|
553
|
+
<tr><td>The browser engine (every browser run: <code>run --direct</code>, suites, monitoring)</td><td><code>engine_model</code>, on GitHub Models</td><td><code>GITHUB_TOKEN</code>, or <code>forgeqe auth login github</code></td></tr>
|
|
554
|
+
<tr><td>The desktop agent</td><td><code>engine_model</code>, on GitHub Models</td><td><code>GITHUB_TOKEN</code>, or <code>forgeqe auth login github</code></td></tr>
|
|
555
|
+
</tbody>
|
|
556
|
+
</table></div>
|
|
557
|
+
<p><code>code.model</code> still overrides <code>model</code> for ForgeQE's agent and Scribe; <code>code.model: github-copilot/<model></code> uses the Copilot sign-in too, with any other profile active.</p>
|
|
558
|
+
<p>What each part costs: the agent's and Scribe's requests count against the seat's premium requests. The engine's GitHub Models calls are billed by GitHub Models: the free allowance is rate-limited, so for suites set <code>github_org</code> to an organization with paid GitHub Models usage turned on. If a Copilot model is refused, check that your organization's Copilot policies allow that model. ForgeQE shows no cost for a <code>github_copilot</code> profile unless you give it a <code>price</code>.</p>
|
|
559
|
+
<p><strong>In pipelines.</strong> <code>forgeqe suite ... --no-code</code>, which pipelines run, uses only the browser engine, so it needs GitHub Models and no Copilot sign-in. In GitHub Actions, give the job <code>permissions: models: read</code> and pass the workflow's own token as <code>GITHUB_TOKEN</code>; on Azure Pipelines or elsewhere, store a fine-grained token with the <code>models:read</code> permission as the <code>GITHUB_TOKEN</code> secret. A Copilot seat belongs to one person, so keep ForgeQE's agent and Scribe on developers' machines rather than signing a shared pipeline in to someone's seat.</p>
|
|
560
|
+
<p><strong>When something goes wrong.</strong></p>
|
|
561
|
+
<div class="tbl"><table>
|
|
562
|
+
<thead>
|
|
563
|
+
<tr><th>ForgeQE says</th><th>What to do</th></tr>
|
|
564
|
+
</thead>
|
|
565
|
+
<tbody>
|
|
566
|
+
<tr><td><code>The github_copilot profile for … needs a GitHub Copilot sign-in</code></td><td>Run <code>forgeqe auth login copilot</code> (or <code>/auth copilot</code> in the shell).</td></tr>
|
|
567
|
+
<tr><td><code>The browser engine reaches GitHub Models with a GitHub sign-in, and there is none</code></td><td>Run <code>forgeqe auth login github</code>, or set <code>GITHUB_TOKEN</code> to a token with <code>models:read</code>.</td></tr>
|
|
568
|
+
<tr><td><code>… must be a Copilot model id</code>, <code>engine_model must be a GitHub Models id</code>, <code>github_org must be a GitHub organization name</code></td><td>Fix that field in <code>llms.json</code>: the sign-in lists the seat's Copilot models, and <a href="https://github.com/marketplace?type=models" rel="noopener">the GitHub Models catalog</a> lists <code>publisher/model</code> ids.</td></tr>
|
|
569
|
+
<tr><td>The agent's model is refused, or not found</td><td>The seat's plan or your organization's Copilot policies do not allow that model: pick one the sign-in listed, or ask a Copilot admin to enable it.</td></tr>
|
|
570
|
+
<tr><td>The browser engine gets <code>429</code> / rate limit errors</td><td>The GitHub Models free allowance is used up: set <code>github_org</code> to an organization with paid GitHub Models usage, or wait for the limit to reset.</td></tr>
|
|
571
|
+
<tr><td>The code page does not open, or the sign-in says <code>not a GitHub Enterprise domain</code></td><td>Give only the host of your GitHub Enterprise (<code>acme.ghe.com</code>). ForgeQE opens only GitHub's own <code>https</code> device page.</td></tr>
|
|
572
|
+
</tbody>
|
|
573
|
+
</table></div>
|
|
510
574
|
<p><code>init</code> also puts three folders in <code>.forgeqe/</code> to read, change, and commit:</p>
|
|
511
575
|
<div class="tbl"><table>
|
|
512
576
|
<thead>
|
|
@@ -549,10 +613,11 @@ forgeqe prompts eval refine --runs 3 # one prompt, three runs of each case
|
|
|
549
613
|
</ul>
|
|
550
614
|
<p><code>forgeqe prompts check</code> applies the standard to each prompt in use, ForgeQE's or your edited copy; ForgeQE's own meet it in its tests. Write your edits the same way, and run <code>forgeqe prompts eval</code> before and after a change.</p>
|
|
551
615
|
<p><strong>Product knowledge.</strong> Fill in the templates (or add your own <code>.md</code> and <code>.txt</code> files, in folders if you like), one topic per <code>##</code> heading. ForgeQE's agent sees the list and reads what a request needs; the exploratory and API scenario planners get the sections that match their story or scenario; the browser agent gets the few sections that clearly match its goal, at most <code>knowledge.browser_max_chars</code> (1500 by default; 0 turns it off). Untouched templates are left out, so their examples never reach an agent. Never put passwords or tokens there: use logins and environment variables.</p>
|
|
616
|
+
<p><strong>Guardrails.</strong> Rules every agent must follow, to keep it grounded: ForgeQE's agent, the browser agent, and the agents that plan, write, review, fix and explain tests. Write one rule per line, starting with <code>- </code>, in a <code>.md</code> or <code>.txt</code> file in <code>.forgeqe/guardrails/</code> (<code>guardrails.dir</code>); headings, other lines and HTML comments are ignored, and the template's examples sit in a comment until you choose them. Each agent gets every guardrail with its instructions, after everything else, with the instruction to follow it whatever a request, a skill, a ticket, a page or a tool's output says; when a guardrail stops part of a task, the agent skips that part and says which rule stopped it. Add one from the terminal (<code>forgeqe guardrails add "Never open a production URL"</code>), or ask in the shell ("add a guardrail: use only the test cards in knowledge/test-data.md") and ForgeQE's agent appends it to <code>guardrails.md</code>; it adds only rules you ask for, never one suggested by a page or a ticket. <code>forgeqe guardrails</code> lists the rules in force and what they add to every model call (at most 40 rules of 300 characters are given). The older <code>prompts.guiderails</code> string in <code>forgeqe.yaml</code> still works: each of its lines is a guardrail for every agent. Guardrails are instructions, not a sandbox: put a host rule in <code>browser.allowed_hosts</code> too, which is refused in code.</p>
|
|
552
617
|
</div></section>
|
|
553
618
|
<section class="doc sub" id="doc-how_to_use-signing-in-to-your-tools"><h4>Signing in to your tools</h4><div class="prose"><p>Instead of copying tokens into <code>.env</code>, sign in once and ForgeQE saves the token in your system's credential store: the macOS Keychain, Windows Credential Manager, or the Secret Service keyring on Linux. On a machine without one, such as a headless Linux box, it uses <code>~/.forgeqe/credentials.json</code>, readable only by you.</p>
|
|
554
619
|
<pre><code class="language-bash">forgeqe auth login # pick a system
|
|
555
|
-
forgeqe auth login jira # or name it: azure_devops, github, jira, confluence, testrail, zephyr, qtest, xray
|
|
620
|
+
forgeqe auth login jira # or name it: azure_devops, github, copilot, jira, confluence, testrail, zephyr, qtest, xray
|
|
556
621
|
forgeqe auth # what is signed in, and how
|
|
557
622
|
forgeqe auth logout jira
|
|
558
623
|
</code></pre>
|
|
@@ -563,7 +628,8 @@ forgeqe auth logout jira
|
|
|
563
628
|
</thead>
|
|
564
629
|
<tbody>
|
|
565
630
|
<tr><td>Azure DevOps</td><td><strong>Sign in with Microsoft in the browser</strong> through the Azure CLI (<code>az login</code>); ForgeQE saves nothing, and the Azure CLI renews the sign-in. Or paste a personal access token: the token page opens with the scopes to tick. Azure DevOps Server (on-premises) uses a personal access token.</td></tr>
|
|
566
|
-
<tr><td>GitHub</td><td><strong>Sign in with GitHub in the browser</strong> through the GitHub CLI (<code>gh auth login</code>): ForgeQE shows the one-time code and opens the page. Or paste a token. Needs <code>gh</code> installed.</td></tr>
|
|
631
|
+
<tr><td>GitHub</td><td><strong>Sign in with GitHub in the browser</strong> through the GitHub CLI (<code>gh auth login</code>): ForgeQE shows the one-time code and opens the page. Or paste a token. Needs <code>gh</code> installed. A <code>github_copilot</code> profile's browser engine uses this sign-in for GitHub Models.</td></tr>
|
|
632
|
+
<tr><td>GitHub Copilot</td><td><code>forgeqe auth login copilot</code>: <strong>Sign in with GitHub in the browser</strong> through the agent runtime's Copilot sign-in, on GitHub.com or GitHub Enterprise (<code>acme.ghe.com</code>). ForgeQE shows the one-time code, opens GitHub's device page, and lists the Copilot models your seat has. For a <code>github_copilot</code> profile; see <a href="../#github-copilot-seats-as-the-model">GitHub Copilot seats as the model</a>.</td></tr>
|
|
567
633
|
<tr><td>Jira, Confluence</td><td>Your Atlassian API tokens page opens (Cloud) or your personal access tokens page (Server/Data Center); create one and paste it. Confluence on the same Cloud site as Jira reuses the Jira sign-in.</td></tr>
|
|
568
634
|
<tr><td>TestRail, Zephyr Scale, qTest, Xray Cloud</td><td>The page with your API key opens, or ForgeQE says where to find it; paste it.</td></tr>
|
|
569
635
|
<tr><td>Anything else</td><td><code>forgeqe auth login OPENAI_API_KEY</code> (any variable name) saves that one secret: a model key, a Teams webhook URL, an SMTP or saved-login password.</td></tr>
|
|
@@ -572,6 +638,7 @@ forgeqe auth logout jira
|
|
|
572
638
|
<p>Atlassian's OAuth sign-in needs a client secret, which a command-line tool cannot keep secret, so Jira and Confluence use API tokens: they are what Atlassian recommends for scripts and tools.</p>
|
|
573
639
|
<p>Each token is checked against the system before it is saved, and the system is switched on: a saved Jira sign-in sets the site URL too, so <code>sources.jira</code> is not needed in <code>forgeqe.yaml</code>. Saved values go in under the variable names the config already reads (<code>JIRA_API_TOKEN</code>, <code>ADO_PAT</code>, ...). A variable exported in the terminal is used before a saved sign-in, and a saved sign-in before a <code>.env</code> file, so pipelines keep using their secrets. The shell commands the agents run never see the saved secrets.</p>
|
|
574
640
|
<p><strong>Prices.</strong> ForgeQE prices every model call at the model's <strong>list price</strong>, from two public price lists: <a href="https://models.dev" rel="noopener">models.dev</a> (the list the agent runtime uses) and LiteLLM's model price list, which also has Azure's Data Zone prices. It keeps the few fields it needs in <code>~/.forgeqe/prices.json</code>, waits a few seconds for it the first time, refreshes it once a day in the background, and uses the last copy offline; only the public lists are downloaded, nothing about your runs is sent (<code>FORGEQE_PRICES=off</code> stops the downloads). Costs at list price say so: <code>$1.73 (list price of gpt-5.4 (Azure Global) from models.dev, 2026-10-06)</code>. For an Azure Data Zone deployment, which costs about 10% more, set <code>"deployment_type": "data_zone"</code> on the profile.</p>
|
|
641
|
+
<p><strong>Model parameters.</strong> Models differ in what a request may carry: reasoning models (o1, o3, o4, gpt-5) take <code>max_completion_tokens</code> instead of <code>max_tokens</code> and only the default temperature, and some OpenAI-compatible servers refuse <code>max_completion_tokens</code>, <code>reasoning_effort</code> or <code>response_format</code>. ForgeQE works this out by itself. A profile is a reasoning model when its <code>model</code> or <code>deployment</code> name says so (an Azure deployment called <code>chat-prod</code> with <code>"model": "gpt-5.4"</code> counts), or when it sets <code>"reasoning": true</code>; <code>"reasoning": false</code> says it is not. When the API still refuses a parameter, the browser, desktop and mobile agents send the call again without it and leave it out of later calls to that model. Nothing needs setting for a new model.</p>
|
|
575
642
|
<p>A profile's own <code>"price": { "input": 2.5, "output": 15, "cached_input": 0.25 }</code> (per million tokens) wins, for a negotiated or enterprise rate. Leave <code>currency</code> out and reports use this machine's currency, or set <code>"currency": "USD"</code>. Without <code>cached_input</code>, cached input gets the model's cache discount from its list price, and the cost is marked as an estimate (<code>≈$1.52</code>); with no list price for the model either, cached input is charged at the full rate and the cost is an upper bound (<code>up to $6.11</code>). When your price differs from the list price, ForgeQE says so (<code>forgeqe models</code>, the shell, and the start of each command), since that is usually a typo or an old price. Anthropic's cache writes are priced at their own rate. A provisioned (PTU) deployment is billed per hour, not per token, so its costs are what the tokens would cost at list price. Requests over a model's long-context threshold (272K tokens for GPT-5.4) cost more and are priced at the normal rate.</p>
|
|
576
643
|
<p><strong>Usage after every job.</strong> Every browser run, Scribe job, heal, suite, and monitor round ends with a <code>Usage</code> line: tokens in (and how many of them were cached), tokens out, and the cost at the profile's price. A request to ForgeQE's agent in the shell, and every <code>forgeqe run</code>, <code>code</code>, <code>suite</code>, and <code>heal</code> command, ends with the total for everything it started, by role:</p>
|
|
577
644
|
<pre><code>This request used 1.1M tokens (1.1M in, 30.0K of it cached · 34.7K out) · $3.07 — ForgeQE's agent 42.0K · $0.12, browser agent 52.7K · $0.15, Scribe 1.0M · $2.80
|
|
@@ -611,7 +678,7 @@ browser:
|
|
|
611
678
|
<p><code>judge</code> is off by default. When it is on, one more model call reads the finished run's steps and screenshots and can fail a run the agent completed, for example when five scores were read off one results page rather than one step per item. That strictness sent runs into loops of re-checking, so turn it on only when you want that second opinion.</p>
|
|
612
679
|
<p>Each step waits for the tab the agent is on to finish loading (up to 8 seconds, 3 for a tab that stays blank) before the page is read, so a result that opens in a new tab is not mistaken for a blank page.</p>
|
|
613
680
|
<p>With a visible browser (<code>headless: false</code> or <code>--headed</code>), ForgeQE opens on its own start page, and every page it drives gets a glowing edge and a bar along the bottom showing, in plain words, what the step is for and what the agent is doing ("Step 19 · Check whether the results can be sorted by Fastest · Looking through the page's buttons"). Your mouse and keyboard are locked while the agent thinks and unlocked while it clicks and types, so a stray click can't change the test. None of this appears in the screenshots the model sees or in the report. <code>overlay: false</code> turns it off. <code>forgeqe login <name></code> shows the same bar without the lock, since you are the one signing in.</p>
|
|
614
|
-
<p><code>start_url</code> is optional. Leave it out and the agent picks the site from the goal; ForgeQE never guesses a URL from the goal text. <code>allowed_hosts</code> empty allows every host. Set <code>["example.com"]</code> to keep the browser on that domain and its subdomains. <code>prompts.
|
|
681
|
+
<p><code>start_url</code> is optional. Leave it out and the agent picks the site from the goal; ForgeQE never guesses a URL from the goal text. <code>allowed_hosts</code> empty allows every host. Set <code>["example.com"]</code> to keep the browser on that domain and its subdomains. <code>prompts.step_instruction</code> is appended to the engine's system prompt; the lines of <code>prompts.guiderails</code> are guardrails for every agent (see Guardrails above).</p>
|
|
615
682
|
<p>List profiles:</p>
|
|
616
683
|
<pre><code class="language-bash">bun src/index.ts models
|
|
617
684
|
</code></pre>
|
|
@@ -667,7 +734,7 @@ Add a GitHub Actions workflow that runs tests/cases.yaml on every PR, and make i
|
|
|
667
734
|
</tbody>
|
|
668
735
|
</table></div>
|
|
669
736
|
<p><code>start_url</code> in <code>forgeqe.yaml</code> or <code>--url</code> pins a start page for browser runs, such as a staging app. Passed runs become tests in the current folder; <code>forgeqe start --repo ../app</code> puts them in that repo instead, and <code>--no-code</code> turns tests off.</p>
|
|
670
|
-
<p>ForgeQE's agent runs on the agent runtime with the active profile's model (or <code>code.model</code>). ForgeQE downloads the agent runtime into <code>~/.forgeqe/tools</code> when it is not installed. When the agent runtime cannot be downloaded, ForgeQE says the agent is off and runs each message as one browser goal.</p>
|
|
737
|
+
<p>ForgeQE's agent runs on the agent runtime with the active profile's model (or <code>code.model</code>); a <code>github_copilot</code> profile runs it on your Copilot seat. ForgeQE downloads the agent runtime into <code>~/.forgeqe/tools</code> when it is not installed. When the agent runtime cannot be downloaded, ForgeQE says the agent is off and runs each message as one browser goal.</p>
|
|
671
738
|
<p>While a run is going, the bar under the prompt shows a spinner, the tokens used, and the elapsed time, and the transcript ends with what the agent is doing now (<code>Thinking about step 3…</code>, <code>Acting in the browser (step 3)…</code>). Each step starts with <code>+ Thought: 4.2s</code>, the time the model took to decide. When the run ends, a summary card closes the transcript:</p>
|
|
672
739
|
<pre><code>┃ ✓ Task complete 10 steps · 1m 42s · 65.3K tokens
|
|
673
740
|
┃
|
|
@@ -1276,187 +1343,700 @@ Acceptance criteria:
|
|
|
1276
1343
|
</div>
|
|
1277
1344
|
</article>
|
|
1278
1345
|
</section>
|
|
1279
|
-
<section class="chapter" id="area-
|
|
1280
|
-
<header class="chead"><span class="mono">06</span><h2 id="area-
|
|
1281
|
-
<article class="spec feat" id="cap-
|
|
1282
|
-
<div class="hd"><span class="mono">
|
|
1283
|
-
<p class="ds">
|
|
1346
|
+
<section class="chapter" id="area-setup" aria-labelledby="area-setup-h">
|
|
1347
|
+
<header class="chead"><span class="mono">06</span><h2 id="area-setup-h">Setup</h2><p>Configuration, skills, and help</p></header>
|
|
1348
|
+
<article class="spec feat" id="cap-test-apps">
|
|
1349
|
+
<div class="hd"><span class="mono">Setup</span><h3>Test environments and their apps</h3></div>
|
|
1350
|
+
<p class="ds">Knows the test environments in env.yaml (QA, UAT, …) and each one's apps: web apps and APIs by URL, Windows desktop apps, and mobile apps, with what each is for. Every agent works from the active one; switches environment on request, picks the apps a scenario needs, and carries values such as an order number from one app to the next.</p>
|
|
1284
1351
|
<div class="facts">
|
|
1285
|
-
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>
|
|
1286
|
-
<div class="fact"><span class="mono">What it needs</span><ul class="needs"><li><span class="need
|
|
1287
|
-
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>
|
|
1352
|
+
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>Place an order in the shop, then check its invoice in Billing, in UAT</li><li>Which apps can you test in QA?</li><li>From now on, use the UAT environment</li></ul></div>
|
|
1353
|
+
<div class="fact"><span class="mono">What it needs</span><ul class="needs"><li><span class="need input">You give</span><span>env.yaml with the environments and apps (forgeqe init writes one)</span></li></ul></div>
|
|
1354
|
+
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>test_apps</code> <code>use_environment</code></p></div>
|
|
1288
1355
|
</div>
|
|
1289
1356
|
<div class="detail"><span class="mono">In detail</span>
|
|
1290
|
-
<section class="doc" id="doc-how_to_use-
|
|
1291
|
-
<h4>
|
|
1292
|
-
<div class="prose"><p>
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
</code
|
|
1296
|
-
<pre><code class="language-yaml">
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
<
|
|
1310
|
-
|
|
1311
|
-
<p>
|
|
1312
|
-
|
|
1313
|
-
<li>
|
|
1314
|
-
|
|
1315
|
-
<
|
|
1316
|
-
|
|
1317
|
-
<li>
|
|
1318
|
-
<
|
|
1319
|
-
<
|
|
1320
|
-
</
|
|
1321
|
-
<
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
</
|
|
1330
|
-
<
|
|
1331
|
-
|
|
1332
|
-
</
|
|
1333
|
-
</
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
</code></
|
|
1345
|
-
<
|
|
1346
|
-
<
|
|
1347
|
-
|
|
1348
|
-
<
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
┃ Run .forgeqe/reports/20260928-172700/
|
|
1355
|
-
┃ Repo ../app
|
|
1356
|
-
┃
|
|
1357
|
-
▸ Handoff written: .forgeqe/reports/20260928-172700/handoff.md (7 steps, 3 pages from page memory)
|
|
1358
|
-
▸ Scribe, the coding agent (azure/gpt-5.4), reads the handoff and writes a test in /work/app
|
|
1359
|
-
⚙ Reading .forgeqe/reports/20260928-172700/handoff.md
|
|
1360
|
-
⚙ Reading tests/pages/LoginPage.ts
|
|
1361
|
-
✎ Editing tests/invoices.spec.ts
|
|
1362
|
-
⚙ Running npx playwright test tests/invoices.spec.ts
|
|
1363
|
-
▸ ForgeQE runs the test: npx playwright test tests/invoices.spec.ts
|
|
1364
|
-
✗ tests/invoices.spec.ts failed, exit 1 (attempt 1/5)
|
|
1365
|
-
▸ tests/invoices.spec.ts passed in 9s in Google Chrome; a reviewer checks it against the goal
|
|
1366
|
-
✓ tests/invoices.spec.ts passed and was approved (attempt 2/5)
|
|
1367
|
-
┃
|
|
1368
|
-
┃ ✓ Test passed 2/5 attempts · 3m 12s · 88.4K tokens · azure/gpt-5.4
|
|
1369
|
-
┃
|
|
1370
|
-
┃ Added openInvoices() to DashboardPage and reused the loginAs fixture.
|
|
1371
|
-
┃
|
|
1372
|
-
┃ Test tests/invoices.spec.ts (in ../app)
|
|
1373
|
-
┃ Command npx playwright test tests/invoices.spec.ts
|
|
1374
|
-
┃ Reused LoginPage, fixtures/auth.ts
|
|
1375
|
-
┃ Changed tests/invoices.spec.ts, tests/pages/DashboardPage.ts
|
|
1376
|
-
┃ Browser Google Chrome
|
|
1377
|
-
┃ Review approved: Opens the invoices page through the dashboard and checks the table columns.
|
|
1378
|
-
</code></pre>
|
|
1379
|
-
<p>After ForgeQE's reply, the request ends with the files it produced:</p>
|
|
1380
|
-
<pre><code>┃ ✓ What this produced
|
|
1381
|
-
┃
|
|
1382
|
-
┃ Report .forgeqe/reports/20260928-172700/report.html
|
|
1383
|
-
┃ ✓ Test ../app/tests/invoices.spec.ts passed when ForgeQE ran it
|
|
1384
|
-
┃ Command npx playwright test tests/invoices.spec.ts
|
|
1385
|
-
┃ Handoff .forgeqe/reports/20260928-172700/handoff.md
|
|
1386
|
-
</code></pre>
|
|
1387
|
-
<p>When no test was written, the Test row says why: the run did not pass, the session has <code>--no-code</code>, or the run was not handed to the coding agent.</p>
|
|
1388
|
-
<p>Scribe uses the active <code>llms.json</code> profile. The key stays in your environment: the agent runtime's config refers to it as <code>{env:WP_AZURE_API_KEY}</code> and never contains the value. An <code>auth: entra</code> profile reaches the model through ForgeQE's local token gateway (see <a href="../#signing-in-to-the-model-without-a-key-microsoft-entra-id">Signing in to the model without a key</a>). An Azure profile goes through the agent runtime's <code>azure</code> provider on <code><endpoint>/openai/v1</code> (the Responses API, with the deployment as the model). <code>openai_compatible</code> and <code>ollama</code> profiles use their <code>base_url</code>. <code>code.model: provider/model</code> (or <code>--code-model</code>) uses one of the agent runtime's built-in providers instead, with that provider's usual key variable (for example <code>ANTHROPIC_API_KEY</code> for <code>anthropic/...</code>).</p>
|
|
1389
|
-
<p>Scribe may edit files and run commands anywhere in the repo without asking. <code>git push</code> in any form is denied (pushing goes through ForgeQE's guarded <code>push_branch</code> tool, section 8), and it cannot touch files outside the repo except the run and memory directories, which it may read.</p>
|
|
1390
|
-
<p>More ways to run it:</p>
|
|
1391
|
-
<pre><code class="language-bash">forgeqe code --repo ../app # the latest run in .forgeqe/reports/
|
|
1392
|
-
forgeqe code .forgeqe/reports/20260928-172700 --repo ../app --attempts 3
|
|
1393
|
-
forgeqe run "..." --no-code # no tests for this request
|
|
1394
|
-
</code></pre>
|
|
1395
|
-
<p>In the shell, <code>forgeqe start --repo ../app</code> writes the tests in that repo instead of the current folder and <code>--no-code</code> turns them off; you can also just ask ("write a test for the last run in ../app"). Esc stops the agent the same way it stops a run.</p>
|
|
1396
|
-
<p>Exit codes for <code>forgeqe code</code> and for <code>forgeqe run --direct</code> with a repo: <code>0</code> test passed, <code>1</code> run or test failed, <code>2</code> stopped. The result (status, attempts, test file, command, what was reused and changed, the browsers ForgeQE saw, the review, the last test output, tokens) is saved as <code>code.json</code> in the run directory, next to <code>handoff.md</code>.</p>
|
|
1397
|
-
<p>If Scribe's job is cut off (ForgeQE closed, the machine restarted, you stopped it, or the model could not be reached), run it again for the same run and repo, for example <code>forgeqe code .forgeqe/reports/20260928-172700 --repo ../app</code>. Scribe reopens the same session, with everything it read and said, and carries on with the files it already wrote; it doesn't read the repo again, and ForgeQE uses the Playwright replay it already finished. The session is kept in <code>scribe-session.json</code> in the run folder until the job ends with a result.</p>
|
|
1357
|
+
<section class="doc" id="doc-how_to_use-environments-and-apps">
|
|
1358
|
+
<h4>Environments and apps</h4>
|
|
1359
|
+
<div class="prose"><p><code>env.yaml</code> (written by <code>forgeqe init</code>, named in forgeqe.yaml <code>environments.file</code>) lists your test environments and the
|
|
1360
|
+
apps in each: web apps by URL, APIs by base URL, Windows desktop apps by <code>.exe</code>, and mobile apps by package id, each
|
|
1361
|
+
with a short description of what it is for. Plain values go under <code>variables</code>; secrets never do, only the names of the
|
|
1362
|
+
environment variables that hold them, under <code>secrets</code>.</p>
|
|
1363
|
+
<pre><code class="language-yaml">default: qa
|
|
1364
|
+
environments:
|
|
1365
|
+
qa:
|
|
1366
|
+
start_app: shop
|
|
1367
|
+
apps:
|
|
1368
|
+
shop: { kind: web, url: "https://qa.shop.example.com", description: Customer web shop where orders are placed }
|
|
1369
|
+
orders: { kind: api, url: "https://qa.api.example.com/v1", description: Orders REST API }
|
|
1370
|
+
billing: { kind: desktop, path: 'C:\Program Files\Contoso\Billing\Billing.exe', args: ["--server", "qa-db01"], description: Back-office invoicing }
|
|
1371
|
+
variables: { company: Contoso QA }
|
|
1372
|
+
secrets: [BILLING_PASSWORD_QA]
|
|
1373
|
+
uat:
|
|
1374
|
+
apps: { ... }
|
|
1375
|
+
</code></pre>
|
|
1376
|
+
<p>Your test cases stay the same in every environment; only where they run changes. The environment supplies the URLs,
|
|
1377
|
+
executables, accounts and values, and the same test, workflow or scenario runs against QA or UAT.</p>
|
|
1378
|
+
<p><strong>Any layout works.</strong> The layout above is ForgeQE's own, but an <code>env.yaml</code> your team already keeps is read as it is:</p>
|
|
1379
|
+
<ul>
|
|
1380
|
+
<li>environments under <code>environments:</code> / <code>envs:</code>, as a list with a <code>name</code> each, or as top-level keys (<code>qa:</code>, <code>uat:</code>);
|
|
1381
|
+
or one file per environment with <code>environments.dir</code> in forgeqe.yaml</li>
|
|
1382
|
+
<li>a <code>common:</code> / <code>defaults:</code> / <code>shared:</code> block merged into each environment</li>
|
|
1383
|
+
<li>any <code>http(s)</code> URL becomes a web app (an API when its key or path says api); an absolute <code>.exe</code> becomes a desktop app
|
|
1384
|
+
(with a sibling <code>args</code>); other values become variables</li>
|
|
1385
|
+
<li>secrets: keys like <code>password</code>, <code>token</code> or <code>api_key</code> are never shown to a model. A reference (<code>${BILLING_PW}</code>,
|
|
1386
|
+
<code>env:BILLING_PW</code>, <code>%BILLING_PW%</code>) is recorded as a secret name; a literal value is replaced with <code><redacted></code></li>
|
|
1387
|
+
</ul>
|
|
1388
|
+
<pre><code class="language-yaml">common: { company: Contoso }
|
|
1389
|
+
qa:
|
|
1390
|
+
url: https://qa.shop.example.com
|
|
1391
|
+
orders_api: https://qa.api.example.com/v1
|
|
1392
|
+
billing_exe: 'C:\Program Files\Contoso\Billing\Billing.exe'
|
|
1393
|
+
db_password: ${BILLING_DB_PW_QA}
|
|
1394
|
+
uat:
|
|
1395
|
+
url: https://uat.shop.example.com
|
|
1396
|
+
</code></pre>
|
|
1397
|
+
<p>ForgeQE's agent also sees the environment as written (secrets replaced), so a request can name anything in it ("log in
|
|
1398
|
+
as the <code>admin_user</code> of UAT").</p>
|
|
1399
|
+
<p>Pick the environment with <code>--env uat</code> on any command (<code>forgeqe start</code>, <code>run</code>, <code>suite</code>, <code>monitor</code>, <code>mcp</code>, …) or
|
|
1400
|
+
<code>FORGEQE_ENV=uat</code>; otherwise <code>default</code> is used. In the shell, <code>/env</code> lists them and <code>/env uat</code> switches; asked in plain
|
|
1401
|
+
language ("from now on, use UAT"), ForgeQE's agent switches with <code>use_environment</code>, and any single request can name an
|
|
1402
|
+
environment ("run the checkout tests in UAT"): each test tool takes an <code>env</code> for that call only. <code>forgeqe apps</code> lists the environments and the active one's apps.</p>
|
|
1403
|
+
<p>What each agent gets from the active environment:</p>
|
|
1404
|
+
<div class="tbl"><table>
|
|
1405
|
+
<thead>
|
|
1406
|
+
<tr><th>Agent</th><th>What it uses</th></tr>
|
|
1407
|
+
</thead>
|
|
1408
|
+
<tbody>
|
|
1409
|
+
<tr><td>ForgeQE's agent</td><td>The environment, its apps (with descriptions) and values in its session notes; plans scenarios across apps from them and carries values (an order number) between them</td></tr>
|
|
1410
|
+
<tr><td>Browser agent</td><td>Starts at <code>start_app</code> when no URL is given; knows the environment's web apps by name and stays on their hosts; its values</td></tr>
|
|
1411
|
+
<tr><td>API tests</td><td>Call <code>api_app</code>'s URL</td></tr>
|
|
1412
|
+
<tr><td>Desktop agent</td><td>Starts the environment's desktop apps; types only its <code>secrets</code></td></tr>
|
|
1413
|
+
<tr><td>Scribe, the healer, the reviewer</td><td>Write tests that read <code>FORGEQE_APP_<APP>_URL</code> (or the repository's own settings), never a fixed environment URL; the reviewer flags one</td></tr>
|
|
1414
|
+
<tr><td>Tests ForgeQE runs, pack pipelines</td><td><code>FORGEQE_ACTIVE_ENV</code>, <code>FORGEQE_APP_<APP>_URL</code> / <code>_PATH</code> / <code>_ID</code>, <code>FORGEQE_VAR_<NAME></code> (pipelines written by <code>forgeqe pack pipeline init</code> take them as the <code>env_vars</code> input)</td></tr>
|
|
1415
|
+
<tr><td>Reports</td><td>Every run's report names the test environment</td></tr>
|
|
1416
|
+
</tbody>
|
|
1417
|
+
</table></div>
|
|
1418
|
+
<p>There is no scenario file to write: ask in plain language ("place an order in the shop, then check its invoice in
|
|
1419
|
+
Billing, in UAT") and ForgeQE's agent picks the apps from their descriptions, shows the plan, runs each step with the
|
|
1420
|
+
right engine, and carries values from one app to the next.</p>
|
|
1398
1421
|
</div>
|
|
1399
1422
|
</section>
|
|
1400
1423
|
</div>
|
|
1401
1424
|
</article>
|
|
1402
|
-
<article class="spec feat" id="cap-
|
|
1403
|
-
<div class="hd"><span class="mono">
|
|
1404
|
-
<p class="ds">
|
|
1425
|
+
<article class="spec feat" id="cap-setup-help">
|
|
1426
|
+
<div class="hd"><span class="mono">Setup</span><h3>Setup, skills, and documentation</h3></div>
|
|
1427
|
+
<p class="ds">Shows the model profiles and MCP servers, installs or repairs the browser engine, creates a forgeqe.yaml, manages skills, and answers questions about ForgeQE from its documentation and this list.</p>
|
|
1405
1428
|
<div class="facts">
|
|
1406
|
-
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>
|
|
1407
|
-
<div class="fact"><span class="mono">What it needs</span><
|
|
1408
|
-
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>
|
|
1429
|
+
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>What can you do?</li><li>Which model am I using?</li><li>How do I set up email alerts?</li></ul></div>
|
|
1430
|
+
<div class="fact"><span class="mono">What it needs</span><p class="muted">Nothing beyond ForgeQE itself.</p></div>
|
|
1431
|
+
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>forgeqe_setup</code> <code>capabilities</code> <code>playbook</code> <code>use_capabilities</code></p></div>
|
|
1409
1432
|
</div>
|
|
1410
1433
|
<div class="detail"><span class="mono">In detail</span>
|
|
1411
|
-
<
|
|
1412
|
-
<
|
|
1413
|
-
<
|
|
1414
|
-
<
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1434
|
+
<p class="see">See <a href="#doc-how_to_use-2-scaffold-config">2. Scaffold config</a>.</p>
|
|
1435
|
+
<section class="doc" id="doc-how_to_use-11-skills">
|
|
1436
|
+
<h4>Skills</h4>
|
|
1437
|
+
<div class="prose"><p>A skill is a folder with a <code>SKILL.md</code>: instructions that teach ForgeQE's agents something a prompt can't know, like your repo's test conventions, a framework, or how to drive an app such as Dynamics 365. The format is the Agent Skills one (Claude Code and others use it), with one ForgeQE field, <code>agents</code>:</p>
|
|
1438
|
+
<pre><code class="language-markdown">---
|
|
1439
|
+
name: contoso-erp
|
|
1440
|
+
description: "Use when the site is the Contoso ERP (erp.contoso.com): the order grid, the customer lookup, and saving orders."
|
|
1441
|
+
agents: [browser]
|
|
1442
|
+
---
|
|
1443
|
+
|
|
1444
|
+
- The customer lookup opens a list after three letters; click the row whose name matches exactly.
|
|
1445
|
+
- Save with the floppy icon in the toolbar; wait until "Saved" shows in the status bar.
|
|
1421
1446
|
</code></pre>
|
|
1422
|
-
<
|
|
1423
|
-
<
|
|
1424
|
-
<
|
|
1425
|
-
<
|
|
1426
|
-
<li>
|
|
1427
|
-
<li>
|
|
1428
|
-
<li>
|
|
1429
|
-
</
|
|
1430
|
-
<
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
┃ Changed tests/pages/LoginPage.ts
|
|
1452
|
-
┃ Report .forgeqe/reports/heal/20260929-101500/heal.md
|
|
1447
|
+
<p><code>name</code> is lowercase words joined by hyphens, the same as the folder. Agents choose skills by <code>description</code> alone, so name the concrete signs it applies (a framework, a config file, a host, a product). Quote it when it contains <code>: </code>.</p>
|
|
1448
|
+
<p><strong>Who uses it.</strong> <code>agents</code> lists <code>forgeqe</code> (ForgeQE's agent: how to handle a kind of request), <code>scribe</code> (writing, healing, finding and reviewing tests), and <code>browser</code> (the browser agent). Without <code>agents</code>, a skill is for <code>forgeqe</code> and <code>scribe</code>; a browser skill must ask for <code>browser</code>.</p>
|
|
1449
|
+
<p><strong>How skills are used.</strong></p>
|
|
1450
|
+
<ul>
|
|
1451
|
+
<li>ForgeQE's agent and Scribe see the name and description of each skill meant for them, and load the ones that fit the job through the agent runtime's skill tool. A skill adds to their instructions and never overrides them. Scribe can also read files next to <code>SKILL.md</code> (for example <code>references/locators.md</code>) when the skill points to them.</li>
|
|
1452
|
+
<li>Browser runs: ForgeQE's agent passes the browser skills a goal needs to <code>browser_run</code> and <code>run_test_cases</code>. For <code>forgeqe suite</code>, <code>forgeqe monitor</code> and <code>forgeqe run --direct</code>, the engine picks them from the goal with one short model call (at most three), and remembers the choice for that goal, so a monitor's later rounds and repeated cases skip it. <code>--skill NAME</code> (repeatable) fixes the list; <code>--skill none</code> runs without any. The chosen skills' text is added to the browser agent's prompt, at most <code>skills.browser_max_chars</code> (6,000) characters in total; the browser agent reads only <code>SKILL.md</code>.</li>
|
|
1453
|
+
<li><code>run.json</code>, the <code>run.done</code> log line and the report list the skills a run used.</li>
|
|
1454
|
+
</ul>
|
|
1455
|
+
<p><strong>Where skills come from</strong>, lowest to highest precedence; a later source replaces a skill of the same name:</p>
|
|
1456
|
+
<div class="tbl"><table>
|
|
1457
|
+
<thead>
|
|
1458
|
+
<tr><th>Source</th><th>Folder</th></tr>
|
|
1459
|
+
</thead>
|
|
1460
|
+
<tbody>
|
|
1461
|
+
<tr><td>Built-in</td><td>the <code>skills/</code> folder ForgeQE ships: <code>playwright-typescript</code> and <code>dotnet-nunit-playwright</code> (Scribe), <code>dynamics-365</code> (browser), <code>writing-skills</code> (ForgeQE's agent)</td></tr>
|
|
1462
|
+
<tr><td>Global, only with <code>skills.external: true</code></td><td><code>~/.claude/skills</code>, <code>~/.agents/skills</code>, and the repo's <code>.claude/skills</code> and <code>.agents/skills</code></td></tr>
|
|
1463
|
+
<tr><td>User</td><td><code>~/.forgeqe/skills</code> (or <code>$FORGEQE_HOME/skills</code>)</td></tr>
|
|
1464
|
+
<tr><td>Project</td><td><code>.forgeqe/skills</code> next to <code>forgeqe.yaml</code>, and the repo's own <code>.forgeqe/skills</code></td></tr>
|
|
1465
|
+
<tr><td>Extra</td><td>each folder in <code>skills.dirs</code></td></tr>
|
|
1466
|
+
</tbody>
|
|
1467
|
+
</table></div>
|
|
1468
|
+
<p>Commit <code>.forgeqe/skills</code> so the team and CI share it. ForgeQE only ignores <code>.forgeqe/reports/</code>; if your <code>.gitignore</code> has <code>.forgeqe/</code>, replace it with <code>.forgeqe/*</code> and <code>!.forgeqe/skills/</code> (<code>forgeqe skills new</code> warns when git would ignore the new skill). To change a built-in skill, make a project skill with the same name.</p>
|
|
1469
|
+
<p>Skills from anywhere else stay out: the agent runtime's own discovery (<code>~/.claude/skills</code>, a repo's <code>.opencode/skills</code>, its built-in skills) is off, and each agent may load only the skills ForgeQE's registry gives it. Skill text is trusted like the repo's own files, so global skills are opt-in.</p>
|
|
1470
|
+
<pre><code class="language-yaml">skills:
|
|
1471
|
+
enabled: true # false (or skills: false) turns skills off
|
|
1472
|
+
dirs: [../shared-skills] # more folders, read last
|
|
1473
|
+
disable: [dynamics-365] # leave these out
|
|
1474
|
+
external: false # true: also the global folders above
|
|
1475
|
+
browser_max_chars: 6000 # most skill text added to one browser run
|
|
1453
1476
|
</code></pre>
|
|
1454
|
-
<p>
|
|
1477
|
+
<p><strong>Commands.</strong></p>
|
|
1478
|
+
<div class="tbl"><table>
|
|
1479
|
+
<thead>
|
|
1480
|
+
<tr><th>Command</th><th>What it does</th></tr>
|
|
1481
|
+
</thead>
|
|
1482
|
+
<tbody>
|
|
1483
|
+
<tr><td><code>forgeqe skills</code></td><td>The skills each agent can use, where each comes from (and what it replaces), disabled skills, problems, and the folders read</td></tr>
|
|
1484
|
+
<tr><td><code>forgeqe skills show <name></code></td><td>One skill's instructions</td></tr>
|
|
1485
|
+
<tr><td><code>forgeqe skills new <name> --description "Use when …" [--agent browser] [--user]</code></td><td>Start a skill in <code>.forgeqe/skills</code> (<code>--user</code>: <code>~/.forgeqe/skills</code>)</td></tr>
|
|
1486
|
+
<tr><td><code>forgeqe skills check</code></td><td>Validate every skill; exit 1 when one has a problem</td></tr>
|
|
1487
|
+
<tr><td><code>forgeqe skills enable|disable <name></code></td><td>Edit <code>skills.disable</code> in <code>forgeqe.yaml</code>, keeping its comments</td></tr>
|
|
1488
|
+
<tr><td><code>/skills [name]</code></td><td>The same list, or one skill, in the shell</td></tr>
|
|
1489
|
+
</tbody>
|
|
1490
|
+
</table></div>
|
|
1491
|
+
<p>In the shell you can also ask: "make a skill for our Dynamics date picker" or "teach Scribe our test conventions". ForgeQE's agent gathers the facts, writes the skill into <code>.forgeqe/skills</code> with <code>forgeqe_setup</code>, and it applies from the next session. It writes skills only when you ask.</p>
|
|
1455
1492
|
</div>
|
|
1456
1493
|
</section>
|
|
1457
1494
|
</div>
|
|
1458
1495
|
</article>
|
|
1459
|
-
|
|
1496
|
+
</section>
|
|
1497
|
+
<section class="chapter" id="area-desktop" aria-labelledby="area-desktop-h">
|
|
1498
|
+
<header class="chead"><span class="mono">07</span><h2 id="area-desktop-h">Desktop</h2><p>Windows thick-client apps: explore a goal, record by hand, replay the saved workflow</p></header>
|
|
1499
|
+
<article class="spec feat" id="cap-desktop-apps">
|
|
1500
|
+
<div class="hd"><span class="mono">Desktop</span><h3>Automate Windows desktop apps</h3></div>
|
|
1501
|
+
<p class="ds">Carries out a plain-language goal in a Windows thick-client app with UFO² and saves it as a workflow with checks of the outcome, or records the steps a tester does by hand, with no model; replays saved workflows with no model, healing a step whose control moved, and fixes a workflow from chat (a timeout, a wait, a dialog to dismiss) after showing the change. Apps and the secrets they may type come from env.yaml or forgeqe.yaml desktop:.</p>
|
|
1502
|
+
<div class="facts">
|
|
1503
|
+
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>Create an invoice for Contoso in the Billing app and save it as a test</li><li>I'll show you the steps in Billing: record them</li><li>Replay the billing workflow with customer Fabrikam</li><li>Step 4 timed out while the list loaded: fix the workflow</li></ul></div>
|
|
1504
|
+
<div class="fact"><span class="mono">What it needs</span><ul class="needs"><li><span class="need opt">Optional</span><span>A live model profile in llms.json (an API key or Microsoft Entra ID sign-in)</span></li><li><span class="need req">Required</span><span>Windows desktop apps: env.yaml (kind: desktop) or desktop.apps in forgeqe.yaml</span></li><li><span class="need input">You give</span><span>a Windows machine, and the app in forgeqe.yaml desktop.apps</span></li></ul></div>
|
|
1505
|
+
<div class="fact"><span class="mono">Limits</span><ul class="plainlist"><li>Windows only, one run at a time per desktop</li><li>Apps whose controls UI Automation cannot see (Citrix, some Java and custom-drawn apps) are not supported yet</li></ul></div>
|
|
1506
|
+
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>desktop_explore</code> <code>desktop_record</code> <code>desktop_replay</code> <code>desktop_edit_workflow</code></p></div>
|
|
1507
|
+
</div>
|
|
1508
|
+
<div class="detail"><span class="mono">In detail</span>
|
|
1509
|
+
<section class="doc" id="doc-how_to_use-desktop-apps-windows">
|
|
1510
|
+
<h4>Desktop apps (Windows)</h4>
|
|
1511
|
+
<div class="prose"><p>ForgeQE also automates Windows thick-client apps (Win32, WinForms, WPF). In the shell, ask for a goal in the app
|
|
1512
|
+
("create an invoice for Contoso in Billing") and ForgeQE's agent uses UFO² to do it, saves the run as a workflow with
|
|
1513
|
+
checks of the outcome, and can replay it with no model. The same from the command line:</p>
|
|
1514
|
+
<pre><code class="language-bash">forgeqe desktop setup --ufo
|
|
1515
|
+
forgeqe desktop explore "Create an invoice for Contoso, net 30" --app billing
|
|
1516
|
+
forgeqe desktop record --app billing # or do the steps yourself; no model
|
|
1517
|
+
forgeqe desktop replay <run folder>/workflow.yaml customer=Fabrikam
|
|
1518
|
+
forgeqe desktop replay workflow.yaml --data customers.csv # once per row; columns are variable names
|
|
1519
|
+
forgeqe desktop replay desktop-tests/ --junit results.xml # every workflow in a folder, results for CI
|
|
1520
|
+
forgeqe desktop doctor # can this machine replay unattended?
|
|
1521
|
+
</code></pre>
|
|
1522
|
+
<p>Workflows wait for slow apps without fixed sleeps: a <code>timeout</code> and <code>retries</code> for the workflow or a step, <code>wait_for</code> a
|
|
1523
|
+
control to appear, show a text or go away (a splash screen), and a <code>dismiss</code> list of dialog buttons to click whenever
|
|
1524
|
+
they are in the way. <code>expect_image</code> compares a control or window with a baseline image (saved on the first run, kept
|
|
1525
|
+
next to the workflow), with a tolerance and regions to ignore. When you know the steps, <code>forgeqe desktop record</code> (or
|
|
1526
|
+
"I'll show you, record it" in the shell) saves what you do by hand: Ctrl+Shift+F11 over a control adds a check of its
|
|
1527
|
+
text, Ctrl+Shift+F12 finishes. After a failed replay, ForgeQE's agent can fix the workflow (a longer timeout, a
|
|
1528
|
+
<code>wait_for</code>, a dialog to dismiss): it shows the diff and writes it only when you agree. For nightly runs on a machine
|
|
1529
|
+
of your own, see <a href="../docs/DESKTOP-CI.md">docs/DESKTOP-CI.md</a>.</p>
|
|
1530
|
+
<p>Apps come from <code>desktop.apps</code> in forgeqe.yaml (an absolute <code>.exe</code> and an argument list), and <code>{{secret:NAME}}</code> reads
|
|
1531
|
+
only the variables listed in <code>desktop.secrets</code>. Explore and replay take over the desktop's mouse and keyboard and ask
|
|
1532
|
+
first; use a test machine and a test account. Details: <a href="../docs/DESKTOP.md">docs/DESKTOP.md</a>.</p>
|
|
1533
|
+
</div>
|
|
1534
|
+
</section>
|
|
1535
|
+
</div>
|
|
1536
|
+
</article>
|
|
1537
|
+
</section>
|
|
1538
|
+
<section class="chapter" id="area-mobile" aria-labelledby="area-mobile-h">
|
|
1539
|
+
<header class="chead"><span class="mono">08</span><h2 id="area-mobile-h">Mobile</h2><p>Android and iOS apps through Appium or Maestro: explore a goal, replay, run the team's flows</p></header>
|
|
1540
|
+
<article class="spec feat" id="cap-mobile-apps">
|
|
1541
|
+
<div class="hd"><span class="mono">Mobile</span><h3>Automate Android and iOS apps</h3></div>
|
|
1542
|
+
<p class="ds">Carries out a plain-language goal in an Android or iOS app through Appium and saves it as a mobile workflow with checks of the outcome; replays workflows with no model on Appium (healing a moved control) or Maestro; runs the team's own Maestro flows after checking them against an allowlist. Apps come from env.yaml (kind: mobile) or forgeqe.yaml mobile:.</p>
|
|
1543
|
+
<div class="facts">
|
|
1544
|
+
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>Sign in to the courier app and accept the first delivery, and save it as a test</li><li>Replay the courier login on Maestro</li><li>Run our Maestro flows in .maestro/</li></ul></div>
|
|
1545
|
+
<div class="fact"><span class="mono">What it needs</span><ul class="needs"><li><span class="need opt">Optional</span><span>A live model profile in llms.json (an API key or Microsoft Entra ID sign-in)</span></li><li><span class="need req">Required</span><span>Mobile apps: env.yaml (kind: mobile) or mobile.apps in forgeqe.yaml</span></li><li><span class="need input">You give</span><span>an Appium server with a device or emulator (or a device cloud), or Maestro installed; the app in env.yaml or forgeqe.yaml mobile.apps</span></li></ul></div>
|
|
1546
|
+
<div class="fact"><span class="mono">Limits</span><ul class="plainlist"><li>Explore runs on Appium only</li><li>Maestro replays do not heal moved controls or compare images (expect_image)</li><li>On TestMu AI, Maestro (through HyperExecute) refuses workflows that type secrets; replay those there with --driver appium</li></ul></div>
|
|
1547
|
+
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>mobile_explore</code> <code>mobile_replay</code> <code>mobile_maestro</code></p></div>
|
|
1548
|
+
</div>
|
|
1549
|
+
<div class="detail"><span class="mono">In detail</span>
|
|
1550
|
+
<section class="doc" id="doc-how_to_use-mobile-apps-android-ios">
|
|
1551
|
+
<h4>Mobile apps (Android, iOS)</h4>
|
|
1552
|
+
<div class="prose"><p>ForgeQE drives Android and iOS apps two ways: through <strong>Appium</strong>, step by step (it can explore a goal with the model,
|
|
1553
|
+
and heals a control that moved), and through <strong>Maestro</strong> (fast replays, and your team's own Maestro flows). In the shell,
|
|
1554
|
+
ask for a goal in the app ("sign in to the courier app and accept the first delivery") and ForgeQE's agent explores it
|
|
1555
|
+
on your device or emulator and saves a workflow with checks of the outcome. The same from the command line:</p>
|
|
1556
|
+
<pre><code class="language-bash">forgeqe mobile doctor # does the Appium server answer, is maestro installed?
|
|
1557
|
+
forgeqe mobile explore "Sign in and accept the first delivery" --app courier
|
|
1558
|
+
forgeqe mobile replay <run folder>/workflow.yaml # on Appium (mobile.driver), healing moved controls
|
|
1559
|
+
forgeqe mobile replay workflow.yaml --driver maestro # the same workflow as a Maestro flow
|
|
1560
|
+
forgeqe mobile replay mobile-tests/ --junit results.xml # every workflow in a folder, results for CI
|
|
1561
|
+
forgeqe mobile maestro .maestro/ # your team's Maestro flows, checked first
|
|
1562
|
+
forgeqe mobile flow workflow.yaml > login.flow.yaml # a workflow as a Maestro flow, to keep
|
|
1563
|
+
</code></pre>
|
|
1564
|
+
<p>Apps come from env.yaml (<code>kind: mobile</code>, <code>platform: android</code> or <code>ios</code>, <code>app_id</code>, and optionally <code>activity</code> and an
|
|
1565
|
+
<code>app</code> build to install) or <code>mobile.apps</code> in forgeqe.yaml. <code>mobile.appium</code> names the Appium server (default
|
|
1566
|
+
<code>http://127.0.0.1:4723</code>, or a device cloud's hub with its user and key in environment variables), and <code>{{secret:NAME}}</code>
|
|
1567
|
+
reads only <code>mobile.secrets</code> (and the environment's secrets). Team Maestro flows run only with UI commands: scripts,
|
|
1568
|
+
JavaScript in <code>${…}</code>, Maestro's AI commands and file writes are refused. Details: <a href="../docs/MOBILE.md">docs/MOBILE.md</a>.</p>
|
|
1569
|
+
<h5>On TestMu AI (formerly LambdaTest)</h5>
|
|
1570
|
+
<p>Both drivers run on TestMu AI's devices. Upload the app build once, then use the <code>lt://</code> id it prints as the app's <code>app</code>
|
|
1571
|
+
(in env.yaml or <code>mobile.apps</code>; ForgeQE adds it to Appium's <code>LT:Options</code> itself):</p>
|
|
1572
|
+
<pre><code class="language-bash">forgeqe mobile upload courier.apk # prints lt://APP…, for TestMu's real devices
|
|
1573
|
+
forgeqe mobile upload courier.apk --virtual # for TestMu's emulators and simulators instead
|
|
1574
|
+
forgeqe mobile upload courier.apk --name courier-nightly # the name TestMu shows for the build
|
|
1575
|
+
</code></pre>
|
|
1576
|
+
<p>The user name and access key come from <code>LT_USERNAME</code> and <code>LT_ACCESS_KEY</code> (or the variables <code>username_env</code> and
|
|
1577
|
+
<code>access_key_env</code> name), for example in a <code>.env</code> file in the project folder. They go only to the mobile engine, never to
|
|
1578
|
+
a flow or a model, and a user or key in the hub URL is refused.</p>
|
|
1579
|
+
<pre><code class="language-yaml">mobile:
|
|
1580
|
+
apps:
|
|
1581
|
+
courier: { platform: android, app_id: com.contoso.courier, app: "lt://APP1016…" }
|
|
1582
|
+
appium: # Appium on TestMu's mobile hub
|
|
1583
|
+
url: https://mobile-hub.lambdatest.com/wd/hub
|
|
1584
|
+
username_env: LT_USERNAME
|
|
1585
|
+
access_key_env: LT_ACCESS_KEY
|
|
1586
|
+
capabilities:
|
|
1587
|
+
"LT:Options": { w3c: true, isRealMobile: true, deviceName: "Pixel 6", platformVersion: "14" }
|
|
1588
|
+
maestro:
|
|
1589
|
+
testmu: # Maestro on TestMu, through HyperExecute
|
|
1590
|
+
cli: ./hyperexecute # TestMu's hyperexecute CLI
|
|
1591
|
+
devices: ["Pixel 6-14"] # name-version, or a pattern such as ".*-.*"
|
|
1592
|
+
real_device: true # false: TestMu's emulators
|
|
1593
|
+
</code></pre>
|
|
1594
|
+
<ul>
|
|
1595
|
+
<li><strong>Appium</strong> needs nothing else: it talks to the hub with the user and key from the two variables.</li>
|
|
1596
|
+
<li><strong>Maestro runs through HyperExecute.</strong> <code>mobile.maestro.testmu</code> names TestMu's <code>hyperexecute</code> CLI (<code>cli</code>), the devices
|
|
1597
|
+
to run on (<code>devices</code>), and real devices or emulators (<code>real_device</code>). ForgeQE writes a HyperExecute project in the run
|
|
1598
|
+
folder (the flows in their folder layout, so <code>runFlow</code> still works, the app, fixed setup and run scripts, and
|
|
1599
|
+
<code>hyperexecute.yaml</code>) and starts the CLI. Team flows are checked against the allowlist first, as always. The access key
|
|
1600
|
+
is removed from the CLI's output and from every file in the run folder; the run prints the HyperExecute job link.</li>
|
|
1601
|
+
<li><strong>Secrets:</strong> a workflow that types a secret is refused on Maestro on TestMu (HyperExecute would need the value in its
|
|
1602
|
+
own vault). Replay it on TestMu with <code>--driver appium</code> instead. Maestro on TestMu runs Android apps; run iOS apps
|
|
1603
|
+
there through Appium.</li>
|
|
1604
|
+
</ul>
|
|
1605
|
+
<p><a href="../samples/mobile-testmu/README.md">samples/mobile-testmu</a> is a ready project with TestMu's Wikipedia sample app: copy
|
|
1606
|
+
<code>.env.example</code> to <code>.env</code>, upload the app with <code>forgeqe mobile upload sample.apk</code>, put the id in its <code>forgeqe.yaml</code>,
|
|
1607
|
+
download the <code>hyperexecute</code> CLI into the folder, then run <code>forgeqe mobile replay wiki.yaml --driver appium</code>,
|
|
1608
|
+
<code>--driver maestro</code>, and <code>forgeqe mobile maestro .maestro</code>. Details: <a href="../docs/MOBILE.md#testmu-ai-formerly-lambdatest">docs/MOBILE.md</a>.</p>
|
|
1609
|
+
</div>
|
|
1610
|
+
</section>
|
|
1611
|
+
</div>
|
|
1612
|
+
</article>
|
|
1613
|
+
</section>
|
|
1614
|
+
<section class="chapter" id="area-operations" aria-labelledby="area-operations-h">
|
|
1615
|
+
<header class="chead"><span class="mono">09</span><h2 id="area-operations-h">Operations</h2><p>Monitoring and CI pipelines</p></header>
|
|
1616
|
+
<article class="spec feat" id="cap-containers">
|
|
1617
|
+
<div class="hd"><span class="mono">Operations</span><h3>Services and mocks in Docker or Podman</h3></div>
|
|
1618
|
+
<p class="ds">Starts the services a test run needs (a database, a queue, the API under test) in Docker or Podman, as forgeqe.yaml declares them, and waits until each is ready; stops them, and shows their status and logs. Starts mocks of an API on WireMock, from the calls a browser run recorded or from an OpenAPI description, and can point an API app of the environment at the mock for the session.</p>
|
|
1619
|
+
<div class="facts">
|
|
1620
|
+
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>Start the database and the queue in Docker</li><li>Start the services from our docker-compose file</li><li>Mock the payments API from the last run</li><li>Create a mock service from openapi.yaml and point order-api at it</li><li>Show the logs of the db container</li></ul></div>
|
|
1621
|
+
<div class="fact"><span class="mono">What it needs</span><ul class="needs"><li><span class="need req">Required</span><span>Docker or Podman on PATH; the services to start in containers.services in forgeqe.yaml (mocks need none)</span></li></ul></div>
|
|
1622
|
+
<div class="fact"><span class="mono">Limits</span><ul class="plainlist"><li>Starts only services declared in forgeqe.yaml containers.services; it never runs an image, flag or mount from a request</li><li>No privileged containers, host network, or access to the container runtime's socket; ports on 127.0.0.1 only; mounts read-only from inside the project</li><li>Mocks answer REST calls (WireMock); GraphQL, SOAP and gRPC are not mocked</li><li>Reads docker-compose files only for their services' image, ports, environment, bind mounts, command and depends_on; refuses build, privileged, network_mode, cap_add, devices and env_file; leaves named volumes out</li></ul></div>
|
|
1623
|
+
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>services</code> <code>mock_service</code></p></div>
|
|
1624
|
+
</div>
|
|
1625
|
+
<div class="detail"><span class="mono">In detail</span>
|
|
1626
|
+
<section class="doc" id="doc-how_to_use-containers-and-mock-services">
|
|
1627
|
+
<h4>Containers and mock services</h4>
|
|
1628
|
+
<div class="prose"><p>When Docker or Podman is installed, ForgeQE can start the services a test run needs, and mock an API the tests should
|
|
1629
|
+
not call. Ask in the shell: "start the database and the queue", "mock the payments API from the last run", "make a mock
|
|
1630
|
+
from openapi.yaml and point order-api at it", "show the db logs". Full reference: <a href="../docs/CONTAINERS.md">docs/CONTAINERS.md</a>.</p>
|
|
1631
|
+
<p>The project declares its services in forgeqe.yaml. ForgeQE starts only these, and builds each command itself:</p>
|
|
1632
|
+
<pre><code class="language-yaml">containers:
|
|
1633
|
+
# runtime: podman # default: docker, else podman, on PATH
|
|
1634
|
+
services:
|
|
1635
|
+
db:
|
|
1636
|
+
image: postgres:16.4 # an exact version or a digest; never latest
|
|
1637
|
+
ports: [5432] # or "15432:5432" (this machine:container); always 127.0.0.1
|
|
1638
|
+
env: { POSTGRES_DB: shop, POSTGRES_USER: shop }
|
|
1639
|
+
secrets: [POSTGRES_PASSWORD] # names of environment variables (.env works), passed by name
|
|
1640
|
+
volumes: ["./db/init:/docker-entrypoint-initdb.d"] # read-only, inside the project
|
|
1641
|
+
ready: { tcp: 5432 } # or { http: /health, port: 8080 }
|
|
1642
|
+
timeout: 90
|
|
1643
|
+
</code></pre>
|
|
1644
|
+
<pre><code class="language-bash">forgeqe services # declared services, what runs, whether Docker answers
|
|
1645
|
+
forgeqe services up db # start and wait until ready; keeps running after the command
|
|
1646
|
+
forgeqe services logs db --lines 50
|
|
1647
|
+
forgeqe services down # stop this project's containers, mocks included
|
|
1648
|
+
</code></pre>
|
|
1649
|
+
<ul>
|
|
1650
|
+
<li><strong>Mocks:</strong> <code>mock_service</code> builds WireMock stubs. From a run, they answer what the app really got. From an
|
|
1651
|
+
OpenAPI or Swagger description, or a contract set, they answer with its examples, or values made from its schemas. The mock
|
|
1652
|
+
runs on a free port of 127.0.0.1 (<code>wiremock/wiremock:3.9.1</code>, or <code>containers.mock.image</code>), and its stubs are in
|
|
1653
|
+
<code>.forgeqe/mocks/<name>/mappings</code>. With <code>for_app</code>, an API app of the active environment points at the mock for the
|
|
1654
|
+
rest of the session, so API tests and the tests ForgeQE runs (<code>FORGEQE_APP_<APP>_URL</code>) use it.</li>
|
|
1655
|
+
<li><strong>Session:</strong> what the agent starts (services or mocks) stops when the shell, <code>forgeqe run</code>, or <code>forgeqe mcp</code> ends.
|
|
1656
|
+
<code>forgeqe services up</code> keeps its services running until <code>forgeqe services down</code>.</li>
|
|
1657
|
+
<li><strong>Asked first:</strong> <code>services up</code> and <code>mock_service</code> ask before they start anything.</li>
|
|
1658
|
+
</ul>
|
|
1659
|
+
<p>Guard rails:</p>
|
|
1660
|
+
<ul>
|
|
1661
|
+
<li><strong>Fixed run settings:</strong> every container gets no new privileges, a process limit, ports on 127.0.0.1, read-only
|
|
1662
|
+
mounts, and labels with the project, so ForgeQE stops only its own containers.</li>
|
|
1663
|
+
<li><strong>Refused service settings:</strong> <code>privileged</code>, <code>network</code>, <code>cap_add</code>, <code>devices</code>, <code>user</code>, and anything else that is
|
|
1664
|
+
not a service setting. Also refused: a port bound to another address, an image without an exact version, and a
|
|
1665
|
+
secret that is one of ForgeQE's own keys.</li>
|
|
1666
|
+
<li><strong>Mounts:</strong> inside the project only. A mount that is or holds a hidden file (<code>.env</code>, <code>.git</code>), a file named like a key,
|
|
1667
|
+
or a link is refused, because a container can send what it reads anywhere.</li>
|
|
1668
|
+
<li><strong>The docker, podman and compose programs only see</strong> the variables they need to find their daemon, plus the service's
|
|
1669
|
+
listed secrets.</li>
|
|
1670
|
+
<li><strong>Not in the agents' shell:</strong> <code>docker</code>, <code>podman</code>, <code>nerdctl</code> and compose commands are refused for ForgeQE's agent and
|
|
1671
|
+
Scribe. Containers start only through these tools.</li>
|
|
1672
|
+
</ul>
|
|
1673
|
+
<p><strong>Compose files.</strong> <code>containers.compose: docker-compose.yml</code> reads the file's services into the same list, checked by the
|
|
1674
|
+
same rules (forgeqe.yaml's own declaration of a service wins):</p>
|
|
1675
|
+
<ul>
|
|
1676
|
+
<li><strong>Translated:</strong> <code>image</code>, <code>ports</code>, <code>environment</code>, bind mounts, <code>command</code> and <code>depends_on</code>. A <code>${NAME}</code> value becomes a
|
|
1677
|
+
secret passed by name.</li>
|
|
1678
|
+
<li><strong>Left out:</strong> named volumes. Mounts made read-only are listed in the notes.</li>
|
|
1679
|
+
<li><strong>Ignored:</strong> <code>container_name</code>, <code>restart</code>, <code>healthcheck</code> and <code>networks</code>.</li>
|
|
1680
|
+
<li><strong>Refused:</strong> <code>build</code>, <code>privileged</code>, <code>network_mode</code>, <code>cap_add</code>, <code>devices</code>, <code>pid</code> and <code>env_file</code>.</li>
|
|
1681
|
+
</ul>
|
|
1682
|
+
<p><code>x-forgeqe: { ready: … }</code> sets the ready check. Without it, a service is ready when its first port takes connections.
|
|
1683
|
+
The project's containers share their own network, so services reach each other by name (<code>db:5432</code>).</p>
|
|
1684
|
+
<p><strong>Tests in a container.</strong> With <code>containers.tests</code>, <code>run_tests</code> runs the repository's tests in that image instead of
|
|
1685
|
+
on this machine, after starting the services it lists:</p>
|
|
1686
|
+
<pre><code class="language-yaml">containers:
|
|
1687
|
+
tests:
|
|
1688
|
+
image: mcr.microsoft.com/playwright:v1.48.2-jammy
|
|
1689
|
+
services: [db]
|
|
1690
|
+
secrets: [TEST_DB_PASSWORD]
|
|
1691
|
+
</code></pre>
|
|
1692
|
+
<ul>
|
|
1693
|
+
<li><strong>What runs:</strong> the command you confirmed, inside the container, in the repository (mounted at its own path, so the
|
|
1694
|
+
result files land where ForgeQE reads them).</li>
|
|
1695
|
+
<li><strong>User:</strong> your user.</li>
|
|
1696
|
+
<li><strong>Variables:</strong> only the environment's <code>FORGEQE_*</code> variables, the saved logins' variables, and the listed env and
|
|
1697
|
+
secrets.</li>
|
|
1698
|
+
<li><strong>To run on this machine instead:</strong> ask to run the tests on this machine (<code>in_container: false</code>).</li>
|
|
1699
|
+
<li><strong>Machines:</strong> Linux and macOS.</li>
|
|
1700
|
+
</ul>
|
|
1701
|
+
<p><strong>Testcontainers.</strong> When a test needs a real database or queue, or the repository already uses Testcontainers, Scribe's
|
|
1702
|
+
<code>testcontainers</code> skill writes the test to start it from code. It reuses the repository's fixtures, pins images, waits
|
|
1703
|
+
for readiness, and uses the mapped port. It never uses privileged mode, the host network, or the Docker socket.</p>
|
|
1704
|
+
</div>
|
|
1705
|
+
</section>
|
|
1706
|
+
</div>
|
|
1707
|
+
</article>
|
|
1708
|
+
<article class="spec feat" id="cap-monitoring">
|
|
1709
|
+
<div class="hd"><span class="mono">Operations</span><h3>Scheduled monitoring with alerts</h3></div>
|
|
1710
|
+
<p class="ds">Runs checks on a schedule and alerts Teams or email when one keeps failing, and again when it recovers.</p>
|
|
1711
|
+
<div class="facts">
|
|
1712
|
+
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>Check the login flow every 15 minutes and alert Teams</li></ul></div>
|
|
1713
|
+
<div class="fact"><span class="mono">What it needs</span><ul class="needs"><li><span class="need req">Required</span><span>A live model profile in llms.json (an API key or Microsoft Entra ID sign-in)</span></li><li><span class="need opt">Optional</span><span>Alert channels: monitor.alerts.teams or monitor.alerts.email in forgeqe.yaml</span></li></ul></div>
|
|
1714
|
+
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>monitor</code></p></div>
|
|
1715
|
+
</div>
|
|
1716
|
+
<div class="detail"><span class="mono">In detail</span>
|
|
1717
|
+
<section class="doc" id="doc-how_to_use-9-monitoring">
|
|
1718
|
+
<h4>Monitoring</h4>
|
|
1719
|
+
<div class="prose"><p><code>forgeqe monitor</code> runs a set of checks on a schedule and alerts Microsoft Teams and email when one breaks. A check is any test case <code>forgeqe suite</code> takes (section 7): a YAML or Gherkin file, a folder, Jira keys or JQL, an Azure DevOps plan, TestRail, and so on.</p>
|
|
1720
|
+
<pre><code class="language-yaml">monitor:
|
|
1721
|
+
checks: [checks/smoke.yaml]
|
|
1722
|
+
every: 15m # 30s, 15m, 1h, 1h30m; at least 10s
|
|
1723
|
+
replay: verify # verify | trust | off
|
|
1724
|
+
parallel: 1
|
|
1725
|
+
fail_after: 1 # failing rounds in a row before the first alert
|
|
1726
|
+
remind_every: 2h # repeat while it keeps failing; default: never
|
|
1727
|
+
keep: 100 # round folders kept
|
|
1728
|
+
write_back: false # post each round's results to the source system
|
|
1729
|
+
# out_dir: ./.forgeqe/reports/monitor
|
|
1730
|
+
# report_url: https://reports.example.com/monitor
|
|
1731
|
+
alerts:
|
|
1732
|
+
teams:
|
|
1733
|
+
webhook_env: TEAMS_WEBHOOK_URL
|
|
1734
|
+
email:
|
|
1735
|
+
host: smtp.office365.com # or SMTP_HOST
|
|
1736
|
+
port: 587 # STARTTLS; 465 is TLS from the start (secure: true)
|
|
1737
|
+
username_env: SMTP_USERNAME
|
|
1738
|
+
password_env: SMTP_PASSWORD
|
|
1739
|
+
# from: ForgeQE <qa-bot@example.com>
|
|
1740
|
+
to: [qa-team@example.com, oncall@example.com]
|
|
1741
|
+
</code></pre>
|
|
1742
|
+
<pre><code class="language-bash">forgeqe monitor --test-alerts # send a sample alert through every channel
|
|
1743
|
+
forgeqe monitor # monitor.checks every monitor.every, until Ctrl+C
|
|
1744
|
+
forgeqe monitor checks/checkout.yaml --every 5m
|
|
1745
|
+
forgeqe monitor --once # one round, then exit (cron, scheduled pipelines)
|
|
1746
|
+
</code></pre>
|
|
1747
|
+
<p><strong>Each round</strong> runs the checks like a suite, without writing tests, and replays the flows from page memory (section 5). A flow that passed before costs one model call to confirm the expected results (<code>replay: verify</code>), or none (<code>replay: trust</code>, which trusts the replay without checking). A new or changed flow falls back to the full agent. Rounds start <code>every</code> apart; a round that overruns is followed by the next one straight away.</p>
|
|
1748
|
+
<p><strong>Alerts</strong> go out when something changes, not every round:</p>
|
|
1749
|
+
<ul>
|
|
1750
|
+
<li>A check that fails <code>fail_after</code> rounds in a row gets an alert. With <code>fail_after: 2</code> a single flaky failure is never reported.</li>
|
|
1751
|
+
<li>While it keeps failing, nothing more is sent, unless <code>remind_every</code> is set.</li>
|
|
1752
|
+
<li>When it passes again, a recovery alert says how long it was down.</li>
|
|
1753
|
+
<li>A round that cannot run at all (a test system is down, a source is empty, the engine will not start) alerts as the check "ForgeQE monitor", and the other checks keep their state.</li>
|
|
1754
|
+
</ul>
|
|
1755
|
+
<p>All changes in one round go out as one message. Teams gets an Adaptive Card with each check, why it failed and at which step, a link to the test case, and an "Open report" button. Email gets the same as text and HTML, with the failing checks' last screenshots attached (up to five).</p>
|
|
1756
|
+
<p>For Teams, create an incoming webhook on the channel, or a Workflows flow with the trigger "When a Teams webhook request is received" and the action "Post card in a chat or channel"; both take the card as it is. Put the URL in <code>TEAMS_WEBHOOK_URL</code>. For email, any SMTP server works: Office 365 (<code>smtp.office365.com:587</code>, with SMTP AUTH allowed for the mailbox), Gmail with an app password, SendGrid, or a relay on your network (no username needed). The webhook URL and the SMTP password stay in the environment; if a send fails, the error never contains them.</p>
|
|
1757
|
+
<p>The terminal shows one block per round:</p>
|
|
1758
|
+
<pre><code>Monitoring checks/smoke.yaml every 15m · replay verify · alerts: Teams, email to qa-team@example.com · Ctrl+C stops
|
|
1759
|
+
[14:15] ✗ 4/5 passing · 38s · 6.2K tokens
|
|
1760
|
+
✗ SHOP-12 Checkout: failing since 14:15
|
|
1761
|
+
The Pay button is missing on the payment page
|
|
1762
|
+
↑ Alert sent: Teams, email
|
|
1763
|
+
Report .forgeqe/reports/monitor/20260929-141500/report.html
|
|
1764
|
+
Next round at 14:30
|
|
1765
|
+
[14:30] ✗ 4/5 passing · 35s · 5.9K tokens
|
|
1766
|
+
✗ SHOP-12 Checkout: failing since 14:15 (2 rounds, already alerted)
|
|
1767
|
+
Report .forgeqe/reports/monitor/20260929-143000/report.html
|
|
1768
|
+
Next round at 14:45
|
|
1769
|
+
</code></pre>
|
|
1770
|
+
<p><strong>Files.</strong> Each round is a suite folder in <code>.forgeqe/reports/monitor/<time>/</code> (<code>report.html</code>, <code>junit.xml</code>, <code>suite.json</code>, the runs, and <code>alerts.json</code> when it alerted), so the report's trends and flaky tests cover the rounds. Folders beyond <code>keep</code> are deleted. <code>state.json</code> in <code>.forgeqe/reports/monitor/</code> remembers which checks are failing, since when, and when they were last alerted; delete it to start fresh. Set <code>report_url</code> to where you publish <code>.forgeqe/reports/monitor/</code> and alerts link <code><report_url>/<round>/report.html</code>; without it they name the local file.</p>
|
|
1771
|
+
<p><strong>Scheduling.</strong> Run <code>forgeqe monitor</code> in a long-lived session (tmux, a service, a container), or <code>forgeqe monitor --once</code> from cron or a scheduled pipeline. <code>--once</code> exits <code>0</code> when every check passed, <code>1</code> when any failed, and <code>2</code> when stopped. In a pipeline, cache <code>.forgeqe/reports/monitor/state.json</code> between runs; without it every run starts fresh, and a failure alerts every time. Only one monitor can use an <code>out_dir</code> at a time, so an overlapping cron run exits instead of racing.</p>
|
|
1772
|
+
<p>You can also ask ForgeQE's agent. In the shell, "check the smoke cases once" runs one round and shows it as a card, and "monitor the smoke cases every 10 minutes" starts a monitor in the background: you keep working, each round appears as a card, "is anything failing?" shows the state, and "stop monitoring" ends it. A background monitor lives as long as the shell (or <code>forgeqe mcp</code>) and stops when it closes; <code>forgeqe run</code> refuses to start one, because it would stop when the command exits. While one is running, a one-off round on the same <code>out_dir</code> is refused rather than run twice. For monitoring that keeps going after you close the shell, "monitor the smoke cases every 10 minutes and alert the QA channel in Teams, always on" makes the agent set up <code>monitor:</code> in <code>forgeqe.yaml</code>, send a test alert, and tell you how to keep <code>forgeqe monitor</code> running (or add a scheduled pipeline).</p>
|
|
1773
|
+
</div>
|
|
1774
|
+
</section>
|
|
1775
|
+
</div>
|
|
1776
|
+
</article>
|
|
1777
|
+
<article class="spec feat" id="cap-notifications">
|
|
1778
|
+
<div class="hd"><span class="mono">Operations</span><h3>Teams and email notifications</h3></div>
|
|
1779
|
+
<p class="ds">Tells the team in Microsoft Teams channels or by email when a suite fails or passes, when a release readiness verdict is in, and when the domain evaluation falls below its gate, with the failing cases, their bugs, and links to the report and the CI build. Each channel picks its events; the agent also posts a summary to a channel when asked.</p>
|
|
1780
|
+
<div class="facts">
|
|
1781
|
+
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>Post the results to the QA channel</li><li>Tell the release channel the verdict is NO-GO</li></ul></div>
|
|
1782
|
+
<div class="fact"><span class="mono">What it needs</span><ul class="needs"><li><span class="need req">Required</span><span>A notification channel: notify.channels in forgeqe.yaml, each a Teams Workflows webhook (in an environment variable) or email</span></li></ul></div>
|
|
1783
|
+
<div class="fact"><span class="mono">Limits</span><ul class="plainlist"><li>Automatic notifications come from runs nobody is watching (forgeqe suite, forgeqe run, pipelines, the MCP server), not from the interactive shell, where the agent posts only when asked.</li></ul></div>
|
|
1784
|
+
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>notify</code></p></div>
|
|
1785
|
+
</div>
|
|
1786
|
+
<div class="detail"><span class="mono">In detail</span>
|
|
1787
|
+
<section class="doc" id="doc-how_to_use-17-notifications">
|
|
1788
|
+
<h4>Notifications</h4>
|
|
1789
|
+
<div class="prose"><p>ForgeQE tells the team in Microsoft Teams channels, or by email, when something they should know happens. Each channel picks its events:</p>
|
|
1790
|
+
<pre><code class="language-yaml">notify:
|
|
1791
|
+
report_url: https://reports.example.com/forgeqe # where .forgeqe/reports is published, so cards link to each report
|
|
1792
|
+
channels:
|
|
1793
|
+
qa:
|
|
1794
|
+
teams: { webhook_env: TEAMS_WEBHOOK_URL }
|
|
1795
|
+
on: [suite_failed, eval_failed]
|
|
1796
|
+
release:
|
|
1797
|
+
teams: { webhook_env: TEAMS_RELEASE_WEBHOOK }
|
|
1798
|
+
on: [release]
|
|
1799
|
+
leads:
|
|
1800
|
+
email: { host: smtp.office365.com, port: 587, username_env: SMTP_USERNAME, password_env: SMTP_PASSWORD, to: [qa-leads@example.com] }
|
|
1801
|
+
on: [suite_failed, release]
|
|
1802
|
+
</code></pre>
|
|
1803
|
+
<div class="tbl"><table>
|
|
1804
|
+
<thead>
|
|
1805
|
+
<tr><th>Event</th><th>When</th><th>The card shows</th></tr>
|
|
1806
|
+
</thead>
|
|
1807
|
+
<tbody>
|
|
1808
|
+
<tr><td><code>suite_failed</code></td><td>A suite ends with a case that did not pass, or a test that did not</td><td>Each failing case (up to 10) with its kind of failure, the test case and bug links, tests written, and the report and build</td></tr>
|
|
1809
|
+
<tr><td><code>suite_passed</code></td><td>Every case passed</td><td>The count, how long it took, where it ran</td></tr>
|
|
1810
|
+
<tr><td><code>release</code></td><td>A release readiness verdict</td><td>GO, GO WITH RISKS, or NO-GO, the counts, every blocker and risk, and the wiki page when published</td></tr>
|
|
1811
|
+
<tr><td><code>eval_failed</code></td><td><code>forgeqe eval domain --min</code> falls below its gate</td><td>Coverage for each variant and invented ids</td></tr>
|
|
1812
|
+
</tbody>
|
|
1813
|
+
</table></div>
|
|
1814
|
+
<p>A channel without <code>on</code> gets <code>suite_failed</code>, <code>release</code>, and <code>eval_failed</code>. Email adds the failure screenshots. A stopped suite is not announced. Monitoring keeps its own <code>monitor.alerts</code>, with the same cards.</p>
|
|
1815
|
+
<p>Events are sent from runs nobody is watching: <code>forgeqe suite</code> in a pipeline or on a schedule, <code>forgeqe run</code>, and the MCP server. In the shell, where people try things out, nothing is sent on its own; ask instead ("post the results to the QA channel", "tell the release channel the verdict"), and ForgeQE's agent posts a card with <code>notify</code>.</p>
|
|
1816
|
+
<p>Links: with <code>report_url</code>, cards open the report where the reports folder is published. In Azure Pipelines and GitHub Actions they also open the build, where the report is an artifact. Without either, the card names the report's path.</p>
|
|
1817
|
+
<p><strong>Setting up a Teams channel:</strong></p>
|
|
1818
|
+
<ol>
|
|
1819
|
+
<li>In the channel, open <strong>Workflows</strong> and choose <strong>Post to a channel when a webhook request is received</strong>, then copy the URL it gives you. Microsoft is retiring the older Office 365 "Incoming Webhook" connectors; Workflows webhooks take the same cards.</li>
|
|
1820
|
+
<li>The URL is the secret (anyone with it can post): keep it in the environment, the system's credential store (<code>forgeqe auth login TEAMS_WEBHOOK_URL</code>), or a pipeline secret, never in <code>forgeqe.yaml</code>.</li>
|
|
1821
|
+
<li>Create the workflow with a shared or service account: a workflow belongs to whoever made it and stops when they leave. Your tenant must allow the Workflows app.</li>
|
|
1822
|
+
<li><code>forgeqe notify test [channel]</code> sends a test card to check it arrives.</li>
|
|
1823
|
+
</ol>
|
|
1824
|
+
</div>
|
|
1825
|
+
</section>
|
|
1826
|
+
</div>
|
|
1827
|
+
</article>
|
|
1828
|
+
<article class="spec feat" id="cap-pipelines">
|
|
1829
|
+
<div class="hd"><span class="mono">Operations</span><h3>CI pipelines</h3></div>
|
|
1830
|
+
<p class="ds">Writes a GitHub Actions or Azure Pipelines workflow that runs ForgeQE, stores the secrets it needs, pushes a branch, and makes the pipeline green.</p>
|
|
1831
|
+
<div class="facts">
|
|
1832
|
+
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>Add a GitHub Actions workflow that runs tests/cases.yaml on every PR, and make it green</li></ul></div>
|
|
1833
|
+
<div class="fact"><span class="mono">What it needs</span><ul class="needs"><li><span class="need req">Required</span><span>git on PATH</span></li><li><span class="need req">Required</span><span>gh on PATH, for GitHub Actions; or az on PATH, for Azure Pipelines</span></li></ul></div>
|
|
1834
|
+
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>push_branch</code> <code>set_ci_secret</code></p></div>
|
|
1835
|
+
</div>
|
|
1836
|
+
<div class="detail"><span class="mono">In detail</span>
|
|
1837
|
+
<section class="doc" id="doc-how_to_use-8-pipelines">
|
|
1838
|
+
<h4>Pipelines</h4>
|
|
1839
|
+
<div class="prose"><p>ForgeQE's agent writes, runs, and fixes GitHub Actions and Azure Pipelines. Run it in the repo (or pass <code>--repo</code>) and ask:</p>
|
|
1840
|
+
<pre><code class="language-bash">forgeqe run "Add a GitHub Actions workflow that runs the smoke cases in cases.yaml on every pull request"
|
|
1841
|
+
forgeqe run "Set up an Azure DevOps pipeline that runs test plan 12 every night and posts results back"
|
|
1842
|
+
forgeqe run "The ForgeQE workflow is failing, fix it"
|
|
1843
|
+
</code></pre>
|
|
1844
|
+
<p>What it does:</p>
|
|
1845
|
+
<ol>
|
|
1846
|
+
<li>Works out the platform from the git remote (github.com, or dev.azure.com / visualstudio.com) and reads the repo's existing CI, so it can add a job to it or write a new file.</li>
|
|
1847
|
+
<li>Writes the pipeline so it installs Bun and uv, runs <code>npm install -g @capagents/forgeqe</code> and <code>forgeqe setup</code>, then <code>forgeqe suite <source...> --no-code --plain --out <dir></code>, and publishes <code>junit.xml</code> and the report folder. On GitHub the suite adds a results table to the job summary.</li>
|
|
1848
|
+
<li>Validates it (actionlint when installed, otherwise by parsing the YAML) and commits it.</li>
|
|
1849
|
+
<li>Pushes to a new branch named <code>forgeqe/ci-<topic></code> with ForgeQE's <code>push_branch</code> tool. That tool refuses the default branch, <code>main</code>, and <code>master</code>, and never forces; <code>git push</code> itself is blocked for the agent. Merging stays with you.</li>
|
|
1850
|
+
<li>Stores the keys the pipeline needs (the model's key from <code>llms.json</code>, and <code>ADO_PAT</code>, <code>JIRA_EMAIL</code> and <code>JIRA_API_TOKEN</code>, or <code>XRAY_CLIENT_ID</code> and <code>XRAY_CLIENT_SECRET</code>) with ForgeQE's <code>set_ci_secret</code> tool. The agent only names the variable; ForgeQE passes the value from its own environment to <code>gh secret set</code> (on stdin) or <code>az pipelines variable</code> (through the environment), and never shows it. A variable that is not set in ForgeQE's environment is reported, not guessed.</li>
|
|
1851
|
+
<li>Triggers the run with <code>gh workflow run</code> or <code>az pipelines run</code>, watches it, reads the logs of failed steps, fixes the pipeline (or the code), pushes again, and repeats until the run passes or <code>code.max_attempts</code> runs have failed. It reports the run link and the result.</li>
|
|
1852
|
+
</ol>
|
|
1853
|
+
<p>It needs <code>gh</code> logged in for GitHub, or <code>az</code> with the <code>azure-devops</code> extension logged in for Azure DevOps. Without them it still writes, validates, and commits the pipeline, then tells you the commands to run. Set <code>llm.active</code> to a live profile in the committed <code>forgeqe.yaml</code>. With a <code>github_copilot</code> profile the pipeline needs only a GitHub Models token: in GitHub Actions, <code>permissions: models: read</code> and the workflow's <code>GITHUB_TOKEN</code> (see <a href="../#github-copilot-seats-as-the-model">In pipelines</a>).</p>
|
|
1854
|
+
</div>
|
|
1855
|
+
</section>
|
|
1856
|
+
</div>
|
|
1857
|
+
</article>
|
|
1858
|
+
</section>
|
|
1859
|
+
<section class="chapter" id="area-automated-tests" aria-labelledby="area-automated-tests-h">
|
|
1860
|
+
<header class="chead"><span class="mono">10</span><h2 id="area-automated-tests-h">Automated tests</h2><p>Write, heal, find, run, and stabilise tests in your repository</p></header>
|
|
1861
|
+
<article class="spec feat" id="cap-write-tests">
|
|
1862
|
+
<div class="hd"><span class="mono">Automated tests</span><h3>Turn a passed run into an automated test</h3></div>
|
|
1863
|
+
<p class="ds">Scribe, the coding agent, writes a test for a passed browser run in the repository's own framework, page objects, and helpers; ForgeQE runs it until it passes and a reviewer approves it.</p>
|
|
1864
|
+
<div class="facts">
|
|
1865
|
+
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>Write a Playwright test for that run in ../app</li></ul></div>
|
|
1866
|
+
<div class="fact"><span class="mono">What it needs</span><ul class="needs"><li><span class="need req">Required</span><span>The coding agent, Scribe: a live model profile (or code.model) and the agent runtime, which forgeqe setup installs</span></li><li><span class="need input">You give</span><span>a repository to write the test in (default: the working directory)</span></li></ul></div>
|
|
1867
|
+
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>write_test</code> <code>run_test</code></p></div>
|
|
1868
|
+
</div>
|
|
1869
|
+
<div class="detail"><span class="mono">In detail</span>
|
|
1870
|
+
<section class="doc" id="doc-how_to_use-6-tests">
|
|
1871
|
+
<h4>Tests</h4>
|
|
1872
|
+
<div class="prose"><p>After a run passes, <strong>Scribe</strong>, ForgeQE's coding agent, turns it into a test in your repo.</p>
|
|
1873
|
+
<p>In the shell and <code>forgeqe run</code>, every passed run then becomes a test, and so does each test case that passes when you ask the shell to run test cases: in the current folder, or in the repo you give per run or in <code>forgeqe.yaml</code>. <code>--no-code</code> turns it off; you can still ask for a test ("write a Playwright test for that run").</p>
|
|
1874
|
+
<pre><code class="language-bash">forgeqe run "Log in and open the invoices page on staging.app.test" --repo ../app
|
|
1875
|
+
</code></pre>
|
|
1876
|
+
<pre><code class="language-yaml">code:
|
|
1877
|
+
repo: ../app # relative to forgeqe.yaml
|
|
1878
|
+
max_attempts: 5 # test runs before giving up
|
|
1879
|
+
test_timeout_seconds: 600 # per test run
|
|
1880
|
+
stall_after: 5 # stop the agent's turn when one command it runs has failed this many times
|
|
1881
|
+
replay: true # before the agent starts, run the steps as a Playwright script and hand over what worked
|
|
1882
|
+
live_browser: true # once a step fails, the agent looks at that page in a Playwright browser
|
|
1883
|
+
graph: true # the code graph: find page objects without reading files, and reuse what already locates an element
|
|
1884
|
+
# model: anthropic/claude-sonnet-4-5
|
|
1885
|
+
# runtime: /path/to/agent-runtime # default: the runtime ForgeQE downloads
|
|
1886
|
+
</code></pre>
|
|
1887
|
+
<p>What happens after the run passes:</p>
|
|
1888
|
+
<ol>
|
|
1889
|
+
<li>
|
|
1890
|
+
<p>ForgeQE writes <code>handoff.md</code> into the run directory: the goal and result, how ForgeQE's browser was set up (browser and version, headless or headed, user agent, <code>navigator.webdriver</code>, launch switches, extra headers, language, time zone, viewport), every step in order with the value typed (secrets as <code>process.env.NAME</code>) and the JavaScript the browser agent ran, the browser agent's own note before each group of steps (what it saw and why it acted, so a sign-in pop-up it closed or steps it redid after a banner covered the page read as such), each element's best locator and the others with their confidence %, and a short table of each visited page's other elements for assertions (weak locators under 30% are left out, and elements the steps already use are listed with the steps, so the handoff stays small; the full lists are in the page-memory files it points to). The agent's result and the final URL are marked as coming from one run on live data, to be checked as rules (the results load, the order holds, the fields have the right format), not copied as expected values. Pop-ups and banners the run met are handled wherever they appear (Playwright's locator handlers or the repository's helper), since the next run can show them at another moment or not at all; the agent's retries are not copied. A site can also serve different versions of a page to different visits (A/B tests, such as a classic search box or a free-text one), so the test handles every version the run or the agent saw, or waits for one control from either version.</p>
|
|
1891
|
+
<p>The handoff also lists the browser agent's <strong>checks</strong>: before it finishes, the browser agent records each result the goal asks to verify with the exact text the page showed (<code>record_check</code>; a text the page does not show is refused). They are saved in <code>workflow.yaml</code> under <code>checks:</code>. The test asserts these and nothing else.</p>
|
|
1892
|
+
</li>
|
|
1893
|
+
<li>
|
|
1894
|
+
<p>ForgeQE replays the run's steps as a Playwright script (<code><run>/replay/flow.mjs</code>), in the run's browser setup (channel, headless or headed, user agent, language, time zone, viewport), with 10 seconds per step. Text is typed key by key, as the browser agent typed it, so search suggestions and other typing handlers fire. If the replay does not reach the run's final page, it runs again without Playwright's automation switches (<code>navigator.webdriver</code> false, as in ForgeQE's browser). The handoff gets the result: which steps worked, and for each failed step, Playwright's reason (for example a dialog that intercepts the click), the page, the dialogs open on it, how many elements each of the element's other locators matched, and a screenshot and page snapshot. It also saves Playwright's page snapshot of each page the replay visited (roles, accessible names, and text, as the replay left the page; the last page once it stops changing), so Scribe takes locators for its steps and assertions from real pages instead of guessing them. On the last page it looks up each of the browser agent's checks and puts the Playwright locator that matched its text into the handoff (a heading or text match, the container it sits in when there are several, or the form field that holds it), so assertions come with a locator that worked. For a heading it also gives the part of the page the heading names (an element labelled by it, the section it heads, or the element right after it), with its locator and how many links and list items it holds, so a check such as "an In the news section with at least one item" is asserted inside that part, not on the heading alone. A step that fails while every step after it still works and the replay reaches the final page (a click that did nothing on the run, or a control this version of the page doesn't have) doesn't count as a failed replay: Scribe decides from the step whether the goal needs it and ports the rest as it is. When only the second launch got through, the handoff says the browser setup is the difference and which launch settings fix it. The first time, Playwright (about 13 MB) is downloaded into <code>~/.forgeqe/tools</code>; it drives the installed Chrome. A run that only opened a page still replays when it recorded checks, to look them up; with neither it skips this, and <code>replay: false</code> turns it off.</p>
|
|
1895
|
+
<p>Scribe first reads the repo (as a read-only agent: it cannot change files or use the live browser yet), while the replay runs when there is one, so the two overlap. What it learns (the framework, the command that runs one test, where page objects and config live, the conventions) is saved with page memory in <code>scribe-notes.json</code>, with the tests each job writes there. The next job in the same repo starts from those notes and that list instead of reading the repo again, and extends those tests' page objects when its flow overlaps; notes older than 30 days are read again.</p>
|
|
1896
|
+
</li>
|
|
1897
|
+
<li>
|
|
1898
|
+
<p>Scribe starts in the repo. It reads <code>AGENTS.md</code>, the test framework config, run scripts, page objects, fixtures, and helpers, and follows the imports between them. It writes the test in the repo's language and framework (a C# NUnit repo gets a C# NUnit test) and to fit its code: it extends an existing page object instead of duplicating it and reuses the login flow, test data, and config. A repo with no tests gets Playwright Test in TypeScript. It may change anything the test needs, including dependencies and config.</p>
|
|
1899
|
+
<p>Before Scribe starts, ForgeQE reads the repo into a <strong>code graph</strong>, without a model: every class, page object, method, property, and test (TypeScript, JavaScript, C#, Java, Python), the calls between them, and the locators each one builds, in any framework's spelling (Playwright, Selenium, Cypress, WebdriverIO, Testing Library). It is kept in <code>~/.forgeqe/cache/code-graph</code>, nothing leaves the machine, and the next job reads again only the files that changed. The graph is tied to page memory, which knows each element by all its locators (its id, role and name, text, test id): so when the run clicked <code>getByRole('button', { name: 'Login' })</code> and the repo's <code>LoginPage.LoginButton</code> uses <code>#login-button</code>, the handoff says under that step "in this repository: LoginPage.LoginButton … used by LoginPage.SignInAsync", and Scribe calls the existing method instead of writing a second locator. Scribe also gets tools that answer from the graph instead of grep and whole-file reads: <code>code_find</code> (classes, methods, and tests by name, or by the text a locator uses), <code>code_outline</code> (a file's or class's signatures and locators, or the source of one member), <code>code_element</code> (which member already locates an element), <code>code_references</code> (what calls a member, before it is changed), and <code>code_impact</code> (the tests a change reaches, through flows and base classes, whatever the test files are called). <code>graph: false</code> turns it off. The handoff also outlines the page objects its steps use and includes the repo's test that uses the most of them, so Scribe takes the test conventions from that test instead of reading other tests, and reads a settings file only to change it.</p>
|
|
1900
|
+
</li>
|
|
1901
|
+
<li>
|
|
1902
|
+
<p>It writes the test from the handoff and the replay script: steps that worked are ported with their locators, steps that failed are fixed first. Every locator, including those for assertions, comes from the replay script, the page snapshots, page memory, or a check on the page; steps where the browser agent ran JavaScript to look around are left out. It does not walk the flow again or build tests to explore. Once a step fails (in the replay, or in its test), ForgeQE switches on its live browser: a Chrome that Playwright drives, set up like ForgeQE's, where it looks at the failing step's page one tool call per action (open, snapshot, click, type, wait, screenshot, run JavaScript, console errors, failed requests). Until then the live browser tools are off, so no tokens go into looking at pages that work. The live browser needs Node.js (<code>npx</code>), and Microsoft's Playwright MCP server is downloaded on first use; with <code>live_browser: false</code>, or without <code>npx</code>, it looks with a small throwaway script instead. When its browser gets a different page than ForgeQE's did (an error, a challenge, a control that will not open), it compares the two browsers (user agent, <code>navigator.webdriver</code>, headless, failed requests and CORS errors, headers the framework adds, console errors) and fixes the difference in the repo's browser configuration. It never judges the site with <code>curl</code> or <code>webfetch</code>.</p>
|
|
1903
|
+
</li>
|
|
1904
|
+
<li>
|
|
1905
|
+
<p>Scribe does not run the test itself: it may build or compile, then reports the test file and the command that runs only that test, and ForgeQE runs it. If one command it runs fails <code>stall_after</code> times in a row within its work (the same build again and again, say), ForgeQE stops that turn and sends the command and its last output back, asking it to change approach. If the same command stalls again, the job stops as a failure with that output in the notes.</p>
|
|
1906
|
+
</li>
|
|
1907
|
+
<li>
|
|
1908
|
+
<p>ForgeQE runs that command (with <code>CI=1</code>) and checks the exit code, and it watches the processes the command starts for a browser launched by Playwright, Puppeteer, Selenium, or Cypress. On a failure it sends the command, exit code, and output back to Scribe. The output is kept short so the error is never cut off: the page dump Playwright puts into a failed assertion ("Aria snapshot:", often thousands of lines) is taken out and saved as <code><run>/test-runs/attempt-N-page.aria.yml</code>, then the start (where the first error and its stack trace are) and the end (the summary) are sent; the full output is in <code><run>/test-runs/attempt-N.log</code>, and the prompt names both files. A command that exits 0 without starting a browser also goes back: the test must drive the app in a browser. If the repo's browser runs on another machine (Selenium Grid, a cloud service, a container), the agent says where and the check is skipped.</p>
|
|
1909
|
+
</li>
|
|
1910
|
+
<li>
|
|
1911
|
+
<p>A passing test then goes to a separate reviewer agent, which can read files and <code>git diff</code> but not change anything. It checks the change against the goal and the test rules: every part of the goal done through the page and asserted, generic page objects with data in the test, no values or URLs copied from the run, browser settings in the repo's configuration, no leftover files. The changed files come inline in its prompt, so it reads nothing again, and it does not ask to replace locators or texts that came from the replay or the checks, since those matched the live site. It judges only what this job changed: ForgeQE records the working tree before the job, so your uncommitted edits, your untracked files, and ForgeQE's own files (<code>.forgeqe/</code>, <code>forgeqe.yaml</code>, <code>llms.json</code>) are never part of the review. It checks the diff of every file that existed before the job: removing or rewriting an existing test, even a commented-out one, is a correctness problem. Credentials come from the environment, except a public demo login that the application itself shows to anyone (such as the logins on saucedemo.com's sign-in page), which may stay in the repo's test data. It gets the outlines of the classes the changed code uses, the repo's browser settings, and the same <code>code_*</code> tools, so it checks reuse and settings without reading those files. The code graph also tells it what the job added that it can check in the code: a new member that locates an element an existing member already locates (to reuse instead), and new XPath or position-based locators. If it finds problems, they go back to Scribe as the next attempt. The terminal shows the exchange as it happens, in order: the reviewer's verdict with its reason and each issue (✗ correctness, ~ coding standard) with the file and the fix it asks for; then why Scribe tries again and what it is fixing (the reviewer's issues, or the test's first error); then what Scribe says it did and which files it changed; then the next verdict:</p>
|
|
1912
|
+
</li>
|
|
1913
|
+
</ol>
|
|
1914
|
+
<pre><code>⏺ Reviewer asked for changes attempt 1 of 5 · 1 to fix, 1 coding-standard
|
|
1915
|
+
⎿ The test never checks that the pie chart is shown.
|
|
1916
|
+
✗ Correctness · tests/PracticeTests.cs: No assertion on the SVG pie chart.
|
|
1917
|
+
Fix: Assert the chart's heading and SVG are visible.
|
|
1918
|
+
~ Coding standard · tests/PracticeTests.cs: Test data is inline.
|
|
1919
|
+
Fix: Move it to the page object.
|
|
1920
|
+
⏺ Scribe tries again · attempt 2 of 5 the reviewer asked for changes
|
|
1921
|
+
⎿ → tests/PracticeTests.cs: No assertion on the SVG pie chart. — fix: Assert the chart's heading and SVG are visible.
|
|
1922
|
+
⎿ Scribe (attempt 2): Added the pie chart assertion · changed tests/PracticeTests.cs, Pages/PracticePage.cs
|
|
1923
|
+
⏺ Reviewer approved the test attempt 2 of 5
|
|
1924
|
+
</code></pre>
|
|
1925
|
+
<p>The Activities panel has one line for each verdict and attempt, and <code>forgeqe</code> commands outside the terminal UI print the same blocks.</p>
|
|
1926
|
+
<p>Scribe cannot edit ForgeQE's files or run folders, and cannot run <code>git clean</code>, <code>git reset --hard</code>, <code>git stash</code>, <code>git restore</code>, or <code>git checkout -- <path></code>. If it still removes a file that existed before the job, or reverts your uncommitted edits, ForgeQE puts the file back after that attempt and tells the agent.
|
|
1927
|
+
8. The job is done when ForgeQE's run passes with a browser and the reviewer approves. It stops as a failure after <code>max_attempts</code> attempts, when the job passes <code>code.max_tokens</code>, when the reviewer rejects a passing test twice, if the review cannot finish, when the test fails the same way three times in a row (the same output apart from numbers such as timings and line numbers; the second time, Scribe is told its change did not change the failure and to look at the failing step before changing code), or straight away on an auth, API, or context-limit error from the model.</p>
|
|
1928
|
+
<p><code>blocked</code> is a last resort. The first time the agent reports that the app blocks it, ForgeQE sends it back to compare its browser with ForgeQE's and fix the difference; only a second blocked report, with what it compared, ends the job as blocked.</p>
|
|
1929
|
+
<p>The transcript shows the handover and Scribe's work as it goes; files it writes or edits are marked ✎:</p>
|
|
1930
|
+
<pre><code>┃
|
|
1931
|
+
┃ ▸ Handing over to the coding agent browser agent → coding agent
|
|
1932
|
+
┃
|
|
1933
|
+
┃ Flow Log in and open the invoices page on staging.app.test
|
|
1934
|
+
┃ Run .forgeqe/reports/20260928-172700/
|
|
1935
|
+
┃ Repo ../app
|
|
1936
|
+
┃
|
|
1937
|
+
▸ Handoff written: .forgeqe/reports/20260928-172700/handoff.md (7 steps, 3 pages from page memory)
|
|
1938
|
+
▸ Scribe, the coding agent (azure/gpt-5.4), reads the handoff and writes a test in /work/app
|
|
1939
|
+
⚙ Reading .forgeqe/reports/20260928-172700/handoff.md
|
|
1940
|
+
⚙ Reading tests/pages/LoginPage.ts
|
|
1941
|
+
✎ Editing tests/invoices.spec.ts
|
|
1942
|
+
⚙ Running npx playwright test tests/invoices.spec.ts
|
|
1943
|
+
▸ ForgeQE runs the test: npx playwright test tests/invoices.spec.ts
|
|
1944
|
+
✗ tests/invoices.spec.ts failed, exit 1 (attempt 1/5)
|
|
1945
|
+
▸ tests/invoices.spec.ts passed in 9s in Google Chrome; a reviewer checks it against the goal
|
|
1946
|
+
✓ tests/invoices.spec.ts passed and was approved (attempt 2/5)
|
|
1947
|
+
┃
|
|
1948
|
+
┃ ✓ Test passed 2/5 attempts · 3m 12s · 88.4K tokens · azure/gpt-5.4
|
|
1949
|
+
┃
|
|
1950
|
+
┃ Added openInvoices() to DashboardPage and reused the loginAs fixture.
|
|
1951
|
+
┃
|
|
1952
|
+
┃ Test tests/invoices.spec.ts (in ../app)
|
|
1953
|
+
┃ Command npx playwright test tests/invoices.spec.ts
|
|
1954
|
+
┃ Reused LoginPage, fixtures/auth.ts
|
|
1955
|
+
┃ Changed tests/invoices.spec.ts, tests/pages/DashboardPage.ts
|
|
1956
|
+
┃ Browser Google Chrome
|
|
1957
|
+
┃ Review approved: Opens the invoices page through the dashboard and checks the table columns.
|
|
1958
|
+
</code></pre>
|
|
1959
|
+
<p>After ForgeQE's reply, the request ends with the files it produced:</p>
|
|
1960
|
+
<pre><code>┃ ✓ What this produced
|
|
1961
|
+
┃
|
|
1962
|
+
┃ Report .forgeqe/reports/20260928-172700/report.html
|
|
1963
|
+
┃ ✓ Test ../app/tests/invoices.spec.ts passed when ForgeQE ran it
|
|
1964
|
+
┃ Command npx playwright test tests/invoices.spec.ts
|
|
1965
|
+
┃ Handoff .forgeqe/reports/20260928-172700/handoff.md
|
|
1966
|
+
</code></pre>
|
|
1967
|
+
<p>When no test was written, the Test row says why: the run did not pass, the session has <code>--no-code</code>, or the run was not handed to the coding agent.</p>
|
|
1968
|
+
<p>Scribe uses the active <code>llms.json</code> profile. The key stays in your environment: the agent runtime's config refers to it as <code>{env:WP_AZURE_API_KEY}</code> and never contains the value. An <code>auth: entra</code> profile reaches the model through ForgeQE's local token gateway (see <a href="../#signing-in-to-the-model-without-a-key-microsoft-entra-id">Signing in to the model without a key</a>). An Azure profile goes through the agent runtime's <code>azure</code> provider on <code><endpoint>/openai/v1</code> (the Responses API, with the deployment as the model). <code>openai_compatible</code> and <code>ollama</code> profiles use their <code>base_url</code>. A <code>github_copilot</code> profile goes through the agent runtime's <code>github-copilot</code> provider with the profile's <code>model</code>, signed in with <code>forgeqe auth login copilot</code>. <code>code.model: provider/model</code> (or <code>--code-model</code>) uses one of the agent runtime's built-in providers instead, with that provider's usual key variable (for example <code>ANTHROPIC_API_KEY</code> for <code>anthropic/...</code>).</p>
|
|
1969
|
+
<p>Scribe may edit files and run commands anywhere in the repo without asking. <code>git push</code> in any form is denied (pushing goes through ForgeQE's guarded <code>push_branch</code> tool, section 8), and it cannot touch files outside the repo except the run and memory directories, which it may read.</p>
|
|
1970
|
+
<p>More ways to run it:</p>
|
|
1971
|
+
<pre><code class="language-bash">forgeqe code --repo ../app # the latest run in .forgeqe/reports/
|
|
1972
|
+
forgeqe code .forgeqe/reports/20260928-172700 --repo ../app --attempts 3
|
|
1973
|
+
forgeqe run "..." --no-code # no tests for this request
|
|
1974
|
+
</code></pre>
|
|
1975
|
+
<p>In the shell, <code>forgeqe start --repo ../app</code> writes the tests in that repo instead of the current folder and <code>--no-code</code> turns them off; you can also just ask ("write a test for the last run in ../app"). Esc stops the agent the same way it stops a run.</p>
|
|
1976
|
+
<p>Exit codes for <code>forgeqe code</code> and for <code>forgeqe run --direct</code> with a repo: <code>0</code> test passed, <code>1</code> run or test failed, <code>2</code> stopped. The result (status, attempts, test file, command, what was reused and changed, the browsers ForgeQE saw, the review, the last test output, tokens) is saved as <code>code.json</code> in the run directory, next to <code>handoff.md</code>.</p>
|
|
1977
|
+
<p>If Scribe's job is cut off (ForgeQE closed, the machine restarted, you stopped it, or the model could not be reached), run it again for the same run and repo, for example <code>forgeqe code .forgeqe/reports/20260928-172700 --repo ../app</code>. Scribe reopens the same session, with everything it read and said, and carries on with the files it already wrote; it doesn't read the repo again, and ForgeQE uses the Playwright replay it already finished. The session is kept in <code>scribe-session.json</code> in the run folder until the job ends with a result.</p>
|
|
1978
|
+
</div>
|
|
1979
|
+
</section>
|
|
1980
|
+
</div>
|
|
1981
|
+
</article>
|
|
1982
|
+
<article class="spec feat" id="cap-heal-tests">
|
|
1983
|
+
<div class="hd"><span class="mono">Automated tests</span><h3>Heal failing tests</h3></div>
|
|
1984
|
+
<p class="ds">Runs a repository's tests and updates the ones the application has outgrown against the live app, or reports a real application bug instead.</p>
|
|
1985
|
+
<div class="facts">
|
|
1986
|
+
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>Heal the failing tests in ../app</li></ul></div>
|
|
1987
|
+
<div class="fact"><span class="mono">What it needs</span><ul class="needs"><li><span class="need req">Required</span><span>The coding agent, Scribe: a live model profile (or code.model) and the agent runtime, which forgeqe setup installs</span></li><li><span class="need input">You give</span><span>a repository with tests</span></li></ul></div>
|
|
1988
|
+
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>heal_tests</code></p></div>
|
|
1989
|
+
</div>
|
|
1990
|
+
<div class="detail"><span class="mono">In detail</span>
|
|
1991
|
+
<section class="doc" id="doc-how_to_use-fixing-broken-tests">
|
|
1992
|
+
<h4>Fixing broken tests</h4>
|
|
1993
|
+
<div class="prose"><p>When a redesign or a renamed button breaks tests, <code>forgeqe heal</code> fixes them, or tells you the app is what broke:</p>
|
|
1994
|
+
<pre><code class="language-bash">forgeqe heal --repo ../app # the command from code.test_command, or the agent finds it
|
|
1995
|
+
forgeqe heal "npx playwright test tests/checkout" --repo ../app
|
|
1996
|
+
forgeqe heal -- npx playwright test --project=chromium # everything after -- is the command
|
|
1997
|
+
</code></pre>
|
|
1998
|
+
<pre><code class="language-yaml">code:
|
|
1999
|
+
repo: ../app
|
|
2000
|
+
test_command: npx playwright test # optional
|
|
2001
|
+
</code></pre>
|
|
2002
|
+
<ol>
|
|
2003
|
+
<li>ForgeQE runs the command. If it passes, there is nothing to heal and no model is called.</li>
|
|
2004
|
+
<li>Scribe starts in the repo as a healing agent, with the failing output. For each failing test it reads the test and the page objects it uses, then re-runs the test's flow in the real browser with ForgeQE's <code>browser_run</code> (the goal in plain words, not the old selectors).</li>
|
|
2005
|
+
<li>If the browser run passes, the test is out of date. The agent reads that run's steps and locators with <code>run_details</code>, updates the test (in the page object when the locators live there), and runs it with <code>run_test</code>.</li>
|
|
2006
|
+
<li>If the browser run fails at the step where the test fails, the app is broken there. The test is left as it is and the failure is reported as an app bug.</li>
|
|
2007
|
+
<li>It never deletes or skips a test, loosens an assertion, or raises a timeout to make a test pass.</li>
|
|
2008
|
+
<li>ForgeQE runs each healed test's own command, then the whole command. A healed test that still fails goes back to the agent with the output, and so does any other failure left in the whole run, until everything passes or <code>code.max_attempts</code> rounds are used.</li>
|
|
2009
|
+
</ol>
|
|
2010
|
+
<pre><code> ▸ ForgeQE runs the tests: npx playwright test
|
|
2011
|
+
✗ Tests failed, exit 1 (41s)
|
|
2012
|
+
▸ Scribe (azure/gpt-5.4) is healing the failing tests in app
|
|
2013
|
+
⚙ read tests/pages/LoginPage.ts
|
|
2014
|
+
▸ Browser: Sign in to staging.app.test and open the invoices page
|
|
2015
|
+
✓ Task complete · 9 steps
|
|
2016
|
+
⚙ edit tests/pages/LoginPage.ts
|
|
2017
|
+
▸ ForgeQE runs: npx playwright test tests/login.spec.ts
|
|
2018
|
+
✓ login › signs in passed in 6s
|
|
2019
|
+
▸ ForgeQE runs the tests: npx playwright test
|
|
2020
|
+
✗ Tests still fail, exit 1 (39s)
|
|
2021
|
+
┃
|
|
2022
|
+
┃ ✗ Tests still failing 2/5 attempts · 2m 40s · 61.2K tokens · azure/gpt-5.4
|
|
2023
|
+
┃
|
|
2024
|
+
┃ ✓ login › signs in healed
|
|
2025
|
+
┃ The Sign in button is now "Log in"; LoginPage uses the new label
|
|
2026
|
+
┃ ! invoices › totals app bug
|
|
2027
|
+
┃ The total column is empty; the browser run failed at the same step
|
|
2028
|
+
┃
|
|
2029
|
+
┃ 1 failure is in the app, not the tests; the tests were left as they are.
|
|
2030
|
+
┃ Command npx playwright test (exit 1 after 39s)
|
|
2031
|
+
┃ Changed tests/pages/LoginPage.ts
|
|
2032
|
+
┃ Report .forgeqe/reports/heal/20260929-101500/heal.md
|
|
2033
|
+
</code></pre>
|
|
2034
|
+
<p>The report is <code>heal.md</code> and <code>heal.json</code> in <code>.forgeqe/reports/heal/<time>/</code>: each test's verdict and reason, the files changed, the browser runs (their folders sit next to the report), and the test output before and after. Exit codes: <code>0</code> the tests pass (or nothing needed healing), <code>1</code> some still fail, <code>2</code> stopped. In the shell or with <code>forgeqe run</code>, "the checkout tests are failing after the redesign, fix them" does the same through ForgeQE's agent.</p>
|
|
2035
|
+
</div>
|
|
2036
|
+
</section>
|
|
2037
|
+
</div>
|
|
2038
|
+
</article>
|
|
2039
|
+
<article class="spec feat" id="cap-find-run-tests">
|
|
1460
2040
|
<div class="hd"><span class="mono">Automated tests</span><h3>Find and run existing tests</h3></div>
|
|
1461
2041
|
<p class="ds">Finds the automated tests that match a feature, tags, ids, a folder, or a sprint, confirms the selection, and runs it once the user agrees, with a verdict per test.</p>
|
|
1462
2042
|
<div class="facts">
|
|
@@ -1586,7 +2166,7 @@ Build a pack for Jira fix version 2.4 and the commits since release/2.3 in ../sh
|
|
|
1586
2166
|
</article>
|
|
1587
2167
|
</section>
|
|
1588
2168
|
<section class="chapter" id="area-pull-requests" aria-labelledby="area-pull-requests-h">
|
|
1589
|
-
<header class="chead"><span class="mono">
|
|
2169
|
+
<header class="chead"><span class="mono">11</span><h2 id="area-pull-requests-h">Pull requests</h2><p>Reviews, impact, and the tests a change can break</p></header>
|
|
1590
2170
|
<article class="spec feat" id="cap-pr-review">
|
|
1591
2171
|
<div class="hd"><span class="mono">Pull requests</span><h3>Review pull requests</h3></div>
|
|
1592
2172
|
<p class="ds">Reviews an Azure DevOps or GitHub pull request from the local clone for bugs, security, requirements, and tests, and posts the review once the user agrees.</p>
|
|
@@ -1658,7 +2238,7 @@ forgeqe pr-tests 42 --run --post # and post the results on p
|
|
|
1658
2238
|
</article>
|
|
1659
2239
|
</section>
|
|
1660
2240
|
<section class="chapter" id="area-api" aria-labelledby="area-api-h">
|
|
1661
|
-
<header class="chead"><span class="mono">
|
|
2241
|
+
<header class="chead"><span class="mono">12</span><h2 id="area-api-h">API</h2><p>Contracts and end-to-end scenarios, without a browser</p></header>
|
|
1662
2242
|
<article class="spec feat" id="cap-api-testing">
|
|
1663
2243
|
<div class="hd"><span class="mono">API</span><h3>API and contract testing</h3></div>
|
|
1664
2244
|
<p class="ds">Builds API contracts (status, schema, key fields) from the calls a browser run saw or from an API description (OpenAPI, Swagger, WSDL, GraphQL, .proto, gRPC reflection), and end-to-end API scenarios from runs, descriptions, test cases, or plain English. Runs them straight against the API without a browser, or writes them as tests in the repository. REST, GraphQL, SOAP, and gRPC.</p>
|
|
@@ -1736,7 +2316,7 @@ forgeqe api tests --scenarios --repo ../api-tests # on the repo's exi
|
|
|
1736
2316
|
</article>
|
|
1737
2317
|
</section>
|
|
1738
2318
|
<section class="chapter" id="area-quality" aria-labelledby="area-quality-h">
|
|
1739
|
-
<header class="chead"><span class="mono">
|
|
2319
|
+
<header class="chead"><span class="mono">13</span><h2 id="area-quality-h">Quality</h2><p>Accessibility, page performance, and visual changes</p></header>
|
|
1740
2320
|
<article class="spec feat" id="cap-accessibility-performance">
|
|
1741
2321
|
<div class="hd"><span class="mono">Quality</span><h3>Accessibility and page performance</h3></div>
|
|
1742
2322
|
<p class="ds">Checks every page a run visits against WCAG rules (axe-core) and measures its Core Web Vitals, against budgets, with trends over time per page.</p>
|
|
@@ -1854,7 +2434,7 @@ forgeqe visual approve --page /home # make this run's /home the baseline (asks
|
|
|
1854
2434
|
</article>
|
|
1855
2435
|
</section>
|
|
1856
2436
|
<section class="chapter" id="area-release" aria-labelledby="area-release-h">
|
|
1857
|
-
<header class="chead"><span class="mono">
|
|
2437
|
+
<header class="chead"><span class="mono">14</span><h2 id="area-release-h">Release</h2><p>Requirement coverage and a release verdict</p></header>
|
|
1858
2438
|
<article class="spec feat" id="cap-release-readiness">
|
|
1859
2439
|
<div class="hd"><span class="mono">Release</span><h3>Requirement coverage and release readiness</h3></div>
|
|
1860
2440
|
<p class="ds">Gives requirement coverage and a GO / GO WITH RISKS / NO-GO verdict for a sprint, area, epic, or stories, from test cases, results, bugs, and flaky tests; can publish it to the wiki.</p>
|
|
@@ -1917,161 +2497,8 @@ forgeqe visual approve --page /home # make this run's /home the baseline (asks
|
|
|
1917
2497
|
</div>
|
|
1918
2498
|
</article>
|
|
1919
2499
|
</section>
|
|
1920
|
-
<section class="chapter" id="area-operations" aria-labelledby="area-operations-h">
|
|
1921
|
-
<header class="chead"><span class="mono">11</span><h2 id="area-operations-h">Operations</h2><p>Monitoring and CI pipelines</p></header>
|
|
1922
|
-
<article class="spec feat" id="cap-monitoring">
|
|
1923
|
-
<div class="hd"><span class="mono">Operations</span><h3>Scheduled monitoring with alerts</h3></div>
|
|
1924
|
-
<p class="ds">Runs checks on a schedule and alerts Teams or email when one keeps failing, and again when it recovers.</p>
|
|
1925
|
-
<div class="facts">
|
|
1926
|
-
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>Check the login flow every 15 minutes and alert Teams</li></ul></div>
|
|
1927
|
-
<div class="fact"><span class="mono">What it needs</span><ul class="needs"><li><span class="need req">Required</span><span>A live model profile in llms.json (an API key or Microsoft Entra ID sign-in)</span></li><li><span class="need opt">Optional</span><span>Alert channels: monitor.alerts.teams or monitor.alerts.email in forgeqe.yaml</span></li></ul></div>
|
|
1928
|
-
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>monitor</code></p></div>
|
|
1929
|
-
</div>
|
|
1930
|
-
<div class="detail"><span class="mono">In detail</span>
|
|
1931
|
-
<section class="doc" id="doc-how_to_use-9-monitoring">
|
|
1932
|
-
<h4>Monitoring</h4>
|
|
1933
|
-
<div class="prose"><p><code>forgeqe monitor</code> runs a set of checks on a schedule and alerts Microsoft Teams and email when one breaks. A check is any test case <code>forgeqe suite</code> takes (section 7): a YAML or Gherkin file, a folder, Jira keys or JQL, an Azure DevOps plan, TestRail, and so on.</p>
|
|
1934
|
-
<pre><code class="language-yaml">monitor:
|
|
1935
|
-
checks: [checks/smoke.yaml]
|
|
1936
|
-
every: 15m # 30s, 15m, 1h, 1h30m; at least 10s
|
|
1937
|
-
replay: verify # verify | trust | off
|
|
1938
|
-
parallel: 1
|
|
1939
|
-
fail_after: 1 # failing rounds in a row before the first alert
|
|
1940
|
-
remind_every: 2h # repeat while it keeps failing; default: never
|
|
1941
|
-
keep: 100 # round folders kept
|
|
1942
|
-
write_back: false # post each round's results to the source system
|
|
1943
|
-
# out_dir: ./.forgeqe/reports/monitor
|
|
1944
|
-
# report_url: https://reports.example.com/monitor
|
|
1945
|
-
alerts:
|
|
1946
|
-
teams:
|
|
1947
|
-
webhook_env: TEAMS_WEBHOOK_URL
|
|
1948
|
-
email:
|
|
1949
|
-
host: smtp.office365.com # or SMTP_HOST
|
|
1950
|
-
port: 587 # STARTTLS; 465 is TLS from the start (secure: true)
|
|
1951
|
-
username_env: SMTP_USERNAME
|
|
1952
|
-
password_env: SMTP_PASSWORD
|
|
1953
|
-
# from: ForgeQE <qa-bot@example.com>
|
|
1954
|
-
to: [qa-team@example.com, oncall@example.com]
|
|
1955
|
-
</code></pre>
|
|
1956
|
-
<pre><code class="language-bash">forgeqe monitor --test-alerts # send a sample alert through every channel
|
|
1957
|
-
forgeqe monitor # monitor.checks every monitor.every, until Ctrl+C
|
|
1958
|
-
forgeqe monitor checks/checkout.yaml --every 5m
|
|
1959
|
-
forgeqe monitor --once # one round, then exit (cron, scheduled pipelines)
|
|
1960
|
-
</code></pre>
|
|
1961
|
-
<p><strong>Each round</strong> runs the checks like a suite, without writing tests, and replays the flows from page memory (section 5). A flow that passed before costs one model call to confirm the expected results (<code>replay: verify</code>), or none (<code>replay: trust</code>, which trusts the replay without checking). A new or changed flow falls back to the full agent. Rounds start <code>every</code> apart; a round that overruns is followed by the next one straight away.</p>
|
|
1962
|
-
<p><strong>Alerts</strong> go out when something changes, not every round:</p>
|
|
1963
|
-
<ul>
|
|
1964
|
-
<li>A check that fails <code>fail_after</code> rounds in a row gets an alert. With <code>fail_after: 2</code> a single flaky failure is never reported.</li>
|
|
1965
|
-
<li>While it keeps failing, nothing more is sent, unless <code>remind_every</code> is set.</li>
|
|
1966
|
-
<li>When it passes again, a recovery alert says how long it was down.</li>
|
|
1967
|
-
<li>A round that cannot run at all (a test system is down, a source is empty, the engine will not start) alerts as the check "ForgeQE monitor", and the other checks keep their state.</li>
|
|
1968
|
-
</ul>
|
|
1969
|
-
<p>All changes in one round go out as one message. Teams gets an Adaptive Card with each check, why it failed and at which step, a link to the test case, and an "Open report" button. Email gets the same as text and HTML, with the failing checks' last screenshots attached (up to five).</p>
|
|
1970
|
-
<p>For Teams, create an incoming webhook on the channel, or a Workflows flow with the trigger "When a Teams webhook request is received" and the action "Post card in a chat or channel"; both take the card as it is. Put the URL in <code>TEAMS_WEBHOOK_URL</code>. For email, any SMTP server works: Office 365 (<code>smtp.office365.com:587</code>, with SMTP AUTH allowed for the mailbox), Gmail with an app password, SendGrid, or a relay on your network (no username needed). The webhook URL and the SMTP password stay in the environment; if a send fails, the error never contains them.</p>
|
|
1971
|
-
<p>The terminal shows one block per round:</p>
|
|
1972
|
-
<pre><code>Monitoring checks/smoke.yaml every 15m · replay verify · alerts: Teams, email to qa-team@example.com · Ctrl+C stops
|
|
1973
|
-
[14:15] ✗ 4/5 passing · 38s · 6.2K tokens
|
|
1974
|
-
✗ SHOP-12 Checkout: failing since 14:15
|
|
1975
|
-
The Pay button is missing on the payment page
|
|
1976
|
-
↑ Alert sent: Teams, email
|
|
1977
|
-
Report .forgeqe/reports/monitor/20260929-141500/report.html
|
|
1978
|
-
Next round at 14:30
|
|
1979
|
-
[14:30] ✗ 4/5 passing · 35s · 5.9K tokens
|
|
1980
|
-
✗ SHOP-12 Checkout: failing since 14:15 (2 rounds, already alerted)
|
|
1981
|
-
Report .forgeqe/reports/monitor/20260929-143000/report.html
|
|
1982
|
-
Next round at 14:45
|
|
1983
|
-
</code></pre>
|
|
1984
|
-
<p><strong>Files.</strong> Each round is a suite folder in <code>.forgeqe/reports/monitor/<time>/</code> (<code>report.html</code>, <code>junit.xml</code>, <code>suite.json</code>, the runs, and <code>alerts.json</code> when it alerted), so the report's trends and flaky tests cover the rounds. Folders beyond <code>keep</code> are deleted. <code>state.json</code> in <code>.forgeqe/reports/monitor/</code> remembers which checks are failing, since when, and when they were last alerted; delete it to start fresh. Set <code>report_url</code> to where you publish <code>.forgeqe/reports/monitor/</code> and alerts link <code><report_url>/<round>/report.html</code>; without it they name the local file.</p>
|
|
1985
|
-
<p><strong>Scheduling.</strong> Run <code>forgeqe monitor</code> in a long-lived session (tmux, a service, a container), or <code>forgeqe monitor --once</code> from cron or a scheduled pipeline. <code>--once</code> exits <code>0</code> when every check passed, <code>1</code> when any failed, and <code>2</code> when stopped. In a pipeline, cache <code>.forgeqe/reports/monitor/state.json</code> between runs; without it every run starts fresh, and a failure alerts every time. Only one monitor can use an <code>out_dir</code> at a time, so an overlapping cron run exits instead of racing.</p>
|
|
1986
|
-
<p>You can also ask ForgeQE's agent. In the shell, "check the smoke cases once" runs one round and shows it as a card, and "monitor the smoke cases every 10 minutes" starts a monitor in the background: you keep working, each round appears as a card, "is anything failing?" shows the state, and "stop monitoring" ends it. A background monitor lives as long as the shell (or <code>forgeqe mcp</code>) and stops when it closes; <code>forgeqe run</code> refuses to start one, because it would stop when the command exits. While one is running, a one-off round on the same <code>out_dir</code> is refused rather than run twice. For monitoring that keeps going after you close the shell, "monitor the smoke cases every 10 minutes and alert the QA channel in Teams, always on" makes the agent set up <code>monitor:</code> in <code>forgeqe.yaml</code>, send a test alert, and tell you how to keep <code>forgeqe monitor</code> running (or add a scheduled pipeline).</p>
|
|
1987
|
-
</div>
|
|
1988
|
-
</section>
|
|
1989
|
-
</div>
|
|
1990
|
-
</article>
|
|
1991
|
-
<article class="spec feat" id="cap-notifications">
|
|
1992
|
-
<div class="hd"><span class="mono">Operations</span><h3>Teams and email notifications</h3></div>
|
|
1993
|
-
<p class="ds">Tells the team in Microsoft Teams channels or by email when a suite fails or passes, when a release readiness verdict is in, and when the domain evaluation falls below its gate, with the failing cases, their bugs, and links to the report and the CI build. Each channel picks its events; the agent also posts a summary to a channel when asked.</p>
|
|
1994
|
-
<div class="facts">
|
|
1995
|
-
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>Post the results to the QA channel</li><li>Tell the release channel the verdict is NO-GO</li></ul></div>
|
|
1996
|
-
<div class="fact"><span class="mono">What it needs</span><ul class="needs"><li><span class="need req">Required</span><span>A notification channel: notify.channels in forgeqe.yaml, each a Teams Workflows webhook (in an environment variable) or email</span></li></ul></div>
|
|
1997
|
-
<div class="fact"><span class="mono">Limits</span><ul class="plainlist"><li>Automatic notifications come from runs nobody is watching (forgeqe suite, forgeqe run, pipelines, the MCP server), not from the interactive shell, where the agent posts only when asked.</li></ul></div>
|
|
1998
|
-
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>notify</code></p></div>
|
|
1999
|
-
</div>
|
|
2000
|
-
<div class="detail"><span class="mono">In detail</span>
|
|
2001
|
-
<section class="doc" id="doc-how_to_use-17-notifications">
|
|
2002
|
-
<h4>Notifications</h4>
|
|
2003
|
-
<div class="prose"><p>ForgeQE tells the team in Microsoft Teams channels, or by email, when something they should know happens. Each channel picks its events:</p>
|
|
2004
|
-
<pre><code class="language-yaml">notify:
|
|
2005
|
-
report_url: https://reports.example.com/forgeqe # where .forgeqe/reports is published, so cards link to each report
|
|
2006
|
-
channels:
|
|
2007
|
-
qa:
|
|
2008
|
-
teams: { webhook_env: TEAMS_WEBHOOK_URL }
|
|
2009
|
-
on: [suite_failed, eval_failed]
|
|
2010
|
-
release:
|
|
2011
|
-
teams: { webhook_env: TEAMS_RELEASE_WEBHOOK }
|
|
2012
|
-
on: [release]
|
|
2013
|
-
leads:
|
|
2014
|
-
email: { host: smtp.office365.com, port: 587, username_env: SMTP_USERNAME, password_env: SMTP_PASSWORD, to: [qa-leads@example.com] }
|
|
2015
|
-
on: [suite_failed, release]
|
|
2016
|
-
</code></pre>
|
|
2017
|
-
<div class="tbl"><table>
|
|
2018
|
-
<thead>
|
|
2019
|
-
<tr><th>Event</th><th>When</th><th>The card shows</th></tr>
|
|
2020
|
-
</thead>
|
|
2021
|
-
<tbody>
|
|
2022
|
-
<tr><td><code>suite_failed</code></td><td>A suite ends with a case that did not pass, or a test that did not</td><td>Each failing case (up to 10) with its kind of failure, the test case and bug links, tests written, and the report and build</td></tr>
|
|
2023
|
-
<tr><td><code>suite_passed</code></td><td>Every case passed</td><td>The count, how long it took, where it ran</td></tr>
|
|
2024
|
-
<tr><td><code>release</code></td><td>A release readiness verdict</td><td>GO, GO WITH RISKS, or NO-GO, the counts, every blocker and risk, and the wiki page when published</td></tr>
|
|
2025
|
-
<tr><td><code>eval_failed</code></td><td><code>forgeqe eval domain --min</code> falls below its gate</td><td>Coverage for each variant and invented ids</td></tr>
|
|
2026
|
-
</tbody>
|
|
2027
|
-
</table></div>
|
|
2028
|
-
<p>A channel without <code>on</code> gets <code>suite_failed</code>, <code>release</code>, and <code>eval_failed</code>. Email adds the failure screenshots. A stopped suite is not announced. Monitoring keeps its own <code>monitor.alerts</code>, with the same cards.</p>
|
|
2029
|
-
<p>Events are sent from runs nobody is watching: <code>forgeqe suite</code> in a pipeline or on a schedule, <code>forgeqe run</code>, and the MCP server. In the shell, where people try things out, nothing is sent on its own; ask instead ("post the results to the QA channel", "tell the release channel the verdict"), and ForgeQE's agent posts a card with <code>notify</code>.</p>
|
|
2030
|
-
<p>Links: with <code>report_url</code>, cards open the report where the reports folder is published. In Azure Pipelines and GitHub Actions they also open the build, where the report is an artifact. Without either, the card names the report's path.</p>
|
|
2031
|
-
<p><strong>Setting up a Teams channel:</strong></p>
|
|
2032
|
-
<ol>
|
|
2033
|
-
<li>In the channel, open <strong>Workflows</strong> and choose <strong>Post to a channel when a webhook request is received</strong>, then copy the URL it gives you. Microsoft is retiring the older Office 365 "Incoming Webhook" connectors; Workflows webhooks take the same cards.</li>
|
|
2034
|
-
<li>The URL is the secret (anyone with it can post): keep it in the environment, the system's credential store (<code>forgeqe auth login TEAMS_WEBHOOK_URL</code>), or a pipeline secret, never in <code>forgeqe.yaml</code>.</li>
|
|
2035
|
-
<li>Create the workflow with a shared or service account: a workflow belongs to whoever made it and stops when they leave. Your tenant must allow the Workflows app.</li>
|
|
2036
|
-
<li><code>forgeqe notify test [channel]</code> sends a test card to check it arrives.</li>
|
|
2037
|
-
</ol>
|
|
2038
|
-
</div>
|
|
2039
|
-
</section>
|
|
2040
|
-
</div>
|
|
2041
|
-
</article>
|
|
2042
|
-
<article class="spec feat" id="cap-pipelines">
|
|
2043
|
-
<div class="hd"><span class="mono">Operations</span><h3>CI pipelines</h3></div>
|
|
2044
|
-
<p class="ds">Writes a GitHub Actions or Azure Pipelines workflow that runs ForgeQE, stores the secrets it needs, pushes a branch, and makes the pipeline green.</p>
|
|
2045
|
-
<div class="facts">
|
|
2046
|
-
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>Add a GitHub Actions workflow that runs tests/cases.yaml on every PR, and make it green</li></ul></div>
|
|
2047
|
-
<div class="fact"><span class="mono">What it needs</span><ul class="needs"><li><span class="need req">Required</span><span>git on PATH</span></li><li><span class="need req">Required</span><span>gh on PATH, for GitHub Actions; or az on PATH, for Azure Pipelines</span></li></ul></div>
|
|
2048
|
-
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>push_branch</code> <code>set_ci_secret</code></p></div>
|
|
2049
|
-
</div>
|
|
2050
|
-
<div class="detail"><span class="mono">In detail</span>
|
|
2051
|
-
<section class="doc" id="doc-how_to_use-8-pipelines">
|
|
2052
|
-
<h4>Pipelines</h4>
|
|
2053
|
-
<div class="prose"><p>ForgeQE's agent writes, runs, and fixes GitHub Actions and Azure Pipelines. Run it in the repo (or pass <code>--repo</code>) and ask:</p>
|
|
2054
|
-
<pre><code class="language-bash">forgeqe run "Add a GitHub Actions workflow that runs the smoke cases in cases.yaml on every pull request"
|
|
2055
|
-
forgeqe run "Set up an Azure DevOps pipeline that runs test plan 12 every night and posts results back"
|
|
2056
|
-
forgeqe run "The ForgeQE workflow is failing, fix it"
|
|
2057
|
-
</code></pre>
|
|
2058
|
-
<p>What it does:</p>
|
|
2059
|
-
<ol>
|
|
2060
|
-
<li>Works out the platform from the git remote (github.com, or dev.azure.com / visualstudio.com) and reads the repo's existing CI, so it can add a job to it or write a new file.</li>
|
|
2061
|
-
<li>Writes the pipeline so it installs Bun and uv, runs <code>npm install -g @capagents/forgeqe</code> and <code>forgeqe setup</code>, then <code>forgeqe suite <source...> --no-code --plain --out <dir></code>, and publishes <code>junit.xml</code> and the report folder. On GitHub the suite adds a results table to the job summary.</li>
|
|
2062
|
-
<li>Validates it (actionlint when installed, otherwise by parsing the YAML) and commits it.</li>
|
|
2063
|
-
<li>Pushes to a new branch named <code>forgeqe/ci-<topic></code> with ForgeQE's <code>push_branch</code> tool. That tool refuses the default branch, <code>main</code>, and <code>master</code>, and never forces; <code>git push</code> itself is blocked for the agent. Merging stays with you.</li>
|
|
2064
|
-
<li>Stores the keys the pipeline needs (the model's key from <code>llms.json</code>, and <code>ADO_PAT</code>, <code>JIRA_EMAIL</code> and <code>JIRA_API_TOKEN</code>, or <code>XRAY_CLIENT_ID</code> and <code>XRAY_CLIENT_SECRET</code>) with ForgeQE's <code>set_ci_secret</code> tool. The agent only names the variable; ForgeQE passes the value from its own environment to <code>gh secret set</code> (on stdin) or <code>az pipelines variable</code> (through the environment), and never shows it. A variable that is not set in ForgeQE's environment is reported, not guessed.</li>
|
|
2065
|
-
<li>Triggers the run with <code>gh workflow run</code> or <code>az pipelines run</code>, watches it, reads the logs of failed steps, fixes the pipeline (or the code), pushes again, and repeats until the run passes or <code>code.max_attempts</code> runs have failed. It reports the run link and the result.</li>
|
|
2066
|
-
</ol>
|
|
2067
|
-
<p>It needs <code>gh</code> logged in for GitHub, or <code>az</code> with the <code>azure-devops</code> extension logged in for Azure DevOps. Without them it still writes, validates, and commits the pipeline, then tells you the commands to run. Set <code>llm.active</code> to a live profile in the committed <code>forgeqe.yaml</code>.</p>
|
|
2068
|
-
</div>
|
|
2069
|
-
</section>
|
|
2070
|
-
</div>
|
|
2071
|
-
</article>
|
|
2072
|
-
</section>
|
|
2073
2500
|
<section class="chapter" id="area-systems" aria-labelledby="area-systems-h">
|
|
2074
|
-
<header class="chead"><span class="mono">
|
|
2501
|
+
<header class="chead"><span class="mono">15</span><h2 id="area-systems-h">Systems</h2><p>Azure DevOps, Jira, and Confluence</p></header>
|
|
2075
2502
|
<article class="spec feat" id="cap-systems">
|
|
2076
2503
|
<div class="hd"><span class="mono">Systems</span><h3>Work with Azure DevOps, Jira, GitHub, and Confluence</h3></div>
|
|
2077
2504
|
<p class="ds">Reads anything in the configured systems (work items, issues, queries, boards, pages), makes changes the user asks for, puts run evidence (screenshots, reports) on work items and issues as attachments or comments, and builds Azure DevOps dashboards and wiki pages.</p>
|
|
@@ -2116,82 +2543,8 @@ mcp:
|
|
|
2116
2543
|
</div>
|
|
2117
2544
|
</article>
|
|
2118
2545
|
</section>
|
|
2119
|
-
<section class="chapter" id="area-setup" aria-labelledby="area-setup-h">
|
|
2120
|
-
<header class="chead"><span class="mono">13</span><h2 id="area-setup-h">Setup</h2><p>Configuration, skills, and help</p></header>
|
|
2121
|
-
<article class="spec feat" id="cap-setup-help">
|
|
2122
|
-
<div class="hd"><span class="mono">Setup</span><h3>Setup, skills, and documentation</h3></div>
|
|
2123
|
-
<p class="ds">Shows the model profiles and MCP servers, installs or repairs the browser engine, creates a forgeqe.yaml, manages skills, and answers questions about ForgeQE from its documentation and this list.</p>
|
|
2124
|
-
<div class="facts">
|
|
2125
|
-
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>What can you do?</li><li>Which model am I using?</li><li>How do I set up email alerts?</li></ul></div>
|
|
2126
|
-
<div class="fact"><span class="mono">What it needs</span><p class="muted">Nothing beyond ForgeQE itself.</p></div>
|
|
2127
|
-
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>forgeqe_setup</code> <code>capabilities</code> <code>playbook</code></p></div>
|
|
2128
|
-
</div>
|
|
2129
|
-
<div class="detail"><span class="mono">In detail</span>
|
|
2130
|
-
<p class="see">See <a href="#doc-how_to_use-2-scaffold-config">2. Scaffold config</a>.</p>
|
|
2131
|
-
<section class="doc" id="doc-how_to_use-11-skills">
|
|
2132
|
-
<h4>Skills</h4>
|
|
2133
|
-
<div class="prose"><p>A skill is a folder with a <code>SKILL.md</code>: instructions that teach ForgeQE's agents something a prompt can't know, like your repo's test conventions, a framework, or how to drive an app such as Dynamics 365. The format is the Agent Skills one (Claude Code and others use it), with one ForgeQE field, <code>agents</code>:</p>
|
|
2134
|
-
<pre><code class="language-markdown">---
|
|
2135
|
-
name: contoso-erp
|
|
2136
|
-
description: "Use when the site is the Contoso ERP (erp.contoso.com): the order grid, the customer lookup, and saving orders."
|
|
2137
|
-
agents: [browser]
|
|
2138
|
-
---
|
|
2139
|
-
|
|
2140
|
-
- The customer lookup opens a list after three letters; click the row whose name matches exactly.
|
|
2141
|
-
- Save with the floppy icon in the toolbar; wait until "Saved" shows in the status bar.
|
|
2142
|
-
</code></pre>
|
|
2143
|
-
<p><code>name</code> is lowercase words joined by hyphens, the same as the folder. Agents choose skills by <code>description</code> alone, so name the concrete signs it applies (a framework, a config file, a host, a product). Quote it when it contains <code>: </code>.</p>
|
|
2144
|
-
<p><strong>Who uses it.</strong> <code>agents</code> lists <code>forgeqe</code> (ForgeQE's agent: how to handle a kind of request), <code>scribe</code> (writing, healing, finding and reviewing tests), and <code>browser</code> (the browser agent). Without <code>agents</code>, a skill is for <code>forgeqe</code> and <code>scribe</code>; a browser skill must ask for <code>browser</code>.</p>
|
|
2145
|
-
<p><strong>How skills are used.</strong></p>
|
|
2146
|
-
<ul>
|
|
2147
|
-
<li>ForgeQE's agent and Scribe see the name and description of each skill meant for them, and load the ones that fit the job through the agent runtime's skill tool. A skill adds to their instructions and never overrides them. Scribe can also read files next to <code>SKILL.md</code> (for example <code>references/locators.md</code>) when the skill points to them.</li>
|
|
2148
|
-
<li>Browser runs: ForgeQE's agent passes the browser skills a goal needs to <code>browser_run</code> and <code>run_test_cases</code>. For <code>forgeqe suite</code>, <code>forgeqe monitor</code> and <code>forgeqe run --direct</code>, the engine picks them from the goal with one short model call (at most three), and remembers the choice for that goal, so a monitor's later rounds and repeated cases skip it. <code>--skill NAME</code> (repeatable) fixes the list; <code>--skill none</code> runs without any. The chosen skills' text is added to the browser agent's prompt, at most <code>skills.browser_max_chars</code> (6,000) characters in total; the browser agent reads only <code>SKILL.md</code>.</li>
|
|
2149
|
-
<li><code>run.json</code>, the <code>run.done</code> log line and the report list the skills a run used.</li>
|
|
2150
|
-
</ul>
|
|
2151
|
-
<p><strong>Where skills come from</strong>, lowest to highest precedence; a later source replaces a skill of the same name:</p>
|
|
2152
|
-
<div class="tbl"><table>
|
|
2153
|
-
<thead>
|
|
2154
|
-
<tr><th>Source</th><th>Folder</th></tr>
|
|
2155
|
-
</thead>
|
|
2156
|
-
<tbody>
|
|
2157
|
-
<tr><td>Built-in</td><td>the <code>skills/</code> folder ForgeQE ships: <code>playwright-typescript</code> and <code>dotnet-nunit-playwright</code> (Scribe), <code>dynamics-365</code> (browser), <code>writing-skills</code> (ForgeQE's agent)</td></tr>
|
|
2158
|
-
<tr><td>Global, only with <code>skills.external: true</code></td><td><code>~/.claude/skills</code>, <code>~/.agents/skills</code>, and the repo's <code>.claude/skills</code> and <code>.agents/skills</code></td></tr>
|
|
2159
|
-
<tr><td>User</td><td><code>~/.forgeqe/skills</code> (or <code>$FORGEQE_HOME/skills</code>)</td></tr>
|
|
2160
|
-
<tr><td>Project</td><td><code>.forgeqe/skills</code> next to <code>forgeqe.yaml</code>, and the repo's own <code>.forgeqe/skills</code></td></tr>
|
|
2161
|
-
<tr><td>Extra</td><td>each folder in <code>skills.dirs</code></td></tr>
|
|
2162
|
-
</tbody>
|
|
2163
|
-
</table></div>
|
|
2164
|
-
<p>Commit <code>.forgeqe/skills</code> so the team and CI share it. ForgeQE only ignores <code>.forgeqe/reports/</code>; if your <code>.gitignore</code> has <code>.forgeqe/</code>, replace it with <code>.forgeqe/*</code> and <code>!.forgeqe/skills/</code> (<code>forgeqe skills new</code> warns when git would ignore the new skill). To change a built-in skill, make a project skill with the same name.</p>
|
|
2165
|
-
<p>Skills from anywhere else stay out: the agent runtime's own discovery (<code>~/.claude/skills</code>, a repo's <code>.opencode/skills</code>, its built-in skills) is off, and each agent may load only the skills ForgeQE's registry gives it. Skill text is trusted like the repo's own files, so global skills are opt-in.</p>
|
|
2166
|
-
<pre><code class="language-yaml">skills:
|
|
2167
|
-
enabled: true # false (or skills: false) turns skills off
|
|
2168
|
-
dirs: [../shared-skills] # more folders, read last
|
|
2169
|
-
disable: [dynamics-365] # leave these out
|
|
2170
|
-
external: false # true: also the global folders above
|
|
2171
|
-
browser_max_chars: 6000 # most skill text added to one browser run
|
|
2172
|
-
</code></pre>
|
|
2173
|
-
<p><strong>Commands.</strong></p>
|
|
2174
|
-
<div class="tbl"><table>
|
|
2175
|
-
<thead>
|
|
2176
|
-
<tr><th>Command</th><th>What it does</th></tr>
|
|
2177
|
-
</thead>
|
|
2178
|
-
<tbody>
|
|
2179
|
-
<tr><td><code>forgeqe skills</code></td><td>The skills each agent can use, where each comes from (and what it replaces), disabled skills, problems, and the folders read</td></tr>
|
|
2180
|
-
<tr><td><code>forgeqe skills show <name></code></td><td>One skill's instructions</td></tr>
|
|
2181
|
-
<tr><td><code>forgeqe skills new <name> --description "Use when …" [--agent browser] [--user]</code></td><td>Start a skill in <code>.forgeqe/skills</code> (<code>--user</code>: <code>~/.forgeqe/skills</code>)</td></tr>
|
|
2182
|
-
<tr><td><code>forgeqe skills check</code></td><td>Validate every skill; exit 1 when one has a problem</td></tr>
|
|
2183
|
-
<tr><td><code>forgeqe skills enable|disable <name></code></td><td>Edit <code>skills.disable</code> in <code>forgeqe.yaml</code>, keeping its comments</td></tr>
|
|
2184
|
-
<tr><td><code>/skills [name]</code></td><td>The same list, or one skill, in the shell</td></tr>
|
|
2185
|
-
</tbody>
|
|
2186
|
-
</table></div>
|
|
2187
|
-
<p>In the shell you can also ask: "make a skill for our Dynamics date picker" or "teach Scribe our test conventions". ForgeQE's agent gathers the facts, writes the skill into <code>.forgeqe/skills</code> with <code>forgeqe_setup</code>, and it applies from the next session. It writes skills only when you ask.</p>
|
|
2188
|
-
</div>
|
|
2189
|
-
</section>
|
|
2190
|
-
</div>
|
|
2191
|
-
</article>
|
|
2192
|
-
</section>
|
|
2193
2546
|
<section class="chapter" id="reference" aria-labelledby="reference-h">
|
|
2194
|
-
<header class="chead"><span class="mono">
|
|
2547
|
+
<header class="chead"><span class="mono">16</span><h2 id="reference-h">Reference</h2><p>MCP, tuning, logs, environment variables, and every command</p></header>
|
|
2195
2548
|
<section class="doc" id="doc-how_to_use-10-mcp">
|
|
2196
2549
|
<h3>MCP</h3>
|
|
2197
2550
|
<div class="prose"></div>
|
|
@@ -2295,6 +2648,31 @@ agents: [browser]
|
|
|
2295
2648
|
<p>To share a session, type <code>/logs</code> in the shell (<code>/logs shots</code> adds screenshots), or run <code>forgeqe logs bundle</code> afterwards. Either writes one <code>.tar.gz</code> with the session log, the run folders it mentions (<code>run.json</code>, <code>steps.json</code>, <code>trace.md</code>, <code>resume.json</code>, <code>workflow.yaml</code>, the failure screenshot), your <code>forgeqe.yaml</code>, your profiles without keys, and the versions. <code>forgeqe logs</code> lists recent logs.</p>
|
|
2296
2649
|
<p>Start with <code>--debug</code> (or set <code>FORGEQE_DEBUG=1</code>) for more: the browser engine logs at debug level, and long values such as tool results are kept whole instead of cut at 8,000 characters.</p>
|
|
2297
2650
|
</div>
|
|
2651
|
+
<section class="doc sub" id="doc-how_to_use-which-tools-forgeqes-agent-gets"><h4>Which tools ForgeQE's agent gets</h4><div class="prose"><p>Every model call carries the definitions of the tools the agent has, so it does not get all of them. Each turn gets:</p>
|
|
2652
|
+
<ul>
|
|
2653
|
+
<li><strong>the core:</strong> help, setup, <code>use_capabilities</code>, and the test environments and their apps;</li>
|
|
2654
|
+
<li><strong>the capabilities the request is about</strong>, picked without a model call from its words (each capability's keywords
|
|
2655
|
+
in <code>src/capabilities/registry.ts</code>), the apps of the environment it names (a mobile app means the mobile tools), and
|
|
2656
|
+
links (the browser);</li>
|
|
2657
|
+
<li><strong>capabilities used in the last few turns</strong>, so a conversation keeps its tools.</li>
|
|
2658
|
+
</ul>
|
|
2659
|
+
<p>A capability this project cannot use yet (no mobile apps, no Jira) is not picked from words alone. When a request
|
|
2660
|
+
needs a capability the turn did not get, the agent calls <code>use_capabilities</code>, and ForgeQE continues the same request
|
|
2661
|
+
with its tools. With fewer tools in view, the agent also cannot mix up similar ones (<code>browser_run</code>, <code>desktop_explore</code>,
|
|
2662
|
+
<code>mobile_explore</code>). <code>forgeqe context</code> shows what a turn carries, and what each capability adds. The session log
|
|
2663
|
+
(<code>/logs</code>) records which capabilities each turn got and why (<code>agent.scope</code>).</p>
|
|
2664
|
+
<p>Pin capabilities on, or switch them off for a project, in forgeqe.yaml (ids from <code>forgeqe features</code>):</p>
|
|
2665
|
+
<pre><code class="language-yaml">agent:
|
|
2666
|
+
capabilities:
|
|
2667
|
+
always: [regression-packs]
|
|
2668
|
+
off: [desktop-apps]
|
|
2669
|
+
</code></pre>
|
|
2670
|
+
<p><code>FORGEQE_TOOL_SCOPE=off</code> gives every tool on every turn.</p>
|
|
2671
|
+
<p>MCP clients of <code>forgeqe mcp</code> (Claude Desktop, Cursor, Claude Code) list tools once rather than per turn, so they get
|
|
2672
|
+
the core and every capability this project can use (no desktop tools without desktop apps, no tracker tools without a
|
|
2673
|
+
tracker). <code>use_capabilities</code> adds others and sends <code>notifications/tools/list_changed</code>, which tells the client to read
|
|
2674
|
+
the list again. For a client that ignores it, <code>forgeqe mcp --all-tools</code> lists every tool.</p>
|
|
2675
|
+
</div></section>
|
|
2298
2676
|
</section>
|
|
2299
2677
|
<section class="doc" id="doc-how_to_use-14-env-overrides">
|
|
2300
2678
|
<h3>Env overrides</h3>
|
|
@@ -2307,9 +2685,13 @@ agents: [browser]
|
|
|
2307
2685
|
<tr><td><code>WP_LLMS_JSON</code></td><td>Path to <code>llms.json</code></td></tr>
|
|
2308
2686
|
<tr><td><code>WP_LLM_PROFILE</code></td><td>Active profile, unless <code>--profile</code> is set</td></tr>
|
|
2309
2687
|
<tr><td><code>WP_AZURE_API_KEY</code> / <code>OPENAI_API_KEY</code> / <code>WP_API_KEY</code></td><td>Keys referenced by profiles</td></tr>
|
|
2688
|
+
<tr><td><code>GITHUB_TOKEN</code></td><td>A <code>github_copilot</code> profile's GitHub Models token for the browser engine (needs <code>models:read</code>), unless the profile's <code>api_key_env</code> names another; else the GitHub CLI's sign-in</td></tr>
|
|
2689
|
+
<tr><td><code>FORGEQE_COPILOT_AUTH</code></td><td>The Copilot sign-in <code>forgeqe auth login copilot</code> saves; set by ForgeQE from the credential store, not by hand</td></tr>
|
|
2310
2690
|
<tr><td><code>FORGEQE_CONFIG_DIR</code></td><td>Global config folder (default <code>~/.config/forgeqe</code>): <code>forgeqe.yaml</code>, <code>llms.json</code> and <code>.env</code> used when a project has none</td></tr>
|
|
2311
2691
|
<tr><td><code>FORGEQE_PYTHON</code></td><td>A Python that already has the engine's libraries, used instead of the managed venv (skipped, with a note, when it cannot run the engine)</td></tr>
|
|
2312
2692
|
<tr><td><code>FORGEQE_DEBUG</code></td><td><code>1</code> logs at debug level, like <code>--debug</code></td></tr>
|
|
2693
|
+
<tr><td><code>FORGEQE_TOOL_SCOPE</code></td><td><code>off</code> gives ForgeQE's agent every tool on every turn, instead of the tools of the capabilities each request is about</td></tr>
|
|
2694
|
+
<tr><td><code>FORGEQE_ROUTING</code></td><td><code>off</code> also puts every playbook in the agent's prompt (for comparing, or ruling out a routing fault)</td></tr>
|
|
2313
2695
|
<tr><td><code>FORGEQE_HOME</code></td><td>Where the managed engine lives, user skills (<code>skills/</code>), and page memory when there is no config file (default <code>~/.forgeqe</code>)</td></tr>
|
|
2314
2696
|
<tr><td><code>ADO_ORG_URL</code> / <code>ADO_PROJECT</code> / <code>ADO_PAT</code></td><td>Azure DevOps organization, project, and token when <code>sources.azure_devops</code> doesn't set them</td></tr>
|
|
2315
2697
|
<tr><td><code>JIRA_URL</code> / <code>JIRA_EMAIL</code> / <code>JIRA_API_TOKEN</code></td><td>Jira site and credentials when <code>sources.jira</code> doesn't set them</td></tr>
|
|
@@ -2338,7 +2720,7 @@ agents: [browser]
|
|
|
2338
2720
|
<tr><td><code>forgeqe features</code></td><td>What ForgeQE can do, and what each capability still needs in this setup (works without a model)</td></tr>
|
|
2339
2721
|
<tr><td><code>forgeqe docs [--guide|--open] [--copy DIR]</code></td><td>The bundled docs; <code>--guide</code> opens the user guide, <code>--open</code> the architecture page</td></tr>
|
|
2340
2722
|
<tr><td><code>forgeqe mcp list</code></td><td>The MCP servers ForgeQE's agent gets from your settings</td></tr>
|
|
2341
|
-
<tr><td><code>forgeqe auth login [system|NAME]</code></td><td>Sign in to Azure DevOps, GitHub, Jira, Confluence, TestRail, Zephyr, qTest, or Xray in the browser, or save one secret, in the system's credential store</td></tr>
|
|
2723
|
+
<tr><td><code>forgeqe auth login [system|NAME]</code></td><td>Sign in to Azure DevOps, GitHub, GitHub Copilot (<code>copilot</code>), Jira, Confluence, TestRail, Zephyr, qTest, or Xray in the browser, or save one secret, in the system's credential store</td></tr>
|
|
2342
2724
|
<tr><td><code>forgeqe auth [status]</code>, <code>forgeqe auth logout <system></code></td><td>What is signed in; forget a sign-in</td></tr>
|
|
2343
2725
|
</tbody>
|
|
2344
2726
|
</table></div>
|