@capagents/forgeqe 2.2.1 → 2.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
1
  <!DOCTYPE html>
2
- <!-- Generated by scripts/gen-guide.ts from src/capabilities/registry.ts and HOW_TO_USE.md. Do not edit: change those and run bun scripts/gen-guide.ts. -->
2
+ <!-- Generated by scripts/gen-guide.ts from src/capabilities/registry.ts, HOW_TO_USE.md and the feature guides in docs/. Do not edit: change those and run bun scripts/gen-guide.ts. -->
3
3
  <html lang="en">
4
4
  <head>
5
5
  <meta charset="utf-8">
@@ -259,7 +259,7 @@ a:hover{text-decoration-color:var(--guide-accent)}
259
259
  .toc{display:none}
260
260
  }
261
261
  @media (max-width:900px){
262
- .shell{grid-template-columns:minmax(0,1fr);grid-template-rows:auto minmax(0,1fr);gap:14px;padding:0}
262
+ .shell{grid-template-columns:minmax(0,1fr);grid-template-rows:auto auto minmax(0,1fr);gap:14px;padding:0}
263
263
  .rail{grid-column:1;grid-row:1;overflow:visible;border-right:0;border-bottom:1px solid var(--guide-line)}
264
264
  .mast{grid-column:1;grid-row:2}
265
265
  .rail-inner{padding:12px 14px}
@@ -341,6 +341,11 @@ a:hover{text-decoration-color:var(--guide-accent)}
341
341
  <li><a href="#reference"><span class="n">17</span><span class="t">Reference</span><span class="d">MCP, tuning, logs, environment variables, and every command</span></a></li>
342
342
  </ol>
343
343
  </section>
344
+ <section class="rail-group" id="rail-hits" hidden>
345
+ <h2 class="rail-title mono">Sections</h2>
346
+ <ol class="chapters" id="rail-hits-list"></ol>
347
+ <p class="muted" id="rail-none" hidden>No section matches</p>
348
+ </section>
344
349
  </div>
345
350
  </nav>
346
351
  <header class="mast">
@@ -379,12 +384,12 @@ forgeqe docs --copy ./docs # copy the guide, the architecture page, and the re
379
384
  </code></pre>
380
385
  </div>
381
386
  <h3 class="band-title">Every feature</h3>
382
- <div class="area"><a class="area-name" href="#area-browser"><span class="mono">Browser</span></a><ul class="tiles"><li><a class="tile" href="#cap-browser-goals"><b>Run any goal in a real browser</b><span>Go to booking.com and search hotels in Mumbai for next weekend</span></a></li><li><a class="tile" href="#cap-logins"><b>Saved logins for the application under test</b><span>Sign in to the admin site and keep the session</span></a></li></ul></div>
387
+ <div class="area"><a class="area-name" href="#area-browser"><span class="mono">Browser</span></a><ul class="tiles"><li><a class="tile" href="#cap-browser-goals"><b>Run any goal in a real browser</b><span>Go to booking.com and search hotels in Mumbai for next weekend</span></a></li><li><a class="tile" href="#cap-logins"><b>Saved logins for the application under test</b><span>Sign in to the admin site and keep the session</span></a></li><li><a class="tile" href="#cap-record-web"><b>Record a web test by hand</b><span>I'll show you the checkout steps in the shop: record them</span></a></li></ul></div>
383
388
  <div class="area"><a class="area-name" href="#area-test-cases"><span class="mono">Test cases</span></a><ul class="tiles"><li><a class="tile" href="#cap-run-test-cases"><b>Run manual test cases in browsers</b><span>Run SHOP-12 and SHOP-14 from Jira, two browsers at once</span></a></li><li><a class="tile" href="#cap-test-case-quality"><b>Check and improve test cases</b><span>Are the cases in test plan 12 good enough to run?</span></a></li><li><a class="tile" href="#cap-write-test-cases"><b>Write test cases from a story</b><span>Write test cases for story 1234 and save them</span></a></li><li><a class="tile" href="#cap-domain-compliance"><b>Domain and regulatory test design</b><span>Write test cases for story 1234, covering our DORA and PSD2 obligations</span></a></li><li><a class="tile" href="#cap-exploratory-testing"><b>Exploratory testing of a story</b><span>Explore story 1234 and find bugs</span></a></li></ul></div>
384
389
  <div class="area"><a class="area-name" href="#area-setup"><span class="mono">Setup</span></a><ul class="tiles"><li><a class="tile" href="#cap-test-apps"><b>Test environments and their apps</b><span>Place an order in the shop, then check its invoice in Billing, in UAT</span></a></li><li><a class="tile" href="#cap-generated-code"><b>What ForgeQE wrote</b><span>How much of the code ForgeQE wrote is still there?</span></a></li><li><a class="tile" href="#cap-forgeqe-commands"><b>Every forgeqe command from the shell</b><span>Show me the context budget</span></a></li><li><a class="tile" href="#cap-setup-help"><b>Setup, skills, and documentation</b><span>What can you do?</span></a></li></ul></div>
385
390
  <div class="area"><a class="area-name" href="#area-desktop"><span class="mono">Desktop</span></a><ul class="tiles"><li><a class="tile" href="#cap-desktop-apps"><b>Automate Windows desktop apps</b><span>Create an invoice for Contoso in the Billing app and save it as a test</span></a></li></ul></div>
386
391
  <div class="area"><a class="area-name" href="#area-mobile"><span class="mono">Mobile</span></a><ul class="tiles"><li><a class="tile" href="#cap-mobile-apps"><b>Automate Android and iOS apps</b><span>Sign in to the courier app and accept the first delivery, and save it as a test</span></a></li></ul></div>
387
- <div class="area"><a class="area-name" href="#area-operations"><span class="mono">Operations</span></a><ul class="tiles"><li><a class="tile" href="#cap-containers"><b>Services and mocks in Docker or Podman</b><span>Start the database and the queue in Docker</span></a></li><li><a class="tile" href="#cap-monitoring"><b>Scheduled monitoring with alerts</b><span>Check the login flow every 15 minutes and alert Teams</span></a></li><li><a class="tile" href="#cap-notifications"><b>Teams and email notifications</b><span>Post the results to the QA channel</span></a></li><li><a class="tile" href="#cap-pipelines"><b>CI pipelines</b><span>Add a GitHub Actions workflow that runs tests/cases.yaml on every PR, and make it green</span></a></li></ul></div>
392
+ <div class="area"><a class="area-name" href="#area-operations"><span class="mono">Operations</span></a><ul class="tiles"><li><a class="tile" href="#cap-containers"><b>Services, mocks, workspace and browser in Docker or Podman</b><span>Start the database and the queue in Docker</span></a></li><li><a class="tile" href="#cap-monitoring"><b>Scheduled monitoring with alerts</b><span>Check the login flow every 15 minutes and alert Teams</span></a></li><li><a class="tile" href="#cap-notifications"><b>Teams and email notifications</b><span>Post the results to the QA channel</span></a></li><li><a class="tile" href="#cap-pipelines"><b>CI pipelines</b><span>Add a GitHub Actions workflow that runs tests/cases.yaml on every PR, and make it green</span></a></li></ul></div>
388
393
  <div class="area"><a class="area-name" href="#area-data"><span class="mono">Data</span></a><ul class="tiles"><li><a class="tile" href="#cap-databases"><b>Ask a database in plain language</b><span>Which customers in India have paid orders? Ask the orders database</span></a></li><li><a class="tile" href="#cap-test-data"><b>Test data on demand</b><span>Generate test data: 20 customers with 1 to 3 orders each in the shop database</span></a></li><li><a class="tile" href="#cap-data-flows"><b>Data flows between systems</b><span>Check that customers from the digital database reach the CRM correctly</span></a></li></ul></div>
389
394
  <div class="area"><a class="area-name" href="#area-automated-tests"><span class="mono">Automated tests</span></a><ul class="tiles"><li><a class="tile" href="#cap-write-tests"><b>Turn a passed run into an automated test</b><span>Write a Playwright test for that run in ../app</span></a></li><li><a class="tile" href="#cap-heal-tests"><b>Heal failing tests</b><span>Heal the failing tests in ../app</span></a></li><li><a class="tile" href="#cap-find-run-tests"><b>Find and run existing tests</b><span>Run the checkout tests</span></a></li><li><a class="tile" href="#cap-regression-packs"><b>Regression packs for a release</b><span>Build a regression pack for sprint 12 and run it on TestMu</span></a></li><li><a class="tile" href="#cap-flaky-tests"><b>Flaky test detection and quarantine</b><span>Which tests are flaky?</span></a></li></ul></div>
390
395
  <div class="area"><a class="area-name" href="#area-pull-requests"><span class="mono">Pull requests</span></a><ul class="tiles"><li><a class="tile" href="#cap-pr-review"><b>Review pull requests</b><span>Review PR 42 and focus on error handling</span></a></li><li><a class="tile" href="#cap-pr-impact"><b>Pull request impact and test selection</b><span>What does PR 42 impact?</span></a></li></ul></div>
@@ -407,7 +412,7 @@ forgeqe docs --copy ./docs # copy the guide, the architecture page, and the re
407
412
  <tbody>
408
413
  <tr><td><strong>npm (Node.js 18+) or pip (Python 3.9+)</strong></td><td>To install <code>@capagents/forgeqe</code> or <code>forgeqe-cli</code>; either one gives the same <code>forgeqe</code> command</td></tr>
409
414
  <tr><td><strong>A terminal</strong></td><td>macOS Terminal, iTerm2, VS Code or Cursor's terminal, Windows Terminal</td></tr>
410
- <tr><td><strong>A model</strong></td><td>A profile in <code>llms.json</code> with a working key, or Microsoft Entra ID sign-in for Azure (Azure OpenAI, OpenAI, an OpenAI-compatible gateway, or Ollama), or a GitHub Copilot seat (see <a href="../#github-copilot-seats-as-the-model">GitHub Copilot seats as the model</a>)</td></tr>
415
+ <tr><td><strong>A model</strong></td><td>A profile in <code>llms.json</code> with a working key, or Microsoft Entra ID sign-in for Azure (Azure OpenAI, OpenAI, an OpenAI-compatible gateway, or Ollama), or a GitHub Copilot seat (see <a href="../HOW_TO_USE.md#github-copilot-seats-as-the-model">GitHub Copilot seats as the model</a>)</td></tr>
411
416
  <tr><td><strong>Internet access on first use</strong></td><td>ForgeQE downloads Bun, the agent runtime, uv, Python, and the engine into <code>~/.forgeqe</code> once</td></tr>
412
417
  </tbody>
413
418
  </table></div>
@@ -440,9 +445,8 @@ export WP_AZURE_API_KEY=... # the key your llms.json profile names
440
445
  forgeqe models # check the profiles
441
446
  forgeqe start # the shell
442
447
  </code></pre>
443
- <p>With GitHub Copilot licences instead of a model key, sign in rather than exporting one, and set <code>llm.active: copilot</code> in <code>forgeqe.yaml</code> (or <code>&quot;default&quot;: &quot;copilot&quot;</code> in <code>llms.json</code>):</p>
444
- <pre><code class="language-bash">forgeqe auth login copilot # ForgeQE's agent and Scribe on your Copilot seat
445
- forgeqe auth login github # the browser engine, through GitHub Models
448
+ <p>With GitHub Copilot licences instead of a model key, sign in rather than exporting one, and set <code>llm.active: copilot</code> in <code>forgeqe.yaml</code> (or <code>&quot;default&quot;: &quot;copilot&quot;</code> in <code>llms.json</code>). The browser engine then runs on your Copilot seat; ForgeQE's agent and Scribe need another profile for now:</p>
449
+ <pre><code class="language-bash">forgeqe auth login copilot # a fine-grained token with Copilot Requests, or the GitHub CLI's sign-in
446
450
  forgeqe models # the copilot profile is active
447
451
  </code></pre>
448
452
  <p>In the shell, type a request and press Enter:</p>
@@ -455,7 +459,7 @@ Write a Playwright test for that run in ../app
455
459
  forgeqe suite cases.yaml --parallel 3 # what pipelines call
456
460
  </code></pre>
457
461
  <p><code>forgeqe models</code> lists the profiles ForgeQE sees and marks the active one. With no profile set up, the shell's footer reads <strong>Model none set up</strong> and any request explains how to add one.</p>
458
- <p>With more than one profile, <code>llm.routing: auto</code> gives each kind of work (agent, browser, coding, review, analysis, planning) the configured model it needs; <code>forgeqe models --routes</code> shows the picks. See <a href="../docs/MODEL_ROUTING.md">docs/MODEL_ROUTING.md</a>.</p>
462
+ <p>With more than one profile, <code>llm.routing: auto</code> gives each kind of work (agent, browser, coding, review, analysis, planning) the configured model it needs; <code>forgeqe models --routes</code> shows the picks. See <a href="#doc-docs-model_routing-model-routing-the-right-configured-model-for-each-task">docs/MODEL_ROUTING.md</a>.</p>
459
463
  <hr />
460
464
  </div>
461
465
  </section>
@@ -520,12 +524,10 @@ bun src/index.ts init --global --profile azure-gpt4o
520
524
  <p>Give that identity the <strong>Cognitive Services OpenAI User</strong> role on the resource. Tokens are renewed before they expire, so long suites and coding sessions keep running. The browser engine signs in itself; the coding agent's runtime only takes a key, so ForgeQE gives it a gateway on <code>127.0.0.1</code> that accepts a one-time random key and passes each request on with a current token. No key is stored anywhere.</p>
521
525
  <p><code>auth: entra</code> also works on an <code>openai_compatible</code> profile, for a gateway such as API Management that checks Entra ID tokens: set <code>token_scope</code> to the scope it expects (for example <code>api://&lt;app id&gt;/.default</code>; the default is <code>https://cognitiveservices.azure.com/.default</code>). The engine needs the <code>azure-identity</code> package, which <code>forgeqe setup</code> installs (and adds to environments set up before it was needed); with <code>FORGEQE_PYTHON</code>, install it there yourself.</p>
522
526
  </div></section>
523
- <section class="doc sub" id="doc-how_to_use-github-copilot-seats-as-the-model"><h4>GitHub Copilot seats as the model</h4><div class="prose"><p>With GitHub Copilot licences (Pro, Pro+, Business or Enterprise), a <code>github_copilot</code> profile runs ForgeQE's agent and Scribe on the Copilot seat. They work through the agent runtime's own GitHub Copilot sign-in, <a href="https://github.blog/changelog/2026-01-16-github-copilot-now-supports-opencode/" rel="noopener">which GitHub supports</a>. Copilot gives other tools no model API of their own, so the browser engine and the desktop agent use <a href="https://docs.github.com/en/github-models" rel="noopener">GitHub Models</a> instead, with your GitHub sign-in:</p>
527
+ <section class="doc sub" id="doc-how_to_use-github-copilot-seats-as-the-model"><h4>GitHub Copilot seats as the model</h4><div class="prose"><p>With GitHub Copilot licences (Pro, Pro+, Business or Enterprise), a <code>github_copilot</code> profile runs the browser engine and the desktop and mobile agents on your Copilot seat, through <a href="https://docs.github.com/en/copilot/how-tos/copilot-sdk" rel="noopener">GitHub's Copilot SDK</a>. Copilot gives other tools no model API of its own, so ForgeQE starts a small gateway on <code>127.0.0.1</code> that runs the SDK and speaks the OpenAI chat API to the engine. It signs in with a GitHub token, so no other app is authorized on your GitHub account.</p>
524
528
  <pre><code class="language-json">&quot;copilot&quot;: {
525
529
  &quot;provider&quot;: &quot;github_copilot&quot;,
526
- &quot;model&quot;: &quot;claude-sonnet-4.6&quot;,
527
- &quot;engine_model&quot;: &quot;openai/gpt-4.1&quot;,
528
- &quot;github_org&quot;: &quot;acme&quot;
530
+ &quot;model&quot;: &quot;gpt-4.1&quot;
529
531
  }
530
532
  </code></pre>
531
533
  <div class="tbl"><table>
@@ -533,90 +535,50 @@ bun src/index.ts init --global --profile azure-gpt4o
533
535
  <tr><th>Field</th><th>What it is</th></tr>
534
536
  </thead>
535
537
  <tbody>
536
- <tr><td><code>model</code></td><td>The Copilot model for ForgeQE's agent and Scribe, as Copilot names it (<code>claude-sonnet-4.6</code>, <code>gpt-5.4</code>). The sign-in lists the ones your seat has.</td></tr>
537
- <tr><td><code>engine_model</code></td><td>The GitHub Models model for the browser engine and the desktop agent, as <code>publisher/model</code>. It must take images. Default <code>openai/gpt-4.1</code>.</td></tr>
538
- <tr><td><code>github_org</code></td><td>Optional. Bills the engine's GitHub Models calls to this organization and applies its policies, rather than your personal free allowance.</td></tr>
539
- <tr><td><code>api_key_env</code></td><td>Optional. The variable holding the GitHub Models token; default <code>GITHUB_TOKEN</code>, else the GitHub CLI's sign-in.</td></tr>
538
+ <tr><td><code>model</code></td><td>The Copilot model, as the Copilot SDK names it (<code>gpt-4.1</code>, <code>claude-sonnet-4.6</code>). It must take screenshots. The sign-in lists your seat's models and which of them take images.</td></tr>
539
+ <tr><td><code>engine_model</code></td><td>Optional. Another Copilot model for the browser engine and the desktop agent.</td></tr>
540
+ <tr><td><code>api_key_env</code></td><td>Optional. The variable holding the GitHub token; without it ForgeQE reads <code>COPILOT_GITHUB_TOKEN</code>, then the token <code>forgeqe auth login copilot</code> saved, then the GitHub CLI's sign-in.</td></tr>
540
541
  </tbody>
541
542
  </table></div>
542
543
  <p>Sign in once per machine:</p>
543
- <pre><code class="language-bash">forgeqe auth login copilot # Copilot, for ForgeQE's agent and Scribe (GitHub.com or GitHub Enterprise, e.g. acme.ghe.com)
544
- forgeqe auth login github # the GitHub CLI, whose sign-in the browser engine uses for GitHub Models
544
+ <pre><code class="language-bash">forgeqe auth login copilot
545
545
  </code></pre>
546
- <p><code>forgeqe auth login copilot</code> shows a one-time code and opens GitHub's device page; authorize with the account that has the seat. The sign-in goes to your system's credential store like ForgeQE's other secrets. The agent runtime gets it in its environment for each session, never in a file, and the agent's shell commands never see it. A Copilot sign-in made with <code>opencode auth login</code> also works. In CI, set <code>GITHUB_TOKEN</code> to a token with the <code>models:read</code> permission for the engine.</p>
547
- <p><code>forgeqe auth</code> shows the sign-in on its <code>Copilot</code> line (<code>saved sign-in as dana</code>, or <code>agent runtime's own sign-in</code> for one made with <code>opencode auth login</code>); <code>forgeqe auth logout copilot</code> forgets it.</p>
546
+ <p>It offers two ways:</p>
547
+ <ul>
548
+ <li><strong>A fine-grained token</strong> (recommended). ForgeQE opens GitHub's token page: set <strong>Resource owner</strong> to your personal account and add the <strong>Copilot Requests</strong> permission, then paste the token (<code>github_pat_...</code>). The token goes to your system's credential store like ForgeQE's other secrets. Classic tokens (<code>ghp_</code>) do not work with Copilot.</li>
549
+ <li><strong>The GitHub CLI's sign-in</strong>, when <code>gh</code> is signed in.</li>
550
+ </ul>
551
+ <p>Either way ForgeQE checks the token by asking Copilot for your seat's models and lists them; the first time, the Copilot SDK downloads Copilot's runtime from GitHub (checked against GitHub's checksums). <code>forgeqe auth</code> shows the sign-in on its <code>Copilot</code> line; <code>forgeqe auth logout copilot</code> forgets a saved token.</p>
552
+ <p>The token reaches only the gateway's process. The engine gets the gateway's address and a one-time key that works nowhere else; the gateway refuses requests without that key and any request a web page sends. Model-run shell commands never see the token or the key.</p>
548
553
  <p>Which part uses what:</p>
549
554
  <div class="tbl"><table>
550
555
  <thead>
551
556
  <tr><th>Part</th><th>Model</th><th>Signs in with</th></tr>
552
557
  </thead>
553
558
  <tbody>
554
- <tr><td>ForgeQE's agent (the shell, <code>forgeqe run</code>)</td><td><code>model</code>, on the Copilot seat</td><td><code>forgeqe auth login copilot</code></td></tr>
555
- <tr><td>Scribe (writing, fixing and reviewing tests)</td><td><code>model</code>, on the Copilot seat</td><td><code>forgeqe auth login copilot</code></td></tr>
556
- <tr><td>The browser engine (every browser run: <code>run --direct</code>, suites, monitoring)</td><td><code>engine_model</code>, on GitHub Models</td><td><code>GITHUB_TOKEN</code>, or <code>forgeqe auth login github</code></td></tr>
557
- <tr><td>The desktop agent</td><td><code>engine_model</code>, on GitHub Models</td><td><code>GITHUB_TOKEN</code>, or <code>forgeqe auth login github</code></td></tr>
559
+ <tr><td>The browser engine (every browser run: <code>run --direct</code>, suites, monitoring)</td><td><code>engine_model</code> or <code>model</code>, on the Copilot seat</td><td><code>forgeqe auth login copilot</code></td></tr>
560
+ <tr><td>The desktop and mobile agents</td><td><code>engine_model</code> or <code>model</code>, on the Copilot seat</td><td><code>forgeqe auth login copilot</code></td></tr>
561
+ <tr><td>ForgeQE's agent and Scribe</td><td>Not on Copilot yet: give them another profile with <code>--profile</code>, or <code>code.model</code></td><td></td></tr>
558
562
  </tbody>
559
563
  </table></div>
560
- <p><code>code.model</code> still overrides <code>model</code> for ForgeQE's agent and Scribe; <code>code.model: github-copilot/&lt;model&gt;</code> uses the Copilot sign-in too, with any other profile active.</p>
561
- <p>What each part costs: the agent's and Scribe's requests count against the seat's premium requests. The engine's GitHub Models calls are billed by GitHub Models: the free allowance is rate-limited, so for suites set <code>github_org</code> to an organization with paid GitHub Models usage turned on. If a Copilot model is refused, check that your organization's Copilot policies allow that model. ForgeQE shows no cost for a <code>github_copilot</code> profile unless you give it a <code>price</code>.</p>
562
- <p><strong>In pipelines.</strong> <code>forgeqe suite ... --no-code</code>, which pipelines run, uses only the browser engine, so it needs GitHub Models and no Copilot sign-in. In GitHub Actions, give the job <code>permissions: models: read</code> and pass the workflow's own token as <code>GITHUB_TOKEN</code>; on Azure Pipelines or elsewhere, store a fine-grained token with the <code>models:read</code> permission as the <code>GITHUB_TOKEN</code> secret. A Copilot seat belongs to one person, so keep ForgeQE's agent and Scribe on developers' machines rather than signing a shared pipeline in to someone's seat.</p>
564
+ <p>Copilot runs one SDK session per model request, with no tools of its own and your request's system prompt in place of Copilot's. Each request is billed to the seat (GitHub AI credits or premium requests, by your plan), and GitHub may rate-limit frequent automated requests. If a model is refused, check that your organization's Copilot policies allow it. ForgeQE shows no cost for a <code>github_copilot</code> profile unless you give it a <code>price</code>.</p>
565
+ <p><strong>In pipelines.</strong> <code>forgeqe suite ... --no-code</code> uses only the browser engine. Set <code>COPILOT_GITHUB_TOKEN</code> to a fine-grained token with the Copilot Requests permission, stored as a pipeline secret. GitHub also documents the workflow's own <code>GITHUB_TOKEN</code> with <code>permissions: copilot-requests: write</code> for organization repositories whose Copilot policy allows it (<a href="https://docs.github.com/en/copilot/how-tos/copilot-sdk/auth/server-to-server-tokens" rel="noopener">server tokens</a>). A Copilot seat belongs to one person: prefer an organization-billed token over signing a shared pipeline in to someone's seat.</p>
566
+ <p>GitHub Models, which ForgeQE used for the engine before, was <a href="https://github.blog/changelog/2026-07-30-github-models-is-now-retired/" rel="noopener">retired by GitHub</a> on July 30, 2026. A profile that still has <code>engine_model: &quot;openai/...&quot;</code> or <code>github_org</code> needs updating: drop <code>github_org</code>, and use a Copilot model id.</p>
563
567
  <p><strong>When something goes wrong.</strong></p>
564
568
  <div class="tbl"><table>
565
569
  <thead>
566
570
  <tr><th>ForgeQE says</th><th>What to do</th></tr>
567
571
  </thead>
568
572
  <tbody>
569
- <tr><td><code>The github_copilot profile for … needs a GitHub Copilot sign-in</code></td><td>Run <code>forgeqe auth login copilot</code> (or <code>/auth copilot</code> in the shell).</td></tr>
570
- <tr><td><code>The browser engine reaches GitHub Models with a GitHub sign-in, and there is none</code></td><td>Run <code>forgeqe auth login github</code>, or set <code>GITHUB_TOKEN</code> to a token with <code>models:read</code>.</td></tr>
571
- <tr><td><code>… must be a Copilot model id</code>, <code>engine_model must be a GitHub Models id</code>, <code>github_org must be a GitHub organization name</code></td><td>Fix that field in <code>llms.json</code>: the sign-in lists the seat's Copilot models, and <a href="https://github.com/marketplace?type=models" rel="noopener">the GitHub Models catalog</a> lists <code>publisher/model</code> ids.</td></tr>
572
- <tr><td>The agent's model is refused, or not found</td><td>The seat's plan or your organization's Copilot policies do not allow that model: pick one the sign-in listed, or ask a Copilot admin to enable it.</td></tr>
573
- <tr><td>The browser engine gets <code>429</code> / rate limit errors</td><td>The GitHub Models free allowance is used up: set <code>github_org</code> to an organization with paid GitHub Models usage, or wait for the limit to reset.</td></tr>
574
- <tr><td>The code page does not open, or the sign-in says <code>not a GitHub Enterprise domain</code></td><td>Give only the host of your GitHub Enterprise (<code>acme.ghe.com</code>). ForgeQE opens only GitHub's own <code>https</code> device page.</td></tr>
575
- </tbody>
576
- </table></div>
577
- <p><code>init</code> also puts three folders in <code>.forgeqe/</code> to read, change, and commit:</p>
578
- <div class="tbl"><table>
579
- <thead>
580
- <tr><th>Folder</th><th>What it holds</th></tr>
581
- </thead>
582
- <tbody>
583
- <tr><td><code>.forgeqe/prompts/</code></td><td>Every prompt ForgeQE's agents work from, one markdown file each: ForgeQE's agent (<code>brain.md</code>), Scribe (<code>test-agent.md</code>, <code>test-rules.md</code>), the reviewer, healing, finding tests, pull request review, API tests and scenarios, exploratory charters, and the browser agent (<code>browser-*.md</code>, plus <code>browser-extra.md</code> for your own instructions).</td></tr>
584
- <tr><td><code>.forgeqe/skills/</code></td><td>The built-in skills (framework and app playbooks), to adapt to your project; add your own next to them.</td></tr>
585
- <tr><td><code>.forgeqe/knowledge/</code></td><td>Product knowledge templates: the product and its flows, roles, glossary, business rules, test data, environments.</td></tr>
573
+ <tr><td><code>GitHub Copilot needs a GitHub token, and there is none</code></td><td>Run <code>forgeqe auth login copilot</code> (or <code>/auth copilot</code> in the shell), or set <code>COPILOT_GITHUB_TOKEN</code>.</td></tr>
574
+ <tr><td><code>GitHub Copilot does not take classic tokens (ghp_)</code></td><td>Make a fine-grained token with the Copilot Requests permission, owned by your personal account.</td></tr>
575
+ <tr><td><code>GitHub Copilot did not accept the token</code></td><td>The token has no Copilot Requests permission, has expired, or the account has no Copilot seat.</td></tr>
576
+ <tr><td><code>… must be a Copilot model id</code>, <code>engine_model must be a Copilot model id</code></td><td>Fix that field in <code>llms.json</code>: the sign-in lists the seat's models. A GitHub Models id such as <code>openai/gpt-4.1</code> is not a Copilot id; use <code>gpt-4.1</code>.</td></tr>
577
+ <tr><td><code>ForgeQE's agent and Scribe cannot run on a github_copilot profile yet</code></td><td>Pick another profile for them (<code>--profile</code>, or Tab in the shell), or set <code>code.model</code>.</td></tr>
578
+ <tr><td>The model is refused, or not found</td><td>The seat's plan or your organization's Copilot policies do not allow that model: pick one the sign-in listed, or ask a Copilot admin to enable it.</td></tr>
579
+ <tr><td><code>GitHub Copilot runs through the ForgeQE engine, which is not set up</code></td><td>Run <code>forgeqe setup</code>.</td></tr>
586
580
  </tbody>
587
581
  </table></div>
588
- <p><strong>Your copy or ForgeQE's.</strong> <code>.forgeqe-manifest.json</code> in each folder records what was copied. A copy you have not changed still counts as ForgeQE's, so it keeps getting ForgeQE's improvements when you update; once you edit it, yours is used. When ForgeQE's version changes after you edited yours, ForgeQE says so at start-up. Running <code>init</code> again adds what is missing and never overwrites an edited file (with <code>--force</code> it refreshes only unchanged copies).</p>
589
- <p><strong>Prompts.</strong> Each file has a short header (<code>name</code>, <code>description</code>, <code>used_by</code>, <code>kind</code>) and the prompt; <code>{{&gt; name}}</code> includes another prompt (<code>test-rules</code> is included in the test, reviewer, and healing prompts, so one edit changes all three). ForgeQE looks in <code>prompts.dir</code>, then <code>.forgeqe/prompts/</code>, then <code>~/.forgeqe/prompts/</code> (for every project), then its own. Changes take effect on the next run or session.</p>
590
- <pre><code class="language-bash">forgeqe prompts # every prompt, what it is for, and where it comes from
591
- forgeqe prompts show test-agent # the prompt as the agent gets it (includes expanded; --raw for the file)
592
- forgeqe prompts diff test-rules # how your copy differs from ForgeQE's
593
- forgeqe prompts reset test-rules # back to ForgeQE's (yours is kept as test-rules.md.bak)
594
- forgeqe prompts check # each prompt against the prompting standard for its kind (exits 1 when one falls short)
595
- forgeqe prompts check brain # what to fix in one prompt: sections, rules, examples, contract, sentences
596
- forgeqe prompts eval # run the evaluation cases with the live model; pass rates against the last run
597
- forgeqe prompts eval refine --runs 3 # one prompt, three runs of each case
598
- </code></pre>
599
- <p><strong>The prompting standard.</strong> Every prompt follows one standard, and each technique is used where it fits the kind of prompt (<code>kind:</code> in its header):</p>
600
- <div class="tbl"><table>
601
- <thead>
602
- <tr><th>Kind</th><th>Prompts</th><th>Structure</th><th>Few-shot examples</th><th>Execution contract</th></tr>
603
- </thead>
604
- <tbody>
605
- <tr><td>agent: works with tools over many steps</td><td>brain, test-agent, heal, api-tests, pr-review, find-tests</td><td>Role, Objective, Execution contract, Rules, Output</td><td>where routing or reply format varies</td><td>Preconditions, Tools, Procedure, Stop when, On failure, Evidence</td></tr>
606
- <tr><td>structured: one answer in a fixed shape</td><td>refine, api-scenarios, explore-charters, eval-judge, test-reviewer, browser-skill-select</td><td>Role, Objective, Inputs, Rules, Output, Examples</td><td>two or more worked examples</td><td>–</td></tr>
607
- <tr><td>fragment: rules included in other prompts</td><td>test-rules, domain-rule, skill-rule, conventions, the browser agent's rules</td><td>rules only</td><td>–</td><td>–</td></tr>
608
- </tbody>
609
- </table></div>
610
- <ul>
611
- <li><strong>ASD-STE100</strong> Simplified Technical English everywhere, so each agent reads one clear instruction at a time: an instruction has 20 words or fewer and a description 25 or fewer, one instruction per sentence, the active voice, and approved words (&quot;make sure&quot;, not &quot;ensure&quot;). At least 80% of sentences. The check covers the rules that can be measured in text and a list of common words the standard does not approve; the standard's full dictionary is licensed and not included.</li>
612
- <li><strong>RFC 2119</strong> key words state the strength of every rule: <strong>MUST</strong> / <strong>MUST NOT</strong> are absolute, <strong>SHOULD</strong> / <strong>SHOULD NOT</strong> are strong defaults the agent may break only for a reason it states, <strong>MAY</strong> is a free choice, and a numbered procedure step is a MUST. A lower-case &quot;must&quot; or &quot;should&quot; is flagged as ambiguous. <code>conventions.md</code> says this at the top of every agent and structured prompt, and the browser engine adds it to the browser agent's instructions.</li>
613
- <li><strong>Structured prompting and few-shot examples</strong> make the output predictable: the same sections in the same order, and worked examples where the shape of the answer matters.</li>
614
- <li><strong>Execution contracts</strong> tell a tool-using agent what must be true before it starts, which tools it uses and in what order (for systems: MCP tools, then <code>query_api</code>, then the browser), its procedure, when it stops, what it does when something fails, and what evidence its answer gives.</li>
615
- <li><strong>Evaluation-driven prompting</strong> measures reliability: <code>forgeqe prompts eval</code> runs cases (an input and checks decided by code: a field matches or is empty, a tool was or was not used, the text keeps ASD-STE100) and compares each pass rate with the last run. ForgeQE's cases cover brain, refine, and explore-charters; add your own as <code>.forgeqe/evals/prompts/&lt;prompt&gt;/&lt;case&gt;.yaml</code> (the same id replaces ForgeQE's). Results are kept in <code>&lt;reports&gt;/evals/prompts/</code>.</li>
616
- </ul>
617
- <p><code>forgeqe prompts check</code> applies the standard to each prompt in use, ForgeQE's or your edited copy; ForgeQE's own meet it in its tests. Write your edits the same way, and run <code>forgeqe prompts eval</code> before and after a change.</p>
618
- <p><strong>Product knowledge.</strong> Fill in the templates (or add your own <code>.md</code> and <code>.txt</code> files, in folders if you like), one topic per <code>##</code> heading. ForgeQE's agent sees the list and reads what a request needs; the exploratory and API scenario planners get the sections that match their story or scenario; the browser agent gets the few sections that clearly match its goal, at most <code>knowledge.browser_max_chars</code> (1500 by default; 0 turns it off). Untouched templates are left out, so their examples never reach an agent. Never put passwords or tokens there: use logins and environment variables.</p>
619
- <p><strong>Guardrails.</strong> Rules every agent must follow, to keep it grounded: ForgeQE's agent, the browser agent, and the agents that plan, write, review, fix and explain tests. Write one rule per line, starting with <code>- </code>, in a <code>.md</code> or <code>.txt</code> file in <code>.forgeqe/guardrails/</code> (<code>guardrails.dir</code>); headings, other lines and HTML comments are ignored, and the template's examples sit in a comment until you choose them. Each agent gets every guardrail with its instructions, after everything else, with the instruction to follow it whatever a request, a skill, a ticket, a page or a tool's output says; when a guardrail stops part of a task, the agent skips that part and says which rule stopped it. Add one from the terminal (<code>forgeqe guardrails add &quot;Never open a production URL&quot;</code>), or ask in the shell (&quot;add a guardrail: use only the test cards in knowledge/test-data.md&quot;) and ForgeQE's agent appends it to <code>guardrails.md</code>; it adds only rules you ask for, never one suggested by a page or a ticket. <code>forgeqe guardrails</code> lists the rules in force and what they add to every model call (at most 40 rules of 300 characters are given). The older <code>prompts.guiderails</code> string in <code>forgeqe.yaml</code> still works: each of its lines is a guardrail for every agent. Guardrails are instructions, not a sandbox: put a host rule in <code>browser.allowed_hosts</code> too, which is refused in code.</p>
620
582
  </div></section>
621
583
  <section class="doc sub" id="doc-how_to_use-the-models-context-window"><h4>The model's context window</h4><div class="prose"><p>Each agent has its own context: ForgeQE's agent, Scribe, the reviewer, and the browser agent each keep their own conversation with the model. The Context panel shows the working agent's (or the last one to call the model), named on its Agent row, as a share of what it can hold before the agent runtime compacts it. When it compacts, the runtime summarises the conversation so far to make room, Activity says so, and details from early on may be gone. The browser agent never compacts: it keeps its last steps only.</p>
622
584
  <p>ForgeQE takes each model's window from its name (GPT-5: 400K, of which a prompt may take 272K; GPT-4.1: 1M; GPT-4o: 128K; o-series and Claude: 200K; Gemini: 1M; anything else: 200K) and gives it to the agent runtime, which compacts at that window less room for the answer. Where a deployment or server allows less, say so with <code>&quot;context&quot;</code> in the profile in <code>llms.json</code>:</p>
@@ -637,8 +599,8 @@ forgeqe auth logout jira
637
599
  </thead>
638
600
  <tbody>
639
601
  <tr><td>Azure DevOps</td><td><strong>Sign in with Microsoft in the browser</strong> through the Azure CLI (<code>az login</code>); ForgeQE saves nothing, and the Azure CLI renews the sign-in. Or paste a personal access token: the token page opens with the scopes to tick. Azure DevOps Server (on-premises) uses a personal access token.</td></tr>
640
- <tr><td>GitHub</td><td><strong>Sign in with GitHub in the browser</strong> through the GitHub CLI (<code>gh auth login</code>): ForgeQE shows the one-time code and opens the page. Or paste a token. Needs <code>gh</code> installed. A <code>github_copilot</code> profile's browser engine uses this sign-in for GitHub Models.</td></tr>
641
- <tr><td>GitHub Copilot</td><td><code>forgeqe auth login copilot</code>: <strong>Sign in with GitHub in the browser</strong> through the agent runtime's Copilot sign-in, on GitHub.com or GitHub Enterprise (<code>acme.ghe.com</code>). ForgeQE shows the one-time code, opens GitHub's device page, and lists the Copilot models your seat has. For a <code>github_copilot</code> profile; see <a href="../#github-copilot-seats-as-the-model">GitHub Copilot seats as the model</a>.</td></tr>
602
+ <tr><td>GitHub</td><td><strong>Sign in with GitHub in the browser</strong> through the GitHub CLI (<code>gh auth login</code>): ForgeQE shows the one-time code and opens the page. Or paste a token. Needs <code>gh</code> installed. <code>forgeqe auth login copilot</code> can use this sign-in for a <code>github_copilot</code> profile.</td></tr>
603
+ <tr><td>GitHub Copilot</td><td><code>forgeqe auth login copilot</code>: <strong>Paste a fine-grained token</strong> with the Copilot Requests permission (ForgeQE opens GitHub's token page), or use the GitHub CLI's sign-in. ForgeQE checks it with Copilot and lists the models your seat has. No other app is authorized on your GitHub account. For a <code>github_copilot</code> profile; see <a href="../HOW_TO_USE.md#github-copilot-seats-as-the-model">GitHub Copilot seats as the model</a>.</td></tr>
642
604
  <tr><td>Jira, Confluence</td><td>Your Atlassian API tokens page opens (Cloud) or your personal access tokens page (Server/Data Center); create one and paste it. Confluence on the same Cloud site as Jira reuses the Jira sign-in.</td></tr>
643
605
  <tr><td>TestRail, Zephyr Scale, qTest, Xray Cloud</td><td>The page with your API key opens, or ForgeQE says where to find it; paste it.</td></tr>
644
606
  <tr><td>Anything else</td><td><code>forgeqe auth login OPENAI_API_KEY</code> (any variable name) saves that one secret: a model key, a Teams webhook URL, an SMTP or saved-login password.</td></tr>
@@ -646,31 +608,8 @@ forgeqe auth logout jira
646
608
  </table></div>
647
609
  <p>Atlassian's OAuth sign-in needs a client secret, which a command-line tool cannot keep secret, so Jira and Confluence use API tokens: they are what Atlassian recommends for scripts and tools.</p>
648
610
  <p>Each token is checked against the system before it is saved, and the system is switched on: a saved Jira sign-in sets the site URL too, so <code>sources.jira</code> is not needed in <code>forgeqe.yaml</code>. Saved values go in under the variable names the config already reads (<code>JIRA_API_TOKEN</code>, <code>ADO_PAT</code>, ...). A variable exported in the terminal is used before a saved sign-in, and a saved sign-in before a <code>.env</code> file, so pipelines keep using their secrets. The shell commands the agents run never see the saved secrets.</p>
649
- <p><strong>Prices.</strong> ForgeQE prices every model call at the model's <strong>list price</strong>, from two public price lists: <a href="https://models.dev" rel="noopener">models.dev</a> (the list the agent runtime uses) and LiteLLM's model price list, which also has Azure's Data Zone prices. It keeps the few fields it needs in <code>~/.forgeqe/prices.json</code>, waits a few seconds for it the first time, refreshes it once a day in the background, and uses the last copy offline; only the public lists are downloaded, nothing about your runs is sent (<code>FORGEQE_PRICES=off</code> stops the downloads). Costs at list price say so: <code>$1.73 (list price of gpt-5.4 (Azure Global) from models.dev, 2026-10-06)</code>. For an Azure Data Zone deployment, which costs about 10% more, set <code>&quot;deployment_type&quot;: &quot;data_zone&quot;</code> on the profile.</p>
650
- <p><strong>Model parameters.</strong> Models differ in what a request may carry: reasoning models (o1, o3, o4, gpt-5) take <code>max_completion_tokens</code> instead of <code>max_tokens</code> and only the default temperature, and some OpenAI-compatible servers refuse <code>max_completion_tokens</code>, <code>reasoning_effort</code> or <code>response_format</code>. ForgeQE works this out by itself. A profile is a reasoning model when its <code>model</code> or <code>deployment</code> name says so (an Azure deployment called <code>chat-prod</code> with <code>&quot;model&quot;: &quot;gpt-5.4&quot;</code> counts), or when it sets <code>&quot;reasoning&quot;: true</code>; <code>&quot;reasoning&quot;: false</code> says it is not. When the API still refuses a parameter, the browser, desktop and mobile agents send the call again without it and leave it out of later calls to that model. Nothing needs setting for a new model.</p>
651
- <p>A profile's own <code>&quot;price&quot;: { &quot;input&quot;: 2.5, &quot;output&quot;: 15, &quot;cached_input&quot;: 0.25 }</code> (per million tokens) wins, for a negotiated or enterprise rate. Leave <code>currency</code> out and reports use this machine's currency, or set <code>&quot;currency&quot;: &quot;USD&quot;</code>. Without <code>cached_input</code>, cached input gets the model's cache discount from its list price, and the cost is marked as an estimate (<code>≈$1.52</code>); with no list price for the model either, cached input is charged at the full rate and the cost is an upper bound (<code>up to $6.11</code>). When your price differs from the list price, ForgeQE says so (<code>forgeqe models</code>, the shell, and the start of each command), since that is usually a typo or an old price. Anthropic's cache writes are priced at their own rate. A provisioned (PTU) deployment is billed per hour, not per token, so its costs are what the tokens would cost at list price. Requests over a model's long-context threshold (272K tokens for GPT-5.4) cost more and are priced at the normal rate.</p>
652
- <p><strong>Usage after every job.</strong> Every browser run, Scribe job, heal, suite, and monitor round ends with a <code>Usage</code> line: tokens in (and how many of them were cached), tokens out, and the cost at the profile's price. A request to ForgeQE's agent in the shell, and every <code>forgeqe run</code>, <code>code</code>, <code>suite</code>, and <code>heal</code> command, ends with the total for everything it started, by role:</p>
653
- <pre><code>This request used 1.1M tokens (1.1M in, 30.0K of it cached · 34.7K out) · $3.07 — ForgeQE's agent 42.0K · $0.12, browser agent 52.7K · $0.15, Scribe 1.0M · $2.80
654
- </code></pre>
655
- <p><strong>Reuse before rebuild, extend before duplicate.</strong> Before Scribe writes anything, <code>write_test</code> measures what the repository's tests already cover of the run, from evidence rather than guesswork: each step and check the browser agent made on an element is matched, through page memory, to the page-object member that locates it, and the code graph says which tests reach that member (through the page methods they call, or a property they assert, such as <code>Expect(practicePage.AjaxStatus)</code>). Steps on elements no page object locates yet are listed as not measured, since a test may do them its own way. Then:</p>
656
- <div class="tbl"><table>
657
- <thead>
658
- <tr><th>What it finds</th><th>What happens</th></tr>
659
- </thead>
660
- <tbody>
661
- <tr><td>One test covers every measured step and check</td><td><strong>Reuse</strong>: Scribe only finds the command that runs it, and changes nothing. If it fails, nothing is changed: you are asked whether to update it or write a new test.</td></tr>
662
- <tr><td>One test covers most (40% or more)</td><td><strong>Extend</strong>: Scribe adds only the missing steps and checks to that test, keeping every step and assertion it has; the reviewer rejects a removed or weakened assertion, or the steps copied into a new file</td></tr>
663
- <tr><td>Two or more tests cover most of it</td><td><strong>You choose</strong>: the evidence and the options (reuse, extend with how many steps it adds, a new test, cancel)</td></tr>
664
- <tr><td>Nothing covers much</td><td><strong>New</strong>: a new test, reusing the page objects and helpers as before</td></tr>
665
- </tbody>
666
- </table></div>
667
- <p>A card says which it did and why (<em>Extending PracticeTests.Flow · covers 12 of 15; only 3 are missing</em>). Say how when you know (&quot;extend the login test&quot;, &quot;write a separate test&quot;, &quot;update it&quot;) and ForgeQE's agent passes that on without asking. In every mode Scribe adds page-object members rather than changing what an existing member does for the other tests that use it.</p>
668
- <p><strong>The same flow asked again.</strong> When a session writes tests and you ask for a flow an earlier job already wrote a test for, ForgeQE runs that test first, before any browser run. If it passes in a real browser, that is the answer: no browser run, no Scribe, and no new test file, in the seconds the test takes (<em>✓ Existing test passed · 19s · no browser run or new test needed</em>). If it fails, the flow runs in the browser and Scribe updates that test instead of writing another. ForgeQE's agent rewords a goal a little each time, so goals count as the same flow when they share their content words (endings, case, and phrasing aside) and the new goal keeps everything the earlier one checked; the same steps with a different final check, a shorter flow, or another site are a new flow. The agent also compares the earlier goal with your request and runs fresh when you ask for a check the earlier test does not have. Ask for &quot;a fresh run&quot; or &quot;a new test&quot; to skip the reuse.</p>
669
- <p><strong>When the flow cannot be reproduced.</strong> If ForgeQE's own replay of the run could not reach the run's final page and Scribe's first test fails too, the job stops there and says which replayed step failed and why, instead of spending its remaining attempts on a flow no script can repeat as recorded.</p>
670
- <p><strong>Why a test failed.</strong> When ForgeQE's run of Scribe's test fails, the ✗ line and the next attempt say why in the test's own words: dotnet test's Error Message, pytest's <code>E</code> lines, or the error with its Expected and Received values (Playwright, Jest, JUnit), not the runner's summary line.</p>
671
- <p><strong>Keeping Scribe's cost down.</strong> ForgeQE reads the repository's test setup itself before Scribe starts (stack, settings files and their browser values, the test id attribute, base classes, page objects and their methods, one test command) and hands it over, so Scribe does not discover it with dozens of file reads. Each attempt after the first starts a fresh session from a summary of the job (handoff, files changed, what each attempt did, the latest failure), so later attempts cost about what the first did. <code>code.max_attempts</code> (default 3) and <code>code.max_tokens</code> (default 1,500,000; <code>0</code> turns it off) cap a job, and the token budget is checked during an attempt too, so one long attempt is stopped when the total passes it. The reviewer sorts its issues into correctness (the test does not test what the goal says, or not reliably) and coding standards (test data or credentials in a page object, naming, hard waits, leftovers). Coding standards never fail a test that works: it is accepted, and the coding agent gets one more round (not counted in <code>max_attempts</code>) to fix them; if that round breaks the test, ForgeQE puts back the version that passed and lists the standards issues in the notes (<code>code.tidy: false</code> skips the round). A test the reviewer rejects twice for correctness stops the job with its issues, since that usually means the test case and the page disagree. Under each step, the handoff shows that step's part of the page (its section and neighbours, from the replay's page snapshot) instead of whole snapshots. When a test fails, ForgeQE shows the failing element on the test's page at the failure and on the replay's page, so most failures are fixed without opening a browser. Scribe's live browser is switched on only when the same failure repeats, and it does not send a page snapshot after every click or keystroke. <code>code.max_steps</code> (default 50) caps the tool calls in one of Scribe's turns. <code>code.review_profile</code> runs the reviewer on a cheaper llms.json profile.</p>
672
- <p><strong>Generated code.</strong> What ForgeQE writes is marked, so the team can see it and measure it. A file ForgeQE created starts with one line in that language's comment, such as <code>// Generated with ForgeQE · fq:k3x7m2qa4bdi · run 20261019-162847</code>; the <code>fq:</code> id is a hash of the file's contents without that line, so it shows whether the file was changed since (<code>code.header: false</code> leaves the line out; a JSON file never gets one). Files it only edited get no header. Every file it created or edited is recorded per run in <code>.forgeqe/generated.jsonl</code> (commit it with the code): the lines it added, as short hashes, never their text. <code>forgeqe generated</code> (or <code>/generated</code> in the shell, <code>--json</code> for a pipeline) says how many lines ForgeQE wrote, how many are still in the code, its share of the files it touched, and which created files are as it wrote them. Commit messages ForgeQE's agent writes end with a <code>Generated-by: ForgeQE (fq:…)</code> trailer, and its pull request descriptions with a line naming the runs' <code>fq:</code> ids and how much they wrote. The ids are plain hashes anyone can recompute: they show what ForgeQE wrote, not who owns the code.</p>
673
- <p><code>forgeqe.yaml</code> selects the active profile:</p>
611
+ </div></section>
612
+ <section class="doc sub" id="doc-how_to_use-choosing-the-model-and-the-browser"><h4>Choosing the model and the browser</h4><div class="prose"><p><code>forgeqe.yaml</code> selects the active profile:</p>
674
613
  <pre><code class="language-yaml">llm:
675
614
  models_file: ./llms.json
676
615
  active: azure-gpt4o
@@ -685,18 +624,15 @@ browser:
685
624
  allowed_hosts: []
686
625
  judge: false # true: a second model call checks each finished run and can fail it
687
626
  overlay: true
627
+ window: maximized # visible browser: maximized | viewport (a window of viewport's size)
628
+ video: true # video.mp4 of each run, shown in its report
688
629
  # start_url: https://staging.example.com
689
630
  </code></pre>
690
631
  <p><code>judge</code> is off by default. When it is on, one more model call reads the finished run's steps and screenshots and can fail a run the agent completed, for example when five scores were read off one results page rather than one step per item. That strictness sent runs into loops of re-checking, so turn it on only when you want that second opinion.</p>
691
632
  <p>Each step waits for the tab the agent is on to finish loading (up to 8 seconds, 3 for a tab that stays blank) before the page is read, so a result that opens in a new tab is not mistaken for a blank page.</p>
692
633
  <p>With a visible browser (<code>headless: false</code> or <code>--headed</code>), ForgeQE opens on its own start page, and every page it drives gets a glowing edge and a bar along the bottom showing, in plain words, what the step is for and what the agent is doing (&quot;Step 19 · Check whether the results can be sorted by Fastest · Looking through the page's buttons&quot;). Your mouse and keyboard are locked while the agent thinks and unlocked while it clicks and types, so a stray click can't change the test. None of this appears in the screenshots the model sees or in the report. <code>overlay: false</code> turns it off. <code>forgeqe login &lt;name&gt;</code> shows the same bar without the lock, since you are the one signing in.</p>
634
+ <p>A visible browser opens maximized. <code>window: viewport</code> opens it at <code>viewport</code>'s width and height instead; a headless page is always that size (1280×800 by default). Each run is recorded to <code>video.mp4</code> in its run folder, and the report plays it on the run's page; <code>video: false</code> turns recording off. The video needs packages that <code>forgeqe setup</code> adds to the engine (an engine set up before this version gets them on its next run).</p>
693
635
  <p><code>start_url</code> is optional. Leave it out and the agent picks the site from the goal; ForgeQE never guesses a URL from the goal text. <code>allowed_hosts</code> empty allows every host. Set <code>[&quot;example.com&quot;]</code> to keep the browser on that domain and its subdomains. <code>prompts.step_instruction</code> is appended to the engine's system prompt; the lines of <code>prompts.guiderails</code> are guardrails for every agent (see Guardrails above).</p>
694
- <p><code>llm.routing</code> is <code>off</code> by default for existing projects (<code>forgeqe init</code> writes <code>auto</code>); <code>shadow</code> logs each pick in the session log but changes nothing. <code>--profile</code> and <code>code.model</code> always win, and <code>llm.routes</code> limits or pins the profiles an area may use. ForgeQE checks and rates the models once and keeps the result in <code>.forgeqe/model-profile.json</code> (you can edit the ratings). Details: <a href="../docs/MODEL_ROUTING.md">docs/MODEL_ROUTING.md</a>.</p>
695
- <p>List profiles, and the model each area gets:</p>
696
- <pre><code class="language-bash">bun src/index.ts models
697
- bun src/index.ts models --routes # the pick per area, and why
698
- bun src/index.ts models --routes --profile-models # check and rate the models again
699
- </code></pre>
700
636
  </div></section>
701
637
  </section>
702
638
  </section>
@@ -752,7 +688,7 @@ Add a GitHub Actions workflow that runs tests/cases.yaml on every PR, and make i
752
688
  <tr><td><code>/init</code></td><td>Set ForgeQE up in this folder: <code>forgeqe.yaml</code>, <code>llms.json</code>, editable prompts, skills, knowledge</td></tr>
753
689
  <tr><td><code>/skills [name]</code></td><td>The skills each agent can use, or one skill's instructions</td></tr>
754
690
  <tr><td><code>/db [test|tables|refresh|batches|examples|add] [name]</code></td><td>The databases set up here (see <a href="#doc-how_to_use-databases">Databases</a>)</td></tr>
755
- <tr><td><code>/report</code>, <code>/report export</code>, <code>/report &lt;ask&gt;</code></td><td>The team report of every run here and the ones teammates shared; your bundle to share; or ask the agent for a report, e.g. <code>/report open defects in all projects</code> (see <a href="../#team-report">Team report</a>)</td></tr>
691
+ <tr><td><code>/report</code>, <code>/report export</code>, <code>/report &lt;ask&gt;</code></td><td>The team report of every run here and the ones teammates shared; your bundle to share; or ask the agent for a report, e.g. <code>/report open defects in all projects</code> (see <a href="#doc-how_to_use-team-report">Team report</a>)</td></tr>
756
692
  <tr><td><code>/auth [system]</code></td><td>Sign in to a system in the browser; <code>/auth</code> alone shows what is signed in</td></tr>
757
693
  <tr><td><code>/clear</code></td><td>Clear the transcript</td></tr>
758
694
  <tr><td><code>/help</code></td><td>Examples and commands</td></tr>
@@ -760,7 +696,7 @@ Add a GitHub Actions workflow that runs tests/cases.yaml on every PR, and make i
760
696
  </tbody>
761
697
  </table></div>
762
698
  <p><code>start_url</code> in <code>forgeqe.yaml</code> or <code>--url</code> pins a start page for browser runs, such as a staging app. Passed runs become tests in the current folder; <code>forgeqe start --repo ../app</code> puts them in that repo instead, and <code>--no-code</code> turns tests off.</p>
763
- <p>ForgeQE's agent runs on the agent runtime with the active profile's model (or <code>code.model</code>), or, with <code>llm.routing: auto</code>, the model the router picks for the <code>agent</code> area when the session starts (<code>--profile</code> and <code>code.model</code> still win); a <code>github_copilot</code> profile runs it on your Copilot seat. ForgeQE downloads the agent runtime into <code>~/.forgeqe/tools</code> when it is not installed. When the agent runtime cannot be downloaded, ForgeQE says the agent is off and runs each message as one browser goal.</p>
699
+ <p>ForgeQE's agent runs on the agent runtime with the active profile's model (or <code>code.model</code>), or, with <code>llm.routing: auto</code>, the model the router picks for the <code>agent</code> area when the session starts (<code>--profile</code> and <code>code.model</code> still win); it cannot run on a <code>github_copilot</code> profile yet. ForgeQE downloads the agent runtime into <code>~/.forgeqe/tools</code> when it is not installed. When the agent runtime cannot be downloaded, ForgeQE says the agent is off and runs each message as one browser goal.</p>
764
700
  <p>While a run is going, the bar under the prompt shows a spinner, the tokens used, and the elapsed time, and the transcript ends with what the agent is doing now (<code>Thinking about step 3…</code>, <code>Acting in the browser (step 3)…</code>). Each step starts with <code>+ Thought: 4.2s</code>, the time the model took to decide. When the run ends, a summary card closes the transcript:</p>
765
701
  <pre><code>┃ ✓ Task complete 10 steps · 1m 42s · 65.3K tokens
766
702
  ┃
@@ -997,6 +933,99 @@ forgeqe memory stats # how often memory stood in for the
997
933
  </section>
998
934
  </div>
999
935
  </article>
936
+ <article class="spec feat" id="cap-record-web">
937
+ <div class="hd"><span class="mono">Browser</span><h3>Record a web test by hand</h3></div>
938
+ <p class="ds">Opens a browser with the Recorder panel docked in the page while the user does the steps of a web test themself, and saves their clicks, typing, choices and keys in order as a run, with the checks, kept values, test data and notes they add; the coding agent then turns it into a test in the repository's framework.</p>
939
+ <div class="facts">
940
+ <div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>I'll show you the checkout steps in the shop: record them</li><li>Record my steps on the cart page and make a test of it</li></ul></div>
941
+ <div class="fact"><span class="mono">What it needs</span><ul class="needs"><li><span class="need opt">Optional</span><span>The coding agent, Scribe: a live model profile (or code.model) and the agent runtime, which forgeqe setup installs</span></li></ul></div>
942
+ <div class="fact"><span class="mono">Limits</span><ul class="plainlist"><li>Local Chrome or Chromium only, one recording at a time; drag and drop and steps inside iframes are not recorded yet</li></ul></div>
943
+ <div class="fact"><span class="mono">Tools</span><p class="tools"><code>web_record</code></p></div>
944
+ </div>
945
+ <div class="detail"><span class="mono">In detail</span>
946
+ <section class="doc" id="doc-how_to_use-recording-a-web-test-by-hand">
947
+ <h4>Recording a web test by hand</h4>
948
+ <div class="prose"><p>When you would rather show the steps than describe them, record them. ForgeQE opens Chrome on the app's page with the
949
+ <strong>Recorder</strong> panel docked in a corner of the page, in the same window: you click, type and choose as usual, and each
950
+ action is listed in the panel as a numbered step, in order. Drag the panel by its title to another corner, or make it
951
+ small (a pill that shows the step count); it comes back where you left it on every page.</p>
952
+ <pre><code class="language-bash">forgeqe record shop --name &quot;Apply a coupon at checkout&quot; # a web app from env.yaml
953
+ forgeqe record --url https://staging.shop.test/cart # or any page
954
+ </code></pre>
955
+ <p>In the shell, say &quot;I'll show you the checkout, record it&quot; (or <code>/record shop</code>). What you can add while recording:</p>
956
+ <div class="tbl"><table>
957
+ <thead>
958
+ <tr><th>In the panel</th><th>In the page: Ctrl+Shift+Space, then</th><th>What it adds</th></tr>
959
+ </thead>
960
+ <tbody>
961
+ <tr><td>Check, then click an element</td><td><strong>A</strong></td><td>A check: its text, or its value for a field, or checked for a checkbox. Change the kind (text, visible, hidden, value) and mark <strong>live data</strong> (check the pattern, not this value) in the panel</td></tr>
962
+ <tr><td>Variable, then click an element</td><td><strong>V</strong></td><td>Keeps the element's text for later steps: typing the same text later is written as <code>{{name}}</code></td></tr>
963
+ <tr><td>Wait for, then click an element</td><td><strong>W</strong></td><td>The test waits for the element to show</td></tr>
964
+ <tr><td>Test data</td><td><strong>D</strong></td><td>The last value you typed becomes test data the test reads from its data</td></tr>
965
+ <tr><td><code>note: …</code> in the box</td><td><strong>N</strong></td><td>A note on the last step: &quot;this is the cart page object&quot;</td></tr>
966
+ <tr><td>Pause / Undo / Stop, ✕ on a step</td><td><strong>P</strong> / <strong>Z</strong> / <strong>S</strong></td><td>Pause records nothing until you resume; ✕ removes that step</td></tr>
967
+ </tbody>
968
+ </table></div>
969
+ <p>Type a check in words in the panel's box, or in the terminal: <code>the total shows &quot;$40.00&quot;</code>. Quoted text the page shows
970
+ becomes a check on the element showing it; without quotes, one model call finds the text that shows it on the page. A
971
+ check the page cannot show yet is kept in your words, and the coding agent writes it. In the terminal, <code>undo</code>, <code>pause</code>,
972
+ <code>delete 4</code> and <code>stop</code> work too.</p>
973
+ <p>What is recorded: clicks (a double click as one step), the final text of each field you typed in (not each key), choices
974
+ in drop-downs, Enter and Escape, key chords, pages you open from the address bar, and Back. A click on a menu item that
975
+ opened when the pointer went over its trigger gets a hover on the trigger first. Password fields are written as an
976
+ environment variable (the saved login's <code>password_env</code>, or <code>&lt;SITE&gt;_PASSWORD</code>), and text equal to a saved login's value
977
+ as its variable: what you type in them is never stored. A file you choose for an upload is recorded by its path in the
978
+ working folder; one outside it, a hidden file (<code>.env</code>, anything under <code>.git</code>), or one named like credentials or keys is
979
+ listed as not recorded. Drag and drop is listed as not recorded. <code>--signed-in</code> starts with the saved sessions of
980
+ <code>logins:</code>; without it, record the sign-in too.</p>
981
+ <p><strong>Data that changes from run to run.</strong> A test that spells out this recording's price, date, or order number fails on
982
+ the next run, so ForgeQE keeps what holds and makes the rest a pattern, a variable, or a value made fresh:</p>
983
+ <div class="tbl"><table>
984
+ <thead>
985
+ <tr><th>In the recording</th><th>In the test</th></tr>
986
+ </thead>
987
+ <tbody>
988
+ <tr><td>A check of a price, date, time, order number, or count (&quot;$14.15&quot;, &quot;2,463 found&quot;)</td><td>A check of its pattern: <code>/^\$\d[\d,]*\.\d{2}$/</code></td></tr>
989
+ <tr><td>The same value checked again later (the product's price, then the cart total)</td><td>The first check keeps it as <code>{{price}}</code>; the later one checks <code>{{price}}</code></td></tr>
990
+ <tr><td>A link or button named with such data (&quot;Combination Pliers $14.15&quot;)</td><td>Found by the part that holds: <code>getByRole(&quot;link&quot;, { name: /Combination Pliers/ })</code></td></tr>
991
+ <tr><td>A date you typed</td><td>The date the same number of days from today: <code>{{$date:+3:%Y-%m-%d}}</code></td></tr>
992
+ <tr><td>A value the app refuses twice (a new account's email)</td><td>A part new on each run: <code>alice+{{$unique}}@example.com</code></td></tr>
993
+ </tbody>
994
+ </table></div>
995
+ <p>Prices, dates, times, ids, and long numbers are found by their form; a small number on its own (&quot;1&quot; in the cart) stays
996
+ exact, as it is usually what the check is about. When you stop, one model call sorts the values the forms cannot
997
+ tell (a new account's email, &quot;today's deal&quot;); without a model, those stay as recorded. Each change shows in the panel
998
+ with a tick box: untick it to keep the value as recorded.</p>
999
+ <p><strong>Recording a test case.</strong> <code>forgeqe record --case SHOP-12</code> (any source ForgeQE reads cases from: Jira, Azure DevOps,
1000
+ Xray, TestRail, a file) starts on the case's page and shows its steps as a checklist in the panel. Click the step you
1001
+ are on (or <strong>Next case step</strong>): what you record belongs to it, and a check you add there takes its expected result.
1002
+ Each expected result shows <strong>✓ checked</strong> or <strong>no check yet</strong>; Stop warns once about the ones with no check, and a second
1003
+ Stop stops. The test is named and tagged for the case, and mapped to it once it passes.</p>
1004
+ <p>When you stop, the run folder holds <code>workflow.yaml</code> and <code>trace.md</code> as for any run. With a repository (<code>--repo</code>, or
1005
+ <code>code.repo</code>), ForgeQE asks whether to turn it into a test. First a planner reads the repository and shows the plan:</p>
1006
+ <pre><code>Test tests/add-to-cart.spec.ts › adds pliers to the cart
1007
+ Steps
1008
+ 2. search: HomePage.searchFor()
1009
+ 4. open the product: new HomePage.product(name)
1010
+ 6. add to the cart: new ProductPage.addToCart()
1011
+ New code
1012
+ ProductPage in pages/ProductPage.ts
1013
+ Checks
1014
+ after step 5: expect(product.price).toHaveText(/^\$\d+\.\d{2}$/)
1015
+ after step 6: expect(product.cartQuantity).toHaveText(&quot;1&quot;)
1016
+ </code></pre>
1017
+ <p>It asks at most three questions where the repository allows two good answers (a new file, or the existing spec?), then
1018
+ <strong>Write it</strong>, <strong>Change the plan</strong> (say what to change, in your words), or <strong>Cancel</strong>. The coding agent replays the
1019
+ recording in Playwright, writes the test to the plan you agreed in the repository's framework, and asserts each check at
1020
+ the step where you added it. In the shell, &quot;change the plan&quot; comes back to ForgeQE's agent, which asks you what to
1021
+ change. <code>--no-plan</code> (or <code>--yes</code>) skips the plan. Later: <code>forgeqe code &lt;run folder&gt;</code>.</p>
1022
+ <p>The recorder runs in an isolated world in each page, so the page's own scripts can neither see it nor add steps, and it
1023
+ records only real input: clicks and keys a script fakes are ignored. The panel's own clicks and keys never reach the page
1024
+ and are never recorded, and it stays above the page's dialogs. Recording uses local Chrome or Chromium.</p>
1025
+ </div>
1026
+ </section>
1027
+ </div>
1028
+ </article>
1000
1029
  </section>
1001
1030
  <section class="chapter" id="area-test-cases" aria-labelledby="area-test-cases-h">
1002
1031
  <header class="chead"><span class="mono">05</span><h2 id="area-test-cases-h">Test cases</h2><p>Run, check, write, and explore manual test cases</p></header>
@@ -1013,17 +1042,6 @@ forgeqe memory stats # how often memory stood in for the
1013
1042
  <h4>Test cases and suites</h4>
1014
1043
  <div class="prose"><p>Test cases you already have can be run two ways. In the shell or with <code>forgeqe run</code>, ask in plain words (&quot;run SHOP-12 and SHOP-14&quot;, &quot;run test plan 12 suite 34&quot;, &quot;run the cases in smoke.yaml&quot;) or paste the cases, and the agent works out the source. <code>forgeqe suite</code> takes explicit references and runs the same way every time, with no agent involved, which is what pipelines call. Each case is one goal: ForgeQE tells the agent to do the steps in order and to check every expected result, and the agent's verdict decides whether the case passed. A case with no steps (plain text, a Jira story) is run from its description.</p>
1015
1044
  </div>
1016
- <section class="doc sub" id="doc-how_to_use-team-report"><h5>Team report</h5><div class="prose"><p><code>forgeqe bi</code> builds one report from every run in the reports folder: browser suites, monitor rounds, release readiness, single browser runs, desktop and mobile runs, exploratory testing, and API runs, each tagged with its run type. It builds an <a href="https://github.com/evidence-dev/evidence" rel="noopener">Evidence</a> site (needs Node.js); without Node.js, or when Evidence fails, it writes ForgeQE's own page, which opens straight from disk (<code>bi:</code> in <code>forgeqe.yaml</code>; see <a href="../README.md#static-bi">README.md</a>).</p>
1017
- <pre><code class="language-bash">forgeqe bi init # scaffold the Evidence project and install its packages
1018
- forgeqe bi build # refresh the data and build (ForgeQE's own page if Evidence cannot build)
1019
- forgeqe bi build --static # ForgeQE's own page only, no Node.js needed
1020
- forgeqe bi sync # refresh the datasets and pages, without building
1021
- forgeqe bi export [--out FILE|DIR] # your runs, in one file to share
1022
- forgeqe bi build --from //share/qa-reports # this machine's runs plus every bundle there (repeatable)
1023
- </code></pre>
1024
- <p>For one report across the team, each person runs <code>forgeqe bi export</code>, which writes their runs and readiness reports to one file in <code>.forgeqe/reports/shared/</code>. Whoever builds puts the files in <code>&lt;out_dir&gt;/collected/</code> (<code>.forgeqe/reports/collected/</code> by default), or lists shared folders under <code>bi.collect</code>, or passes <code>--from</code>, and runs <code>forgeqe bi build</code>. Every table gets <code>user</code> and <code>machine</code> columns; the report adds <strong>Results by run type</strong> and <strong>Case results by person</strong> tables. A run that arrives twice (in two bundles, or a bundle and this machine) counts once. A bundle is read as data only: a file that is not one, or is larger than 64 MB, is skipped with a note.</p>
1025
- <p>In the shell, <code>/report</code> builds ForgeQE's own page from the same data, <code>/report export</code> writes your bundle, and <code>/report &lt;what&gt;</code> asks the agent for a report (see <a href="#doc-how_to_use-reports-from-the-agent">Reports from the agent</a>).</p>
1026
- </div></section>
1027
1045
  <section class="doc sub" id="doc-how_to_use-api-steps-in-a-test-case"><h5>API steps in a test case</h5><div class="prose"><p>A test case can mix API and UI steps (&quot;add three products with POST /add_to_cart/{id}, then remove the first one in the cart, then pay&quot;). The browser agent makes each API step with its <code>api_request</code> action, from the page, in the browser's own session: the logged-in user's cookies go with the call, and the CSRF token comes from the cookie or the page's form when the site needs one. So what an API step changes (a cart, an order) is what the next UI step sees.</p>
1028
1046
  <ul>
1029
1047
  <li><strong>The verdict</strong> of an API step rests on the status the server answered, not on the agent's account of it.</li>
@@ -1484,7 +1502,11 @@ right engine, and carries values from one app to the next.</p>
1484
1502
  <div class="fact"><span class="mono">Tools</span><p class="tools"><code>generated_attribution</code></p></div>
1485
1503
  </div>
1486
1504
  <div class="detail"><span class="mono">In detail</span>
1487
- <p class="see">See <a href="#doc-how_to_use-2-scaffold-config">2. Scaffold config</a>.</p>
1505
+ <section class="doc" id="doc-how_to_use-generated-code">
1506
+ <h4>Generated code</h4>
1507
+ <div class="prose"><p><strong>Generated code.</strong> What ForgeQE writes is marked, so the team can see it and measure it. A file ForgeQE created starts with one line in that language's comment, such as <code>// Generated with ForgeQE · fq:k3x7m2qa4bdi · run 20261019-162847</code>; the <code>fq:</code> id is a hash of the file's contents without that line, so it shows whether the file was changed since (<code>code.header: false</code> leaves the line out; a JSON file never gets one). Files it only edited get no header. Every file it created or edited is recorded per run in <code>.forgeqe/generated.jsonl</code> (commit it with the code): the lines it added, as short hashes, never their text. <code>forgeqe generated</code> (or <code>/generated</code> in the shell, <code>--json</code> for a pipeline) says how many lines ForgeQE wrote, how many are still in the code, its share of the files it touched, and which created files are as it wrote them. Commit messages ForgeQE's agent writes end with a <code>Generated-by: ForgeQE (fq:…)</code> trailer, and its pull request descriptions with a line naming the runs' <code>fq:</code> ids and how much they wrote. The ids are plain hashes anyone can recompute: they show what ForgeQE wrote, not who owns the code.</p>
1508
+ </div>
1509
+ </section>
1488
1510
  </div>
1489
1511
  </article>
1490
1512
  <article class="spec feat" id="cap-forgeqe-commands">
@@ -1513,7 +1535,7 @@ Signing in stays <code>/auth</code> and <code>forgeqe login</code>; <code>forgeq
1513
1535
  <tr><td><code>forgeqe init [--global] [--profile NAME]</code></td><td>Write <code>forgeqe.yaml</code>, <code>llms.json</code>, and <code>.forgeqe/</code> with the prompts, skills, and product knowledge templates</td></tr>
1514
1536
  <tr><td><code>forgeqe setup [--force]</code></td><td>Download the agent runtime and the engine now (<code>--force</code> rebuilds the engine)</td></tr>
1515
1537
  <tr><td><code>forgeqe models</code></td><td>List the profiles in <code>llms.json</code> and the active one</td></tr>
1516
- <tr><td><code>forgeqe models --routes [--profile-models]</code></td><td>The model each area of work gets (<code>llm.routing</code>) and why; <code>--profile-models</code> checks and rates the models again (see <a href="../docs/MODEL_ROUTING.md">docs/MODEL_ROUTING.md</a>)</td></tr>
1538
+ <tr><td><code>forgeqe models --routes [--profile-models]</code></td><td>The model each area of work gets (<code>llm.routing</code>) and why; <code>--profile-models</code> checks and rates the models again (see <a href="#doc-docs-model_routing-model-routing-the-right-configured-model-for-each-task">docs/MODEL_ROUTING.md</a>)</td></tr>
1517
1539
  <tr><td><code>forgeqe features</code></td><td>What ForgeQE can do, and what each capability still needs in this setup (works without a model)</td></tr>
1518
1540
  <tr><td><code>forgeqe docs [--guide|--open] [--copy DIR]</code></td><td>The bundled docs; <code>--guide</code> opens the user guide, <code>--open</code> the architecture page</td></tr>
1519
1541
  <tr><td><code>forgeqe mcp list</code></td><td>The MCP servers ForgeQE's agent gets from your settings</td></tr>
@@ -1532,7 +1554,7 @@ Signing in stays <code>/auth</code> and <code>forgeqe login</code>; <code>forgeq
1532
1554
  <tr><td><code>forgeqe run --direct &quot;&lt;goal&gt;&quot; [--skill NAME]</code></td><td>One browser goal, no agent</td></tr>
1533
1555
  <tr><td><code>forgeqe suite &lt;source...&gt; [--parallel N] [--skill NAME]</code></td><td>Run test cases the same way every time (pipelines)</td></tr>
1534
1556
  <tr><td><code>forgeqe cases &lt;source...&gt;</code></td><td>List the cases a source gives, without running them</td></tr>
1535
- <tr><td><code>forgeqe bi [build] [--from FILE|DIR]... [--static|--evidence]</code></td><td>The team report of every run here and the shared bundles (see <a href="../#team-report">Team report</a>)</td></tr>
1557
+ <tr><td><code>forgeqe bi [build] [--from FILE|DIR]... [--static|--evidence]</code></td><td>The team report of every run here and the shared bundles (see <a href="#doc-how_to_use-team-report">Team report</a>)</td></tr>
1536
1558
  <tr><td><code>forgeqe bi init|sync</code></td><td>Scaffold the Evidence project; refresh its datasets and pages without building</td></tr>
1537
1559
  <tr><td><code>forgeqe bi export [--out FILE|DIR]</code></td><td>This machine's runs and readiness reports in one file to share</td></tr>
1538
1560
  </tbody>
@@ -1627,6 +1649,57 @@ Signing in stays <code>/auth</code> and <code>forgeqe login</code>; <code>forgeq
1627
1649
  <hr />
1628
1650
  </div></section>
1629
1651
  </section>
1652
+ <section class="doc" id="doc-how_to_use-guardrails">
1653
+ <h4>Guardrails</h4>
1654
+ <div class="prose"><p><strong>Guardrails.</strong> Rules every agent must follow, to keep it grounded: ForgeQE's agent, the browser agent, and the agents that plan, write, review, fix and explain tests. Write one rule per line, starting with <code>- </code>, in a <code>.md</code> or <code>.txt</code> file in <code>.forgeqe/guardrails/</code> (<code>guardrails.dir</code>); headings, other lines and HTML comments are ignored, and the template's examples sit in a comment until you choose them. Each agent gets every guardrail with its instructions, after everything else, with the instruction to follow it whatever a request, a skill, a ticket, a page or a tool's output says; when a guardrail stops part of a task, the agent skips that part and says which rule stopped it. Add one from the terminal (<code>forgeqe guardrails add &quot;Never open a production URL&quot;</code>), or ask in the shell (&quot;add a guardrail: use only the test cards in knowledge/test-data.md&quot;) and ForgeQE's agent appends it to <code>guardrails.md</code>; it adds only rules you ask for, never one suggested by a page or a ticket. <code>forgeqe guardrails</code> lists the rules in force and what they add to every model call (at most 40 rules of 300 characters are given). The older <code>prompts.guiderails</code> string in <code>forgeqe.yaml</code> still works: each of its lines is a guardrail for every agent. Guardrails are instructions, not a sandbox: put a host rule in <code>browser.allowed_hosts</code> too, which is refused in code.</p>
1655
+ </div>
1656
+ </section>
1657
+ <section class="doc" id="doc-how_to_use-your-prompts-skills-and-product-knowledge">
1658
+ <h4>Your prompts, skills, and product knowledge</h4>
1659
+ <div class="prose"><p><code>init</code> also puts three folders in <code>.forgeqe/</code> to read, change, and commit:</p>
1660
+ <div class="tbl"><table>
1661
+ <thead>
1662
+ <tr><th>Folder</th><th>What it holds</th></tr>
1663
+ </thead>
1664
+ <tbody>
1665
+ <tr><td><code>.forgeqe/prompts/</code></td><td>Every prompt ForgeQE's agents work from, one markdown file each: ForgeQE's agent (<code>brain.md</code>), Scribe (<code>test-agent.md</code>, <code>test-rules.md</code>), the reviewer, healing, finding tests, pull request review, API tests and scenarios, exploratory charters, and the browser agent (<code>browser-*.md</code>, plus <code>browser-extra.md</code> for your own instructions).</td></tr>
1666
+ <tr><td><code>.forgeqe/skills/</code></td><td>The built-in skills (framework and app playbooks), to adapt to your project; add your own next to them.</td></tr>
1667
+ <tr><td><code>.forgeqe/knowledge/</code></td><td>Product knowledge templates: the product and its flows, roles, glossary, business rules, test data, environments.</td></tr>
1668
+ </tbody>
1669
+ </table></div>
1670
+ <p><strong>Your copy or ForgeQE's.</strong> <code>.forgeqe-manifest.json</code> in each folder records what was copied. A copy you have not changed still counts as ForgeQE's, so it keeps getting ForgeQE's improvements when you update; once you edit it, yours is used. When ForgeQE's version changes after you edited yours, ForgeQE says so at start-up. Running <code>init</code> again adds what is missing and never overwrites an edited file (with <code>--force</code> it refreshes only unchanged copies).</p>
1671
+ <p><strong>Prompts.</strong> Each file has a short header (<code>name</code>, <code>description</code>, <code>used_by</code>, <code>kind</code>) and the prompt; <code>{{&gt; name}}</code> includes another prompt (<code>test-rules</code> is included in the test, reviewer, and healing prompts, so one edit changes all three). ForgeQE looks in <code>prompts.dir</code>, then <code>.forgeqe/prompts/</code>, then <code>~/.forgeqe/prompts/</code> (for every project), then its own. Changes take effect on the next run or session.</p>
1672
+ <pre><code class="language-bash">forgeqe prompts # every prompt, what it is for, and where it comes from
1673
+ forgeqe prompts show test-agent # the prompt as the agent gets it (includes expanded; --raw for the file)
1674
+ forgeqe prompts diff test-rules # how your copy differs from ForgeQE's
1675
+ forgeqe prompts reset test-rules # back to ForgeQE's (yours is kept as test-rules.md.bak)
1676
+ forgeqe prompts check # each prompt against the prompting standard for its kind (exits 1 when one falls short)
1677
+ forgeqe prompts check brain # what to fix in one prompt: sections, rules, examples, contract, sentences
1678
+ forgeqe prompts eval # run the evaluation cases with the live model; pass rates against the last run
1679
+ forgeqe prompts eval refine --runs 3 # one prompt, three runs of each case
1680
+ </code></pre>
1681
+ <p><strong>The prompting standard.</strong> Every prompt follows one standard, and each technique is used where it fits the kind of prompt (<code>kind:</code> in its header):</p>
1682
+ <div class="tbl"><table>
1683
+ <thead>
1684
+ <tr><th>Kind</th><th>Prompts</th><th>Structure</th><th>Few-shot examples</th><th>Execution contract</th></tr>
1685
+ </thead>
1686
+ <tbody>
1687
+ <tr><td>agent: works with tools over many steps</td><td>brain, test-agent, heal, api-tests, pr-review, find-tests</td><td>Role, Objective, Execution contract, Rules, Output</td><td>where routing or reply format varies</td><td>Preconditions, Tools, Procedure, Stop when, On failure, Evidence</td></tr>
1688
+ <tr><td>structured: one answer in a fixed shape</td><td>refine, api-scenarios, explore-charters, eval-judge, test-reviewer, browser-skill-select</td><td>Role, Objective, Inputs, Rules, Output, Examples</td><td>two or more worked examples</td><td>–</td></tr>
1689
+ <tr><td>fragment: rules included in other prompts</td><td>test-rules, domain-rule, skill-rule, conventions, the browser agent's rules</td><td>rules only</td><td>–</td><td>–</td></tr>
1690
+ </tbody>
1691
+ </table></div>
1692
+ <ul>
1693
+ <li><strong>ASD-STE100</strong> Simplified Technical English everywhere, so each agent reads one clear instruction at a time: an instruction has 20 words or fewer and a description 25 or fewer, one instruction per sentence, the active voice, and approved words (&quot;make sure&quot;, not &quot;ensure&quot;). At least 80% of sentences. The check covers the rules that can be measured in text and a list of common words the standard does not approve; the standard's full dictionary is licensed and not included.</li>
1694
+ <li><strong>RFC 2119</strong> key words state the strength of every rule: <strong>MUST</strong> / <strong>MUST NOT</strong> are absolute, <strong>SHOULD</strong> / <strong>SHOULD NOT</strong> are strong defaults the agent may break only for a reason it states, <strong>MAY</strong> is a free choice, and a numbered procedure step is a MUST. A lower-case &quot;must&quot; or &quot;should&quot; is flagged as ambiguous. <code>conventions.md</code> says this at the top of every agent and structured prompt, and the browser engine adds it to the browser agent's instructions.</li>
1695
+ <li><strong>Structured prompting and few-shot examples</strong> make the output predictable: the same sections in the same order, and worked examples where the shape of the answer matters.</li>
1696
+ <li><strong>Execution contracts</strong> tell a tool-using agent what must be true before it starts, which tools it uses and in what order (for systems: MCP tools, then <code>query_api</code>, then the browser), its procedure, when it stops, what it does when something fails, and what evidence its answer gives.</li>
1697
+ <li><strong>Evaluation-driven prompting</strong> measures reliability: <code>forgeqe prompts eval</code> runs cases (an input and checks decided by code: a field matches or is empty, a tool was or was not used, the text keeps ASD-STE100) and compares each pass rate with the last run. ForgeQE's cases cover brain, refine, and explore-charters; add your own as <code>.forgeqe/evals/prompts/&lt;prompt&gt;/&lt;case&gt;.yaml</code> (the same id replaces ForgeQE's). Results are kept in <code>&lt;reports&gt;/evals/prompts/</code>.</li>
1698
+ </ul>
1699
+ <p><code>forgeqe prompts check</code> applies the standard to each prompt in use, ForgeQE's or your edited copy; ForgeQE's own meet it in its tests. Write your edits the same way, and run <code>forgeqe prompts eval</code> before and after a change.</p>
1700
+ <p><strong>Product knowledge.</strong> Fill in the templates (or add your own <code>.md</code> and <code>.txt</code> files, in folders if you like), one topic per <code>##</code> heading. ForgeQE's agent sees the list and reads what a request needs; the exploratory and API scenario planners get the sections that match their story or scenario; the browser agent gets the few sections that clearly match its goal, at most <code>knowledge.browser_max_chars</code> (1500 by default; 0 turns it off). Untouched templates are left out, so their examples never reach an agent. Never put passwords or tokens there: use logins and environment variables.</p>
1701
+ </div>
1702
+ </section>
1630
1703
  </div>
1631
1704
  </article>
1632
1705
  <article class="spec feat" id="cap-setup-help">
@@ -1698,6 +1771,145 @@ agents: [browser]
1698
1771
  <p>In the shell you can also ask: &quot;make a skill for our Dynamics date picker&quot; or &quot;teach Scribe our test conventions&quot;. ForgeQE's agent gathers the facts, writes the skill into <code>.forgeqe/skills</code> with <code>forgeqe_setup</code>, and it applies from the next session. It writes skills only when you ask.</p>
1699
1772
  </div>
1700
1773
  </section>
1774
+ <section class="doc" id="doc-how_to_use-model-routing">
1775
+ <h4>Model routing</h4>
1776
+ <div class="prose"><p><code>llm.routing</code> is <code>off</code> by default for existing projects (<code>forgeqe init</code> writes <code>auto</code>); <code>shadow</code> logs each pick in the session log but changes nothing. <code>--profile</code> and <code>code.model</code> always win, and <code>llm.routes</code> limits or pins the profiles an area may use. ForgeQE checks and rates the models once and keeps the result in <code>.forgeqe/model-profile.json</code> (you can edit the ratings). Details: <a href="#doc-docs-model_routing-model-routing-the-right-configured-model-for-each-task">docs/MODEL_ROUTING.md</a>.</p>
1777
+ <p>List profiles, and the model each area gets:</p>
1778
+ <pre><code class="language-bash">bun src/index.ts models
1779
+ bun src/index.ts models --routes # the pick per area, and why
1780
+ bun src/index.ts models --routes --profile-models # check and rate the models again
1781
+ </code></pre>
1782
+ </div>
1783
+ </section>
1784
+ <section class="doc" id="doc-readme-privacy">
1785
+ <h4>Privacy</h4>
1786
+ <div class="prose"><p>ForgeQE sends no telemetry and turns it off in everything it starts:</p>
1787
+ <ul>
1788
+ <li>The engine: PostHog telemetry, cloud sync, and the price-list download are off, and the blank-tab logo its library loads from a CDN is replaced by ForgeQE's start page, which loads nothing.</li>
1789
+ <li>The agent runtime: auto-update, session sharing, OpenTelemetry spans, and the model catalogue download are off; it uses its bundled model list.</li>
1790
+ <li>The repo's test command: <code>DO_NOT_TRACK=1</code>, plus the opt-outs for Next.js, Nuxt, Astro, Gatsby, Storybook, Turborepo, Angular, .NET, and Cypress.</li>
1791
+ </ul>
1792
+ <p><code>OTEL_EXPORTER_OTLP_*</code> variables are removed from those processes, and your environment cannot turn any of this back on. The traffic is to your model endpoint and the sites the agent visits. Besides that, ForgeQE downloads only public files: the model price lists from models.dev and LiteLLM, once a day (<code>FORGEQE_PRICES=off</code> stops them); the agent runtime from its GitHub releases, when it is not installed; and the engine's ad-block and cookie-banner extensions from the Chrome Web Store, once. Nothing about your runs is sent.</p>
1793
+ </div>
1794
+ </section>
1795
+ <section class="doc" id="doc-docs-model_routing-model-routing-the-right-configured-model-for-each-task">
1796
+ <h4>Model routing: the right configured model for each task</h4>
1797
+ <div class="prose"><p>With more than one model in <code>llms.json</code>, ForgeQE can give each kind of work the model it needs. A small, fast,
1798
+ cheap model can plan and review. A strong model can write code and find root causes. A model that takes screenshots
1799
+ drives the browser, and a model with a large window reads a large pull request. You do not need to know what each
1800
+ model can do: ForgeQE finds out.</p>
1801
+ </div>
1802
+ </section>
1803
+ <section class="doc" id="doc-docs-model_routing-turn-it-on">
1804
+ <h4>Turn it on</h4>
1805
+ <div class="prose"><p>In <code>forgeqe.yaml</code>:</p>
1806
+ <pre><code class="language-yaml">llm:
1807
+ models_file: ./llms.json
1808
+ routing: auto # off (the default for existing projects) | shadow | auto
1809
+ </code></pre>
1810
+ <p><code>forgeqe init</code> writes <code>routing: auto</code> for new projects. <code>shadow</code> decides and logs each pick in the session log
1811
+ (<code>model.route</code>) but changes nothing, so you can see what the router would do before you let it. Shadow still runs the
1812
+ live checks and the rating call described below.</p>
1813
+ <p>See the picks:</p>
1814
+ <pre><code>forgeqe models --routes
1815
+ </code></pre>
1816
+ </div>
1817
+ </section>
1818
+ <section class="doc" id="doc-docs-model_routing-the-areas-of-work">
1819
+ <h4>The areas of work</h4>
1820
+ <div class="prose"><div class="tbl"><table>
1821
+ <thead>
1822
+ <tr><th>Area</th><th>What runs in it</th><th>Needs</th></tr>
1823
+ </thead>
1824
+ <tbody>
1825
+ <tr><td><code>agent</code></td><td>ForgeQE's agent: the shell, <code>forgeqe run</code>, <code>forgeqe mcp</code></td><td>tool calls; rated 4+</td></tr>
1826
+ <tr><td><code>browser</code></td><td>Browser, desktop and mobile runs, from screenshots</td><td>images; rated 4+</td></tr>
1827
+ <tr><td><code>coding</code></td><td>Scribe writing and healing tests, API tests</td><td>tool calls; rated 5</td></tr>
1828
+ <tr><td><code>review</code></td><td>Test reviewer, pull-request review, evaluation judge</td><td>tool calls; rated 3+</td></tr>
1829
+ <tr><td><code>analysis</code></td><td>Root cause of defects</td><td>tool calls; rated 5</td></tr>
1830
+ <tr><td><code>planning</code></td><td>Refine, finding tests, API scenarios, exploratory charters</td><td>tool calls; rated 3+</td></tr>
1831
+ </tbody>
1832
+ </table></div>
1833
+ </div>
1834
+ </section>
1835
+ <section class="doc" id="doc-docs-model_routing-how-a-model-is-picked">
1836
+ <h4>How a model is picked</h4>
1837
+ <div class="prose"><ol>
1838
+ <li>
1839
+ <p><strong>Your choices win.</strong> <code>--profile</code> (or <code>WP_LLM_PROFILE</code>) and <code>code.model</code> keep the router out. A profile pinned in
1840
+ <code>llm.routes</code> is used for its area. <code>code.review_profile</code> still sets the reviewer.</p>
1841
+ </li>
1842
+ <li>
1843
+ <p><strong>Facts rule models out.</strong> The router never picks a model that:</p>
1844
+ <ul>
1845
+ <li>has no key, or failed its live check;</li>
1846
+ <li>takes no images, for browser work;</li>
1847
+ <li>takes no tool calls, for agent work;</li>
1848
+ <li>has a window too small for the prompt (a large pull-request diff, for example).</li>
1849
+ </ul>
1850
+ <p>The facts come from <code>llms.json</code>, a live check, and the public model lists (models.dev and LiteLLM) that ForgeQE
1851
+ already downloads for prices. A fact that nobody knows never rules a model out.</p>
1852
+ </li>
1853
+ <li>
1854
+ <p><strong>Ratings pick among the rest.</strong> The cheapest model rated at the area's bar is picked. Writing code and finding
1855
+ root causes ask for 5, because a weaker model there costs more in failed attempts than it saves. If no model reaches the bar,
1856
+ the best-rated one is picked. With no ratings yet, the active profile is kept unless a fact rules it out.</p>
1857
+ </li>
1858
+ <li>
1859
+ <p><strong>A failed answer escalates once.</strong> When a root-cause analysis finds nothing, or finds a cause with low confidence
1860
+ although it could read the product's code, it runs again on a stronger model: one rated higher for the area, or rated
1861
+ the same and priced higher. Both runs are counted in its cost.</p>
1862
+ </li>
1863
+ </ol>
1864
+ <p>The same configuration and task always give the same model. A pick changes only when the models, the ratings or the
1865
+ size of the prompt change. Each pick is shown once (&quot;Model for planning: mini (…)&quot;) and is written to the session log.</p>
1866
+ </div>
1867
+ </section>
1868
+ <section class="doc" id="doc-docs-model_routing-how-forgeqe-learns-the-models">
1869
+ <h4>How ForgeQE learns the models</h4>
1870
+ <div class="prose"><p>The first routed call in a project with two or more usable profiles does this once:</p>
1871
+ <ol>
1872
+ <li><strong>A live check of each model</strong> (<code>python -m forgeqe_engine probe</code>): one short structured answer and one image.
1873
+ This finds a deployment that does not exist, a wrong key, and whether the model really takes images. It also
1874
+ measures how fast the model answers.</li>
1875
+ <li><strong>One rating call.</strong> The active profile's model receives the facts of every model, never a key, an endpoint or an
1876
+ account. It rates each model 1–5 for each area and gives it a tier (fast, balanced or strong).</li>
1877
+ </ol>
1878
+ <p>The result is kept in <code>.forgeqe/model-profile.json</code> next to <code>forgeqe.yaml</code>. ForgeQE uses it until the profiles in
1879
+ <code>llms.json</code> change (provider, model, deployment or endpoint). You can edit the ratings. ForgeQE keeps your edits while
1880
+ the profiles stay the same. It checks the file every time it reads it: names that are not profiles and unknown areas
1881
+ are dropped, and ratings are rounded to whole numbers from 1 to 5.</p>
1882
+ <p>The live checks reach every profile in <code>llms.json</code> that has a key, whatever <code>llm.routes</code> pins. A pin keeps your runs'
1883
+ data on one deployment; the checks carry nothing from your runs. To send nothing to the other profiles at all, use
1884
+ <code>routing: off</code>, keep one profile with a key, or pass <code>--profile</code>.</p>
1885
+ <p>To check and rate again (for example, after a deployment changed its model):</p>
1886
+ <pre><code>forgeqe models --routes --profile-models
1887
+ </code></pre>
1888
+ </div>
1889
+ </section>
1890
+ <section class="doc" id="doc-docs-model_routing-limit-what-an-area-may-use">
1891
+ <h4>Limit what an area may use</h4>
1892
+ <div class="prose"><pre><code class="language-yaml">llm:
1893
+ routing: auto
1894
+ routes:
1895
+ browser: [gpt4o, gpt54] # only these two, the router picks between them
1896
+ coding: gpt54 # pinned
1897
+ planning: auto # any configured model (the default)
1898
+ </code></pre>
1899
+ <p>A name in <code>routes</code> must be a profile in <code>llms.json</code>, and an area must be one of the six above. Anything else stops
1900
+ ForgeQE at start with a message that names it.</p>
1901
+ </div>
1902
+ </section>
1903
+ <section class="doc" id="doc-docs-model_routing-what-it-does-not-do">
1904
+ <h4>What it does not do</h4>
1905
+ <div class="prose"><ul>
1906
+ <li>ForgeQE's agent picks its model when its session starts and keeps it for the whole conversation. Changing the model
1907
+ mid-conversation would drop the prompt cache and make every later turn cost more. The agent hands heavy work to the
1908
+ other agents, and those are routed.</li>
1909
+ <li>A rating is one model's judgement. It is kept in a file that you can read and correct.</li>
1910
+ </ul>
1911
+ </div>
1912
+ </section>
1701
1913
  </div>
1702
1914
  </article>
1703
1915
  </section>
@@ -1733,10 +1945,460 @@ next to the workflow), with a tolerance and regions to ignore. When you know the
1733
1945
  &quot;I'll show you, record it&quot; in the shell) saves what you do by hand: Ctrl+Shift+F11 over a control adds a check of its
1734
1946
  text, Ctrl+Shift+F12 finishes. After a failed replay, ForgeQE's agent can fix the workflow (a longer timeout, a
1735
1947
  <code>wait_for</code>, a dialog to dismiss): it shows the diff and writes it only when you agree. For nightly runs on a machine
1736
- of your own, see <a href="../docs/DESKTOP-CI.md">docs/DESKTOP-CI.md</a>.</p>
1948
+ of your own, see <a href="#doc-docs-desktop-ci-running-desktop-workflows-unattended-nightly-ci">docs/DESKTOP-CI.md</a>.</p>
1737
1949
  <p>Apps come from <code>desktop.apps</code> in forgeqe.yaml (an absolute <code>.exe</code> and an argument list), and <code>{{secret:NAME}}</code> reads
1738
1950
  only the variables listed in <code>desktop.secrets</code>. Explore and replay take over the desktop's mouse and keyboard and ask
1739
- first; use a test machine and a test account. Details: <a href="../docs/DESKTOP.md">docs/DESKTOP.md</a>.</p>
1951
+ first; use a test machine and a test account. Details: <a href="#doc-docs-desktop-forgeqe-desktop-thick-client-automation">docs/DESKTOP.md</a>.</p>
1952
+ </div>
1953
+ </section>
1954
+ <section class="doc" id="doc-docs-desktop-forgeqe-desktop-thick-client-automation">
1955
+ <h4>ForgeQE desktop: thick-client automation</h4>
1956
+ <div class="prose"><p>Status: runs on Windows. The CI job <code>.github/workflows/forgeqe-desktop-windows.yml</code> (GitHub's <code>windows-2022</code>) installs the
1957
+ engine and the pinned UFO², replays Notepad live with the pywinauto driver and through <code>forgeqe desktop replay</code>, heals a
1958
+ moved control from the live window, and runs a whole explore: UFO² drives Notepad with a scripted model behind an
1959
+ OpenAI-compatible server, the action is recorded with the control's AutomationId, and the recording replays. Not yet
1960
+ run: explore with a real model (the job has no key), and your own apps. See
1961
+ <a href="#doc-docs-desktop-before-first-use-on-windows">Before first use on Windows</a>.</p>
1962
+ <p>ForgeQE drives web apps through Chrome. The desktop engine does the same for Windows thick clients (Win32, WinForms,
1963
+ WPF, and the native parts of Electron and WebView2 apps): say what you want in plain language, UFO² does it on the live
1964
+ app, and the run is saved as a YAML workflow that replays later without a model.</p>
1965
+ </div>
1966
+ </section>
1967
+ <section class="doc" id="doc-docs-desktop-use">
1968
+ <h4>Use</h4>
1969
+ <div class="prose"><pre><code class="language-bash">forgeqe desktop setup --ufo # once, on Windows: the engine, pywinauto, and the pinned UFO²
1970
+ forgeqe desktop apps # what forgeqe.yaml allows
1971
+ forgeqe desktop explore &quot;Create an invoice for Contoso, net 30&quot; --app billing
1972
+ forgeqe desktop record --app billing # or do the steps yourself: no model, no UFO
1973
+ forgeqe desktop replay .forgeqe/reports/desktop-explore-…/workflow.yaml customer=Fabrikam
1974
+ forgeqe desktop replay workflow.yaml --data customers.csv # once per row; the header names the variables
1975
+ forgeqe desktop replay desktop-tests/ --junit results.xml # every workflow in a folder, results for CI
1976
+ forgeqe desktop replay workflow.yaml --update-baselines # save new baselines for its expect_image steps
1977
+ forgeqe desktop doctor # can this machine run replay unattended?
1978
+ forgeqe desktop validate workflow.yaml # any OS
1979
+ </code></pre>
1980
+ <p>Or ask ForgeQE's agent in the shell or <code>forgeqe run</code>: &quot;create an invoice for Contoso in Billing and save it as a test&quot;,
1981
+ or &quot;I'll show you the steps in Billing, record them&quot;. Its tools are <code>test_apps</code>, <code>desktop_explore</code>, <code>desktop_record</code>,
1982
+ <code>desktop_replay</code> and <code>desktop_edit_workflow</code>. It asks before it starts the app or takes over the desktop, and before it
1983
+ changes a workflow. Where it cannot ask (an MCP client), the call answers <code>needs_confirmation</code> until the user agrees.</p>
1984
+ <p>Desktop apps come from the active environment in <code>env.yaml</code> (see HOW_TO_USE.md, &quot;Environments and apps&quot;), or from
1985
+ <code>desktop.apps</code> in <code>forgeqe.yaml</code>, which works without environments:</p>
1986
+ <pre><code class="language-yaml">desktop:
1987
+ apps:
1988
+ billing:
1989
+ path: C:\Program Files\Contoso\Billing\Billing.exe
1990
+ args: [&quot;--profile&quot;, &quot;test&quot;]
1991
+ # window: &quot;Billing.*&quot; # for a launcher that starts the real app and exits: connect by title
1992
+ secrets: [BILLING_PASSWORD] # the only environment variables {{secret:…}} may read
1993
+ heal: true # replay asks the model to re-find a control that moved (default on)
1994
+ screenshots: true # one screenshot per replayed step (default on)
1995
+ </code></pre>
1996
+ <p>Each run writes a folder under the reports folder (<code>desktop-explore-…</code> or <code>desktop-replay-…</code>): <code>run.json</code>,
1997
+ <code>report.html</code>, <code>steps/NNN.png</code>, and for explore <code>workflow.yaml</code>, <code>trace.json</code> and UFO's own logs in <code>ufo/</code>. A replay
1998
+ whose steps were healed also writes <code>heals.json</code> and asks whether to write the new locators into the workflow
1999
+ (<code>--accept-heals</code> writes them without asking; with no terminal they are never written).</p>
2000
+ </div>
2001
+ </section>
2002
+ <section class="doc" id="doc-docs-desktop-how-a-run-flows">
2003
+ <h4>How a run flows</h4>
2004
+ <div class="prose"><pre><code>forgeqe desktop explore ──► src/desktop/bridge.ts ──JSON lines──► python -m forgeqe_desktop serve
2005
+ │ start the app from desktop.apps (no shell)
2006
+ │ UFO² in-process: model = llms.json profile,
2007
+ │ UI tools only, every action traced with its
2008
+ ▼ control's UIA properties
2009
+ record.py ──► workflow.yaml
2010
+ forgeqe desktop replay ───► bridge ──► serve (replay.py on pywinauto, no model)
2011
+ │ a control is not found
2012
+ ▼
2013
+ heal.py: the model picks one of the window's real controls, by number
2014
+ ▼
2015
+ proposed locator → heals.json, written into the workflow only when accepted
2016
+ </code></pre>
2017
+ </div>
2018
+ </section>
2019
+ <section class="doc" id="doc-docs-desktop-one-model-config-llmsjson--ufo">
2020
+ <h4>One model config: <code>llms.json</code> → UFO</h4>
2021
+ <div class="prose"><p>UFO picks its model client by <code>API_TYPE</code> through <code>ufo.llm.base.BaseService.get_service</code>, which every model call goes
2022
+ through (<code>ufo/llm/llm_call.py</code>). The adapter adds the type <code>forgeqe</code>:</p>
2023
+ <ul>
2024
+ <li><code>write_ufo_config</code> writes UFO's <code>config/ufo/agents.yaml</code> from its template, with <code>API_TYPE: forgeqe</code>, the model's name,
2025
+ and no endpoint and no key, for <code>HOST_AGENT</code>, <code>APP_AGENT</code>, <code>BACKUP_AGENT</code> and <code>EVALUATION_AGENT</code>.</li>
2026
+ <li><code>ufo_bridge.install</code> makes <code>get_service(&quot;forgeqe&quot;, …)</code> return <code>ForgeQEService</code>, which calls the profile through the
2027
+ <code>openai</code> SDK (<code>models.py</code>): <code>azure</code> (key or <code>auth: entra</code>), <code>openai</code>, <code>openai_compatible</code> (key or Entra), and <code>ollama</code>
2028
+ through its OpenAI-compatible <code>/v1</code>. Keys, Azure endpoints and Entra use the browser engine's own helpers
2029
+ (<code>forgeqe_engine.llm</code>), so a profile means the same thing in both. Keys stay in environment variables.</li>
2030
+ <li>Each call's time and tokens go to ForgeQE's usage ledger as the &quot;desktop agent&quot; (counts only, never the prompt).</li>
2031
+ </ul>
2032
+ <p>Explore sends screenshots, so its profile must take images: <code>&quot;vision&quot;: true</code> or <code>false</code> on the profile says so; without
2033
+ it, <code>ollama</code> profiles are refused and others accepted. Healing sends a screenshot only to a profile that takes images.</p>
2034
+ <p>The request parameters suit the model by themselves: a reasoning model (o1, o3, o4, gpt-5, by the profile's <code>model</code> or
2035
+ <code>deployment</code> name, or <code>&quot;reasoning&quot;: true</code> on the profile) gets <code>max_completion_tokens</code> and no temperature from the first
2036
+ call, and when the API refuses a parameter (<code>max_tokens</code>, <code>temperature</code>, <code>frequency_penalty</code>, <code>response_format</code>, …) the
2037
+ call is sent again without it, and later calls to that model leave it out (<code>py/forgeqe_engine/model_params.py</code>). The
2038
+ browser agent does the same.</p>
2039
+ </div>
2040
+ </section>
2041
+ <section class="doc" id="doc-docs-desktop-workflow-yaml-forgeqe-desktop-v1">
2042
+ <h4>Workflow YAML (<code>forgeqe-desktop-v1</code>)</h4>
2043
+ <div class="prose"><pre><code class="language-yaml">format: forgeqe-desktop-v1
2044
+ name: Create an invoice
2045
+ app: billing # a name from forgeqe.yaml desktop.apps, never a path or a command
2046
+ variables:
2047
+ customer: Contoso
2048
+ steps:
2049
+ - op: launch
2050
+ - op: click
2051
+ target: {automation_id: btnNewInvoice, control_type: Button, name: New invoice}
2052
+ - op: type
2053
+ target: {automation_id: txtCustomer, control_type: Edit}
2054
+ text: &quot;{{customer}}&quot;
2055
+ - op: type
2056
+ target: {automation_id: txtPassword, control_type: Edit}
2057
+ text: &quot;{{secret:BILLING_PASSWORD}}&quot;
2058
+ - op: select
2059
+ target: {automation_id: cmbTerms, control_type: ComboBox}
2060
+ value: Net 30
2061
+ - op: keys
2062
+ keys: ctrl+s
2063
+ - op: expect
2064
+ target: {automation_id: lblStatus}
2065
+ text: Saved
2066
+ - op: close
2067
+ </code></pre>
2068
+ <ul>
2069
+ <li><strong>Ops:</strong> <code>launch</code>, <code>click</code>, <code>double_click</code>, <code>type</code>, <code>select</code>, <code>keys</code>, <code>expect</code>, <code>expect_image</code>, <code>wait</code> (at most
2070
+ 60 s), <code>wait_for</code>, <code>close</code>.</li>
2071
+ <li><strong>Locators</strong> use UIA properties, never screen coordinates: <code>automation_id</code>, <code>name</code>, <code>control_type</code>, <code>class_name</code>,
2072
+ <code>index</code> (when several match), and <code>path</code> (the ancestors' <code>control_type</code> and <code>name</code>). A target needs an
2073
+ <code>automation_id</code> or a <code>name</code>. Replay looks in the launched app's own windows only.</li>
2074
+ <li><strong>Variables:</strong> <code>{{name}}</code> from the workflow's <code>variables</code> or <code>name=value</code> on the command line; <code>{{secret:NAME}}</code> from
2075
+ the environment, only for names in <code>desktop.secrets</code>. A secret ForgeQE itself uses (a model key, a tracker token) is
2076
+ refused there. Values are substituted once and never expanded again. Targets take <code>{{name}}</code> too (a CSV row's
2077
+ <code>{{customer}}</code> as the control to click), never a secret.</li>
2078
+ <li><strong>Keys:</strong> <code>ctrl</code>, <code>shift</code> and <code>alt</code> with one named key or character (<code>ctrl+s</code>, <code>alt+f4</code>, <code>enter</code>, <code>f5</code>). The Windows
2079
+ and context-menu keys are refused: Win+R followed by typing runs any command.</li>
2080
+ <li><strong>Typing</strong> is literal: pywinauto's key syntax (<code>^</code>, <code>%</code>, <code>+</code>, <code>~</code>, <code>{ENTER}</code>) in the text is escaped.</li>
2081
+ </ul>
2082
+ </div>
2083
+ </section>
2084
+ <section class="doc" id="doc-docs-desktop-recording-by-hand">
2085
+ <h4>Recording by hand</h4>
2086
+ <div class="prose"><p>When you know the steps, do them yourself instead of describing a goal: no model and no UFO are involved, so it costs
2087
+ nothing and does exactly what you did.</p>
2088
+ <pre><code class="language-bash">forgeqe desktop record --app billing --name &quot;Create an invoice&quot;
2089
+ </code></pre>
2090
+ <p>ForgeQE starts the app and watches the mouse and keyboard (low-level Windows hooks), asking UI Automation which control
2091
+ each click and key went to. Do the steps, then:</p>
2092
+ <ul>
2093
+ <li><strong>Ctrl+Shift+F11</strong> with the mouse over a control adds a check: an <code>expect</code> step with the text it shows now (a
2094
+ status message, a saved total). Add one at the end, so the replay tests the outcome, not only the steps.</li>
2095
+ <li><strong>Ctrl+Shift+F12</strong>, or closing the app, finishes. The console shows each step as it is recorded.</li>
2096
+ </ul>
2097
+ <p>What becomes a step:</p>
2098
+ <div class="tbl"><table>
2099
+ <thead>
2100
+ <tr><th>You do</th><th>Step</th></tr>
2101
+ </thead>
2102
+ <tbody>
2103
+ <tr><td>Click a control (two quick clicks: <code>double_click</code>)</td><td><code>click</code> with its AutomationId, name and control type</td></tr>
2104
+ <tr><td>Pick an item in a drop-down</td><td><code>select</code> on the drop-down, with the item's name</td></tr>
2105
+ <tr><td>Type into a field (corrections and pastes included)</td><td>one <code>type</code> with the field's text when you move on, read from the field</td></tr>
2106
+ <tr><td>A key chord outside a text field (<code>ctrl+s</code>, <code>enter</code>, <code>f5</code>)</td><td><code>keys</code></td></tr>
2107
+ <tr><td>Ctrl+Shift+F11 over a control</td><td><code>expect</code> with its text</td></tr>
2108
+ </tbody>
2109
+ </table></div>
2110
+ <ul>
2111
+ <li><strong>Passwords:</strong> a password field is never read; its step types <code>{{secret:NAME}}</code> (the one name in <code>desktop.secrets</code>,
2112
+ or <code>&lt;APP&gt;_PASSWORD</code>, which the result tells you to set). Text equal to a <code>desktop.secrets</code> value is written as its
2113
+ placeholder. The typed characters themselves are never kept, only the field's text when you move on.</li>
2114
+ <li><strong>Only the app:</strong> clicks and keys outside the app's own process are not recorded; nor are the Windows and menu keys,
2115
+ or controls with neither an AutomationId nor a name. The result lists what was left out.</li>
2116
+ <li>The run folder has <code>workflow.yaml</code>, <code>run.json</code> and <code>report.html</code>, like explore. Replay it with
2117
+ <code>forgeqe desktop replay</code>.</li>
2118
+ </ul>
2119
+ </div>
2120
+ </section>
2121
+ <section class="doc" id="doc-docs-desktop-fixing-a-workflow-from-chat">
2122
+ <h4>Fixing a workflow from chat</h4>
2123
+ <div class="prose"><p>When a replayed step fails because the app was slow or a dialog was in the way, ForgeQE's agent fixes the workflow
2124
+ with <code>desktop_edit_workflow</code>: it proposes the change (a longer <code>timeout</code> or <code>retries</code> on the step, a <code>wait_for</code> before
2125
+ it, a dialog button in <code>dismiss</code>, a healed locator, a corrected text), shows you the diff, and writes the file only
2126
+ when you agree. For example, after &quot;step 4 timed out while the invoice list loaded&quot;:</p>
2127
+ <pre><code class="language-text">Change invoice.yaml?
2128
+ • new wait_for step before step 4
2129
+ • step 4: retries 2
2130
+ - op: click
2131
+ + target: {automation_id: btnPost}
2132
+ + retries: 2
2133
+ </code></pre>
2134
+ <ul>
2135
+ <li>Each change has a fixed shape (<code>set</code>, <code>unset</code>, <code>set_step</code>, <code>add_dismiss</code>, <code>insert_step</code>, <code>remove_step</code>,
2136
+ <code>set_variable</code>); step numbers are the replay report's. Anything else is refused.</li>
2137
+ <li>The changed workflow is checked by the engine's own validator before it is shown; one that would not run is not
2138
+ offered. Comments and layout are kept.</li>
2139
+ <li>The old text is kept in the reports folder (<code>desktop-edits/</code>), and a file that changed since the edit was proposed is
2140
+ not overwritten.</li>
2141
+ <li>The workflow must be a <code>.yaml</code> inside the working folder or the reports folder.</li>
2142
+ </ul>
2143
+ </div>
2144
+ </section>
2145
+ <section class="doc" id="doc-docs-desktop-waits-retries-and-dialogs">
2146
+ <h4>Waits, retries and dialogs</h4>
2147
+ <div class="prose"><p>Real apps are slow in places: a splash screen, a form that loads its data, a &quot;Tip of the day&quot; or &quot;Save changes?&quot;
2148
+ dialog. A workflow says how to wait for them, without fixed <code>wait</code> steps:</p>
2149
+ <pre><code class="language-yaml">timeout: 20 # seconds a step waits for its control (default 10, at most 300)
2150
+ retries: 1 # tries a failed action again, after a short pause (default 0, at most 5)
2151
+ dismiss: # clicked whenever one of them is in the way
2152
+ - {name: OK, control_type: Button, automation_id: btnTipClose}
2153
+ - {name: &quot;Don't Save&quot;, control_type: Button}
2154
+ steps:
2155
+ - op: launch
2156
+ timeout: 90 # a slow start: how long to wait for the app's window
2157
+ - op: wait_for # the splash screen goes away
2158
+ target: {automation_id: splashForm}
2159
+ gone: true
2160
+ timeout: 120
2161
+ - op: wait_for # the form has loaded its data
2162
+ target: {automation_id: lblStatus}
2163
+ text: Ready
2164
+ - op: click
2165
+ target: {automation_id: btnPost}
2166
+ timeout: 60 # per step, over the workflow's timeout
2167
+ retries: 3
2168
+ </code></pre>
2169
+ <ul>
2170
+ <li><strong>Each step waits</strong> for its control up to its <code>timeout</code>. <code>click</code>, <code>type</code> and <code>select</code> also wait until the control is
2171
+ enabled and visible (a form that is still loading greys its buttons out).</li>
2172
+ <li><strong><code>wait_for</code></strong> waits until a control is there, shows a <code>text</code>, or with <code>gone: true</code> has gone.</li>
2173
+ <li><strong><code>retries</code></strong> applies to <code>click</code>, <code>double_click</code>, <code>type</code>, <code>select</code>, <code>keys</code>, <code>expect</code> and <code>expect_image</code>. Between
2174
+ tries, replay pauses (1 s, 2 s, … up to 5 s) and closes any <code>dismiss</code> dialog. An error in the workflow itself (a
2175
+ missing variable) is not retried.</li>
2176
+ <li><strong><code>dismiss</code></strong> lists at most 10 targets, checked like any target. When a control is not found, or something waits to
2177
+ go away, replay clicks the <code>dismiss</code> controls that are on screen and looks again, up to 3 times per lookup. The
2178
+ report and the console name every dialog closed, and how many tries a step took.</li>
2179
+ </ul>
2180
+ </div>
2181
+ </section>
2182
+ <section class="doc" id="doc-docs-desktop-visual-checks-expect_image">
2183
+ <h4>Visual checks (<code>expect_image</code>)</h4>
2184
+ <div class="prose"><p><code>expect</code> compares text. When what matters is drawn (a chart, a signature, a colour, a layout), <code>expect_image</code>
2185
+ compares pixels with a baseline image:</p>
2186
+ <pre><code class="language-yaml"> - op: expect_image
2187
+ target: {automation_id: chartRevenue} # without a target: the app's top window
2188
+ image: revenue-chart.png # a file name in &lt;workflow name&gt;.baselines/ next to the workflow
2189
+ tolerance: 0.01 # share of pixels that may differ (default 0.01, i.e. 1%)
2190
+ ignore: [[0, 0, 120, 24]] # regions [x, y, width, height] that change every run (a date, a clock)
2191
+ </code></pre>
2192
+ <ul>
2193
+ <li><strong>The first run</strong> with no baseline saves what it captured and passes; the report says so. Commit the baselines
2194
+ folder with the workflow. <code>--update-baselines</code> replaces them after an intended change.</li>
2195
+ <li><strong>A pixel counts as different</strong> when a colour channel differs by more than 32 of 255, so anti-aliasing and ClearType
2196
+ do not fail a check. A different size (another screen scale or window size) fails with both sizes named.</li>
2197
+ <li><strong>The report</strong> shows the baseline, what was captured, and the differences in red, with the share of pixels that
2198
+ differ (<code>images/NNN-baseline|actual|diff.png</code> in the run folder).</li>
2199
+ <li><strong>Image names</strong> are file names only (letters, digits, <code>_</code>, <code>-</code>, ending <code>.png</code>). A path, <code>..</code>, or a link that leads
2200
+ out of the baselines folder is refused.</li>
2201
+ <li>Run visual checks on one machine with a fixed resolution and scale (see <a href="#doc-docs-desktop-ci-running-desktop-workflows-unattended-nightly-ci">DESKTOP-CI.md</a>).</li>
2202
+ </ul>
2203
+ </div>
2204
+ </section>
2205
+ <section class="doc" id="doc-docs-desktop-unattended-runs">
2206
+ <h4>Unattended runs</h4>
2207
+ <div class="prose"><p>Replay needs a signed-in, unlocked Windows session. Before it starts the app, it checks the session and fails at once
2208
+ with the reason if it is a Windows service (session 0), a locked screen, or a disconnected Remote Desktop session.
2209
+ <code>forgeqe desktop doctor</code> runs the same check. To run the suite every night on a machine of your own, see
2210
+ <a href="#doc-docs-desktop-ci-running-desktop-workflows-unattended-nightly-ci">DESKTOP-CI.md</a>: a runner set up in the user's session (<code>ci/desktop-runner/prepare-runner.ps1</code>), a
2211
+ nightly workflow (<code>ci/desktop-runner/desktop-nightly.example.yml</code>), and <code>--junit</code> results.</p>
2212
+ </div>
2213
+ </section>
2214
+ <section class="doc" id="doc-docs-desktop-updating-ufo²">
2215
+ <h4>Updating UFO²</h4>
2216
+ <div class="prose"><p>UFO² is pinned to one commit. Moving it is checked before it is pinned:</p>
2217
+ <ol>
2218
+ <li><code>.github/workflows/forgeqe-desktop-ufo-update.yml</code> runs weekly, or by hand with a commit. It sets up that UFO on
2219
+ Windows (<code>FORGEQE_UFO_COMMIT</code>, which takes a full 40-character commit id only), runs the adapter tests against its
2220
+ real code and a scripted explore of Notepad, and says in the job summary whether it passed.</li>
2221
+ <li>When it passed: <code>bun scripts/ufo-bump.ts &lt;commit&gt;</code> writes the new pin into <code>src/desktop/setup.ts</code>,
2222
+ <code>py/forgeqe_desktop/ufo_runner.py</code> and this page. Run the tests and push; the Windows CI job tests the pin again.</li>
2223
+ </ol>
2224
+ <p>When it fails, the failing step names what changed in UFO (usually the action executor or the config layout that the
2225
+ adapter wraps). Keep the pin until the adapter is updated for it.</p>
2226
+ </div>
2227
+ </section>
2228
+ <section class="doc" id="doc-docs-desktop-checks-of-the-outcome">
2229
+ <h4>Checks of the outcome</h4>
2230
+ <div class="prose"><p>A recording of actions only proves the steps run. When UFO has finished, and before the app closes, explore shows the
2231
+ model the goal and the app's controls with the text each shows now, and the model picks up to three whose text proves
2232
+ the goal (a status message, a saved value), by number, with the part of the text to expect. A check is kept only when
2233
+ that text really is in that control, so the model can choose among what is on screen but never invent it; text holding
2234
+ a <code>desktop.secrets</code> value is never offered. The checks are saved as <code>expect</code> steps at the end of the workflow, and the
2235
+ report says how many there are. With none, the report and the agent say so: add <code>expect</code> steps by hand.</p>
2236
+ </div>
2237
+ </section>
2238
+ <section class="doc" id="doc-docs-desktop-recording">
2239
+ <h4>Recording</h4>
2240
+ <div class="prose"><p>UFO's own step log names the control and a per-step id, not its AutomationId, so the adapter wraps UFO's
2241
+ <code>ActionExecutor.execute</code> and records each action with the UIA properties of the control it acted on. <code>record.py</code> turns
2242
+ that trace into steps: <code>click_input</code> → <code>click</code>/<code>double_click</code>, <code>set_edit_text</code> → <code>type</code>, <code>keyboard_input</code> → <code>keys</code> (one
2243
+ allowed chord) or <code>type</code>, <code>select</code> → <code>select</code>. Coordinate clicks, scrolls and anything else are listed as not recorded
2244
+ in the report. A typed value equal to a <code>desktop.secrets</code> value is written as its placeholder; typed text that only
2245
+ looks like <code>{{…}}</code> is kept literal, so a model told to type <code>{{secret:X}}</code> cannot make replay type the secret.</p>
2246
+ </div>
2247
+ </section>
2248
+ <section class="doc" id="doc-docs-desktop-security">
2249
+ <h4>Security</h4>
2250
+ <div class="prose"><p>The YAML, the trace, the app's screen text and the model's replies can all be written by someone else. The rules in the
2251
+ repository's <code>CLAUDE.md</code> apply; for this engine:</p>
2252
+ <ul>
2253
+ <li><strong>No command in the workflow.</strong> <code>app:</code> names a <code>desktop.apps</code> entry: an absolute <code>.exe</code> (a <code>.bat</code> or <code>.cmd</code> would run
2254
+ through <code>cmd.exe</code>, which reparses its arguments) and an argument list, started with <code>subprocess.Popen([path, *args])</code>.</li>
2255
+ <li><strong>UFO gets UI tools only.</strong> Each explore writes UFO's <code>mcp.yaml</code> with <code>UICollector</code>, <code>HostUIExecutor</code> and
2256
+ <code>AppUIExecutor</code>. UFO's default <code>CommandLineExecutor</code>, which runs shell commands, and its Word, Excel, PowerPoint and
2257
+ PDF executors are left out. The adapter starts the app, so UFO never launches a program.</li>
2258
+ <li><strong>Keys:</strong> during explore the adapter refuses UFO's Windows and menu keys too, and the report lists what it refused.</li>
2259
+ <li><strong>No one at the console:</strong> UFO's safe guard asks before sensitive actions; explore answers no, since nobody is there.</li>
2260
+ <li><strong>Environment:</strong> the engine process gets system settings, the profile's key variable (or Entra's) and
2261
+ <code>desktop.secrets</code>, nothing else (<code>desktopEnvNames</code>).</li>
2262
+ <li><strong>Explore controls the real mouse and keyboard.</strong> It asks before starting (<code>--yes</code> to skip; required with no
2263
+ terminal). Use a test machine or VM and a test account.</li>
2264
+ <li><strong>Agent tools:</strong> a workflow path the agent gives must be a <code>.yaml</code> inside the working folder or the reports folder (real
2265
+ path, so a link out is refused); variables are names and text only; heals are reported, never written.</li>
2266
+ <li><strong>Healing:</strong> the model picks one of the window's real controls by number; control text is passed as data, and any
2267
+ other answer heals nothing. Heals are written into the workflow only when accepted.</li>
2268
+ <li><strong>Report:</strong> every value is escaped; images are only the engine's own <code>steps/NNN.png</code> and
2269
+ <code>images/NNN-baseline|actual|diff.png</code>.</li>
2270
+ <li><strong>Baselines:</strong> <code>expect_image</code> names are file names, and their real path must be inside the workflow's baselines
2271
+ folder, so a workflow cannot read or overwrite another file.</li>
2272
+ <li><strong>Unattended machines:</strong> <code>prepare-runner.ps1</code> never handles a password; automatic sign-in uses Sysinternals
2273
+ Autologon, which stores it encrypted.</li>
2274
+ <li><strong>Install:</strong> UFO is cloned and checked out at the pinned commit (or <code>FORGEQE_UFO_COMMIT</code>, a full commit id only),
2275
+ and the checkout's <code>HEAD</code> is verified; replay's packages are pinned to exact versions.</li>
2276
+ </ul>
2277
+ </div>
2278
+ </section>
2279
+ <section class="doc" id="doc-docs-desktop-before-first-use-on-windows">
2280
+ <h4>Before first use on Windows</h4>
2281
+ <div class="prose"><p>Verified in CI: setup with and without <code>--ufo</code>, replay, healing, the CLI, and UFO's own code with the adapter (its
2282
+ config, its model calls routed to the profile, the action trace, UI tools only). Still to check, on a Windows test
2283
+ machine with your apps:</p>
2284
+ <ol>
2285
+ <li><strong>UIA coverage:</strong> <code>Inspect.exe</code> shows the controls that matter with stable <code>AutomationId</code>s. Custom grids, Java
2286
+ (needs Java Access Bridge), Delphi, SAP GUI and Citrix (vision only) are where UIA falls short.</li>
2287
+ <li><strong>Explore with your model:</strong> one goal with an Azure profile and one with <code>auth: entra</code>; <code>trace.json</code> has the
2288
+ controls' AutomationIds, and <code>ufo/</code> has UFO's logs.</li>
2289
+ <li><strong>Replay:</strong> the recorded workflow passes 10 runs out of 10; a renamed control is healed.</li>
2290
+ <li><strong>Cost and time</strong> of one explore, from <code>forgeqe</code> usage.</li>
2291
+ </ol>
2292
+ <p>Found by the Windows runs and fixed: UFO's pins need Python 3.10 (pandas 1.4.3); UFO prints emoji panels to stdout, which
2293
+ is the engine's event stream and crashed on the cp1252 console (now sent to stderr, characters replaced); an error in
2294
+ UFO's own evaluation lost the recorded actions (the trace is now kept and saved).</p>
2295
+ </div>
2296
+ </section>
2297
+ <section class="doc" id="doc-docs-desktop-next-steps">
2298
+ <h4>Next steps</h4>
2299
+ <div class="prose"><ol>
2300
+ <li>The checks above.</li>
2301
+ <li><code>forgeqe suite</code> and test cases from Azure DevOps, Jira and Xray for desktop workflows, with results written back.</li>
2302
+ <li>Scribe: export a workflow as pywinauto code (or C# with FlaUI) in the user's repo.</li>
2303
+ <li>App adapters where UIA falls short: Java Access Bridge, SAP GUI scripting, vision-only for Citrix.</li>
2304
+ <li>A pipeline template for a self-hosted Windows runner signed in to a desktop session.</li>
2305
+ </ol>
2306
+ </div>
2307
+ </section>
2308
+ <section class="doc" id="doc-docs-desktop-ci-running-desktop-workflows-unattended-nightly-ci">
2309
+ <h4>Running desktop workflows unattended (nightly CI)</h4>
2310
+ <div class="prose"><p>Desktop replay clicks and types into real windows. It needs what a person at the machine would have: a signed-in
2311
+ Windows session, unlocked, with a screen. GitHub's hosted <code>windows-2022</code> runners have that, and ForgeQE's own CI runs
2312
+ Notepad there. Your own apps usually need a machine of your own, with the app installed, its test database, and a
2313
+ network route to its servers. This page sets one up as a GitHub Actions self-hosted runner. The same rules apply to
2314
+ Azure DevOps agents and Jenkins nodes.</p>
2315
+ </div>
2316
+ </section>
2317
+ <section class="doc" id="doc-docs-desktop-ci-what-breaks-a-desktop-run-and-how-forgeqe-tells-you">
2318
+ <h4>What breaks a desktop run, and how ForgeQE tells you</h4>
2319
+ <div class="prose"><div class="tbl"><table>
2320
+ <thead>
2321
+ <tr><th>Situation</th><th>Effect</th><th>Fix</th></tr>
2322
+ </thead>
2323
+ <tbody>
2324
+ <tr><td>The runner is a Windows service</td><td>Session 0 has no desktop; every control lookup times out</td><td>Run the runner in the user's session (below)</td></tr>
2325
+ <tr><td>The screen is locked</td><td>Windows sends no input to apps</td><td>No inactivity lock, no screen-saver lock</td></tr>
2326
+ <tr><td>A Remote Desktop session was disconnected</td><td>The session is locked</td><td>Leave Remote Desktop with <code>tscon</code> (below), or don't use it on the test machine</td></tr>
2327
+ <tr><td>The machine slept or the screen turned off</td><td>Input and screenshots stop</td><td>Power plan: never sleep, never turn off the screen</td></tr>
2328
+ <tr><td>After a restart nobody signs in</td><td>No session at all</td><td>Sysinternals Autologon</td></tr>
2329
+ </tbody>
2330
+ </table></div>
2331
+ <p>Before each replay or explore, the engine checks the session (<code>py/forgeqe_desktop/session.py</code>). On a session it cannot
2332
+ drive, the run fails at once with the reason, instead of timing out step by step. <code>forgeqe desktop doctor</code> runs the
2333
+ same check from a shell or a job:</p>
2334
+ <pre><code class="language-text">engine ready (C:\Users\qa\.forgeqe\desktop\.venv\Scripts\python.exe)
2335
+ explore not set up (forgeqe desktop setup --ufo)
2336
+ session 1, screen 1920x1080
2337
+ desktop logged in and unlocked
2338
+ </code></pre>
2339
+ <p>It exits with 1 when the desktop cannot be driven, so a job stops before the suite runs.</p>
2340
+ </div>
2341
+ </section>
2342
+ <section class="doc" id="doc-docs-desktop-ci-set-up-the-test-machine">
2343
+ <h4>Set up the test machine</h4>
2344
+ <div class="prose"><p>Use a test machine or VM, and a test account: the runner's jobs control its mouse and keyboard.</p>
2345
+ <ol>
2346
+ <li>
2347
+ <p><strong>Sign in</strong> as the account the tests will run as.</p>
2348
+ </li>
2349
+ <li>
2350
+ <p><strong>Install the runner</strong> (GitHub: Settings → Actions → Runners → New self-hosted runner). Give it the label
2351
+ <code>forgeqe-desktop</code>. When <code>config.cmd</code> asks whether to run as a service, answer <strong>no</strong>.</p>
2352
+ </li>
2353
+ <li>
2354
+ <p><strong>Run <code>ci/desktop-runner/prepare-runner.ps1</code></strong> in an elevated PowerShell (<code>-WhatIf</code> first to see the changes):</p>
2355
+ <pre><code class="language-powershell">.\prepare-runner.ps1 -RunnerDir C:\actions-runner -WhatIf
2356
+ .\prepare-runner.ps1 -RunnerDir C:\actions-runner
2357
+ </code></pre>
2358
+ <p>It stops sleep, screen-off and the screen-saver lock. It turns the inactivity lock off and registers a logon task
2359
+ that starts <code>run.cmd</code> in this account's session. It warns you if the runner is also installed as a service. It
2360
+ never handles a password.</p>
2361
+ </li>
2362
+ <li>
2363
+ <p><strong>Sign in automatically after a restart</strong> with
2364
+ <a href="https://learn.microsoft.com/sysinternals/downloads/autologon" rel="noopener">Sysinternals Autologon</a>, run as the test account.
2365
+ It stores the password encrypted by Windows. Never put the password in a script, a registry value you write
2366
+ yourself, or the repository.</p>
2367
+ </li>
2368
+ <li>
2369
+ <p><strong>Fix the screen size.</strong> A visual check (<code>expect_image</code>) compares pixels, so the resolution and scale (100%, 125%)
2370
+ must match the machine where its baselines were saved. Set them once and leave them.</p>
2371
+ </li>
2372
+ <li>
2373
+ <p><strong>Restart</strong> and check that the runner shows as idle on GitHub. Then run <code>forgeqe desktop doctor</code> from a job.</p>
2374
+ </li>
2375
+ </ol>
2376
+ </div>
2377
+ <section class="doc sub" id="doc-docs-desktop-ci-leaving-remote-desktop-without-locking-the-session"><h5>Leaving Remote Desktop without locking the session</h5><div class="prose"><p>Closing a Remote Desktop window disconnects the session, and Windows locks it. To leave it running instead, hand the
2378
+ session back to the machine's console. In an elevated prompt inside the Remote Desktop session:</p>
2379
+ <pre><code class="language-bat">for /f &quot;skip=1 tokens=3&quot; %s in ('query user %USERNAME%') do tscon %s /dest:console
2380
+ </code></pre>
2381
+ <p>On a VM, the hypervisor's console (Hyper-V, VMware, VirtualBox) needs no such step.</p>
2382
+ </div></section>
2383
+ </section>
2384
+ <section class="doc" id="doc-docs-desktop-ci-the-nightly-job">
2385
+ <h4>The nightly job</h4>
2386
+ <div class="prose"><p><code>ci/desktop-runner/desktop-nightly.example.yml</code> is a ready workflow. Copy it into <code>.github/workflows/</code> in the
2387
+ repository that holds your desktop workflows. It:</p>
2388
+ <ul>
2389
+ <li>runs at night on <code>[self-hosted, windows, forgeqe-desktop]</code>, one run at a time;</li>
2390
+ <li>installs a pinned ForgeQE and runs <code>forgeqe desktop setup</code> and <code>forgeqe desktop doctor</code>;</li>
2391
+ <li>replays every workflow in a folder: <code>forgeqe desktop replay desktop-tests/ --junit results/desktop.xml</code>;</li>
2392
+ <li>keeps the reports, screenshots and image diffs as an artifact.</li>
2393
+ </ul>
2394
+ <p><code>--junit</code> writes one test case per workflow (and per CSV row with <code>--data</code>), which CI servers show as test results.
2395
+ <code>runs.json</code> in the output folder has the same, with the path of each report.</p>
2396
+ <p>Secrets the workflows type (<code>{{secret:NAME}}</code>) come from the CI's secret store as environment variables, and must be
2397
+ listed in <code>desktop.secrets</code> or the environment's <code>secrets</code> in <code>env.yaml</code>. Pick the environment with <code>FORGEQE_ENV</code> or
2398
+ <code>--env</code>.</p>
2399
+ <p>Healing calls the model. In a nightly run, either give the job the model's key so moved controls are healed and
2400
+ reported in <code>heals.json</code> (never written into the workflow without <code>--accept-heals</code>), or pass <code>--no-heal</code> so a moved
2401
+ control fails the run.</p>
1740
2402
  </div>
1741
2403
  </section>
1742
2404
  </div>
@@ -1774,7 +2436,7 @@ forgeqe mobile flow workflow.yaml &gt; login.flow.yaml # a workflow as
1774
2436
  reads only <code>mobile.secrets</code> (and the environment's secrets). A flow kept with <code>forgeqe mobile flow</code> reads each secret
1775
2437
  as <code>${MAESTRO_&lt;NAME&gt;}</code>, never its value, so set <code>MAESTRO_&lt;NAME&gt;</code> when you run it outside ForgeQE; a workflow that types a
1776
2438
  secret not listed there is refused. Team Maestro flows run only with UI commands: scripts,
1777
- JavaScript in <code>${…}</code>, Maestro's AI commands and file writes are refused. Details: <a href="../docs/MOBILE.md">docs/MOBILE.md</a>.</p>
2439
+ JavaScript in <code>${…}</code>, Maestro's AI commands and file writes are refused. Details: <a href="#doc-docs-mobile-forgeqe-mobile-android-and-ios-apps">docs/MOBILE.md</a>.</p>
1778
2440
  <h5>On TestMu AI (formerly LambdaTest)</h5>
1779
2441
  <p>Both drivers run on TestMu AI's devices. Upload the app build once, then use the <code>lt://</code> id it prints as the app's <code>app</code>
1780
2442
  (in env.yaml or <code>mobile.apps</code>; ForgeQE adds it to Appium's <code>LT:Options</code> itself):</p>
@@ -1817,7 +2479,222 @@ set there). The build must be inside the working folder; hidden folders and file
1817
2479
  <p><a href="../samples/mobile-testmu/README.md">samples/mobile-testmu</a> is a ready project with TestMu's Wikipedia sample app: copy
1818
2480
  <code>.env.example</code> to <code>.env</code>, upload the app with <code>forgeqe mobile upload sample.apk</code>, put the id in its <code>forgeqe.yaml</code>,
1819
2481
  download the <code>hyperexecute</code> CLI into the folder, then run <code>forgeqe mobile replay wiki.yaml --driver appium</code>,
1820
- <code>--driver maestro</code>, and <code>forgeqe mobile maestro .maestro</code>. Details: <a href="../docs/MOBILE.md#testmu-ai-formerly-lambdatest">docs/MOBILE.md</a>.</p>
2482
+ <code>--driver maestro</code>, and <code>forgeqe mobile maestro .maestro</code>. Details: <a href="#doc-docs-mobile-testmu-ai-formerly-lambdatest">docs/MOBILE.md</a>.</p>
2483
+ </div>
2484
+ </section>
2485
+ <section class="doc" id="doc-docs-mobile-forgeqe-mobile-android-and-ios-apps">
2486
+ <h4>ForgeQE mobile: Android and iOS apps</h4>
2487
+ <div class="prose"><p>Status: the engine and its tests run on any OS, against a fake Appium server and a stand-in for Maestro. The CI job
2488
+ <code>.github/workflows/forgeqe-mobile-android.yml</code> runs it on an Android emulator: Appium explores and replays the
2489
+ emulator's Settings app, and Maestro replays the same workflow. iOS runs the same code with the XCUITest driver, but
2490
+ no CI job tests it yet (it needs a macOS runner with a simulator).</p>
2491
+ <p>ForgeQE drives mobile apps two ways, and one workflow format runs on both:</p>
2492
+ <div class="tbl"><table>
2493
+ <thead>
2494
+ <tr><th></th><th>Appium</th><th>Maestro</th></tr>
2495
+ </thead>
2496
+ <tbody>
2497
+ <tr><td>Runs</td><td>step by step, through an Appium server (yours or a device cloud's)</td><td>the whole workflow as one Maestro flow</td></tr>
2498
+ <tr><td>Explore a goal with the model</td><td>yes</td><td>no</td></tr>
2499
+ <tr><td>Heal a control that moved</td><td>yes, one model call, reported for review</td><td>no</td></tr>
2500
+ <tr><td>Visual checks (<code>expect_image</code>)</td><td>yes</td><td>no</td></tr>
2501
+ <tr><td>Your team's own flows</td><td>n/a</td><td><code>forgeqe mobile maestro</code>, checked against an allowlist</td></tr>
2502
+ <tr><td>Needs</td><td><code>appium</code> with UiAutomator2 (Android) or XCUITest (iOS), a device or emulator</td><td><code>maestro</code> and a device or emulator</td></tr>
2503
+ </tbody>
2504
+ </table></div>
2505
+ </div>
2506
+ </section>
2507
+ <section class="doc" id="doc-docs-mobile-use">
2508
+ <h4>Use</h4>
2509
+ <div class="prose"><pre><code class="language-bash">forgeqe mobile doctor # does the Appium server answer, is maestro installed?
2510
+ forgeqe mobile apps # the mobile apps of the active environment
2511
+ forgeqe mobile explore &quot;Sign in as the courier and accept the first delivery&quot; --app courier
2512
+ forgeqe mobile replay .forgeqe/reports/mobile-explore-…/workflow.yaml
2513
+ forgeqe mobile replay workflow.yaml --driver maestro
2514
+ forgeqe mobile replay mobile-tests/ --data couriers.csv --junit results.xml
2515
+ forgeqe mobile maestro .maestro/ # the team's flows
2516
+ forgeqe mobile flow workflow.yaml # print a workflow as a Maestro flow
2517
+ forgeqe mobile validate workflow.yaml
2518
+ </code></pre>
2519
+ <p>Or ask ForgeQE's agent in the shell or <code>forgeqe run</code>: &quot;sign in to the courier app and save it as a test&quot;, &quot;replay the
2520
+ courier login on Maestro&quot;, &quot;run our Maestro flows&quot;. Its tools are <code>mobile_explore</code>, <code>mobile_replay</code> and
2521
+ <code>mobile_maestro</code>. It asks before each one starts, and <code>desktop_edit_workflow</code> fixes a mobile workflow after a failed
2522
+ replay, the same way as a desktop one.</p>
2523
+ </div>
2524
+ </section>
2525
+ <section class="doc" id="doc-docs-mobile-apps-and-settings">
2526
+ <h4>Apps and settings</h4>
2527
+ <div class="prose"><p>Mobile apps come from the active environment in env.yaml, or from <code>mobile.apps</code> in forgeqe.yaml:</p>
2528
+ <pre><code class="language-yaml"># env.yaml
2529
+ environments:
2530
+ qa:
2531
+ apps:
2532
+ courier:
2533
+ kind: mobile
2534
+ platform: android # or ios
2535
+ app_id: com.contoso.courier # Android package or iOS bundle id
2536
+ activity: .MainActivity # Android only, optional
2537
+ app: /builds/courier-qa.apk # optional: install this build (.apk/.aab/.ipa/.app/.zip, https URL, bs://…)
2538
+ description: Driver app that confirms delivery
2539
+ secrets: [COURIER_PASSWORD_QA]
2540
+ </code></pre>
2541
+ <pre><code class="language-yaml"># forgeqe.yaml
2542
+ mobile:
2543
+ driver: appium # appium | maestro: the default for replays
2544
+ appium:
2545
+ url: http://127.0.0.1:4723
2546
+ capabilities: # added to every session
2547
+ appium:udid: emulator-5554
2548
+ # username_env: BROWSERSTACK_USERNAME
2549
+ # access_key_env: BROWSERSTACK_ACCESS_KEY
2550
+ maestro:
2551
+ command: [maestro]
2552
+ # device: emulator-5554
2553
+ secrets: [COURIER_PASSWORD]
2554
+ heal: true
2555
+ screenshots: true
2556
+ </code></pre>
2557
+ <p>Each run writes a folder under the reports folder (<code>mobile-explore-…</code>, <code>mobile-replay-…</code>, <code>mobile-maestro-…</code>) with
2558
+ <code>run.json</code>, <code>report.html</code>, step screenshots, and for Maestro runs the flow, <code>maestro.log</code> and Maestro's JUnit report.</p>
2559
+ </div>
2560
+ </section>
2561
+ <section class="doc" id="doc-docs-mobile-workflow-yaml-forgeqe-mobile-v1">
2562
+ <h4>Workflow YAML (<code>forgeqe-mobile-v1</code>)</h4>
2563
+ <div class="prose"><pre><code class="language-yaml">format: forgeqe-mobile-v1
2564
+ name: Courier signs in
2565
+ app: courier
2566
+ timeout: 20
2567
+ dismiss:
2568
+ - {name: Allow, control_type: android.widget.Button} # permission dialogs, when they are in the way
2569
+ steps:
2570
+ - op: launch
2571
+ - op: type
2572
+ target: {automation_id: username}
2573
+ text: &quot;{{courier}}&quot;
2574
+ - op: type
2575
+ target: {automation_id: password}
2576
+ text: &quot;{{secret:COURIER_PASSWORD}}&quot;
2577
+ - op: tap
2578
+ target: {name: Sign in}
2579
+ - op: wait_for
2580
+ target: {automation_id: progress}
2581
+ gone: true
2582
+ - op: scroll_to
2583
+ target: {name: Deliveries}
2584
+ - op: expect
2585
+ target: {automation_id: status}
2586
+ text: Welcome
2587
+ </code></pre>
2588
+ <ul>
2589
+ <li><strong>Ops:</strong> <code>launch</code>, <code>tap</code>, <code>double_tap</code>, <code>long_press</code>, <code>type</code>, <code>expect</code>, <code>expect_image</code>, <code>swipe</code> (<code>direction</code> up,
2590
+ down, left or right, optionally on a <code>target</code>), <code>scroll_to</code> (scrolls until the target is on screen), <code>back</code>,
2591
+ <code>hide_keyboard</code>, <code>keys</code> (<code>enter</code>, <code>back</code>, <code>tab</code>, <code>delete</code>, <code>search</code>), <code>wait</code>, <code>wait_for</code>, <code>close</code>. Timeouts,
2592
+ retries and <code>dismiss</code> work as in desktop workflows. Explore saves a tap on a system dialog (another package's
2593
+ control, such as Android's notification prompt) under <code>dismiss</code>, not as a step, so the workflow still runs where the
2594
+ dialog never shows; on Maestro, each <code>dismiss</code> target is tapped after a launch if it is there.</li>
2595
+ <li><strong>Targets</strong> use the desktop keys, mapped per platform: <code>automation_id</code> is the Android resource-id (with or without the
2596
+ <code>package:id/</code> prefix) or the iOS accessibility identifier; <code>name</code> is the Android text or content-desc, or the iOS
2597
+ label; <code>control_type</code> is the Android class (<code>android.widget.Button</code>) or the iOS type (<code>XCUIElementTypeButton</code>);
2598
+ <code>index</code> picks one of several matches.</li>
2599
+ </ul>
2600
+ </div>
2601
+ </section>
2602
+ <section class="doc" id="doc-docs-mobile-explore">
2603
+ <h4>Explore</h4>
2604
+ <div class="prose"><p>The model sees the goal, its steps so far, the numbered controls on screen with the text each shows, and a screenshot
2605
+ when the profile takes images. It answers with one action on a control by its number (tap, long press, type), a swipe
2606
+ or scroll, back, or done. So it can only act on a control that is on screen. When it says done, it picks checks of the
2607
+ outcome from the controls' current text, kept only when that text is really there, as on the desktop. The app's text
2608
+ is passed as data.</p>
2609
+ </div>
2610
+ </section>
2611
+ <section class="doc" id="doc-docs-mobile-maestro">
2612
+ <h4>Maestro</h4>
2613
+ <div class="prose"><p><strong>ForgeQE workflows on Maestro.</strong> <code>--driver maestro</code> writes the workflow as a Maestro flow (<code>flow.yaml</code> in the run
2614
+ folder) and runs <code>maestro test</code>. Maestro's debug output gives each workflow step its result. Variables are filled in
2615
+ first. Values are matched literally, so Maestro's regular expressions never see them as patterns. A secret becomes
2616
+ <code>${MAESTRO_NAME}</code>, which Maestro reads from the environment ForgeQE gives it, so a secret's value never lands in the flow
2617
+ file or on a command line. A value holding <code>${…}</code> is refused, because Maestro would run it as JavaScript.</p>
2618
+ <p><strong>Your team's flows.</strong> <code>forgeqe mobile maestro &lt;files or folders&gt;</code> checks each flow, and every flow it runs, before
2619
+ Maestro gets it:</p>
2620
+ <ul>
2621
+ <li>Only UI commands pass: <code>launchApp</code>, <code>tapOn</code>, <code>inputText</code>, <code>assertVisible</code>, <code>scrollUntilVisible</code>, <code>swipe</code>,
2622
+ <code>extendedWaitUntil</code>, <code>runFlow</code>, <code>repeat</code>, <code>retry</code> and the like.</li>
2623
+ <li>These are refused:
2624
+ <ul>
2625
+ <li><code>runScript</code>, <code>evalScript</code> and <code>assertTrue</code>, which run JavaScript.</li>
2626
+ <li>Maestro's AI commands, which send screenshots to a service.</li>
2627
+ <li><code>takeScreenshot</code>, recordings and <code>addMedia</code>, which read or write files.</li>
2628
+ </ul>
2629
+ </li>
2630
+ <li><code>${…}</code> may only name a variable (<code>${USERNAME}</code>), never an expression.</li>
2631
+ <li><code>when: {true: …}</code> is refused, because it is JavaScript.</li>
2632
+ <li><code>openLink</code> takes http(s) or the app's own deep links, never <code>javascript:</code>, <code>file:</code> or <code>intent:</code>.</li>
2633
+ <li><code>runFlow</code> files must be <code>.yaml</code> inside the working folder; links out of it are refused too. Nesting stops at 5 levels.</li>
2634
+ </ul>
2635
+ <p>A refused flow does not run, and the error names the command.</p>
2636
+ </div>
2637
+ </section>
2638
+ <section class="doc" id="doc-docs-mobile-testmu-ai-formerly-lambdatest">
2639
+ <h4>TestMu AI (formerly LambdaTest)</h4>
2640
+ <div class="prose"><p>Both drivers run on TestMu's devices. Upload the app once, then use the <code>lt://</code> id it returns as the app's <code>app</code>:</p>
2641
+ <pre><code class="language-bash">forgeqe mobile upload courier.apk # → lt://APP… (--virtual for TestMu's emulators)
2642
+ </code></pre>
2643
+ <p>From the shell, ask in words (&quot;upload builds/courier.apk to TestMu for courier&quot;): <code>mobile_upload</code> asks first, uploads
2644
+ the build (it must be inside the working folder), and saves the <code>lt://</code> id as that app's <code>app:</code> in forgeqe.yaml.</p>
2645
+ <p>The user and key come from <code>LT_USERNAME</code> and <code>LT_ACCESS_KEY</code> (or the variables the config names), for example in a
2646
+ <code>.env</code> file in the project folder. <code>samples/mobile-testmu/</code> is a ready project with TestMu's Wikipedia sample app.</p>
2647
+ <pre><code class="language-yaml">mobile:
2648
+ apps:
2649
+ courier: { platform: android, app_id: com.contoso.courier, app: &quot;lt://APP1016…&quot; }
2650
+ appium: # Appium on TestMu's mobile hub
2651
+ url: https://mobile-hub.lambdatest.com/wd/hub
2652
+ username_env: LT_USERNAME
2653
+ access_key_env: LT_ACCESS_KEY
2654
+ capabilities:
2655
+ &quot;LT:Options&quot;: { w3c: true, isRealMobile: true, deviceName: &quot;Pixel 6&quot;, platformVersion: &quot;14&quot;, build: nightly } # app: added from the app's own
2656
+ maestro:
2657
+ testmu: # Maestro on TestMu, through HyperExecute
2658
+ cli: /opt/hyperexecute # TestMu's hyperexecute CLI
2659
+ devices: [&quot;Pixel 6-14&quot;] # name-version, or a pattern such as &quot;.*-.*&quot;
2660
+ real_device: true # false: TestMu's emulators
2661
+ # username_env: LT_USERNAME # the defaults
2662
+ # access_key_env: LT_ACCESS_KEY
2663
+ </code></pre>
2664
+ <ul>
2665
+ <li><strong>Appium</strong> needs nothing else: the Appium driver talks to the hub with the user and key from the two variables.</li>
2666
+ <li><strong>Maestro</strong> runs through HyperExecute. ForgeQE writes a HyperExecute project in the run folder and starts the CLI.
2667
+ The project holds the flows in their folder layout (so <code>runFlow</code> still works), the app (an <code>lt://</code> id is passed as
2668
+ the device's <code>appId</code>; a build file is uploaded with the project), a fixed setup script (Android platform tools 35.0.2
2669
+ and Maestro 2.6.1 from its release, both checked against their SHA-256) and a fixed run script, and
2670
+ <code>hyperexecute.yaml</code>. Team flows are checked against the allowlist first, as always. Flow file names may only use
2671
+ letters, digits and <code>_ . - /</code> (and one leading dot, for <code>.maestro/</code>), because HyperExecute's shell reads them. After
2672
+ the job, ForgeQE downloads its artifacts (Maestro's command results and JUnit reports), so each step gets its own
2673
+ result in the report.</li>
2674
+ <li><strong>Secrets on Maestro on TestMu:</strong> a workflow that types a secret is refused there for now (HyperExecute would need
2675
+ the value in its own vault). Replay it on TestMu with <code>--driver appium</code>.</li>
2676
+ <li>The access key is passed to the CLI as an argument. It is removed from the CLI's output and from every file in the
2677
+ run folder. Neither the user name nor the key is passed to the flows' environment.</li>
2678
+ <li><code>.github/workflows/forgeqe-mobile-testmu.yml</code> runs all three on TestMu (Appium, Maestro with a workflow, a team flow)
2679
+ with TestMu's Wikipedia sample app, once the repository has the secrets <code>LT_USERNAME</code> and <code>LT_ACCESS_KEY</code>.</li>
2680
+ </ul>
2681
+ </div>
2682
+ </section>
2683
+ <section class="doc" id="doc-docs-mobile-security">
2684
+ <h4>Security</h4>
2685
+ <div class="prose"><ul>
2686
+ <li><strong>No commands from the workflow or the model.</strong> <code>app:</code> names a configured app (a platform and an id). Appium is
2687
+ spoken to over HTTP, and <code>maestro</code> is started with an argument list, never a shell.</li>
2688
+ <li><strong>Locators</strong> go into XPath as quoted literals, so a target cannot change the query.</li>
2689
+ <li><strong>Secrets:</strong> <code>{{secret:NAME}}</code> reads only <code>mobile.secrets</code> and the environment's secrets, and ForgeQE's own keys are
2690
+ refused there. Explore types a secret from its placeholder, so the model never sees the value.</li>
2691
+ <li><strong>The engine's environment</strong> holds system settings, the profile's key, the listed secrets, the cloud's user and key,
2692
+ and <code>JAVA_HOME</code> / <code>ANDROID_HOME</code>, nothing else.</li>
2693
+ <li><strong>Cloud credentials</strong> come from the variables <code>mobile.appium.username_env</code> and <code>access_key_env</code> name; a user or key in
2694
+ the URL is refused. TLS is verified as usual.</li>
2695
+ <li><strong>App builds</strong> are absolute paths to an app file, https URLs, or a device cloud's own upload ids (<code>bs://</code>, <code>lt://</code>,
2696
+ <code>storage:filename=</code>); <code>file://</code> and other schemes are refused.</li>
2697
+ </ul>
1821
2698
  </div>
1822
2699
  </section>
1823
2700
  </div>
@@ -1826,12 +2703,12 @@ download the <code>hyperexecute</code> CLI into the folder, then run <code>forge
1826
2703
  <section class="chapter" id="area-operations" aria-labelledby="area-operations-h">
1827
2704
  <header class="chead"><span class="mono">09</span><h2 id="area-operations-h">Operations</h2><p>Monitoring and CI pipelines</p></header>
1828
2705
  <article class="spec feat" id="cap-containers">
1829
- <div class="hd"><span class="mono">Operations</span><h3>Services and mocks in Docker or Podman</h3></div>
1830
- <p class="ds">Starts the services a test run needs (a database, a queue, the API under test) in Docker or Podman, as forgeqe.yaml declares them, and waits until each is ready; stops them, and shows their status and logs. Starts mocks of an API on WireMock, from the calls a browser run recorded or from an OpenAPI description, and can point an API app of the environment at the mock for the session.</p>
2706
+ <div class="hd"><span class="mono">Operations</span><h3>Services, mocks, workspace and browser in Docker or Podman</h3></div>
2707
+ <p class="ds">Starts the services a test run needs (a database, a queue, the API under test) in Docker or Podman, as forgeqe.yaml declares them, and waits until each is ready; stops them, and shows their status and logs. Starts mocks of an API on WireMock, from the calls a browser run recorded or from an OpenAPI description, and can point an API app of the environment at the mock for the session. With containers.workspace, the coding agents' shell commands (build, restore, run) and every test ForgeQE runs go to a container of the project's image; with browser.provider docker, the browser agent, the replay and the coding agent's live browser use Chromium in a container.</p>
1831
2708
  <div class="facts">
1832
2709
  <div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>Start the database and the queue in Docker</li><li>Start the services from our docker-compose file</li><li>Mock the payments API from the last run</li><li>Create a mock service from openapi.yaml and point order-api at it</li><li>Show the logs of the db container</li></ul></div>
1833
2710
  <div class="fact"><span class="mono">What it needs</span><ul class="needs"><li><span class="need req">Required</span><span>Docker or Podman on PATH; the services to start in containers.services in forgeqe.yaml (mocks need none)</span></li></ul></div>
1834
- <div class="fact"><span class="mono">Limits</span><ul class="plainlist"><li>Starts only services declared in forgeqe.yaml containers.services; it never runs an image, flag or mount from a request</li><li>No privileged containers, host network, or access to the container runtime's socket; ports on 127.0.0.1 only; mounts read-only from inside the project</li><li>Mocks answer REST calls (WireMock); GraphQL, SOAP and gRPC are not mocked</li><li>Reads docker-compose files only for their services' image, ports, environment, bind mounts, command and depends_on; refuses build, privileged, network_mode, cap_add, devices and env_file; leaves named volumes out</li></ul></div>
2711
+ <div class="fact"><span class="mono">Limits</span><ul class="plainlist"><li>Starts only services declared in forgeqe.yaml containers.services; it never runs an image, flag or mount from a request</li><li>No privileged containers, host network, or access to the container runtime's socket; ports on 127.0.0.1 only; mounts read-only from inside the project</li><li>Mocks answer REST calls (WireMock); GraphQL, SOAP and gRPC are not mocked</li><li>Reads docker-compose files only for their services' image, ports, environment, bind mounts, command and depends_on; refuses build, privileged, network_mode, cap_add, devices and env_file; leaves named volumes out</li><li>The workspace mounts the repository with its credential files (.env, .npmrc, keys) covered; the agents and the model key stay on this machine; ForgeQE's own agent keeps this machine's shell</li><li>A browser in a container reaches this machine's apps as host.docker.internal, not localhost; it is headless, or visible on a virtual display, never on this screen</li></ul></div>
1835
2712
  <div class="fact"><span class="mono">Tools</span><p class="tools"><code>services</code> <code>mock_service</code></p></div>
1836
2713
  </div>
1837
2714
  <div class="detail"><span class="mono">In detail</span>
@@ -1839,7 +2716,7 @@ download the <code>hyperexecute</code> CLI into the folder, then run <code>forge
1839
2716
  <h4>Containers and mock services</h4>
1840
2717
  <div class="prose"><p>When Docker or Podman is installed, ForgeQE can start the services a test run needs, and mock an API the tests should
1841
2718
  not call. Ask in the shell: &quot;start the database and the queue&quot;, &quot;mock the payments API from the last run&quot;, &quot;make a mock
1842
- from openapi.yaml and point order-api at it&quot;, &quot;show the db logs&quot;. Full reference: <a href="../docs/CONTAINERS.md">docs/CONTAINERS.md</a>.</p>
2719
+ from openapi.yaml and point order-api at it&quot;, &quot;show the db logs&quot;. Full reference: <a href="#doc-docs-containers-forgeqe-containers-services-and-mocks-in-docker-or-podman">docs/CONTAINERS.md</a>.</p>
1843
2720
  <p>The project declares its services in forgeqe.yaml. ForgeQE starts only these, and builds each command itself:</p>
1844
2721
  <pre><code class="language-yaml">containers:
1845
2722
  # runtime: podman # default: docker, else podman, on PATH
@@ -1893,6 +2770,27 @@ secret passed by name.</li>
1893
2770
  </ul>
1894
2771
  <p><code>x-forgeqe: { ready: … }</code> sets the ready check. Without it, a service is ready when its first port takes connections.
1895
2772
  The project's containers share their own network, so services reach each other by name (<code>db:5432</code>).</p>
2773
+ <p><strong>Docker as the workspace and the browser.</strong> Two settings move the rest of the work into containers:</p>
2774
+ <pre><code class="language-yaml">browser:
2775
+ provider: docker # the browser agent, the replay and Scribe's live browser use Chromium in a container
2776
+ containers:
2777
+ workspace: # Scribe's shell commands and every test ForgeQE runs use this image
2778
+ image: mcr.microsoft.com/playwright/dotnet:v1.63.0-noble
2779
+ env: { NEXUS_ENVIRONMENT: CI }
2780
+ browser: # optional
2781
+ headless: false # visible on a virtual display inside the container (some sites refuse headless)
2782
+ </code></pre>
2783
+ <ul>
2784
+ <li><strong>Workspace:</strong> Scribe builds, restores, runs and fixes tests in a Linux container of the project's image, with the
2785
+ repository at the same path. Each test run gets its own container from that image, and <code>HOME</code> is a volume of the
2786
+ project's, so package caches outlive the session. The agents and the model key stay on this machine, and ForgeQE's
2787
+ own agent keeps this machine's shell. <code>.env</code>, keys and other credential files in the repository are covered inside
2788
+ the containers.</li>
2789
+ <li><strong>Browser:</strong> each browser run gets a fresh container, reached through a gateway that wants a random token. A
2790
+ declared service is reached by its name; an app on this machine is <code>host.docker.internal</code>, not <code>localhost</code>.
2791
+ <code>--browser-provider docker</code> picks it for one run.</li>
2792
+ <li><code>FORGEQE_HOST_RUN=1</code> runs Scribe's commands and the tests on this machine for a session.</li>
2793
+ </ul>
1896
2794
  <p><strong>Tests in a container.</strong> With <code>containers.tests</code>, <code>run_tests</code> runs the repository's tests in that image instead of
1897
2795
  on this machine, after starting the services it lists:</p>
1898
2796
  <pre><code class="language-yaml">containers:
@@ -1915,6 +2813,312 @@ secrets.</li>
1915
2813
  for readiness, and uses the mapped port. It never uses privileged mode, the host network, or the Docker socket.</p>
1916
2814
  </div>
1917
2815
  </section>
2816
+ <section class="doc" id="doc-docs-containers-forgeqe-containers-services-and-mocks-in-docker-or-podman">
2817
+ <h4>ForgeQE containers: services and mocks in Docker or Podman</h4>
2818
+ <div class="prose"><p>Status: the tests run against a fake runtime on any OS. They check the commands ForgeQE builds, the checks on config,
2819
+ compose files, mounts and secrets, the mock stubs, and tests in a container. <code>tests/containers.live.test.ts</code> runs on
2820
+ real Docker (and is skipped without it):</p>
2821
+ <ul>
2822
+ <li>Postgres from forgeqe.yaml and from a compose file.</li>
2823
+ <li>A WireMock mock that answers a recorded run's calls.</li>
2824
+ <li>A tests container that reaches the database by name and writes its results into the repository as this machine's
2825
+ user.</li>
2826
+ </ul>
2827
+ <p><code>.github/workflows/forgeqe-containers.yml</code> runs both on GitHub's Docker. There, a skipped live test fails the job.</p>
2828
+ <p>ForgeQE uses Docker or Podman, when one is installed, for five jobs:</p>
2829
+ <ul>
2830
+ <li><strong>Services a test run needs:</strong> a database, a queue, a cache, or the API under test. The project declares them in
2831
+ forgeqe.yaml, or in a docker-compose file that forgeqe.yaml names.</li>
2832
+ <li><strong>Mocks of an API:</strong> WireMock stubs made from the calls a browser run recorded, or from an OpenAPI description, so
2833
+ the tests can run without the real service.</li>
2834
+ <li><strong>The repository's tests in a container:</strong> <code>run_tests</code> runs the command the user confirmed in an image, instead of
2835
+ on this machine, after starting the services the tests need.</li>
2836
+ <li><strong>A workspace for the coding agents:</strong> with <code>containers.workspace</code>, Scribe's shell commands (build, restore, run,
2837
+ git) and every test ForgeQE runs for it go to a container of the project's image. The agents stay on this machine.</li>
2838
+ <li><strong>A browser in a container:</strong> with <code>browser.provider: docker</code>, the browser agent, the replay and Scribe's live
2839
+ browser drive Chromium in a container of Microsoft's Playwright image.</li>
2840
+ </ul>
2841
+ <p>Scribe also has a <code>testcontainers</code> skill, for tests that start their own services from code (below).</p>
2842
+ <p>ForgeQE's agent uses them through two tools, <code>services</code> and <code>mock_service</code>. The CLI has <code>forgeqe services</code>. When
2843
+ neither Docker nor Podman is on PATH, the agent isn't offered these tools.</p>
2844
+ </div>
2845
+ </section>
2846
+ <section class="doc" id="doc-docs-containers-use">
2847
+ <h4>Use</h4>
2848
+ <div class="prose"><pre><code class="language-bash">forgeqe services # the declared services, what runs, whether the runtime answers
2849
+ forgeqe services up # start every declared service and wait until each is ready
2850
+ forgeqe services up db queue # just these
2851
+ forgeqe services logs db --lines 50 # last lines, secrets removed
2852
+ forgeqe services down # stop and remove this project's containers (mocks included)
2853
+ </code></pre>
2854
+ <p>In the shell or <code>forgeqe run</code>, you can ask, for example:</p>
2855
+ <ul>
2856
+ <li>&quot;start the database&quot;</li>
2857
+ <li>&quot;is the queue up?&quot;</li>
2858
+ <li>&quot;mock the payments API from the last run&quot;</li>
2859
+ <li>&quot;make a mock service from openapi.yaml and point order-api at it&quot;</li>
2860
+ <li>&quot;stop the services&quot;</li>
2861
+ </ul>
2862
+ <p>The agent asks before it starts anything.</p>
2863
+ </div>
2864
+ </section>
2865
+ <section class="doc" id="doc-docs-containers-declaring-services">
2866
+ <h4>Declaring services</h4>
2867
+ <div class="prose"><pre><code class="language-yaml"># forgeqe.yaml
2868
+ containers:
2869
+ runtime: docker # docker | podman; default: docker, else podman, on PATH
2870
+ services:
2871
+ db:
2872
+ description: Orders database for the API tests
2873
+ image: postgres:16.4 # an exact version, or name@sha256:&lt;digest&gt;
2874
+ ports: [5432] # 127.0.0.1:5432 -&gt; 5432; &quot;15432:5432&quot; maps another port of this machine
2875
+ env: # plain values only
2876
+ POSTGRES_DB: shop
2877
+ POSTGRES_USER: shop
2878
+ secrets: [POSTGRES_PASSWORD] # environment variable names; their values come from the environment or .env
2879
+ volumes:
2880
+ - ./db/init:/docker-entrypoint-initdb.d # read-only; a folder or file inside the project
2881
+ command: [&quot;-c&quot;, &quot;log_statement=all&quot;] # arguments for the image's entrypoint
2882
+ ready: { tcp: 5432 } # or { http: /health, port: 8080 }; without it: the container runs
2883
+ timeout: 90 # seconds to wait until ready (default 60)
2884
+ queue:
2885
+ image: rabbitmq:3.13.7-management
2886
+ ports: [5672, 15672]
2887
+ ready: { http: /, port: 15672 }
2888
+ mock:
2889
+ image: wiremock/wiremock:3.9.1 # the default
2890
+ </code></pre>
2891
+ <div class="tbl"><table>
2892
+ <thead>
2893
+ <tr><th>Setting</th><th>Rule</th></tr>
2894
+ </thead>
2895
+ <tbody>
2896
+ <tr><td>Service name</td><td>Lower-case letters, digits and <code>-</code>, starting with a letter. Names starting with <code>mock-</code> are kept for mocks.</td></tr>
2897
+ <tr><td><code>image</code></td><td>Must have an exact tag or a digest. An untagged image or <code>latest</code> is refused, because it changes under the same name.</td></tr>
2898
+ <tr><td><code>ports</code></td><td>Always bound to 127.0.0.1. An address such as <code>0.0.0.0:5432:5432</code> is refused.</td></tr>
2899
+ <tr><td><code>env</code></td><td>Plain values on one line. Secrets go under <code>secrets</code>.</td></tr>
2900
+ <tr><td><code>secrets</code></td><td>Variable names. They are passed to the runtime by name (<code>--env NAME</code>), so the value is never on a command line. One of ForgeQE's own keys (a model key, a tracker token) is refused.</td></tr>
2901
+ <tr><td><code>volumes</code></td><td>Always read-only. Must be inside the folder of forgeqe.yaml, and can't be the whole folder.</td></tr>
2902
+ <tr><td><code>command</code></td><td>A list of arguments, never one command string.</td></tr>
2903
+ <tr><td><code>depends_on</code></td><td>Services to start first.</td></tr>
2904
+ <tr><td>Anything else</td><td>Refused with an error, not ignored. That includes <code>privileged</code>, <code>network</code>, <code>cap_add</code>, <code>devices</code>, <code>user</code>, <code>pid</code> and the socket.</td></tr>
2905
+ </tbody>
2906
+ </table></div>
2907
+ <p>Every service, whether from forgeqe.yaml or a compose file, can list <code>depends_on</code>. <code>up</code> starts a service after the
2908
+ ones it depends on; a circle is refused.</p>
2909
+ </div>
2910
+ </section>
2911
+ <section class="doc" id="doc-docs-containers-compose-files">
2912
+ <h4>Compose files</h4>
2913
+ <div class="prose"><pre><code class="language-yaml">containers:
2914
+ compose: docker-compose.yml # or a list; inside the project
2915
+ </code></pre>
2916
+ <p>ForgeQE reads each service of the file and translates it into a forgeqe.yaml service. The same rules then check it,
2917
+ and a service declared in forgeqe.yaml wins over the compose file's service of the same name. Compose itself never
2918
+ runs.</p>
2919
+ <div class="tbl"><table>
2920
+ <thead>
2921
+ <tr><th>Compose</th><th>ForgeQE</th></tr>
2922
+ </thead>
2923
+ <tbody>
2924
+ <tr><td><code>image</code></td><td>Required; an exact version or a digest. <code>build</code> is refused: build and tag the image first.</td></tr>
2925
+ <tr><td><code>ports</code></td><td><code>&quot;5432&quot;</code>, <code>&quot;15432:5432&quot;</code>, <code>&quot;127.0.0.1:15432:5432&quot;</code>, or the long form; always bound to 127.0.0.1. Any other address, and UDP, are refused.</td></tr>
2926
+ <tr><td><code>environment</code></td><td>Plain values are passed as they are. <code>KEY: ${KEY}</code>, <code>KEY</code> alone in the list form, and <code>KEY: ${KEY:-default}</code> pass the variable by name as a secret. <code>${OTHER}</code> under another name, and values that mix text and <code>${…}</code>, are refused.</td></tr>
2927
+ <tr><td><code>volumes</code></td><td>Bind mounts (<code>./db/init:/docker-entrypoint-initdb.d</code>) are mounted read-only, and <code>:rw</code> is noted. Named and anonymous volumes are left out, so the container keeps no data between runs. A path outside the project is refused when the service starts.</td></tr>
2928
+ <tr><td><code>command</code></td><td>A list, or a string split into arguments (compose uses no shell, and neither does ForgeQE).</td></tr>
2929
+ <tr><td><code>depends_on</code></td><td>List or map form: the start order. <code>condition</code> is not used; ForgeQE waits until each service is ready.</td></tr>
2930
+ <tr><td><code>x-forgeqe</code></td><td><code>ready</code>, <code>timeout</code>, <code>description</code>, as in forgeqe.yaml. Without <code>ready</code>, a service with ports is ready when its first port takes connections.</td></tr>
2931
+ <tr><td><code>container_name</code>, <code>restart</code>, <code>labels</code>, <code>healthcheck</code>, <code>networks</code>, <code>expose</code>, <code>deploy</code>, <code>profiles</code>, …</td><td>Ignored: ForgeQE names, labels and networks the containers itself.</td></tr>
2932
+ <tr><td><code>privileged</code>, <code>network_mode</code>, <code>cap_add</code>, <code>devices</code>, <code>pid</code>, <code>ipc</code>, <code>user</code>, <code>security_opt</code>, <code>volumes_from</code>, <code>env_file</code>, <code>entrypoint</code>, <code>extends</code>, …</td><td>Refused, and the error names the setting.</td></tr>
2933
+ </tbody>
2934
+ </table></div>
2935
+ <p>What was left out (a named volume, a mount made read-only) is listed in <code>notes</code>.</p>
2936
+ </div>
2937
+ </section>
2938
+ <section class="doc" id="doc-docs-containers-the-project-network">
2939
+ <h4>The project network</h4>
2940
+ <div class="prose"><p>Each project's containers join their own network, <code>forgeqe-&lt;project&gt;</code>, under their service name. So the API container
2941
+ reaches the database as <code>db:5432</code>, and the tests container reaches both. No other container joins the network, and no
2942
+ container is on the host's network. <code>forgeqe services down</code> with no names also removes the network.</p>
2943
+ </div>
2944
+ </section>
2945
+ <section class="doc" id="doc-docs-containers-tests-in-a-container">
2946
+ <h4>Tests in a container</h4>
2947
+ <div class="prose"><pre><code class="language-yaml">containers:
2948
+ tests:
2949
+ image: mcr.microsoft.com/playwright:v1.48.2-jammy # an exact version; the tests' toolchain
2950
+ services: [db, order-api] # started first (when not running yet); the tests reach them by name
2951
+ env: { DB_HOST: db }
2952
+ secrets: [TEST_DB_PASSWORD]
2953
+ </code></pre>
2954
+ <p>With <code>containers.tests</code> set, <code>run_tests</code> runs the tests in that image unless it is called with <code>in_container: false</code>.
2955
+ Its confirmation says where the tests will run. The container:</p>
2956
+ <ul>
2957
+ <li><strong>Runs the command the user confirmed</strong> with the image's own shell, <code>sh -c &quot;&lt;command&gt;&quot;</code>, inside the container,
2958
+ never on this machine.</li>
2959
+ <li><strong>Mounts the repository read-write at its own path</strong> and works in it, so results land where ForgeQE reads them.
2960
+ On this machine, the tests already have the whole disk. A folder outside the repository that the results go to is
2961
+ mounted the same way.</li>
2962
+ <li><strong>Runs as this machine's user</strong> (<code>--user uid:gid</code>), so files it writes are yours. It also gets
2963
+ <code>--security-opt no-new-privileges</code>, <code>--pids-limit 4096</code> and <code>--rm</code>, and it is removed by name when the run ends or is
2964
+ stopped.</li>
2965
+ <li><strong>Gets only these variables:</strong> <code>CI=1</code>, <code>NO_COLOR</code>, <code>HOME=/tmp</code>, the environment's <code>FORGEQE_ACTIVE_ENV</code>,
2966
+ <code>FORGEQE_APP_*</code> and <code>FORGEQE_VAR_*</code>, the saved logins' variables, and <code>containers.tests</code> <code>env</code> and <code>secrets</code>.
2967
+ Secrets are passed by name. ForgeQE's own keys are never passed.</li>
2968
+ <li><strong>Has secret values removed from its output</strong>, as on this machine.</li>
2969
+ </ul>
2970
+ <p>This needs a Linux or macOS machine, because the repository is mounted at its own path.</p>
2971
+ </div>
2972
+ </section>
2973
+ <section class="doc" id="doc-docs-containers-the-workspace">
2974
+ <h4>The workspace</h4>
2975
+ <div class="prose"><pre><code class="language-yaml">containers:
2976
+ workspace:
2977
+ image: mcr.microsoft.com/playwright/dotnet:v1.63.0-noble # an exact version; the project's toolchain
2978
+ services: [db] # started first; reached by name
2979
+ env: { NEXUS_ENVIRONMENT: CI, DOTNET_ROLL_FORWARD: Major }
2980
+ secrets: [TEST_DB_PASSWORD] # by name; values from the environment or .env
2981
+ </code></pre>
2982
+ <p>With <code>containers.workspace</code> set:</p>
2983
+ <ul>
2984
+ <li><strong>Scribe's shell runs in the container.</strong> The agent runtime runs each command as <code>$SHELL -c &lt;command&gt;</code>. ForgeQE
2985
+ sets <code>SHELL</code> to a script that runs the same command with <code>docker exec … sh -c &lt;command&gt;</code>, in the same folder,
2986
+ in the workspace container. A folder outside the repository is refused, since the container has only the
2987
+ repository. The agent is told its commands run in a Linux container of that image.</li>
2988
+ <li><strong>Every test ForgeQE runs goes to the same image:</strong> the coding loop's runs, <code>heal</code>, <code>suite</code>'s existing tests and
2989
+ <code>run_tests</code>. Each run gets its own container (<code>docker run --rm</code>), so stopping a run removes it. Browsers the test
2990
+ starts are found with <code>docker top</code>, as ForgeQE finds them on this machine.</li>
2991
+ <li><strong>What stays on this machine:</strong> the agent runtime, the model key, ForgeQE's other secrets, the file tools (read,
2992
+ edit, write; the repository is the same folder in both places), and ForgeQE's own agent, which works with you
2993
+ on this machine and keeps its shell here. <code>FORGEQE_HOST_RUN=1</code> runs everything on this machine for a session.</li>
2994
+ <li><strong>The container:</strong>
2995
+ <ul>
2996
+ <li>one per ForgeQE process and project, <code>forgeqe-&lt;project&gt;-workspace-&lt;id&gt;</code>, removed when ForgeQE ends;</li>
2997
+ <li>the repository mounted read-write at its own path; credential files in it (<code>.env</code>, <code>.env.*</code> except templates,
2998
+ <code>.npmrc</code>, <code>.netrc</code>, <code>*.pem</code>, <code>*.key</code>, <code>id_*</code> keys) covered with an empty read-only file, and <code>.ssh</code>, <code>.aws</code>,
2999
+ <code>.azure</code>, <code>.docker</code>, <code>.kube</code> and <code>.forgeqe/sessions</code> with an empty folder;</li>
3000
+ <li><code>HOME</code> on the project's volume <code>forgeqe-&lt;project&gt;-home</code>, so NuGet, npm and pip caches outlive the session. Docker
3001
+ creates the volume for root, so ForgeQE runs <code>chown</code> in it once, with a fixed command;</li>
3002
+ <li>this machine's user (<code>--user uid:gid</code>), <code>no-new-privileges</code>, <code>--pids-limit 4096</code>, the project network;</li>
3003
+ <li>only <code>CI=1</code>, <code>HOME</code>, the <code>env</code> and <code>secrets</code> listed, and, in each command, the environment's <code>FORGEQE_APP_*</code> and
3004
+ <code>FORGEQE_VAR_*</code> and the saved logins' variables, by name.</li>
3005
+ </ul>
3006
+ </li>
3007
+ <li><strong><code>containers.tests</code></strong> still names the image <code>run_tests</code> uses when it is set. Without it, <code>run_tests</code> uses the
3008
+ workspace's image.</li>
3009
+ </ul>
3010
+ <p>This needs a Linux or macOS machine, as tests in a container do.</p>
3011
+ </div>
3012
+ </section>
3013
+ <section class="doc" id="doc-docs-containers-the-browser-in-a-container">
3014
+ <h4>The browser in a container</h4>
3015
+ <div class="prose"><pre><code class="language-yaml">browser:
3016
+ provider: docker # or per run: --browser-provider docker
3017
+ containers:
3018
+ browser: # optional
3019
+ image: mcr.microsoft.com/playwright:v1.63.0-noble # default; needs Node.js and /ms-playwright
3020
+ headless: false # default: browser.headless; a visible browser runs on a virtual display (Xvfb)
3021
+ </code></pre>
3022
+ <ul>
3023
+ <li><strong>One container per use:</strong> each browser run (a suite runs <code>suite.parallel</code> at once) gets its own, and a coding job
3024
+ gets one that its replay and live browser share. Each is removed when its run or job ends.</li>
3025
+ <li><strong>Connection:</strong> Chromium listens on 127.0.0.1 inside the container. CDP has no authentication of its own, so a small
3026
+ gateway in front of it (<code>BROWSER_SERVER</code> in <code>src/containers/browser.ts</code>, run with the image's Node.js) accepts a
3027
+ connection only with a random 48-character token in its path, compared in constant time, and refuses a browser
3028
+ <code>Origin</code> that is not local. The port is published on 127.0.0.1 only. The token reaches the engine, the replay and
3029
+ the live browser's MCP server through the environment, never a file or a command line.</li>
3030
+ <li><strong>Visible or headless:</strong> a visible browser runs on Xvfb inside the container. Sites that turn headless browsers
3031
+ away see an ordinary one, but you do not see it on your screen. Runs keep screenshots and traces as usual.</li>
3032
+ <li><strong>Reaching the app:</strong> the container joins the project network, so a declared service is reached by its name
3033
+ (<code>http://order-api:8080</code>). An app this machine serves is <code>http://host.docker.internal:&lt;port&gt;</code>; <code>localhost</code> is the
3034
+ container itself, and ForgeQE warns when a run starts there.</li>
3035
+ <li><strong>Replay:</strong> one pass, in a new browser context. The second pass (without Playwright's automation switches) needs a
3036
+ launch, which a running browser does not have.</li>
3037
+ <li>A run explored in a container is replayed in one even when the provider is changed back later (<code>run.json</code>).</li>
3038
+ </ul>
3039
+ </div>
3040
+ </section>
3041
+ <section class="doc" id="doc-docs-containers-testcontainers-scribe">
3042
+ <h4>Testcontainers (Scribe)</h4>
3043
+ <div class="prose"><p>When a test needs a real service, or the repository already uses Testcontainers (Java, .NET, Node, Python, Go), Scribe
3044
+ loads the <code>testcontainers</code> skill. It reuses the repository's containers and fixtures, pins images, waits for
3045
+ readiness, uses the mapped port, and sets the app's own configuration from it. It never uses privileged mode, the
3046
+ host network, the Docker socket, or a <code>docker</code> command from a test. These containers belong to the tests in the
3047
+ repository: the tests start and stop them, wherever they run.</p>
3048
+ </div>
3049
+ </section>
3050
+ <section class="doc" id="doc-docs-containers-mocks">
3051
+ <h4>Mocks</h4>
3052
+ <div class="prose"><p><code>mock_service</code> builds WireMock stubs and runs WireMock on a free port of 127.0.0.1:</p>
3053
+ <ul>
3054
+ <li><strong>From a run</strong> (the default: the latest run, or <code>run_dir</code>):
3055
+ <ul>
3056
+ <li>One stub per method, path and query of the app's own API calls, answering with the last status, content type
3057
+ and body the app got.</li>
3058
+ <li>Other sites' calls are left out unless <code>third_party</code> is set.</li>
3059
+ <li>Failed calls are left out.</li>
3060
+ <li>Answers that were cut when recorded are listed under <code>skipped</code>.</li>
3061
+ </ul>
3062
+ </li>
3063
+ <li><strong>From a description</strong> (<code>spec</code>: OpenAPI 3 or Swagger 2, a link or a file):
3064
+ <ul>
3065
+ <li>One stub per REST operation.</li>
3066
+ <li>Each stub answers with the operation's first success status and a body from the schema's examples, or values
3067
+ made from its types.</li>
3068
+ <li>A path parameter (<code>/orders/{id}</code>) matches any one path segment.</li>
3069
+ <li>Instead of a description, you can give a contract set from <code>api_contracts</code> (<code>set_id</code>).</li>
3070
+ </ul>
3071
+ </li>
3072
+ <li><strong>GraphQL, SOAP and gRPC</strong> are listed under <code>skipped</code>, not mocked.</li>
3073
+ </ul>
3074
+ <p>A spec given as a link is fetched by ForgeQE itself, with no credentials, as <code>api_contracts</code> does. The stubs are written to <code>.forgeqe/mocks/&lt;name&gt;/mappings/</code>. You can read and edit them there. The folder is mounted
3075
+ read-only at <code>/home/wiremock/mappings</code>. WireMock runs with <code>--enable-stub-cors</code>, so a web app on another local port
3076
+ can call it.</p>
3077
+ <p>With <code>for_app: &lt;api app&gt;</code>, the active environment's API app points at the mock for the rest of the session. That
3078
+ covers the API tools, and the tests ForgeQE runs, which read <code>FORGEQE_APP_&lt;APP&gt;_URL</code>. env.yaml itself is not
3079
+ changed.</p>
3080
+ </div>
3081
+ </section>
3082
+ <section class="doc" id="doc-docs-containers-lifetime">
3083
+ <h4>Lifetime</h4>
3084
+ <div class="prose"><ul>
3085
+ <li><strong>Services and mocks the agent starts</strong> are removed when the shell, <code>forgeqe run</code>, or <code>forgeqe mcp</code> ends, as are
3086
+ the workspace and browser containers (<code>forgeqe.kind</code> <code>workspace</code> and <code>browser</code>). The home volume stays.</li>
3087
+ <li><strong><code>forgeqe services up</code></strong> keeps its services running after the command ends, until <code>forgeqe services down</code>.</li>
3088
+ <li><strong>Container names</strong> are <code>forgeqe-&lt;project&gt;-&lt;service&gt;</code>. <code>&lt;project&gt;</code> is a hash of the project folder, so two projects
3089
+ never share a container. Starting a service replaces its earlier container. Removing a container also removes the anonymous volumes its image declares (a database's data folder), so nothing is left on the machine. A container is never reused with old
3090
+ settings.</li>
3091
+ <li><strong>Labels:</strong> each container carries <code>forgeqe=1</code>, <code>forgeqe.project</code>, <code>forgeqe.service</code> and <code>forgeqe.kind</code>.
3092
+ <code>down</code> and <code>status</code> look only at this project's containers.</li>
3093
+ </ul>
3094
+ </div>
3095
+ </section>
3096
+ <section class="doc" id="doc-docs-containers-security">
3097
+ <h4>Security</h4>
3098
+ <div class="prose"><p>ForgeQE's agent reads text other people control: tickets, web pages, repositories. A container command built from that
3099
+ text could mount the whole disk or run privileged, so:</p>
3100
+ <ul>
3101
+ <li><strong>No commands from the model.</strong> The agent names services that forgeqe.yaml declares, and ForgeQE builds the
3102
+ argument array (<code>docker run …</code>). It never uses a shell. Images, flags and mounts never come from a request.</li>
3103
+ <li><strong>The same restrictions for every container:</strong> <code>--security-opt no-new-privileges</code>, <code>--pids-limit 1024</code>, ports on
3104
+ 127.0.0.1, and read-only mounts. There is no setting to turn these off.</li>
3105
+ <li><strong>Mounts are checked like uploads</strong> (except a mock's stubs folder, which ForgeQE writes itself and mounts read-only). The path is resolved with <code>realpath</code> and must be inside the project. A mount
3106
+ that is or holds a hidden file or folder (<code>.env</code>, <code>.git</code>, <code>.npmrc</code>), a file named like a key (<code>*.pem</code>, <code>id_rsa</code>,
3107
+ <code>llms.json</code>) or a link is refused. A container can send what it reads anywhere.</li>
3108
+ <li><strong>Environment:</strong>
3109
+ <ul>
3110
+ <li>The <code>docker</code> and <code>podman</code> programs see only the system settings and the variables they need to find their daemon
3111
+ (<code>DOCKER_HOST</code>, <code>DOCKER_CONTEXT</code>, <code>CONTAINER_HOST</code>, …), plus the service's listed secrets.</li>
3112
+ <li>A container gets only its <code>env</code> and <code>secrets</code>.</li>
3113
+ <li>Logs and errors have secret values replaced with <code>***</code>.</li>
3114
+ </ul>
3115
+ </li>
3116
+ <li><strong>Not in the agents' shell:</strong> <code>docker</code>, <code>podman</code>, <code>nerdctl</code>, <code>docker-compose</code> and <code>podman-compose</code> are denied in the
3117
+ shell of ForgeQE's agent and of Scribe. So the only way to start a container is these tools.</li>
3118
+ <li><strong>Asked first:</strong> <code>services up</code> and <code>mock_service</code> ask the user before starting, as <code>run_tests</code> does.</li>
3119
+ </ul>
3120
+ </div>
3121
+ </section>
1918
3122
  </div>
1919
3123
  </article>
1920
3124
  <article class="spec feat" id="cap-monitoring">
@@ -2062,7 +3266,7 @@ forgeqe run &quot;The ForgeQE workflow is failing, fix it&quot;
2062
3266
  <li>Stores the keys the pipeline needs (the model's key from <code>llms.json</code>, and <code>ADO_PAT</code>, <code>JIRA_EMAIL</code> and <code>JIRA_API_TOKEN</code>, or <code>XRAY_CLIENT_ID</code> and <code>XRAY_CLIENT_SECRET</code>) with ForgeQE's <code>set_ci_secret</code> tool. The agent only names the variable; ForgeQE passes the value from its own environment to <code>gh secret set</code> (on stdin) or <code>az pipelines variable</code> (through the environment), and never shows it. A variable that is not set in ForgeQE's environment is reported, not guessed.</li>
2063
3267
  <li>Triggers the run with <code>gh workflow run</code> or <code>az pipelines run</code>, watches it, reads the logs of failed steps, fixes the pipeline (or the code), pushes again, and repeats until the run passes or <code>code.max_attempts</code> runs have failed. It reports the run link and the result.</li>
2064
3268
  </ol>
2065
- <p>It needs <code>gh</code> logged in for GitHub, or <code>az</code> with the <code>azure-devops</code> extension logged in for Azure DevOps. Without them it still writes, validates, and commits the pipeline, then tells you the commands to run. Set <code>llm.active</code> to a live profile in the committed <code>forgeqe.yaml</code>. With a <code>github_copilot</code> profile the pipeline needs only a GitHub Models token: in GitHub Actions, <code>permissions: models: read</code> and the workflow's <code>GITHUB_TOKEN</code> (see <a href="../#github-copilot-seats-as-the-model">In pipelines</a>).</p>
3269
+ <p>It needs <code>gh</code> logged in for GitHub, or <code>az</code> with the <code>azure-devops</code> extension logged in for Azure DevOps. Without them it still writes, validates, and commits the pipeline, then tells you the commands to run. Set <code>llm.active</code> to a live profile in the committed <code>forgeqe.yaml</code>. With a <code>github_copilot</code> profile the pipeline needs a GitHub token for Copilot in <code>COPILOT_GITHUB_TOKEN</code> (see <a href="../HOW_TO_USE.md#github-copilot-seats-as-the-model">In pipelines</a>).</p>
2066
3270
  </div>
2067
3271
  </section>
2068
3272
  </div>
@@ -2538,7 +3742,7 @@ tokens and the like) show as <code>•••</code> in results. A check compares
2538
3742
  ┃ Handoff .forgeqe/reports/20260928-172700/handoff.md
2539
3743
  </code></pre>
2540
3744
  <p>When no test was written, the Test row says why: the run did not pass, the session has <code>--no-code</code>, or the run was not handed to the coding agent.</p>
2541
- <p>Scribe uses the active <code>llms.json</code> profile, or, with <code>llm.routing: auto</code>, the profile the router picks for the <code>coding</code> area (<code>--profile</code> and <code>code.model</code> still win; see <a href="../docs/MODEL_ROUTING.md">docs/MODEL_ROUTING.md</a>). The key stays in your environment: the agent runtime's config refers to it as <code>{env:WP_AZURE_API_KEY}</code> and never contains the value. An <code>auth: entra</code> profile reaches the model through ForgeQE's local token gateway (see <a href="../#signing-in-to-the-model-without-a-key-microsoft-entra-id">Signing in to the model without a key</a>). An Azure profile goes through the agent runtime's <code>azure</code> provider on <code>&lt;endpoint&gt;/openai/v1</code> (the Responses API, with the deployment as the model). <code>openai_compatible</code> and <code>ollama</code> profiles use their <code>base_url</code>. A <code>github_copilot</code> profile goes through the agent runtime's <code>github-copilot</code> provider with the profile's <code>model</code>, signed in with <code>forgeqe auth login copilot</code>. <code>code.model: provider/model</code> (or <code>--code-model</code>) uses one of the agent runtime's built-in providers instead, with that provider's usual key variable (for example <code>ANTHROPIC_API_KEY</code> for <code>anthropic/...</code>).</p>
3745
+ <p>Scribe uses the active <code>llms.json</code> profile, or, with <code>llm.routing: auto</code>, the profile the router picks for the <code>coding</code> area (<code>--profile</code> and <code>code.model</code> still win; see <a href="#doc-docs-model_routing-model-routing-the-right-configured-model-for-each-task">docs/MODEL_ROUTING.md</a>). The key stays in your environment: the agent runtime's config refers to it as <code>{env:WP_AZURE_API_KEY}</code> and never contains the value. An <code>auth: entra</code> profile reaches the model through ForgeQE's local token gateway (see <a href="../HOW_TO_USE.md#signing-in-to-the-model-without-a-key-microsoft-entra-id">Signing in to the model without a key</a>). An Azure profile goes through the agent runtime's <code>azure</code> provider on <code>&lt;endpoint&gt;/openai/v1</code> (the Responses API, with the deployment as the model). <code>openai_compatible</code> and <code>ollama</code> profiles use their <code>base_url</code>. A <code>github_copilot</code> profile cannot run Scribe yet, and <code>code.model: github-copilot/...</code> is refused: the agent runtime's own Copilot provider signs in through a third-party app on your GitHub account. <code>code.model: provider/model</code> (or <code>--code-model</code>) uses one of the agent runtime's built-in providers instead, with that provider's usual key variable (for example <code>ANTHROPIC_API_KEY</code> for <code>anthropic/...</code>).</p>
2542
3746
  <p>Scribe may edit files and run commands anywhere in the repo without asking. <code>git push</code> in any form is denied (pushing goes through ForgeQE's guarded <code>push_branch</code> tool, section 8), and it cannot touch files outside the repo except the run and memory directories, which it may read.</p>
2543
3747
  <p>More ways to run it:</p>
2544
3748
  <pre><code class="language-bash">forgeqe code --repo ../app # the latest run in .forgeqe/reports/
@@ -2550,6 +3754,28 @@ forgeqe run &quot;...&quot; --no-code # no tests
2550
3754
  <p>If Scribe's job is cut off (ForgeQE closed, the machine restarted, you stopped it, or the model could not be reached), run it again for the same run and repo, for example <code>forgeqe code .forgeqe/reports/20260928-172700 --repo ../app</code>. Scribe reopens the same session, with everything it read and said, and carries on with the files it already wrote; it doesn't read the repo again, and ForgeQE uses the Playwright replay it already finished. The session is kept in <code>scribe-session.json</code> in the run folder until the job ends with a result.</p>
2551
3755
  </div>
2552
3756
  </section>
3757
+ <section class="doc" id="doc-how_to_use-how-scribe-writes-a-test">
3758
+ <h4>How Scribe writes a test</h4>
3759
+ <div class="prose"><p><strong>Reuse before rebuild, extend before duplicate.</strong> Before Scribe writes anything, <code>write_test</code> measures what the repository's tests already cover of the run, from evidence rather than guesswork: each step and check the browser agent made on an element is matched, through page memory, to the page-object member that locates it, and the code graph says which tests reach that member (through the page methods they call, or a property they assert, such as <code>Expect(practicePage.AjaxStatus)</code>). Steps on elements no page object locates yet are listed as not measured, since a test may do them its own way. Then:</p>
3760
+ <div class="tbl"><table>
3761
+ <thead>
3762
+ <tr><th>What it finds</th><th>What happens</th></tr>
3763
+ </thead>
3764
+ <tbody>
3765
+ <tr><td>One test covers every measured step and check</td><td><strong>Reuse</strong>: Scribe only finds the command that runs it, and changes nothing. If it fails, nothing is changed: you are asked whether to update it or write a new test.</td></tr>
3766
+ <tr><td>One test covers most (40% or more)</td><td><strong>Extend</strong>: Scribe adds only the missing steps and checks to that test, keeping every step and assertion it has; the reviewer rejects a removed or weakened assertion, or the steps copied into a new file</td></tr>
3767
+ <tr><td>Two or more tests cover most of it</td><td><strong>You choose</strong>: the evidence and the options (reuse, extend with how many steps it adds, a new test, cancel)</td></tr>
3768
+ <tr><td>Nothing covers much</td><td><strong>New</strong>: a new test, reusing the page objects and helpers as before</td></tr>
3769
+ </tbody>
3770
+ </table></div>
3771
+ <p>A card says which it did and why (<em>Extending PracticeTests.Flow · covers 12 of 15; only 3 are missing</em>). Say how when you know (&quot;extend the login test&quot;, &quot;write a separate test&quot;, &quot;update it&quot;) and ForgeQE's agent passes that on without asking. In every mode Scribe adds page-object members rather than changing what an existing member does for the other tests that use it.</p>
3772
+ <p><strong>The same flow asked again.</strong> When a session writes tests and you ask for a flow an earlier job already wrote a test for, ForgeQE runs that test first, before any browser run. If it passes in a real browser, that is the answer: no browser run, no Scribe, and no new test file, in the seconds the test takes (<em>✓ Existing test passed · 19s · no browser run or new test needed</em>). If it fails, the flow runs in the browser and Scribe updates that test instead of writing another. ForgeQE's agent rewords a goal a little each time, so goals count as the same flow when they share their content words (endings, case, and phrasing aside) and the new goal keeps everything the earlier one checked; the same steps with a different final check, a shorter flow, or another site are a new flow. The agent also compares the earlier goal with your request and runs fresh when you ask for a check the earlier test does not have. Ask for &quot;a fresh run&quot; or &quot;a new test&quot; to skip the reuse.</p>
3773
+ <p><strong>When the flow cannot be reproduced.</strong> If ForgeQE's own replay of the run could not reach the run's final page and Scribe's first test fails too, the job stops there and says which replayed step failed and why, instead of spending its remaining attempts on a flow no script can repeat as recorded.</p>
3774
+ <p><strong>Why a test failed.</strong> When ForgeQE's run of Scribe's test fails, the ✗ line and the next attempt say why in the test's own words: dotnet test's Error Message, pytest's <code>E</code> lines, or the error with its Expected and Received values (Playwright, Jest, JUnit), not the runner's summary line.</p>
3775
+ <p><strong>Keeping Scribe's cost down.</strong> ForgeQE reads the repository's test setup itself before Scribe starts (stack, settings files and their browser values, the test id attribute, base classes, page objects and their methods, one test command) and hands it over, so Scribe does not discover it with dozens of file reads. Each attempt after the first starts a fresh session from a summary of the job (handoff, files changed, what each attempt did, the latest failure), so later attempts cost about what the first did. <code>code.max_attempts</code> (default 3) and <code>code.max_tokens</code> (default 1,500,000; <code>0</code> turns it off) cap a job, and the token budget is checked during an attempt too, so one long attempt is stopped when the total passes it. The reviewer sorts its issues into correctness (the test does not test what the goal says, or not reliably) and coding standards (test data or credentials in a page object, naming, hard waits, leftovers). Coding standards never fail a test that works: it is accepted, and the coding agent gets one more round (not counted in <code>max_attempts</code>) to fix them; if that round breaks the test, ForgeQE puts back the version that passed and lists the standards issues in the notes (<code>code.tidy: false</code> skips the round). A test the reviewer rejects twice for correctness stops the job with its issues, since that usually means the test case and the page disagree. Under each step, the handoff shows that step's part of the page (its section and neighbours, from the replay's page snapshot) instead of whole snapshots. When a test fails, ForgeQE shows the failing element on the test's page at the failure and on the replay's page, so most failures are fixed without opening a browser. Scribe's live browser is switched on only when the same failure repeats, and it does not send a page snapshot after every click or keystroke. <code>code.max_steps</code> (default 50) caps the tool calls in one of Scribe's turns. <code>code.review_profile</code> runs the reviewer on a cheaper llms.json profile.</p>
3776
+ </div>
3777
+ </section>
3778
+ <p class="see">See <a href="#doc-how_to_use-generated-code">Generated code</a>.</p>
2553
3779
  </div>
2554
3780
  </article>
2555
3781
  <article class="spec feat" id="cap-heal-tests">
@@ -2653,11 +3879,26 @@ Run the tests added since 2026-09-01
2653
3879
  <p>Through <code>forgeqe mcp</code>, ForgeQE cannot ask you, so <code>run_tests</code> returns the list and runs only when called again with <code>confirmed: true</code>; the MCP client should ask you first.</p>
2654
3880
  </div>
2655
3881
  </section>
3882
+ <section class="doc" id="doc-how_to_use-reports-for-tests-run-outside-forgeqe">
3883
+ <h4>Reports for tests run outside ForgeQE</h4>
3884
+ <div class="prose"><p>ForgeQE's HTML report is not only for runs it makes. <code>run_tests</code> and <code>pr-tests --run</code> write one, and so can any run in a terminal, an IDE's test runner, or CI:</p>
3885
+ <pre><code class="language-bash">forgeqe report test-results/junit.xml # from JUnit XML or TRX: any framework, after the run
3886
+ forgeqe report setup # a report after every Playwright or pytest run, automatically
3887
+ </code></pre>
3888
+ <p><code>forgeqe report</code> reads JUnit XML or TRX files, or the folder they are in (Playwright <code>--reporter=junit</code>, pytest <code>--junitxml=results.xml</code>, <code>dotnet test --logger trx</code>), and exits 1 when a test failed. Each report is saved in <code>&lt;out_dir&gt;/results/&lt;time&gt;/</code>, so its Trends and Flaky tests pages cover the earlier runs, whoever ran them. JUnit carries one line per failure; a skipped test is counted, not shown.</p>
3889
+ <p><code>forgeqe report setup</code> sets up the framework the repository uses:</p>
3890
+ <ul>
3891
+ <li><strong>Playwright</strong>: it copies a reporter to <code>.forgeqe/playwright-reporter.mjs</code> (commit it) and prints the line to add to <code>playwright.config</code>: <code>reporter: [[&quot;list&quot;], [&quot;./.forgeqe/playwright-reporter.mjs&quot;]]</code>. Every run then ends with a report, from <code>npx playwright test</code>, the VS Code Playwright extension, or CI. With <code>screenshot: &quot;only-on-failure&quot;</code> and <code>trace: &quot;retain-on-failure&quot;</code>, each failure shows its screenshot, the full error and stack, and the trace.</li>
3892
+ <li><strong>pytest</strong>: install <code>forgeqe-cli</code> in the environment that runs the tests (<code>pip install forgeqe-cli</code>). Its plugin loads by itself, from the terminal, VS Code's test explorer, or CI, in a folder with <code>forgeqe.yaml</code> or <code>.forgeqe/</code> (elsewhere with <code>pytest --forgeqe-report</code>). A test that passes when pytest-rerunfailures runs it again is reported as flaky.</li>
3893
+ </ul>
3894
+ <p><code>FORGEQE_REPORT=0</code> turns the reporters off for a run (ForgeQE's own <code>run_tests</code> does this, as it writes its own report). The reporters call <code>forgeqe</code> on PATH, or <code>FORGEQE_BIN</code>; without it they keep the results file and print the command that builds the report. Screenshots and traces are copied into the report only from the working folder, never from a hidden folder or a file named like a key.</p>
3895
+ </div>
3896
+ </section>
2656
3897
  </div>
2657
3898
  </article>
2658
3899
  <article class="spec feat" id="cap-regression-packs">
2659
3900
  <div class="hd"><span class="mono">Automated tests</span><h3>Regression packs for a release</h3></div>
2660
- <p class="ds">Builds a regression pack from what goes into a release (an Azure DevOps sprint or work items, a commit range, pull requests, or changes read from any connected tool): the tests those changes can break, each with why, plus risk-area and smoke tests, and the changes no test covers. Runs it locally, on TestMu AI, or in a pipeline, and compares each run with the last.</p>
3901
+ <p class="ds">Builds a regression pack from what goes into a release (an Azure DevOps sprint or work items, a commit range, pull requests, or changes read from any connected tool): the tests those changes can break, each with why, plus risk-area and smoke tests, and the changes no test covers. Runs it locally, on TestMu AI, or in a pipeline, compares each run with the last, and triages each failure: flaky, environment, broken test code, heal, or an AI browser re-run to tell a test the app outgrew from an app bug.</p>
2661
3902
  <div class="facts">
2662
3903
  <div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>Build a regression pack for sprint 12 and run it on TestMu</li><li>Which tests should we run for the PRs merged this week?</li></ul></div>
2663
3904
  <div class="fact"><span class="mono">What it needs</span><ul class="needs"><li><span class="need input">You give</span><span>a repository with tests</span></li><li><span class="need opt">Optional</span><span>Azure DevOps or Jira, to read a sprint's or release's work items: set up under sources: in forgeqe.yaml, or sign in with /auth</span></li></ul></div>
@@ -2697,7 +3938,22 @@ Build a pack for Jira fix version 2.4 and the commits since release/2.3 in ../sh
2697
3938
  <li><em>This machine</em>: ForgeQE builds one command for exactly the pack's tests, the way the repository already runs a test (for .NET, a <code>--filter</code> with each test, the settings an earlier job used, TRX results), runs it, and reads each result. A failed test runs once more; one that then passes is flaky.</li>
2698
3939
  <li><em>TestMu AI</em>: the same command, with the variable <code>FORGEQE_PW_WS_ENDPOINT</code> set to TestMu AI's Playwright endpoint for the browser and platform in <code>testmu.yaml</code> (the access key goes in the variable, never in the command, and is removed from the output). The tests must connect to it when it is set: for NUnit, override <code>ConnectOptionsAsync</code> in the test base class; for Playwright Test, set <code>connectOptions</code> from it in <code>playwright.config</code>. ForgeQE refuses a TestMu run when the tests do not read the variable, so a run never looks remote while it ran locally.</li>
2699
3940
  </ul>
2700
- <p>Each run is kept with the pack: the card says which tests fail now that passed in the last run, and which were fixed. Packs are saved in <code>.forgeqe/packs/&lt;name&gt;.json</code> and <code>&lt;name&gt;.md</code>; run output and results in <code>.forgeqe/reports/packs/&lt;name&gt;/&lt;time&gt;/</code>.</p>
3941
+ <p>Each run is kept with the pack: the card says which tests fail now that passed in the last run, and which were fixed.</p>
3942
+ <p><strong>Triage: heal, re-run, or file a bug?</strong> After each run ForgeQE sorts every failed or flaky test into one route, from evidence that costs nothing first (the retry in the same run, the failure message, the test's history), so the AI is spent only where it decides something:</p>
3943
+ <div class="tbl"><table>
3944
+ <thead>
3945
+ <tr><th>Route</th><th>When</th><th>What happens</th></tr>
3946
+ </thead>
3947
+ <tbody>
3948
+ <tr><td>flaky</td><td>it passed when run again in the same run, or its history flip-flops</td><td>not healed: quarantine it, or fix the cause it flakes on</td></tr>
3949
+ <tr><td>environment</td><td>network errors, 5xx, a crashed browser, sign-in failing</td><td>run again once the environment works; no heal and no bug</td></tr>
3950
+ <tr><td>test code broken</td><td>a missing module, a compile error, a missing fixture</td><td>fix the test's code; healing against the page cannot fix it</td></tr>
3951
+ <tr><td>heal</td><td>an element the test uses is no longer found (high confidence when it passed its last runs)</td><td>the healing agent re-runs the flow in the browser and updates the steps and locators</td></tr>
3952
+ <tr><td>AI re-run decides</td><td>an expected value does not hold, a timeout, or something covers the element</td><td>the healing agent re-runs the flow in the browser: heals the test when the flow works, reports an app bug when it fails at the same step</td></tr>
3953
+ </tbody>
3954
+ </table></div>
3955
+ <p>Failures that name the same element or the same message are shown as <strong>one cause</strong> (a page object or a shared step changed), so one fix can heal them together. When most failures are the environment, ForgeQE says to run the pack again before healing anything, and <code>heal_tests</code> refuses until you name the tests.</p>
3956
+ <p>The triage is on the run's card and in <code>triage.md</code> and <code>triage.json</code> in the run's folder. To heal only the tests it sent to healing, ask &quot;heal the failures of pack release-1.5&quot;, or run <code>forgeqe pack heal release-1.5</code> (<code>--tests</code> for some of them): ForgeQE builds a command for just those tests and the healing agent re-runs each flow in the browser before it changes anything. <code>forgeqe pack triage &lt;name&gt;</code> shows the last run's triage again. Packs are saved in <code>.forgeqe/packs/&lt;name&gt;.json</code> and <code>&lt;name&gt;.md</code>; run output and results in <code>.forgeqe/reports/packs/&lt;name&gt;/&lt;time&gt;/</code>.</p>
2701
3957
  </div>
2702
3958
  </section>
2703
3959
  </div>
@@ -2867,7 +4123,7 @@ forgeqe api tests --repo ../app # Scribe writes them as tests
2867
4123
  </tbody>
2868
4124
  </table></div>
2869
4125
  <p>The planner only uses operations the set has: a step no operation does is listed as unmatched, never invented. Give it the API's operations with <code>--spec</code> (an OpenAPI or Swagger link or file) or a saved set; without a description, make contracts from a browser run first. Every plan is checked before it is saved (each operation exists, each variable is saved before it is used) and shown step by step; nothing runs until you run it.</p>
2870
- <p><code>run_api_scenarios</code> (or <code>forgeqe api run-scenarios</code>) runs them against the API, without a browser or a model: each step in order, the values passed on, every step's checks. A failed step stops its scenario, and the scenario's <strong>cleanup</strong> steps (deleting what it created) run, those whose values exist. Scenarios that change data run only after you agree (in CI: <code>--allow-writes</code>), on a test environment. Results go to <code>scenario-results.json</code> and <code>scenarios-junit.xml</code>, with <code>report.html</code> (each scenario a case, its calls the steps) and <code>suite.json</code> beside them. API runs count in the <a href="../#team-report">team report</a>.</p>
4126
+ <p><code>run_api_scenarios</code> (or <code>forgeqe api run-scenarios</code>) runs them against the API, without a browser or a model: each step in order, the values passed on, every step's checks. A failed step stops its scenario, and the scenario's <strong>cleanup</strong> steps (deleting what it created) run, those whose values exist. Scenarios that change data run only after you agree (in CI: <code>--allow-writes</code>), on a test environment. Results go to <code>scenario-results.json</code> and <code>scenarios-junit.xml</code>, with <code>report.html</code> (each scenario a case, its calls the steps) and <code>suite.json</code> beside them. API runs count in the <a href="#doc-how_to_use-team-report">team report</a>.</p>
2871
4127
  <p><strong>On your existing API automation.</strong> <code>write_api_tests</code> with <code>scenarios</code> (or <code>forgeqe api tests --scenarios</code>) has Scribe write each scenario as an end-to-end test in the repository, built on what is already there:</p>
2872
4128
  <ol>
2873
4129
  <li>It maps the repository's API layer first: clients, request helpers, auth and base URL fixtures, data builders, schemas.</li>
@@ -3054,7 +4310,7 @@ forgeqe visual approve --page /home # make this run's /home the baseline (asks
3054
4310
  <ul>
3055
4311
  <li>ForgeQE keeps the result of every tool call in the session (its own tools, and the Azure DevOps and Jira MCP servers). A dataset of the report names the calls it comes from, and ForgeQE takes the rows from their results.</li>
3056
4312
  <li>Or the dataset is a read-only query that ForgeQE runs itself when it writes the report: a WIQL query reads up to 1000 work items with their project, type, state, priority, severity, assignee, area, iteration, dates, and a link. The agent uses this for &quot;all&quot; of something.</li>
3057
- <li>Or the dataset is ForgeQE's own past results (<code>history:</code>): <code>suites</code> (one row per run, with its run type in <code>suite_scope</code>), <code>case_results</code>, <code>step_results</code>, <code>writebacks</code>, <code>readiness</code>, <code>readiness_stories</code>, or <code>readiness_cases</code>, from every run on this machine and the bundles teammates shared with <code>forgeqe bi export</code> (see <a href="../#team-report">Team report</a>). Every row has <code>user</code> and <code>machine</code>.</li>
4313
+ <li>Or the dataset is ForgeQE's own past results (<code>history:</code>): <code>suites</code> (one row per run, with its run type in <code>suite_scope</code>), <code>case_results</code>, <code>step_results</code>, <code>writebacks</code>, <code>readiness</code>, <code>readiness_stories</code>, or <code>readiness_cases</code>, from every run on this machine and the bundles teammates shared with <code>forgeqe bi export</code> (see <a href="#doc-how_to_use-team-report">Team report</a>). Every row has <code>user</code> and <code>machine</code>.</li>
3058
4314
  <li>The agent writes only the summary paragraph.</li>
3059
4315
  </ul>
3060
4316
  <p>The report goes to the reports folder (<code>export.out_dir</code>, <code>.forgeqe/reports/</code> by default):</p>
@@ -3068,6 +4324,84 @@ forgeqe visual approve --page /home # make this run's /home the baseline (asks
3068
4324
  <p>The chat gets a short summary and a card with the paths as links, and <strong>Open the report</strong> appears as a next step. To post it, ask for it (&quot;post the report to the release channel&quot;) once <code>notify.channels</code> is set up; the reports folder is linked when it is served (<code>notify.report_url</code>).</p>
3069
4325
  </div>
3070
4326
  </section>
4327
+ <section class="doc" id="doc-how_to_use-team-report">
4328
+ <h4>Team report</h4>
4329
+ <div class="prose"><p><code>forgeqe bi</code> builds one report from every run in the reports folder: browser suites, monitor rounds, release readiness, single browser runs, desktop and mobile runs, exploratory testing, and API runs, each tagged with its run type. It builds an <a href="https://github.com/evidence-dev/evidence" rel="noopener">Evidence</a> site (needs Node.js); without Node.js, or when Evidence fails, it writes ForgeQE's own page, which opens straight from disk (<code>bi:</code> in <code>forgeqe.yaml</code>; see <a href="#doc-readme-static-bi">README.md</a>).</p>
4330
+ <pre><code class="language-bash">forgeqe bi init # scaffold the Evidence project and install its packages
4331
+ forgeqe bi build # refresh the data and build (ForgeQE's own page if Evidence cannot build)
4332
+ forgeqe bi build --static # ForgeQE's own page only, no Node.js needed
4333
+ forgeqe bi sync # refresh the datasets and pages, without building
4334
+ forgeqe bi export [--out FILE|DIR] # your runs, in one file to share
4335
+ forgeqe bi build --from //share/qa-reports # this machine's runs plus every bundle there (repeatable)
4336
+ </code></pre>
4337
+ <p>For one report across the team, each person runs <code>forgeqe bi export</code>, which writes their runs and readiness reports to one file in <code>.forgeqe/reports/shared/</code>. Whoever builds puts the files in <code>&lt;out_dir&gt;/collected/</code> (<code>.forgeqe/reports/collected/</code> by default), or lists shared folders under <code>bi.collect</code>, or passes <code>--from</code>, and runs <code>forgeqe bi build</code>. Every table gets <code>user</code> and <code>machine</code> columns; the report adds <strong>Results by run type</strong> and <strong>Case results by person</strong> tables. A run that arrives twice (in two bundles, or a bundle and this machine) counts once. A bundle is read as data only: a file that is not one, or is larger than 64 MB, is skipped with a note.</p>
4338
+ <p>In the shell, <code>/report</code> builds ForgeQE's own page from the same data, <code>/report export</code> writes your bundle, and <code>/report &lt;what&gt;</code> asks the agent for a report (see <a href="#doc-how_to_use-reports-from-the-agent">Reports from the agent</a>).</p>
4339
+ </div>
4340
+ </section>
4341
+ <section class="doc" id="doc-readme-static-bi">
4342
+ <h4>Static BI</h4>
4343
+ <div class="prose"><p><code>forgeqe bi</code> adds an offline BI layer on top of those reports. It exports curated CSV datasets from ForgeQE's own suite history and release-readiness reports, plus optional Azure DevOps WIQL and Jira JQL datasets from <code>bi.datasets</code> in <code>forgeqe.yaml</code>, into an <a href="https://github.com/evidence-dev/evidence" rel="noopener">Evidence</a> project that reads them through its CSV source, then runs <code>evidence sources</code> and <code>evidence build</code> to produce a static site in the configured build directory. ForgeQE uses the open-source Evidence line (<code>@evidence-dev/evidence</code> 40.x, pinned in the generated <code>package.json</code>), because the newer Evidence Studio CLI has no local build and keeps data in its cloud. Building it needs Node.js 18 or later; serve the folder to view it (<code>npx serve</code>), since it cannot load its data when opened from disk. When Node.js is missing or the Evidence build fails, <code>forgeqe bi build</code> writes ForgeQE's own self-contained page from the same data instead, which opens straight from disk, and says why; <code>--static</code> always does that, and <code>--evidence</code> fails instead of falling back. Dataset ids become table names (<code>forgeqe.&lt;id&gt;</code>), so they use letters, digits, and <code>_</code>. Pages ForgeQE generates carry a &quot;Managed by forgeqe bi&quot; comment in their front matter; a page without it is yours and is never overwritten. Tables with no rows yet stay out of the Evidence project (Evidence cannot build over an empty table), and their sections read &quot;None yet.&quot; No backend service stays up between runs.</p>
4344
+ <pre><code class="language-yaml">bi:
4345
+ title: ForgeQE BI
4346
+ datasets:
4347
+ ado_current_sprint:
4348
+ kind: ado_wiql
4349
+ project: Shop
4350
+ wiql: |
4351
+ SELECT [System.Id]
4352
+ FROM WorkItems
4353
+ WHERE [System.TeamProject] = @project
4354
+ AND [System.WorkItemType] IN GROUP 'Microsoft.RequirementCategory'
4355
+ jira_open_bugs:
4356
+ kind: jira_jql
4357
+ jql: project = SHOP AND issuetype = Bug AND statusCategory != Done
4358
+ </code></pre>
4359
+ <pre><code class="language-bash">forgeqe bi init # scaffold the Evidence project and install its packages
4360
+ forgeqe bi build # refresh the data and build the site (ForgeQE's own page if Evidence cannot build)
4361
+ forgeqe bi build --static # ForgeQE's own page only, no Node.js needed
4362
+ </code></pre>
4363
+ </div>
4364
+ </section>
4365
+ <section class="doc" id="doc-readme-one-report-for-the-whole-team">
4366
+ <h4>One report for the whole team</h4>
4367
+ <div class="prose"><p>Each person runs <code>forgeqe bi export</code>, which writes every kind of run they have (suites, monitor rounds, single browser runs, desktop, mobile, exploratory and API runs, and release-readiness reports) to one JSON file in <code>.forgeqe/reports/shared/</code>, each run tagged with its run type. Whoever builds the report puts the files they receive in <code>.forgeqe/reports/collected/</code> (or lists shared folders under <code>bi.collect</code> in <code>forgeqe.yaml</code>, or passes <code>--from FILE|DIR</code>, repeatable) and runs <code>forgeqe bi build</code>. Every table gets <code>user</code> and <code>machine</code> columns, the home page and ForgeQE's own page gain &quot;Results by run type&quot; and &quot;Case results by person&quot; tables, and a run that arrives twice (from two bundles, or a bundle and this machine) counts once. A bundle is read as data only: anything that is not one, or is larger than 64 MB, is skipped with a note, and a malformed run inside one is skipped whole.</p>
4368
+ <p>In the shell, <code>/report</code> builds ForgeQE's own page from the same data, <code>/report export</code> writes your bundle, and <code>/report &lt;what you want&gt;</code> asks the agent, which can now build <code>write_report</code> datasets from these past results (<code>history: case_results</code>, <code>suites</code>, <code>readiness</code>, …) as well as from live Azure DevOps or Jira queries.</p>
4369
+ <pre><code class="language-bash">forgeqe bi export # your runs, in one file to share
4370
+ forgeqe bi build --from //share/qa-reports # this machine's runs plus every bundle in that folder
4371
+ </code></pre>
4372
+ </div>
4373
+ </section>
4374
+ <section class="doc" id="doc-how_to_use-model-prices-and-usage">
4375
+ <h4>Model prices and usage</h4>
4376
+ <div class="prose"><p><strong>Prices.</strong> ForgeQE prices every model call at the model's <strong>list price</strong>, from two public price lists: <a href="https://models.dev" rel="noopener">models.dev</a> (the list the agent runtime uses) and LiteLLM's model price list, which also has Azure's Data Zone prices. It keeps the few fields it needs in <code>~/.forgeqe/prices.json</code>, waits a few seconds for it the first time, refreshes it once a day in the background, and uses the last copy offline; only the public lists are downloaded, nothing about your runs is sent (<code>FORGEQE_PRICES=off</code> stops the downloads). Costs at list price say so: <code>$1.73 (list price of gpt-5.4 (Azure Global) from models.dev, 2026-10-06)</code>. For an Azure Data Zone deployment, which costs about 10% more, set <code>&quot;deployment_type&quot;: &quot;data_zone&quot;</code> on the profile.</p>
4377
+ <p><strong>Model parameters.</strong> Models differ in what a request may carry: reasoning models (o1, o3, o4, gpt-5) take <code>max_completion_tokens</code> instead of <code>max_tokens</code> and only the default temperature, and some OpenAI-compatible servers refuse <code>max_completion_tokens</code>, <code>reasoning_effort</code> or <code>response_format</code>. ForgeQE works this out by itself. A profile is a reasoning model when its <code>model</code> or <code>deployment</code> name says so (an Azure deployment called <code>chat-prod</code> with <code>&quot;model&quot;: &quot;gpt-5.4&quot;</code> counts), or when it sets <code>&quot;reasoning&quot;: true</code>; <code>&quot;reasoning&quot;: false</code> says it is not. When the API still refuses a parameter, the browser, desktop and mobile agents send the call again without it and leave it out of later calls to that model. Nothing needs setting for a new model.</p>
4378
+ <p>A profile's own <code>&quot;price&quot;: { &quot;input&quot;: 2.5, &quot;output&quot;: 15, &quot;cached_input&quot;: 0.25 }</code> (per million tokens) wins, for a negotiated or enterprise rate. Leave <code>currency</code> out and reports use this machine's currency, or set <code>&quot;currency&quot;: &quot;USD&quot;</code>. Without <code>cached_input</code>, cached input gets the model's cache discount from its list price, and the cost is marked as an estimate (<code>≈$1.52</code>); with no list price for the model either, cached input is charged at the full rate and the cost is an upper bound (<code>up to $6.11</code>). When your price differs from the list price, ForgeQE says so (<code>forgeqe models</code>, the shell, and the start of each command), since that is usually a typo or an old price. Anthropic's cache writes are priced at their own rate. A provisioned (PTU) deployment is billed per hour, not per token, so its costs are what the tokens would cost at list price. Requests over a model's long-context threshold (272K tokens for GPT-5.4) cost more and are priced at the normal rate.</p>
4379
+ <p><strong>Usage after every job.</strong> Every browser run, Scribe job, heal, suite, and monitor round ends with a <code>Usage</code> line: tokens in (and how many of them were cached), tokens out, and the cost at the profile's price. A request to ForgeQE's agent in the shell, and every <code>forgeqe run</code>, <code>code</code>, <code>suite</code>, and <code>heal</code> command, ends with the total for everything it started, by role:</p>
4380
+ <pre><code>This request used 1.1M tokens (1.1M in, 30.0K of it cached · 34.7K out) · $3.07 — ForgeQE's agent 42.0K · $0.12, browser agent 52.7K · $0.15, Scribe 1.0M · $2.80
4381
+ </code></pre>
4382
+ </div>
4383
+ </section>
4384
+ <section class="doc" id="doc-how_to_use-spending-budgets">
4385
+ <h4>Spending budgets</h4>
4386
+ <div class="prose"><p>A budget warns you before model spend gets out of hand. Set it under <code>budget:</code> in <code>forgeqe.yaml</code>, in the currency of your prices (see <a href="#doc-how_to_use-model-prices-and-usage">Model prices and usage</a>):</p>
4387
+ <pre><code class="language-yaml">budget:
4388
+ run: 2 # one browser run
4389
+ day: 25
4390
+ month: 400
4391
+ # currency: USD # default: the currency of the model's price
4392
+ </code></pre>
4393
+ <p>Set any of the three. A budget <strong>only warns; it never stops a run</strong>. Each budget warns once at 80%, once at 90%, and once at 100%. When spend jumps past more than one of these at once, you get one warning, for the highest:</p>
4394
+ <pre><code>Budget: this month's model spend is at $245.10, 82% of the $300.00 month budget (budget.month in forgeqe.yaml). Nothing is stopped.
4395
+ </code></pre>
4396
+ <ul>
4397
+ <li><strong>Where you see it.</strong> In the shell, the warning is a line in the conversation. Every other <code>forgeqe</code> command writes it to the terminal's error output.</li>
4398
+ <li><strong>What counts.</strong> <code>run</code> is the spend of one browser run. <code>day</code> and <code>month</code> add up every ForgeQE process on this machine (days and months are in UTC). Runs that the agent starts through the MCP server count too, because every process writes its model calls to the usage ledger in <code>~/.forgeqe/usage/</code> (<code>FORGEQE_USAGE_DIR</code> moves it). The ledger holds counts, times, and names, never prompt or response text. With <code>FORGEQE_USAGE=off</code>, nothing is kept on disk, and a budget counts only the calls of the process that warns.</li>
4399
+ <li><strong>What does not count.</strong> A call with no price, such as a <code>github_copilot</code> profile without a <code>price</code>, and a call priced in a different currency than the budget.</li>
4400
+ <li><strong>Spend from before.</strong> If you start ForgeQE when the day or month is already past 80%, it does not repeat that warning. It warns only for a threshold that spend passes from then on.</li>
4401
+ <li><strong>In reports.</strong> The <strong>Trends and forecast</strong> page of a report's Tokenomics shows spend on this machine and the next days at the recent pace, against the budget.</li>
4402
+ </ul>
4403
+ </div>
4404
+ </section>
3071
4405
  </div>
3072
4406
  </article>
3073
4407
  </section>
@@ -3283,8 +4617,8 @@ the list again. For a client that ignores it, <code>forgeqe mcp --all-tools</cod
3283
4617
  <tr><td><code>WP_LLMS_JSON</code></td><td>Path to <code>llms.json</code></td></tr>
3284
4618
  <tr><td><code>WP_LLM_PROFILE</code></td><td>Active profile, unless <code>--profile</code> is set</td></tr>
3285
4619
  <tr><td><code>WP_AZURE_API_KEY</code> / <code>OPENAI_API_KEY</code> / <code>WP_API_KEY</code></td><td>Keys referenced by profiles</td></tr>
3286
- <tr><td><code>GITHUB_TOKEN</code></td><td>A <code>github_copilot</code> profile's GitHub Models token for the browser engine (needs <code>models:read</code>), unless the profile's <code>api_key_env</code> names another; else the GitHub CLI's sign-in</td></tr>
3287
- <tr><td><code>FORGEQE_COPILOT_AUTH</code></td><td>The Copilot sign-in <code>forgeqe auth login copilot</code> saves; set by ForgeQE from the credential store, not by hand</td></tr>
4620
+ <tr><td><code>COPILOT_GITHUB_TOKEN</code></td><td>A <code>github_copilot</code> profile's GitHub token (fine-grained, with the Copilot Requests permission), unless the profile's <code>api_key_env</code> names another; else the saved token, then the GitHub CLI's sign-in</td></tr>
4621
+ <tr><td><code>FORGEQE_COPILOT_TOKEN</code></td><td>The token <code>forgeqe auth login copilot</code> saves; set by ForgeQE from the credential store, not by hand</td></tr>
3288
4622
  <tr><td><code>FORGEQE_CONFIG_DIR</code></td><td>Global config folder (default <code>~/.config/forgeqe</code>): <code>forgeqe.yaml</code>, <code>llms.json</code> and <code>.env</code> used when a project has none</td></tr>
3289
4623
  <tr><td><code>FORGEQE_PYTHON</code></td><td>A Python that already has the engine's libraries, used instead of the managed venv (skipped, with a note, when it cannot run the engine)</td></tr>
3290
4624
  <tr><td><code>FORGEQE_DEBUG</code></td><td><code>1</code> logs at debug level, like <code>--debug</code></td></tr>
@@ -3338,12 +4672,16 @@ the list again. For a client that ignores it, <code>forgeqe mcp --all-tools</cod
3338
4672
  var panel = document.getElementById("panel");
3339
4673
  var links = Array.prototype.slice.call(document.querySelectorAll(".chapters a"));
3340
4674
  var railSearch = document.getElementById("rail-search");
3341
- var railGroups = Array.prototype.slice.call(document.querySelectorAll(".rail-group"));
4675
+ var railGroups = Array.prototype.slice.call(document.querySelectorAll(".rail-group:not(#rail-hits)"));
4676
+ var railHits = document.getElementById("rail-hits");
4677
+ var railHitsList = document.getElementById("rail-hits-list");
4678
+ var railNone = document.getElementById("rail-none");
3342
4679
  var tocCurrent = document.getElementById("toc-current");
3343
4680
  var tocLinks = document.getElementById("toc-links");
3344
4681
  var byId = {};
3345
4682
  links.forEach(function (a) { byId[a.getAttribute("href").slice(1)] = a; });
3346
4683
  var filterRail = function (term) {
4684
+ var chaptersShown = 0;
3347
4685
  railGroups.forEach(function (group) {
3348
4686
  var items = Array.prototype.slice.call(group.querySelectorAll(".chapters li"));
3349
4687
  var shown = 0;
@@ -3355,7 +4693,26 @@ the list again. For a client that ignores it, <code>forgeqe mcp --all-tools</cod
3355
4693
  if (on) shown += 1;
3356
4694
  });
3357
4695
  group.style.display = shown ? "" : "none";
4696
+ chaptersShown += shown;
3358
4697
  });
4698
+ // Every section and capability card whose heading or chapter matches, so a topic inside a chapter is found too.
4699
+ railHitsList.innerHTML = "";
4700
+ var hits = term ? sections.filter(function (s) { return s.text.indexOf(term) !== -1; }).slice(0, 30) : [];
4701
+ hits.forEach(function (s) {
4702
+ var li = document.createElement("li");
4703
+ var a = document.createElement("a");
4704
+ a.setAttribute("href", "#" + s.id);
4705
+ [["n", s.n], ["t", s.title], ["d", s.chapter]].forEach(function (part) {
4706
+ var span = document.createElement("span");
4707
+ span.className = part[0];
4708
+ span.textContent = part[1];
4709
+ a.appendChild(span);
4710
+ });
4711
+ li.appendChild(a);
4712
+ railHitsList.appendChild(li);
4713
+ });
4714
+ railHits.hidden = !term || (!hits.length && chaptersShown > 0);
4715
+ railNone.hidden = hits.length > 0 || chaptersShown > 0;
3359
4716
  };
3360
4717
  var chapters = Array.prototype.slice.call(document.querySelectorAll(".chapter"));
3361
4718
  var itemText = function (node) {
@@ -3418,6 +4775,15 @@ the list again. For a client that ignores it, <code>forgeqe mcp --all-tools</cod
3418
4775
  ticking = true;
3419
4776
  requestAnimationFrame(function () { ticking = false; mark(current()); });
3420
4777
  }, { passive: true });
4778
+ var sections = [];
4779
+ chapters.forEach(function (chapter, i) {
4780
+ var head = chapter.querySelector(".chead h2");
4781
+ var name = head ? head.textContent.trim() : "";
4782
+ tocItems(chapter).forEach(function (node) {
4783
+ var title = itemText(node);
4784
+ sections.push({ id: node.id, title: title, chapter: name, n: String(i + 1).padStart(2, "0"), text: (title + " " + name).toLowerCase() });
4785
+ });
4786
+ });
3421
4787
  if (railSearch) railSearch.addEventListener("input", function () { filterRail(railSearch.value.trim().toLowerCase()); });
3422
4788
  filterRail("");
3423
4789
  mark(location.hash && document.getElementById(location.hash.slice(1)) ? locateChapter(location.hash.slice(1)) : current());