@capagents/forgeqe 2.2.2 → 2.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HOW_TO_USE.md +102 -37
- package/README.md +2 -2
- package/THIRD_PARTY_NOTICES.md +2 -1
- package/dist/forgeqe.js +1005 -898
- package/docs/ARCHITECTURE.md +6 -2
- package/docs/CONTAINERS.md +80 -4
- package/docs/user-guide.html +1383 -140
- package/forgeqe.example.yaml +3 -1
- package/llms.example.json +2 -4
- package/package.json +4 -2
- package/py/forgeqe_engine/browser.py +106 -1
- package/py/forgeqe_engine/cli.py +13 -0
- package/py/forgeqe_engine/copilot_gateway.py +369 -0
- package/py/forgeqe_engine/service.py +29 -0
package/docs/user-guide.html
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
<!DOCTYPE html>
|
|
2
|
-
<!-- Generated by scripts/gen-guide.ts from src/capabilities/registry.ts
|
|
2
|
+
<!-- Generated by scripts/gen-guide.ts from src/capabilities/registry.ts, HOW_TO_USE.md and the feature guides in docs/. Do not edit: change those and run bun scripts/gen-guide.ts. -->
|
|
3
3
|
<html lang="en">
|
|
4
4
|
<head>
|
|
5
5
|
<meta charset="utf-8">
|
|
@@ -259,7 +259,7 @@ a:hover{text-decoration-color:var(--guide-accent)}
|
|
|
259
259
|
.toc{display:none}
|
|
260
260
|
}
|
|
261
261
|
@media (max-width:900px){
|
|
262
|
-
.shell{grid-template-columns:minmax(0,1fr);grid-template-rows:auto minmax(0,1fr);gap:14px;padding:0}
|
|
262
|
+
.shell{grid-template-columns:minmax(0,1fr);grid-template-rows:auto auto minmax(0,1fr);gap:14px;padding:0}
|
|
263
263
|
.rail{grid-column:1;grid-row:1;overflow:visible;border-right:0;border-bottom:1px solid var(--guide-line)}
|
|
264
264
|
.mast{grid-column:1;grid-row:2}
|
|
265
265
|
.rail-inner{padding:12px 14px}
|
|
@@ -341,6 +341,11 @@ a:hover{text-decoration-color:var(--guide-accent)}
|
|
|
341
341
|
<li><a href="#reference"><span class="n">17</span><span class="t">Reference</span><span class="d">MCP, tuning, logs, environment variables, and every command</span></a></li>
|
|
342
342
|
</ol>
|
|
343
343
|
</section>
|
|
344
|
+
<section class="rail-group" id="rail-hits" hidden>
|
|
345
|
+
<h2 class="rail-title mono">Sections</h2>
|
|
346
|
+
<ol class="chapters" id="rail-hits-list"></ol>
|
|
347
|
+
<p class="muted" id="rail-none" hidden>No section matches</p>
|
|
348
|
+
</section>
|
|
344
349
|
</div>
|
|
345
350
|
</nav>
|
|
346
351
|
<header class="mast">
|
|
@@ -384,7 +389,7 @@ forgeqe docs --copy ./docs # copy the guide, the architecture page, and the re
|
|
|
384
389
|
<div class="area"><a class="area-name" href="#area-setup"><span class="mono">Setup</span></a><ul class="tiles"><li><a class="tile" href="#cap-test-apps"><b>Test environments and their apps</b><span>Place an order in the shop, then check its invoice in Billing, in UAT</span></a></li><li><a class="tile" href="#cap-generated-code"><b>What ForgeQE wrote</b><span>How much of the code ForgeQE wrote is still there?</span></a></li><li><a class="tile" href="#cap-forgeqe-commands"><b>Every forgeqe command from the shell</b><span>Show me the context budget</span></a></li><li><a class="tile" href="#cap-setup-help"><b>Setup, skills, and documentation</b><span>What can you do?</span></a></li></ul></div>
|
|
385
390
|
<div class="area"><a class="area-name" href="#area-desktop"><span class="mono">Desktop</span></a><ul class="tiles"><li><a class="tile" href="#cap-desktop-apps"><b>Automate Windows desktop apps</b><span>Create an invoice for Contoso in the Billing app and save it as a test</span></a></li></ul></div>
|
|
386
391
|
<div class="area"><a class="area-name" href="#area-mobile"><span class="mono">Mobile</span></a><ul class="tiles"><li><a class="tile" href="#cap-mobile-apps"><b>Automate Android and iOS apps</b><span>Sign in to the courier app and accept the first delivery, and save it as a test</span></a></li></ul></div>
|
|
387
|
-
<div class="area"><a class="area-name" href="#area-operations"><span class="mono">Operations</span></a><ul class="tiles"><li><a class="tile" href="#cap-containers"><b>Services and
|
|
392
|
+
<div class="area"><a class="area-name" href="#area-operations"><span class="mono">Operations</span></a><ul class="tiles"><li><a class="tile" href="#cap-containers"><b>Services, mocks, workspace and browser in Docker or Podman</b><span>Start the database and the queue in Docker</span></a></li><li><a class="tile" href="#cap-monitoring"><b>Scheduled monitoring with alerts</b><span>Check the login flow every 15 minutes and alert Teams</span></a></li><li><a class="tile" href="#cap-notifications"><b>Teams and email notifications</b><span>Post the results to the QA channel</span></a></li><li><a class="tile" href="#cap-pipelines"><b>CI pipelines</b><span>Add a GitHub Actions workflow that runs tests/cases.yaml on every PR, and make it green</span></a></li></ul></div>
|
|
388
393
|
<div class="area"><a class="area-name" href="#area-data"><span class="mono">Data</span></a><ul class="tiles"><li><a class="tile" href="#cap-databases"><b>Ask a database in plain language</b><span>Which customers in India have paid orders? Ask the orders database</span></a></li><li><a class="tile" href="#cap-test-data"><b>Test data on demand</b><span>Generate test data: 20 customers with 1 to 3 orders each in the shop database</span></a></li><li><a class="tile" href="#cap-data-flows"><b>Data flows between systems</b><span>Check that customers from the digital database reach the CRM correctly</span></a></li></ul></div>
|
|
389
394
|
<div class="area"><a class="area-name" href="#area-automated-tests"><span class="mono">Automated tests</span></a><ul class="tiles"><li><a class="tile" href="#cap-write-tests"><b>Turn a passed run into an automated test</b><span>Write a Playwright test for that run in ../app</span></a></li><li><a class="tile" href="#cap-heal-tests"><b>Heal failing tests</b><span>Heal the failing tests in ../app</span></a></li><li><a class="tile" href="#cap-find-run-tests"><b>Find and run existing tests</b><span>Run the checkout tests</span></a></li><li><a class="tile" href="#cap-regression-packs"><b>Regression packs for a release</b><span>Build a regression pack for sprint 12 and run it on TestMu</span></a></li><li><a class="tile" href="#cap-flaky-tests"><b>Flaky test detection and quarantine</b><span>Which tests are flaky?</span></a></li></ul></div>
|
|
390
395
|
<div class="area"><a class="area-name" href="#area-pull-requests"><span class="mono">Pull requests</span></a><ul class="tiles"><li><a class="tile" href="#cap-pr-review"><b>Review pull requests</b><span>Review PR 42 and focus on error handling</span></a></li><li><a class="tile" href="#cap-pr-impact"><b>Pull request impact and test selection</b><span>What does PR 42 impact?</span></a></li></ul></div>
|
|
@@ -407,7 +412,7 @@ forgeqe docs --copy ./docs # copy the guide, the architecture page, and the re
|
|
|
407
412
|
<tbody>
|
|
408
413
|
<tr><td><strong>npm (Node.js 18+) or pip (Python 3.9+)</strong></td><td>To install <code>@capagents/forgeqe</code> or <code>forgeqe-cli</code>; either one gives the same <code>forgeqe</code> command</td></tr>
|
|
409
414
|
<tr><td><strong>A terminal</strong></td><td>macOS Terminal, iTerm2, VS Code or Cursor's terminal, Windows Terminal</td></tr>
|
|
410
|
-
<tr><td><strong>A model</strong></td><td>A profile in <code>llms.json</code> with a working key, or Microsoft Entra ID sign-in for Azure (Azure OpenAI, OpenAI, an OpenAI-compatible gateway, or Ollama), or a GitHub Copilot seat (see <a href="
|
|
415
|
+
<tr><td><strong>A model</strong></td><td>A profile in <code>llms.json</code> with a working key, or Microsoft Entra ID sign-in for Azure (Azure OpenAI, OpenAI, an OpenAI-compatible gateway, or Ollama), or a GitHub Copilot seat (see <a href="../HOW_TO_USE.md#github-copilot-seats-as-the-model">GitHub Copilot seats as the model</a>)</td></tr>
|
|
411
416
|
<tr><td><strong>Internet access on first use</strong></td><td>ForgeQE downloads Bun, the agent runtime, uv, Python, and the engine into <code>~/.forgeqe</code> once</td></tr>
|
|
412
417
|
</tbody>
|
|
413
418
|
</table></div>
|
|
@@ -440,9 +445,8 @@ export WP_AZURE_API_KEY=... # the key your llms.json profile names
|
|
|
440
445
|
forgeqe models # check the profiles
|
|
441
446
|
forgeqe start # the shell
|
|
442
447
|
</code></pre>
|
|
443
|
-
<p>With GitHub Copilot licences instead of a model key, sign in rather than exporting one, and set <code>llm.active: copilot</code> in <code>forgeqe.yaml</code> (or <code>"default": "copilot"</code> in <code>llms.json</code>):</p>
|
|
444
|
-
<pre><code class="language-bash">forgeqe auth login copilot #
|
|
445
|
-
forgeqe auth login github # the browser engine, through GitHub Models
|
|
448
|
+
<p>With GitHub Copilot licences instead of a model key, sign in rather than exporting one, and set <code>llm.active: copilot</code> in <code>forgeqe.yaml</code> (or <code>"default": "copilot"</code> in <code>llms.json</code>). The browser engine then runs on your Copilot seat; ForgeQE's agent and Scribe need another profile for now:</p>
|
|
449
|
+
<pre><code class="language-bash">forgeqe auth login copilot # a fine-grained token with Copilot Requests, or the GitHub CLI's sign-in
|
|
446
450
|
forgeqe models # the copilot profile is active
|
|
447
451
|
</code></pre>
|
|
448
452
|
<p>In the shell, type a request and press Enter:</p>
|
|
@@ -455,7 +459,7 @@ Write a Playwright test for that run in ../app
|
|
|
455
459
|
forgeqe suite cases.yaml --parallel 3 # what pipelines call
|
|
456
460
|
</code></pre>
|
|
457
461
|
<p><code>forgeqe models</code> lists the profiles ForgeQE sees and marks the active one. With no profile set up, the shell's footer reads <strong>Model none set up</strong> and any request explains how to add one.</p>
|
|
458
|
-
<p>With more than one profile, <code>llm.routing: auto</code> gives each kind of work (agent, browser, coding, review, analysis, planning) the configured model it needs; <code>forgeqe models --routes</code> shows the picks. See <a href="
|
|
462
|
+
<p>With more than one profile, <code>llm.routing: auto</code> gives each kind of work (agent, browser, coding, review, analysis, planning) the configured model it needs; <code>forgeqe models --routes</code> shows the picks. See <a href="#doc-docs-model_routing-model-routing-the-right-configured-model-for-each-task">docs/MODEL_ROUTING.md</a>.</p>
|
|
459
463
|
<hr />
|
|
460
464
|
</div>
|
|
461
465
|
</section>
|
|
@@ -520,12 +524,10 @@ bun src/index.ts init --global --profile azure-gpt4o
|
|
|
520
524
|
<p>Give that identity the <strong>Cognitive Services OpenAI User</strong> role on the resource. Tokens are renewed before they expire, so long suites and coding sessions keep running. The browser engine signs in itself; the coding agent's runtime only takes a key, so ForgeQE gives it a gateway on <code>127.0.0.1</code> that accepts a one-time random key and passes each request on with a current token. No key is stored anywhere.</p>
|
|
521
525
|
<p><code>auth: entra</code> also works on an <code>openai_compatible</code> profile, for a gateway such as API Management that checks Entra ID tokens: set <code>token_scope</code> to the scope it expects (for example <code>api://<app id>/.default</code>; the default is <code>https://cognitiveservices.azure.com/.default</code>). The engine needs the <code>azure-identity</code> package, which <code>forgeqe setup</code> installs (and adds to environments set up before it was needed); with <code>FORGEQE_PYTHON</code>, install it there yourself.</p>
|
|
522
526
|
</div></section>
|
|
523
|
-
<section class="doc sub" id="doc-how_to_use-github-copilot-seats-as-the-model"><h4>GitHub Copilot seats as the model</h4><div class="prose"><p>With GitHub Copilot licences (Pro, Pro+, Business or Enterprise), a <code>github_copilot</code> profile runs
|
|
527
|
+
<section class="doc sub" id="doc-how_to_use-github-copilot-seats-as-the-model"><h4>GitHub Copilot seats as the model</h4><div class="prose"><p>With GitHub Copilot licences (Pro, Pro+, Business or Enterprise), a <code>github_copilot</code> profile runs the browser engine and the desktop and mobile agents on your Copilot seat, through <a href="https://docs.github.com/en/copilot/how-tos/copilot-sdk" rel="noopener">GitHub's Copilot SDK</a>. Copilot gives other tools no model API of its own, so ForgeQE starts a small gateway on <code>127.0.0.1</code> that runs the SDK and speaks the OpenAI chat API to the engine. It signs in with a GitHub token, so no other app is authorized on your GitHub account.</p>
|
|
524
528
|
<pre><code class="language-json">"copilot": {
|
|
525
529
|
"provider": "github_copilot",
|
|
526
|
-
"model": "
|
|
527
|
-
"engine_model": "openai/gpt-4.1",
|
|
528
|
-
"github_org": "acme"
|
|
530
|
+
"model": "gpt-4.1"
|
|
529
531
|
}
|
|
530
532
|
</code></pre>
|
|
531
533
|
<div class="tbl"><table>
|
|
@@ -533,90 +535,50 @@ bun src/index.ts init --global --profile azure-gpt4o
|
|
|
533
535
|
<tr><th>Field</th><th>What it is</th></tr>
|
|
534
536
|
</thead>
|
|
535
537
|
<tbody>
|
|
536
|
-
<tr><td><code>model</code></td><td>The Copilot model
|
|
537
|
-
<tr><td><code>engine_model</code></td><td>
|
|
538
|
-
<tr><td><code>
|
|
539
|
-
<tr><td><code>api_key_env</code></td><td>Optional. The variable holding the GitHub Models token; default <code>GITHUB_TOKEN</code>, else the GitHub CLI's sign-in.</td></tr>
|
|
538
|
+
<tr><td><code>model</code></td><td>The Copilot model, as the Copilot SDK names it (<code>gpt-4.1</code>, <code>claude-sonnet-4.6</code>). It must take screenshots. The sign-in lists your seat's models and which of them take images.</td></tr>
|
|
539
|
+
<tr><td><code>engine_model</code></td><td>Optional. Another Copilot model for the browser engine and the desktop agent.</td></tr>
|
|
540
|
+
<tr><td><code>api_key_env</code></td><td>Optional. The variable holding the GitHub token; without it ForgeQE reads <code>COPILOT_GITHUB_TOKEN</code>, then the token <code>forgeqe auth login copilot</code> saved, then the GitHub CLI's sign-in.</td></tr>
|
|
540
541
|
</tbody>
|
|
541
542
|
</table></div>
|
|
542
543
|
<p>Sign in once per machine:</p>
|
|
543
|
-
<pre><code class="language-bash">forgeqe auth login copilot
|
|
544
|
-
forgeqe auth login github # the GitHub CLI, whose sign-in the browser engine uses for GitHub Models
|
|
544
|
+
<pre><code class="language-bash">forgeqe auth login copilot
|
|
545
545
|
</code></pre>
|
|
546
|
-
<p
|
|
547
|
-
<
|
|
546
|
+
<p>It offers two ways:</p>
|
|
547
|
+
<ul>
|
|
548
|
+
<li><strong>A fine-grained token</strong> (recommended). ForgeQE opens GitHub's token page: set <strong>Resource owner</strong> to your personal account and add the <strong>Copilot Requests</strong> permission, then paste the token (<code>github_pat_...</code>). The token goes to your system's credential store like ForgeQE's other secrets. Classic tokens (<code>ghp_</code>) do not work with Copilot.</li>
|
|
549
|
+
<li><strong>The GitHub CLI's sign-in</strong>, when <code>gh</code> is signed in.</li>
|
|
550
|
+
</ul>
|
|
551
|
+
<p>Either way ForgeQE checks the token by asking Copilot for your seat's models and lists them; the first time, the Copilot SDK downloads Copilot's runtime from GitHub (checked against GitHub's checksums). <code>forgeqe auth</code> shows the sign-in on its <code>Copilot</code> line; <code>forgeqe auth logout copilot</code> forgets a saved token.</p>
|
|
552
|
+
<p>The token reaches only the gateway's process. The engine gets the gateway's address and a one-time key that works nowhere else; the gateway refuses requests without that key and any request a web page sends. Model-run shell commands never see the token or the key.</p>
|
|
548
553
|
<p>Which part uses what:</p>
|
|
549
554
|
<div class="tbl"><table>
|
|
550
555
|
<thead>
|
|
551
556
|
<tr><th>Part</th><th>Model</th><th>Signs in with</th></tr>
|
|
552
557
|
</thead>
|
|
553
558
|
<tbody>
|
|
554
|
-
<tr><td>
|
|
555
|
-
<tr><td>
|
|
556
|
-
<tr><td>
|
|
557
|
-
<tr><td>The desktop agent</td><td><code>engine_model</code>, on GitHub Models</td><td><code>GITHUB_TOKEN</code>, or <code>forgeqe auth login github</code></td></tr>
|
|
559
|
+
<tr><td>The browser engine (every browser run: <code>run --direct</code>, suites, monitoring)</td><td><code>engine_model</code> or <code>model</code>, on the Copilot seat</td><td><code>forgeqe auth login copilot</code></td></tr>
|
|
560
|
+
<tr><td>The desktop and mobile agents</td><td><code>engine_model</code> or <code>model</code>, on the Copilot seat</td><td><code>forgeqe auth login copilot</code></td></tr>
|
|
561
|
+
<tr><td>ForgeQE's agent and Scribe</td><td>Not on Copilot yet: give them another profile with <code>--profile</code>, or <code>code.model</code></td><td></td></tr>
|
|
558
562
|
</tbody>
|
|
559
563
|
</table></div>
|
|
560
|
-
<p
|
|
561
|
-
<p>
|
|
562
|
-
<p
|
|
564
|
+
<p>Copilot runs one SDK session per model request, with no tools of its own and your request's system prompt in place of Copilot's. Each request is billed to the seat (GitHub AI credits or premium requests, by your plan), and GitHub may rate-limit frequent automated requests. If a model is refused, check that your organization's Copilot policies allow it. ForgeQE shows no cost for a <code>github_copilot</code> profile unless you give it a <code>price</code>.</p>
|
|
565
|
+
<p><strong>In pipelines.</strong> <code>forgeqe suite ... --no-code</code> uses only the browser engine. Set <code>COPILOT_GITHUB_TOKEN</code> to a fine-grained token with the Copilot Requests permission, stored as a pipeline secret. GitHub also documents the workflow's own <code>GITHUB_TOKEN</code> with <code>permissions: copilot-requests: write</code> for organization repositories whose Copilot policy allows it (<a href="https://docs.github.com/en/copilot/how-tos/copilot-sdk/auth/server-to-server-tokens" rel="noopener">server tokens</a>). A Copilot seat belongs to one person: prefer an organization-billed token over signing a shared pipeline in to someone's seat.</p>
|
|
566
|
+
<p>GitHub Models, which ForgeQE used for the engine before, was <a href="https://github.blog/changelog/2026-07-30-github-models-is-now-retired/" rel="noopener">retired by GitHub</a> on July 30, 2026. A profile that still has <code>engine_model: "openai/..."</code> or <code>github_org</code> needs updating: drop <code>github_org</code>, and use a Copilot model id.</p>
|
|
563
567
|
<p><strong>When something goes wrong.</strong></p>
|
|
564
568
|
<div class="tbl"><table>
|
|
565
569
|
<thead>
|
|
566
570
|
<tr><th>ForgeQE says</th><th>What to do</th></tr>
|
|
567
571
|
</thead>
|
|
568
572
|
<tbody>
|
|
569
|
-
<tr><td><code>
|
|
570
|
-
<tr><td><code>
|
|
571
|
-
<tr><td><code
|
|
572
|
-
<tr><td
|
|
573
|
-
<tr><td>
|
|
574
|
-
<tr><td>The
|
|
573
|
+
<tr><td><code>GitHub Copilot needs a GitHub token, and there is none</code></td><td>Run <code>forgeqe auth login copilot</code> (or <code>/auth copilot</code> in the shell), or set <code>COPILOT_GITHUB_TOKEN</code>.</td></tr>
|
|
574
|
+
<tr><td><code>GitHub Copilot does not take classic tokens (ghp_)</code></td><td>Make a fine-grained token with the Copilot Requests permission, owned by your personal account.</td></tr>
|
|
575
|
+
<tr><td><code>GitHub Copilot did not accept the token</code></td><td>The token has no Copilot Requests permission, has expired, or the account has no Copilot seat.</td></tr>
|
|
576
|
+
<tr><td><code>… must be a Copilot model id</code>, <code>engine_model must be a Copilot model id</code></td><td>Fix that field in <code>llms.json</code>: the sign-in lists the seat's models. A GitHub Models id such as <code>openai/gpt-4.1</code> is not a Copilot id; use <code>gpt-4.1</code>.</td></tr>
|
|
577
|
+
<tr><td><code>ForgeQE's agent and Scribe cannot run on a github_copilot profile yet</code></td><td>Pick another profile for them (<code>--profile</code>, or Tab in the shell), or set <code>code.model</code>.</td></tr>
|
|
578
|
+
<tr><td>The model is refused, or not found</td><td>The seat's plan or your organization's Copilot policies do not allow that model: pick one the sign-in listed, or ask a Copilot admin to enable it.</td></tr>
|
|
579
|
+
<tr><td><code>GitHub Copilot runs through the ForgeQE engine, which is not set up</code></td><td>Run <code>forgeqe setup</code>.</td></tr>
|
|
575
580
|
</tbody>
|
|
576
581
|
</table></div>
|
|
577
|
-
<p><code>init</code> also puts three folders in <code>.forgeqe/</code> to read, change, and commit:</p>
|
|
578
|
-
<div class="tbl"><table>
|
|
579
|
-
<thead>
|
|
580
|
-
<tr><th>Folder</th><th>What it holds</th></tr>
|
|
581
|
-
</thead>
|
|
582
|
-
<tbody>
|
|
583
|
-
<tr><td><code>.forgeqe/prompts/</code></td><td>Every prompt ForgeQE's agents work from, one markdown file each: ForgeQE's agent (<code>brain.md</code>), Scribe (<code>test-agent.md</code>, <code>test-rules.md</code>), the reviewer, healing, finding tests, pull request review, API tests and scenarios, exploratory charters, and the browser agent (<code>browser-*.md</code>, plus <code>browser-extra.md</code> for your own instructions).</td></tr>
|
|
584
|
-
<tr><td><code>.forgeqe/skills/</code></td><td>The built-in skills (framework and app playbooks), to adapt to your project; add your own next to them.</td></tr>
|
|
585
|
-
<tr><td><code>.forgeqe/knowledge/</code></td><td>Product knowledge templates: the product and its flows, roles, glossary, business rules, test data, environments.</td></tr>
|
|
586
|
-
</tbody>
|
|
587
|
-
</table></div>
|
|
588
|
-
<p><strong>Your copy or ForgeQE's.</strong> <code>.forgeqe-manifest.json</code> in each folder records what was copied. A copy you have not changed still counts as ForgeQE's, so it keeps getting ForgeQE's improvements when you update; once you edit it, yours is used. When ForgeQE's version changes after you edited yours, ForgeQE says so at start-up. Running <code>init</code> again adds what is missing and never overwrites an edited file (with <code>--force</code> it refreshes only unchanged copies).</p>
|
|
589
|
-
<p><strong>Prompts.</strong> Each file has a short header (<code>name</code>, <code>description</code>, <code>used_by</code>, <code>kind</code>) and the prompt; <code>{{> name}}</code> includes another prompt (<code>test-rules</code> is included in the test, reviewer, and healing prompts, so one edit changes all three). ForgeQE looks in <code>prompts.dir</code>, then <code>.forgeqe/prompts/</code>, then <code>~/.forgeqe/prompts/</code> (for every project), then its own. Changes take effect on the next run or session.</p>
|
|
590
|
-
<pre><code class="language-bash">forgeqe prompts # every prompt, what it is for, and where it comes from
|
|
591
|
-
forgeqe prompts show test-agent # the prompt as the agent gets it (includes expanded; --raw for the file)
|
|
592
|
-
forgeqe prompts diff test-rules # how your copy differs from ForgeQE's
|
|
593
|
-
forgeqe prompts reset test-rules # back to ForgeQE's (yours is kept as test-rules.md.bak)
|
|
594
|
-
forgeqe prompts check # each prompt against the prompting standard for its kind (exits 1 when one falls short)
|
|
595
|
-
forgeqe prompts check brain # what to fix in one prompt: sections, rules, examples, contract, sentences
|
|
596
|
-
forgeqe prompts eval # run the evaluation cases with the live model; pass rates against the last run
|
|
597
|
-
forgeqe prompts eval refine --runs 3 # one prompt, three runs of each case
|
|
598
|
-
</code></pre>
|
|
599
|
-
<p><strong>The prompting standard.</strong> Every prompt follows one standard, and each technique is used where it fits the kind of prompt (<code>kind:</code> in its header):</p>
|
|
600
|
-
<div class="tbl"><table>
|
|
601
|
-
<thead>
|
|
602
|
-
<tr><th>Kind</th><th>Prompts</th><th>Structure</th><th>Few-shot examples</th><th>Execution contract</th></tr>
|
|
603
|
-
</thead>
|
|
604
|
-
<tbody>
|
|
605
|
-
<tr><td>agent: works with tools over many steps</td><td>brain, test-agent, heal, api-tests, pr-review, find-tests</td><td>Role, Objective, Execution contract, Rules, Output</td><td>where routing or reply format varies</td><td>Preconditions, Tools, Procedure, Stop when, On failure, Evidence</td></tr>
|
|
606
|
-
<tr><td>structured: one answer in a fixed shape</td><td>refine, api-scenarios, explore-charters, eval-judge, test-reviewer, browser-skill-select</td><td>Role, Objective, Inputs, Rules, Output, Examples</td><td>two or more worked examples</td><td>–</td></tr>
|
|
607
|
-
<tr><td>fragment: rules included in other prompts</td><td>test-rules, domain-rule, skill-rule, conventions, the browser agent's rules</td><td>rules only</td><td>–</td><td>–</td></tr>
|
|
608
|
-
</tbody>
|
|
609
|
-
</table></div>
|
|
610
|
-
<ul>
|
|
611
|
-
<li><strong>ASD-STE100</strong> Simplified Technical English everywhere, so each agent reads one clear instruction at a time: an instruction has 20 words or fewer and a description 25 or fewer, one instruction per sentence, the active voice, and approved words ("make sure", not "ensure"). At least 80% of sentences. The check covers the rules that can be measured in text and a list of common words the standard does not approve; the standard's full dictionary is licensed and not included.</li>
|
|
612
|
-
<li><strong>RFC 2119</strong> key words state the strength of every rule: <strong>MUST</strong> / <strong>MUST NOT</strong> are absolute, <strong>SHOULD</strong> / <strong>SHOULD NOT</strong> are strong defaults the agent may break only for a reason it states, <strong>MAY</strong> is a free choice, and a numbered procedure step is a MUST. A lower-case "must" or "should" is flagged as ambiguous. <code>conventions.md</code> says this at the top of every agent and structured prompt, and the browser engine adds it to the browser agent's instructions.</li>
|
|
613
|
-
<li><strong>Structured prompting and few-shot examples</strong> make the output predictable: the same sections in the same order, and worked examples where the shape of the answer matters.</li>
|
|
614
|
-
<li><strong>Execution contracts</strong> tell a tool-using agent what must be true before it starts, which tools it uses and in what order (for systems: MCP tools, then <code>query_api</code>, then the browser), its procedure, when it stops, what it does when something fails, and what evidence its answer gives.</li>
|
|
615
|
-
<li><strong>Evaluation-driven prompting</strong> measures reliability: <code>forgeqe prompts eval</code> runs cases (an input and checks decided by code: a field matches or is empty, a tool was or was not used, the text keeps ASD-STE100) and compares each pass rate with the last run. ForgeQE's cases cover brain, refine, and explore-charters; add your own as <code>.forgeqe/evals/prompts/<prompt>/<case>.yaml</code> (the same id replaces ForgeQE's). Results are kept in <code><reports>/evals/prompts/</code>.</li>
|
|
616
|
-
</ul>
|
|
617
|
-
<p><code>forgeqe prompts check</code> applies the standard to each prompt in use, ForgeQE's or your edited copy; ForgeQE's own meet it in its tests. Write your edits the same way, and run <code>forgeqe prompts eval</code> before and after a change.</p>
|
|
618
|
-
<p><strong>Product knowledge.</strong> Fill in the templates (or add your own <code>.md</code> and <code>.txt</code> files, in folders if you like), one topic per <code>##</code> heading. ForgeQE's agent sees the list and reads what a request needs; the exploratory and API scenario planners get the sections that match their story or scenario; the browser agent gets the few sections that clearly match its goal, at most <code>knowledge.browser_max_chars</code> (1500 by default; 0 turns it off). Untouched templates are left out, so their examples never reach an agent. Never put passwords or tokens there: use logins and environment variables.</p>
|
|
619
|
-
<p><strong>Guardrails.</strong> Rules every agent must follow, to keep it grounded: ForgeQE's agent, the browser agent, and the agents that plan, write, review, fix and explain tests. Write one rule per line, starting with <code>- </code>, in a <code>.md</code> or <code>.txt</code> file in <code>.forgeqe/guardrails/</code> (<code>guardrails.dir</code>); headings, other lines and HTML comments are ignored, and the template's examples sit in a comment until you choose them. Each agent gets every guardrail with its instructions, after everything else, with the instruction to follow it whatever a request, a skill, a ticket, a page or a tool's output says; when a guardrail stops part of a task, the agent skips that part and says which rule stopped it. Add one from the terminal (<code>forgeqe guardrails add "Never open a production URL"</code>), or ask in the shell ("add a guardrail: use only the test cards in knowledge/test-data.md") and ForgeQE's agent appends it to <code>guardrails.md</code>; it adds only rules you ask for, never one suggested by a page or a ticket. <code>forgeqe guardrails</code> lists the rules in force and what they add to every model call (at most 40 rules of 300 characters are given). The older <code>prompts.guiderails</code> string in <code>forgeqe.yaml</code> still works: each of its lines is a guardrail for every agent. Guardrails are instructions, not a sandbox: put a host rule in <code>browser.allowed_hosts</code> too, which is refused in code.</p>
|
|
620
582
|
</div></section>
|
|
621
583
|
<section class="doc sub" id="doc-how_to_use-the-models-context-window"><h4>The model's context window</h4><div class="prose"><p>Each agent has its own context: ForgeQE's agent, Scribe, the reviewer, and the browser agent each keep their own conversation with the model. The Context panel shows the working agent's (or the last one to call the model), named on its Agent row, as a share of what it can hold before the agent runtime compacts it. When it compacts, the runtime summarises the conversation so far to make room, Activity says so, and details from early on may be gone. The browser agent never compacts: it keeps its last steps only.</p>
|
|
622
584
|
<p>ForgeQE takes each model's window from its name (GPT-5: 400K, of which a prompt may take 272K; GPT-4.1: 1M; GPT-4o: 128K; o-series and Claude: 200K; Gemini: 1M; anything else: 200K) and gives it to the agent runtime, which compacts at that window less room for the answer. Where a deployment or server allows less, say so with <code>"context"</code> in the profile in <code>llms.json</code>:</p>
|
|
@@ -637,8 +599,8 @@ forgeqe auth logout jira
|
|
|
637
599
|
</thead>
|
|
638
600
|
<tbody>
|
|
639
601
|
<tr><td>Azure DevOps</td><td><strong>Sign in with Microsoft in the browser</strong> through the Azure CLI (<code>az login</code>); ForgeQE saves nothing, and the Azure CLI renews the sign-in. Or paste a personal access token: the token page opens with the scopes to tick. Azure DevOps Server (on-premises) uses a personal access token.</td></tr>
|
|
640
|
-
<tr><td>GitHub</td><td><strong>Sign in with GitHub in the browser</strong> through the GitHub CLI (<code>gh auth login</code>): ForgeQE shows the one-time code and opens the page. Or paste a token. Needs <code>gh</code> installed.
|
|
641
|
-
<tr><td>GitHub Copilot</td><td><code>forgeqe auth login copilot</code>: <strong>
|
|
602
|
+
<tr><td>GitHub</td><td><strong>Sign in with GitHub in the browser</strong> through the GitHub CLI (<code>gh auth login</code>): ForgeQE shows the one-time code and opens the page. Or paste a token. Needs <code>gh</code> installed. <code>forgeqe auth login copilot</code> can use this sign-in for a <code>github_copilot</code> profile.</td></tr>
|
|
603
|
+
<tr><td>GitHub Copilot</td><td><code>forgeqe auth login copilot</code>: <strong>Paste a fine-grained token</strong> with the Copilot Requests permission (ForgeQE opens GitHub's token page), or use the GitHub CLI's sign-in. ForgeQE checks it with Copilot and lists the models your seat has. No other app is authorized on your GitHub account. For a <code>github_copilot</code> profile; see <a href="../HOW_TO_USE.md#github-copilot-seats-as-the-model">GitHub Copilot seats as the model</a>.</td></tr>
|
|
642
604
|
<tr><td>Jira, Confluence</td><td>Your Atlassian API tokens page opens (Cloud) or your personal access tokens page (Server/Data Center); create one and paste it. Confluence on the same Cloud site as Jira reuses the Jira sign-in.</td></tr>
|
|
643
605
|
<tr><td>TestRail, Zephyr Scale, qTest, Xray Cloud</td><td>The page with your API key opens, or ForgeQE says where to find it; paste it.</td></tr>
|
|
644
606
|
<tr><td>Anything else</td><td><code>forgeqe auth login OPENAI_API_KEY</code> (any variable name) saves that one secret: a model key, a Teams webhook URL, an SMTP or saved-login password.</td></tr>
|
|
@@ -646,31 +608,8 @@ forgeqe auth logout jira
|
|
|
646
608
|
</table></div>
|
|
647
609
|
<p>Atlassian's OAuth sign-in needs a client secret, which a command-line tool cannot keep secret, so Jira and Confluence use API tokens: they are what Atlassian recommends for scripts and tools.</p>
|
|
648
610
|
<p>Each token is checked against the system before it is saved, and the system is switched on: a saved Jira sign-in sets the site URL too, so <code>sources.jira</code> is not needed in <code>forgeqe.yaml</code>. Saved values go in under the variable names the config already reads (<code>JIRA_API_TOKEN</code>, <code>ADO_PAT</code>, ...). A variable exported in the terminal is used before a saved sign-in, and a saved sign-in before a <code>.env</code> file, so pipelines keep using their secrets. The shell commands the agents run never see the saved secrets.</p>
|
|
649
|
-
|
|
650
|
-
<
|
|
651
|
-
<p>A profile's own <code>"price": { "input": 2.5, "output": 15, "cached_input": 0.25 }</code> (per million tokens) wins, for a negotiated or enterprise rate. Leave <code>currency</code> out and reports use this machine's currency, or set <code>"currency": "USD"</code>. Without <code>cached_input</code>, cached input gets the model's cache discount from its list price, and the cost is marked as an estimate (<code>≈$1.52</code>); with no list price for the model either, cached input is charged at the full rate and the cost is an upper bound (<code>up to $6.11</code>). When your price differs from the list price, ForgeQE says so (<code>forgeqe models</code>, the shell, and the start of each command), since that is usually a typo or an old price. Anthropic's cache writes are priced at their own rate. A provisioned (PTU) deployment is billed per hour, not per token, so its costs are what the tokens would cost at list price. Requests over a model's long-context threshold (272K tokens for GPT-5.4) cost more and are priced at the normal rate.</p>
|
|
652
|
-
<p><strong>Usage after every job.</strong> Every browser run, Scribe job, heal, suite, and monitor round ends with a <code>Usage</code> line: tokens in (and how many of them were cached), tokens out, and the cost at the profile's price. A request to ForgeQE's agent in the shell, and every <code>forgeqe run</code>, <code>code</code>, <code>suite</code>, and <code>heal</code> command, ends with the total for everything it started, by role:</p>
|
|
653
|
-
<pre><code>This request used 1.1M tokens (1.1M in, 30.0K of it cached · 34.7K out) · $3.07 — ForgeQE's agent 42.0K · $0.12, browser agent 52.7K · $0.15, Scribe 1.0M · $2.80
|
|
654
|
-
</code></pre>
|
|
655
|
-
<p><strong>Reuse before rebuild, extend before duplicate.</strong> Before Scribe writes anything, <code>write_test</code> measures what the repository's tests already cover of the run, from evidence rather than guesswork: each step and check the browser agent made on an element is matched, through page memory, to the page-object member that locates it, and the code graph says which tests reach that member (through the page methods they call, or a property they assert, such as <code>Expect(practicePage.AjaxStatus)</code>). Steps on elements no page object locates yet are listed as not measured, since a test may do them its own way. Then:</p>
|
|
656
|
-
<div class="tbl"><table>
|
|
657
|
-
<thead>
|
|
658
|
-
<tr><th>What it finds</th><th>What happens</th></tr>
|
|
659
|
-
</thead>
|
|
660
|
-
<tbody>
|
|
661
|
-
<tr><td>One test covers every measured step and check</td><td><strong>Reuse</strong>: Scribe only finds the command that runs it, and changes nothing. If it fails, nothing is changed: you are asked whether to update it or write a new test.</td></tr>
|
|
662
|
-
<tr><td>One test covers most (40% or more)</td><td><strong>Extend</strong>: Scribe adds only the missing steps and checks to that test, keeping every step and assertion it has; the reviewer rejects a removed or weakened assertion, or the steps copied into a new file</td></tr>
|
|
663
|
-
<tr><td>Two or more tests cover most of it</td><td><strong>You choose</strong>: the evidence and the options (reuse, extend with how many steps it adds, a new test, cancel)</td></tr>
|
|
664
|
-
<tr><td>Nothing covers much</td><td><strong>New</strong>: a new test, reusing the page objects and helpers as before</td></tr>
|
|
665
|
-
</tbody>
|
|
666
|
-
</table></div>
|
|
667
|
-
<p>A card says which it did and why (<em>Extending PracticeTests.Flow · covers 12 of 15; only 3 are missing</em>). Say how when you know ("extend the login test", "write a separate test", "update it") and ForgeQE's agent passes that on without asking. In every mode Scribe adds page-object members rather than changing what an existing member does for the other tests that use it.</p>
|
|
668
|
-
<p><strong>The same flow asked again.</strong> When a session writes tests and you ask for a flow an earlier job already wrote a test for, ForgeQE runs that test first, before any browser run. If it passes in a real browser, that is the answer: no browser run, no Scribe, and no new test file, in the seconds the test takes (<em>✓ Existing test passed · 19s · no browser run or new test needed</em>). If it fails, the flow runs in the browser and Scribe updates that test instead of writing another. ForgeQE's agent rewords a goal a little each time, so goals count as the same flow when they share their content words (endings, case, and phrasing aside) and the new goal keeps everything the earlier one checked; the same steps with a different final check, a shorter flow, or another site are a new flow. The agent also compares the earlier goal with your request and runs fresh when you ask for a check the earlier test does not have. Ask for "a fresh run" or "a new test" to skip the reuse.</p>
|
|
669
|
-
<p><strong>When the flow cannot be reproduced.</strong> If ForgeQE's own replay of the run could not reach the run's final page and Scribe's first test fails too, the job stops there and says which replayed step failed and why, instead of spending its remaining attempts on a flow no script can repeat as recorded.</p>
|
|
670
|
-
<p><strong>Why a test failed.</strong> When ForgeQE's run of Scribe's test fails, the ✗ line and the next attempt say why in the test's own words: dotnet test's Error Message, pytest's <code>E</code> lines, or the error with its Expected and Received values (Playwright, Jest, JUnit), not the runner's summary line.</p>
|
|
671
|
-
<p><strong>Keeping Scribe's cost down.</strong> ForgeQE reads the repository's test setup itself before Scribe starts (stack, settings files and their browser values, the test id attribute, base classes, page objects and their methods, one test command) and hands it over, so Scribe does not discover it with dozens of file reads. Each attempt after the first starts a fresh session from a summary of the job (handoff, files changed, what each attempt did, the latest failure), so later attempts cost about what the first did. <code>code.max_attempts</code> (default 3) and <code>code.max_tokens</code> (default 1,500,000; <code>0</code> turns it off) cap a job, and the token budget is checked during an attempt too, so one long attempt is stopped when the total passes it. The reviewer sorts its issues into correctness (the test does not test what the goal says, or not reliably) and coding standards (test data or credentials in a page object, naming, hard waits, leftovers). Coding standards never fail a test that works: it is accepted, and the coding agent gets one more round (not counted in <code>max_attempts</code>) to fix them; if that round breaks the test, ForgeQE puts back the version that passed and lists the standards issues in the notes (<code>code.tidy: false</code> skips the round). A test the reviewer rejects twice for correctness stops the job with its issues, since that usually means the test case and the page disagree. Under each step, the handoff shows that step's part of the page (its section and neighbours, from the replay's page snapshot) instead of whole snapshots. When a test fails, ForgeQE shows the failing element on the test's page at the failure and on the replay's page, so most failures are fixed without opening a browser. Scribe's live browser is switched on only when the same failure repeats, and it does not send a page snapshot after every click or keystroke. <code>code.max_steps</code> (default 50) caps the tool calls in one of Scribe's turns. <code>code.review_profile</code> runs the reviewer on a cheaper llms.json profile.</p>
|
|
672
|
-
<p><strong>Generated code.</strong> What ForgeQE writes is marked, so the team can see it and measure it. A file ForgeQE created starts with one line in that language's comment, such as <code>// Generated with ForgeQE · fq:k3x7m2qa4bdi · run 20261019-162847</code>; the <code>fq:</code> id is a hash of the file's contents without that line, so it shows whether the file was changed since (<code>code.header: false</code> leaves the line out; a JSON file never gets one). Files it only edited get no header. Every file it created or edited is recorded per run in <code>.forgeqe/generated.jsonl</code> (commit it with the code): the lines it added, as short hashes, never their text. <code>forgeqe generated</code> (or <code>/generated</code> in the shell, <code>--json</code> for a pipeline) says how many lines ForgeQE wrote, how many are still in the code, its share of the files it touched, and which created files are as it wrote them. Commit messages ForgeQE's agent writes end with a <code>Generated-by: ForgeQE (fq:…)</code> trailer, and its pull request descriptions with a line naming the runs' <code>fq:</code> ids and how much they wrote. The ids are plain hashes anyone can recompute: they show what ForgeQE wrote, not who owns the code.</p>
|
|
673
|
-
<p><code>forgeqe.yaml</code> selects the active profile:</p>
|
|
611
|
+
</div></section>
|
|
612
|
+
<section class="doc sub" id="doc-how_to_use-choosing-the-model-and-the-browser"><h4>Choosing the model and the browser</h4><div class="prose"><p><code>forgeqe.yaml</code> selects the active profile:</p>
|
|
674
613
|
<pre><code class="language-yaml">llm:
|
|
675
614
|
models_file: ./llms.json
|
|
676
615
|
active: azure-gpt4o
|
|
@@ -685,18 +624,15 @@ browser:
|
|
|
685
624
|
allowed_hosts: []
|
|
686
625
|
judge: false # true: a second model call checks each finished run and can fail it
|
|
687
626
|
overlay: true
|
|
627
|
+
window: maximized # visible browser: maximized | viewport (a window of viewport's size)
|
|
628
|
+
video: true # video.mp4 of each run, shown in its report
|
|
688
629
|
# start_url: https://staging.example.com
|
|
689
630
|
</code></pre>
|
|
690
631
|
<p><code>judge</code> is off by default. When it is on, one more model call reads the finished run's steps and screenshots and can fail a run the agent completed, for example when five scores were read off one results page rather than one step per item. That strictness sent runs into loops of re-checking, so turn it on only when you want that second opinion.</p>
|
|
691
632
|
<p>Each step waits for the tab the agent is on to finish loading (up to 8 seconds, 3 for a tab that stays blank) before the page is read, so a result that opens in a new tab is not mistaken for a blank page.</p>
|
|
692
633
|
<p>With a visible browser (<code>headless: false</code> or <code>--headed</code>), ForgeQE opens on its own start page, and every page it drives gets a glowing edge and a bar along the bottom showing, in plain words, what the step is for and what the agent is doing ("Step 19 · Check whether the results can be sorted by Fastest · Looking through the page's buttons"). Your mouse and keyboard are locked while the agent thinks and unlocked while it clicks and types, so a stray click can't change the test. None of this appears in the screenshots the model sees or in the report. <code>overlay: false</code> turns it off. <code>forgeqe login <name></code> shows the same bar without the lock, since you are the one signing in.</p>
|
|
634
|
+
<p>A visible browser opens maximized. <code>window: viewport</code> opens it at <code>viewport</code>'s width and height instead; a headless page is always that size (1280×800 by default). Each run is recorded to <code>video.mp4</code> in its run folder, and the report plays it on the run's page; <code>video: false</code> turns recording off. The video needs packages that <code>forgeqe setup</code> adds to the engine (an engine set up before this version gets them on its next run).</p>
|
|
693
635
|
<p><code>start_url</code> is optional. Leave it out and the agent picks the site from the goal; ForgeQE never guesses a URL from the goal text. <code>allowed_hosts</code> empty allows every host. Set <code>["example.com"]</code> to keep the browser on that domain and its subdomains. <code>prompts.step_instruction</code> is appended to the engine's system prompt; the lines of <code>prompts.guiderails</code> are guardrails for every agent (see Guardrails above).</p>
|
|
694
|
-
<p><code>llm.routing</code> is <code>off</code> by default for existing projects (<code>forgeqe init</code> writes <code>auto</code>); <code>shadow</code> logs each pick in the session log but changes nothing. <code>--profile</code> and <code>code.model</code> always win, and <code>llm.routes</code> limits or pins the profiles an area may use. ForgeQE checks and rates the models once and keeps the result in <code>.forgeqe/model-profile.json</code> (you can edit the ratings). Details: <a href="../docs/MODEL_ROUTING.md">docs/MODEL_ROUTING.md</a>.</p>
|
|
695
|
-
<p>List profiles, and the model each area gets:</p>
|
|
696
|
-
<pre><code class="language-bash">bun src/index.ts models
|
|
697
|
-
bun src/index.ts models --routes # the pick per area, and why
|
|
698
|
-
bun src/index.ts models --routes --profile-models # check and rate the models again
|
|
699
|
-
</code></pre>
|
|
700
636
|
</div></section>
|
|
701
637
|
</section>
|
|
702
638
|
</section>
|
|
@@ -752,7 +688,7 @@ Add a GitHub Actions workflow that runs tests/cases.yaml on every PR, and make i
|
|
|
752
688
|
<tr><td><code>/init</code></td><td>Set ForgeQE up in this folder: <code>forgeqe.yaml</code>, <code>llms.json</code>, editable prompts, skills, knowledge</td></tr>
|
|
753
689
|
<tr><td><code>/skills [name]</code></td><td>The skills each agent can use, or one skill's instructions</td></tr>
|
|
754
690
|
<tr><td><code>/db [test|tables|refresh|batches|examples|add] [name]</code></td><td>The databases set up here (see <a href="#doc-how_to_use-databases">Databases</a>)</td></tr>
|
|
755
|
-
<tr><td><code>/report</code>, <code>/report export</code>, <code>/report <ask></code></td><td>The team report of every run here and the ones teammates shared; your bundle to share; or ask the agent for a report, e.g. <code>/report open defects in all projects</code> (see <a href="
|
|
691
|
+
<tr><td><code>/report</code>, <code>/report export</code>, <code>/report <ask></code></td><td>The team report of every run here and the ones teammates shared; your bundle to share; or ask the agent for a report, e.g. <code>/report open defects in all projects</code> (see <a href="#doc-how_to_use-team-report">Team report</a>)</td></tr>
|
|
756
692
|
<tr><td><code>/auth [system]</code></td><td>Sign in to a system in the browser; <code>/auth</code> alone shows what is signed in</td></tr>
|
|
757
693
|
<tr><td><code>/clear</code></td><td>Clear the transcript</td></tr>
|
|
758
694
|
<tr><td><code>/help</code></td><td>Examples and commands</td></tr>
|
|
@@ -760,7 +696,7 @@ Add a GitHub Actions workflow that runs tests/cases.yaml on every PR, and make i
|
|
|
760
696
|
</tbody>
|
|
761
697
|
</table></div>
|
|
762
698
|
<p><code>start_url</code> in <code>forgeqe.yaml</code> or <code>--url</code> pins a start page for browser runs, such as a staging app. Passed runs become tests in the current folder; <code>forgeqe start --repo ../app</code> puts them in that repo instead, and <code>--no-code</code> turns tests off.</p>
|
|
763
|
-
<p>ForgeQE's agent runs on the agent runtime with the active profile's model (or <code>code.model</code>), or, with <code>llm.routing: auto</code>, the model the router picks for the <code>agent</code> area when the session starts (<code>--profile</code> and <code>code.model</code> still win); a <code>github_copilot</code> profile
|
|
699
|
+
<p>ForgeQE's agent runs on the agent runtime with the active profile's model (or <code>code.model</code>), or, with <code>llm.routing: auto</code>, the model the router picks for the <code>agent</code> area when the session starts (<code>--profile</code> and <code>code.model</code> still win); it cannot run on a <code>github_copilot</code> profile yet. ForgeQE downloads the agent runtime into <code>~/.forgeqe/tools</code> when it is not installed. When the agent runtime cannot be downloaded, ForgeQE says the agent is off and runs each message as one browser goal.</p>
|
|
764
700
|
<p>While a run is going, the bar under the prompt shows a spinner, the tokens used, and the elapsed time, and the transcript ends with what the agent is doing now (<code>Thinking about step 3…</code>, <code>Acting in the browser (step 3)…</code>). Each step starts with <code>+ Thought: 4.2s</code>, the time the model took to decide. When the run ends, a summary card closes the transcript:</p>
|
|
765
701
|
<pre><code>┃ ✓ Task complete 10 steps · 1m 42s · 65.3K tokens
|
|
766
702
|
┃
|
|
@@ -1106,17 +1042,6 @@ and are never recorded, and it stays above the page's dialogs. Recording uses lo
|
|
|
1106
1042
|
<h4>Test cases and suites</h4>
|
|
1107
1043
|
<div class="prose"><p>Test cases you already have can be run two ways. In the shell or with <code>forgeqe run</code>, ask in plain words ("run SHOP-12 and SHOP-14", "run test plan 12 suite 34", "run the cases in smoke.yaml") or paste the cases, and the agent works out the source. <code>forgeqe suite</code> takes explicit references and runs the same way every time, with no agent involved, which is what pipelines call. Each case is one goal: ForgeQE tells the agent to do the steps in order and to check every expected result, and the agent's verdict decides whether the case passed. A case with no steps (plain text, a Jira story) is run from its description.</p>
|
|
1108
1044
|
</div>
|
|
1109
|
-
<section class="doc sub" id="doc-how_to_use-team-report"><h5>Team report</h5><div class="prose"><p><code>forgeqe bi</code> builds one report from every run in the reports folder: browser suites, monitor rounds, release readiness, single browser runs, desktop and mobile runs, exploratory testing, and API runs, each tagged with its run type. It builds an <a href="https://github.com/evidence-dev/evidence" rel="noopener">Evidence</a> site (needs Node.js); without Node.js, or when Evidence fails, it writes ForgeQE's own page, which opens straight from disk (<code>bi:</code> in <code>forgeqe.yaml</code>; see <a href="../README.md#static-bi">README.md</a>).</p>
|
|
1110
|
-
<pre><code class="language-bash">forgeqe bi init # scaffold the Evidence project and install its packages
|
|
1111
|
-
forgeqe bi build # refresh the data and build (ForgeQE's own page if Evidence cannot build)
|
|
1112
|
-
forgeqe bi build --static # ForgeQE's own page only, no Node.js needed
|
|
1113
|
-
forgeqe bi sync # refresh the datasets and pages, without building
|
|
1114
|
-
forgeqe bi export [--out FILE|DIR] # your runs, in one file to share
|
|
1115
|
-
forgeqe bi build --from //share/qa-reports # this machine's runs plus every bundle there (repeatable)
|
|
1116
|
-
</code></pre>
|
|
1117
|
-
<p>For one report across the team, each person runs <code>forgeqe bi export</code>, which writes their runs and readiness reports to one file in <code>.forgeqe/reports/shared/</code>. Whoever builds puts the files in <code><out_dir>/collected/</code> (<code>.forgeqe/reports/collected/</code> by default), or lists shared folders under <code>bi.collect</code>, or passes <code>--from</code>, and runs <code>forgeqe bi build</code>. Every table gets <code>user</code> and <code>machine</code> columns; the report adds <strong>Results by run type</strong> and <strong>Case results by person</strong> tables. A run that arrives twice (in two bundles, or a bundle and this machine) counts once. A bundle is read as data only: a file that is not one, or is larger than 64 MB, is skipped with a note.</p>
|
|
1118
|
-
<p>In the shell, <code>/report</code> builds ForgeQE's own page from the same data, <code>/report export</code> writes your bundle, and <code>/report <what></code> asks the agent for a report (see <a href="#doc-how_to_use-reports-from-the-agent">Reports from the agent</a>).</p>
|
|
1119
|
-
</div></section>
|
|
1120
1045
|
<section class="doc sub" id="doc-how_to_use-api-steps-in-a-test-case"><h5>API steps in a test case</h5><div class="prose"><p>A test case can mix API and UI steps ("add three products with POST /add_to_cart/{id}, then remove the first one in the cart, then pay"). The browser agent makes each API step with its <code>api_request</code> action, from the page, in the browser's own session: the logged-in user's cookies go with the call, and the CSRF token comes from the cookie or the page's form when the site needs one. So what an API step changes (a cart, an order) is what the next UI step sees.</p>
|
|
1121
1046
|
<ul>
|
|
1122
1047
|
<li><strong>The verdict</strong> of an API step rests on the status the server answered, not on the agent's account of it.</li>
|
|
@@ -1577,7 +1502,11 @@ right engine, and carries values from one app to the next.</p>
|
|
|
1577
1502
|
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>generated_attribution</code></p></div>
|
|
1578
1503
|
</div>
|
|
1579
1504
|
<div class="detail"><span class="mono">In detail</span>
|
|
1580
|
-
<
|
|
1505
|
+
<section class="doc" id="doc-how_to_use-generated-code">
|
|
1506
|
+
<h4>Generated code</h4>
|
|
1507
|
+
<div class="prose"><p><strong>Generated code.</strong> What ForgeQE writes is marked, so the team can see it and measure it. A file ForgeQE created starts with one line in that language's comment, such as <code>// Generated with ForgeQE · fq:k3x7m2qa4bdi · run 20261019-162847</code>; the <code>fq:</code> id is a hash of the file's contents without that line, so it shows whether the file was changed since (<code>code.header: false</code> leaves the line out; a JSON file never gets one). Files it only edited get no header. Every file it created or edited is recorded per run in <code>.forgeqe/generated.jsonl</code> (commit it with the code): the lines it added, as short hashes, never their text. <code>forgeqe generated</code> (or <code>/generated</code> in the shell, <code>--json</code> for a pipeline) says how many lines ForgeQE wrote, how many are still in the code, its share of the files it touched, and which created files are as it wrote them. Commit messages ForgeQE's agent writes end with a <code>Generated-by: ForgeQE (fq:…)</code> trailer, and its pull request descriptions with a line naming the runs' <code>fq:</code> ids and how much they wrote. The ids are plain hashes anyone can recompute: they show what ForgeQE wrote, not who owns the code.</p>
|
|
1508
|
+
</div>
|
|
1509
|
+
</section>
|
|
1581
1510
|
</div>
|
|
1582
1511
|
</article>
|
|
1583
1512
|
<article class="spec feat" id="cap-forgeqe-commands">
|
|
@@ -1606,7 +1535,7 @@ Signing in stays <code>/auth</code> and <code>forgeqe login</code>; <code>forgeq
|
|
|
1606
1535
|
<tr><td><code>forgeqe init [--global] [--profile NAME]</code></td><td>Write <code>forgeqe.yaml</code>, <code>llms.json</code>, and <code>.forgeqe/</code> with the prompts, skills, and product knowledge templates</td></tr>
|
|
1607
1536
|
<tr><td><code>forgeqe setup [--force]</code></td><td>Download the agent runtime and the engine now (<code>--force</code> rebuilds the engine)</td></tr>
|
|
1608
1537
|
<tr><td><code>forgeqe models</code></td><td>List the profiles in <code>llms.json</code> and the active one</td></tr>
|
|
1609
|
-
<tr><td><code>forgeqe models --routes [--profile-models]</code></td><td>The model each area of work gets (<code>llm.routing</code>) and why; <code>--profile-models</code> checks and rates the models again (see <a href="
|
|
1538
|
+
<tr><td><code>forgeqe models --routes [--profile-models]</code></td><td>The model each area of work gets (<code>llm.routing</code>) and why; <code>--profile-models</code> checks and rates the models again (see <a href="#doc-docs-model_routing-model-routing-the-right-configured-model-for-each-task">docs/MODEL_ROUTING.md</a>)</td></tr>
|
|
1610
1539
|
<tr><td><code>forgeqe features</code></td><td>What ForgeQE can do, and what each capability still needs in this setup (works without a model)</td></tr>
|
|
1611
1540
|
<tr><td><code>forgeqe docs [--guide|--open] [--copy DIR]</code></td><td>The bundled docs; <code>--guide</code> opens the user guide, <code>--open</code> the architecture page</td></tr>
|
|
1612
1541
|
<tr><td><code>forgeqe mcp list</code></td><td>The MCP servers ForgeQE's agent gets from your settings</td></tr>
|
|
@@ -1625,7 +1554,7 @@ Signing in stays <code>/auth</code> and <code>forgeqe login</code>; <code>forgeq
|
|
|
1625
1554
|
<tr><td><code>forgeqe run --direct "<goal>" [--skill NAME]</code></td><td>One browser goal, no agent</td></tr>
|
|
1626
1555
|
<tr><td><code>forgeqe suite <source...> [--parallel N] [--skill NAME]</code></td><td>Run test cases the same way every time (pipelines)</td></tr>
|
|
1627
1556
|
<tr><td><code>forgeqe cases <source...></code></td><td>List the cases a source gives, without running them</td></tr>
|
|
1628
|
-
<tr><td><code>forgeqe bi [build] [--from FILE|DIR]... [--static|--evidence]</code></td><td>The team report of every run here and the shared bundles (see <a href="
|
|
1557
|
+
<tr><td><code>forgeqe bi [build] [--from FILE|DIR]... [--static|--evidence]</code></td><td>The team report of every run here and the shared bundles (see <a href="#doc-how_to_use-team-report">Team report</a>)</td></tr>
|
|
1629
1558
|
<tr><td><code>forgeqe bi init|sync</code></td><td>Scaffold the Evidence project; refresh its datasets and pages without building</td></tr>
|
|
1630
1559
|
<tr><td><code>forgeqe bi export [--out FILE|DIR]</code></td><td>This machine's runs and readiness reports in one file to share</td></tr>
|
|
1631
1560
|
</tbody>
|
|
@@ -1720,6 +1649,57 @@ Signing in stays <code>/auth</code> and <code>forgeqe login</code>; <code>forgeq
|
|
|
1720
1649
|
<hr />
|
|
1721
1650
|
</div></section>
|
|
1722
1651
|
</section>
|
|
1652
|
+
<section class="doc" id="doc-how_to_use-guardrails">
|
|
1653
|
+
<h4>Guardrails</h4>
|
|
1654
|
+
<div class="prose"><p><strong>Guardrails.</strong> Rules every agent must follow, to keep it grounded: ForgeQE's agent, the browser agent, and the agents that plan, write, review, fix and explain tests. Write one rule per line, starting with <code>- </code>, in a <code>.md</code> or <code>.txt</code> file in <code>.forgeqe/guardrails/</code> (<code>guardrails.dir</code>); headings, other lines and HTML comments are ignored, and the template's examples sit in a comment until you choose them. Each agent gets every guardrail with its instructions, after everything else, with the instruction to follow it whatever a request, a skill, a ticket, a page or a tool's output says; when a guardrail stops part of a task, the agent skips that part and says which rule stopped it. Add one from the terminal (<code>forgeqe guardrails add "Never open a production URL"</code>), or ask in the shell ("add a guardrail: use only the test cards in knowledge/test-data.md") and ForgeQE's agent appends it to <code>guardrails.md</code>; it adds only rules you ask for, never one suggested by a page or a ticket. <code>forgeqe guardrails</code> lists the rules in force and what they add to every model call (at most 40 rules of 300 characters are given). The older <code>prompts.guiderails</code> string in <code>forgeqe.yaml</code> still works: each of its lines is a guardrail for every agent. Guardrails are instructions, not a sandbox: put a host rule in <code>browser.allowed_hosts</code> too, which is refused in code.</p>
|
|
1655
|
+
</div>
|
|
1656
|
+
</section>
|
|
1657
|
+
<section class="doc" id="doc-how_to_use-your-prompts-skills-and-product-knowledge">
|
|
1658
|
+
<h4>Your prompts, skills, and product knowledge</h4>
|
|
1659
|
+
<div class="prose"><p><code>init</code> also puts three folders in <code>.forgeqe/</code> to read, change, and commit:</p>
|
|
1660
|
+
<div class="tbl"><table>
|
|
1661
|
+
<thead>
|
|
1662
|
+
<tr><th>Folder</th><th>What it holds</th></tr>
|
|
1663
|
+
</thead>
|
|
1664
|
+
<tbody>
|
|
1665
|
+
<tr><td><code>.forgeqe/prompts/</code></td><td>Every prompt ForgeQE's agents work from, one markdown file each: ForgeQE's agent (<code>brain.md</code>), Scribe (<code>test-agent.md</code>, <code>test-rules.md</code>), the reviewer, healing, finding tests, pull request review, API tests and scenarios, exploratory charters, and the browser agent (<code>browser-*.md</code>, plus <code>browser-extra.md</code> for your own instructions).</td></tr>
|
|
1666
|
+
<tr><td><code>.forgeqe/skills/</code></td><td>The built-in skills (framework and app playbooks), to adapt to your project; add your own next to them.</td></tr>
|
|
1667
|
+
<tr><td><code>.forgeqe/knowledge/</code></td><td>Product knowledge templates: the product and its flows, roles, glossary, business rules, test data, environments.</td></tr>
|
|
1668
|
+
</tbody>
|
|
1669
|
+
</table></div>
|
|
1670
|
+
<p><strong>Your copy or ForgeQE's.</strong> <code>.forgeqe-manifest.json</code> in each folder records what was copied. A copy you have not changed still counts as ForgeQE's, so it keeps getting ForgeQE's improvements when you update; once you edit it, yours is used. When ForgeQE's version changes after you edited yours, ForgeQE says so at start-up. Running <code>init</code> again adds what is missing and never overwrites an edited file (with <code>--force</code> it refreshes only unchanged copies).</p>
|
|
1671
|
+
<p><strong>Prompts.</strong> Each file has a short header (<code>name</code>, <code>description</code>, <code>used_by</code>, <code>kind</code>) and the prompt; <code>{{> name}}</code> includes another prompt (<code>test-rules</code> is included in the test, reviewer, and healing prompts, so one edit changes all three). ForgeQE looks in <code>prompts.dir</code>, then <code>.forgeqe/prompts/</code>, then <code>~/.forgeqe/prompts/</code> (for every project), then its own. Changes take effect on the next run or session.</p>
|
|
1672
|
+
<pre><code class="language-bash">forgeqe prompts # every prompt, what it is for, and where it comes from
|
|
1673
|
+
forgeqe prompts show test-agent # the prompt as the agent gets it (includes expanded; --raw for the file)
|
|
1674
|
+
forgeqe prompts diff test-rules # how your copy differs from ForgeQE's
|
|
1675
|
+
forgeqe prompts reset test-rules # back to ForgeQE's (yours is kept as test-rules.md.bak)
|
|
1676
|
+
forgeqe prompts check # each prompt against the prompting standard for its kind (exits 1 when one falls short)
|
|
1677
|
+
forgeqe prompts check brain # what to fix in one prompt: sections, rules, examples, contract, sentences
|
|
1678
|
+
forgeqe prompts eval # run the evaluation cases with the live model; pass rates against the last run
|
|
1679
|
+
forgeqe prompts eval refine --runs 3 # one prompt, three runs of each case
|
|
1680
|
+
</code></pre>
|
|
1681
|
+
<p><strong>The prompting standard.</strong> Every prompt follows one standard, and each technique is used where it fits the kind of prompt (<code>kind:</code> in its header):</p>
|
|
1682
|
+
<div class="tbl"><table>
|
|
1683
|
+
<thead>
|
|
1684
|
+
<tr><th>Kind</th><th>Prompts</th><th>Structure</th><th>Few-shot examples</th><th>Execution contract</th></tr>
|
|
1685
|
+
</thead>
|
|
1686
|
+
<tbody>
|
|
1687
|
+
<tr><td>agent: works with tools over many steps</td><td>brain, test-agent, heal, api-tests, pr-review, find-tests</td><td>Role, Objective, Execution contract, Rules, Output</td><td>where routing or reply format varies</td><td>Preconditions, Tools, Procedure, Stop when, On failure, Evidence</td></tr>
|
|
1688
|
+
<tr><td>structured: one answer in a fixed shape</td><td>refine, api-scenarios, explore-charters, eval-judge, test-reviewer, browser-skill-select</td><td>Role, Objective, Inputs, Rules, Output, Examples</td><td>two or more worked examples</td><td>–</td></tr>
|
|
1689
|
+
<tr><td>fragment: rules included in other prompts</td><td>test-rules, domain-rule, skill-rule, conventions, the browser agent's rules</td><td>rules only</td><td>–</td><td>–</td></tr>
|
|
1690
|
+
</tbody>
|
|
1691
|
+
</table></div>
|
|
1692
|
+
<ul>
|
|
1693
|
+
<li><strong>ASD-STE100</strong> Simplified Technical English everywhere, so each agent reads one clear instruction at a time: an instruction has 20 words or fewer and a description 25 or fewer, one instruction per sentence, the active voice, and approved words ("make sure", not "ensure"). At least 80% of sentences. The check covers the rules that can be measured in text and a list of common words the standard does not approve; the standard's full dictionary is licensed and not included.</li>
|
|
1694
|
+
<li><strong>RFC 2119</strong> key words state the strength of every rule: <strong>MUST</strong> / <strong>MUST NOT</strong> are absolute, <strong>SHOULD</strong> / <strong>SHOULD NOT</strong> are strong defaults the agent may break only for a reason it states, <strong>MAY</strong> is a free choice, and a numbered procedure step is a MUST. A lower-case "must" or "should" is flagged as ambiguous. <code>conventions.md</code> says this at the top of every agent and structured prompt, and the browser engine adds it to the browser agent's instructions.</li>
|
|
1695
|
+
<li><strong>Structured prompting and few-shot examples</strong> make the output predictable: the same sections in the same order, and worked examples where the shape of the answer matters.</li>
|
|
1696
|
+
<li><strong>Execution contracts</strong> tell a tool-using agent what must be true before it starts, which tools it uses and in what order (for systems: MCP tools, then <code>query_api</code>, then the browser), its procedure, when it stops, what it does when something fails, and what evidence its answer gives.</li>
|
|
1697
|
+
<li><strong>Evaluation-driven prompting</strong> measures reliability: <code>forgeqe prompts eval</code> runs cases (an input and checks decided by code: a field matches or is empty, a tool was or was not used, the text keeps ASD-STE100) and compares each pass rate with the last run. ForgeQE's cases cover brain, refine, and explore-charters; add your own as <code>.forgeqe/evals/prompts/<prompt>/<case>.yaml</code> (the same id replaces ForgeQE's). Results are kept in <code><reports>/evals/prompts/</code>.</li>
|
|
1698
|
+
</ul>
|
|
1699
|
+
<p><code>forgeqe prompts check</code> applies the standard to each prompt in use, ForgeQE's or your edited copy; ForgeQE's own meet it in its tests. Write your edits the same way, and run <code>forgeqe prompts eval</code> before and after a change.</p>
|
|
1700
|
+
<p><strong>Product knowledge.</strong> Fill in the templates (or add your own <code>.md</code> and <code>.txt</code> files, in folders if you like), one topic per <code>##</code> heading. ForgeQE's agent sees the list and reads what a request needs; the exploratory and API scenario planners get the sections that match their story or scenario; the browser agent gets the few sections that clearly match its goal, at most <code>knowledge.browser_max_chars</code> (1500 by default; 0 turns it off). Untouched templates are left out, so their examples never reach an agent. Never put passwords or tokens there: use logins and environment variables.</p>
|
|
1701
|
+
</div>
|
|
1702
|
+
</section>
|
|
1723
1703
|
</div>
|
|
1724
1704
|
</article>
|
|
1725
1705
|
<article class="spec feat" id="cap-setup-help">
|
|
@@ -1791,6 +1771,145 @@ agents: [browser]
|
|
|
1791
1771
|
<p>In the shell you can also ask: "make a skill for our Dynamics date picker" or "teach Scribe our test conventions". ForgeQE's agent gathers the facts, writes the skill into <code>.forgeqe/skills</code> with <code>forgeqe_setup</code>, and it applies from the next session. It writes skills only when you ask.</p>
|
|
1792
1772
|
</div>
|
|
1793
1773
|
</section>
|
|
1774
|
+
<section class="doc" id="doc-how_to_use-model-routing">
|
|
1775
|
+
<h4>Model routing</h4>
|
|
1776
|
+
<div class="prose"><p><code>llm.routing</code> is <code>off</code> by default for existing projects (<code>forgeqe init</code> writes <code>auto</code>); <code>shadow</code> logs each pick in the session log but changes nothing. <code>--profile</code> and <code>code.model</code> always win, and <code>llm.routes</code> limits or pins the profiles an area may use. ForgeQE checks and rates the models once and keeps the result in <code>.forgeqe/model-profile.json</code> (you can edit the ratings). Details: <a href="#doc-docs-model_routing-model-routing-the-right-configured-model-for-each-task">docs/MODEL_ROUTING.md</a>.</p>
|
|
1777
|
+
<p>List profiles, and the model each area gets:</p>
|
|
1778
|
+
<pre><code class="language-bash">bun src/index.ts models
|
|
1779
|
+
bun src/index.ts models --routes # the pick per area, and why
|
|
1780
|
+
bun src/index.ts models --routes --profile-models # check and rate the models again
|
|
1781
|
+
</code></pre>
|
|
1782
|
+
</div>
|
|
1783
|
+
</section>
|
|
1784
|
+
<section class="doc" id="doc-readme-privacy">
|
|
1785
|
+
<h4>Privacy</h4>
|
|
1786
|
+
<div class="prose"><p>ForgeQE sends no telemetry and turns it off in everything it starts:</p>
|
|
1787
|
+
<ul>
|
|
1788
|
+
<li>The engine: PostHog telemetry, cloud sync, and the price-list download are off, and the blank-tab logo its library loads from a CDN is replaced by ForgeQE's start page, which loads nothing.</li>
|
|
1789
|
+
<li>The agent runtime: auto-update, session sharing, OpenTelemetry spans, and the model catalogue download are off; it uses its bundled model list.</li>
|
|
1790
|
+
<li>The repo's test command: <code>DO_NOT_TRACK=1</code>, plus the opt-outs for Next.js, Nuxt, Astro, Gatsby, Storybook, Turborepo, Angular, .NET, and Cypress.</li>
|
|
1791
|
+
</ul>
|
|
1792
|
+
<p><code>OTEL_EXPORTER_OTLP_*</code> variables are removed from those processes, and your environment cannot turn any of this back on. The traffic is to your model endpoint and the sites the agent visits. Besides that, ForgeQE downloads only public files: the model price lists from models.dev and LiteLLM, once a day (<code>FORGEQE_PRICES=off</code> stops them); the agent runtime from its GitHub releases, when it is not installed; and the engine's ad-block and cookie-banner extensions from the Chrome Web Store, once. Nothing about your runs is sent.</p>
|
|
1793
|
+
</div>
|
|
1794
|
+
</section>
|
|
1795
|
+
<section class="doc" id="doc-docs-model_routing-model-routing-the-right-configured-model-for-each-task">
|
|
1796
|
+
<h4>Model routing: the right configured model for each task</h4>
|
|
1797
|
+
<div class="prose"><p>With more than one model in <code>llms.json</code>, ForgeQE can give each kind of work the model it needs. A small, fast,
|
|
1798
|
+
cheap model can plan and review. A strong model can write code and find root causes. A model that takes screenshots
|
|
1799
|
+
drives the browser, and a model with a large window reads a large pull request. You do not need to know what each
|
|
1800
|
+
model can do: ForgeQE finds out.</p>
|
|
1801
|
+
</div>
|
|
1802
|
+
</section>
|
|
1803
|
+
<section class="doc" id="doc-docs-model_routing-turn-it-on">
|
|
1804
|
+
<h4>Turn it on</h4>
|
|
1805
|
+
<div class="prose"><p>In <code>forgeqe.yaml</code>:</p>
|
|
1806
|
+
<pre><code class="language-yaml">llm:
|
|
1807
|
+
models_file: ./llms.json
|
|
1808
|
+
routing: auto # off (the default for existing projects) | shadow | auto
|
|
1809
|
+
</code></pre>
|
|
1810
|
+
<p><code>forgeqe init</code> writes <code>routing: auto</code> for new projects. <code>shadow</code> decides and logs each pick in the session log
|
|
1811
|
+
(<code>model.route</code>) but changes nothing, so you can see what the router would do before you let it. Shadow still runs the
|
|
1812
|
+
live checks and the rating call described below.</p>
|
|
1813
|
+
<p>See the picks:</p>
|
|
1814
|
+
<pre><code>forgeqe models --routes
|
|
1815
|
+
</code></pre>
|
|
1816
|
+
</div>
|
|
1817
|
+
</section>
|
|
1818
|
+
<section class="doc" id="doc-docs-model_routing-the-areas-of-work">
|
|
1819
|
+
<h4>The areas of work</h4>
|
|
1820
|
+
<div class="prose"><div class="tbl"><table>
|
|
1821
|
+
<thead>
|
|
1822
|
+
<tr><th>Area</th><th>What runs in it</th><th>Needs</th></tr>
|
|
1823
|
+
</thead>
|
|
1824
|
+
<tbody>
|
|
1825
|
+
<tr><td><code>agent</code></td><td>ForgeQE's agent: the shell, <code>forgeqe run</code>, <code>forgeqe mcp</code></td><td>tool calls; rated 4+</td></tr>
|
|
1826
|
+
<tr><td><code>browser</code></td><td>Browser, desktop and mobile runs, from screenshots</td><td>images; rated 4+</td></tr>
|
|
1827
|
+
<tr><td><code>coding</code></td><td>Scribe writing and healing tests, API tests</td><td>tool calls; rated 5</td></tr>
|
|
1828
|
+
<tr><td><code>review</code></td><td>Test reviewer, pull-request review, evaluation judge</td><td>tool calls; rated 3+</td></tr>
|
|
1829
|
+
<tr><td><code>analysis</code></td><td>Root cause of defects</td><td>tool calls; rated 5</td></tr>
|
|
1830
|
+
<tr><td><code>planning</code></td><td>Refine, finding tests, API scenarios, exploratory charters</td><td>tool calls; rated 3+</td></tr>
|
|
1831
|
+
</tbody>
|
|
1832
|
+
</table></div>
|
|
1833
|
+
</div>
|
|
1834
|
+
</section>
|
|
1835
|
+
<section class="doc" id="doc-docs-model_routing-how-a-model-is-picked">
|
|
1836
|
+
<h4>How a model is picked</h4>
|
|
1837
|
+
<div class="prose"><ol>
|
|
1838
|
+
<li>
|
|
1839
|
+
<p><strong>Your choices win.</strong> <code>--profile</code> (or <code>WP_LLM_PROFILE</code>) and <code>code.model</code> keep the router out. A profile pinned in
|
|
1840
|
+
<code>llm.routes</code> is used for its area. <code>code.review_profile</code> still sets the reviewer.</p>
|
|
1841
|
+
</li>
|
|
1842
|
+
<li>
|
|
1843
|
+
<p><strong>Facts rule models out.</strong> The router never picks a model that:</p>
|
|
1844
|
+
<ul>
|
|
1845
|
+
<li>has no key, or failed its live check;</li>
|
|
1846
|
+
<li>takes no images, for browser work;</li>
|
|
1847
|
+
<li>takes no tool calls, for agent work;</li>
|
|
1848
|
+
<li>has a window too small for the prompt (a large pull-request diff, for example).</li>
|
|
1849
|
+
</ul>
|
|
1850
|
+
<p>The facts come from <code>llms.json</code>, a live check, and the public model lists (models.dev and LiteLLM) that ForgeQE
|
|
1851
|
+
already downloads for prices. A fact that nobody knows never rules a model out.</p>
|
|
1852
|
+
</li>
|
|
1853
|
+
<li>
|
|
1854
|
+
<p><strong>Ratings pick among the rest.</strong> The cheapest model rated at the area's bar is picked. Writing code and finding
|
|
1855
|
+
root causes ask for 5, because a weaker model there costs more in failed attempts than it saves. If no model reaches the bar,
|
|
1856
|
+
the best-rated one is picked. With no ratings yet, the active profile is kept unless a fact rules it out.</p>
|
|
1857
|
+
</li>
|
|
1858
|
+
<li>
|
|
1859
|
+
<p><strong>A failed answer escalates once.</strong> When a root-cause analysis finds nothing, or finds a cause with low confidence
|
|
1860
|
+
although it could read the product's code, it runs again on a stronger model: one rated higher for the area, or rated
|
|
1861
|
+
the same and priced higher. Both runs are counted in its cost.</p>
|
|
1862
|
+
</li>
|
|
1863
|
+
</ol>
|
|
1864
|
+
<p>The same configuration and task always give the same model. A pick changes only when the models, the ratings or the
|
|
1865
|
+
size of the prompt change. Each pick is shown once ("Model for planning: mini (…)") and is written to the session log.</p>
|
|
1866
|
+
</div>
|
|
1867
|
+
</section>
|
|
1868
|
+
<section class="doc" id="doc-docs-model_routing-how-forgeqe-learns-the-models">
|
|
1869
|
+
<h4>How ForgeQE learns the models</h4>
|
|
1870
|
+
<div class="prose"><p>The first routed call in a project with two or more usable profiles does this once:</p>
|
|
1871
|
+
<ol>
|
|
1872
|
+
<li><strong>A live check of each model</strong> (<code>python -m forgeqe_engine probe</code>): one short structured answer and one image.
|
|
1873
|
+
This finds a deployment that does not exist, a wrong key, and whether the model really takes images. It also
|
|
1874
|
+
measures how fast the model answers.</li>
|
|
1875
|
+
<li><strong>One rating call.</strong> The active profile's model receives the facts of every model, never a key, an endpoint or an
|
|
1876
|
+
account. It rates each model 1–5 for each area and gives it a tier (fast, balanced or strong).</li>
|
|
1877
|
+
</ol>
|
|
1878
|
+
<p>The result is kept in <code>.forgeqe/model-profile.json</code> next to <code>forgeqe.yaml</code>. ForgeQE uses it until the profiles in
|
|
1879
|
+
<code>llms.json</code> change (provider, model, deployment or endpoint). You can edit the ratings. ForgeQE keeps your edits while
|
|
1880
|
+
the profiles stay the same. It checks the file every time it reads it: names that are not profiles and unknown areas
|
|
1881
|
+
are dropped, and ratings are rounded to whole numbers from 1 to 5.</p>
|
|
1882
|
+
<p>The live checks reach every profile in <code>llms.json</code> that has a key, whatever <code>llm.routes</code> pins. A pin keeps your runs'
|
|
1883
|
+
data on one deployment; the checks carry nothing from your runs. To send nothing to the other profiles at all, use
|
|
1884
|
+
<code>routing: off</code>, keep one profile with a key, or pass <code>--profile</code>.</p>
|
|
1885
|
+
<p>To check and rate again (for example, after a deployment changed its model):</p>
|
|
1886
|
+
<pre><code>forgeqe models --routes --profile-models
|
|
1887
|
+
</code></pre>
|
|
1888
|
+
</div>
|
|
1889
|
+
</section>
|
|
1890
|
+
<section class="doc" id="doc-docs-model_routing-limit-what-an-area-may-use">
|
|
1891
|
+
<h4>Limit what an area may use</h4>
|
|
1892
|
+
<div class="prose"><pre><code class="language-yaml">llm:
|
|
1893
|
+
routing: auto
|
|
1894
|
+
routes:
|
|
1895
|
+
browser: [gpt4o, gpt54] # only these two, the router picks between them
|
|
1896
|
+
coding: gpt54 # pinned
|
|
1897
|
+
planning: auto # any configured model (the default)
|
|
1898
|
+
</code></pre>
|
|
1899
|
+
<p>A name in <code>routes</code> must be a profile in <code>llms.json</code>, and an area must be one of the six above. Anything else stops
|
|
1900
|
+
ForgeQE at start with a message that names it.</p>
|
|
1901
|
+
</div>
|
|
1902
|
+
</section>
|
|
1903
|
+
<section class="doc" id="doc-docs-model_routing-what-it-does-not-do">
|
|
1904
|
+
<h4>What it does not do</h4>
|
|
1905
|
+
<div class="prose"><ul>
|
|
1906
|
+
<li>ForgeQE's agent picks its model when its session starts and keeps it for the whole conversation. Changing the model
|
|
1907
|
+
mid-conversation would drop the prompt cache and make every later turn cost more. The agent hands heavy work to the
|
|
1908
|
+
other agents, and those are routed.</li>
|
|
1909
|
+
<li>A rating is one model's judgement. It is kept in a file that you can read and correct.</li>
|
|
1910
|
+
</ul>
|
|
1911
|
+
</div>
|
|
1912
|
+
</section>
|
|
1794
1913
|
</div>
|
|
1795
1914
|
</article>
|
|
1796
1915
|
</section>
|
|
@@ -1826,10 +1945,460 @@ next to the workflow), with a tolerance and regions to ignore. When you know the
|
|
|
1826
1945
|
"I'll show you, record it" in the shell) saves what you do by hand: Ctrl+Shift+F11 over a control adds a check of its
|
|
1827
1946
|
text, Ctrl+Shift+F12 finishes. After a failed replay, ForgeQE's agent can fix the workflow (a longer timeout, a
|
|
1828
1947
|
<code>wait_for</code>, a dialog to dismiss): it shows the diff and writes it only when you agree. For nightly runs on a machine
|
|
1829
|
-
of your own, see <a href="
|
|
1948
|
+
of your own, see <a href="#doc-docs-desktop-ci-running-desktop-workflows-unattended-nightly-ci">docs/DESKTOP-CI.md</a>.</p>
|
|
1830
1949
|
<p>Apps come from <code>desktop.apps</code> in forgeqe.yaml (an absolute <code>.exe</code> and an argument list), and <code>{{secret:NAME}}</code> reads
|
|
1831
1950
|
only the variables listed in <code>desktop.secrets</code>. Explore and replay take over the desktop's mouse and keyboard and ask
|
|
1832
|
-
first; use a test machine and a test account. Details: <a href="
|
|
1951
|
+
first; use a test machine and a test account. Details: <a href="#doc-docs-desktop-forgeqe-desktop-thick-client-automation">docs/DESKTOP.md</a>.</p>
|
|
1952
|
+
</div>
|
|
1953
|
+
</section>
|
|
1954
|
+
<section class="doc" id="doc-docs-desktop-forgeqe-desktop-thick-client-automation">
|
|
1955
|
+
<h4>ForgeQE desktop: thick-client automation</h4>
|
|
1956
|
+
<div class="prose"><p>Status: runs on Windows. The CI job <code>.github/workflows/forgeqe-desktop-windows.yml</code> (GitHub's <code>windows-2022</code>) installs the
|
|
1957
|
+
engine and the pinned UFO², replays Notepad live with the pywinauto driver and through <code>forgeqe desktop replay</code>, heals a
|
|
1958
|
+
moved control from the live window, and runs a whole explore: UFO² drives Notepad with a scripted model behind an
|
|
1959
|
+
OpenAI-compatible server, the action is recorded with the control's AutomationId, and the recording replays. Not yet
|
|
1960
|
+
run: explore with a real model (the job has no key), and your own apps. See
|
|
1961
|
+
<a href="#doc-docs-desktop-before-first-use-on-windows">Before first use on Windows</a>.</p>
|
|
1962
|
+
<p>ForgeQE drives web apps through Chrome. The desktop engine does the same for Windows thick clients (Win32, WinForms,
|
|
1963
|
+
WPF, and the native parts of Electron and WebView2 apps): say what you want in plain language, UFO² does it on the live
|
|
1964
|
+
app, and the run is saved as a YAML workflow that replays later without a model.</p>
|
|
1965
|
+
</div>
|
|
1966
|
+
</section>
|
|
1967
|
+
<section class="doc" id="doc-docs-desktop-use">
|
|
1968
|
+
<h4>Use</h4>
|
|
1969
|
+
<div class="prose"><pre><code class="language-bash">forgeqe desktop setup --ufo # once, on Windows: the engine, pywinauto, and the pinned UFO²
|
|
1970
|
+
forgeqe desktop apps # what forgeqe.yaml allows
|
|
1971
|
+
forgeqe desktop explore "Create an invoice for Contoso, net 30" --app billing
|
|
1972
|
+
forgeqe desktop record --app billing # or do the steps yourself: no model, no UFO
|
|
1973
|
+
forgeqe desktop replay .forgeqe/reports/desktop-explore-…/workflow.yaml customer=Fabrikam
|
|
1974
|
+
forgeqe desktop replay workflow.yaml --data customers.csv # once per row; the header names the variables
|
|
1975
|
+
forgeqe desktop replay desktop-tests/ --junit results.xml # every workflow in a folder, results for CI
|
|
1976
|
+
forgeqe desktop replay workflow.yaml --update-baselines # save new baselines for its expect_image steps
|
|
1977
|
+
forgeqe desktop doctor # can this machine run replay unattended?
|
|
1978
|
+
forgeqe desktop validate workflow.yaml # any OS
|
|
1979
|
+
</code></pre>
|
|
1980
|
+
<p>Or ask ForgeQE's agent in the shell or <code>forgeqe run</code>: "create an invoice for Contoso in Billing and save it as a test",
|
|
1981
|
+
or "I'll show you the steps in Billing, record them". Its tools are <code>test_apps</code>, <code>desktop_explore</code>, <code>desktop_record</code>,
|
|
1982
|
+
<code>desktop_replay</code> and <code>desktop_edit_workflow</code>. It asks before it starts the app or takes over the desktop, and before it
|
|
1983
|
+
changes a workflow. Where it cannot ask (an MCP client), the call answers <code>needs_confirmation</code> until the user agrees.</p>
|
|
1984
|
+
<p>Desktop apps come from the active environment in <code>env.yaml</code> (see HOW_TO_USE.md, "Environments and apps"), or from
|
|
1985
|
+
<code>desktop.apps</code> in <code>forgeqe.yaml</code>, which works without environments:</p>
|
|
1986
|
+
<pre><code class="language-yaml">desktop:
|
|
1987
|
+
apps:
|
|
1988
|
+
billing:
|
|
1989
|
+
path: C:\Program Files\Contoso\Billing\Billing.exe
|
|
1990
|
+
args: ["--profile", "test"]
|
|
1991
|
+
# window: "Billing.*" # for a launcher that starts the real app and exits: connect by title
|
|
1992
|
+
secrets: [BILLING_PASSWORD] # the only environment variables {{secret:…}} may read
|
|
1993
|
+
heal: true # replay asks the model to re-find a control that moved (default on)
|
|
1994
|
+
screenshots: true # one screenshot per replayed step (default on)
|
|
1995
|
+
</code></pre>
|
|
1996
|
+
<p>Each run writes a folder under the reports folder (<code>desktop-explore-…</code> or <code>desktop-replay-…</code>): <code>run.json</code>,
|
|
1997
|
+
<code>report.html</code>, <code>steps/NNN.png</code>, and for explore <code>workflow.yaml</code>, <code>trace.json</code> and UFO's own logs in <code>ufo/</code>. A replay
|
|
1998
|
+
whose steps were healed also writes <code>heals.json</code> and asks whether to write the new locators into the workflow
|
|
1999
|
+
(<code>--accept-heals</code> writes them without asking; with no terminal they are never written).</p>
|
|
2000
|
+
</div>
|
|
2001
|
+
</section>
|
|
2002
|
+
<section class="doc" id="doc-docs-desktop-how-a-run-flows">
|
|
2003
|
+
<h4>How a run flows</h4>
|
|
2004
|
+
<div class="prose"><pre><code>forgeqe desktop explore ──► src/desktop/bridge.ts ──JSON lines──► python -m forgeqe_desktop serve
|
|
2005
|
+
│ start the app from desktop.apps (no shell)
|
|
2006
|
+
│ UFO² in-process: model = llms.json profile,
|
|
2007
|
+
│ UI tools only, every action traced with its
|
|
2008
|
+
▼ control's UIA properties
|
|
2009
|
+
record.py ──► workflow.yaml
|
|
2010
|
+
forgeqe desktop replay ───► bridge ──► serve (replay.py on pywinauto, no model)
|
|
2011
|
+
│ a control is not found
|
|
2012
|
+
▼
|
|
2013
|
+
heal.py: the model picks one of the window's real controls, by number
|
|
2014
|
+
▼
|
|
2015
|
+
proposed locator → heals.json, written into the workflow only when accepted
|
|
2016
|
+
</code></pre>
|
|
2017
|
+
</div>
|
|
2018
|
+
</section>
|
|
2019
|
+
<section class="doc" id="doc-docs-desktop-one-model-config-llmsjson--ufo">
|
|
2020
|
+
<h4>One model config: <code>llms.json</code> → UFO</h4>
|
|
2021
|
+
<div class="prose"><p>UFO picks its model client by <code>API_TYPE</code> through <code>ufo.llm.base.BaseService.get_service</code>, which every model call goes
|
|
2022
|
+
through (<code>ufo/llm/llm_call.py</code>). The adapter adds the type <code>forgeqe</code>:</p>
|
|
2023
|
+
<ul>
|
|
2024
|
+
<li><code>write_ufo_config</code> writes UFO's <code>config/ufo/agents.yaml</code> from its template, with <code>API_TYPE: forgeqe</code>, the model's name,
|
|
2025
|
+
and no endpoint and no key, for <code>HOST_AGENT</code>, <code>APP_AGENT</code>, <code>BACKUP_AGENT</code> and <code>EVALUATION_AGENT</code>.</li>
|
|
2026
|
+
<li><code>ufo_bridge.install</code> makes <code>get_service("forgeqe", …)</code> return <code>ForgeQEService</code>, which calls the profile through the
|
|
2027
|
+
<code>openai</code> SDK (<code>models.py</code>): <code>azure</code> (key or <code>auth: entra</code>), <code>openai</code>, <code>openai_compatible</code> (key or Entra), and <code>ollama</code>
|
|
2028
|
+
through its OpenAI-compatible <code>/v1</code>. Keys, Azure endpoints and Entra use the browser engine's own helpers
|
|
2029
|
+
(<code>forgeqe_engine.llm</code>), so a profile means the same thing in both. Keys stay in environment variables.</li>
|
|
2030
|
+
<li>Each call's time and tokens go to ForgeQE's usage ledger as the "desktop agent" (counts only, never the prompt).</li>
|
|
2031
|
+
</ul>
|
|
2032
|
+
<p>Explore sends screenshots, so its profile must take images: <code>"vision": true</code> or <code>false</code> on the profile says so; without
|
|
2033
|
+
it, <code>ollama</code> profiles are refused and others accepted. Healing sends a screenshot only to a profile that takes images.</p>
|
|
2034
|
+
<p>The request parameters suit the model by themselves: a reasoning model (o1, o3, o4, gpt-5, by the profile's <code>model</code> or
|
|
2035
|
+
<code>deployment</code> name, or <code>"reasoning": true</code> on the profile) gets <code>max_completion_tokens</code> and no temperature from the first
|
|
2036
|
+
call, and when the API refuses a parameter (<code>max_tokens</code>, <code>temperature</code>, <code>frequency_penalty</code>, <code>response_format</code>, …) the
|
|
2037
|
+
call is sent again without it, and later calls to that model leave it out (<code>py/forgeqe_engine/model_params.py</code>). The
|
|
2038
|
+
browser agent does the same.</p>
|
|
2039
|
+
</div>
|
|
2040
|
+
</section>
|
|
2041
|
+
<section class="doc" id="doc-docs-desktop-workflow-yaml-forgeqe-desktop-v1">
|
|
2042
|
+
<h4>Workflow YAML (<code>forgeqe-desktop-v1</code>)</h4>
|
|
2043
|
+
<div class="prose"><pre><code class="language-yaml">format: forgeqe-desktop-v1
|
|
2044
|
+
name: Create an invoice
|
|
2045
|
+
app: billing # a name from forgeqe.yaml desktop.apps, never a path or a command
|
|
2046
|
+
variables:
|
|
2047
|
+
customer: Contoso
|
|
2048
|
+
steps:
|
|
2049
|
+
- op: launch
|
|
2050
|
+
- op: click
|
|
2051
|
+
target: {automation_id: btnNewInvoice, control_type: Button, name: New invoice}
|
|
2052
|
+
- op: type
|
|
2053
|
+
target: {automation_id: txtCustomer, control_type: Edit}
|
|
2054
|
+
text: "{{customer}}"
|
|
2055
|
+
- op: type
|
|
2056
|
+
target: {automation_id: txtPassword, control_type: Edit}
|
|
2057
|
+
text: "{{secret:BILLING_PASSWORD}}"
|
|
2058
|
+
- op: select
|
|
2059
|
+
target: {automation_id: cmbTerms, control_type: ComboBox}
|
|
2060
|
+
value: Net 30
|
|
2061
|
+
- op: keys
|
|
2062
|
+
keys: ctrl+s
|
|
2063
|
+
- op: expect
|
|
2064
|
+
target: {automation_id: lblStatus}
|
|
2065
|
+
text: Saved
|
|
2066
|
+
- op: close
|
|
2067
|
+
</code></pre>
|
|
2068
|
+
<ul>
|
|
2069
|
+
<li><strong>Ops:</strong> <code>launch</code>, <code>click</code>, <code>double_click</code>, <code>type</code>, <code>select</code>, <code>keys</code>, <code>expect</code>, <code>expect_image</code>, <code>wait</code> (at most
|
|
2070
|
+
60 s), <code>wait_for</code>, <code>close</code>.</li>
|
|
2071
|
+
<li><strong>Locators</strong> use UIA properties, never screen coordinates: <code>automation_id</code>, <code>name</code>, <code>control_type</code>, <code>class_name</code>,
|
|
2072
|
+
<code>index</code> (when several match), and <code>path</code> (the ancestors' <code>control_type</code> and <code>name</code>). A target needs an
|
|
2073
|
+
<code>automation_id</code> or a <code>name</code>. Replay looks in the launched app's own windows only.</li>
|
|
2074
|
+
<li><strong>Variables:</strong> <code>{{name}}</code> from the workflow's <code>variables</code> or <code>name=value</code> on the command line; <code>{{secret:NAME}}</code> from
|
|
2075
|
+
the environment, only for names in <code>desktop.secrets</code>. A secret ForgeQE itself uses (a model key, a tracker token) is
|
|
2076
|
+
refused there. Values are substituted once and never expanded again. Targets take <code>{{name}}</code> too (a CSV row's
|
|
2077
|
+
<code>{{customer}}</code> as the control to click), never a secret.</li>
|
|
2078
|
+
<li><strong>Keys:</strong> <code>ctrl</code>, <code>shift</code> and <code>alt</code> with one named key or character (<code>ctrl+s</code>, <code>alt+f4</code>, <code>enter</code>, <code>f5</code>). The Windows
|
|
2079
|
+
and context-menu keys are refused: Win+R followed by typing runs any command.</li>
|
|
2080
|
+
<li><strong>Typing</strong> is literal: pywinauto's key syntax (<code>^</code>, <code>%</code>, <code>+</code>, <code>~</code>, <code>{ENTER}</code>) in the text is escaped.</li>
|
|
2081
|
+
</ul>
|
|
2082
|
+
</div>
|
|
2083
|
+
</section>
|
|
2084
|
+
<section class="doc" id="doc-docs-desktop-recording-by-hand">
|
|
2085
|
+
<h4>Recording by hand</h4>
|
|
2086
|
+
<div class="prose"><p>When you know the steps, do them yourself instead of describing a goal: no model and no UFO are involved, so it costs
|
|
2087
|
+
nothing and does exactly what you did.</p>
|
|
2088
|
+
<pre><code class="language-bash">forgeqe desktop record --app billing --name "Create an invoice"
|
|
2089
|
+
</code></pre>
|
|
2090
|
+
<p>ForgeQE starts the app and watches the mouse and keyboard (low-level Windows hooks), asking UI Automation which control
|
|
2091
|
+
each click and key went to. Do the steps, then:</p>
|
|
2092
|
+
<ul>
|
|
2093
|
+
<li><strong>Ctrl+Shift+F11</strong> with the mouse over a control adds a check: an <code>expect</code> step with the text it shows now (a
|
|
2094
|
+
status message, a saved total). Add one at the end, so the replay tests the outcome, not only the steps.</li>
|
|
2095
|
+
<li><strong>Ctrl+Shift+F12</strong>, or closing the app, finishes. The console shows each step as it is recorded.</li>
|
|
2096
|
+
</ul>
|
|
2097
|
+
<p>What becomes a step:</p>
|
|
2098
|
+
<div class="tbl"><table>
|
|
2099
|
+
<thead>
|
|
2100
|
+
<tr><th>You do</th><th>Step</th></tr>
|
|
2101
|
+
</thead>
|
|
2102
|
+
<tbody>
|
|
2103
|
+
<tr><td>Click a control (two quick clicks: <code>double_click</code>)</td><td><code>click</code> with its AutomationId, name and control type</td></tr>
|
|
2104
|
+
<tr><td>Pick an item in a drop-down</td><td><code>select</code> on the drop-down, with the item's name</td></tr>
|
|
2105
|
+
<tr><td>Type into a field (corrections and pastes included)</td><td>one <code>type</code> with the field's text when you move on, read from the field</td></tr>
|
|
2106
|
+
<tr><td>A key chord outside a text field (<code>ctrl+s</code>, <code>enter</code>, <code>f5</code>)</td><td><code>keys</code></td></tr>
|
|
2107
|
+
<tr><td>Ctrl+Shift+F11 over a control</td><td><code>expect</code> with its text</td></tr>
|
|
2108
|
+
</tbody>
|
|
2109
|
+
</table></div>
|
|
2110
|
+
<ul>
|
|
2111
|
+
<li><strong>Passwords:</strong> a password field is never read; its step types <code>{{secret:NAME}}</code> (the one name in <code>desktop.secrets</code>,
|
|
2112
|
+
or <code><APP>_PASSWORD</code>, which the result tells you to set). Text equal to a <code>desktop.secrets</code> value is written as its
|
|
2113
|
+
placeholder. The typed characters themselves are never kept, only the field's text when you move on.</li>
|
|
2114
|
+
<li><strong>Only the app:</strong> clicks and keys outside the app's own process are not recorded; nor are the Windows and menu keys,
|
|
2115
|
+
or controls with neither an AutomationId nor a name. The result lists what was left out.</li>
|
|
2116
|
+
<li>The run folder has <code>workflow.yaml</code>, <code>run.json</code> and <code>report.html</code>, like explore. Replay it with
|
|
2117
|
+
<code>forgeqe desktop replay</code>.</li>
|
|
2118
|
+
</ul>
|
|
2119
|
+
</div>
|
|
2120
|
+
</section>
|
|
2121
|
+
<section class="doc" id="doc-docs-desktop-fixing-a-workflow-from-chat">
|
|
2122
|
+
<h4>Fixing a workflow from chat</h4>
|
|
2123
|
+
<div class="prose"><p>When a replayed step fails because the app was slow or a dialog was in the way, ForgeQE's agent fixes the workflow
|
|
2124
|
+
with <code>desktop_edit_workflow</code>: it proposes the change (a longer <code>timeout</code> or <code>retries</code> on the step, a <code>wait_for</code> before
|
|
2125
|
+
it, a dialog button in <code>dismiss</code>, a healed locator, a corrected text), shows you the diff, and writes the file only
|
|
2126
|
+
when you agree. For example, after "step 4 timed out while the invoice list loaded":</p>
|
|
2127
|
+
<pre><code class="language-text">Change invoice.yaml?
|
|
2128
|
+
• new wait_for step before step 4
|
|
2129
|
+
• step 4: retries 2
|
|
2130
|
+
- op: click
|
|
2131
|
+
+ target: {automation_id: btnPost}
|
|
2132
|
+
+ retries: 2
|
|
2133
|
+
</code></pre>
|
|
2134
|
+
<ul>
|
|
2135
|
+
<li>Each change has a fixed shape (<code>set</code>, <code>unset</code>, <code>set_step</code>, <code>add_dismiss</code>, <code>insert_step</code>, <code>remove_step</code>,
|
|
2136
|
+
<code>set_variable</code>); step numbers are the replay report's. Anything else is refused.</li>
|
|
2137
|
+
<li>The changed workflow is checked by the engine's own validator before it is shown; one that would not run is not
|
|
2138
|
+
offered. Comments and layout are kept.</li>
|
|
2139
|
+
<li>The old text is kept in the reports folder (<code>desktop-edits/</code>), and a file that changed since the edit was proposed is
|
|
2140
|
+
not overwritten.</li>
|
|
2141
|
+
<li>The workflow must be a <code>.yaml</code> inside the working folder or the reports folder.</li>
|
|
2142
|
+
</ul>
|
|
2143
|
+
</div>
|
|
2144
|
+
</section>
|
|
2145
|
+
<section class="doc" id="doc-docs-desktop-waits-retries-and-dialogs">
|
|
2146
|
+
<h4>Waits, retries and dialogs</h4>
|
|
2147
|
+
<div class="prose"><p>Real apps are slow in places: a splash screen, a form that loads its data, a "Tip of the day" or "Save changes?"
|
|
2148
|
+
dialog. A workflow says how to wait for them, without fixed <code>wait</code> steps:</p>
|
|
2149
|
+
<pre><code class="language-yaml">timeout: 20 # seconds a step waits for its control (default 10, at most 300)
|
|
2150
|
+
retries: 1 # tries a failed action again, after a short pause (default 0, at most 5)
|
|
2151
|
+
dismiss: # clicked whenever one of them is in the way
|
|
2152
|
+
- {name: OK, control_type: Button, automation_id: btnTipClose}
|
|
2153
|
+
- {name: "Don't Save", control_type: Button}
|
|
2154
|
+
steps:
|
|
2155
|
+
- op: launch
|
|
2156
|
+
timeout: 90 # a slow start: how long to wait for the app's window
|
|
2157
|
+
- op: wait_for # the splash screen goes away
|
|
2158
|
+
target: {automation_id: splashForm}
|
|
2159
|
+
gone: true
|
|
2160
|
+
timeout: 120
|
|
2161
|
+
- op: wait_for # the form has loaded its data
|
|
2162
|
+
target: {automation_id: lblStatus}
|
|
2163
|
+
text: Ready
|
|
2164
|
+
- op: click
|
|
2165
|
+
target: {automation_id: btnPost}
|
|
2166
|
+
timeout: 60 # per step, over the workflow's timeout
|
|
2167
|
+
retries: 3
|
|
2168
|
+
</code></pre>
|
|
2169
|
+
<ul>
|
|
2170
|
+
<li><strong>Each step waits</strong> for its control up to its <code>timeout</code>. <code>click</code>, <code>type</code> and <code>select</code> also wait until the control is
|
|
2171
|
+
enabled and visible (a form that is still loading greys its buttons out).</li>
|
|
2172
|
+
<li><strong><code>wait_for</code></strong> waits until a control is there, shows a <code>text</code>, or with <code>gone: true</code> has gone.</li>
|
|
2173
|
+
<li><strong><code>retries</code></strong> applies to <code>click</code>, <code>double_click</code>, <code>type</code>, <code>select</code>, <code>keys</code>, <code>expect</code> and <code>expect_image</code>. Between
|
|
2174
|
+
tries, replay pauses (1 s, 2 s, … up to 5 s) and closes any <code>dismiss</code> dialog. An error in the workflow itself (a
|
|
2175
|
+
missing variable) is not retried.</li>
|
|
2176
|
+
<li><strong><code>dismiss</code></strong> lists at most 10 targets, checked like any target. When a control is not found, or something waits to
|
|
2177
|
+
go away, replay clicks the <code>dismiss</code> controls that are on screen and looks again, up to 3 times per lookup. The
|
|
2178
|
+
report and the console name every dialog closed, and how many tries a step took.</li>
|
|
2179
|
+
</ul>
|
|
2180
|
+
</div>
|
|
2181
|
+
</section>
|
|
2182
|
+
<section class="doc" id="doc-docs-desktop-visual-checks-expect_image">
|
|
2183
|
+
<h4>Visual checks (<code>expect_image</code>)</h4>
|
|
2184
|
+
<div class="prose"><p><code>expect</code> compares text. When what matters is drawn (a chart, a signature, a colour, a layout), <code>expect_image</code>
|
|
2185
|
+
compares pixels with a baseline image:</p>
|
|
2186
|
+
<pre><code class="language-yaml"> - op: expect_image
|
|
2187
|
+
target: {automation_id: chartRevenue} # without a target: the app's top window
|
|
2188
|
+
image: revenue-chart.png # a file name in <workflow name>.baselines/ next to the workflow
|
|
2189
|
+
tolerance: 0.01 # share of pixels that may differ (default 0.01, i.e. 1%)
|
|
2190
|
+
ignore: [[0, 0, 120, 24]] # regions [x, y, width, height] that change every run (a date, a clock)
|
|
2191
|
+
</code></pre>
|
|
2192
|
+
<ul>
|
|
2193
|
+
<li><strong>The first run</strong> with no baseline saves what it captured and passes; the report says so. Commit the baselines
|
|
2194
|
+
folder with the workflow. <code>--update-baselines</code> replaces them after an intended change.</li>
|
|
2195
|
+
<li><strong>A pixel counts as different</strong> when a colour channel differs by more than 32 of 255, so anti-aliasing and ClearType
|
|
2196
|
+
do not fail a check. A different size (another screen scale or window size) fails with both sizes named.</li>
|
|
2197
|
+
<li><strong>The report</strong> shows the baseline, what was captured, and the differences in red, with the share of pixels that
|
|
2198
|
+
differ (<code>images/NNN-baseline|actual|diff.png</code> in the run folder).</li>
|
|
2199
|
+
<li><strong>Image names</strong> are file names only (letters, digits, <code>_</code>, <code>-</code>, ending <code>.png</code>). A path, <code>..</code>, or a link that leads
|
|
2200
|
+
out of the baselines folder is refused.</li>
|
|
2201
|
+
<li>Run visual checks on one machine with a fixed resolution and scale (see <a href="#doc-docs-desktop-ci-running-desktop-workflows-unattended-nightly-ci">DESKTOP-CI.md</a>).</li>
|
|
2202
|
+
</ul>
|
|
2203
|
+
</div>
|
|
2204
|
+
</section>
|
|
2205
|
+
<section class="doc" id="doc-docs-desktop-unattended-runs">
|
|
2206
|
+
<h4>Unattended runs</h4>
|
|
2207
|
+
<div class="prose"><p>Replay needs a signed-in, unlocked Windows session. Before it starts the app, it checks the session and fails at once
|
|
2208
|
+
with the reason if it is a Windows service (session 0), a locked screen, or a disconnected Remote Desktop session.
|
|
2209
|
+
<code>forgeqe desktop doctor</code> runs the same check. To run the suite every night on a machine of your own, see
|
|
2210
|
+
<a href="#doc-docs-desktop-ci-running-desktop-workflows-unattended-nightly-ci">DESKTOP-CI.md</a>: a runner set up in the user's session (<code>ci/desktop-runner/prepare-runner.ps1</code>), a
|
|
2211
|
+
nightly workflow (<code>ci/desktop-runner/desktop-nightly.example.yml</code>), and <code>--junit</code> results.</p>
|
|
2212
|
+
</div>
|
|
2213
|
+
</section>
|
|
2214
|
+
<section class="doc" id="doc-docs-desktop-updating-ufo²">
|
|
2215
|
+
<h4>Updating UFO²</h4>
|
|
2216
|
+
<div class="prose"><p>UFO² is pinned to one commit. Moving it is checked before it is pinned:</p>
|
|
2217
|
+
<ol>
|
|
2218
|
+
<li><code>.github/workflows/forgeqe-desktop-ufo-update.yml</code> runs weekly, or by hand with a commit. It sets up that UFO on
|
|
2219
|
+
Windows (<code>FORGEQE_UFO_COMMIT</code>, which takes a full 40-character commit id only), runs the adapter tests against its
|
|
2220
|
+
real code and a scripted explore of Notepad, and says in the job summary whether it passed.</li>
|
|
2221
|
+
<li>When it passed: <code>bun scripts/ufo-bump.ts <commit></code> writes the new pin into <code>src/desktop/setup.ts</code>,
|
|
2222
|
+
<code>py/forgeqe_desktop/ufo_runner.py</code> and this page. Run the tests and push; the Windows CI job tests the pin again.</li>
|
|
2223
|
+
</ol>
|
|
2224
|
+
<p>When it fails, the failing step names what changed in UFO (usually the action executor or the config layout that the
|
|
2225
|
+
adapter wraps). Keep the pin until the adapter is updated for it.</p>
|
|
2226
|
+
</div>
|
|
2227
|
+
</section>
|
|
2228
|
+
<section class="doc" id="doc-docs-desktop-checks-of-the-outcome">
|
|
2229
|
+
<h4>Checks of the outcome</h4>
|
|
2230
|
+
<div class="prose"><p>A recording of actions only proves the steps run. When UFO has finished, and before the app closes, explore shows the
|
|
2231
|
+
model the goal and the app's controls with the text each shows now, and the model picks up to three whose text proves
|
|
2232
|
+
the goal (a status message, a saved value), by number, with the part of the text to expect. A check is kept only when
|
|
2233
|
+
that text really is in that control, so the model can choose among what is on screen but never invent it; text holding
|
|
2234
|
+
a <code>desktop.secrets</code> value is never offered. The checks are saved as <code>expect</code> steps at the end of the workflow, and the
|
|
2235
|
+
report says how many there are. With none, the report and the agent say so: add <code>expect</code> steps by hand.</p>
|
|
2236
|
+
</div>
|
|
2237
|
+
</section>
|
|
2238
|
+
<section class="doc" id="doc-docs-desktop-recording">
|
|
2239
|
+
<h4>Recording</h4>
|
|
2240
|
+
<div class="prose"><p>UFO's own step log names the control and a per-step id, not its AutomationId, so the adapter wraps UFO's
|
|
2241
|
+
<code>ActionExecutor.execute</code> and records each action with the UIA properties of the control it acted on. <code>record.py</code> turns
|
|
2242
|
+
that trace into steps: <code>click_input</code> → <code>click</code>/<code>double_click</code>, <code>set_edit_text</code> → <code>type</code>, <code>keyboard_input</code> → <code>keys</code> (one
|
|
2243
|
+
allowed chord) or <code>type</code>, <code>select</code> → <code>select</code>. Coordinate clicks, scrolls and anything else are listed as not recorded
|
|
2244
|
+
in the report. A typed value equal to a <code>desktop.secrets</code> value is written as its placeholder; typed text that only
|
|
2245
|
+
looks like <code>{{…}}</code> is kept literal, so a model told to type <code>{{secret:X}}</code> cannot make replay type the secret.</p>
|
|
2246
|
+
</div>
|
|
2247
|
+
</section>
|
|
2248
|
+
<section class="doc" id="doc-docs-desktop-security">
|
|
2249
|
+
<h4>Security</h4>
|
|
2250
|
+
<div class="prose"><p>The YAML, the trace, the app's screen text and the model's replies can all be written by someone else. The rules in the
|
|
2251
|
+
repository's <code>CLAUDE.md</code> apply; for this engine:</p>
|
|
2252
|
+
<ul>
|
|
2253
|
+
<li><strong>No command in the workflow.</strong> <code>app:</code> names a <code>desktop.apps</code> entry: an absolute <code>.exe</code> (a <code>.bat</code> or <code>.cmd</code> would run
|
|
2254
|
+
through <code>cmd.exe</code>, which reparses its arguments) and an argument list, started with <code>subprocess.Popen([path, *args])</code>.</li>
|
|
2255
|
+
<li><strong>UFO gets UI tools only.</strong> Each explore writes UFO's <code>mcp.yaml</code> with <code>UICollector</code>, <code>HostUIExecutor</code> and
|
|
2256
|
+
<code>AppUIExecutor</code>. UFO's default <code>CommandLineExecutor</code>, which runs shell commands, and its Word, Excel, PowerPoint and
|
|
2257
|
+
PDF executors are left out. The adapter starts the app, so UFO never launches a program.</li>
|
|
2258
|
+
<li><strong>Keys:</strong> during explore the adapter refuses UFO's Windows and menu keys too, and the report lists what it refused.</li>
|
|
2259
|
+
<li><strong>No one at the console:</strong> UFO's safe guard asks before sensitive actions; explore answers no, since nobody is there.</li>
|
|
2260
|
+
<li><strong>Environment:</strong> the engine process gets system settings, the profile's key variable (or Entra's) and
|
|
2261
|
+
<code>desktop.secrets</code>, nothing else (<code>desktopEnvNames</code>).</li>
|
|
2262
|
+
<li><strong>Explore controls the real mouse and keyboard.</strong> It asks before starting (<code>--yes</code> to skip; required with no
|
|
2263
|
+
terminal). Use a test machine or VM and a test account.</li>
|
|
2264
|
+
<li><strong>Agent tools:</strong> a workflow path the agent gives must be a <code>.yaml</code> inside the working folder or the reports folder (real
|
|
2265
|
+
path, so a link out is refused); variables are names and text only; heals are reported, never written.</li>
|
|
2266
|
+
<li><strong>Healing:</strong> the model picks one of the window's real controls by number; control text is passed as data, and any
|
|
2267
|
+
other answer heals nothing. Heals are written into the workflow only when accepted.</li>
|
|
2268
|
+
<li><strong>Report:</strong> every value is escaped; images are only the engine's own <code>steps/NNN.png</code> and
|
|
2269
|
+
<code>images/NNN-baseline|actual|diff.png</code>.</li>
|
|
2270
|
+
<li><strong>Baselines:</strong> <code>expect_image</code> names are file names, and their real path must be inside the workflow's baselines
|
|
2271
|
+
folder, so a workflow cannot read or overwrite another file.</li>
|
|
2272
|
+
<li><strong>Unattended machines:</strong> <code>prepare-runner.ps1</code> never handles a password; automatic sign-in uses Sysinternals
|
|
2273
|
+
Autologon, which stores it encrypted.</li>
|
|
2274
|
+
<li><strong>Install:</strong> UFO is cloned and checked out at the pinned commit (or <code>FORGEQE_UFO_COMMIT</code>, a full commit id only),
|
|
2275
|
+
and the checkout's <code>HEAD</code> is verified; replay's packages are pinned to exact versions.</li>
|
|
2276
|
+
</ul>
|
|
2277
|
+
</div>
|
|
2278
|
+
</section>
|
|
2279
|
+
<section class="doc" id="doc-docs-desktop-before-first-use-on-windows">
|
|
2280
|
+
<h4>Before first use on Windows</h4>
|
|
2281
|
+
<div class="prose"><p>Verified in CI: setup with and without <code>--ufo</code>, replay, healing, the CLI, and UFO's own code with the adapter (its
|
|
2282
|
+
config, its model calls routed to the profile, the action trace, UI tools only). Still to check, on a Windows test
|
|
2283
|
+
machine with your apps:</p>
|
|
2284
|
+
<ol>
|
|
2285
|
+
<li><strong>UIA coverage:</strong> <code>Inspect.exe</code> shows the controls that matter with stable <code>AutomationId</code>s. Custom grids, Java
|
|
2286
|
+
(needs Java Access Bridge), Delphi, SAP GUI and Citrix (vision only) are where UIA falls short.</li>
|
|
2287
|
+
<li><strong>Explore with your model:</strong> one goal with an Azure profile and one with <code>auth: entra</code>; <code>trace.json</code> has the
|
|
2288
|
+
controls' AutomationIds, and <code>ufo/</code> has UFO's logs.</li>
|
|
2289
|
+
<li><strong>Replay:</strong> the recorded workflow passes 10 runs out of 10; a renamed control is healed.</li>
|
|
2290
|
+
<li><strong>Cost and time</strong> of one explore, from <code>forgeqe</code> usage.</li>
|
|
2291
|
+
</ol>
|
|
2292
|
+
<p>Found by the Windows runs and fixed: UFO's pins need Python 3.10 (pandas 1.4.3); UFO prints emoji panels to stdout, which
|
|
2293
|
+
is the engine's event stream and crashed on the cp1252 console (now sent to stderr, characters replaced); an error in
|
|
2294
|
+
UFO's own evaluation lost the recorded actions (the trace is now kept and saved).</p>
|
|
2295
|
+
</div>
|
|
2296
|
+
</section>
|
|
2297
|
+
<section class="doc" id="doc-docs-desktop-next-steps">
|
|
2298
|
+
<h4>Next steps</h4>
|
|
2299
|
+
<div class="prose"><ol>
|
|
2300
|
+
<li>The checks above.</li>
|
|
2301
|
+
<li><code>forgeqe suite</code> and test cases from Azure DevOps, Jira and Xray for desktop workflows, with results written back.</li>
|
|
2302
|
+
<li>Scribe: export a workflow as pywinauto code (or C# with FlaUI) in the user's repo.</li>
|
|
2303
|
+
<li>App adapters where UIA falls short: Java Access Bridge, SAP GUI scripting, vision-only for Citrix.</li>
|
|
2304
|
+
<li>A pipeline template for a self-hosted Windows runner signed in to a desktop session.</li>
|
|
2305
|
+
</ol>
|
|
2306
|
+
</div>
|
|
2307
|
+
</section>
|
|
2308
|
+
<section class="doc" id="doc-docs-desktop-ci-running-desktop-workflows-unattended-nightly-ci">
|
|
2309
|
+
<h4>Running desktop workflows unattended (nightly CI)</h4>
|
|
2310
|
+
<div class="prose"><p>Desktop replay clicks and types into real windows. It needs what a person at the machine would have: a signed-in
|
|
2311
|
+
Windows session, unlocked, with a screen. GitHub's hosted <code>windows-2022</code> runners have that, and ForgeQE's own CI runs
|
|
2312
|
+
Notepad there. Your own apps usually need a machine of your own, with the app installed, its test database, and a
|
|
2313
|
+
network route to its servers. This page sets one up as a GitHub Actions self-hosted runner. The same rules apply to
|
|
2314
|
+
Azure DevOps agents and Jenkins nodes.</p>
|
|
2315
|
+
</div>
|
|
2316
|
+
</section>
|
|
2317
|
+
<section class="doc" id="doc-docs-desktop-ci-what-breaks-a-desktop-run-and-how-forgeqe-tells-you">
|
|
2318
|
+
<h4>What breaks a desktop run, and how ForgeQE tells you</h4>
|
|
2319
|
+
<div class="prose"><div class="tbl"><table>
|
|
2320
|
+
<thead>
|
|
2321
|
+
<tr><th>Situation</th><th>Effect</th><th>Fix</th></tr>
|
|
2322
|
+
</thead>
|
|
2323
|
+
<tbody>
|
|
2324
|
+
<tr><td>The runner is a Windows service</td><td>Session 0 has no desktop; every control lookup times out</td><td>Run the runner in the user's session (below)</td></tr>
|
|
2325
|
+
<tr><td>The screen is locked</td><td>Windows sends no input to apps</td><td>No inactivity lock, no screen-saver lock</td></tr>
|
|
2326
|
+
<tr><td>A Remote Desktop session was disconnected</td><td>The session is locked</td><td>Leave Remote Desktop with <code>tscon</code> (below), or don't use it on the test machine</td></tr>
|
|
2327
|
+
<tr><td>The machine slept or the screen turned off</td><td>Input and screenshots stop</td><td>Power plan: never sleep, never turn off the screen</td></tr>
|
|
2328
|
+
<tr><td>After a restart nobody signs in</td><td>No session at all</td><td>Sysinternals Autologon</td></tr>
|
|
2329
|
+
</tbody>
|
|
2330
|
+
</table></div>
|
|
2331
|
+
<p>Before each replay or explore, the engine checks the session (<code>py/forgeqe_desktop/session.py</code>). On a session it cannot
|
|
2332
|
+
drive, the run fails at once with the reason, instead of timing out step by step. <code>forgeqe desktop doctor</code> runs the
|
|
2333
|
+
same check from a shell or a job:</p>
|
|
2334
|
+
<pre><code class="language-text">engine ready (C:\Users\qa\.forgeqe\desktop\.venv\Scripts\python.exe)
|
|
2335
|
+
explore not set up (forgeqe desktop setup --ufo)
|
|
2336
|
+
session 1, screen 1920x1080
|
|
2337
|
+
desktop logged in and unlocked
|
|
2338
|
+
</code></pre>
|
|
2339
|
+
<p>It exits with 1 when the desktop cannot be driven, so a job stops before the suite runs.</p>
|
|
2340
|
+
</div>
|
|
2341
|
+
</section>
|
|
2342
|
+
<section class="doc" id="doc-docs-desktop-ci-set-up-the-test-machine">
|
|
2343
|
+
<h4>Set up the test machine</h4>
|
|
2344
|
+
<div class="prose"><p>Use a test machine or VM, and a test account: the runner's jobs control its mouse and keyboard.</p>
|
|
2345
|
+
<ol>
|
|
2346
|
+
<li>
|
|
2347
|
+
<p><strong>Sign in</strong> as the account the tests will run as.</p>
|
|
2348
|
+
</li>
|
|
2349
|
+
<li>
|
|
2350
|
+
<p><strong>Install the runner</strong> (GitHub: Settings → Actions → Runners → New self-hosted runner). Give it the label
|
|
2351
|
+
<code>forgeqe-desktop</code>. When <code>config.cmd</code> asks whether to run as a service, answer <strong>no</strong>.</p>
|
|
2352
|
+
</li>
|
|
2353
|
+
<li>
|
|
2354
|
+
<p><strong>Run <code>ci/desktop-runner/prepare-runner.ps1</code></strong> in an elevated PowerShell (<code>-WhatIf</code> first to see the changes):</p>
|
|
2355
|
+
<pre><code class="language-powershell">.\prepare-runner.ps1 -RunnerDir C:\actions-runner -WhatIf
|
|
2356
|
+
.\prepare-runner.ps1 -RunnerDir C:\actions-runner
|
|
2357
|
+
</code></pre>
|
|
2358
|
+
<p>It stops sleep, screen-off and the screen-saver lock. It turns the inactivity lock off and registers a logon task
|
|
2359
|
+
that starts <code>run.cmd</code> in this account's session. It warns you if the runner is also installed as a service. It
|
|
2360
|
+
never handles a password.</p>
|
|
2361
|
+
</li>
|
|
2362
|
+
<li>
|
|
2363
|
+
<p><strong>Sign in automatically after a restart</strong> with
|
|
2364
|
+
<a href="https://learn.microsoft.com/sysinternals/downloads/autologon" rel="noopener">Sysinternals Autologon</a>, run as the test account.
|
|
2365
|
+
It stores the password encrypted by Windows. Never put the password in a script, a registry value you write
|
|
2366
|
+
yourself, or the repository.</p>
|
|
2367
|
+
</li>
|
|
2368
|
+
<li>
|
|
2369
|
+
<p><strong>Fix the screen size.</strong> A visual check (<code>expect_image</code>) compares pixels, so the resolution and scale (100%, 125%)
|
|
2370
|
+
must match the machine where its baselines were saved. Set them once and leave them.</p>
|
|
2371
|
+
</li>
|
|
2372
|
+
<li>
|
|
2373
|
+
<p><strong>Restart</strong> and check that the runner shows as idle on GitHub. Then run <code>forgeqe desktop doctor</code> from a job.</p>
|
|
2374
|
+
</li>
|
|
2375
|
+
</ol>
|
|
2376
|
+
</div>
|
|
2377
|
+
<section class="doc sub" id="doc-docs-desktop-ci-leaving-remote-desktop-without-locking-the-session"><h5>Leaving Remote Desktop without locking the session</h5><div class="prose"><p>Closing a Remote Desktop window disconnects the session, and Windows locks it. To leave it running instead, hand the
|
|
2378
|
+
session back to the machine's console. In an elevated prompt inside the Remote Desktop session:</p>
|
|
2379
|
+
<pre><code class="language-bat">for /f "skip=1 tokens=3" %s in ('query user %USERNAME%') do tscon %s /dest:console
|
|
2380
|
+
</code></pre>
|
|
2381
|
+
<p>On a VM, the hypervisor's console (Hyper-V, VMware, VirtualBox) needs no such step.</p>
|
|
2382
|
+
</div></section>
|
|
2383
|
+
</section>
|
|
2384
|
+
<section class="doc" id="doc-docs-desktop-ci-the-nightly-job">
|
|
2385
|
+
<h4>The nightly job</h4>
|
|
2386
|
+
<div class="prose"><p><code>ci/desktop-runner/desktop-nightly.example.yml</code> is a ready workflow. Copy it into <code>.github/workflows/</code> in the
|
|
2387
|
+
repository that holds your desktop workflows. It:</p>
|
|
2388
|
+
<ul>
|
|
2389
|
+
<li>runs at night on <code>[self-hosted, windows, forgeqe-desktop]</code>, one run at a time;</li>
|
|
2390
|
+
<li>installs a pinned ForgeQE and runs <code>forgeqe desktop setup</code> and <code>forgeqe desktop doctor</code>;</li>
|
|
2391
|
+
<li>replays every workflow in a folder: <code>forgeqe desktop replay desktop-tests/ --junit results/desktop.xml</code>;</li>
|
|
2392
|
+
<li>keeps the reports, screenshots and image diffs as an artifact.</li>
|
|
2393
|
+
</ul>
|
|
2394
|
+
<p><code>--junit</code> writes one test case per workflow (and per CSV row with <code>--data</code>), which CI servers show as test results.
|
|
2395
|
+
<code>runs.json</code> in the output folder has the same, with the path of each report.</p>
|
|
2396
|
+
<p>Secrets the workflows type (<code>{{secret:NAME}}</code>) come from the CI's secret store as environment variables, and must be
|
|
2397
|
+
listed in <code>desktop.secrets</code> or the environment's <code>secrets</code> in <code>env.yaml</code>. Pick the environment with <code>FORGEQE_ENV</code> or
|
|
2398
|
+
<code>--env</code>.</p>
|
|
2399
|
+
<p>Healing calls the model. In a nightly run, either give the job the model's key so moved controls are healed and
|
|
2400
|
+
reported in <code>heals.json</code> (never written into the workflow without <code>--accept-heals</code>), or pass <code>--no-heal</code> so a moved
|
|
2401
|
+
control fails the run.</p>
|
|
1833
2402
|
</div>
|
|
1834
2403
|
</section>
|
|
1835
2404
|
</div>
|
|
@@ -1867,7 +2436,7 @@ forgeqe mobile flow workflow.yaml > login.flow.yaml # a workflow as
|
|
|
1867
2436
|
reads only <code>mobile.secrets</code> (and the environment's secrets). A flow kept with <code>forgeqe mobile flow</code> reads each secret
|
|
1868
2437
|
as <code>${MAESTRO_<NAME>}</code>, never its value, so set <code>MAESTRO_<NAME></code> when you run it outside ForgeQE; a workflow that types a
|
|
1869
2438
|
secret not listed there is refused. Team Maestro flows run only with UI commands: scripts,
|
|
1870
|
-
JavaScript in <code>${…}</code>, Maestro's AI commands and file writes are refused. Details: <a href="
|
|
2439
|
+
JavaScript in <code>${…}</code>, Maestro's AI commands and file writes are refused. Details: <a href="#doc-docs-mobile-forgeqe-mobile-android-and-ios-apps">docs/MOBILE.md</a>.</p>
|
|
1871
2440
|
<h5>On TestMu AI (formerly LambdaTest)</h5>
|
|
1872
2441
|
<p>Both drivers run on TestMu AI's devices. Upload the app build once, then use the <code>lt://</code> id it prints as the app's <code>app</code>
|
|
1873
2442
|
(in env.yaml or <code>mobile.apps</code>; ForgeQE adds it to Appium's <code>LT:Options</code> itself):</p>
|
|
@@ -1910,7 +2479,222 @@ set there). The build must be inside the working folder; hidden folders and file
|
|
|
1910
2479
|
<p><a href="../samples/mobile-testmu/README.md">samples/mobile-testmu</a> is a ready project with TestMu's Wikipedia sample app: copy
|
|
1911
2480
|
<code>.env.example</code> to <code>.env</code>, upload the app with <code>forgeqe mobile upload sample.apk</code>, put the id in its <code>forgeqe.yaml</code>,
|
|
1912
2481
|
download the <code>hyperexecute</code> CLI into the folder, then run <code>forgeqe mobile replay wiki.yaml --driver appium</code>,
|
|
1913
|
-
<code>--driver maestro</code>, and <code>forgeqe mobile maestro .maestro</code>. Details: <a href="
|
|
2482
|
+
<code>--driver maestro</code>, and <code>forgeqe mobile maestro .maestro</code>. Details: <a href="#doc-docs-mobile-testmu-ai-formerly-lambdatest">docs/MOBILE.md</a>.</p>
|
|
2483
|
+
</div>
|
|
2484
|
+
</section>
|
|
2485
|
+
<section class="doc" id="doc-docs-mobile-forgeqe-mobile-android-and-ios-apps">
|
|
2486
|
+
<h4>ForgeQE mobile: Android and iOS apps</h4>
|
|
2487
|
+
<div class="prose"><p>Status: the engine and its tests run on any OS, against a fake Appium server and a stand-in for Maestro. The CI job
|
|
2488
|
+
<code>.github/workflows/forgeqe-mobile-android.yml</code> runs it on an Android emulator: Appium explores and replays the
|
|
2489
|
+
emulator's Settings app, and Maestro replays the same workflow. iOS runs the same code with the XCUITest driver, but
|
|
2490
|
+
no CI job tests it yet (it needs a macOS runner with a simulator).</p>
|
|
2491
|
+
<p>ForgeQE drives mobile apps two ways, and one workflow format runs on both:</p>
|
|
2492
|
+
<div class="tbl"><table>
|
|
2493
|
+
<thead>
|
|
2494
|
+
<tr><th></th><th>Appium</th><th>Maestro</th></tr>
|
|
2495
|
+
</thead>
|
|
2496
|
+
<tbody>
|
|
2497
|
+
<tr><td>Runs</td><td>step by step, through an Appium server (yours or a device cloud's)</td><td>the whole workflow as one Maestro flow</td></tr>
|
|
2498
|
+
<tr><td>Explore a goal with the model</td><td>yes</td><td>no</td></tr>
|
|
2499
|
+
<tr><td>Heal a control that moved</td><td>yes, one model call, reported for review</td><td>no</td></tr>
|
|
2500
|
+
<tr><td>Visual checks (<code>expect_image</code>)</td><td>yes</td><td>no</td></tr>
|
|
2501
|
+
<tr><td>Your team's own flows</td><td>n/a</td><td><code>forgeqe mobile maestro</code>, checked against an allowlist</td></tr>
|
|
2502
|
+
<tr><td>Needs</td><td><code>appium</code> with UiAutomator2 (Android) or XCUITest (iOS), a device or emulator</td><td><code>maestro</code> and a device or emulator</td></tr>
|
|
2503
|
+
</tbody>
|
|
2504
|
+
</table></div>
|
|
2505
|
+
</div>
|
|
2506
|
+
</section>
|
|
2507
|
+
<section class="doc" id="doc-docs-mobile-use">
|
|
2508
|
+
<h4>Use</h4>
|
|
2509
|
+
<div class="prose"><pre><code class="language-bash">forgeqe mobile doctor # does the Appium server answer, is maestro installed?
|
|
2510
|
+
forgeqe mobile apps # the mobile apps of the active environment
|
|
2511
|
+
forgeqe mobile explore "Sign in as the courier and accept the first delivery" --app courier
|
|
2512
|
+
forgeqe mobile replay .forgeqe/reports/mobile-explore-…/workflow.yaml
|
|
2513
|
+
forgeqe mobile replay workflow.yaml --driver maestro
|
|
2514
|
+
forgeqe mobile replay mobile-tests/ --data couriers.csv --junit results.xml
|
|
2515
|
+
forgeqe mobile maestro .maestro/ # the team's flows
|
|
2516
|
+
forgeqe mobile flow workflow.yaml # print a workflow as a Maestro flow
|
|
2517
|
+
forgeqe mobile validate workflow.yaml
|
|
2518
|
+
</code></pre>
|
|
2519
|
+
<p>Or ask ForgeQE's agent in the shell or <code>forgeqe run</code>: "sign in to the courier app and save it as a test", "replay the
|
|
2520
|
+
courier login on Maestro", "run our Maestro flows". Its tools are <code>mobile_explore</code>, <code>mobile_replay</code> and
|
|
2521
|
+
<code>mobile_maestro</code>. It asks before each one starts, and <code>desktop_edit_workflow</code> fixes a mobile workflow after a failed
|
|
2522
|
+
replay, the same way as a desktop one.</p>
|
|
2523
|
+
</div>
|
|
2524
|
+
</section>
|
|
2525
|
+
<section class="doc" id="doc-docs-mobile-apps-and-settings">
|
|
2526
|
+
<h4>Apps and settings</h4>
|
|
2527
|
+
<div class="prose"><p>Mobile apps come from the active environment in env.yaml, or from <code>mobile.apps</code> in forgeqe.yaml:</p>
|
|
2528
|
+
<pre><code class="language-yaml"># env.yaml
|
|
2529
|
+
environments:
|
|
2530
|
+
qa:
|
|
2531
|
+
apps:
|
|
2532
|
+
courier:
|
|
2533
|
+
kind: mobile
|
|
2534
|
+
platform: android # or ios
|
|
2535
|
+
app_id: com.contoso.courier # Android package or iOS bundle id
|
|
2536
|
+
activity: .MainActivity # Android only, optional
|
|
2537
|
+
app: /builds/courier-qa.apk # optional: install this build (.apk/.aab/.ipa/.app/.zip, https URL, bs://…)
|
|
2538
|
+
description: Driver app that confirms delivery
|
|
2539
|
+
secrets: [COURIER_PASSWORD_QA]
|
|
2540
|
+
</code></pre>
|
|
2541
|
+
<pre><code class="language-yaml"># forgeqe.yaml
|
|
2542
|
+
mobile:
|
|
2543
|
+
driver: appium # appium | maestro: the default for replays
|
|
2544
|
+
appium:
|
|
2545
|
+
url: http://127.0.0.1:4723
|
|
2546
|
+
capabilities: # added to every session
|
|
2547
|
+
appium:udid: emulator-5554
|
|
2548
|
+
# username_env: BROWSERSTACK_USERNAME
|
|
2549
|
+
# access_key_env: BROWSERSTACK_ACCESS_KEY
|
|
2550
|
+
maestro:
|
|
2551
|
+
command: [maestro]
|
|
2552
|
+
# device: emulator-5554
|
|
2553
|
+
secrets: [COURIER_PASSWORD]
|
|
2554
|
+
heal: true
|
|
2555
|
+
screenshots: true
|
|
2556
|
+
</code></pre>
|
|
2557
|
+
<p>Each run writes a folder under the reports folder (<code>mobile-explore-…</code>, <code>mobile-replay-…</code>, <code>mobile-maestro-…</code>) with
|
|
2558
|
+
<code>run.json</code>, <code>report.html</code>, step screenshots, and for Maestro runs the flow, <code>maestro.log</code> and Maestro's JUnit report.</p>
|
|
2559
|
+
</div>
|
|
2560
|
+
</section>
|
|
2561
|
+
<section class="doc" id="doc-docs-mobile-workflow-yaml-forgeqe-mobile-v1">
|
|
2562
|
+
<h4>Workflow YAML (<code>forgeqe-mobile-v1</code>)</h4>
|
|
2563
|
+
<div class="prose"><pre><code class="language-yaml">format: forgeqe-mobile-v1
|
|
2564
|
+
name: Courier signs in
|
|
2565
|
+
app: courier
|
|
2566
|
+
timeout: 20
|
|
2567
|
+
dismiss:
|
|
2568
|
+
- {name: Allow, control_type: android.widget.Button} # permission dialogs, when they are in the way
|
|
2569
|
+
steps:
|
|
2570
|
+
- op: launch
|
|
2571
|
+
- op: type
|
|
2572
|
+
target: {automation_id: username}
|
|
2573
|
+
text: "{{courier}}"
|
|
2574
|
+
- op: type
|
|
2575
|
+
target: {automation_id: password}
|
|
2576
|
+
text: "{{secret:COURIER_PASSWORD}}"
|
|
2577
|
+
- op: tap
|
|
2578
|
+
target: {name: Sign in}
|
|
2579
|
+
- op: wait_for
|
|
2580
|
+
target: {automation_id: progress}
|
|
2581
|
+
gone: true
|
|
2582
|
+
- op: scroll_to
|
|
2583
|
+
target: {name: Deliveries}
|
|
2584
|
+
- op: expect
|
|
2585
|
+
target: {automation_id: status}
|
|
2586
|
+
text: Welcome
|
|
2587
|
+
</code></pre>
|
|
2588
|
+
<ul>
|
|
2589
|
+
<li><strong>Ops:</strong> <code>launch</code>, <code>tap</code>, <code>double_tap</code>, <code>long_press</code>, <code>type</code>, <code>expect</code>, <code>expect_image</code>, <code>swipe</code> (<code>direction</code> up,
|
|
2590
|
+
down, left or right, optionally on a <code>target</code>), <code>scroll_to</code> (scrolls until the target is on screen), <code>back</code>,
|
|
2591
|
+
<code>hide_keyboard</code>, <code>keys</code> (<code>enter</code>, <code>back</code>, <code>tab</code>, <code>delete</code>, <code>search</code>), <code>wait</code>, <code>wait_for</code>, <code>close</code>. Timeouts,
|
|
2592
|
+
retries and <code>dismiss</code> work as in desktop workflows. Explore saves a tap on a system dialog (another package's
|
|
2593
|
+
control, such as Android's notification prompt) under <code>dismiss</code>, not as a step, so the workflow still runs where the
|
|
2594
|
+
dialog never shows; on Maestro, each <code>dismiss</code> target is tapped after a launch if it is there.</li>
|
|
2595
|
+
<li><strong>Targets</strong> use the desktop keys, mapped per platform: <code>automation_id</code> is the Android resource-id (with or without the
|
|
2596
|
+
<code>package:id/</code> prefix) or the iOS accessibility identifier; <code>name</code> is the Android text or content-desc, or the iOS
|
|
2597
|
+
label; <code>control_type</code> is the Android class (<code>android.widget.Button</code>) or the iOS type (<code>XCUIElementTypeButton</code>);
|
|
2598
|
+
<code>index</code> picks one of several matches.</li>
|
|
2599
|
+
</ul>
|
|
2600
|
+
</div>
|
|
2601
|
+
</section>
|
|
2602
|
+
<section class="doc" id="doc-docs-mobile-explore">
|
|
2603
|
+
<h4>Explore</h4>
|
|
2604
|
+
<div class="prose"><p>The model sees the goal, its steps so far, the numbered controls on screen with the text each shows, and a screenshot
|
|
2605
|
+
when the profile takes images. It answers with one action on a control by its number (tap, long press, type), a swipe
|
|
2606
|
+
or scroll, back, or done. So it can only act on a control that is on screen. When it says done, it picks checks of the
|
|
2607
|
+
outcome from the controls' current text, kept only when that text is really there, as on the desktop. The app's text
|
|
2608
|
+
is passed as data.</p>
|
|
2609
|
+
</div>
|
|
2610
|
+
</section>
|
|
2611
|
+
<section class="doc" id="doc-docs-mobile-maestro">
|
|
2612
|
+
<h4>Maestro</h4>
|
|
2613
|
+
<div class="prose"><p><strong>ForgeQE workflows on Maestro.</strong> <code>--driver maestro</code> writes the workflow as a Maestro flow (<code>flow.yaml</code> in the run
|
|
2614
|
+
folder) and runs <code>maestro test</code>. Maestro's debug output gives each workflow step its result. Variables are filled in
|
|
2615
|
+
first. Values are matched literally, so Maestro's regular expressions never see them as patterns. A secret becomes
|
|
2616
|
+
<code>${MAESTRO_NAME}</code>, which Maestro reads from the environment ForgeQE gives it, so a secret's value never lands in the flow
|
|
2617
|
+
file or on a command line. A value holding <code>${…}</code> is refused, because Maestro would run it as JavaScript.</p>
|
|
2618
|
+
<p><strong>Your team's flows.</strong> <code>forgeqe mobile maestro <files or folders></code> checks each flow, and every flow it runs, before
|
|
2619
|
+
Maestro gets it:</p>
|
|
2620
|
+
<ul>
|
|
2621
|
+
<li>Only UI commands pass: <code>launchApp</code>, <code>tapOn</code>, <code>inputText</code>, <code>assertVisible</code>, <code>scrollUntilVisible</code>, <code>swipe</code>,
|
|
2622
|
+
<code>extendedWaitUntil</code>, <code>runFlow</code>, <code>repeat</code>, <code>retry</code> and the like.</li>
|
|
2623
|
+
<li>These are refused:
|
|
2624
|
+
<ul>
|
|
2625
|
+
<li><code>runScript</code>, <code>evalScript</code> and <code>assertTrue</code>, which run JavaScript.</li>
|
|
2626
|
+
<li>Maestro's AI commands, which send screenshots to a service.</li>
|
|
2627
|
+
<li><code>takeScreenshot</code>, recordings and <code>addMedia</code>, which read or write files.</li>
|
|
2628
|
+
</ul>
|
|
2629
|
+
</li>
|
|
2630
|
+
<li><code>${…}</code> may only name a variable (<code>${USERNAME}</code>), never an expression.</li>
|
|
2631
|
+
<li><code>when: {true: …}</code> is refused, because it is JavaScript.</li>
|
|
2632
|
+
<li><code>openLink</code> takes http(s) or the app's own deep links, never <code>javascript:</code>, <code>file:</code> or <code>intent:</code>.</li>
|
|
2633
|
+
<li><code>runFlow</code> files must be <code>.yaml</code> inside the working folder; links out of it are refused too. Nesting stops at 5 levels.</li>
|
|
2634
|
+
</ul>
|
|
2635
|
+
<p>A refused flow does not run, and the error names the command.</p>
|
|
2636
|
+
</div>
|
|
2637
|
+
</section>
|
|
2638
|
+
<section class="doc" id="doc-docs-mobile-testmu-ai-formerly-lambdatest">
|
|
2639
|
+
<h4>TestMu AI (formerly LambdaTest)</h4>
|
|
2640
|
+
<div class="prose"><p>Both drivers run on TestMu's devices. Upload the app once, then use the <code>lt://</code> id it returns as the app's <code>app</code>:</p>
|
|
2641
|
+
<pre><code class="language-bash">forgeqe mobile upload courier.apk # → lt://APP… (--virtual for TestMu's emulators)
|
|
2642
|
+
</code></pre>
|
|
2643
|
+
<p>From the shell, ask in words ("upload builds/courier.apk to TestMu for courier"): <code>mobile_upload</code> asks first, uploads
|
|
2644
|
+
the build (it must be inside the working folder), and saves the <code>lt://</code> id as that app's <code>app:</code> in forgeqe.yaml.</p>
|
|
2645
|
+
<p>The user and key come from <code>LT_USERNAME</code> and <code>LT_ACCESS_KEY</code> (or the variables the config names), for example in a
|
|
2646
|
+
<code>.env</code> file in the project folder. <code>samples/mobile-testmu/</code> is a ready project with TestMu's Wikipedia sample app.</p>
|
|
2647
|
+
<pre><code class="language-yaml">mobile:
|
|
2648
|
+
apps:
|
|
2649
|
+
courier: { platform: android, app_id: com.contoso.courier, app: "lt://APP1016…" }
|
|
2650
|
+
appium: # Appium on TestMu's mobile hub
|
|
2651
|
+
url: https://mobile-hub.lambdatest.com/wd/hub
|
|
2652
|
+
username_env: LT_USERNAME
|
|
2653
|
+
access_key_env: LT_ACCESS_KEY
|
|
2654
|
+
capabilities:
|
|
2655
|
+
"LT:Options": { w3c: true, isRealMobile: true, deviceName: "Pixel 6", platformVersion: "14", build: nightly } # app: added from the app's own
|
|
2656
|
+
maestro:
|
|
2657
|
+
testmu: # Maestro on TestMu, through HyperExecute
|
|
2658
|
+
cli: /opt/hyperexecute # TestMu's hyperexecute CLI
|
|
2659
|
+
devices: ["Pixel 6-14"] # name-version, or a pattern such as ".*-.*"
|
|
2660
|
+
real_device: true # false: TestMu's emulators
|
|
2661
|
+
# username_env: LT_USERNAME # the defaults
|
|
2662
|
+
# access_key_env: LT_ACCESS_KEY
|
|
2663
|
+
</code></pre>
|
|
2664
|
+
<ul>
|
|
2665
|
+
<li><strong>Appium</strong> needs nothing else: the Appium driver talks to the hub with the user and key from the two variables.</li>
|
|
2666
|
+
<li><strong>Maestro</strong> runs through HyperExecute. ForgeQE writes a HyperExecute project in the run folder and starts the CLI.
|
|
2667
|
+
The project holds the flows in their folder layout (so <code>runFlow</code> still works), the app (an <code>lt://</code> id is passed as
|
|
2668
|
+
the device's <code>appId</code>; a build file is uploaded with the project), a fixed setup script (Android platform tools 35.0.2
|
|
2669
|
+
and Maestro 2.6.1 from its release, both checked against their SHA-256) and a fixed run script, and
|
|
2670
|
+
<code>hyperexecute.yaml</code>. Team flows are checked against the allowlist first, as always. Flow file names may only use
|
|
2671
|
+
letters, digits and <code>_ . - /</code> (and one leading dot, for <code>.maestro/</code>), because HyperExecute's shell reads them. After
|
|
2672
|
+
the job, ForgeQE downloads its artifacts (Maestro's command results and JUnit reports), so each step gets its own
|
|
2673
|
+
result in the report.</li>
|
|
2674
|
+
<li><strong>Secrets on Maestro on TestMu:</strong> a workflow that types a secret is refused there for now (HyperExecute would need
|
|
2675
|
+
the value in its own vault). Replay it on TestMu with <code>--driver appium</code>.</li>
|
|
2676
|
+
<li>The access key is passed to the CLI as an argument. It is removed from the CLI's output and from every file in the
|
|
2677
|
+
run folder. Neither the user name nor the key is passed to the flows' environment.</li>
|
|
2678
|
+
<li><code>.github/workflows/forgeqe-mobile-testmu.yml</code> runs all three on TestMu (Appium, Maestro with a workflow, a team flow)
|
|
2679
|
+
with TestMu's Wikipedia sample app, once the repository has the secrets <code>LT_USERNAME</code> and <code>LT_ACCESS_KEY</code>.</li>
|
|
2680
|
+
</ul>
|
|
2681
|
+
</div>
|
|
2682
|
+
</section>
|
|
2683
|
+
<section class="doc" id="doc-docs-mobile-security">
|
|
2684
|
+
<h4>Security</h4>
|
|
2685
|
+
<div class="prose"><ul>
|
|
2686
|
+
<li><strong>No commands from the workflow or the model.</strong> <code>app:</code> names a configured app (a platform and an id). Appium is
|
|
2687
|
+
spoken to over HTTP, and <code>maestro</code> is started with an argument list, never a shell.</li>
|
|
2688
|
+
<li><strong>Locators</strong> go into XPath as quoted literals, so a target cannot change the query.</li>
|
|
2689
|
+
<li><strong>Secrets:</strong> <code>{{secret:NAME}}</code> reads only <code>mobile.secrets</code> and the environment's secrets, and ForgeQE's own keys are
|
|
2690
|
+
refused there. Explore types a secret from its placeholder, so the model never sees the value.</li>
|
|
2691
|
+
<li><strong>The engine's environment</strong> holds system settings, the profile's key, the listed secrets, the cloud's user and key,
|
|
2692
|
+
and <code>JAVA_HOME</code> / <code>ANDROID_HOME</code>, nothing else.</li>
|
|
2693
|
+
<li><strong>Cloud credentials</strong> come from the variables <code>mobile.appium.username_env</code> and <code>access_key_env</code> name; a user or key in
|
|
2694
|
+
the URL is refused. TLS is verified as usual.</li>
|
|
2695
|
+
<li><strong>App builds</strong> are absolute paths to an app file, https URLs, or a device cloud's own upload ids (<code>bs://</code>, <code>lt://</code>,
|
|
2696
|
+
<code>storage:filename=</code>); <code>file://</code> and other schemes are refused.</li>
|
|
2697
|
+
</ul>
|
|
1914
2698
|
</div>
|
|
1915
2699
|
</section>
|
|
1916
2700
|
</div>
|
|
@@ -1919,12 +2703,12 @@ download the <code>hyperexecute</code> CLI into the folder, then run <code>forge
|
|
|
1919
2703
|
<section class="chapter" id="area-operations" aria-labelledby="area-operations-h">
|
|
1920
2704
|
<header class="chead"><span class="mono">09</span><h2 id="area-operations-h">Operations</h2><p>Monitoring and CI pipelines</p></header>
|
|
1921
2705
|
<article class="spec feat" id="cap-containers">
|
|
1922
|
-
<div class="hd"><span class="mono">Operations</span><h3>Services and
|
|
1923
|
-
<p class="ds">Starts the services a test run needs (a database, a queue, the API under test) in Docker or Podman, as forgeqe.yaml declares them, and waits until each is ready; stops them, and shows their status and logs. Starts mocks of an API on WireMock, from the calls a browser run recorded or from an OpenAPI description, and can point an API app of the environment at the mock for the session.</p>
|
|
2706
|
+
<div class="hd"><span class="mono">Operations</span><h3>Services, mocks, workspace and browser in Docker or Podman</h3></div>
|
|
2707
|
+
<p class="ds">Starts the services a test run needs (a database, a queue, the API under test) in Docker or Podman, as forgeqe.yaml declares them, and waits until each is ready; stops them, and shows their status and logs. Starts mocks of an API on WireMock, from the calls a browser run recorded or from an OpenAPI description, and can point an API app of the environment at the mock for the session. With containers.workspace, the coding agents' shell commands (build, restore, run) and every test ForgeQE runs go to a container of the project's image; with browser.provider docker, the browser agent, the replay and the coding agent's live browser use Chromium in a container.</p>
|
|
1924
2708
|
<div class="facts">
|
|
1925
2709
|
<div class="fact"><span class="mono">Ask it</span><ul class="asks"><li>Start the database and the queue in Docker</li><li>Start the services from our docker-compose file</li><li>Mock the payments API from the last run</li><li>Create a mock service from openapi.yaml and point order-api at it</li><li>Show the logs of the db container</li></ul></div>
|
|
1926
2710
|
<div class="fact"><span class="mono">What it needs</span><ul class="needs"><li><span class="need req">Required</span><span>Docker or Podman on PATH; the services to start in containers.services in forgeqe.yaml (mocks need none)</span></li></ul></div>
|
|
1927
|
-
<div class="fact"><span class="mono">Limits</span><ul class="plainlist"><li>Starts only services declared in forgeqe.yaml containers.services; it never runs an image, flag or mount from a request</li><li>No privileged containers, host network, or access to the container runtime's socket; ports on 127.0.0.1 only; mounts read-only from inside the project</li><li>Mocks answer REST calls (WireMock); GraphQL, SOAP and gRPC are not mocked</li><li>Reads docker-compose files only for their services' image, ports, environment, bind mounts, command and depends_on; refuses build, privileged, network_mode, cap_add, devices and env_file; leaves named volumes out</li></ul></div>
|
|
2711
|
+
<div class="fact"><span class="mono">Limits</span><ul class="plainlist"><li>Starts only services declared in forgeqe.yaml containers.services; it never runs an image, flag or mount from a request</li><li>No privileged containers, host network, or access to the container runtime's socket; ports on 127.0.0.1 only; mounts read-only from inside the project</li><li>Mocks answer REST calls (WireMock); GraphQL, SOAP and gRPC are not mocked</li><li>Reads docker-compose files only for their services' image, ports, environment, bind mounts, command and depends_on; refuses build, privileged, network_mode, cap_add, devices and env_file; leaves named volumes out</li><li>The workspace mounts the repository with its credential files (.env, .npmrc, keys) covered; the agents and the model key stay on this machine; ForgeQE's own agent keeps this machine's shell</li><li>A browser in a container reaches this machine's apps as host.docker.internal, not localhost; it is headless, or visible on a virtual display, never on this screen</li></ul></div>
|
|
1928
2712
|
<div class="fact"><span class="mono">Tools</span><p class="tools"><code>services</code> <code>mock_service</code></p></div>
|
|
1929
2713
|
</div>
|
|
1930
2714
|
<div class="detail"><span class="mono">In detail</span>
|
|
@@ -1932,7 +2716,7 @@ download the <code>hyperexecute</code> CLI into the folder, then run <code>forge
|
|
|
1932
2716
|
<h4>Containers and mock services</h4>
|
|
1933
2717
|
<div class="prose"><p>When Docker or Podman is installed, ForgeQE can start the services a test run needs, and mock an API the tests should
|
|
1934
2718
|
not call. Ask in the shell: "start the database and the queue", "mock the payments API from the last run", "make a mock
|
|
1935
|
-
from openapi.yaml and point order-api at it", "show the db logs". Full reference: <a href="
|
|
2719
|
+
from openapi.yaml and point order-api at it", "show the db logs". Full reference: <a href="#doc-docs-containers-forgeqe-containers-services-and-mocks-in-docker-or-podman">docs/CONTAINERS.md</a>.</p>
|
|
1936
2720
|
<p>The project declares its services in forgeqe.yaml. ForgeQE starts only these, and builds each command itself:</p>
|
|
1937
2721
|
<pre><code class="language-yaml">containers:
|
|
1938
2722
|
# runtime: podman # default: docker, else podman, on PATH
|
|
@@ -1986,6 +2770,27 @@ secret passed by name.</li>
|
|
|
1986
2770
|
</ul>
|
|
1987
2771
|
<p><code>x-forgeqe: { ready: … }</code> sets the ready check. Without it, a service is ready when its first port takes connections.
|
|
1988
2772
|
The project's containers share their own network, so services reach each other by name (<code>db:5432</code>).</p>
|
|
2773
|
+
<p><strong>Docker as the workspace and the browser.</strong> Two settings move the rest of the work into containers:</p>
|
|
2774
|
+
<pre><code class="language-yaml">browser:
|
|
2775
|
+
provider: docker # the browser agent, the replay and Scribe's live browser use Chromium in a container
|
|
2776
|
+
containers:
|
|
2777
|
+
workspace: # Scribe's shell commands and every test ForgeQE runs use this image
|
|
2778
|
+
image: mcr.microsoft.com/playwright/dotnet:v1.63.0-noble
|
|
2779
|
+
env: { NEXUS_ENVIRONMENT: CI }
|
|
2780
|
+
browser: # optional
|
|
2781
|
+
headless: false # visible on a virtual display inside the container (some sites refuse headless)
|
|
2782
|
+
</code></pre>
|
|
2783
|
+
<ul>
|
|
2784
|
+
<li><strong>Workspace:</strong> Scribe builds, restores, runs and fixes tests in a Linux container of the project's image, with the
|
|
2785
|
+
repository at the same path. Each test run gets its own container from that image, and <code>HOME</code> is a volume of the
|
|
2786
|
+
project's, so package caches outlive the session. The agents and the model key stay on this machine, and ForgeQE's
|
|
2787
|
+
own agent keeps this machine's shell. <code>.env</code>, keys and other credential files in the repository are covered inside
|
|
2788
|
+
the containers.</li>
|
|
2789
|
+
<li><strong>Browser:</strong> each browser run gets a fresh container, reached through a gateway that wants a random token. A
|
|
2790
|
+
declared service is reached by its name; an app on this machine is <code>host.docker.internal</code>, not <code>localhost</code>.
|
|
2791
|
+
<code>--browser-provider docker</code> picks it for one run.</li>
|
|
2792
|
+
<li><code>FORGEQE_HOST_RUN=1</code> runs Scribe's commands and the tests on this machine for a session.</li>
|
|
2793
|
+
</ul>
|
|
1989
2794
|
<p><strong>Tests in a container.</strong> With <code>containers.tests</code>, <code>run_tests</code> runs the repository's tests in that image instead of
|
|
1990
2795
|
on this machine, after starting the services it lists:</p>
|
|
1991
2796
|
<pre><code class="language-yaml">containers:
|
|
@@ -2008,6 +2813,312 @@ secrets.</li>
|
|
|
2008
2813
|
for readiness, and uses the mapped port. It never uses privileged mode, the host network, or the Docker socket.</p>
|
|
2009
2814
|
</div>
|
|
2010
2815
|
</section>
|
|
2816
|
+
<section class="doc" id="doc-docs-containers-forgeqe-containers-services-and-mocks-in-docker-or-podman">
|
|
2817
|
+
<h4>ForgeQE containers: services and mocks in Docker or Podman</h4>
|
|
2818
|
+
<div class="prose"><p>Status: the tests run against a fake runtime on any OS. They check the commands ForgeQE builds, the checks on config,
|
|
2819
|
+
compose files, mounts and secrets, the mock stubs, and tests in a container. <code>tests/containers.live.test.ts</code> runs on
|
|
2820
|
+
real Docker (and is skipped without it):</p>
|
|
2821
|
+
<ul>
|
|
2822
|
+
<li>Postgres from forgeqe.yaml and from a compose file.</li>
|
|
2823
|
+
<li>A WireMock mock that answers a recorded run's calls.</li>
|
|
2824
|
+
<li>A tests container that reaches the database by name and writes its results into the repository as this machine's
|
|
2825
|
+
user.</li>
|
|
2826
|
+
</ul>
|
|
2827
|
+
<p><code>.github/workflows/forgeqe-containers.yml</code> runs both on GitHub's Docker. There, a skipped live test fails the job.</p>
|
|
2828
|
+
<p>ForgeQE uses Docker or Podman, when one is installed, for five jobs:</p>
|
|
2829
|
+
<ul>
|
|
2830
|
+
<li><strong>Services a test run needs:</strong> a database, a queue, a cache, or the API under test. The project declares them in
|
|
2831
|
+
forgeqe.yaml, or in a docker-compose file that forgeqe.yaml names.</li>
|
|
2832
|
+
<li><strong>Mocks of an API:</strong> WireMock stubs made from the calls a browser run recorded, or from an OpenAPI description, so
|
|
2833
|
+
the tests can run without the real service.</li>
|
|
2834
|
+
<li><strong>The repository's tests in a container:</strong> <code>run_tests</code> runs the command the user confirmed in an image, instead of
|
|
2835
|
+
on this machine, after starting the services the tests need.</li>
|
|
2836
|
+
<li><strong>A workspace for the coding agents:</strong> with <code>containers.workspace</code>, Scribe's shell commands (build, restore, run,
|
|
2837
|
+
git) and every test ForgeQE runs for it go to a container of the project's image. The agents stay on this machine.</li>
|
|
2838
|
+
<li><strong>A browser in a container:</strong> with <code>browser.provider: docker</code>, the browser agent, the replay and Scribe's live
|
|
2839
|
+
browser drive Chromium in a container of Microsoft's Playwright image.</li>
|
|
2840
|
+
</ul>
|
|
2841
|
+
<p>Scribe also has a <code>testcontainers</code> skill, for tests that start their own services from code (below).</p>
|
|
2842
|
+
<p>ForgeQE's agent uses them through two tools, <code>services</code> and <code>mock_service</code>. The CLI has <code>forgeqe services</code>. When
|
|
2843
|
+
neither Docker nor Podman is on PATH, the agent isn't offered these tools.</p>
|
|
2844
|
+
</div>
|
|
2845
|
+
</section>
|
|
2846
|
+
<section class="doc" id="doc-docs-containers-use">
|
|
2847
|
+
<h4>Use</h4>
|
|
2848
|
+
<div class="prose"><pre><code class="language-bash">forgeqe services # the declared services, what runs, whether the runtime answers
|
|
2849
|
+
forgeqe services up # start every declared service and wait until each is ready
|
|
2850
|
+
forgeqe services up db queue # just these
|
|
2851
|
+
forgeqe services logs db --lines 50 # last lines, secrets removed
|
|
2852
|
+
forgeqe services down # stop and remove this project's containers (mocks included)
|
|
2853
|
+
</code></pre>
|
|
2854
|
+
<p>In the shell or <code>forgeqe run</code>, you can ask, for example:</p>
|
|
2855
|
+
<ul>
|
|
2856
|
+
<li>"start the database"</li>
|
|
2857
|
+
<li>"is the queue up?"</li>
|
|
2858
|
+
<li>"mock the payments API from the last run"</li>
|
|
2859
|
+
<li>"make a mock service from openapi.yaml and point order-api at it"</li>
|
|
2860
|
+
<li>"stop the services"</li>
|
|
2861
|
+
</ul>
|
|
2862
|
+
<p>The agent asks before it starts anything.</p>
|
|
2863
|
+
</div>
|
|
2864
|
+
</section>
|
|
2865
|
+
<section class="doc" id="doc-docs-containers-declaring-services">
|
|
2866
|
+
<h4>Declaring services</h4>
|
|
2867
|
+
<div class="prose"><pre><code class="language-yaml"># forgeqe.yaml
|
|
2868
|
+
containers:
|
|
2869
|
+
runtime: docker # docker | podman; default: docker, else podman, on PATH
|
|
2870
|
+
services:
|
|
2871
|
+
db:
|
|
2872
|
+
description: Orders database for the API tests
|
|
2873
|
+
image: postgres:16.4 # an exact version, or name@sha256:<digest>
|
|
2874
|
+
ports: [5432] # 127.0.0.1:5432 -> 5432; "15432:5432" maps another port of this machine
|
|
2875
|
+
env: # plain values only
|
|
2876
|
+
POSTGRES_DB: shop
|
|
2877
|
+
POSTGRES_USER: shop
|
|
2878
|
+
secrets: [POSTGRES_PASSWORD] # environment variable names; their values come from the environment or .env
|
|
2879
|
+
volumes:
|
|
2880
|
+
- ./db/init:/docker-entrypoint-initdb.d # read-only; a folder or file inside the project
|
|
2881
|
+
command: ["-c", "log_statement=all"] # arguments for the image's entrypoint
|
|
2882
|
+
ready: { tcp: 5432 } # or { http: /health, port: 8080 }; without it: the container runs
|
|
2883
|
+
timeout: 90 # seconds to wait until ready (default 60)
|
|
2884
|
+
queue:
|
|
2885
|
+
image: rabbitmq:3.13.7-management
|
|
2886
|
+
ports: [5672, 15672]
|
|
2887
|
+
ready: { http: /, port: 15672 }
|
|
2888
|
+
mock:
|
|
2889
|
+
image: wiremock/wiremock:3.9.1 # the default
|
|
2890
|
+
</code></pre>
|
|
2891
|
+
<div class="tbl"><table>
|
|
2892
|
+
<thead>
|
|
2893
|
+
<tr><th>Setting</th><th>Rule</th></tr>
|
|
2894
|
+
</thead>
|
|
2895
|
+
<tbody>
|
|
2896
|
+
<tr><td>Service name</td><td>Lower-case letters, digits and <code>-</code>, starting with a letter. Names starting with <code>mock-</code> are kept for mocks.</td></tr>
|
|
2897
|
+
<tr><td><code>image</code></td><td>Must have an exact tag or a digest. An untagged image or <code>latest</code> is refused, because it changes under the same name.</td></tr>
|
|
2898
|
+
<tr><td><code>ports</code></td><td>Always bound to 127.0.0.1. An address such as <code>0.0.0.0:5432:5432</code> is refused.</td></tr>
|
|
2899
|
+
<tr><td><code>env</code></td><td>Plain values on one line. Secrets go under <code>secrets</code>.</td></tr>
|
|
2900
|
+
<tr><td><code>secrets</code></td><td>Variable names. They are passed to the runtime by name (<code>--env NAME</code>), so the value is never on a command line. One of ForgeQE's own keys (a model key, a tracker token) is refused.</td></tr>
|
|
2901
|
+
<tr><td><code>volumes</code></td><td>Always read-only. Must be inside the folder of forgeqe.yaml, and can't be the whole folder.</td></tr>
|
|
2902
|
+
<tr><td><code>command</code></td><td>A list of arguments, never one command string.</td></tr>
|
|
2903
|
+
<tr><td><code>depends_on</code></td><td>Services to start first.</td></tr>
|
|
2904
|
+
<tr><td>Anything else</td><td>Refused with an error, not ignored. That includes <code>privileged</code>, <code>network</code>, <code>cap_add</code>, <code>devices</code>, <code>user</code>, <code>pid</code> and the socket.</td></tr>
|
|
2905
|
+
</tbody>
|
|
2906
|
+
</table></div>
|
|
2907
|
+
<p>Every service, whether from forgeqe.yaml or a compose file, can list <code>depends_on</code>. <code>up</code> starts a service after the
|
|
2908
|
+
ones it depends on; a circle is refused.</p>
|
|
2909
|
+
</div>
|
|
2910
|
+
</section>
|
|
2911
|
+
<section class="doc" id="doc-docs-containers-compose-files">
|
|
2912
|
+
<h4>Compose files</h4>
|
|
2913
|
+
<div class="prose"><pre><code class="language-yaml">containers:
|
|
2914
|
+
compose: docker-compose.yml # or a list; inside the project
|
|
2915
|
+
</code></pre>
|
|
2916
|
+
<p>ForgeQE reads each service of the file and translates it into a forgeqe.yaml service. The same rules then check it,
|
|
2917
|
+
and a service declared in forgeqe.yaml wins over the compose file's service of the same name. Compose itself never
|
|
2918
|
+
runs.</p>
|
|
2919
|
+
<div class="tbl"><table>
|
|
2920
|
+
<thead>
|
|
2921
|
+
<tr><th>Compose</th><th>ForgeQE</th></tr>
|
|
2922
|
+
</thead>
|
|
2923
|
+
<tbody>
|
|
2924
|
+
<tr><td><code>image</code></td><td>Required; an exact version or a digest. <code>build</code> is refused: build and tag the image first.</td></tr>
|
|
2925
|
+
<tr><td><code>ports</code></td><td><code>"5432"</code>, <code>"15432:5432"</code>, <code>"127.0.0.1:15432:5432"</code>, or the long form; always bound to 127.0.0.1. Any other address, and UDP, are refused.</td></tr>
|
|
2926
|
+
<tr><td><code>environment</code></td><td>Plain values are passed as they are. <code>KEY: ${KEY}</code>, <code>KEY</code> alone in the list form, and <code>KEY: ${KEY:-default}</code> pass the variable by name as a secret. <code>${OTHER}</code> under another name, and values that mix text and <code>${…}</code>, are refused.</td></tr>
|
|
2927
|
+
<tr><td><code>volumes</code></td><td>Bind mounts (<code>./db/init:/docker-entrypoint-initdb.d</code>) are mounted read-only, and <code>:rw</code> is noted. Named and anonymous volumes are left out, so the container keeps no data between runs. A path outside the project is refused when the service starts.</td></tr>
|
|
2928
|
+
<tr><td><code>command</code></td><td>A list, or a string split into arguments (compose uses no shell, and neither does ForgeQE).</td></tr>
|
|
2929
|
+
<tr><td><code>depends_on</code></td><td>List or map form: the start order. <code>condition</code> is not used; ForgeQE waits until each service is ready.</td></tr>
|
|
2930
|
+
<tr><td><code>x-forgeqe</code></td><td><code>ready</code>, <code>timeout</code>, <code>description</code>, as in forgeqe.yaml. Without <code>ready</code>, a service with ports is ready when its first port takes connections.</td></tr>
|
|
2931
|
+
<tr><td><code>container_name</code>, <code>restart</code>, <code>labels</code>, <code>healthcheck</code>, <code>networks</code>, <code>expose</code>, <code>deploy</code>, <code>profiles</code>, …</td><td>Ignored: ForgeQE names, labels and networks the containers itself.</td></tr>
|
|
2932
|
+
<tr><td><code>privileged</code>, <code>network_mode</code>, <code>cap_add</code>, <code>devices</code>, <code>pid</code>, <code>ipc</code>, <code>user</code>, <code>security_opt</code>, <code>volumes_from</code>, <code>env_file</code>, <code>entrypoint</code>, <code>extends</code>, …</td><td>Refused, and the error names the setting.</td></tr>
|
|
2933
|
+
</tbody>
|
|
2934
|
+
</table></div>
|
|
2935
|
+
<p>What was left out (a named volume, a mount made read-only) is listed in <code>notes</code>.</p>
|
|
2936
|
+
</div>
|
|
2937
|
+
</section>
|
|
2938
|
+
<section class="doc" id="doc-docs-containers-the-project-network">
|
|
2939
|
+
<h4>The project network</h4>
|
|
2940
|
+
<div class="prose"><p>Each project's containers join their own network, <code>forgeqe-<project></code>, under their service name. So the API container
|
|
2941
|
+
reaches the database as <code>db:5432</code>, and the tests container reaches both. No other container joins the network, and no
|
|
2942
|
+
container is on the host's network. <code>forgeqe services down</code> with no names also removes the network.</p>
|
|
2943
|
+
</div>
|
|
2944
|
+
</section>
|
|
2945
|
+
<section class="doc" id="doc-docs-containers-tests-in-a-container">
|
|
2946
|
+
<h4>Tests in a container</h4>
|
|
2947
|
+
<div class="prose"><pre><code class="language-yaml">containers:
|
|
2948
|
+
tests:
|
|
2949
|
+
image: mcr.microsoft.com/playwright:v1.48.2-jammy # an exact version; the tests' toolchain
|
|
2950
|
+
services: [db, order-api] # started first (when not running yet); the tests reach them by name
|
|
2951
|
+
env: { DB_HOST: db }
|
|
2952
|
+
secrets: [TEST_DB_PASSWORD]
|
|
2953
|
+
</code></pre>
|
|
2954
|
+
<p>With <code>containers.tests</code> set, <code>run_tests</code> runs the tests in that image unless it is called with <code>in_container: false</code>.
|
|
2955
|
+
Its confirmation says where the tests will run. The container:</p>
|
|
2956
|
+
<ul>
|
|
2957
|
+
<li><strong>Runs the command the user confirmed</strong> with the image's own shell, <code>sh -c "<command>"</code>, inside the container,
|
|
2958
|
+
never on this machine.</li>
|
|
2959
|
+
<li><strong>Mounts the repository read-write at its own path</strong> and works in it, so results land where ForgeQE reads them.
|
|
2960
|
+
On this machine, the tests already have the whole disk. A folder outside the repository that the results go to is
|
|
2961
|
+
mounted the same way.</li>
|
|
2962
|
+
<li><strong>Runs as this machine's user</strong> (<code>--user uid:gid</code>), so files it writes are yours. It also gets
|
|
2963
|
+
<code>--security-opt no-new-privileges</code>, <code>--pids-limit 4096</code> and <code>--rm</code>, and it is removed by name when the run ends or is
|
|
2964
|
+
stopped.</li>
|
|
2965
|
+
<li><strong>Gets only these variables:</strong> <code>CI=1</code>, <code>NO_COLOR</code>, <code>HOME=/tmp</code>, the environment's <code>FORGEQE_ACTIVE_ENV</code>,
|
|
2966
|
+
<code>FORGEQE_APP_*</code> and <code>FORGEQE_VAR_*</code>, the saved logins' variables, and <code>containers.tests</code> <code>env</code> and <code>secrets</code>.
|
|
2967
|
+
Secrets are passed by name. ForgeQE's own keys are never passed.</li>
|
|
2968
|
+
<li><strong>Has secret values removed from its output</strong>, as on this machine.</li>
|
|
2969
|
+
</ul>
|
|
2970
|
+
<p>This needs a Linux or macOS machine, because the repository is mounted at its own path.</p>
|
|
2971
|
+
</div>
|
|
2972
|
+
</section>
|
|
2973
|
+
<section class="doc" id="doc-docs-containers-the-workspace">
|
|
2974
|
+
<h4>The workspace</h4>
|
|
2975
|
+
<div class="prose"><pre><code class="language-yaml">containers:
|
|
2976
|
+
workspace:
|
|
2977
|
+
image: mcr.microsoft.com/playwright/dotnet:v1.63.0-noble # an exact version; the project's toolchain
|
|
2978
|
+
services: [db] # started first; reached by name
|
|
2979
|
+
env: { NEXUS_ENVIRONMENT: CI, DOTNET_ROLL_FORWARD: Major }
|
|
2980
|
+
secrets: [TEST_DB_PASSWORD] # by name; values from the environment or .env
|
|
2981
|
+
</code></pre>
|
|
2982
|
+
<p>With <code>containers.workspace</code> set:</p>
|
|
2983
|
+
<ul>
|
|
2984
|
+
<li><strong>Scribe's shell runs in the container.</strong> The agent runtime runs each command as <code>$SHELL -c <command></code>. ForgeQE
|
|
2985
|
+
sets <code>SHELL</code> to a script that runs the same command with <code>docker exec … sh -c <command></code>, in the same folder,
|
|
2986
|
+
in the workspace container. A folder outside the repository is refused, since the container has only the
|
|
2987
|
+
repository. The agent is told its commands run in a Linux container of that image.</li>
|
|
2988
|
+
<li><strong>Every test ForgeQE runs goes to the same image:</strong> the coding loop's runs, <code>heal</code>, <code>suite</code>'s existing tests and
|
|
2989
|
+
<code>run_tests</code>. Each run gets its own container (<code>docker run --rm</code>), so stopping a run removes it. Browsers the test
|
|
2990
|
+
starts are found with <code>docker top</code>, as ForgeQE finds them on this machine.</li>
|
|
2991
|
+
<li><strong>What stays on this machine:</strong> the agent runtime, the model key, ForgeQE's other secrets, the file tools (read,
|
|
2992
|
+
edit, write; the repository is the same folder in both places), and ForgeQE's own agent, which works with you
|
|
2993
|
+
on this machine and keeps its shell here. <code>FORGEQE_HOST_RUN=1</code> runs everything on this machine for a session.</li>
|
|
2994
|
+
<li><strong>The container:</strong>
|
|
2995
|
+
<ul>
|
|
2996
|
+
<li>one per ForgeQE process and project, <code>forgeqe-<project>-workspace-<id></code>, removed when ForgeQE ends;</li>
|
|
2997
|
+
<li>the repository mounted read-write at its own path; credential files in it (<code>.env</code>, <code>.env.*</code> except templates,
|
|
2998
|
+
<code>.npmrc</code>, <code>.netrc</code>, <code>*.pem</code>, <code>*.key</code>, <code>id_*</code> keys) covered with an empty read-only file, and <code>.ssh</code>, <code>.aws</code>,
|
|
2999
|
+
<code>.azure</code>, <code>.docker</code>, <code>.kube</code> and <code>.forgeqe/sessions</code> with an empty folder;</li>
|
|
3000
|
+
<li><code>HOME</code> on the project's volume <code>forgeqe-<project>-home</code>, so NuGet, npm and pip caches outlive the session. Docker
|
|
3001
|
+
creates the volume for root, so ForgeQE runs <code>chown</code> in it once, with a fixed command;</li>
|
|
3002
|
+
<li>this machine's user (<code>--user uid:gid</code>), <code>no-new-privileges</code>, <code>--pids-limit 4096</code>, the project network;</li>
|
|
3003
|
+
<li>only <code>CI=1</code>, <code>HOME</code>, the <code>env</code> and <code>secrets</code> listed, and, in each command, the environment's <code>FORGEQE_APP_*</code> and
|
|
3004
|
+
<code>FORGEQE_VAR_*</code> and the saved logins' variables, by name.</li>
|
|
3005
|
+
</ul>
|
|
3006
|
+
</li>
|
|
3007
|
+
<li><strong><code>containers.tests</code></strong> still names the image <code>run_tests</code> uses when it is set. Without it, <code>run_tests</code> uses the
|
|
3008
|
+
workspace's image.</li>
|
|
3009
|
+
</ul>
|
|
3010
|
+
<p>This needs a Linux or macOS machine, as tests in a container do.</p>
|
|
3011
|
+
</div>
|
|
3012
|
+
</section>
|
|
3013
|
+
<section class="doc" id="doc-docs-containers-the-browser-in-a-container">
|
|
3014
|
+
<h4>The browser in a container</h4>
|
|
3015
|
+
<div class="prose"><pre><code class="language-yaml">browser:
|
|
3016
|
+
provider: docker # or per run: --browser-provider docker
|
|
3017
|
+
containers:
|
|
3018
|
+
browser: # optional
|
|
3019
|
+
image: mcr.microsoft.com/playwright:v1.63.0-noble # default; needs Node.js and /ms-playwright
|
|
3020
|
+
headless: false # default: browser.headless; a visible browser runs on a virtual display (Xvfb)
|
|
3021
|
+
</code></pre>
|
|
3022
|
+
<ul>
|
|
3023
|
+
<li><strong>One container per use:</strong> each browser run (a suite runs <code>suite.parallel</code> at once) gets its own, and a coding job
|
|
3024
|
+
gets one that its replay and live browser share. Each is removed when its run or job ends.</li>
|
|
3025
|
+
<li><strong>Connection:</strong> Chromium listens on 127.0.0.1 inside the container. CDP has no authentication of its own, so a small
|
|
3026
|
+
gateway in front of it (<code>BROWSER_SERVER</code> in <code>src/containers/browser.ts</code>, run with the image's Node.js) accepts a
|
|
3027
|
+
connection only with a random 48-character token in its path, compared in constant time, and refuses a browser
|
|
3028
|
+
<code>Origin</code> that is not local. The port is published on 127.0.0.1 only. The token reaches the engine, the replay and
|
|
3029
|
+
the live browser's MCP server through the environment, never a file or a command line.</li>
|
|
3030
|
+
<li><strong>Visible or headless:</strong> a visible browser runs on Xvfb inside the container. Sites that turn headless browsers
|
|
3031
|
+
away see an ordinary one, but you do not see it on your screen. Runs keep screenshots and traces as usual.</li>
|
|
3032
|
+
<li><strong>Reaching the app:</strong> the container joins the project network, so a declared service is reached by its name
|
|
3033
|
+
(<code>http://order-api:8080</code>). An app this machine serves is <code>http://host.docker.internal:<port></code>; <code>localhost</code> is the
|
|
3034
|
+
container itself, and ForgeQE warns when a run starts there.</li>
|
|
3035
|
+
<li><strong>Replay:</strong> one pass, in a new browser context. The second pass (without Playwright's automation switches) needs a
|
|
3036
|
+
launch, which a running browser does not have.</li>
|
|
3037
|
+
<li>A run explored in a container is replayed in one even when the provider is changed back later (<code>run.json</code>).</li>
|
|
3038
|
+
</ul>
|
|
3039
|
+
</div>
|
|
3040
|
+
</section>
|
|
3041
|
+
<section class="doc" id="doc-docs-containers-testcontainers-scribe">
|
|
3042
|
+
<h4>Testcontainers (Scribe)</h4>
|
|
3043
|
+
<div class="prose"><p>When a test needs a real service, or the repository already uses Testcontainers (Java, .NET, Node, Python, Go), Scribe
|
|
3044
|
+
loads the <code>testcontainers</code> skill. It reuses the repository's containers and fixtures, pins images, waits for
|
|
3045
|
+
readiness, uses the mapped port, and sets the app's own configuration from it. It never uses privileged mode, the
|
|
3046
|
+
host network, the Docker socket, or a <code>docker</code> command from a test. These containers belong to the tests in the
|
|
3047
|
+
repository: the tests start and stop them, wherever they run.</p>
|
|
3048
|
+
</div>
|
|
3049
|
+
</section>
|
|
3050
|
+
<section class="doc" id="doc-docs-containers-mocks">
|
|
3051
|
+
<h4>Mocks</h4>
|
|
3052
|
+
<div class="prose"><p><code>mock_service</code> builds WireMock stubs and runs WireMock on a free port of 127.0.0.1:</p>
|
|
3053
|
+
<ul>
|
|
3054
|
+
<li><strong>From a run</strong> (the default: the latest run, or <code>run_dir</code>):
|
|
3055
|
+
<ul>
|
|
3056
|
+
<li>One stub per method, path and query of the app's own API calls, answering with the last status, content type
|
|
3057
|
+
and body the app got.</li>
|
|
3058
|
+
<li>Other sites' calls are left out unless <code>third_party</code> is set.</li>
|
|
3059
|
+
<li>Failed calls are left out.</li>
|
|
3060
|
+
<li>Answers that were cut when recorded are listed under <code>skipped</code>.</li>
|
|
3061
|
+
</ul>
|
|
3062
|
+
</li>
|
|
3063
|
+
<li><strong>From a description</strong> (<code>spec</code>: OpenAPI 3 or Swagger 2, a link or a file):
|
|
3064
|
+
<ul>
|
|
3065
|
+
<li>One stub per REST operation.</li>
|
|
3066
|
+
<li>Each stub answers with the operation's first success status and a body from the schema's examples, or values
|
|
3067
|
+
made from its types.</li>
|
|
3068
|
+
<li>A path parameter (<code>/orders/{id}</code>) matches any one path segment.</li>
|
|
3069
|
+
<li>Instead of a description, you can give a contract set from <code>api_contracts</code> (<code>set_id</code>).</li>
|
|
3070
|
+
</ul>
|
|
3071
|
+
</li>
|
|
3072
|
+
<li><strong>GraphQL, SOAP and gRPC</strong> are listed under <code>skipped</code>, not mocked.</li>
|
|
3073
|
+
</ul>
|
|
3074
|
+
<p>A spec given as a link is fetched by ForgeQE itself, with no credentials, as <code>api_contracts</code> does. The stubs are written to <code>.forgeqe/mocks/<name>/mappings/</code>. You can read and edit them there. The folder is mounted
|
|
3075
|
+
read-only at <code>/home/wiremock/mappings</code>. WireMock runs with <code>--enable-stub-cors</code>, so a web app on another local port
|
|
3076
|
+
can call it.</p>
|
|
3077
|
+
<p>With <code>for_app: <api app></code>, the active environment's API app points at the mock for the rest of the session. That
|
|
3078
|
+
covers the API tools, and the tests ForgeQE runs, which read <code>FORGEQE_APP_<APP>_URL</code>. env.yaml itself is not
|
|
3079
|
+
changed.</p>
|
|
3080
|
+
</div>
|
|
3081
|
+
</section>
|
|
3082
|
+
<section class="doc" id="doc-docs-containers-lifetime">
|
|
3083
|
+
<h4>Lifetime</h4>
|
|
3084
|
+
<div class="prose"><ul>
|
|
3085
|
+
<li><strong>Services and mocks the agent starts</strong> are removed when the shell, <code>forgeqe run</code>, or <code>forgeqe mcp</code> ends, as are
|
|
3086
|
+
the workspace and browser containers (<code>forgeqe.kind</code> <code>workspace</code> and <code>browser</code>). The home volume stays.</li>
|
|
3087
|
+
<li><strong><code>forgeqe services up</code></strong> keeps its services running after the command ends, until <code>forgeqe services down</code>.</li>
|
|
3088
|
+
<li><strong>Container names</strong> are <code>forgeqe-<project>-<service></code>. <code><project></code> is a hash of the project folder, so two projects
|
|
3089
|
+
never share a container. Starting a service replaces its earlier container. Removing a container also removes the anonymous volumes its image declares (a database's data folder), so nothing is left on the machine. A container is never reused with old
|
|
3090
|
+
settings.</li>
|
|
3091
|
+
<li><strong>Labels:</strong> each container carries <code>forgeqe=1</code>, <code>forgeqe.project</code>, <code>forgeqe.service</code> and <code>forgeqe.kind</code>.
|
|
3092
|
+
<code>down</code> and <code>status</code> look only at this project's containers.</li>
|
|
3093
|
+
</ul>
|
|
3094
|
+
</div>
|
|
3095
|
+
</section>
|
|
3096
|
+
<section class="doc" id="doc-docs-containers-security">
|
|
3097
|
+
<h4>Security</h4>
|
|
3098
|
+
<div class="prose"><p>ForgeQE's agent reads text other people control: tickets, web pages, repositories. A container command built from that
|
|
3099
|
+
text could mount the whole disk or run privileged, so:</p>
|
|
3100
|
+
<ul>
|
|
3101
|
+
<li><strong>No commands from the model.</strong> The agent names services that forgeqe.yaml declares, and ForgeQE builds the
|
|
3102
|
+
argument array (<code>docker run …</code>). It never uses a shell. Images, flags and mounts never come from a request.</li>
|
|
3103
|
+
<li><strong>The same restrictions for every container:</strong> <code>--security-opt no-new-privileges</code>, <code>--pids-limit 1024</code>, ports on
|
|
3104
|
+
127.0.0.1, and read-only mounts. There is no setting to turn these off.</li>
|
|
3105
|
+
<li><strong>Mounts are checked like uploads</strong> (except a mock's stubs folder, which ForgeQE writes itself and mounts read-only). The path is resolved with <code>realpath</code> and must be inside the project. A mount
|
|
3106
|
+
that is or holds a hidden file or folder (<code>.env</code>, <code>.git</code>, <code>.npmrc</code>), a file named like a key (<code>*.pem</code>, <code>id_rsa</code>,
|
|
3107
|
+
<code>llms.json</code>) or a link is refused. A container can send what it reads anywhere.</li>
|
|
3108
|
+
<li><strong>Environment:</strong>
|
|
3109
|
+
<ul>
|
|
3110
|
+
<li>The <code>docker</code> and <code>podman</code> programs see only the system settings and the variables they need to find their daemon
|
|
3111
|
+
(<code>DOCKER_HOST</code>, <code>DOCKER_CONTEXT</code>, <code>CONTAINER_HOST</code>, …), plus the service's listed secrets.</li>
|
|
3112
|
+
<li>A container gets only its <code>env</code> and <code>secrets</code>.</li>
|
|
3113
|
+
<li>Logs and errors have secret values replaced with <code>***</code>.</li>
|
|
3114
|
+
</ul>
|
|
3115
|
+
</li>
|
|
3116
|
+
<li><strong>Not in the agents' shell:</strong> <code>docker</code>, <code>podman</code>, <code>nerdctl</code>, <code>docker-compose</code> and <code>podman-compose</code> are denied in the
|
|
3117
|
+
shell of ForgeQE's agent and of Scribe. So the only way to start a container is these tools.</li>
|
|
3118
|
+
<li><strong>Asked first:</strong> <code>services up</code> and <code>mock_service</code> ask the user before starting, as <code>run_tests</code> does.</li>
|
|
3119
|
+
</ul>
|
|
3120
|
+
</div>
|
|
3121
|
+
</section>
|
|
2011
3122
|
</div>
|
|
2012
3123
|
</article>
|
|
2013
3124
|
<article class="spec feat" id="cap-monitoring">
|
|
@@ -2155,7 +3266,7 @@ forgeqe run "The ForgeQE workflow is failing, fix it"
|
|
|
2155
3266
|
<li>Stores the keys the pipeline needs (the model's key from <code>llms.json</code>, and <code>ADO_PAT</code>, <code>JIRA_EMAIL</code> and <code>JIRA_API_TOKEN</code>, or <code>XRAY_CLIENT_ID</code> and <code>XRAY_CLIENT_SECRET</code>) with ForgeQE's <code>set_ci_secret</code> tool. The agent only names the variable; ForgeQE passes the value from its own environment to <code>gh secret set</code> (on stdin) or <code>az pipelines variable</code> (through the environment), and never shows it. A variable that is not set in ForgeQE's environment is reported, not guessed.</li>
|
|
2156
3267
|
<li>Triggers the run with <code>gh workflow run</code> or <code>az pipelines run</code>, watches it, reads the logs of failed steps, fixes the pipeline (or the code), pushes again, and repeats until the run passes or <code>code.max_attempts</code> runs have failed. It reports the run link and the result.</li>
|
|
2157
3268
|
</ol>
|
|
2158
|
-
<p>It needs <code>gh</code> logged in for GitHub, or <code>az</code> with the <code>azure-devops</code> extension logged in for Azure DevOps. Without them it still writes, validates, and commits the pipeline, then tells you the commands to run. Set <code>llm.active</code> to a live profile in the committed <code>forgeqe.yaml</code>. With a <code>github_copilot</code> profile the pipeline needs
|
|
3269
|
+
<p>It needs <code>gh</code> logged in for GitHub, or <code>az</code> with the <code>azure-devops</code> extension logged in for Azure DevOps. Without them it still writes, validates, and commits the pipeline, then tells you the commands to run. Set <code>llm.active</code> to a live profile in the committed <code>forgeqe.yaml</code>. With a <code>github_copilot</code> profile the pipeline needs a GitHub token for Copilot in <code>COPILOT_GITHUB_TOKEN</code> (see <a href="../HOW_TO_USE.md#github-copilot-seats-as-the-model">In pipelines</a>).</p>
|
|
2159
3270
|
</div>
|
|
2160
3271
|
</section>
|
|
2161
3272
|
</div>
|
|
@@ -2631,7 +3742,7 @@ tokens and the like) show as <code>•••</code> in results. A check compares
|
|
|
2631
3742
|
┃ Handoff .forgeqe/reports/20260928-172700/handoff.md
|
|
2632
3743
|
</code></pre>
|
|
2633
3744
|
<p>When no test was written, the Test row says why: the run did not pass, the session has <code>--no-code</code>, or the run was not handed to the coding agent.</p>
|
|
2634
|
-
<p>Scribe uses the active <code>llms.json</code> profile, or, with <code>llm.routing: auto</code>, the profile the router picks for the <code>coding</code> area (<code>--profile</code> and <code>code.model</code> still win; see <a href="
|
|
3745
|
+
<p>Scribe uses the active <code>llms.json</code> profile, or, with <code>llm.routing: auto</code>, the profile the router picks for the <code>coding</code> area (<code>--profile</code> and <code>code.model</code> still win; see <a href="#doc-docs-model_routing-model-routing-the-right-configured-model-for-each-task">docs/MODEL_ROUTING.md</a>). The key stays in your environment: the agent runtime's config refers to it as <code>{env:WP_AZURE_API_KEY}</code> and never contains the value. An <code>auth: entra</code> profile reaches the model through ForgeQE's local token gateway (see <a href="../HOW_TO_USE.md#signing-in-to-the-model-without-a-key-microsoft-entra-id">Signing in to the model without a key</a>). An Azure profile goes through the agent runtime's <code>azure</code> provider on <code><endpoint>/openai/v1</code> (the Responses API, with the deployment as the model). <code>openai_compatible</code> and <code>ollama</code> profiles use their <code>base_url</code>. A <code>github_copilot</code> profile cannot run Scribe yet, and <code>code.model: github-copilot/...</code> is refused: the agent runtime's own Copilot provider signs in through a third-party app on your GitHub account. <code>code.model: provider/model</code> (or <code>--code-model</code>) uses one of the agent runtime's built-in providers instead, with that provider's usual key variable (for example <code>ANTHROPIC_API_KEY</code> for <code>anthropic/...</code>).</p>
|
|
2635
3746
|
<p>Scribe may edit files and run commands anywhere in the repo without asking. <code>git push</code> in any form is denied (pushing goes through ForgeQE's guarded <code>push_branch</code> tool, section 8), and it cannot touch files outside the repo except the run and memory directories, which it may read.</p>
|
|
2636
3747
|
<p>More ways to run it:</p>
|
|
2637
3748
|
<pre><code class="language-bash">forgeqe code --repo ../app # the latest run in .forgeqe/reports/
|
|
@@ -2643,6 +3754,28 @@ forgeqe run "..." --no-code # no tests
|
|
|
2643
3754
|
<p>If Scribe's job is cut off (ForgeQE closed, the machine restarted, you stopped it, or the model could not be reached), run it again for the same run and repo, for example <code>forgeqe code .forgeqe/reports/20260928-172700 --repo ../app</code>. Scribe reopens the same session, with everything it read and said, and carries on with the files it already wrote; it doesn't read the repo again, and ForgeQE uses the Playwright replay it already finished. The session is kept in <code>scribe-session.json</code> in the run folder until the job ends with a result.</p>
|
|
2644
3755
|
</div>
|
|
2645
3756
|
</section>
|
|
3757
|
+
<section class="doc" id="doc-how_to_use-how-scribe-writes-a-test">
|
|
3758
|
+
<h4>How Scribe writes a test</h4>
|
|
3759
|
+
<div class="prose"><p><strong>Reuse before rebuild, extend before duplicate.</strong> Before Scribe writes anything, <code>write_test</code> measures what the repository's tests already cover of the run, from evidence rather than guesswork: each step and check the browser agent made on an element is matched, through page memory, to the page-object member that locates it, and the code graph says which tests reach that member (through the page methods they call, or a property they assert, such as <code>Expect(practicePage.AjaxStatus)</code>). Steps on elements no page object locates yet are listed as not measured, since a test may do them its own way. Then:</p>
|
|
3760
|
+
<div class="tbl"><table>
|
|
3761
|
+
<thead>
|
|
3762
|
+
<tr><th>What it finds</th><th>What happens</th></tr>
|
|
3763
|
+
</thead>
|
|
3764
|
+
<tbody>
|
|
3765
|
+
<tr><td>One test covers every measured step and check</td><td><strong>Reuse</strong>: Scribe only finds the command that runs it, and changes nothing. If it fails, nothing is changed: you are asked whether to update it or write a new test.</td></tr>
|
|
3766
|
+
<tr><td>One test covers most (40% or more)</td><td><strong>Extend</strong>: Scribe adds only the missing steps and checks to that test, keeping every step and assertion it has; the reviewer rejects a removed or weakened assertion, or the steps copied into a new file</td></tr>
|
|
3767
|
+
<tr><td>Two or more tests cover most of it</td><td><strong>You choose</strong>: the evidence and the options (reuse, extend with how many steps it adds, a new test, cancel)</td></tr>
|
|
3768
|
+
<tr><td>Nothing covers much</td><td><strong>New</strong>: a new test, reusing the page objects and helpers as before</td></tr>
|
|
3769
|
+
</tbody>
|
|
3770
|
+
</table></div>
|
|
3771
|
+
<p>A card says which it did and why (<em>Extending PracticeTests.Flow · covers 12 of 15; only 3 are missing</em>). Say how when you know ("extend the login test", "write a separate test", "update it") and ForgeQE's agent passes that on without asking. In every mode Scribe adds page-object members rather than changing what an existing member does for the other tests that use it.</p>
|
|
3772
|
+
<p><strong>The same flow asked again.</strong> When a session writes tests and you ask for a flow an earlier job already wrote a test for, ForgeQE runs that test first, before any browser run. If it passes in a real browser, that is the answer: no browser run, no Scribe, and no new test file, in the seconds the test takes (<em>✓ Existing test passed · 19s · no browser run or new test needed</em>). If it fails, the flow runs in the browser and Scribe updates that test instead of writing another. ForgeQE's agent rewords a goal a little each time, so goals count as the same flow when they share their content words (endings, case, and phrasing aside) and the new goal keeps everything the earlier one checked; the same steps with a different final check, a shorter flow, or another site are a new flow. The agent also compares the earlier goal with your request and runs fresh when you ask for a check the earlier test does not have. Ask for "a fresh run" or "a new test" to skip the reuse.</p>
|
|
3773
|
+
<p><strong>When the flow cannot be reproduced.</strong> If ForgeQE's own replay of the run could not reach the run's final page and Scribe's first test fails too, the job stops there and says which replayed step failed and why, instead of spending its remaining attempts on a flow no script can repeat as recorded.</p>
|
|
3774
|
+
<p><strong>Why a test failed.</strong> When ForgeQE's run of Scribe's test fails, the ✗ line and the next attempt say why in the test's own words: dotnet test's Error Message, pytest's <code>E</code> lines, or the error with its Expected and Received values (Playwright, Jest, JUnit), not the runner's summary line.</p>
|
|
3775
|
+
<p><strong>Keeping Scribe's cost down.</strong> ForgeQE reads the repository's test setup itself before Scribe starts (stack, settings files and their browser values, the test id attribute, base classes, page objects and their methods, one test command) and hands it over, so Scribe does not discover it with dozens of file reads. Each attempt after the first starts a fresh session from a summary of the job (handoff, files changed, what each attempt did, the latest failure), so later attempts cost about what the first did. <code>code.max_attempts</code> (default 3) and <code>code.max_tokens</code> (default 1,500,000; <code>0</code> turns it off) cap a job, and the token budget is checked during an attempt too, so one long attempt is stopped when the total passes it. The reviewer sorts its issues into correctness (the test does not test what the goal says, or not reliably) and coding standards (test data or credentials in a page object, naming, hard waits, leftovers). Coding standards never fail a test that works: it is accepted, and the coding agent gets one more round (not counted in <code>max_attempts</code>) to fix them; if that round breaks the test, ForgeQE puts back the version that passed and lists the standards issues in the notes (<code>code.tidy: false</code> skips the round). A test the reviewer rejects twice for correctness stops the job with its issues, since that usually means the test case and the page disagree. Under each step, the handoff shows that step's part of the page (its section and neighbours, from the replay's page snapshot) instead of whole snapshots. When a test fails, ForgeQE shows the failing element on the test's page at the failure and on the replay's page, so most failures are fixed without opening a browser. Scribe's live browser is switched on only when the same failure repeats, and it does not send a page snapshot after every click or keystroke. <code>code.max_steps</code> (default 50) caps the tool calls in one of Scribe's turns. <code>code.review_profile</code> runs the reviewer on a cheaper llms.json profile.</p>
|
|
3776
|
+
</div>
|
|
3777
|
+
</section>
|
|
3778
|
+
<p class="see">See <a href="#doc-how_to_use-generated-code">Generated code</a>.</p>
|
|
2646
3779
|
</div>
|
|
2647
3780
|
</article>
|
|
2648
3781
|
<article class="spec feat" id="cap-heal-tests">
|
|
@@ -2990,7 +4123,7 @@ forgeqe api tests --repo ../app # Scribe writes them as tests
|
|
|
2990
4123
|
</tbody>
|
|
2991
4124
|
</table></div>
|
|
2992
4125
|
<p>The planner only uses operations the set has: a step no operation does is listed as unmatched, never invented. Give it the API's operations with <code>--spec</code> (an OpenAPI or Swagger link or file) or a saved set; without a description, make contracts from a browser run first. Every plan is checked before it is saved (each operation exists, each variable is saved before it is used) and shown step by step; nothing runs until you run it.</p>
|
|
2993
|
-
<p><code>run_api_scenarios</code> (or <code>forgeqe api run-scenarios</code>) runs them against the API, without a browser or a model: each step in order, the values passed on, every step's checks. A failed step stops its scenario, and the scenario's <strong>cleanup</strong> steps (deleting what it created) run, those whose values exist. Scenarios that change data run only after you agree (in CI: <code>--allow-writes</code>), on a test environment. Results go to <code>scenario-results.json</code> and <code>scenarios-junit.xml</code>, with <code>report.html</code> (each scenario a case, its calls the steps) and <code>suite.json</code> beside them. API runs count in the <a href="
|
|
4126
|
+
<p><code>run_api_scenarios</code> (or <code>forgeqe api run-scenarios</code>) runs them against the API, without a browser or a model: each step in order, the values passed on, every step's checks. A failed step stops its scenario, and the scenario's <strong>cleanup</strong> steps (deleting what it created) run, those whose values exist. Scenarios that change data run only after you agree (in CI: <code>--allow-writes</code>), on a test environment. Results go to <code>scenario-results.json</code> and <code>scenarios-junit.xml</code>, with <code>report.html</code> (each scenario a case, its calls the steps) and <code>suite.json</code> beside them. API runs count in the <a href="#doc-how_to_use-team-report">team report</a>.</p>
|
|
2994
4127
|
<p><strong>On your existing API automation.</strong> <code>write_api_tests</code> with <code>scenarios</code> (or <code>forgeqe api tests --scenarios</code>) has Scribe write each scenario as an end-to-end test in the repository, built on what is already there:</p>
|
|
2995
4128
|
<ol>
|
|
2996
4129
|
<li>It maps the repository's API layer first: clients, request helpers, auth and base URL fixtures, data builders, schemas.</li>
|
|
@@ -3177,7 +4310,7 @@ forgeqe visual approve --page /home # make this run's /home the baseline (asks
|
|
|
3177
4310
|
<ul>
|
|
3178
4311
|
<li>ForgeQE keeps the result of every tool call in the session (its own tools, and the Azure DevOps and Jira MCP servers). A dataset of the report names the calls it comes from, and ForgeQE takes the rows from their results.</li>
|
|
3179
4312
|
<li>Or the dataset is a read-only query that ForgeQE runs itself when it writes the report: a WIQL query reads up to 1000 work items with their project, type, state, priority, severity, assignee, area, iteration, dates, and a link. The agent uses this for "all" of something.</li>
|
|
3180
|
-
<li>Or the dataset is ForgeQE's own past results (<code>history:</code>): <code>suites</code> (one row per run, with its run type in <code>suite_scope</code>), <code>case_results</code>, <code>step_results</code>, <code>writebacks</code>, <code>readiness</code>, <code>readiness_stories</code>, or <code>readiness_cases</code>, from every run on this machine and the bundles teammates shared with <code>forgeqe bi export</code> (see <a href="
|
|
4313
|
+
<li>Or the dataset is ForgeQE's own past results (<code>history:</code>): <code>suites</code> (one row per run, with its run type in <code>suite_scope</code>), <code>case_results</code>, <code>step_results</code>, <code>writebacks</code>, <code>readiness</code>, <code>readiness_stories</code>, or <code>readiness_cases</code>, from every run on this machine and the bundles teammates shared with <code>forgeqe bi export</code> (see <a href="#doc-how_to_use-team-report">Team report</a>). Every row has <code>user</code> and <code>machine</code>.</li>
|
|
3181
4314
|
<li>The agent writes only the summary paragraph.</li>
|
|
3182
4315
|
</ul>
|
|
3183
4316
|
<p>The report goes to the reports folder (<code>export.out_dir</code>, <code>.forgeqe/reports/</code> by default):</p>
|
|
@@ -3191,6 +4324,84 @@ forgeqe visual approve --page /home # make this run's /home the baseline (asks
|
|
|
3191
4324
|
<p>The chat gets a short summary and a card with the paths as links, and <strong>Open the report</strong> appears as a next step. To post it, ask for it ("post the report to the release channel") once <code>notify.channels</code> is set up; the reports folder is linked when it is served (<code>notify.report_url</code>).</p>
|
|
3192
4325
|
</div>
|
|
3193
4326
|
</section>
|
|
4327
|
+
<section class="doc" id="doc-how_to_use-team-report">
|
|
4328
|
+
<h4>Team report</h4>
|
|
4329
|
+
<div class="prose"><p><code>forgeqe bi</code> builds one report from every run in the reports folder: browser suites, monitor rounds, release readiness, single browser runs, desktop and mobile runs, exploratory testing, and API runs, each tagged with its run type. It builds an <a href="https://github.com/evidence-dev/evidence" rel="noopener">Evidence</a> site (needs Node.js); without Node.js, or when Evidence fails, it writes ForgeQE's own page, which opens straight from disk (<code>bi:</code> in <code>forgeqe.yaml</code>; see <a href="#doc-readme-static-bi">README.md</a>).</p>
|
|
4330
|
+
<pre><code class="language-bash">forgeqe bi init # scaffold the Evidence project and install its packages
|
|
4331
|
+
forgeqe bi build # refresh the data and build (ForgeQE's own page if Evidence cannot build)
|
|
4332
|
+
forgeqe bi build --static # ForgeQE's own page only, no Node.js needed
|
|
4333
|
+
forgeqe bi sync # refresh the datasets and pages, without building
|
|
4334
|
+
forgeqe bi export [--out FILE|DIR] # your runs, in one file to share
|
|
4335
|
+
forgeqe bi build --from //share/qa-reports # this machine's runs plus every bundle there (repeatable)
|
|
4336
|
+
</code></pre>
|
|
4337
|
+
<p>For one report across the team, each person runs <code>forgeqe bi export</code>, which writes their runs and readiness reports to one file in <code>.forgeqe/reports/shared/</code>. Whoever builds puts the files in <code><out_dir>/collected/</code> (<code>.forgeqe/reports/collected/</code> by default), or lists shared folders under <code>bi.collect</code>, or passes <code>--from</code>, and runs <code>forgeqe bi build</code>. Every table gets <code>user</code> and <code>machine</code> columns; the report adds <strong>Results by run type</strong> and <strong>Case results by person</strong> tables. A run that arrives twice (in two bundles, or a bundle and this machine) counts once. A bundle is read as data only: a file that is not one, or is larger than 64 MB, is skipped with a note.</p>
|
|
4338
|
+
<p>In the shell, <code>/report</code> builds ForgeQE's own page from the same data, <code>/report export</code> writes your bundle, and <code>/report <what></code> asks the agent for a report (see <a href="#doc-how_to_use-reports-from-the-agent">Reports from the agent</a>).</p>
|
|
4339
|
+
</div>
|
|
4340
|
+
</section>
|
|
4341
|
+
<section class="doc" id="doc-readme-static-bi">
|
|
4342
|
+
<h4>Static BI</h4>
|
|
4343
|
+
<div class="prose"><p><code>forgeqe bi</code> adds an offline BI layer on top of those reports. It exports curated CSV datasets from ForgeQE's own suite history and release-readiness reports, plus optional Azure DevOps WIQL and Jira JQL datasets from <code>bi.datasets</code> in <code>forgeqe.yaml</code>, into an <a href="https://github.com/evidence-dev/evidence" rel="noopener">Evidence</a> project that reads them through its CSV source, then runs <code>evidence sources</code> and <code>evidence build</code> to produce a static site in the configured build directory. ForgeQE uses the open-source Evidence line (<code>@evidence-dev/evidence</code> 40.x, pinned in the generated <code>package.json</code>), because the newer Evidence Studio CLI has no local build and keeps data in its cloud. Building it needs Node.js 18 or later; serve the folder to view it (<code>npx serve</code>), since it cannot load its data when opened from disk. When Node.js is missing or the Evidence build fails, <code>forgeqe bi build</code> writes ForgeQE's own self-contained page from the same data instead, which opens straight from disk, and says why; <code>--static</code> always does that, and <code>--evidence</code> fails instead of falling back. Dataset ids become table names (<code>forgeqe.<id></code>), so they use letters, digits, and <code>_</code>. Pages ForgeQE generates carry a "Managed by forgeqe bi" comment in their front matter; a page without it is yours and is never overwritten. Tables with no rows yet stay out of the Evidence project (Evidence cannot build over an empty table), and their sections read "None yet." No backend service stays up between runs.</p>
|
|
4344
|
+
<pre><code class="language-yaml">bi:
|
|
4345
|
+
title: ForgeQE BI
|
|
4346
|
+
datasets:
|
|
4347
|
+
ado_current_sprint:
|
|
4348
|
+
kind: ado_wiql
|
|
4349
|
+
project: Shop
|
|
4350
|
+
wiql: |
|
|
4351
|
+
SELECT [System.Id]
|
|
4352
|
+
FROM WorkItems
|
|
4353
|
+
WHERE [System.TeamProject] = @project
|
|
4354
|
+
AND [System.WorkItemType] IN GROUP 'Microsoft.RequirementCategory'
|
|
4355
|
+
jira_open_bugs:
|
|
4356
|
+
kind: jira_jql
|
|
4357
|
+
jql: project = SHOP AND issuetype = Bug AND statusCategory != Done
|
|
4358
|
+
</code></pre>
|
|
4359
|
+
<pre><code class="language-bash">forgeqe bi init # scaffold the Evidence project and install its packages
|
|
4360
|
+
forgeqe bi build # refresh the data and build the site (ForgeQE's own page if Evidence cannot build)
|
|
4361
|
+
forgeqe bi build --static # ForgeQE's own page only, no Node.js needed
|
|
4362
|
+
</code></pre>
|
|
4363
|
+
</div>
|
|
4364
|
+
</section>
|
|
4365
|
+
<section class="doc" id="doc-readme-one-report-for-the-whole-team">
|
|
4366
|
+
<h4>One report for the whole team</h4>
|
|
4367
|
+
<div class="prose"><p>Each person runs <code>forgeqe bi export</code>, which writes every kind of run they have (suites, monitor rounds, single browser runs, desktop, mobile, exploratory and API runs, and release-readiness reports) to one JSON file in <code>.forgeqe/reports/shared/</code>, each run tagged with its run type. Whoever builds the report puts the files they receive in <code>.forgeqe/reports/collected/</code> (or lists shared folders under <code>bi.collect</code> in <code>forgeqe.yaml</code>, or passes <code>--from FILE|DIR</code>, repeatable) and runs <code>forgeqe bi build</code>. Every table gets <code>user</code> and <code>machine</code> columns, the home page and ForgeQE's own page gain "Results by run type" and "Case results by person" tables, and a run that arrives twice (from two bundles, or a bundle and this machine) counts once. A bundle is read as data only: anything that is not one, or is larger than 64 MB, is skipped with a note, and a malformed run inside one is skipped whole.</p>
|
|
4368
|
+
<p>In the shell, <code>/report</code> builds ForgeQE's own page from the same data, <code>/report export</code> writes your bundle, and <code>/report <what you want></code> asks the agent, which can now build <code>write_report</code> datasets from these past results (<code>history: case_results</code>, <code>suites</code>, <code>readiness</code>, …) as well as from live Azure DevOps or Jira queries.</p>
|
|
4369
|
+
<pre><code class="language-bash">forgeqe bi export # your runs, in one file to share
|
|
4370
|
+
forgeqe bi build --from //share/qa-reports # this machine's runs plus every bundle in that folder
|
|
4371
|
+
</code></pre>
|
|
4372
|
+
</div>
|
|
4373
|
+
</section>
|
|
4374
|
+
<section class="doc" id="doc-how_to_use-model-prices-and-usage">
|
|
4375
|
+
<h4>Model prices and usage</h4>
|
|
4376
|
+
<div class="prose"><p><strong>Prices.</strong> ForgeQE prices every model call at the model's <strong>list price</strong>, from two public price lists: <a href="https://models.dev" rel="noopener">models.dev</a> (the list the agent runtime uses) and LiteLLM's model price list, which also has Azure's Data Zone prices. It keeps the few fields it needs in <code>~/.forgeqe/prices.json</code>, waits a few seconds for it the first time, refreshes it once a day in the background, and uses the last copy offline; only the public lists are downloaded, nothing about your runs is sent (<code>FORGEQE_PRICES=off</code> stops the downloads). Costs at list price say so: <code>$1.73 (list price of gpt-5.4 (Azure Global) from models.dev, 2026-10-06)</code>. For an Azure Data Zone deployment, which costs about 10% more, set <code>"deployment_type": "data_zone"</code> on the profile.</p>
|
|
4377
|
+
<p><strong>Model parameters.</strong> Models differ in what a request may carry: reasoning models (o1, o3, o4, gpt-5) take <code>max_completion_tokens</code> instead of <code>max_tokens</code> and only the default temperature, and some OpenAI-compatible servers refuse <code>max_completion_tokens</code>, <code>reasoning_effort</code> or <code>response_format</code>. ForgeQE works this out by itself. A profile is a reasoning model when its <code>model</code> or <code>deployment</code> name says so (an Azure deployment called <code>chat-prod</code> with <code>"model": "gpt-5.4"</code> counts), or when it sets <code>"reasoning": true</code>; <code>"reasoning": false</code> says it is not. When the API still refuses a parameter, the browser, desktop and mobile agents send the call again without it and leave it out of later calls to that model. Nothing needs setting for a new model.</p>
|
|
4378
|
+
<p>A profile's own <code>"price": { "input": 2.5, "output": 15, "cached_input": 0.25 }</code> (per million tokens) wins, for a negotiated or enterprise rate. Leave <code>currency</code> out and reports use this machine's currency, or set <code>"currency": "USD"</code>. Without <code>cached_input</code>, cached input gets the model's cache discount from its list price, and the cost is marked as an estimate (<code>≈$1.52</code>); with no list price for the model either, cached input is charged at the full rate and the cost is an upper bound (<code>up to $6.11</code>). When your price differs from the list price, ForgeQE says so (<code>forgeqe models</code>, the shell, and the start of each command), since that is usually a typo or an old price. Anthropic's cache writes are priced at their own rate. A provisioned (PTU) deployment is billed per hour, not per token, so its costs are what the tokens would cost at list price. Requests over a model's long-context threshold (272K tokens for GPT-5.4) cost more and are priced at the normal rate.</p>
|
|
4379
|
+
<p><strong>Usage after every job.</strong> Every browser run, Scribe job, heal, suite, and monitor round ends with a <code>Usage</code> line: tokens in (and how many of them were cached), tokens out, and the cost at the profile's price. A request to ForgeQE's agent in the shell, and every <code>forgeqe run</code>, <code>code</code>, <code>suite</code>, and <code>heal</code> command, ends with the total for everything it started, by role:</p>
|
|
4380
|
+
<pre><code>This request used 1.1M tokens (1.1M in, 30.0K of it cached · 34.7K out) · $3.07 — ForgeQE's agent 42.0K · $0.12, browser agent 52.7K · $0.15, Scribe 1.0M · $2.80
|
|
4381
|
+
</code></pre>
|
|
4382
|
+
</div>
|
|
4383
|
+
</section>
|
|
4384
|
+
<section class="doc" id="doc-how_to_use-spending-budgets">
|
|
4385
|
+
<h4>Spending budgets</h4>
|
|
4386
|
+
<div class="prose"><p>A budget warns you before model spend gets out of hand. Set it under <code>budget:</code> in <code>forgeqe.yaml</code>, in the currency of your prices (see <a href="#doc-how_to_use-model-prices-and-usage">Model prices and usage</a>):</p>
|
|
4387
|
+
<pre><code class="language-yaml">budget:
|
|
4388
|
+
run: 2 # one browser run
|
|
4389
|
+
day: 25
|
|
4390
|
+
month: 400
|
|
4391
|
+
# currency: USD # default: the currency of the model's price
|
|
4392
|
+
</code></pre>
|
|
4393
|
+
<p>Set any of the three. A budget <strong>only warns; it never stops a run</strong>. Each budget warns once at 80%, once at 90%, and once at 100%. When spend jumps past more than one of these at once, you get one warning, for the highest:</p>
|
|
4394
|
+
<pre><code>Budget: this month's model spend is at $245.10, 82% of the $300.00 month budget (budget.month in forgeqe.yaml). Nothing is stopped.
|
|
4395
|
+
</code></pre>
|
|
4396
|
+
<ul>
|
|
4397
|
+
<li><strong>Where you see it.</strong> In the shell, the warning is a line in the conversation. Every other <code>forgeqe</code> command writes it to the terminal's error output.</li>
|
|
4398
|
+
<li><strong>What counts.</strong> <code>run</code> is the spend of one browser run. <code>day</code> and <code>month</code> add up every ForgeQE process on this machine (days and months are in UTC). Runs that the agent starts through the MCP server count too, because every process writes its model calls to the usage ledger in <code>~/.forgeqe/usage/</code> (<code>FORGEQE_USAGE_DIR</code> moves it). The ledger holds counts, times, and names, never prompt or response text. With <code>FORGEQE_USAGE=off</code>, nothing is kept on disk, and a budget counts only the calls of the process that warns.</li>
|
|
4399
|
+
<li><strong>What does not count.</strong> A call with no price, such as a <code>github_copilot</code> profile without a <code>price</code>, and a call priced in a different currency than the budget.</li>
|
|
4400
|
+
<li><strong>Spend from before.</strong> If you start ForgeQE when the day or month is already past 80%, it does not repeat that warning. It warns only for a threshold that spend passes from then on.</li>
|
|
4401
|
+
<li><strong>In reports.</strong> The <strong>Trends and forecast</strong> page of a report's Tokenomics shows spend on this machine and the next days at the recent pace, against the budget.</li>
|
|
4402
|
+
</ul>
|
|
4403
|
+
</div>
|
|
4404
|
+
</section>
|
|
3194
4405
|
</div>
|
|
3195
4406
|
</article>
|
|
3196
4407
|
</section>
|
|
@@ -3406,8 +4617,8 @@ the list again. For a client that ignores it, <code>forgeqe mcp --all-tools</cod
|
|
|
3406
4617
|
<tr><td><code>WP_LLMS_JSON</code></td><td>Path to <code>llms.json</code></td></tr>
|
|
3407
4618
|
<tr><td><code>WP_LLM_PROFILE</code></td><td>Active profile, unless <code>--profile</code> is set</td></tr>
|
|
3408
4619
|
<tr><td><code>WP_AZURE_API_KEY</code> / <code>OPENAI_API_KEY</code> / <code>WP_API_KEY</code></td><td>Keys referenced by profiles</td></tr>
|
|
3409
|
-
<tr><td><code>
|
|
3410
|
-
<tr><td><code>
|
|
4620
|
+
<tr><td><code>COPILOT_GITHUB_TOKEN</code></td><td>A <code>github_copilot</code> profile's GitHub token (fine-grained, with the Copilot Requests permission), unless the profile's <code>api_key_env</code> names another; else the saved token, then the GitHub CLI's sign-in</td></tr>
|
|
4621
|
+
<tr><td><code>FORGEQE_COPILOT_TOKEN</code></td><td>The token <code>forgeqe auth login copilot</code> saves; set by ForgeQE from the credential store, not by hand</td></tr>
|
|
3411
4622
|
<tr><td><code>FORGEQE_CONFIG_DIR</code></td><td>Global config folder (default <code>~/.config/forgeqe</code>): <code>forgeqe.yaml</code>, <code>llms.json</code> and <code>.env</code> used when a project has none</td></tr>
|
|
3412
4623
|
<tr><td><code>FORGEQE_PYTHON</code></td><td>A Python that already has the engine's libraries, used instead of the managed venv (skipped, with a note, when it cannot run the engine)</td></tr>
|
|
3413
4624
|
<tr><td><code>FORGEQE_DEBUG</code></td><td><code>1</code> logs at debug level, like <code>--debug</code></td></tr>
|
|
@@ -3461,12 +4672,16 @@ the list again. For a client that ignores it, <code>forgeqe mcp --all-tools</cod
|
|
|
3461
4672
|
var panel = document.getElementById("panel");
|
|
3462
4673
|
var links = Array.prototype.slice.call(document.querySelectorAll(".chapters a"));
|
|
3463
4674
|
var railSearch = document.getElementById("rail-search");
|
|
3464
|
-
var railGroups = Array.prototype.slice.call(document.querySelectorAll(".rail-group"));
|
|
4675
|
+
var railGroups = Array.prototype.slice.call(document.querySelectorAll(".rail-group:not(#rail-hits)"));
|
|
4676
|
+
var railHits = document.getElementById("rail-hits");
|
|
4677
|
+
var railHitsList = document.getElementById("rail-hits-list");
|
|
4678
|
+
var railNone = document.getElementById("rail-none");
|
|
3465
4679
|
var tocCurrent = document.getElementById("toc-current");
|
|
3466
4680
|
var tocLinks = document.getElementById("toc-links");
|
|
3467
4681
|
var byId = {};
|
|
3468
4682
|
links.forEach(function (a) { byId[a.getAttribute("href").slice(1)] = a; });
|
|
3469
4683
|
var filterRail = function (term) {
|
|
4684
|
+
var chaptersShown = 0;
|
|
3470
4685
|
railGroups.forEach(function (group) {
|
|
3471
4686
|
var items = Array.prototype.slice.call(group.querySelectorAll(".chapters li"));
|
|
3472
4687
|
var shown = 0;
|
|
@@ -3478,7 +4693,26 @@ the list again. For a client that ignores it, <code>forgeqe mcp --all-tools</cod
|
|
|
3478
4693
|
if (on) shown += 1;
|
|
3479
4694
|
});
|
|
3480
4695
|
group.style.display = shown ? "" : "none";
|
|
4696
|
+
chaptersShown += shown;
|
|
3481
4697
|
});
|
|
4698
|
+
// Every section and capability card whose heading or chapter matches, so a topic inside a chapter is found too.
|
|
4699
|
+
railHitsList.innerHTML = "";
|
|
4700
|
+
var hits = term ? sections.filter(function (s) { return s.text.indexOf(term) !== -1; }).slice(0, 30) : [];
|
|
4701
|
+
hits.forEach(function (s) {
|
|
4702
|
+
var li = document.createElement("li");
|
|
4703
|
+
var a = document.createElement("a");
|
|
4704
|
+
a.setAttribute("href", "#" + s.id);
|
|
4705
|
+
[["n", s.n], ["t", s.title], ["d", s.chapter]].forEach(function (part) {
|
|
4706
|
+
var span = document.createElement("span");
|
|
4707
|
+
span.className = part[0];
|
|
4708
|
+
span.textContent = part[1];
|
|
4709
|
+
a.appendChild(span);
|
|
4710
|
+
});
|
|
4711
|
+
li.appendChild(a);
|
|
4712
|
+
railHitsList.appendChild(li);
|
|
4713
|
+
});
|
|
4714
|
+
railHits.hidden = !term || (!hits.length && chaptersShown > 0);
|
|
4715
|
+
railNone.hidden = hits.length > 0 || chaptersShown > 0;
|
|
3482
4716
|
};
|
|
3483
4717
|
var chapters = Array.prototype.slice.call(document.querySelectorAll(".chapter"));
|
|
3484
4718
|
var itemText = function (node) {
|
|
@@ -3541,6 +4775,15 @@ the list again. For a client that ignores it, <code>forgeqe mcp --all-tools</cod
|
|
|
3541
4775
|
ticking = true;
|
|
3542
4776
|
requestAnimationFrame(function () { ticking = false; mark(current()); });
|
|
3543
4777
|
}, { passive: true });
|
|
4778
|
+
var sections = [];
|
|
4779
|
+
chapters.forEach(function (chapter, i) {
|
|
4780
|
+
var head = chapter.querySelector(".chead h2");
|
|
4781
|
+
var name = head ? head.textContent.trim() : "";
|
|
4782
|
+
tocItems(chapter).forEach(function (node) {
|
|
4783
|
+
var title = itemText(node);
|
|
4784
|
+
sections.push({ id: node.id, title: title, chapter: name, n: String(i + 1).padStart(2, "0"), text: (title + " " + name).toLowerCase() });
|
|
4785
|
+
});
|
|
4786
|
+
});
|
|
3544
4787
|
if (railSearch) railSearch.addEventListener("input", function () { filterRail(railSearch.value.trim().toLowerCase()); });
|
|
3545
4788
|
filterRail("");
|
|
3546
4789
|
mark(location.hash && document.getElementById(location.hash.slice(1)) ? locateChapter(location.hash.slice(1)) : current());
|