@capagents/forgeqe 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HOW_TO_USE.md +1639 -0
- package/README.md +338 -2
- package/bin/forgeqe.d.ts +3 -0
- package/bin/forgeqe.js +188 -0
- package/docs/ARCHITECTURE.md +408 -0
- package/docs/security-data-flow.html +1870 -0
- package/docs/user-guide.html +2094 -0
- package/evals/domain/access-role-change.yaml +37 -0
- package/evals/domain/banking-payment-transfer.yaml +41 -0
- package/evals/domain/gdpr-account-erasure.yaml +30 -0
- package/evals/domain/insurance-claim-fnol.yaml +30 -0
- package/evals/domain/insurance-policy-renewal.yaml +37 -0
- package/evals/domain/lending-loan-application.yaml +36 -0
- package/forgeqe-architecture.html +718 -0
- package/forgeqe.example.yaml +203 -0
- package/knowledge/README.md +18 -0
- package/knowledge/business-rules.md +10 -0
- package/knowledge/compliance.md +16 -0
- package/knowledge/environments.md +5 -0
- package/knowledge/glossary.md +5 -0
- package/knowledge/product.md +16 -0
- package/knowledge/roles.md +9 -0
- package/knowledge/test-data.md +7 -0
- package/llms.example.json +51 -0
- package/package.json +63 -4
- package/prompts/README.md +34 -0
- package/prompts/api-scenarios.md +21 -0
- package/prompts/api-tests.md +39 -0
- package/prompts/brain.md +166 -0
- package/prompts/browser-capture.md +7 -0
- package/prompts/browser-case-report.md +7 -0
- package/prompts/browser-checks.md +14 -0
- package/prompts/browser-efficiency.md +7 -0
- package/prompts/browser-error-recovery.md +15 -0
- package/prompts/browser-extra.md +5 -0
- package/prompts/browser-goal-verification.md +13 -0
- package/prompts/browser-skill-select.md +6 -0
- package/prompts/domain-rule.md +13 -0
- package/prompts/eval-judge.md +19 -0
- package/prompts/explore-charters.md +27 -0
- package/prompts/find-tests.md +28 -0
- package/prompts/heal.md +40 -0
- package/prompts/pr-review.md +29 -0
- package/prompts/skill-rule.md +6 -0
- package/prompts/test-agent.md +34 -0
- package/prompts/test-reviewer.md +30 -0
- package/prompts/test-rules.md +28 -0
- package/py/forgeqe_engine/__init__.py +3 -0
- package/py/forgeqe_engine/__main__.py +3 -0
- package/py/forgeqe_engine/brand.py +453 -0
- package/py/forgeqe_engine/brand_assets.py +6 -0
- package/py/forgeqe_engine/browser.py +196 -0
- package/py/forgeqe_engine/capture.py +538 -0
- package/py/forgeqe_engine/checks.py +98 -0
- package/py/forgeqe_engine/cli.py +188 -0
- package/py/forgeqe_engine/diagnostics.py +609 -0
- package/py/forgeqe_engine/llm.py +195 -0
- package/py/forgeqe_engine/logins.py +251 -0
- package/py/forgeqe_engine/loops.py +224 -0
- package/py/forgeqe_engine/memory.py +1004 -0
- package/py/forgeqe_engine/network.py +244 -0
- package/py/forgeqe_engine/prompts.py +20 -0
- package/py/forgeqe_engine/replay.py +240 -0
- package/py/forgeqe_engine/report.py +173 -0
- package/py/forgeqe_engine/service.py +1335 -0
- package/py/forgeqe_engine/settings.py +93 -0
- package/py/forgeqe_engine/skills.py +88 -0
- package/py/forgeqe_engine/summary.py +239 -0
- package/py/forgeqe_engine/urls.py +81 -0
- package/py/forgeqe_engine/workflow_yaml.py +689 -0
- package/skills/dotnet-nunit-playwright/SKILL.md +34 -0
- package/skills/dynamics-365/SKILL.md +21 -0
- package/skills/playwright-typescript/SKILL.md +35 -0
- package/skills/writing-skills/SKILL.md +30 -0
- package/src/api/plan.ts +246 -0
- package/src/api/run.ts +204 -0
- package/src/api/scenario.ts +502 -0
- package/src/api/schema.ts +170 -0
- package/src/api/spec.ts +624 -0
- package/src/api/store.ts +232 -0
- package/src/api/traffic.ts +281 -0
- package/src/api/types.ts +163 -0
- package/src/api/write.ts +272 -0
- package/src/auth/signin.ts +494 -0
- package/src/auth/store.ts +170 -0
- package/src/brain/index.ts +189 -0
- package/src/brain/prompt.ts +19 -0
- package/src/capabilities/docs.ts +101 -0
- package/src/capabilities/guide.ts +455 -0
- package/src/capabilities/registry.ts +343 -0
- package/src/capabilities/render.ts +106 -0
- package/src/capabilities/status.ts +137 -0
- package/src/card.ts +11 -0
- package/src/cases/ado.ts +325 -0
- package/src/cases/api.ts +117 -0
- package/src/cases/bugs.ts +223 -0
- package/src/cases/confluence.ts +111 -0
- package/src/cases/context.ts +223 -0
- package/src/cases/dashboards.ts +137 -0
- package/src/cases/failure.ts +97 -0
- package/src/cases/files.ts +295 -0
- package/src/cases/goal.ts +53 -0
- package/src/cases/http.ts +111 -0
- package/src/cases/index.ts +131 -0
- package/src/cases/jira.ts +122 -0
- package/src/cases/qtest.ts +108 -0
- package/src/cases/quality.ts +190 -0
- package/src/cases/rag.ts +100 -0
- package/src/cases/readiness.ts +284 -0
- package/src/cases/save.ts +149 -0
- package/src/cases/sprints.ts +148 -0
- package/src/cases/testrail.ts +176 -0
- package/src/cases/types.ts +107 -0
- package/src/cases/wiki.ts +105 -0
- package/src/cases/xray.ts +206 -0
- package/src/cases/zephyr.ts +153 -0
- package/src/childEnv.ts +86 -0
- package/src/cli/args.ts +114 -0
- package/src/cli/help.ts +167 -0
- package/src/cli/index.ts +1142 -0
- package/src/cli/terminal.ts +187 -0
- package/src/code/areas.ts +337 -0
- package/src/code/attention.ts +214 -0
- package/src/code/find.ts +540 -0
- package/src/code/handoff.ts +541 -0
- package/src/code/history.ts +170 -0
- package/src/code/impact.ts +231 -0
- package/src/code/index.ts +34 -0
- package/src/code/livebrowser.ts +67 -0
- package/src/code/loop.ts +603 -0
- package/src/code/notes.ts +75 -0
- package/src/code/profile.ts +226 -0
- package/src/code/prompt.ts +303 -0
- package/src/code/prreview.ts +649 -0
- package/src/code/prtests.ts +103 -0
- package/src/code/replay.ts +772 -0
- package/src/code/review.ts +202 -0
- package/src/code/runtime.ts +757 -0
- package/src/code/slices.ts +159 -0
- package/src/code/testrun.ts +187 -0
- package/src/code/traceability.ts +100 -0
- package/src/code/worktree.ts +153 -0
- package/src/commands.ts +101 -0
- package/src/config.ts +970 -0
- package/src/detail.ts +65 -0
- package/src/domain.ts +19 -0
- package/src/engine/bridge.ts +359 -0
- package/src/engine/mock.ts +102 -0
- package/src/engine/setup.ts +298 -0
- package/src/entra.ts +154 -0
- package/src/env.ts +24 -0
- package/src/evals/domain.ts +338 -0
- package/src/explore/charters.ts +151 -0
- package/src/explore/run.ts +280 -0
- package/src/explore/story.ts +106 -0
- package/src/guard.ts +51 -0
- package/src/heal/loop.ts +357 -0
- package/src/heal/prompt.ts +143 -0
- package/src/heal/report.ts +95 -0
- package/src/index.ts +12 -0
- package/src/init.ts +102 -0
- package/src/knowledge.ts +114 -0
- package/src/legacyEnv.ts +9 -0
- package/src/llm.ts +61 -0
- package/src/locale.ts +25 -0
- package/src/logins.ts +148 -0
- package/src/managed.ts +101 -0
- package/src/mcp/apitools.ts +335 -0
- package/src/mcp/captools.ts +25 -0
- package/src/mcp/casetools.ts +223 -0
- package/src/mcp/commands.ts +421 -0
- package/src/mcp/exploretools.ts +129 -0
- package/src/mcp/external.ts +172 -0
- package/src/mcp/git.ts +78 -0
- package/src/mcp/notifytools.ts +49 -0
- package/src/mcp/prtools.ts +261 -0
- package/src/mcp/qualitytools.ts +154 -0
- package/src/mcp/repotests.ts +286 -0
- package/src/mcp/server.ts +234 -0
- package/src/mcp/tools.ts +810 -0
- package/src/memory.ts +250 -0
- package/src/money.ts +39 -0
- package/src/monitor/alerts.ts +94 -0
- package/src/monitor/background.ts +68 -0
- package/src/monitor/run.ts +263 -0
- package/src/monitor/state.ts +105 -0
- package/src/notify/events.ts +144 -0
- package/src/notify/notice.ts +177 -0
- package/src/prices.ts +175 -0
- package/src/print.ts +54 -0
- package/src/privacy.ts +35 -0
- package/src/prompts.ts +165 -0
- package/src/quality/budgets.ts +249 -0
- package/src/quality/history.ts +149 -0
- package/src/quality/visual.ts +201 -0
- package/src/report/data.ts +475 -0
- package/src/report/html.ts +106 -0
- package/src/report/template.html +84 -0
- package/src/run.ts +243 -0
- package/src/sessionLog.ts +216 -0
- package/src/sessionLogins.ts +94 -0
- package/src/skills/registry.ts +217 -0
- package/src/skills/scaffold.ts +65 -0
- package/src/skills/select.ts +37 -0
- package/src/skills/view.ts +44 -0
- package/src/suite/report.ts +135 -0
- package/src/suite/run.ts +303 -0
- package/src/summary.ts +188 -0
- package/src/tools.ts +130 -0
- package/src/tui/App.tsx +983 -0
- package/src/tui/clipboard.ts +36 -0
- package/src/tui/components.tsx +263 -0
- package/src/tui/dialogs.ts +65 -0
- package/src/tui/main.tsx +20 -0
- package/src/tui/text.ts +28 -0
- package/src/tui/theme.ts +104 -0
- package/src/tui/transcript.tsx +344 -0
- package/src/types.ts +136 -0
- package/src/update.ts +311 -0
- package/src/usage.ts +158 -0
package/src/detail.ts
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import type { AgentEvent, CodeEvent } from "./types.ts"
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* How much of the browser and coding agents' work the terminal shows. brief: what a person follows (what each browser
|
|
5
|
+
* step is for and its result, warnings and errors, the coding agent's milestones, files it writes, test results).
|
|
6
|
+
* full: everything, including the browser agent's whole reasoning, page memory, engine logs, and every file the coding
|
|
7
|
+
* agent reads or searches.
|
|
8
|
+
* The session log records everything either way.
|
|
9
|
+
*/
|
|
10
|
+
export type Detail = "brief" | "full"
|
|
11
|
+
|
|
12
|
+
/** Coding agent tool calls that change files: worth a line even in brief. */
|
|
13
|
+
export const WRITES_FILE = /^(Writing|Editing) /
|
|
14
|
+
|
|
15
|
+
/** Whether an event gets a line of its own at this level; hidden events still drive the shell's activity line. */
|
|
16
|
+
export function shows(event: AgentEvent | CodeEvent, detail: Detail): boolean {
|
|
17
|
+
if (detail === "full") return true
|
|
18
|
+
switch (event.type) {
|
|
19
|
+
case "log":
|
|
20
|
+
return event.level !== "info"
|
|
21
|
+
case "outcome":
|
|
22
|
+
return true
|
|
23
|
+
// One line per step in brief: what the browser agent is about to do (stepIntent).
|
|
24
|
+
case "thought":
|
|
25
|
+
return true
|
|
26
|
+
case "step":
|
|
27
|
+
case "memory":
|
|
28
|
+
return false
|
|
29
|
+
case "code":
|
|
30
|
+
if (event.kind === "info" || event.kind === "test") return true
|
|
31
|
+
if (event.kind === "tool") return !event.ok || WRITES_FILE.test(event.text)
|
|
32
|
+
return false
|
|
33
|
+
default:
|
|
34
|
+
return false
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** What a browser step is for: "Step 4 · Fill in the contact form" in brief (its first sentence); whole in full. */
|
|
39
|
+
export function stepIntent(step: number, text: string, detail: Detail): string {
|
|
40
|
+
if (detail === "full") return text
|
|
41
|
+
const line = (text.split("\n").find((l) => l.trim()) || "").trim()
|
|
42
|
+
const sentence = /^(.{20,}?[.!?])\s/.exec(line)?.[1] || line
|
|
43
|
+
return `Step ${step} · ${sentence.length <= 140 ? sentence : `${sentence.slice(0, 139)}…`}`
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Reading and searching files: noise in brief. Everything else ForgeQE's agent calls (MCP tools, commands) is shown. */
|
|
47
|
+
const LOOKS_AROUND = /^(Reading|Finding files|Searching for|Listing) /
|
|
48
|
+
|
|
49
|
+
/** Whether a tool call of ForgeQE's own agent (not Scribe) gets a line: in brief, all but its file reads and searches. */
|
|
50
|
+
export function showsAgentTool(text: string, ok: boolean, detail: Detail): boolean {
|
|
51
|
+
return detail === "full" || !ok || !LOOKS_AROUND.test(text)
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** A step's result as one short line in brief; whole in full. */
|
|
55
|
+
export function briefText(text: string, detail: Detail, max = 160): string {
|
|
56
|
+
if (detail === "full") return text
|
|
57
|
+
const line = text.split("\n").find((l) => l.trim()) || ""
|
|
58
|
+
const flat = line.trim()
|
|
59
|
+
return flat.length <= max ? flat : `${flat.slice(0, max - 1)}…`
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** --verbose (or FORGEQE_VERBOSE=1) shows everything. */
|
|
63
|
+
export function detailFrom(verbose: boolean | undefined): Detail {
|
|
64
|
+
return verbose || /^(1|true|yes|on)$/i.test(process.env.FORGEQE_VERBOSE || "") ? "full" : "brief"
|
|
65
|
+
}
|
package/src/domain.ts
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { AppConfig } from "./config.ts"
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The product's domain (domain: in forgeqe.yaml) as the lines every agent that designs or judges tests is given, so
|
|
5
|
+
* it applies the industry's risks and the declared regulations' obligations; prompts/domain-rule.md says how.
|
|
6
|
+
* The regulations themselves the model knows; what applies here, with the organisation's control ids, is in the
|
|
7
|
+
* knowledge folder's compliance file.
|
|
8
|
+
*/
|
|
9
|
+
export function domainNotes(cfg: Pick<AppConfig, "domain" | "knowledge">): string {
|
|
10
|
+
const d = cfg.domain
|
|
11
|
+
if (!d) return "Domain: not declared (domain: in forgeqe.yaml). Work out the industry and any regulations from the story, the product knowledge, and the application, and say which you assumed."
|
|
12
|
+
const controls = cfg.knowledge ? ` The controls that apply, with their ids, are in ${cfg.knowledge.dir}/compliance.md when the team has filled it in.` : ""
|
|
13
|
+
return [
|
|
14
|
+
`Domain: ${d.industry || "not named"}.`,
|
|
15
|
+
d.regulations.length ? ` Regulations and standards in scope: ${d.regulations.join(", ")}.` : "",
|
|
16
|
+
d.notes ? ` ${d.notes}` : "",
|
|
17
|
+
controls,
|
|
18
|
+
].join("")
|
|
19
|
+
}
|
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
import { spawn } from "node:child_process"
|
|
2
|
+
import { createInterface } from "node:readline"
|
|
3
|
+
|
|
4
|
+
import type { LlmProfile } from "../llm.ts"
|
|
5
|
+
import type { AgentEvent, CaseReport, MemoryInfo, ReplayMode, RunResult } from "../types.ts"
|
|
6
|
+
import { loginEnvNames } from "../childEnv.ts"
|
|
7
|
+
import { recordUsage } from "../usage.ts"
|
|
8
|
+
import type { LoginConfig } from "../config.ts"
|
|
9
|
+
import { debugLogging, logEvent } from "../sessionLog.ts"
|
|
10
|
+
import { engineEnv } from "./setup.ts"
|
|
11
|
+
import { AZURE_IDENTITY_ENV, entraPayload } from "../entra.ts"
|
|
12
|
+
|
|
13
|
+
export type EngineRequest = {
|
|
14
|
+
goal: string
|
|
15
|
+
outDir: string
|
|
16
|
+
llm: LlmProfile
|
|
17
|
+
startUrl?: string
|
|
18
|
+
headed: boolean
|
|
19
|
+
maxSteps: number
|
|
20
|
+
allowedHosts: string[]
|
|
21
|
+
extendSystemMessage?: string
|
|
22
|
+
/** Page memory folder and replay mode; omit to run without memory. */
|
|
23
|
+
memory?: { dir: string; replay: ReplayMode }
|
|
24
|
+
/** BROWSER_USE_* tuning for the engine process. */
|
|
25
|
+
env?: Record<string, string>
|
|
26
|
+
/** Saved logins: the engine reads values from the environment by name, or from `values` (logins remembered for this session). */
|
|
27
|
+
logins?: (LoginConfig & { values?: { username?: string; password?: string; totp?: string } })[]
|
|
28
|
+
/** Steps in the test case this goal came from; the agent reports a verdict for each. */
|
|
29
|
+
caseSteps?: number
|
|
30
|
+
screenshots?: "all" | "failures" | "off"
|
|
31
|
+
accessibility?: boolean
|
|
32
|
+
/** Record the application's API calls (network.json). */
|
|
33
|
+
network?: boolean
|
|
34
|
+
/** axe-core for the accessibility checks: the script and the rule tags of the standard. */
|
|
35
|
+
axe?: { script: string; tags: string[] }
|
|
36
|
+
/** Core Web Vitals and load timings (performance.json). */
|
|
37
|
+
vitals?: boolean
|
|
38
|
+
/** The browser agent's prompts (prompts/browser-*.md, resolved), by name; the engine's own text for any not given. */
|
|
39
|
+
prompts?: Record<string, string>
|
|
40
|
+
/** Screenshots of each page for visual comparison (visual.json): masks by path prefix, and full page or not. */
|
|
41
|
+
visual?: { mask: string[]; pages: Record<string, string[]>; full_page: boolean }
|
|
42
|
+
overlay?: boolean
|
|
43
|
+
/** A failed run's resume.json: its steps are replayed first, then the agent carries on. */
|
|
44
|
+
resume?: Record<string, unknown>
|
|
45
|
+
/** Browser skills: `given` adds them all to the prompt, `choose` has the model pick the ones the goal needs. */
|
|
46
|
+
skills?: { name: string; description: string; body: string }[]
|
|
47
|
+
skillsPick?: "given" | "choose"
|
|
48
|
+
skillsMaxChars?: number
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function loginPayload(logins: EngineRequest["logins"]): Record<string, unknown>[] {
|
|
52
|
+
return (logins || []).map((l) => ({
|
|
53
|
+
name: l.name,
|
|
54
|
+
domains: l.domains,
|
|
55
|
+
username_env: l.usernameEnv || null,
|
|
56
|
+
password_env: l.passwordEnv || null,
|
|
57
|
+
totp_env: l.totpEnv || null,
|
|
58
|
+
session_file: l.sessionFile || null,
|
|
59
|
+
...(l.values ? { values: l.values } : {}),
|
|
60
|
+
}))
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
type EngineMemory = {
|
|
64
|
+
dir?: string
|
|
65
|
+
mode?: ReplayMode
|
|
66
|
+
replayed?: number
|
|
67
|
+
replay_total?: number
|
|
68
|
+
replay_stopped?: string
|
|
69
|
+
llm_steps?: number
|
|
70
|
+
pages?: number
|
|
71
|
+
elements?: number
|
|
72
|
+
new_elements?: number
|
|
73
|
+
flow?: "saved" | "updated" | null
|
|
74
|
+
resume?: { steps?: number; failed_at?: number; reason?: string; lines?: string[]; file?: string }
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
type EngineResult = {
|
|
78
|
+
success?: boolean
|
|
79
|
+
summary?: string
|
|
80
|
+
error?: string
|
|
81
|
+
final_url?: string
|
|
82
|
+
usage?: { input?: number; output?: number; cached?: number }
|
|
83
|
+
files?: string[]
|
|
84
|
+
cancelled?: boolean
|
|
85
|
+
memory?: EngineMemory
|
|
86
|
+
case_report?: CaseReport
|
|
87
|
+
skills?: string[]
|
|
88
|
+
/** "synthetic" from the mock engine: no browser ran, so the run cannot become a test. */
|
|
89
|
+
driver?: "synthetic"
|
|
90
|
+
outcome?: "passed" | "assertion_failed" | "error"
|
|
91
|
+
checks?: { expected?: string; text?: string; holds?: boolean }[]
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
type EngineEvent =
|
|
95
|
+
| { type: "log"; level?: string; text?: string }
|
|
96
|
+
| { type: "memory"; text?: string }
|
|
97
|
+
| {
|
|
98
|
+
type: "step"
|
|
99
|
+
source?: "memory"
|
|
100
|
+
step_number?: number
|
|
101
|
+
url?: string
|
|
102
|
+
title?: string
|
|
103
|
+
next_goal?: string
|
|
104
|
+
memory?: string
|
|
105
|
+
evaluation_previous_goal?: string
|
|
106
|
+
action_lines?: string[]
|
|
107
|
+
}
|
|
108
|
+
| { type: "result"; step?: number; items?: { ok?: boolean; done?: boolean; text?: string }[] }
|
|
109
|
+
| { type: "usage"; input?: number; output?: number; cached?: number }
|
|
110
|
+
| { type: "done"; result?: EngineResult }
|
|
111
|
+
|
|
112
|
+
function toMemoryInfo(m: EngineMemory | undefined): MemoryInfo | undefined {
|
|
113
|
+
if (!m) return undefined
|
|
114
|
+
return {
|
|
115
|
+
dir: String(m.dir || ""),
|
|
116
|
+
mode: m.mode || "verify",
|
|
117
|
+
replayed: Number(m.replayed || 0),
|
|
118
|
+
replayTotal: Number(m.replay_total || 0),
|
|
119
|
+
replayStopped: m.replay_stopped || undefined,
|
|
120
|
+
llmSteps: Number(m.llm_steps || 0),
|
|
121
|
+
pages: Number(m.pages || 0),
|
|
122
|
+
elements: Number(m.elements || 0),
|
|
123
|
+
newElements: Number(m.new_elements || 0),
|
|
124
|
+
flow: m.flow || null,
|
|
125
|
+
...(m.resume?.steps
|
|
126
|
+
? {
|
|
127
|
+
resume: {
|
|
128
|
+
steps: Number(m.resume.steps),
|
|
129
|
+
failedAt: Number(m.resume.failed_at || 0),
|
|
130
|
+
reason: String(m.resume.reason || ""),
|
|
131
|
+
lines: (m.resume.lines || []).map(String),
|
|
132
|
+
file: m.resume.file || undefined,
|
|
133
|
+
},
|
|
134
|
+
}
|
|
135
|
+
: {}),
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function llmPayload(p: LlmProfile): Record<string, unknown> {
|
|
140
|
+
const out: Record<string, unknown> = { provider: p.provider, model: p.model }
|
|
141
|
+
if (p.endpoint) out.endpoint = p.endpoint
|
|
142
|
+
if (p.deployment) out.deployment = p.deployment
|
|
143
|
+
if (p.apiVersion) out.api_version = p.apiVersion
|
|
144
|
+
if (p.apiKey && p.auth !== "entra") out.api_key = p.apiKey
|
|
145
|
+
if (p.auth === "entra") Object.assign(out, entraPayload(p))
|
|
146
|
+
else if (p.apiKeyEnv) out.api_key_env = p.apiKeyEnv
|
|
147
|
+
if (typeof p.temperature === "number") out.temperature = p.temperature
|
|
148
|
+
return out
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export function toRunResult(goal: string, startUrl: string, r: EngineResult): RunResult {
|
|
152
|
+
// The agent's evidence wins over its verdict: a check that does not hold, or a failed case step, fails the run even
|
|
153
|
+
// when the agent called done with success (engines before this rule did not apply it themselves).
|
|
154
|
+
const contradicted =
|
|
155
|
+
Boolean(r.success) && ((r.checks || []).some((c) => c.holds === false) || (r.case_report?.steps || []).some((s) => s.status === "failed"))
|
|
156
|
+
if (contradicted) r = { ...r, success: false, outcome: "assertion_failed" }
|
|
157
|
+
return {
|
|
158
|
+
status: r.success ? "complete" : r.cancelled ? "stopped" : "fail",
|
|
159
|
+
goal,
|
|
160
|
+
startUrl,
|
|
161
|
+
finalUrl: r.final_url || "",
|
|
162
|
+
summary: String(r.summary || r.error || ""),
|
|
163
|
+
usage: { input: Number(r.usage?.input || 0), output: Number(r.usage?.output || 0), ...(r.usage?.cached ? { cached: Number(r.usage.cached) } : {}) },
|
|
164
|
+
driver: r.driver === "synthetic" ? "synthetic" : "engine",
|
|
165
|
+
memory: toMemoryInfo(r.memory),
|
|
166
|
+
...(r.case_report?.steps ? { caseReport: r.case_report } : {}),
|
|
167
|
+
...(r.outcome ? { outcome: r.outcome } : {}),
|
|
168
|
+
...(r.outcome === "assertion_failed"
|
|
169
|
+
? { failedChecks: (r.checks || []).filter((c) => c.holds === false).map((c) => ({ expected: String(c.expected || ""), actual: String(c.text || "") })) }
|
|
170
|
+
: {}),
|
|
171
|
+
...(Array.isArray(r.skills) ? { skills: r.skills.map(String) } : {}),
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** After "stop", how long the engine gets to write its artifacts before it is terminated, then killed. */
|
|
176
|
+
export const STOP_GRACE_MS = 30_000
|
|
177
|
+
export const KILL_AFTER_MS = 10_000
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Run one goal in the Python ForgeQE engine and translate its JSON-line events for the TUI and CLI. A stop (the
|
|
181
|
+
* signal, or `deadlineMs` passing) asks the engine to stop; one that has not exited `STOP_GRACE_MS` later gets
|
|
182
|
+
* SIGTERM, then SIGKILL, so a hung engine never hangs the caller.
|
|
183
|
+
*/
|
|
184
|
+
export async function runEngine(opts: {
|
|
185
|
+
python: string
|
|
186
|
+
/** The engine command; default `<python> -m forgeqe_engine run`. The mock profile runs mock.ts instead. */
|
|
187
|
+
command?: string[]
|
|
188
|
+
request: EngineRequest
|
|
189
|
+
signal?: AbortSignal
|
|
190
|
+
onEvent?: (event: AgentEvent) => void
|
|
191
|
+
/** Hard ceiling for the whole run, setup and replay included; the engine's own timeout covers only the agent. */
|
|
192
|
+
deadlineMs?: number
|
|
193
|
+
/** Overrides STOP_GRACE_MS and KILL_AFTER_MS (tests). */
|
|
194
|
+
graceMs?: { stop: number; kill: number }
|
|
195
|
+
}): Promise<{ result: RunResult; files: string[] }> {
|
|
196
|
+
const { request: req, onEvent } = opts
|
|
197
|
+
const [bin, ...args] = opts.command || [opts.python, "-m", "forgeqe_engine", "run"]
|
|
198
|
+
const child = spawn(bin!, args, {
|
|
199
|
+
env: engineEnv(debugLogging() ? { ...req.env, BROWSER_USE_LOGGING_LEVEL: "debug" } : req.env, [
|
|
200
|
+
...(req.llm.auth === "entra" ? AZURE_IDENTITY_ENV : req.llm.apiKeyEnv ? [req.llm.apiKeyEnv] : []),
|
|
201
|
+
...loginEnvNames(req.logins || []),
|
|
202
|
+
]),
|
|
203
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
204
|
+
})
|
|
205
|
+
|
|
206
|
+
const stderrTail: string[] = []
|
|
207
|
+
child.stderr.setEncoding("utf8")
|
|
208
|
+
child.stderr.on("data", (chunk: string) => {
|
|
209
|
+
for (const line of chunk.split(/\r?\n/)) {
|
|
210
|
+
if (!line.trim()) continue
|
|
211
|
+
logEvent("engine.log", line)
|
|
212
|
+
stderrTail.push(line)
|
|
213
|
+
if (stderrTail.length > 40) stderrTail.shift()
|
|
214
|
+
}
|
|
215
|
+
})
|
|
216
|
+
|
|
217
|
+
const payload = {
|
|
218
|
+
goal: req.goal,
|
|
219
|
+
out_dir: req.outDir,
|
|
220
|
+
llm: llmPayload(req.llm),
|
|
221
|
+
start_url: req.startUrl || null,
|
|
222
|
+
headed: req.headed,
|
|
223
|
+
max_steps: req.maxSteps,
|
|
224
|
+
allowed_hosts: req.allowedHosts,
|
|
225
|
+
extend_system_message: req.extendSystemMessage || null,
|
|
226
|
+
memory: req.memory || null,
|
|
227
|
+
logins: loginPayload(req.logins),
|
|
228
|
+
case_steps: req.caseSteps || 0,
|
|
229
|
+
screenshots: req.screenshots || "all",
|
|
230
|
+
accessibility: req.accessibility !== false,
|
|
231
|
+
network: req.network !== false,
|
|
232
|
+
...(req.axe ? { axe: req.axe } : {}),
|
|
233
|
+
vitals: req.vitals !== false,
|
|
234
|
+
...(req.prompts ? { prompts: req.prompts } : {}),
|
|
235
|
+
...(req.visual ? { visual: req.visual } : {}),
|
|
236
|
+
overlay: req.overlay !== false,
|
|
237
|
+
resume: req.resume || null,
|
|
238
|
+
...(req.skills?.length ? { skills: req.skills, skills_pick: req.skillsPick || "given", skills_max_chars: req.skillsMaxChars || 6000 } : {}),
|
|
239
|
+
}
|
|
240
|
+
child.stdin.write(JSON.stringify(payload) + "\n")
|
|
241
|
+
logEvent("engine.request", payload)
|
|
242
|
+
|
|
243
|
+
const grace = opts.graceMs || { stop: STOP_GRACE_MS, kill: KILL_AFTER_MS }
|
|
244
|
+
const timers: ReturnType<typeof setTimeout>[] = []
|
|
245
|
+
let stopping: "user" | "deadline" | null = null
|
|
246
|
+
let killed = false
|
|
247
|
+
const exited = () => child.exitCode !== null || child.signalCode !== null
|
|
248
|
+
const stop = (why: "user" | "deadline") => {
|
|
249
|
+
if (stopping || exited()) return
|
|
250
|
+
stopping = why
|
|
251
|
+
if (child.stdin.writable) child.stdin.write("stop\n")
|
|
252
|
+
timers.push(
|
|
253
|
+
setTimeout(() => {
|
|
254
|
+
if (exited()) return
|
|
255
|
+
killed = true
|
|
256
|
+
logEvent("engine.kill", { signal: "SIGTERM", why })
|
|
257
|
+
child.kill("SIGTERM")
|
|
258
|
+
timers.push(setTimeout(() => !exited() && child.kill("SIGKILL"), grace.kill))
|
|
259
|
+
}, grace.stop),
|
|
260
|
+
)
|
|
261
|
+
}
|
|
262
|
+
const onAbort = () => stop("user")
|
|
263
|
+
if (opts.signal?.aborted) onAbort()
|
|
264
|
+
opts.signal?.addEventListener("abort", onAbort, { once: true })
|
|
265
|
+
if (opts.deadlineMs) timers.push(setTimeout(() => stop("deadline"), opts.deadlineMs))
|
|
266
|
+
|
|
267
|
+
let pendingActions: string[] = []
|
|
268
|
+
let done: EngineResult | null = null
|
|
269
|
+
|
|
270
|
+
const handle = (ev: EngineEvent) => {
|
|
271
|
+
logEvent("engine.event", ev)
|
|
272
|
+
switch (ev.type) {
|
|
273
|
+
case "log": {
|
|
274
|
+
if (ev.level === "debug") return
|
|
275
|
+
const level = ev.level === "error" ? "error" : ev.level === "warn" ? "warn" : "info"
|
|
276
|
+
if (ev.text) onEvent?.({ type: "log", level, text: ev.text })
|
|
277
|
+
return
|
|
278
|
+
}
|
|
279
|
+
case "memory":
|
|
280
|
+
if (ev.text) onEvent?.({ type: "memory", text: ev.text })
|
|
281
|
+
return
|
|
282
|
+
case "step": {
|
|
283
|
+
const step = Number(ev.step_number || 0)
|
|
284
|
+
onEvent?.({ type: "step", step, url: ev.url || "", title: ev.title || "", category: "", source: ev.source })
|
|
285
|
+
const thought = ev.next_goal || ev.memory || ev.evaluation_previous_goal || ""
|
|
286
|
+
if (thought) onEvent?.({ type: "thought", step, text: thought })
|
|
287
|
+
pendingActions = ev.action_lines || []
|
|
288
|
+
return
|
|
289
|
+
}
|
|
290
|
+
case "result": {
|
|
291
|
+
const step = Number(ev.step || 0)
|
|
292
|
+
const items = ev.items || []
|
|
293
|
+
items.forEach((item, i) => {
|
|
294
|
+
if (item.done) return
|
|
295
|
+
const action = pendingActions[i] || "action"
|
|
296
|
+
// A check recorded as not holding worked as an action, but it is the test failing: show it as a failure.
|
|
297
|
+
const notMet = /^Recorded: expected result not met/.test(item.text || "")
|
|
298
|
+
const ok = Boolean(item.ok) && !notMet
|
|
299
|
+
const text = ok ? action : notMet ? `${action}: expected result not met` : `${action}: ${item.text || "failed"}`
|
|
300
|
+
onEvent?.({ type: "outcome", step, ok, text })
|
|
301
|
+
})
|
|
302
|
+
pendingActions = []
|
|
303
|
+
return
|
|
304
|
+
}
|
|
305
|
+
case "usage":
|
|
306
|
+
onEvent?.({ type: "usage", usage: { input: Number(ev.input || 0), output: Number(ev.output || 0), ...(ev.cached ? { cached: Number(ev.cached) } : {}) } })
|
|
307
|
+
return
|
|
308
|
+
case "done":
|
|
309
|
+
done = ev.result || {}
|
|
310
|
+
setTimeout(() => child.kill(), 10_000).unref()
|
|
311
|
+
return
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
const lines = createInterface({ input: child.stdout })
|
|
316
|
+
lines.on("line", (line) => {
|
|
317
|
+
if (!line.trim()) return
|
|
318
|
+
try {
|
|
319
|
+
handle(JSON.parse(line) as EngineEvent)
|
|
320
|
+
} catch {
|
|
321
|
+
onEvent?.({ type: "log", level: "info", text: line })
|
|
322
|
+
}
|
|
323
|
+
})
|
|
324
|
+
|
|
325
|
+
const code = await new Promise<number>((resolve, reject) => {
|
|
326
|
+
child.on("error", reject)
|
|
327
|
+
child.on("close", (c) => resolve(c ?? 1))
|
|
328
|
+
}).finally(() => {
|
|
329
|
+
for (const t of timers) clearTimeout(t)
|
|
330
|
+
opts.signal?.removeEventListener("abort", onAbort)
|
|
331
|
+
})
|
|
332
|
+
logEvent("engine.exit", { code, finished: Boolean(done), stopping, killed })
|
|
333
|
+
|
|
334
|
+
if (!done && stopping) {
|
|
335
|
+
const seconds = Math.round(grace.stop / 1000)
|
|
336
|
+
done = {
|
|
337
|
+
success: false,
|
|
338
|
+
cancelled: stopping === "user",
|
|
339
|
+
error:
|
|
340
|
+
stopping === "user"
|
|
341
|
+
? `Browser task stopped by user; the engine did not finish within ${seconds}s, so it was terminated and its artifacts may be incomplete.`
|
|
342
|
+
: `Browser task exceeded its ${Math.round((opts.deadlineMs || 0) / 1000)}s limit and was terminated.`,
|
|
343
|
+
}
|
|
344
|
+
} else if (done && stopping === "deadline") {
|
|
345
|
+
// The engine reports any stop as the user's; this one was the deadline.
|
|
346
|
+
const why = `Browser task exceeded its ${Math.round((opts.deadlineMs || 0) / 1000)}s limit and was stopped.`
|
|
347
|
+
done = { ...(done as EngineResult), success: false, cancelled: false, error: why, summary: why }
|
|
348
|
+
}
|
|
349
|
+
if (!done) {
|
|
350
|
+
const detail = stderrTail.slice(-12).join("\n")
|
|
351
|
+
throw new Error(`The ForgeQE engine exited (${code}) without a result.${detail ? `\n${detail}` : ""}`)
|
|
352
|
+
}
|
|
353
|
+
const finished: EngineResult = done
|
|
354
|
+
const result = toRunResult(req.goal, req.startUrl || "", finished)
|
|
355
|
+
recordUsage("browser agent", result.usage, req.llm.price)
|
|
356
|
+
onEvent?.({ type: "usage", usage: result.usage })
|
|
357
|
+
onEvent?.({ type: "done", result })
|
|
358
|
+
return { result, files: finished.files || [] }
|
|
359
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The mock profile's engine: speaks the Python engine's protocol (bridge.ts) without a browser or a model, so offline
|
|
3
|
+
* runs and tests go through the same bridge, events, and reports as live ones.
|
|
4
|
+
*
|
|
5
|
+
* stdin line 1 run request JSON; later "stop"
|
|
6
|
+
* stdout one JSON event per line, ending with done
|
|
7
|
+
*
|
|
8
|
+
* With a start URL it opens it and finishes; without one it fails, since it cannot choose a website.
|
|
9
|
+
*/
|
|
10
|
+
import { mkdirSync, writeFileSync } from "node:fs"
|
|
11
|
+
import path from "node:path"
|
|
12
|
+
import { createInterface } from "node:readline"
|
|
13
|
+
|
|
14
|
+
import YAML from "yaml"
|
|
15
|
+
|
|
16
|
+
export const MOCK_ENGINE = import.meta.path
|
|
17
|
+
|
|
18
|
+
type Request = { goal?: string; out_dir?: string; start_url?: string | null }
|
|
19
|
+
|
|
20
|
+
const emit = (event: Record<string, unknown>) => process.stdout.write(`${JSON.stringify(event)}\n`)
|
|
21
|
+
|
|
22
|
+
function hostOf(url: string): string {
|
|
23
|
+
try {
|
|
24
|
+
return new URL(url).hostname
|
|
25
|
+
} catch {
|
|
26
|
+
return url
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function writeArtifacts(dir: string, run: Record<string, unknown>, steps: { url: string; goal: string; actions: { text: string; ok: boolean }[] }[]): string[] {
|
|
31
|
+
mkdirSync(dir, { recursive: true })
|
|
32
|
+
const trace = [
|
|
33
|
+
"# ForgeQE run",
|
|
34
|
+
"",
|
|
35
|
+
`- Status: ${run.status}`,
|
|
36
|
+
`- Goal: ${run.goal}`,
|
|
37
|
+
`- Start URL: ${run.start_url || "(none)"}`,
|
|
38
|
+
`- Final URL: ${run.final_url || "(none)"}`,
|
|
39
|
+
"- Driver: mock engine (no browser or model)",
|
|
40
|
+
"",
|
|
41
|
+
`## Summary\n\n${run.summary}`,
|
|
42
|
+
"",
|
|
43
|
+
].join("\n")
|
|
44
|
+
const files: [string, string][] = [
|
|
45
|
+
["run.json", JSON.stringify(run, null, 2)],
|
|
46
|
+
["workflow.yaml", YAML.stringify({ format: "forgeqe-mock", goal: run.goal, start_url: run.start_url, final_url: run.final_url, status: run.status, steps: steps.map((s, i) => ({ step: i + 1, url: s.url, goal: s.goal })) })],
|
|
47
|
+
["trace.md", trace],
|
|
48
|
+
["steps.json", JSON.stringify(steps.map((s, i) => ({ n: i + 1, source: "agent", url: s.url, title: hostOf(s.url), goal: s.goal, actions: s.actions })), null, 2)],
|
|
49
|
+
]
|
|
50
|
+
return files.map(([name, body]) => {
|
|
51
|
+
const file = path.join(dir, name)
|
|
52
|
+
writeFileSync(file, body.endsWith("\n") ? body : `${body}\n`)
|
|
53
|
+
return file
|
|
54
|
+
})
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function main() {
|
|
58
|
+
const lines = createInterface({ input: process.stdin })[Symbol.asyncIterator]()
|
|
59
|
+
const first = await lines.next()
|
|
60
|
+
const req: Request = first.done ? {} : JSON.parse(first.value)
|
|
61
|
+
let stopped = false
|
|
62
|
+
void (async () => {
|
|
63
|
+
for (let l = await lines.next(); !l.done; l = await lines.next()) if (l.value.trim().toLowerCase() === "stop") stopped = true
|
|
64
|
+
stopped = true
|
|
65
|
+
})()
|
|
66
|
+
|
|
67
|
+
const goal = String(req.goal || "")
|
|
68
|
+
const startUrl = req.start_url || ""
|
|
69
|
+
const dir = String(req.out_dir || ".")
|
|
70
|
+
const steps: { url: string; goal: string; actions: { text: string; ok: boolean }[] }[] = []
|
|
71
|
+
const finish = (success: boolean, summary: string, finalUrl: string, extra: Record<string, unknown> = {}) => {
|
|
72
|
+
const usage = success ? { input: 12, output: 8 } : { input: 0, output: 0 }
|
|
73
|
+
const status = success ? "complete" : extra.cancelled ? "stopped" : "fail"
|
|
74
|
+
const files = writeArtifacts(dir, { status, goal, start_url: startUrl, final_url: finalUrl, summary, driver: "synthetic", usage }, steps)
|
|
75
|
+
emit({ type: "usage", ...usage })
|
|
76
|
+
emit({ type: "done", result: { success, summary, final_url: finalUrl, usage, files, driver: "synthetic", ...extra } })
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if (!startUrl) {
|
|
80
|
+
const why = "The mock profile cannot choose a website. Press Tab for a live profile, or pin one with --url or start_url."
|
|
81
|
+
steps.push({ url: "about:blank", goal: "There is no page to act on.", actions: [{ text: `fail ${why}`, ok: false }] })
|
|
82
|
+
emit({ type: "step", step_number: 1, url: "about:blank", title: "", next_goal: "There is no page to act on.", action_lines: ["FAIL"] })
|
|
83
|
+
emit({ type: "result", step: 1, items: [{ ok: false, text: why }] })
|
|
84
|
+
return finish(false, why, "")
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
steps.push({ url: startUrl, goal: "Opening the start URL.", actions: [{ text: `navigate ${startUrl}`, ok: true }] })
|
|
88
|
+
emit({ type: "step", step_number: 1, url: "about:blank", title: "", next_goal: "Opening the start URL.", action_lines: [`NAVIGATE ${startUrl}`] })
|
|
89
|
+
emit({ type: "result", step: 1, items: [{ ok: true }] })
|
|
90
|
+
if (stopped) return finish(false, "Browser task stopped by user", startUrl, { cancelled: true })
|
|
91
|
+
|
|
92
|
+
const thought = `Observed ${hostOf(startUrl)}. Mock provider does not drive a live model.`
|
|
93
|
+
steps.push({ url: startUrl, goal: thought, actions: [{ text: "complete", ok: true }] })
|
|
94
|
+
emit({ type: "step", step_number: 2, url: startUrl, title: hostOf(startUrl), next_goal: thought, action_lines: ["COMPLETE"] })
|
|
95
|
+
emit({ type: "result", step: 2, items: [{ ok: true, done: true }] })
|
|
96
|
+
finish(true, `Mock run finished on ${startUrl}`, startUrl)
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
if (import.meta.main) {
|
|
100
|
+
await main()
|
|
101
|
+
process.exit(0)
|
|
102
|
+
}
|