@capagents/forgeqe 0.0.0-stage → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. package/HOW_TO_USE.md +1639 -0
  2. package/README.md +338 -2
  3. package/bin/forgeqe.d.ts +3 -0
  4. package/bin/forgeqe.js +188 -0
  5. package/docs/ARCHITECTURE.md +408 -0
  6. package/docs/security-data-flow.html +1870 -0
  7. package/docs/user-guide.html +2094 -0
  8. package/evals/domain/access-role-change.yaml +37 -0
  9. package/evals/domain/banking-payment-transfer.yaml +41 -0
  10. package/evals/domain/gdpr-account-erasure.yaml +30 -0
  11. package/evals/domain/insurance-claim-fnol.yaml +30 -0
  12. package/evals/domain/insurance-policy-renewal.yaml +37 -0
  13. package/evals/domain/lending-loan-application.yaml +36 -0
  14. package/forgeqe-architecture.html +718 -0
  15. package/forgeqe.example.yaml +203 -0
  16. package/knowledge/README.md +18 -0
  17. package/knowledge/business-rules.md +10 -0
  18. package/knowledge/compliance.md +16 -0
  19. package/knowledge/environments.md +5 -0
  20. package/knowledge/glossary.md +5 -0
  21. package/knowledge/product.md +16 -0
  22. package/knowledge/roles.md +9 -0
  23. package/knowledge/test-data.md +7 -0
  24. package/llms.example.json +51 -0
  25. package/package.json +63 -4
  26. package/prompts/README.md +34 -0
  27. package/prompts/api-scenarios.md +21 -0
  28. package/prompts/api-tests.md +39 -0
  29. package/prompts/brain.md +166 -0
  30. package/prompts/browser-capture.md +7 -0
  31. package/prompts/browser-case-report.md +7 -0
  32. package/prompts/browser-checks.md +14 -0
  33. package/prompts/browser-efficiency.md +7 -0
  34. package/prompts/browser-error-recovery.md +15 -0
  35. package/prompts/browser-extra.md +5 -0
  36. package/prompts/browser-goal-verification.md +13 -0
  37. package/prompts/browser-skill-select.md +6 -0
  38. package/prompts/domain-rule.md +13 -0
  39. package/prompts/eval-judge.md +19 -0
  40. package/prompts/explore-charters.md +27 -0
  41. package/prompts/find-tests.md +28 -0
  42. package/prompts/heal.md +40 -0
  43. package/prompts/pr-review.md +29 -0
  44. package/prompts/skill-rule.md +6 -0
  45. package/prompts/test-agent.md +34 -0
  46. package/prompts/test-reviewer.md +30 -0
  47. package/prompts/test-rules.md +28 -0
  48. package/py/forgeqe_engine/__init__.py +3 -0
  49. package/py/forgeqe_engine/__main__.py +3 -0
  50. package/py/forgeqe_engine/brand.py +453 -0
  51. package/py/forgeqe_engine/brand_assets.py +6 -0
  52. package/py/forgeqe_engine/browser.py +196 -0
  53. package/py/forgeqe_engine/capture.py +538 -0
  54. package/py/forgeqe_engine/checks.py +98 -0
  55. package/py/forgeqe_engine/cli.py +188 -0
  56. package/py/forgeqe_engine/diagnostics.py +609 -0
  57. package/py/forgeqe_engine/llm.py +195 -0
  58. package/py/forgeqe_engine/logins.py +251 -0
  59. package/py/forgeqe_engine/loops.py +224 -0
  60. package/py/forgeqe_engine/memory.py +1004 -0
  61. package/py/forgeqe_engine/network.py +244 -0
  62. package/py/forgeqe_engine/prompts.py +20 -0
  63. package/py/forgeqe_engine/replay.py +240 -0
  64. package/py/forgeqe_engine/report.py +173 -0
  65. package/py/forgeqe_engine/service.py +1335 -0
  66. package/py/forgeqe_engine/settings.py +93 -0
  67. package/py/forgeqe_engine/skills.py +88 -0
  68. package/py/forgeqe_engine/summary.py +239 -0
  69. package/py/forgeqe_engine/urls.py +81 -0
  70. package/py/forgeqe_engine/workflow_yaml.py +689 -0
  71. package/skills/dotnet-nunit-playwright/SKILL.md +34 -0
  72. package/skills/dynamics-365/SKILL.md +21 -0
  73. package/skills/playwright-typescript/SKILL.md +35 -0
  74. package/skills/writing-skills/SKILL.md +30 -0
  75. package/src/api/plan.ts +246 -0
  76. package/src/api/run.ts +204 -0
  77. package/src/api/scenario.ts +502 -0
  78. package/src/api/schema.ts +170 -0
  79. package/src/api/spec.ts +624 -0
  80. package/src/api/store.ts +232 -0
  81. package/src/api/traffic.ts +281 -0
  82. package/src/api/types.ts +163 -0
  83. package/src/api/write.ts +272 -0
  84. package/src/auth/signin.ts +494 -0
  85. package/src/auth/store.ts +170 -0
  86. package/src/brain/index.ts +189 -0
  87. package/src/brain/prompt.ts +19 -0
  88. package/src/capabilities/docs.ts +101 -0
  89. package/src/capabilities/guide.ts +455 -0
  90. package/src/capabilities/registry.ts +343 -0
  91. package/src/capabilities/render.ts +106 -0
  92. package/src/capabilities/status.ts +137 -0
  93. package/src/card.ts +11 -0
  94. package/src/cases/ado.ts +325 -0
  95. package/src/cases/api.ts +117 -0
  96. package/src/cases/bugs.ts +223 -0
  97. package/src/cases/confluence.ts +111 -0
  98. package/src/cases/context.ts +223 -0
  99. package/src/cases/dashboards.ts +137 -0
  100. package/src/cases/failure.ts +97 -0
  101. package/src/cases/files.ts +295 -0
  102. package/src/cases/goal.ts +53 -0
  103. package/src/cases/http.ts +111 -0
  104. package/src/cases/index.ts +131 -0
  105. package/src/cases/jira.ts +122 -0
  106. package/src/cases/qtest.ts +108 -0
  107. package/src/cases/quality.ts +190 -0
  108. package/src/cases/rag.ts +100 -0
  109. package/src/cases/readiness.ts +284 -0
  110. package/src/cases/save.ts +149 -0
  111. package/src/cases/sprints.ts +148 -0
  112. package/src/cases/testrail.ts +176 -0
  113. package/src/cases/types.ts +107 -0
  114. package/src/cases/wiki.ts +105 -0
  115. package/src/cases/xray.ts +206 -0
  116. package/src/cases/zephyr.ts +153 -0
  117. package/src/childEnv.ts +86 -0
  118. package/src/cli/args.ts +114 -0
  119. package/src/cli/help.ts +167 -0
  120. package/src/cli/index.ts +1142 -0
  121. package/src/cli/terminal.ts +187 -0
  122. package/src/code/areas.ts +337 -0
  123. package/src/code/attention.ts +214 -0
  124. package/src/code/find.ts +540 -0
  125. package/src/code/handoff.ts +541 -0
  126. package/src/code/history.ts +170 -0
  127. package/src/code/impact.ts +231 -0
  128. package/src/code/index.ts +34 -0
  129. package/src/code/livebrowser.ts +67 -0
  130. package/src/code/loop.ts +603 -0
  131. package/src/code/notes.ts +75 -0
  132. package/src/code/profile.ts +226 -0
  133. package/src/code/prompt.ts +303 -0
  134. package/src/code/prreview.ts +649 -0
  135. package/src/code/prtests.ts +103 -0
  136. package/src/code/replay.ts +772 -0
  137. package/src/code/review.ts +202 -0
  138. package/src/code/runtime.ts +781 -0
  139. package/src/code/slices.ts +159 -0
  140. package/src/code/testrun.ts +187 -0
  141. package/src/code/traceability.ts +100 -0
  142. package/src/code/worktree.ts +153 -0
  143. package/src/commands.ts +101 -0
  144. package/src/config.ts +970 -0
  145. package/src/detail.ts +65 -0
  146. package/src/domain.ts +19 -0
  147. package/src/engine/bridge.ts +359 -0
  148. package/src/engine/mock.ts +102 -0
  149. package/src/engine/setup.ts +298 -0
  150. package/src/entra.ts +154 -0
  151. package/src/env.ts +24 -0
  152. package/src/evals/domain.ts +338 -0
  153. package/src/explore/charters.ts +151 -0
  154. package/src/explore/run.ts +280 -0
  155. package/src/explore/story.ts +106 -0
  156. package/src/guard.ts +51 -0
  157. package/src/heal/loop.ts +357 -0
  158. package/src/heal/prompt.ts +143 -0
  159. package/src/heal/report.ts +95 -0
  160. package/src/index.ts +12 -0
  161. package/src/init.ts +102 -0
  162. package/src/knowledge.ts +114 -0
  163. package/src/legacyEnv.ts +9 -0
  164. package/src/llm.ts +61 -0
  165. package/src/locale.ts +25 -0
  166. package/src/logins.ts +148 -0
  167. package/src/managed.ts +101 -0
  168. package/src/mcp/apitools.ts +335 -0
  169. package/src/mcp/captools.ts +25 -0
  170. package/src/mcp/casetools.ts +223 -0
  171. package/src/mcp/commands.ts +421 -0
  172. package/src/mcp/exploretools.ts +129 -0
  173. package/src/mcp/external.ts +172 -0
  174. package/src/mcp/git.ts +78 -0
  175. package/src/mcp/notifytools.ts +49 -0
  176. package/src/mcp/prtools.ts +261 -0
  177. package/src/mcp/qualitytools.ts +154 -0
  178. package/src/mcp/repotests.ts +286 -0
  179. package/src/mcp/server.ts +234 -0
  180. package/src/mcp/tools.ts +810 -0
  181. package/src/memory.ts +250 -0
  182. package/src/money.ts +39 -0
  183. package/src/monitor/alerts.ts +94 -0
  184. package/src/monitor/background.ts +68 -0
  185. package/src/monitor/run.ts +263 -0
  186. package/src/monitor/state.ts +105 -0
  187. package/src/notify/events.ts +144 -0
  188. package/src/notify/notice.ts +177 -0
  189. package/src/prices.ts +175 -0
  190. package/src/print.ts +54 -0
  191. package/src/privacy.ts +35 -0
  192. package/src/prompts.ts +165 -0
  193. package/src/quality/budgets.ts +249 -0
  194. package/src/quality/history.ts +149 -0
  195. package/src/quality/visual.ts +201 -0
  196. package/src/report/data.ts +475 -0
  197. package/src/report/html.ts +106 -0
  198. package/src/report/template.html +84 -0
  199. package/src/run.ts +243 -0
  200. package/src/sessionLog.ts +216 -0
  201. package/src/sessionLogins.ts +94 -0
  202. package/src/skills/registry.ts +217 -0
  203. package/src/skills/scaffold.ts +65 -0
  204. package/src/skills/select.ts +37 -0
  205. package/src/skills/view.ts +44 -0
  206. package/src/suite/report.ts +135 -0
  207. package/src/suite/run.ts +303 -0
  208. package/src/summary.ts +188 -0
  209. package/src/tools.ts +130 -0
  210. package/src/tui/App.tsx +983 -0
  211. package/src/tui/clipboard.ts +36 -0
  212. package/src/tui/components.tsx +263 -0
  213. package/src/tui/dialogs.ts +65 -0
  214. package/src/tui/main.tsx +20 -0
  215. package/src/tui/text.ts +28 -0
  216. package/src/tui/theme.ts +104 -0
  217. package/src/tui/transcript.tsx +344 -0
  218. package/src/types.ts +146 -0
  219. package/src/update.ts +311 -0
  220. package/src/usage.ts +158 -0
@@ -0,0 +1,170 @@
1
+ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"
2
+ import path from "node:path"
3
+
4
+ import { projectDir } from "../tools.ts"
5
+
6
+ /**
7
+ * Each automated test's recent results in a repository, to tell a flaky test from a broken one, and the tests in
8
+ * quarantine. Kept in .forgeqe/test-history.json in the repository (commit it to share the history; CI can keep it
9
+ * between runs as a cache). Quarantine changes no test code: a quarantined test still runs, so its history goes on,
10
+ * but its failure does not fail the run.
11
+ */
12
+
13
+ export const HISTORY_FILE = path.join(".forgeqe", "test-history.json")
14
+ const KEEP = 30
15
+ const WINDOW = 20
16
+ /** Consecutive clean passes after which a quarantined test is offered for release. */
17
+ export const RELEASE_AFTER = 5
18
+
19
+ export type Result = { at: string; status: "passed" | "failed" | "skipped"; /** Failed, then passed when run again. */ retried?: boolean; message?: string; seconds?: number }
20
+ export type Quarantine = { at: string; reason: string; by: "auto" | "user" }
21
+ export type TestHistory = { runs: Result[]; quarantined?: Quarantine }
22
+ type HistoryFile = { version: 1; tests: Record<string, TestHistory> }
23
+
24
+ export type Cause = "timing" | "overlay" | "data" | "network" | "environment" | "unknown"
25
+
26
+ export const CAUSE_TEXT: Record<Cause, string> = {
27
+ timing: "timing: waits that are too short or race the page (timeouts, elements not ready, detached)",
28
+ overlay: "something covers the element now and then (a banner, a dialog, a spinner intercepts the click)",
29
+ data: "data that changes between runs (other values, other counts, data left by another test)",
30
+ network: "network or backend errors now and then (failed requests, 5xx, resets)",
31
+ environment: "the environment (the browser crashed or closed, the server was not ready)",
32
+ unknown: "no pattern in the failure messages",
33
+ }
34
+
35
+ export type Flakiness = {
36
+ name: string
37
+ runs: number
38
+ passRate: number
39
+ /** Runs where it failed, then passed when run again, in the last WINDOW runs. */
40
+ retryPasses: number
41
+ /** Pass/fail changes in the last WINDOW runs. */
42
+ flips: number
43
+ score: number
44
+ flaky: boolean
45
+ cause: Cause
46
+ example?: string
47
+ /** The last results, oldest first: ✓ passed, ✗ failed, ↻ passed on retry, – skipped. */
48
+ trail: string
49
+ quarantined?: Quarantine
50
+ /** Quarantined and clean for RELEASE_AFTER runs in a row. */
51
+ releasable: boolean
52
+ }
53
+
54
+ const file = (repo: string) => path.join(projectDir(repo), path.basename(HISTORY_FILE))
55
+
56
+ export function readHistory(repo: string): HistoryFile {
57
+ try {
58
+ const data = JSON.parse(readFileSync(file(repo), "utf8")) as HistoryFile
59
+ return { version: 1, tests: data?.tests && typeof data.tests === "object" ? data.tests : {} }
60
+ } catch {
61
+ return { version: 1, tests: {} }
62
+ }
63
+ }
64
+
65
+ function save(repo: string, h: HistoryFile) {
66
+ mkdirSync(path.dirname(file(repo)), { recursive: true })
67
+ writeFileSync(file(repo), `${JSON.stringify(h, null, 1)}\n`)
68
+ }
69
+
70
+ /** Add one run's results. */
71
+ export function recordResults(repo: string, results: { name: string; status: Result["status"]; retried?: boolean; message?: string; seconds?: number }[], at = new Date().toISOString()): void {
72
+ if (!results.length) return
73
+ const h = readHistory(repo)
74
+ for (const r of results) {
75
+ const t = h.tests[r.name] || { runs: [] }
76
+ t.runs.push({ at, status: r.status, ...(r.retried ? { retried: true } : {}), ...(r.message ? { message: r.message.slice(0, 300) } : {}), ...(r.seconds !== undefined ? { seconds: r.seconds } : {}) })
77
+ t.runs = t.runs.slice(-KEEP)
78
+ h.tests[r.name] = t
79
+ }
80
+ save(repo, h)
81
+ }
82
+
83
+ /** The likely cause behind a test's failure messages. */
84
+ export function likelyCause(messages: string[]): { cause: Cause; example?: string } {
85
+ const rules: [Cause, RegExp][] = [
86
+ ["overlay", /intercepts pointer events|element is not visible|is covered|obscured|not clickable at point|other element would receive the click|ElementClickInterceptedException/i],
87
+ ["network", /net::ERR_|ECONNRESET|ECONNREFUSED|ETIMEDOUT|socket hang up|\b5\d\d\b.*(error|status|response)|request failed|fetch failed/i],
88
+ ["environment", /browser has been closed|Target (page, context or browser )?closed|browser crashed|Protocol error|WebDriverException|session not created|server (is )?not (ready|running)/i],
89
+ ["timing", /timeout|timed out|exceeded|waiting for|detached from (the )?DOM|stale element|not attached|toBeVisible|toHaveText.*\bTimeout\b/i],
90
+ ["data", /expected[\s\S]{0,80}(received|but (was|got)|actual)|toEqual|toBe\(|AssertionError|assert .* ==|Expected: .*\n?\s*Received:/i],
91
+ ]
92
+ const counts = new Map<Cause, { n: number; example: string }>()
93
+ for (const m of messages.filter(Boolean)) {
94
+ const hit = rules.find(([, r]) => r.test(m))
95
+ if (!hit) continue
96
+ const c = counts.get(hit[0]) || { n: 0, example: m }
97
+ c.n++
98
+ counts.set(hit[0], c)
99
+ }
100
+ const best = [...counts].sort((a, b) => b[1].n - a[1].n)[0]
101
+ return best ? { cause: best[0], example: best[1].example.slice(0, 200) } : { cause: "unknown" }
102
+ }
103
+
104
+ /** How flaky a test is from its history: passes on retry and flip-flops, but a test that only fails is broken, not flaky. */
105
+ export function flakiness(name: string, t: TestHistory, flakyAfter = 2): Flakiness {
106
+ const ran = t.runs.filter((r) => r.status !== "skipped")
107
+ const recent = ran.slice(-WINDOW)
108
+ let flips = 0
109
+ for (let i = 1; i < recent.length; i++) if (recent[i].status !== recent[i - 1].status) flips++
110
+ const retryPasses = recent.filter((r) => r.retried).length
111
+ const passes = recent.filter((r) => r.status === "passed").length
112
+ const failures = recent.filter((r) => r.status === "failed").length + retryPasses
113
+ const score = retryPasses + Math.floor(flips / 2)
114
+ const tail = ran.slice(-RELEASE_AFTER)
115
+ const { cause, example } = likelyCause(t.runs.slice(-WINDOW).filter((r) => r.message).map((r) => r.message!))
116
+ return {
117
+ name,
118
+ runs: ran.length,
119
+ passRate: recent.length ? Math.round((passes / recent.length) * 100) / 100 : 0,
120
+ retryPasses,
121
+ flips,
122
+ score,
123
+ flaky: score >= flakyAfter && passes > 0 && failures > 0,
124
+ cause,
125
+ ...(example ? { example } : {}),
126
+ trail: t.runs.slice(-WINDOW).map((r) => (r.status === "skipped" ? "–" : r.retried ? "↻" : r.status === "passed" ? "✓" : "✗")).join(""),
127
+ ...(t.quarantined ? { quarantined: t.quarantined } : {}),
128
+ releasable: Boolean(t.quarantined) && tail.length === RELEASE_AFTER && tail.every((r) => r.status === "passed" && !r.retried),
129
+ }
130
+ }
131
+
132
+ /** Every test's flakiness, flakiest first. */
133
+ export function flakyReport(repo: string, flakyAfter = 2): Flakiness[] {
134
+ return Object.entries(readHistory(repo).tests)
135
+ .map(([name, t]) => flakiness(name, t, flakyAfter))
136
+ .sort((a, b) => Number(Boolean(b.quarantined)) - Number(Boolean(a.quarantined)) || b.score - a.score || a.passRate - b.passRate)
137
+ }
138
+
139
+ export function quarantined(repo: string): Map<string, Quarantine> {
140
+ return new Map(Object.entries(readHistory(repo).tests).flatMap(([n, t]) => (t.quarantined ? [[n, t.quarantined] as [string, Quarantine]] : [])))
141
+ }
142
+
143
+ /** Put tests in quarantine, or take them out (reason null). Returns the names changed. */
144
+ export function setQuarantine(repo: string, names: string[], reason: string | null, by: Quarantine["by"] = "user"): string[] {
145
+ const h = readHistory(repo)
146
+ const changed: string[] = []
147
+ for (const n of names) {
148
+ const t = h.tests[n] || { runs: [] }
149
+ if (reason === null) {
150
+ if (!t.quarantined) continue
151
+ delete t.quarantined
152
+ } else {
153
+ if (t.quarantined) continue
154
+ t.quarantined = { at: new Date().toISOString(), reason, by }
155
+ }
156
+ h.tests[n] = t
157
+ changed.push(n)
158
+ }
159
+ if (changed.length) save(repo, h)
160
+ return changed
161
+ }
162
+
163
+ /** A test name ForgeQE knows, from a name the user gave: exact, else the one that contains it. */
164
+ export function matchTest(repo: string, given: string): string | null {
165
+ const names = Object.keys(readHistory(repo).tests)
166
+ if (names.includes(given)) return given
167
+ const low = given.toLowerCase()
168
+ const hits = names.filter((n) => n.toLowerCase().includes(low))
169
+ return hits.length === 1 ? hits[0] : null
170
+ }
@@ -0,0 +1,231 @@
1
+ import { spawnSync } from "node:child_process"
2
+ import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs"
3
+ import path from "node:path"
4
+
5
+ import type { AppConfig } from "../config.ts"
6
+ import { adoSettings } from "../cases/ado.ts"
7
+ import { http } from "../cases/http.ts"
8
+ import { runDirectory } from "../run.ts"
9
+ import type { TestSelection } from "./find.ts"
10
+ import { readMappings } from "./traceability.ts"
11
+
12
+ /**
13
+ * Which of a repository's tests a change affects, without a model: the test files it changes, the tests that import
14
+ * a changed file (following imports through page objects and helpers), the tests that name a changed class or page
15
+ * object, and the tests mapped (.forgeqe/test-cases.json) to the test cases of the work items the change is linked
16
+ * to. Each test comes with its reasons, so a person, or the find-tests agent, can see why it was picked.
17
+ */
18
+
19
+ export type ImpactedTest = { file: string; reasons: string[] }
20
+ export type Impact = { changed: string[]; tests: ImpactedTest[]; untested: string[] }
21
+
22
+ // By name, as the frameworks find them: a helper in a tests/ folder is not a test, and imports go on through it.
23
+ const TEST_FILE = /\.(spec|test|e2e|cy)\.[cm]?[jt]sx?$|(^|\/)test_[^/]*\.py$|_test\.py$|Tests?\.(cs|java|kt)$|Spec\.(cs|java|kt)$/i
24
+ const CODE = /\.([cm]?[jt]sx?|py|cs|java|kt)$/i
25
+ const SKIP = /(^|\/)(node_modules|dist|build|out|bin|obj|\.forgeqe|\.venv|venv|__pycache__|coverage|test-results|playwright-report|needs-attention)\//
26
+ const MAX_FILES = 6000
27
+ const MAX_READ = 300_000
28
+
29
+ const posix = (p: string) => p.split(path.sep).join("/")
30
+ const git = (repo: string, args: string[]) => spawnSync("git", ["-c", "core.quotepath=false", ...args], { cwd: repo, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 })
31
+
32
+ export const isTestFile = (rel: string) => TEST_FILE.test(rel) && CODE.test(rel)
33
+
34
+ /** Files changed between the merge base of `base` and `head`, existing in the working tree or not. */
35
+ export function changedBetween(repo: string, base: string, head = "HEAD"): string[] {
36
+ const mb = git(repo, ["merge-base", base, head])
37
+ if (mb.status !== 0) throw new Error(`No common history between ${base} and ${head}: ${mb.stderr.trim().slice(0, 300)}`)
38
+ const out = git(repo, ["diff", "--name-only", "-M", mb.stdout.trim(), head])
39
+ if (out.status !== 0) throw new Error(`git diff failed: ${out.stderr.trim().slice(0, 300)}`)
40
+ return out.stdout.split("\n").map((l) => l.trim()).filter(Boolean)
41
+ }
42
+
43
+ function trackedCode(repo: string): string[] {
44
+ const out = git(repo, ["ls-files", "--cached", "--others", "--exclude-standard"])
45
+ return out.stdout
46
+ .split("\n")
47
+ .map((l) => l.trim())
48
+ .filter((f) => f && CODE.test(f) && !SKIP.test(f))
49
+ .slice(0, MAX_FILES)
50
+ }
51
+
52
+ function read(repo: string, rel: string): string {
53
+ try {
54
+ const file = path.join(repo, rel)
55
+ return statSync(file).size <= MAX_READ ? readFileSync(file, "utf8") : ""
56
+ } catch {
57
+ return ""
58
+ }
59
+ }
60
+
61
+ /** The files a JS/TS or Python file imports, as repository paths (only those that exist). */
62
+ export function importsOf(rel: string, text: string, files: Set<string>): string[] {
63
+ const dir = path.posix.dirname(rel)
64
+ const out: string[] = []
65
+ if (/\.[cm]?[jt]sx?$/i.test(rel)) {
66
+ for (const m of text.matchAll(/(?:\bfrom\s*|\bimport\s*\(\s*|\brequire\s*\(\s*|^\s*import\s+)['"](\.{1,2}\/[^'"]+)['"]/gm)) {
67
+ const base = path.posix.normalize(path.posix.join(dir, m[1]))
68
+ const hit = [base, ...[".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"].map((e) => base.replace(/\.(js|mjs|cjs)$/, "") + e), ...["ts", "tsx", "js"].map((e) => `${base}/index.${e}`)].find((c) => files.has(c))
69
+ if (hit) out.push(hit)
70
+ }
71
+ } else if (/\.py$/i.test(rel)) {
72
+ for (const m of text.matchAll(/^\s*(?:from\s+([\w.]+)\s+import|import\s+([\w.]+))/gm)) {
73
+ const mod = (m[1] || m[2]).replace(/^\.+/, "")
74
+ if (!mod) continue
75
+ const tail = mod.split(".").join("/")
76
+ const hit = [...files].find((f) => f.endsWith(`${tail}.py`) || f.endsWith(`${tail}/__init__.py`))
77
+ if (hit) out.push(hit)
78
+ }
79
+ }
80
+ return out
81
+ }
82
+
83
+ /** A changed file's names a test can mention: its class or module name (LoginPage from pages/LoginPage.ts). */
84
+ function namesOf(rel: string): string[] {
85
+ const stem = path.posix.basename(rel).replace(/\.(spec|test)?\.?[cm]?[jt]sx?$|\.(py|cs|java|kt)$/i, "")
86
+ return stem.length >= 5 && /[A-Z_]/.test(stem) ? [stem] : []
87
+ }
88
+
89
+ /** The tests a change affects, each with why. `mapped` adds tests found from linked test cases. */
90
+ export function analyzeImpact(repo: string, changed: string[], mapped: ImpactedTest[] = []): Impact {
91
+ const files = trackedCode(repo)
92
+ const set = new Set(files)
93
+ const tests = new Map<string, Set<string>>()
94
+ const add = (file: string, why: string) => {
95
+ if (!existsSync(path.join(repo, file))) return
96
+ if (!tests.has(file)) tests.set(file, new Set())
97
+ tests.get(file)!.add(why)
98
+ }
99
+ for (const f of changed) if (isTestFile(f)) add(f, "changed in this pull request")
100
+
101
+ // Who imports whom, reversed, so a changed module leads to every test that reaches it.
102
+ const importers = new Map<string, string[]>()
103
+ const texts = new Map<string, string>()
104
+ for (const f of files) {
105
+ const text = read(repo, f)
106
+ texts.set(f, text)
107
+ for (const dep of importsOf(f, text, set)) importers.set(dep, [...(importers.get(dep) || []), f])
108
+ }
109
+ const source = changed.filter((f) => CODE.test(f) && !isTestFile(f))
110
+ const reached = new Set<string>()
111
+ for (const f of source) {
112
+ const queue = [{ file: f, depth: 0 }]
113
+ const seen = new Set([f])
114
+ while (queue.length) {
115
+ const { file, depth } = queue.shift()!
116
+ for (const imp of importers.get(file) || []) {
117
+ if (seen.has(imp) || depth >= 6) continue
118
+ seen.add(imp)
119
+ if (isTestFile(imp)) {
120
+ add(imp, `imports ${f}${depth ? " (through other modules)" : ""}`)
121
+ reached.add(f)
122
+ } else queue.push({ file: imp, depth: depth + 1 })
123
+ }
124
+ }
125
+ }
126
+ // C#, Java, and anything imports cannot show: tests that name a changed class or page object.
127
+ const testFiles = files.filter(isTestFile)
128
+ for (const f of source) {
129
+ for (const name of namesOf(f)) {
130
+ const word = new RegExp(`\\b${name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`)
131
+ for (const t of testFiles) {
132
+ if (word.test(texts.get(t) || "")) {
133
+ add(t, `uses ${name} (${f})`)
134
+ reached.add(f)
135
+ }
136
+ }
137
+ }
138
+ }
139
+ for (const m of mapped) for (const why of m.reasons) add(m.file, why)
140
+ return {
141
+ changed,
142
+ tests: [...tests].map(([file, reasons]) => ({ file, reasons: [...reasons] })).sort((a, b) => a.file.localeCompare(b.file)),
143
+ untested: source.filter((f) => !reached.has(f)),
144
+ }
145
+ }
146
+
147
+ /**
148
+ * Tests mapped to the test cases that test the given Azure DevOps work items (a story or bug linked to the pull
149
+ * request, "Tested By" its test cases): from .forgeqe/test-cases.json.
150
+ */
151
+ export async function testsForWorkItems(cfg: AppConfig, repo: string, ids: number[]): Promise<ImpactedTest[]> {
152
+ if (!ids.length || !cfg.sources.ado) return []
153
+ const map = readMappings(repo).cases
154
+ const { org, auth } = adoSettings(cfg.sources)
155
+ const out: ImpactedTest[] = []
156
+ for (const id of ids.slice(0, 20)) {
157
+ try {
158
+ const { data } = await http<{ relations?: { rel: string; url: string }[] }>(`${org}/_apis/wit/workitems/${id}?$expand=relations&api-version=7.1`, { auth })
159
+ for (const r of data?.relations || []) {
160
+ if (r.rel !== "Microsoft.VSTS.Common.TestedBy-Forward") continue
161
+ const caseId = `ADO-${r.url.split("/").pop()}`
162
+ const m = map[caseId]
163
+ if (m?.testFile && m.status !== "needs_attention") out.push({ file: m.testFile, reasons: [`automates ${caseId}, which tests work item ${id} linked to the pull request`] })
164
+ }
165
+ } catch {
166
+ // A work item the token cannot read: the other signals still count.
167
+ }
168
+ }
169
+ return out
170
+ }
171
+
172
+ /** The nearest folder above `file` (inside the repo) that holds one of `names`. */
173
+ function nearest(repo: string, file: string, names: string[]): string | null {
174
+ let dir = path.dirname(path.join(repo, file))
175
+ while (!path.relative(repo, dir).startsWith("..")) {
176
+ if (names.some((n) => existsSync(path.join(dir, n)))) return dir
177
+ const up = path.dirname(dir)
178
+ if (up === dir) break
179
+ dir = up
180
+ }
181
+ return null
182
+ }
183
+
184
+ const quote = (s: string) => (/^[\w./@:-]+$/.test(s) ? s : `'${s.replace(/'/g, `'\\''`)}'`)
185
+
186
+ /**
187
+ * A run command for the impacted test files, without a model, writing JUnit XML to `junit`: Playwright and pytest.
188
+ * Null for other frameworks (the find-tests agent works those out).
189
+ */
190
+ export function impactCommand(repo: string, files: string[], junit: string): { command: string; framework: string } | null {
191
+ if (!files.length) return null
192
+ const js = files.filter((f) => /\.[cm]?[jt]sx?$/i.test(f))
193
+ const py = files.filter((f) => /\.py$/i.test(f))
194
+ const env = (name: string, value: string) => (process.platform === "win32" ? `set ${name}=${value}&& ` : `${name}=${quote(value)} `)
195
+ if (js.length === files.length) {
196
+ const dir = nearest(repo, js[0], ["playwright.config.ts", "playwright.config.js", "playwright.config.mjs"])
197
+ if (!dir || js.some((f) => nearest(repo, f, ["playwright.config.ts", "playwright.config.js", "playwright.config.mjs"]) !== dir)) return null
198
+ const rel = js.map((f) => quote(posix(path.relative(dir, path.join(repo, f)))))
199
+ const cd = posix(path.relative(repo, dir))
200
+ return { framework: "Playwright", command: `${cd ? `cd ${quote(cd)} && ` : ""}${env("PLAYWRIGHT_JUNIT_OUTPUT_NAME", junit)}npx playwright test ${rel.join(" ")} --reporter=line,junit` }
201
+ }
202
+ if (py.length === files.length) return { framework: "pytest", command: `python -m pytest ${py.map(quote).join(" ")} --junitxml=${quote(junit)}` }
203
+ return null
204
+ }
205
+
206
+ /** A test selection run_tests can run (after the user confirms), from the impact analysis alone. */
207
+ export function impactSelection(cfg: AppConfig, repo: string, request: string, impact: Impact): TestSelection | null {
208
+ const dir = runDirectory(path.join(cfg.export.outDir, "tests"))
209
+ const junit = path.join(dir, "results.xml")
210
+ const cmd = impactCommand(repo, impact.tests.map((t) => t.file), junit)
211
+ if (!cmd) return null
212
+ const sel: TestSelection = {
213
+ id: path.basename(dir),
214
+ dir,
215
+ repo,
216
+ request,
217
+ framework: cmd.framework,
218
+ tests: impact.tests.map((t) => ({ name: t.file, file: t.file, reason: t.reasons.join("; ") })),
219
+ related: [],
220
+ listCommand: "",
221
+ listed: false,
222
+ runCommand: cmd.command,
223
+ results: [junit],
224
+ notes: impact.untested.length ? `No test reaches: ${impact.untested.join(", ")}.` : "",
225
+ model: "",
226
+ usage: { input: 0, output: 0 },
227
+ createdAt: new Date().toISOString(),
228
+ }
229
+ writeFileSync(path.join(dir, "selection.json"), JSON.stringify(sel, null, 2))
230
+ return sel
231
+ }
@@ -0,0 +1,34 @@
1
+ import { existsSync, readdirSync, statSync } from "node:fs"
2
+ import path from "node:path"
3
+
4
+ import type { RunResult } from "../types.ts"
5
+ import type { AppConfig } from "../config.ts"
6
+ import { FAILURE_ACTION, FAILURE_LABEL, type Failure } from "../cases/failure.ts"
7
+
8
+ export { runCodeAgent } from "./loop.ts"
9
+
10
+ /** Newest run directory under the export folder that has a run.json. */
11
+ export function latestRunDir(outRoot: string): string | undefined {
12
+ if (!existsSync(outRoot)) return undefined
13
+ return readdirSync(outRoot)
14
+ .map((name) => path.join(outRoot, name))
15
+ .filter((dir) => existsSync(path.join(dir, "run.json")))
16
+ .sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs)[0]
17
+ }
18
+
19
+ /**
20
+ * Why the coding step should not follow this run, or null when it should. A run that found a functional defect goes
21
+ * to the coding agent too: the test asserts the correct behaviour and is marked as an expected failure.
22
+ */
23
+ export function codeSkipReason(cfg: AppConfig, result: Pick<RunResult, "status" | "driver" | "outcome" | "failedChecks">, failure?: Failure): string | null {
24
+ if (failure && failure.type !== "functional_defect") return `${FAILURE_LABEL[failure.type]}: ${FAILURE_ACTION[failure.type]}`
25
+ if (!failure && result.outcome === "assertion_failed") {
26
+ const what = (result.failedChecks || []).map((c) => `expected ${c.expected}, the page shows ${c.actual}`).join("; ")
27
+ return `the test case failed: an expected result does not hold (${what || "see the run"}); a test is written from a passing run`
28
+ }
29
+ if (!failure && result.status !== "complete") return "the browser run did not pass"
30
+ if (result.driver === "synthetic") return "mock runs have no real page to test"
31
+ if (cfg.llm.provider === "mock" && !cfg.code.model) return "the mock profile has no model; pick a live profile or set code.model"
32
+ if (cfg.llm.provider === "none" && !cfg.code.model) return "no model is set up; add a profile to llms.json or set code.model"
33
+ return null
34
+ }
@@ -0,0 +1,67 @@
1
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"
2
+ import path from "node:path"
3
+
4
+ /** The agent runtime names this server's tools `live_<tool>`, e.g. `live_browser_snapshot`. */
5
+ export const LIVE_BROWSER_SERVER = "live"
6
+ export const PLAYWRIGHT_MCP = "@playwright/mcp@0.0.83"
7
+ /** Also the time allowed for the first start, when npx downloads the server. */
8
+ const TIMEOUT_MS = 120_000
9
+
10
+ type BrowserProfile = {
11
+ channel?: string
12
+ executable?: string
13
+ headless?: boolean
14
+ user_agent?: string
15
+ language?: string
16
+ timezone?: string
17
+ viewport?: number[]
18
+ }
19
+
20
+ export type LiveBrowser = {
21
+ /** MCP server config for the agent runtime. */
22
+ config: Record<string, unknown>
23
+ /** Where the server saves screenshots and snapshots; the coding agent may read it. */
24
+ dir: string
25
+ }
26
+
27
+ function runBrowser(runDir: string): BrowserProfile {
28
+ try {
29
+ return (JSON.parse(readFileSync(path.join(runDir, "run.json"), "utf8")).browser || {}) as BrowserProfile
30
+ } catch {
31
+ return {}
32
+ }
33
+ }
34
+
35
+ /**
36
+ * Microsoft's Playwright MCP server on a browser set up like the one the run used (channel, headless, user agent,
37
+ * language, time zone, viewport), so the coding agent can open pages, read their elements, try locators, and act in
38
+ * seconds, through the same library the repository's tests use, instead of building and running a test to look.
39
+ * Null without Node.js (npx).
40
+ */
41
+ export function liveBrowser(runDir: string, which: (bin: string) => string | null = Bun.which): LiveBrowser | null {
42
+ if (!which("npx")) return null
43
+ const b = runBrowser(runDir)
44
+ const dir = path.join(runDir, "live-browser")
45
+ mkdirSync(dir, { recursive: true })
46
+ const launch: Record<string, unknown> = { headless: Boolean(b.headless) }
47
+ if (b.channel) launch.channel = b.channel
48
+ else if (b.executable && existsSync(b.executable)) launch.executablePath = b.executable
49
+ else launch.channel = "chrome"
50
+ const context: Record<string, unknown> = {}
51
+ if (b.viewport?.length === 2) context.viewport = { width: b.viewport[0], height: b.viewport[1] }
52
+ if (b.language) context.locale = b.language
53
+ if (b.timezone) context.timezoneId = b.timezone
54
+ if (b.user_agent) context.userAgent = b.user_agent
55
+ const file = path.join(dir, "config.json")
56
+ writeFileSync(
57
+ file,
58
+ `${JSON.stringify({ browser: { browserName: "chromium", isolated: true, launchOptions: launch, contextOptions: context }, outputDir: dir }, null, 2)}\n`,
59
+ )
60
+ return {
61
+ dir,
62
+ // --snapshot-mode none: clicks, typing, and navigation do not return a full page snapshot each. Every tool result
63
+ // stays in the coding agent's conversation and is sent again with each later call, so on a heavy page those
64
+ // snapshots made up most of a job's tokens. The agent takes a snapshot (or finds one element) when it needs one.
65
+ config: { type: "local", command: ["npx", "-y", PLAYWRIGHT_MCP, "--config", file, "--snapshot-mode", "none"], enabled: true, timeout: TIMEOUT_MS },
66
+ }
67
+ }