@capagents/forgeqe 0.0.0-stage → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HOW_TO_USE.md +1639 -0
- package/README.md +338 -2
- package/bin/forgeqe.d.ts +3 -0
- package/bin/forgeqe.js +188 -0
- package/docs/ARCHITECTURE.md +408 -0
- package/docs/security-data-flow.html +1870 -0
- package/docs/user-guide.html +2094 -0
- package/evals/domain/access-role-change.yaml +37 -0
- package/evals/domain/banking-payment-transfer.yaml +41 -0
- package/evals/domain/gdpr-account-erasure.yaml +30 -0
- package/evals/domain/insurance-claim-fnol.yaml +30 -0
- package/evals/domain/insurance-policy-renewal.yaml +37 -0
- package/evals/domain/lending-loan-application.yaml +36 -0
- package/forgeqe-architecture.html +718 -0
- package/forgeqe.example.yaml +203 -0
- package/knowledge/README.md +18 -0
- package/knowledge/business-rules.md +10 -0
- package/knowledge/compliance.md +16 -0
- package/knowledge/environments.md +5 -0
- package/knowledge/glossary.md +5 -0
- package/knowledge/product.md +16 -0
- package/knowledge/roles.md +9 -0
- package/knowledge/test-data.md +7 -0
- package/llms.example.json +51 -0
- package/package.json +63 -4
- package/prompts/README.md +34 -0
- package/prompts/api-scenarios.md +21 -0
- package/prompts/api-tests.md +39 -0
- package/prompts/brain.md +166 -0
- package/prompts/browser-capture.md +7 -0
- package/prompts/browser-case-report.md +7 -0
- package/prompts/browser-checks.md +14 -0
- package/prompts/browser-efficiency.md +7 -0
- package/prompts/browser-error-recovery.md +15 -0
- package/prompts/browser-extra.md +5 -0
- package/prompts/browser-goal-verification.md +13 -0
- package/prompts/browser-skill-select.md +6 -0
- package/prompts/domain-rule.md +13 -0
- package/prompts/eval-judge.md +19 -0
- package/prompts/explore-charters.md +27 -0
- package/prompts/find-tests.md +28 -0
- package/prompts/heal.md +40 -0
- package/prompts/pr-review.md +29 -0
- package/prompts/skill-rule.md +6 -0
- package/prompts/test-agent.md +34 -0
- package/prompts/test-reviewer.md +30 -0
- package/prompts/test-rules.md +28 -0
- package/py/forgeqe_engine/__init__.py +3 -0
- package/py/forgeqe_engine/__main__.py +3 -0
- package/py/forgeqe_engine/brand.py +453 -0
- package/py/forgeqe_engine/brand_assets.py +6 -0
- package/py/forgeqe_engine/browser.py +196 -0
- package/py/forgeqe_engine/capture.py +538 -0
- package/py/forgeqe_engine/checks.py +98 -0
- package/py/forgeqe_engine/cli.py +188 -0
- package/py/forgeqe_engine/diagnostics.py +609 -0
- package/py/forgeqe_engine/llm.py +195 -0
- package/py/forgeqe_engine/logins.py +251 -0
- package/py/forgeqe_engine/loops.py +224 -0
- package/py/forgeqe_engine/memory.py +1004 -0
- package/py/forgeqe_engine/network.py +244 -0
- package/py/forgeqe_engine/prompts.py +20 -0
- package/py/forgeqe_engine/replay.py +240 -0
- package/py/forgeqe_engine/report.py +173 -0
- package/py/forgeqe_engine/service.py +1335 -0
- package/py/forgeqe_engine/settings.py +93 -0
- package/py/forgeqe_engine/skills.py +88 -0
- package/py/forgeqe_engine/summary.py +239 -0
- package/py/forgeqe_engine/urls.py +81 -0
- package/py/forgeqe_engine/workflow_yaml.py +689 -0
- package/skills/dotnet-nunit-playwright/SKILL.md +34 -0
- package/skills/dynamics-365/SKILL.md +21 -0
- package/skills/playwright-typescript/SKILL.md +35 -0
- package/skills/writing-skills/SKILL.md +30 -0
- package/src/api/plan.ts +246 -0
- package/src/api/run.ts +204 -0
- package/src/api/scenario.ts +502 -0
- package/src/api/schema.ts +170 -0
- package/src/api/spec.ts +624 -0
- package/src/api/store.ts +232 -0
- package/src/api/traffic.ts +281 -0
- package/src/api/types.ts +163 -0
- package/src/api/write.ts +272 -0
- package/src/auth/signin.ts +494 -0
- package/src/auth/store.ts +170 -0
- package/src/brain/index.ts +189 -0
- package/src/brain/prompt.ts +19 -0
- package/src/capabilities/docs.ts +101 -0
- package/src/capabilities/guide.ts +455 -0
- package/src/capabilities/registry.ts +343 -0
- package/src/capabilities/render.ts +106 -0
- package/src/capabilities/status.ts +137 -0
- package/src/card.ts +11 -0
- package/src/cases/ado.ts +325 -0
- package/src/cases/api.ts +117 -0
- package/src/cases/bugs.ts +223 -0
- package/src/cases/confluence.ts +111 -0
- package/src/cases/context.ts +223 -0
- package/src/cases/dashboards.ts +137 -0
- package/src/cases/failure.ts +97 -0
- package/src/cases/files.ts +295 -0
- package/src/cases/goal.ts +53 -0
- package/src/cases/http.ts +111 -0
- package/src/cases/index.ts +131 -0
- package/src/cases/jira.ts +122 -0
- package/src/cases/qtest.ts +108 -0
- package/src/cases/quality.ts +190 -0
- package/src/cases/rag.ts +100 -0
- package/src/cases/readiness.ts +284 -0
- package/src/cases/save.ts +149 -0
- package/src/cases/sprints.ts +148 -0
- package/src/cases/testrail.ts +176 -0
- package/src/cases/types.ts +107 -0
- package/src/cases/wiki.ts +105 -0
- package/src/cases/xray.ts +206 -0
- package/src/cases/zephyr.ts +153 -0
- package/src/childEnv.ts +86 -0
- package/src/cli/args.ts +114 -0
- package/src/cli/help.ts +167 -0
- package/src/cli/index.ts +1142 -0
- package/src/cli/terminal.ts +187 -0
- package/src/code/areas.ts +337 -0
- package/src/code/attention.ts +214 -0
- package/src/code/find.ts +540 -0
- package/src/code/handoff.ts +541 -0
- package/src/code/history.ts +170 -0
- package/src/code/impact.ts +231 -0
- package/src/code/index.ts +34 -0
- package/src/code/livebrowser.ts +67 -0
- package/src/code/loop.ts +603 -0
- package/src/code/notes.ts +75 -0
- package/src/code/profile.ts +226 -0
- package/src/code/prompt.ts +303 -0
- package/src/code/prreview.ts +649 -0
- package/src/code/prtests.ts +103 -0
- package/src/code/replay.ts +772 -0
- package/src/code/review.ts +202 -0
- package/src/code/runtime.ts +781 -0
- package/src/code/slices.ts +159 -0
- package/src/code/testrun.ts +187 -0
- package/src/code/traceability.ts +100 -0
- package/src/code/worktree.ts +153 -0
- package/src/commands.ts +101 -0
- package/src/config.ts +970 -0
- package/src/detail.ts +65 -0
- package/src/domain.ts +19 -0
- package/src/engine/bridge.ts +359 -0
- package/src/engine/mock.ts +102 -0
- package/src/engine/setup.ts +298 -0
- package/src/entra.ts +154 -0
- package/src/env.ts +24 -0
- package/src/evals/domain.ts +338 -0
- package/src/explore/charters.ts +151 -0
- package/src/explore/run.ts +280 -0
- package/src/explore/story.ts +106 -0
- package/src/guard.ts +51 -0
- package/src/heal/loop.ts +357 -0
- package/src/heal/prompt.ts +143 -0
- package/src/heal/report.ts +95 -0
- package/src/index.ts +12 -0
- package/src/init.ts +102 -0
- package/src/knowledge.ts +114 -0
- package/src/legacyEnv.ts +9 -0
- package/src/llm.ts +61 -0
- package/src/locale.ts +25 -0
- package/src/logins.ts +148 -0
- package/src/managed.ts +101 -0
- package/src/mcp/apitools.ts +335 -0
- package/src/mcp/captools.ts +25 -0
- package/src/mcp/casetools.ts +223 -0
- package/src/mcp/commands.ts +421 -0
- package/src/mcp/exploretools.ts +129 -0
- package/src/mcp/external.ts +172 -0
- package/src/mcp/git.ts +78 -0
- package/src/mcp/notifytools.ts +49 -0
- package/src/mcp/prtools.ts +261 -0
- package/src/mcp/qualitytools.ts +154 -0
- package/src/mcp/repotests.ts +286 -0
- package/src/mcp/server.ts +234 -0
- package/src/mcp/tools.ts +810 -0
- package/src/memory.ts +250 -0
- package/src/money.ts +39 -0
- package/src/monitor/alerts.ts +94 -0
- package/src/monitor/background.ts +68 -0
- package/src/monitor/run.ts +263 -0
- package/src/monitor/state.ts +105 -0
- package/src/notify/events.ts +144 -0
- package/src/notify/notice.ts +177 -0
- package/src/prices.ts +175 -0
- package/src/print.ts +54 -0
- package/src/privacy.ts +35 -0
- package/src/prompts.ts +165 -0
- package/src/quality/budgets.ts +249 -0
- package/src/quality/history.ts +149 -0
- package/src/quality/visual.ts +201 -0
- package/src/report/data.ts +475 -0
- package/src/report/html.ts +106 -0
- package/src/report/template.html +84 -0
- package/src/run.ts +243 -0
- package/src/sessionLog.ts +216 -0
- package/src/sessionLogins.ts +94 -0
- package/src/skills/registry.ts +217 -0
- package/src/skills/scaffold.ts +65 -0
- package/src/skills/select.ts +37 -0
- package/src/skills/view.ts +44 -0
- package/src/suite/report.ts +135 -0
- package/src/suite/run.ts +303 -0
- package/src/summary.ts +188 -0
- package/src/tools.ts +130 -0
- package/src/tui/App.tsx +983 -0
- package/src/tui/clipboard.ts +36 -0
- package/src/tui/components.tsx +263 -0
- package/src/tui/dialogs.ts +65 -0
- package/src/tui/main.tsx +20 -0
- package/src/tui/text.ts +28 -0
- package/src/tui/theme.ts +104 -0
- package/src/tui/transcript.tsx +344 -0
- package/src/types.ts +146 -0
- package/src/update.ts +311 -0
- package/src/usage.ts +158 -0
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
import { mkdirSync, readFileSync, writeFileSync } from "node:fs"
|
|
2
|
+
import path from "node:path"
|
|
3
|
+
|
|
4
|
+
import { projectDir } from "../tools.ts"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Each automated test's recent results in a repository, to tell a flaky test from a broken one, and the tests in
|
|
8
|
+
* quarantine. Kept in .forgeqe/test-history.json in the repository (commit it to share the history; CI can keep it
|
|
9
|
+
* between runs as a cache). Quarantine changes no test code: a quarantined test still runs, so its history goes on,
|
|
10
|
+
* but its failure does not fail the run.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
export const HISTORY_FILE = path.join(".forgeqe", "test-history.json")
|
|
14
|
+
const KEEP = 30
|
|
15
|
+
const WINDOW = 20
|
|
16
|
+
/** Consecutive clean passes after which a quarantined test is offered for release. */
|
|
17
|
+
export const RELEASE_AFTER = 5
|
|
18
|
+
|
|
19
|
+
export type Result = { at: string; status: "passed" | "failed" | "skipped"; /** Failed, then passed when run again. */ retried?: boolean; message?: string; seconds?: number }
|
|
20
|
+
export type Quarantine = { at: string; reason: string; by: "auto" | "user" }
|
|
21
|
+
export type TestHistory = { runs: Result[]; quarantined?: Quarantine }
|
|
22
|
+
type HistoryFile = { version: 1; tests: Record<string, TestHistory> }
|
|
23
|
+
|
|
24
|
+
export type Cause = "timing" | "overlay" | "data" | "network" | "environment" | "unknown"
|
|
25
|
+
|
|
26
|
+
export const CAUSE_TEXT: Record<Cause, string> = {
|
|
27
|
+
timing: "timing: waits that are too short or race the page (timeouts, elements not ready, detached)",
|
|
28
|
+
overlay: "something covers the element now and then (a banner, a dialog, a spinner intercepts the click)",
|
|
29
|
+
data: "data that changes between runs (other values, other counts, data left by another test)",
|
|
30
|
+
network: "network or backend errors now and then (failed requests, 5xx, resets)",
|
|
31
|
+
environment: "the environment (the browser crashed or closed, the server was not ready)",
|
|
32
|
+
unknown: "no pattern in the failure messages",
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export type Flakiness = {
|
|
36
|
+
name: string
|
|
37
|
+
runs: number
|
|
38
|
+
passRate: number
|
|
39
|
+
/** Runs where it failed, then passed when run again, in the last WINDOW runs. */
|
|
40
|
+
retryPasses: number
|
|
41
|
+
/** Pass/fail changes in the last WINDOW runs. */
|
|
42
|
+
flips: number
|
|
43
|
+
score: number
|
|
44
|
+
flaky: boolean
|
|
45
|
+
cause: Cause
|
|
46
|
+
example?: string
|
|
47
|
+
/** The last results, oldest first: ✓ passed, ✗ failed, ↻ passed on retry, – skipped. */
|
|
48
|
+
trail: string
|
|
49
|
+
quarantined?: Quarantine
|
|
50
|
+
/** Quarantined and clean for RELEASE_AFTER runs in a row. */
|
|
51
|
+
releasable: boolean
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const file = (repo: string) => path.join(projectDir(repo), path.basename(HISTORY_FILE))
|
|
55
|
+
|
|
56
|
+
export function readHistory(repo: string): HistoryFile {
|
|
57
|
+
try {
|
|
58
|
+
const data = JSON.parse(readFileSync(file(repo), "utf8")) as HistoryFile
|
|
59
|
+
return { version: 1, tests: data?.tests && typeof data.tests === "object" ? data.tests : {} }
|
|
60
|
+
} catch {
|
|
61
|
+
return { version: 1, tests: {} }
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function save(repo: string, h: HistoryFile) {
|
|
66
|
+
mkdirSync(path.dirname(file(repo)), { recursive: true })
|
|
67
|
+
writeFileSync(file(repo), `${JSON.stringify(h, null, 1)}\n`)
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Add one run's results. */
|
|
71
|
+
export function recordResults(repo: string, results: { name: string; status: Result["status"]; retried?: boolean; message?: string; seconds?: number }[], at = new Date().toISOString()): void {
|
|
72
|
+
if (!results.length) return
|
|
73
|
+
const h = readHistory(repo)
|
|
74
|
+
for (const r of results) {
|
|
75
|
+
const t = h.tests[r.name] || { runs: [] }
|
|
76
|
+
t.runs.push({ at, status: r.status, ...(r.retried ? { retried: true } : {}), ...(r.message ? { message: r.message.slice(0, 300) } : {}), ...(r.seconds !== undefined ? { seconds: r.seconds } : {}) })
|
|
77
|
+
t.runs = t.runs.slice(-KEEP)
|
|
78
|
+
h.tests[r.name] = t
|
|
79
|
+
}
|
|
80
|
+
save(repo, h)
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** The likely cause behind a test's failure messages. */
|
|
84
|
+
export function likelyCause(messages: string[]): { cause: Cause; example?: string } {
|
|
85
|
+
const rules: [Cause, RegExp][] = [
|
|
86
|
+
["overlay", /intercepts pointer events|element is not visible|is covered|obscured|not clickable at point|other element would receive the click|ElementClickInterceptedException/i],
|
|
87
|
+
["network", /net::ERR_|ECONNRESET|ECONNREFUSED|ETIMEDOUT|socket hang up|\b5\d\d\b.*(error|status|response)|request failed|fetch failed/i],
|
|
88
|
+
["environment", /browser has been closed|Target (page, context or browser )?closed|browser crashed|Protocol error|WebDriverException|session not created|server (is )?not (ready|running)/i],
|
|
89
|
+
["timing", /timeout|timed out|exceeded|waiting for|detached from (the )?DOM|stale element|not attached|toBeVisible|toHaveText.*\bTimeout\b/i],
|
|
90
|
+
["data", /expected[\s\S]{0,80}(received|but (was|got)|actual)|toEqual|toBe\(|AssertionError|assert .* ==|Expected: .*\n?\s*Received:/i],
|
|
91
|
+
]
|
|
92
|
+
const counts = new Map<Cause, { n: number; example: string }>()
|
|
93
|
+
for (const m of messages.filter(Boolean)) {
|
|
94
|
+
const hit = rules.find(([, r]) => r.test(m))
|
|
95
|
+
if (!hit) continue
|
|
96
|
+
const c = counts.get(hit[0]) || { n: 0, example: m }
|
|
97
|
+
c.n++
|
|
98
|
+
counts.set(hit[0], c)
|
|
99
|
+
}
|
|
100
|
+
const best = [...counts].sort((a, b) => b[1].n - a[1].n)[0]
|
|
101
|
+
return best ? { cause: best[0], example: best[1].example.slice(0, 200) } : { cause: "unknown" }
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** How flaky a test is from its history: passes on retry and flip-flops, but a test that only fails is broken, not flaky. */
|
|
105
|
+
export function flakiness(name: string, t: TestHistory, flakyAfter = 2): Flakiness {
|
|
106
|
+
const ran = t.runs.filter((r) => r.status !== "skipped")
|
|
107
|
+
const recent = ran.slice(-WINDOW)
|
|
108
|
+
let flips = 0
|
|
109
|
+
for (let i = 1; i < recent.length; i++) if (recent[i].status !== recent[i - 1].status) flips++
|
|
110
|
+
const retryPasses = recent.filter((r) => r.retried).length
|
|
111
|
+
const passes = recent.filter((r) => r.status === "passed").length
|
|
112
|
+
const failures = recent.filter((r) => r.status === "failed").length + retryPasses
|
|
113
|
+
const score = retryPasses + Math.floor(flips / 2)
|
|
114
|
+
const tail = ran.slice(-RELEASE_AFTER)
|
|
115
|
+
const { cause, example } = likelyCause(t.runs.slice(-WINDOW).filter((r) => r.message).map((r) => r.message!))
|
|
116
|
+
return {
|
|
117
|
+
name,
|
|
118
|
+
runs: ran.length,
|
|
119
|
+
passRate: recent.length ? Math.round((passes / recent.length) * 100) / 100 : 0,
|
|
120
|
+
retryPasses,
|
|
121
|
+
flips,
|
|
122
|
+
score,
|
|
123
|
+
flaky: score >= flakyAfter && passes > 0 && failures > 0,
|
|
124
|
+
cause,
|
|
125
|
+
...(example ? { example } : {}),
|
|
126
|
+
trail: t.runs.slice(-WINDOW).map((r) => (r.status === "skipped" ? "–" : r.retried ? "↻" : r.status === "passed" ? "✓" : "✗")).join(""),
|
|
127
|
+
...(t.quarantined ? { quarantined: t.quarantined } : {}),
|
|
128
|
+
releasable: Boolean(t.quarantined) && tail.length === RELEASE_AFTER && tail.every((r) => r.status === "passed" && !r.retried),
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Every test's flakiness, flakiest first. */
|
|
133
|
+
export function flakyReport(repo: string, flakyAfter = 2): Flakiness[] {
|
|
134
|
+
return Object.entries(readHistory(repo).tests)
|
|
135
|
+
.map(([name, t]) => flakiness(name, t, flakyAfter))
|
|
136
|
+
.sort((a, b) => Number(Boolean(b.quarantined)) - Number(Boolean(a.quarantined)) || b.score - a.score || a.passRate - b.passRate)
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export function quarantined(repo: string): Map<string, Quarantine> {
|
|
140
|
+
return new Map(Object.entries(readHistory(repo).tests).flatMap(([n, t]) => (t.quarantined ? [[n, t.quarantined] as [string, Quarantine]] : [])))
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Put tests in quarantine, or take them out (reason null). Returns the names changed. */
|
|
144
|
+
export function setQuarantine(repo: string, names: string[], reason: string | null, by: Quarantine["by"] = "user"): string[] {
|
|
145
|
+
const h = readHistory(repo)
|
|
146
|
+
const changed: string[] = []
|
|
147
|
+
for (const n of names) {
|
|
148
|
+
const t = h.tests[n] || { runs: [] }
|
|
149
|
+
if (reason === null) {
|
|
150
|
+
if (!t.quarantined) continue
|
|
151
|
+
delete t.quarantined
|
|
152
|
+
} else {
|
|
153
|
+
if (t.quarantined) continue
|
|
154
|
+
t.quarantined = { at: new Date().toISOString(), reason, by }
|
|
155
|
+
}
|
|
156
|
+
h.tests[n] = t
|
|
157
|
+
changed.push(n)
|
|
158
|
+
}
|
|
159
|
+
if (changed.length) save(repo, h)
|
|
160
|
+
return changed
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** A test name ForgeQE knows, from a name the user gave: exact, else the one that contains it. */
|
|
164
|
+
export function matchTest(repo: string, given: string): string | null {
|
|
165
|
+
const names = Object.keys(readHistory(repo).tests)
|
|
166
|
+
if (names.includes(given)) return given
|
|
167
|
+
const low = given.toLowerCase()
|
|
168
|
+
const hits = names.filter((n) => n.toLowerCase().includes(low))
|
|
169
|
+
return hits.length === 1 ? hits[0] : null
|
|
170
|
+
}
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process"
|
|
2
|
+
import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs"
|
|
3
|
+
import path from "node:path"
|
|
4
|
+
|
|
5
|
+
import type { AppConfig } from "../config.ts"
|
|
6
|
+
import { adoSettings } from "../cases/ado.ts"
|
|
7
|
+
import { http } from "../cases/http.ts"
|
|
8
|
+
import { runDirectory } from "../run.ts"
|
|
9
|
+
import type { TestSelection } from "./find.ts"
|
|
10
|
+
import { readMappings } from "./traceability.ts"
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Which of a repository's tests a change affects, without a model: the test files it changes, the tests that import
|
|
14
|
+
* a changed file (following imports through page objects and helpers), the tests that name a changed class or page
|
|
15
|
+
* object, and the tests mapped (.forgeqe/test-cases.json) to the test cases of the work items the change is linked
|
|
16
|
+
* to. Each test comes with its reasons, so a person, or the find-tests agent, can see why it was picked.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
export type ImpactedTest = { file: string; reasons: string[] }
|
|
20
|
+
export type Impact = { changed: string[]; tests: ImpactedTest[]; untested: string[] }
|
|
21
|
+
|
|
22
|
+
// By name, as the frameworks find them: a helper in a tests/ folder is not a test, and imports go on through it.
|
|
23
|
+
const TEST_FILE = /\.(spec|test|e2e|cy)\.[cm]?[jt]sx?$|(^|\/)test_[^/]*\.py$|_test\.py$|Tests?\.(cs|java|kt)$|Spec\.(cs|java|kt)$/i
|
|
24
|
+
const CODE = /\.([cm]?[jt]sx?|py|cs|java|kt)$/i
|
|
25
|
+
const SKIP = /(^|\/)(node_modules|dist|build|out|bin|obj|\.forgeqe|\.venv|venv|__pycache__|coverage|test-results|playwright-report|needs-attention)\//
|
|
26
|
+
const MAX_FILES = 6000
|
|
27
|
+
const MAX_READ = 300_000
|
|
28
|
+
|
|
29
|
+
const posix = (p: string) => p.split(path.sep).join("/")
|
|
30
|
+
const git = (repo: string, args: string[]) => spawnSync("git", ["-c", "core.quotepath=false", ...args], { cwd: repo, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 })
|
|
31
|
+
|
|
32
|
+
export const isTestFile = (rel: string) => TEST_FILE.test(rel) && CODE.test(rel)
|
|
33
|
+
|
|
34
|
+
/** Files changed between the merge base of `base` and `head`, existing in the working tree or not. */
|
|
35
|
+
export function changedBetween(repo: string, base: string, head = "HEAD"): string[] {
|
|
36
|
+
const mb = git(repo, ["merge-base", base, head])
|
|
37
|
+
if (mb.status !== 0) throw new Error(`No common history between ${base} and ${head}: ${mb.stderr.trim().slice(0, 300)}`)
|
|
38
|
+
const out = git(repo, ["diff", "--name-only", "-M", mb.stdout.trim(), head])
|
|
39
|
+
if (out.status !== 0) throw new Error(`git diff failed: ${out.stderr.trim().slice(0, 300)}`)
|
|
40
|
+
return out.stdout.split("\n").map((l) => l.trim()).filter(Boolean)
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function trackedCode(repo: string): string[] {
|
|
44
|
+
const out = git(repo, ["ls-files", "--cached", "--others", "--exclude-standard"])
|
|
45
|
+
return out.stdout
|
|
46
|
+
.split("\n")
|
|
47
|
+
.map((l) => l.trim())
|
|
48
|
+
.filter((f) => f && CODE.test(f) && !SKIP.test(f))
|
|
49
|
+
.slice(0, MAX_FILES)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function read(repo: string, rel: string): string {
|
|
53
|
+
try {
|
|
54
|
+
const file = path.join(repo, rel)
|
|
55
|
+
return statSync(file).size <= MAX_READ ? readFileSync(file, "utf8") : ""
|
|
56
|
+
} catch {
|
|
57
|
+
return ""
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** The files a JS/TS or Python file imports, as repository paths (only those that exist). */
|
|
62
|
+
export function importsOf(rel: string, text: string, files: Set<string>): string[] {
|
|
63
|
+
const dir = path.posix.dirname(rel)
|
|
64
|
+
const out: string[] = []
|
|
65
|
+
if (/\.[cm]?[jt]sx?$/i.test(rel)) {
|
|
66
|
+
for (const m of text.matchAll(/(?:\bfrom\s*|\bimport\s*\(\s*|\brequire\s*\(\s*|^\s*import\s+)['"](\.{1,2}\/[^'"]+)['"]/gm)) {
|
|
67
|
+
const base = path.posix.normalize(path.posix.join(dir, m[1]))
|
|
68
|
+
const hit = [base, ...[".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"].map((e) => base.replace(/\.(js|mjs|cjs)$/, "") + e), ...["ts", "tsx", "js"].map((e) => `${base}/index.${e}`)].find((c) => files.has(c))
|
|
69
|
+
if (hit) out.push(hit)
|
|
70
|
+
}
|
|
71
|
+
} else if (/\.py$/i.test(rel)) {
|
|
72
|
+
for (const m of text.matchAll(/^\s*(?:from\s+([\w.]+)\s+import|import\s+([\w.]+))/gm)) {
|
|
73
|
+
const mod = (m[1] || m[2]).replace(/^\.+/, "")
|
|
74
|
+
if (!mod) continue
|
|
75
|
+
const tail = mod.split(".").join("/")
|
|
76
|
+
const hit = [...files].find((f) => f.endsWith(`${tail}.py`) || f.endsWith(`${tail}/__init__.py`))
|
|
77
|
+
if (hit) out.push(hit)
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return out
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** A changed file's names a test can mention: its class or module name (LoginPage from pages/LoginPage.ts). */
|
|
84
|
+
function namesOf(rel: string): string[] {
|
|
85
|
+
const stem = path.posix.basename(rel).replace(/\.(spec|test)?\.?[cm]?[jt]sx?$|\.(py|cs|java|kt)$/i, "")
|
|
86
|
+
return stem.length >= 5 && /[A-Z_]/.test(stem) ? [stem] : []
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** The tests a change affects, each with why. `mapped` adds tests found from linked test cases. */
|
|
90
|
+
export function analyzeImpact(repo: string, changed: string[], mapped: ImpactedTest[] = []): Impact {
|
|
91
|
+
const files = trackedCode(repo)
|
|
92
|
+
const set = new Set(files)
|
|
93
|
+
const tests = new Map<string, Set<string>>()
|
|
94
|
+
const add = (file: string, why: string) => {
|
|
95
|
+
if (!existsSync(path.join(repo, file))) return
|
|
96
|
+
if (!tests.has(file)) tests.set(file, new Set())
|
|
97
|
+
tests.get(file)!.add(why)
|
|
98
|
+
}
|
|
99
|
+
for (const f of changed) if (isTestFile(f)) add(f, "changed in this pull request")
|
|
100
|
+
|
|
101
|
+
// Who imports whom, reversed, so a changed module leads to every test that reaches it.
|
|
102
|
+
const importers = new Map<string, string[]>()
|
|
103
|
+
const texts = new Map<string, string>()
|
|
104
|
+
for (const f of files) {
|
|
105
|
+
const text = read(repo, f)
|
|
106
|
+
texts.set(f, text)
|
|
107
|
+
for (const dep of importsOf(f, text, set)) importers.set(dep, [...(importers.get(dep) || []), f])
|
|
108
|
+
}
|
|
109
|
+
const source = changed.filter((f) => CODE.test(f) && !isTestFile(f))
|
|
110
|
+
const reached = new Set<string>()
|
|
111
|
+
for (const f of source) {
|
|
112
|
+
const queue = [{ file: f, depth: 0 }]
|
|
113
|
+
const seen = new Set([f])
|
|
114
|
+
while (queue.length) {
|
|
115
|
+
const { file, depth } = queue.shift()!
|
|
116
|
+
for (const imp of importers.get(file) || []) {
|
|
117
|
+
if (seen.has(imp) || depth >= 6) continue
|
|
118
|
+
seen.add(imp)
|
|
119
|
+
if (isTestFile(imp)) {
|
|
120
|
+
add(imp, `imports ${f}${depth ? " (through other modules)" : ""}`)
|
|
121
|
+
reached.add(f)
|
|
122
|
+
} else queue.push({ file: imp, depth: depth + 1 })
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
// C#, Java, and anything imports cannot show: tests that name a changed class or page object.
|
|
127
|
+
const testFiles = files.filter(isTestFile)
|
|
128
|
+
for (const f of source) {
|
|
129
|
+
for (const name of namesOf(f)) {
|
|
130
|
+
const word = new RegExp(`\\b${name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`)
|
|
131
|
+
for (const t of testFiles) {
|
|
132
|
+
if (word.test(texts.get(t) || "")) {
|
|
133
|
+
add(t, `uses ${name} (${f})`)
|
|
134
|
+
reached.add(f)
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
for (const m of mapped) for (const why of m.reasons) add(m.file, why)
|
|
140
|
+
return {
|
|
141
|
+
changed,
|
|
142
|
+
tests: [...tests].map(([file, reasons]) => ({ file, reasons: [...reasons] })).sort((a, b) => a.file.localeCompare(b.file)),
|
|
143
|
+
untested: source.filter((f) => !reached.has(f)),
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Tests mapped to the test cases that test the given Azure DevOps work items (a story or bug linked to the pull
|
|
149
|
+
* request, "Tested By" its test cases): from .forgeqe/test-cases.json.
|
|
150
|
+
*/
|
|
151
|
+
export async function testsForWorkItems(cfg: AppConfig, repo: string, ids: number[]): Promise<ImpactedTest[]> {
|
|
152
|
+
if (!ids.length || !cfg.sources.ado) return []
|
|
153
|
+
const map = readMappings(repo).cases
|
|
154
|
+
const { org, auth } = adoSettings(cfg.sources)
|
|
155
|
+
const out: ImpactedTest[] = []
|
|
156
|
+
for (const id of ids.slice(0, 20)) {
|
|
157
|
+
try {
|
|
158
|
+
const { data } = await http<{ relations?: { rel: string; url: string }[] }>(`${org}/_apis/wit/workitems/${id}?$expand=relations&api-version=7.1`, { auth })
|
|
159
|
+
for (const r of data?.relations || []) {
|
|
160
|
+
if (r.rel !== "Microsoft.VSTS.Common.TestedBy-Forward") continue
|
|
161
|
+
const caseId = `ADO-${r.url.split("/").pop()}`
|
|
162
|
+
const m = map[caseId]
|
|
163
|
+
if (m?.testFile && m.status !== "needs_attention") out.push({ file: m.testFile, reasons: [`automates ${caseId}, which tests work item ${id} linked to the pull request`] })
|
|
164
|
+
}
|
|
165
|
+
} catch {
|
|
166
|
+
// A work item the token cannot read: the other signals still count.
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return out
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** The nearest folder above `file` (inside the repo) that holds one of `names`. */
|
|
173
|
+
function nearest(repo: string, file: string, names: string[]): string | null {
|
|
174
|
+
let dir = path.dirname(path.join(repo, file))
|
|
175
|
+
while (!path.relative(repo, dir).startsWith("..")) {
|
|
176
|
+
if (names.some((n) => existsSync(path.join(dir, n)))) return dir
|
|
177
|
+
const up = path.dirname(dir)
|
|
178
|
+
if (up === dir) break
|
|
179
|
+
dir = up
|
|
180
|
+
}
|
|
181
|
+
return null
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
const quote = (s: string) => (/^[\w./@:-]+$/.test(s) ? s : `'${s.replace(/'/g, `'\\''`)}'`)
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* A run command for the impacted test files, without a model, writing JUnit XML to `junit`: Playwright and pytest.
|
|
188
|
+
* Null for other frameworks (the find-tests agent works those out).
|
|
189
|
+
*/
|
|
190
|
+
export function impactCommand(repo: string, files: string[], junit: string): { command: string; framework: string } | null {
|
|
191
|
+
if (!files.length) return null
|
|
192
|
+
const js = files.filter((f) => /\.[cm]?[jt]sx?$/i.test(f))
|
|
193
|
+
const py = files.filter((f) => /\.py$/i.test(f))
|
|
194
|
+
const env = (name: string, value: string) => (process.platform === "win32" ? `set ${name}=${value}&& ` : `${name}=${quote(value)} `)
|
|
195
|
+
if (js.length === files.length) {
|
|
196
|
+
const dir = nearest(repo, js[0], ["playwright.config.ts", "playwright.config.js", "playwright.config.mjs"])
|
|
197
|
+
if (!dir || js.some((f) => nearest(repo, f, ["playwright.config.ts", "playwright.config.js", "playwright.config.mjs"]) !== dir)) return null
|
|
198
|
+
const rel = js.map((f) => quote(posix(path.relative(dir, path.join(repo, f)))))
|
|
199
|
+
const cd = posix(path.relative(repo, dir))
|
|
200
|
+
return { framework: "Playwright", command: `${cd ? `cd ${quote(cd)} && ` : ""}${env("PLAYWRIGHT_JUNIT_OUTPUT_NAME", junit)}npx playwright test ${rel.join(" ")} --reporter=line,junit` }
|
|
201
|
+
}
|
|
202
|
+
if (py.length === files.length) return { framework: "pytest", command: `python -m pytest ${py.map(quote).join(" ")} --junitxml=${quote(junit)}` }
|
|
203
|
+
return null
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/** A test selection run_tests can run (after the user confirms), from the impact analysis alone. */
|
|
207
|
+
export function impactSelection(cfg: AppConfig, repo: string, request: string, impact: Impact): TestSelection | null {
|
|
208
|
+
const dir = runDirectory(path.join(cfg.export.outDir, "tests"))
|
|
209
|
+
const junit = path.join(dir, "results.xml")
|
|
210
|
+
const cmd = impactCommand(repo, impact.tests.map((t) => t.file), junit)
|
|
211
|
+
if (!cmd) return null
|
|
212
|
+
const sel: TestSelection = {
|
|
213
|
+
id: path.basename(dir),
|
|
214
|
+
dir,
|
|
215
|
+
repo,
|
|
216
|
+
request,
|
|
217
|
+
framework: cmd.framework,
|
|
218
|
+
tests: impact.tests.map((t) => ({ name: t.file, file: t.file, reason: t.reasons.join("; ") })),
|
|
219
|
+
related: [],
|
|
220
|
+
listCommand: "",
|
|
221
|
+
listed: false,
|
|
222
|
+
runCommand: cmd.command,
|
|
223
|
+
results: [junit],
|
|
224
|
+
notes: impact.untested.length ? `No test reaches: ${impact.untested.join(", ")}.` : "",
|
|
225
|
+
model: "",
|
|
226
|
+
usage: { input: 0, output: 0 },
|
|
227
|
+
createdAt: new Date().toISOString(),
|
|
228
|
+
}
|
|
229
|
+
writeFileSync(path.join(dir, "selection.json"), JSON.stringify(sel, null, 2))
|
|
230
|
+
return sel
|
|
231
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { existsSync, readdirSync, statSync } from "node:fs"
|
|
2
|
+
import path from "node:path"
|
|
3
|
+
|
|
4
|
+
import type { RunResult } from "../types.ts"
|
|
5
|
+
import type { AppConfig } from "../config.ts"
|
|
6
|
+
import { FAILURE_ACTION, FAILURE_LABEL, type Failure } from "../cases/failure.ts"
|
|
7
|
+
|
|
8
|
+
export { runCodeAgent } from "./loop.ts"
|
|
9
|
+
|
|
10
|
+
/** Newest run directory under the export folder that has a run.json. */
|
|
11
|
+
export function latestRunDir(outRoot: string): string | undefined {
|
|
12
|
+
if (!existsSync(outRoot)) return undefined
|
|
13
|
+
return readdirSync(outRoot)
|
|
14
|
+
.map((name) => path.join(outRoot, name))
|
|
15
|
+
.filter((dir) => existsSync(path.join(dir, "run.json")))
|
|
16
|
+
.sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs)[0]
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Why the coding step should not follow this run, or null when it should. A run that found a functional defect goes
|
|
21
|
+
* to the coding agent too: the test asserts the correct behaviour and is marked as an expected failure.
|
|
22
|
+
*/
|
|
23
|
+
export function codeSkipReason(cfg: AppConfig, result: Pick<RunResult, "status" | "driver" | "outcome" | "failedChecks">, failure?: Failure): string | null {
|
|
24
|
+
if (failure && failure.type !== "functional_defect") return `${FAILURE_LABEL[failure.type]}: ${FAILURE_ACTION[failure.type]}`
|
|
25
|
+
if (!failure && result.outcome === "assertion_failed") {
|
|
26
|
+
const what = (result.failedChecks || []).map((c) => `expected ${c.expected}, the page shows ${c.actual}`).join("; ")
|
|
27
|
+
return `the test case failed: an expected result does not hold (${what || "see the run"}); a test is written from a passing run`
|
|
28
|
+
}
|
|
29
|
+
if (!failure && result.status !== "complete") return "the browser run did not pass"
|
|
30
|
+
if (result.driver === "synthetic") return "mock runs have no real page to test"
|
|
31
|
+
if (cfg.llm.provider === "mock" && !cfg.code.model) return "the mock profile has no model; pick a live profile or set code.model"
|
|
32
|
+
if (cfg.llm.provider === "none" && !cfg.code.model) return "no model is set up; add a profile to llms.json or set code.model"
|
|
33
|
+
return null
|
|
34
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"
|
|
2
|
+
import path from "node:path"
|
|
3
|
+
|
|
4
|
+
/** The agent runtime names this server's tools `live_<tool>`, e.g. `live_browser_snapshot`. */
|
|
5
|
+
export const LIVE_BROWSER_SERVER = "live"
|
|
6
|
+
export const PLAYWRIGHT_MCP = "@playwright/mcp@0.0.83"
|
|
7
|
+
/** Also the time allowed for the first start, when npx downloads the server. */
|
|
8
|
+
const TIMEOUT_MS = 120_000
|
|
9
|
+
|
|
10
|
+
type BrowserProfile = {
|
|
11
|
+
channel?: string
|
|
12
|
+
executable?: string
|
|
13
|
+
headless?: boolean
|
|
14
|
+
user_agent?: string
|
|
15
|
+
language?: string
|
|
16
|
+
timezone?: string
|
|
17
|
+
viewport?: number[]
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export type LiveBrowser = {
|
|
21
|
+
/** MCP server config for the agent runtime. */
|
|
22
|
+
config: Record<string, unknown>
|
|
23
|
+
/** Where the server saves screenshots and snapshots; the coding agent may read it. */
|
|
24
|
+
dir: string
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function runBrowser(runDir: string): BrowserProfile {
|
|
28
|
+
try {
|
|
29
|
+
return (JSON.parse(readFileSync(path.join(runDir, "run.json"), "utf8")).browser || {}) as BrowserProfile
|
|
30
|
+
} catch {
|
|
31
|
+
return {}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Microsoft's Playwright MCP server on a browser set up like the one the run used (channel, headless, user agent,
|
|
37
|
+
* language, time zone, viewport), so the coding agent can open pages, read their elements, try locators, and act in
|
|
38
|
+
* seconds, through the same library the repository's tests use, instead of building and running a test to look.
|
|
39
|
+
* Null without Node.js (npx).
|
|
40
|
+
*/
|
|
41
|
+
export function liveBrowser(runDir: string, which: (bin: string) => string | null = Bun.which): LiveBrowser | null {
|
|
42
|
+
if (!which("npx")) return null
|
|
43
|
+
const b = runBrowser(runDir)
|
|
44
|
+
const dir = path.join(runDir, "live-browser")
|
|
45
|
+
mkdirSync(dir, { recursive: true })
|
|
46
|
+
const launch: Record<string, unknown> = { headless: Boolean(b.headless) }
|
|
47
|
+
if (b.channel) launch.channel = b.channel
|
|
48
|
+
else if (b.executable && existsSync(b.executable)) launch.executablePath = b.executable
|
|
49
|
+
else launch.channel = "chrome"
|
|
50
|
+
const context: Record<string, unknown> = {}
|
|
51
|
+
if (b.viewport?.length === 2) context.viewport = { width: b.viewport[0], height: b.viewport[1] }
|
|
52
|
+
if (b.language) context.locale = b.language
|
|
53
|
+
if (b.timezone) context.timezoneId = b.timezone
|
|
54
|
+
if (b.user_agent) context.userAgent = b.user_agent
|
|
55
|
+
const file = path.join(dir, "config.json")
|
|
56
|
+
writeFileSync(
|
|
57
|
+
file,
|
|
58
|
+
`${JSON.stringify({ browser: { browserName: "chromium", isolated: true, launchOptions: launch, contextOptions: context }, outputDir: dir }, null, 2)}\n`,
|
|
59
|
+
)
|
|
60
|
+
return {
|
|
61
|
+
dir,
|
|
62
|
+
// --snapshot-mode none: clicks, typing, and navigation do not return a full page snapshot each. Every tool result
|
|
63
|
+
// stays in the coding agent's conversation and is sent again with each later call, so on a heavy page those
|
|
64
|
+
// snapshots made up most of a job's tokens. The agent takes a snapshot (or finds one element) when it needs one.
|
|
65
|
+
config: { type: "local", command: ["npx", "-y", PLAYWRIGHT_MCP, "--config", file, "--snapshot-mode", "none"], enabled: true, timeout: TIMEOUT_MS },
|
|
66
|
+
}
|
|
67
|
+
}
|