@capagents/forgeqe 0.0.0-stage → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. package/HOW_TO_USE.md +1639 -0
  2. package/README.md +338 -2
  3. package/bin/forgeqe.d.ts +3 -0
  4. package/bin/forgeqe.js +188 -0
  5. package/docs/ARCHITECTURE.md +408 -0
  6. package/docs/security-data-flow.html +1870 -0
  7. package/docs/user-guide.html +2094 -0
  8. package/evals/domain/access-role-change.yaml +37 -0
  9. package/evals/domain/banking-payment-transfer.yaml +41 -0
  10. package/evals/domain/gdpr-account-erasure.yaml +30 -0
  11. package/evals/domain/insurance-claim-fnol.yaml +30 -0
  12. package/evals/domain/insurance-policy-renewal.yaml +37 -0
  13. package/evals/domain/lending-loan-application.yaml +36 -0
  14. package/forgeqe-architecture.html +718 -0
  15. package/forgeqe.example.yaml +203 -0
  16. package/knowledge/README.md +18 -0
  17. package/knowledge/business-rules.md +10 -0
  18. package/knowledge/compliance.md +16 -0
  19. package/knowledge/environments.md +5 -0
  20. package/knowledge/glossary.md +5 -0
  21. package/knowledge/product.md +16 -0
  22. package/knowledge/roles.md +9 -0
  23. package/knowledge/test-data.md +7 -0
  24. package/llms.example.json +51 -0
  25. package/package.json +63 -4
  26. package/prompts/README.md +34 -0
  27. package/prompts/api-scenarios.md +21 -0
  28. package/prompts/api-tests.md +39 -0
  29. package/prompts/brain.md +166 -0
  30. package/prompts/browser-capture.md +7 -0
  31. package/prompts/browser-case-report.md +7 -0
  32. package/prompts/browser-checks.md +14 -0
  33. package/prompts/browser-efficiency.md +7 -0
  34. package/prompts/browser-error-recovery.md +15 -0
  35. package/prompts/browser-extra.md +5 -0
  36. package/prompts/browser-goal-verification.md +13 -0
  37. package/prompts/browser-skill-select.md +6 -0
  38. package/prompts/domain-rule.md +13 -0
  39. package/prompts/eval-judge.md +19 -0
  40. package/prompts/explore-charters.md +27 -0
  41. package/prompts/find-tests.md +28 -0
  42. package/prompts/heal.md +40 -0
  43. package/prompts/pr-review.md +29 -0
  44. package/prompts/skill-rule.md +6 -0
  45. package/prompts/test-agent.md +34 -0
  46. package/prompts/test-reviewer.md +30 -0
  47. package/prompts/test-rules.md +28 -0
  48. package/py/forgeqe_engine/__init__.py +3 -0
  49. package/py/forgeqe_engine/__main__.py +3 -0
  50. package/py/forgeqe_engine/brand.py +453 -0
  51. package/py/forgeqe_engine/brand_assets.py +6 -0
  52. package/py/forgeqe_engine/browser.py +196 -0
  53. package/py/forgeqe_engine/capture.py +538 -0
  54. package/py/forgeqe_engine/checks.py +98 -0
  55. package/py/forgeqe_engine/cli.py +188 -0
  56. package/py/forgeqe_engine/diagnostics.py +609 -0
  57. package/py/forgeqe_engine/llm.py +195 -0
  58. package/py/forgeqe_engine/logins.py +251 -0
  59. package/py/forgeqe_engine/loops.py +224 -0
  60. package/py/forgeqe_engine/memory.py +1004 -0
  61. package/py/forgeqe_engine/network.py +244 -0
  62. package/py/forgeqe_engine/prompts.py +20 -0
  63. package/py/forgeqe_engine/replay.py +240 -0
  64. package/py/forgeqe_engine/report.py +173 -0
  65. package/py/forgeqe_engine/service.py +1335 -0
  66. package/py/forgeqe_engine/settings.py +93 -0
  67. package/py/forgeqe_engine/skills.py +88 -0
  68. package/py/forgeqe_engine/summary.py +239 -0
  69. package/py/forgeqe_engine/urls.py +81 -0
  70. package/py/forgeqe_engine/workflow_yaml.py +689 -0
  71. package/skills/dotnet-nunit-playwright/SKILL.md +34 -0
  72. package/skills/dynamics-365/SKILL.md +21 -0
  73. package/skills/playwright-typescript/SKILL.md +35 -0
  74. package/skills/writing-skills/SKILL.md +30 -0
  75. package/src/api/plan.ts +246 -0
  76. package/src/api/run.ts +204 -0
  77. package/src/api/scenario.ts +502 -0
  78. package/src/api/schema.ts +170 -0
  79. package/src/api/spec.ts +624 -0
  80. package/src/api/store.ts +232 -0
  81. package/src/api/traffic.ts +281 -0
  82. package/src/api/types.ts +163 -0
  83. package/src/api/write.ts +272 -0
  84. package/src/auth/signin.ts +494 -0
  85. package/src/auth/store.ts +170 -0
  86. package/src/brain/index.ts +189 -0
  87. package/src/brain/prompt.ts +19 -0
  88. package/src/capabilities/docs.ts +101 -0
  89. package/src/capabilities/guide.ts +455 -0
  90. package/src/capabilities/registry.ts +343 -0
  91. package/src/capabilities/render.ts +106 -0
  92. package/src/capabilities/status.ts +137 -0
  93. package/src/card.ts +11 -0
  94. package/src/cases/ado.ts +325 -0
  95. package/src/cases/api.ts +117 -0
  96. package/src/cases/bugs.ts +223 -0
  97. package/src/cases/confluence.ts +111 -0
  98. package/src/cases/context.ts +223 -0
  99. package/src/cases/dashboards.ts +137 -0
  100. package/src/cases/failure.ts +97 -0
  101. package/src/cases/files.ts +295 -0
  102. package/src/cases/goal.ts +53 -0
  103. package/src/cases/http.ts +111 -0
  104. package/src/cases/index.ts +131 -0
  105. package/src/cases/jira.ts +122 -0
  106. package/src/cases/qtest.ts +108 -0
  107. package/src/cases/quality.ts +190 -0
  108. package/src/cases/rag.ts +100 -0
  109. package/src/cases/readiness.ts +284 -0
  110. package/src/cases/save.ts +149 -0
  111. package/src/cases/sprints.ts +148 -0
  112. package/src/cases/testrail.ts +176 -0
  113. package/src/cases/types.ts +107 -0
  114. package/src/cases/wiki.ts +105 -0
  115. package/src/cases/xray.ts +206 -0
  116. package/src/cases/zephyr.ts +153 -0
  117. package/src/childEnv.ts +86 -0
  118. package/src/cli/args.ts +114 -0
  119. package/src/cli/help.ts +167 -0
  120. package/src/cli/index.ts +1142 -0
  121. package/src/cli/terminal.ts +187 -0
  122. package/src/code/areas.ts +337 -0
  123. package/src/code/attention.ts +214 -0
  124. package/src/code/find.ts +540 -0
  125. package/src/code/handoff.ts +541 -0
  126. package/src/code/history.ts +170 -0
  127. package/src/code/impact.ts +231 -0
  128. package/src/code/index.ts +34 -0
  129. package/src/code/livebrowser.ts +67 -0
  130. package/src/code/loop.ts +603 -0
  131. package/src/code/notes.ts +75 -0
  132. package/src/code/profile.ts +226 -0
  133. package/src/code/prompt.ts +303 -0
  134. package/src/code/prreview.ts +649 -0
  135. package/src/code/prtests.ts +103 -0
  136. package/src/code/replay.ts +772 -0
  137. package/src/code/review.ts +202 -0
  138. package/src/code/runtime.ts +757 -0
  139. package/src/code/slices.ts +159 -0
  140. package/src/code/testrun.ts +187 -0
  141. package/src/code/traceability.ts +100 -0
  142. package/src/code/worktree.ts +153 -0
  143. package/src/commands.ts +101 -0
  144. package/src/config.ts +970 -0
  145. package/src/detail.ts +65 -0
  146. package/src/domain.ts +19 -0
  147. package/src/engine/bridge.ts +359 -0
  148. package/src/engine/mock.ts +102 -0
  149. package/src/engine/setup.ts +298 -0
  150. package/src/entra.ts +154 -0
  151. package/src/env.ts +24 -0
  152. package/src/evals/domain.ts +338 -0
  153. package/src/explore/charters.ts +151 -0
  154. package/src/explore/run.ts +280 -0
  155. package/src/explore/story.ts +106 -0
  156. package/src/guard.ts +51 -0
  157. package/src/heal/loop.ts +357 -0
  158. package/src/heal/prompt.ts +143 -0
  159. package/src/heal/report.ts +95 -0
  160. package/src/index.ts +12 -0
  161. package/src/init.ts +102 -0
  162. package/src/knowledge.ts +114 -0
  163. package/src/legacyEnv.ts +9 -0
  164. package/src/llm.ts +61 -0
  165. package/src/locale.ts +25 -0
  166. package/src/logins.ts +148 -0
  167. package/src/managed.ts +101 -0
  168. package/src/mcp/apitools.ts +335 -0
  169. package/src/mcp/captools.ts +25 -0
  170. package/src/mcp/casetools.ts +223 -0
  171. package/src/mcp/commands.ts +421 -0
  172. package/src/mcp/exploretools.ts +129 -0
  173. package/src/mcp/external.ts +172 -0
  174. package/src/mcp/git.ts +78 -0
  175. package/src/mcp/notifytools.ts +49 -0
  176. package/src/mcp/prtools.ts +261 -0
  177. package/src/mcp/qualitytools.ts +154 -0
  178. package/src/mcp/repotests.ts +286 -0
  179. package/src/mcp/server.ts +234 -0
  180. package/src/mcp/tools.ts +810 -0
  181. package/src/memory.ts +250 -0
  182. package/src/money.ts +39 -0
  183. package/src/monitor/alerts.ts +94 -0
  184. package/src/monitor/background.ts +68 -0
  185. package/src/monitor/run.ts +263 -0
  186. package/src/monitor/state.ts +105 -0
  187. package/src/notify/events.ts +144 -0
  188. package/src/notify/notice.ts +177 -0
  189. package/src/prices.ts +175 -0
  190. package/src/print.ts +54 -0
  191. package/src/privacy.ts +35 -0
  192. package/src/prompts.ts +165 -0
  193. package/src/quality/budgets.ts +249 -0
  194. package/src/quality/history.ts +149 -0
  195. package/src/quality/visual.ts +201 -0
  196. package/src/report/data.ts +475 -0
  197. package/src/report/html.ts +106 -0
  198. package/src/report/template.html +84 -0
  199. package/src/run.ts +243 -0
  200. package/src/sessionLog.ts +216 -0
  201. package/src/sessionLogins.ts +94 -0
  202. package/src/skills/registry.ts +217 -0
  203. package/src/skills/scaffold.ts +65 -0
  204. package/src/skills/select.ts +37 -0
  205. package/src/skills/view.ts +44 -0
  206. package/src/suite/report.ts +135 -0
  207. package/src/suite/run.ts +303 -0
  208. package/src/summary.ts +188 -0
  209. package/src/tools.ts +130 -0
  210. package/src/tui/App.tsx +983 -0
  211. package/src/tui/clipboard.ts +36 -0
  212. package/src/tui/components.tsx +263 -0
  213. package/src/tui/dialogs.ts +65 -0
  214. package/src/tui/main.tsx +20 -0
  215. package/src/tui/text.ts +28 -0
  216. package/src/tui/theme.ts +104 -0
  217. package/src/tui/transcript.tsx +344 -0
  218. package/src/types.ts +136 -0
  219. package/src/update.ts +311 -0
  220. package/src/usage.ts +158 -0
@@ -0,0 +1,176 @@
1
+ import { existsSync, readFileSync, statSync } from "node:fs"
2
+ import path from "node:path"
3
+
4
+ import type { SourcesConfig } from "../config.ts"
5
+ import { basic, envSecret, http } from "./http.ts"
6
+ import { resultText } from "./jira.ts"
7
+ import type { CaseResult, CaseStep, TestCase, WriteBack } from "./types.ts"
8
+
9
+ type TrCase = {
10
+ id: number
11
+ title: string
12
+ suite_id?: number
13
+ section_id?: number
14
+ refs?: string | null
15
+ custom_preconds?: string | null
16
+ custom_steps?: string | null
17
+ custom_expected?: string | null
18
+ custom_steps_separated?: { content?: string; expected?: string; additional_info?: string }[] | null
19
+ }
20
+ type TrTest = Omit<TrCase, "id"> & { id: number; case_id: number; run_id: number }
21
+
22
+ export function testrailApi(cfg: SourcesConfig): { base: string; api: string; auth: string; projectId?: number } {
23
+ const t = cfg.testrail
24
+ if (!t) throw new Error("TestRail is not set up. Add sources.testrail.url to forgeqe.yaml, or set TESTRAIL_URL.")
25
+ return { base: t.url, api: `${t.url}/index.php?/api/v2`, auth: basic(envSecret(t.emailEnv, "TestRail"), envSecret(t.tokenEnv, "TestRail")), projectId: t.projectId }
26
+ }
27
+
28
+ /** Every page of a TestRail list endpoint; handles both the paginated ({ key, _links.next }) and the old array shape. */
29
+ async function list<T>(base: string, first: string, key: string, auth: string): Promise<T[]> {
30
+ const out: T[] = []
31
+ let url: string | null = first
32
+ while (url) {
33
+ const { data }: { data: unknown } = await http(url, { auth })
34
+ if (Array.isArray(data)) return [...out, ...(data as T[])]
35
+ const page = data as Record<string, unknown> & { _links?: { next?: string | null } }
36
+ out.push(...((page[key] as T[]) || []))
37
+ const next = page._links?.next
38
+ url = next ? `${base}/index.php?/${next.replace(/^\/?(index\.php\?)?\/?/, "")}` : null
39
+ }
40
+ return out
41
+ }
42
+
43
+ /** Section names by id, for grouping cases in reports; a section that cannot be read is left out. */
44
+ async function sectionNames(cases: TrCase[], api: string, auth: string): Promise<Map<number, string>> {
45
+ const names = new Map<number, string>()
46
+ for (const id of new Set(cases.map((c) => c.section_id).filter((id): id is number => typeof id === "number"))) {
47
+ try {
48
+ const { data } = await http<{ name?: string }>(`${api}/get_section/${id}`, { auth })
49
+ if (data?.name) names.set(id, data.name)
50
+ } catch {}
51
+ }
52
+ return names
53
+ }
54
+
55
+ function toCase(c: TrCase, base: string, run?: number, testId?: number, sections?: Map<number, string>): TestCase {
56
+ const separated = (c.custom_steps_separated || []).filter((s) => s.content?.trim())
57
+ const steps: CaseStep[] = separated.length
58
+ ? separated.map((s) => ({ action: s.content!.trim(), expected: s.expected?.trim() || undefined, data: s.additional_info?.trim() || undefined }))
59
+ : c.custom_steps?.trim()
60
+ ? [{ action: c.custom_steps.trim(), expected: c.custom_expected?.trim() || undefined }]
61
+ : []
62
+ return {
63
+ id: `C${c.id}`,
64
+ title: c.title,
65
+ source: "testrail",
66
+ url: `${base}/index.php?/cases/view/${c.id}`,
67
+ preconditions: c.custom_preconds?.trim() ? [c.custom_preconds.trim()] : undefined,
68
+ steps,
69
+ tags: c.refs ? c.refs.split(",").map((r) => r.trim()).filter(Boolean) : undefined,
70
+ group: c.section_id !== undefined ? sections?.get(c.section_id) : undefined,
71
+ requirements: c.refs ? c.refs.split(",").map((r) => r.trim()).filter(Boolean) : undefined,
72
+ ref: { case: c.id, ...(c.suite_id ? { suite: c.suite_id } : {}), ...(run ? { run } : {}), ...(testId ? { test: testId } : {}), separated: separated.length },
73
+ }
74
+ }
75
+
76
+ /** `testrail:C1,C2`, `testrail:run=12` (results go to that run), `testrail:plan=5` (every run in the plan), `testrail:suite=3`. */
77
+ export async function testrailCases(ref: string, cfg: SourcesConfig): Promise<TestCase[]> {
78
+ const t = testrailApi(cfg)
79
+ const run = ref.match(/^run=R?(\d+)$/i)?.[1]
80
+ const plan = ref.match(/^plan=R?(\d+)$/i)?.[1]
81
+ const suite = ref.match(/^suite=S?(\d+)$/i)?.[1]
82
+ const runCases = async (runId: number) =>
83
+ (await list<TrTest>(t.base, `${t.api}/get_tests/${runId}`, "tests", t.auth)).map((test) => toCase({ ...test, id: test.case_id }, t.base, runId, test.id))
84
+ if (run) return nonEmpty(await runCases(Number(run)), `TestRail run ${run}`)
85
+ if (plan) {
86
+ const { data } = await http<{ entries?: { runs?: { id: number }[] }[] }>(`${t.api}/get_plan/${plan}`, { auth: t.auth })
87
+ const cases: TestCase[] = []
88
+ for (const e of data?.entries || []) for (const r of e.runs || []) cases.push(...(await runCases(r.id)))
89
+ return nonEmpty(cases, `TestRail plan ${plan}`)
90
+ }
91
+ if (suite) {
92
+ if (!t.projectId) throw new Error("TestRail suites need sources.testrail.project_id in forgeqe.yaml.")
93
+ const cases = await list<TrCase>(t.base, `${t.api}/get_cases/${t.projectId}&suite_id=${suite}`, "cases", t.auth)
94
+ const sections = await sectionNames(cases, t.api, t.auth)
95
+ return nonEmpty(cases.map((c) => toCase(c, t.base, undefined, undefined, sections)), `TestRail suite ${suite}`)
96
+ }
97
+ const ids = ref.split(/[,\s]+/).map((s) => Number(s.replace(/^C/i, ""))).filter((n) => Number.isInteger(n) && n > 0)
98
+ if (!ids.length) throw new Error(`TestRail reference "${ref}": use testrail:C1,C2, testrail:run=12, testrail:plan=5, or testrail:suite=3.`)
99
+ const found: TrCase[] = []
100
+ for (const id of ids) found.push((await http<TrCase>(`${t.api}/get_case/${id}`, { auth: t.auth })).data)
101
+ const sections = await sectionNames(found, t.api, t.auth)
102
+ return found.map((c) => toCase(c, t.base, undefined, undefined, sections))
103
+ }
104
+
105
+ function nonEmpty(cases: TestCase[], what: string): TestCase[] {
106
+ if (!cases.length) throw new Error(`No test cases in ${what}.`)
107
+ return cases
108
+ }
109
+
110
+ const STATUS = { passed: 1, failed: 5, error: 2 } as const
111
+ const STEP_STATUS = { passed: 1, failed: 5, not_run: 3 } as const
112
+
113
+ function resultFor(r: CaseResult, withSteps: boolean): Record<string, unknown> {
114
+ const verdicts = new Map((r.stepResults || []).map((v) => [v.step, v]))
115
+ const out: Record<string, unknown> = {
116
+ case_id: Number(r.case.ref?.case),
117
+ status_id: STATUS[r.status as keyof typeof STATUS] ?? 2,
118
+ comment: resultText(r),
119
+ elapsed: `${Math.max(1, Math.round(r.ms / 1000))}s`,
120
+ }
121
+ if (withSteps && verdicts.size && Number(r.case.ref?.separated)) {
122
+ out.custom_step_results = r.case.steps.map((s, i) => {
123
+ const v = verdicts.get(i + 1)
124
+ return { content: s.action, expected: s.expected || "", actual: v?.actual || "", status_id: v ? STEP_STATUS[v.status] : 3 }
125
+ })
126
+ }
127
+ return out
128
+ }
129
+
130
+ /** Results go to the run the cases came from; cases read by id get a new run per suite. Screenshots attach to failed results. */
131
+ export async function testrailWriteBack(results: CaseResult[], cfg: SourcesConfig, runName: string): Promise<WriteBack[]> {
132
+ const t = testrailApi(cfg)
133
+ const out: WriteBack[] = []
134
+ const groups = new Map<string, CaseResult[]>()
135
+ for (const r of results) {
136
+ const key = r.case.ref?.run ? `run:${r.case.ref.run}` : `suite:${r.case.ref?.suite || ""}`
137
+ groups.set(key, [...(groups.get(key) || []), r])
138
+ }
139
+ for (const [key, list] of groups) {
140
+ try {
141
+ let runId = Number(key.startsWith("run:") ? key.slice(4) : 0)
142
+ let created = false
143
+ if (!runId) {
144
+ if (!t.projectId) throw new Error("Recording results for cases read by id needs sources.testrail.project_id, so ForgeQE can create a run.")
145
+ const suite = key.slice(6)
146
+ const { data } = await http<{ id: number }>(`${t.api}/add_run/${t.projectId}`, {
147
+ auth: t.auth,
148
+ body: { name: runName, include_all: false, case_ids: [...new Set(list.map((r) => Number(r.case.ref?.case)))], ...(suite ? { suite_id: Number(suite) } : {}) },
149
+ })
150
+ runId = data.id
151
+ created = true
152
+ }
153
+ const post = (withSteps: boolean) =>
154
+ http<{ id: number; test_id?: number }[]>(`${t.api}/add_results_for_cases/${runId}`, { auth: t.auth, body: { results: list.map((r) => resultFor(r, withSteps)) } })
155
+ const { data: posted } = await post(true).catch(() => post(false))
156
+ let attached = 0
157
+ for (const [i, r] of list.entries()) {
158
+ const shot = r.status !== "passed" && r.runDir ? path.join(r.runDir, "failure.png") : ""
159
+ const resultId = posted?.[i]?.id
160
+ if (!shot || !resultId || !existsSync(shot) || statSync(shot).size > 8 * 1024 * 1024) continue
161
+ const form = new FormData()
162
+ form.append("attachment", new Blob([new Uint8Array(readFileSync(shot))], { type: "image/png" }), "failure.png")
163
+ const res = await fetch(`${t.api}/add_attachment_to_result/${resultId}`, { method: "POST", headers: { Authorization: t.auth }, body: form })
164
+ if (res.ok) attached++
165
+ }
166
+ out.push({
167
+ source: "testrail",
168
+ text: `TestRail run R${runId}${created ? " (new)" : ""}: ${list.length} result${list.length === 1 ? "" : "s"}${attached ? `, ${attached} screenshot${attached === 1 ? "" : "s"}` : ""}`,
169
+ url: `${t.base}/index.php?/runs/view/${runId}`,
170
+ })
171
+ } catch (err) {
172
+ out.push({ source: "testrail", text: `TestRail ${key.replace(":", " ")}: results not recorded`, error: (err as Error).message })
173
+ }
174
+ }
175
+ return out
176
+ }
@@ -0,0 +1,107 @@
1
+ import type { CodeStatus, StepVerdict, Usage } from "../types.ts"
2
+ import type { Cost } from "../usage.ts"
3
+ import type { Failure } from "./failure.ts"
4
+
5
+ export type CaseStep = { action: string; data?: string; expected?: string }
6
+
7
+ export type CaseSource = "text" | "file" | "ado" | "jira" | "xray" | "testrail" | "zephyr" | "qtest"
8
+
9
+ /** A test case from any source, in one shape. It becomes one browser goal and, once it passes, one test. */
10
+ export type TestCase = {
11
+ /** Stable id shown everywhere and used to name the test, e.g. "ADO-1234", "SHOP-12", "login.yaml#2". */
12
+ id: string
13
+ title: string
14
+ source: CaseSource
15
+ /** Link to the item in its system. */
16
+ url?: string
17
+ description?: string
18
+ preconditions?: string[]
19
+ steps: CaseStep[]
20
+ /** Page to open first; otherwise browser.start_url, otherwise the agent chooses. */
21
+ startUrl?: string
22
+ tags?: string[]
23
+ /** Where the case lives in its system: area path, section, folder, or component. */
24
+ group?: string
25
+ /** Stories or requirements the case covers, e.g. TestRail refs or Jira issues it tests. */
26
+ requirements?: string[]
27
+ /** What the source needs to post results back (plan and point ids, issue ids, …). */
28
+ ref?: Record<string, string | number>
29
+ }
30
+
31
+ export type CaseStatus = "passed" | "failed" | "stopped" | "error"
32
+
33
+ export type CaseResult = {
34
+ case: TestCase
35
+ status: CaseStatus
36
+ /** The browser agent's account of the run, or why it could not run. */
37
+ summary: string
38
+ startedAt: string
39
+ finishedAt: string
40
+ ms: number
41
+ tokens: number
42
+ /** The browser agent's tokens, and what they cost at the profile's price. */
43
+ usage?: Usage
44
+ cost?: Cost
45
+ /** Scribe's tokens when it wrote a test for this case, and what they cost. */
46
+ codeUsage?: Usage
47
+ codeCost?: Cost
48
+ runDir?: string
49
+ finalUrl?: string
50
+ /** One line per browser step, for reports. */
51
+ steps?: { text: string; ok: boolean }[]
52
+ /** The browser agent's verdict on each of the case's numbered steps. */
53
+ stepResults?: StepVerdict[]
54
+ /** Set when the coding agent ran for this case. */
55
+ code?: {
56
+ status: CodeStatus
57
+ testFile?: string
58
+ command?: string
59
+ attempts: number
60
+ skills?: string[]
61
+ handoff?: string
62
+ error?: string
63
+ /** The test's full name as its framework reports it. */
64
+ testName?: string
65
+ /** The test asserts the correct behaviour of a known defect, marked as an expected failure. */
66
+ knownDefect?: boolean
67
+ /** The needs-attention folder the unfinished test was moved to. */
68
+ attention?: string
69
+ /** The test the case already had: it passed as it is (no coding job), or the job updated it. */
70
+ existing?: "reused" | "updated"
71
+ }
72
+ /** Why no test was written for this case, when a repo was set and the case did not go to the coding agent. */
73
+ codeSkipped?: string
74
+ /** For a case that did not pass: what kind of failure it was, and so what happened next (see src/cases/failure.ts). */
75
+ failure?: Failure
76
+ /** The bug filed (or found open) for a functional defect, when bugs are set up. */
77
+ bug?: { label: string; url?: string }
78
+ /** The browser run was repeated once because the first one met an environment problem. */
79
+ retried?: boolean
80
+ }
81
+
82
+ /** Where a result was recorded, for the suite summary, e.g. "Azure DevOps run 812: https://…". */
83
+ export type WriteBack = {
84
+ source: CaseSource | "bugs"
85
+ text: string
86
+ url?: string
87
+ error?: string
88
+ /** Set when the record is about one case (a comment, a bug), so the report can show it with that case. */
89
+ caseId?: string
90
+ }
91
+
92
+ const STEP_CAUSE: Record<NonNullable<StepVerdict["cause"]>, string> = { app: "application", test_case: "test case", environment: "environment", automation: "automation" }
93
+
94
+ /** The case's steps with the agent's verdicts, one line each, for comments and bug reports. */
95
+ export function stepVerdictLines(r: CaseResult): string[] {
96
+ const verdicts = new Map((r.stepResults || []).map((v) => [v.step, v]))
97
+ if (!verdicts.size) return []
98
+ const mark = { passed: "✓", failed: "✗", not_run: "–" } as const
99
+ return r.case.steps.map((s, i) => {
100
+ const v = verdicts.get(i + 1)
101
+ const actual = v?.actual?.trim() ? `: ${v.actual.trim()}` : ""
102
+ const cause = v?.status === "failed" && v.cause ? ` [${STEP_CAUSE[v.cause]}]` : ""
103
+ return `${v ? mark[v.status] : "?"} Step ${i + 1} ${v ? v.status.replace("_", " ") : "not reported"}${cause} — ${s.action.trim()}${v && v.status !== "passed" ? actual : ""}`
104
+ })
105
+ }
106
+
107
+ export { costOf, type Cost } from "../usage.ts"
@@ -0,0 +1,105 @@
1
+ import type { SourcesConfig } from "../config.ts"
2
+ import { adoSettings } from "./ado.ts"
3
+ import { http } from "./http.ts"
4
+
5
+ const API = "api-version=7.1"
6
+
7
+ export type WikiPageRequest = {
8
+ /** Project the wiki is in; sources.azure_devops.project when left out. */
9
+ project?: string
10
+ /** Wiki name or id; the project wiki when left out (created when the project has none). */
11
+ wiki?: string
12
+ /** Page path such as /Release notes/Sprint 12; missing parent pages are created empty. */
13
+ path: string
14
+ /** Markdown. */
15
+ content: string
16
+ }
17
+
18
+ export type WikiPageResult = {
19
+ project: string
20
+ wiki: string
21
+ wikiCreated: boolean
22
+ path: string
23
+ created: boolean
24
+ parentsCreated: string[]
25
+ url: string
26
+ }
27
+
28
+ type Wiki = { id: string; name: string; type: "projectWiki" | "codeWiki"; versions?: { version: string }[] }
29
+ type Page = { path?: string; remoteUrl?: string }
30
+
31
+ /**
32
+ * Creates or updates a wiki page. The project wiki is created first when the project has none, since the Azure DevOps
33
+ * MCP server can only write pages into a wiki that exists.
34
+ */
35
+ export async function writeAdoWikiPage(cfg: SourcesConfig, req: WikiPageRequest): Promise<WikiPageResult> {
36
+ const { auth, org, project: configured } = adoSettings(cfg)
37
+ const project = (req.project || configured || "").trim()
38
+ if (!project) throw new Error("Name the Azure DevOps project for the wiki page, or set sources.azure_devops.project.")
39
+ const path = pagePath(req.path)
40
+ if (path === "/") throw new Error("The wiki page needs a path such as /Release notes.")
41
+ const base = `${org}/${encodeURIComponent(project)}`
42
+
43
+ const { data } = await http<{ value: Wiki[] }>(`${base}/_apis/wiki/wikis?${API}`, { auth })
44
+ const wikis = data?.value || []
45
+ let wiki: Wiki | undefined
46
+ let wikiCreated = false
47
+ if (req.wiki?.trim()) {
48
+ const want = req.wiki.trim().toLowerCase()
49
+ wiki = wikis.find((w) => w.name.toLowerCase() === want || w.id.toLowerCase() === want)
50
+ if (!wiki) throw new Error(`No wiki "${req.wiki}" in ${project}. Its wikis: ${wikis.map((w) => w.name).join(", ") || "none"}.`)
51
+ } else {
52
+ wiki = wikis.find((w) => w.type === "projectWiki") || (wikis.length === 1 ? wikis[0] : undefined)
53
+ if (!wiki && wikis.length) throw new Error(`${project} has no project wiki but has code wikis (${wikis.map((w) => w.name).join(", ")}); name one as wiki.`)
54
+ if (!wiki) {
55
+ const { data: p } = await http<{ id: string }>(`${org}/_apis/projects/${encodeURIComponent(project)}?${API}`, { auth })
56
+ wiki = (await http<Wiki>(`${base}/_apis/wiki/wikis?${API}`, { auth, body: { name: `${project}.wiki`, projectId: p.id, type: "projectWiki" } })).data
57
+ wikiCreated = true
58
+ }
59
+ }
60
+
61
+ const branch = wiki.type === "codeWiki" && wiki.versions?.[0]?.version
62
+ const pages = `${base}/_apis/wiki/wikis/${encodeURIComponent(wiki.id)}/pages`
63
+ const at = (p: string) =>
64
+ `${pages}?path=${encodeURIComponent(p)}&${API}${branch ? `&versionDescriptor.version=${encodeURIComponent(branch)}&versionDescriptor.versionType=branch` : ""}`
65
+ const etag = async (p: string): Promise<string | null> => {
66
+ try {
67
+ const { headers } = await http(at(p), { auth })
68
+ return headers.get("etag") || ""
69
+ } catch (err) {
70
+ if (/failed: 404\b/.test((err as Error).message)) return null
71
+ throw err
72
+ }
73
+ }
74
+
75
+ const parentsCreated: string[] = []
76
+ const parts = path.split("/").filter(Boolean)
77
+ for (let i = 1; i < parts.length; i++) {
78
+ const parent = `/${parts.slice(0, i).join("/")}`
79
+ if ((await etag(parent)) !== null) continue
80
+ await http(at(parent), { auth, method: "PUT", body: { content: "" } })
81
+ parentsCreated.push(parent)
82
+ }
83
+
84
+ const existing = await etag(path)
85
+ const { data: page } = await http<Page>(at(path), {
86
+ auth,
87
+ method: "PUT",
88
+ body: { content: req.content },
89
+ ...(existing ? { headers: { "If-Match": existing } } : {}),
90
+ })
91
+ return {
92
+ project,
93
+ wiki: wiki.name,
94
+ wikiCreated,
95
+ path,
96
+ created: existing === null,
97
+ parentsCreated,
98
+ url: page?.remoteUrl || `${base}/_wiki/wikis/${encodeURIComponent(wiki.name)}?pagePath=${encodeURIComponent(path)}`,
99
+ }
100
+ }
101
+
102
+ function pagePath(raw: string): string {
103
+ const parts = String(raw || "").split("/").map((s) => s.trim()).filter(Boolean)
104
+ return `/${parts.join("/")}`
105
+ }
@@ -0,0 +1,206 @@
1
+ import { existsSync, readFileSync, statSync } from "node:fs"
2
+ import path from "node:path"
3
+
4
+ import type { SourcesConfig } from "../config.ts"
5
+ import { adfToText, envSecret, http } from "./http.ts"
6
+ import { jiraApi, jiraIssues, resultText } from "./jira.ts"
7
+ import type { CaseResult, CaseStep, TestCase, WriteBack } from "./types.ts"
8
+
9
+ const TEST_FIELDS = `issueId testType { name kind } steps { action data result } gherkin unstructured
10
+ preconditions(limit: 10) { results { definition jira(fields: ["key", "summary"]) } }
11
+ jira(fields: ["key", "summary", "description", "labels"])`
12
+
13
+ type CloudTest = {
14
+ issueId: string
15
+ testType?: { name?: string; kind?: string }
16
+ steps?: { action?: string; data?: string; result?: string }[]
17
+ gherkin?: string
18
+ unstructured?: string
19
+ preconditions?: { results?: { definition?: string; jira?: { key?: string; summary?: string } }[] }
20
+ jira: { key: string; summary?: string; description?: unknown; labels?: string[] }
21
+ }
22
+
23
+ const server = (cfg: SourcesConfig) => Boolean(cfg.jira && !cfg.jira.cloud)
24
+
25
+ export async function xrayCloudToken(cfg: SourcesConfig): Promise<string> {
26
+ const { data } = await http<string>(`${cfg.xray.url}/api/v2/authenticate`, {
27
+ auth: "",
28
+ body: { client_id: envSecret(cfg.xray.clientIdEnv, "Xray Cloud"), client_secret: envSecret(cfg.xray.clientSecretEnv, "Xray Cloud") },
29
+ })
30
+ return `Bearer ${String(data).replace(/^"|"$/g, "")}`
31
+ }
32
+
33
+ export async function xrayGraphql<T>(cfg: SourcesConfig, auth: string, query: string, variables: Record<string, unknown>): Promise<T> {
34
+ const { data } = await http<{ data?: T; errors?: { message: string }[] }>(`${cfg.xray.url}/api/v2/graphql`, { auth, body: { query, variables } })
35
+ if (data?.errors?.length) throw new Error(`Xray: ${data.errors.map((e) => e.message).join("; ")}`)
36
+ return data!.data as T
37
+ }
38
+
39
+ /** `xray:SHOP-1,SHOP-2`, `xray:jql=<query>`, or `xray:plan=SHOP-100` (the plan's tests; results link to the plan). */
40
+ export async function xrayCases(ref: string, cfg: SourcesConfig): Promise<TestCase[]> {
41
+ const plan = ref.match(/^plan=([A-Z][A-Z0-9_]*-\d+)$/i)?.[1]?.toUpperCase()
42
+ const jql = ref.match(/^jql=(.+)$/s)?.[1]
43
+ const keys = !plan && !jql ? ref.split(/[,\s]+/).map((k) => k.trim().toUpperCase()).filter(Boolean) : []
44
+ if (!plan && !jql && !keys.length) throw new Error(`Xray reference "${ref}": use xray:SHOP-1,SHOP-2, xray:jql=<query>, or xray:plan=SHOP-100.`)
45
+ const cases = server(cfg) ? await serverCases({ plan, jql, keys }, cfg) : await cloudCases({ plan, jql, keys }, cfg)
46
+ if (!cases.length) throw new Error(`No Xray tests for "${ref}".`)
47
+ return plan ? cases.map((c) => ({ ...c, ref: { ...c.ref, plan } })) : cases
48
+ }
49
+
50
+ async function cloudCases(sel: { plan?: string; jql?: string; keys: string[] }, cfg: SourcesConfig): Promise<TestCase[]> {
51
+ const auth = await xrayCloudToken(cfg)
52
+ const tests: CloudTest[] = []
53
+ for (let start = 0; ; start += 100) {
54
+ let page: { total: number; results: CloudTest[] }
55
+ if (sel.plan) {
56
+ const data = await xrayGraphql<{ getTestPlans: { results: { tests: { total: number; results: CloudTest[] } }[] } }>(
57
+ cfg,
58
+ auth,
59
+ `query($jql: String, $start: Int) { getTestPlans(jql: $jql, limit: 1) { results { tests(limit: 100, start: $start) { total results { ${TEST_FIELDS} } } } } }`,
60
+ { jql: `key = ${sel.plan}`, start },
61
+ )
62
+ const found = data.getTestPlans.results[0]
63
+ if (!found) throw new Error(`Xray test plan ${sel.plan} not found.`)
64
+ page = found.tests
65
+ } else {
66
+ const jql = sel.jql || `key in (${sel.keys.join(",")})`
67
+ const data = await xrayGraphql<{ getTests: { total: number; results: CloudTest[] } }>(
68
+ cfg,
69
+ auth,
70
+ `query($jql: String, $start: Int) { getTests(jql: $jql, limit: 100, start: $start) { total results { ${TEST_FIELDS} } } }`,
71
+ { jql, start },
72
+ )
73
+ page = data.getTests
74
+ }
75
+ tests.push(...page.results)
76
+ if (!page.results.length || tests.length >= page.total) break
77
+ }
78
+ const browse = cfg.jira ? `${cfg.jira.url}/browse/` : undefined
79
+ return tests.map((t) => {
80
+ const steps: CaseStep[] = (t.steps || [])
81
+ .filter((s) => s.action?.trim())
82
+ .map((s) => ({ action: s.action!.trim(), data: s.data?.trim() || undefined, expected: s.result?.trim() || undefined }))
83
+ const body = [adfToText(t.jira.description), t.gherkin?.trim() ? `Scenario:\n${t.gherkin.trim()}` : "", t.unstructured?.trim() || ""]
84
+ const pre = (t.preconditions?.results || []).map((p) => [p.jira?.summary, p.definition].filter(Boolean).join(": ")).filter(Boolean)
85
+ return {
86
+ id: t.jira.key,
87
+ title: t.jira.summary || t.jira.key,
88
+ source: "xray" as const,
89
+ url: browse ? browse + t.jira.key : undefined,
90
+ description: body.filter(Boolean).join("\n\n") || undefined,
91
+ preconditions: pre.length ? pre : undefined,
92
+ steps,
93
+ tags: t.jira.labels?.length ? t.jira.labels : undefined,
94
+ ref: { test: t.jira.key },
95
+ }
96
+ })
97
+ }
98
+
99
+ type ServerTest = {
100
+ key: string
101
+ type?: string
102
+ definition?: string
103
+ steps?: { step?: { raw?: string }; data?: { raw?: string }; result?: { raw?: string } }[]
104
+ }
105
+
106
+ async function serverCases(sel: { plan?: string; jql?: string; keys: string[] }, cfg: SourcesConfig): Promise<TestCase[]> {
107
+ const j = jiraApi(cfg, "Xray Server")
108
+ const q = sel.keys.length ? `keys=${sel.keys.join(";")}` : `jql=${encodeURIComponent(sel.plan ? `issue in testPlanTests("${sel.plan}")` : sel.jql!)}`
109
+ const { data } = await http<ServerTest[]>(`${j.url}/rest/raven/1.0/api/test?${q}`, { auth: j.auth })
110
+ const tests = data || []
111
+ if (!tests.length) return []
112
+ const issues = new Map((await jiraIssues(tests.map((t) => t.key).join(","), j)).map((i) => [i.key, i]))
113
+ return tests.map((t) => {
114
+ const issue = issues.get(t.key)
115
+ const steps: CaseStep[] = (t.steps || [])
116
+ .filter((s) => s.step?.raw?.trim())
117
+ .map((s) => ({ action: s.step!.raw!.trim(), data: s.data?.raw?.trim() || undefined, expected: s.result?.raw?.trim() || undefined }))
118
+ const body = [adfToText(issue?.fields.description), !steps.length && t.definition?.trim() ? t.definition.trim() : ""]
119
+ return {
120
+ id: t.key,
121
+ title: issue?.fields.summary || t.key,
122
+ source: "xray" as const,
123
+ url: `${j.url}/browse/${t.key}`,
124
+ description: body.filter(Boolean).join("\n\n") || undefined,
125
+ steps,
126
+ tags: issue?.fields.labels?.length ? issue.fields.labels : undefined,
127
+ ref: { test: t.key },
128
+ }
129
+ })
130
+ }
131
+
132
+ const MAX_EVIDENCE = 4 * 1024 * 1024
133
+
134
+ function evidence(r: CaseResult): { data: string; filename: string; contentType: string }[] {
135
+ if (!r.runDir) return []
136
+ const out = []
137
+ for (const [name, type] of [["failure.png", "image/png"]] as const) {
138
+ const file = path.join(r.runDir, name)
139
+ if (existsSync(file) && statSync(file).size <= MAX_EVIDENCE) {
140
+ out.push({ data: readFileSync(file).toString("base64"), filename: name, contentType: type })
141
+ }
142
+ }
143
+ return out
144
+ }
145
+
146
+ /** One Test Execution per test plan (and one for tests without a plan), with each test's status and comment. */
147
+ export async function xrayWriteBack(results: CaseResult[], cfg: SourcesConfig, runName: string): Promise<WriteBack[]> {
148
+ const isServer = server(cfg)
149
+ const status = (s: CaseResult["status"]) => (s === "passed" ? (isServer ? "PASS" : "PASSED") : s === "failed" ? (isServer ? "FAIL" : "FAILED") : "TODO")
150
+ const groups = new Map<string, CaseResult[]>()
151
+ for (const r of results) {
152
+ const plan = String(r.case.ref?.plan || "")
153
+ groups.set(plan, [...(groups.get(plan) || []), r])
154
+ }
155
+ const out: WriteBack[] = []
156
+ const auth = isServer ? jiraApi(cfg, "Xray Server").auth : await xrayCloudToken(cfg)
157
+ const url = isServer ? `${cfg.jira!.url}/rest/raven/1.0/import/execution` : `${cfg.xray.url}/api/v2/import/execution`
158
+ for (const [plan, list] of groups) {
159
+ const body = {
160
+ info: {
161
+ summary: runName,
162
+ description: `Run by ForgeQE: ${list.filter((r) => r.status === "passed").length}/${list.length} passed.`,
163
+ startDate: list.map((r) => r.startedAt).sort()[0],
164
+ finishDate: list.map((r) => r.finishedAt).sort().at(-1),
165
+ ...(plan ? { testPlanKey: plan } : {}),
166
+ },
167
+ tests: list.map((r) => {
168
+ const files = evidence(r)
169
+ const verdicts = new Map((r.stepResults || []).map((v) => [v.step, v]))
170
+ const stepStatus = { passed: status("passed"), failed: status("failed"), not_run: "TODO" } as const
171
+ const steps =
172
+ verdicts.size && r.case.steps.length
173
+ ? r.case.steps.map((_, i) => {
174
+ const v = verdicts.get(i + 1)
175
+ return { status: v ? stepStatus[v.status] : "TODO", ...(v?.actual ? { actualResult: v.actual.slice(0, 2000) } : {}) }
176
+ })
177
+ : undefined
178
+ return {
179
+ testKey: String(r.case.ref?.test || r.case.id),
180
+ start: r.startedAt,
181
+ finish: r.finishedAt,
182
+ status: status(r.status),
183
+ comment: resultText(r).slice(0, 4000),
184
+ ...(steps ? { steps } : {}),
185
+ ...(files.length ? { [isServer ? "evidences" : "evidence"]: files } : {}),
186
+ }
187
+ }),
188
+ }
189
+ try {
190
+ type Imported = { key?: string; self?: string; testExecIssue?: { key?: string } }
191
+ const withoutSteps = { ...body, tests: body.tests.map(({ steps: _steps, ...t }) => t) }
192
+ const { data } = await http<Imported>(url, { auth, body }).catch((err) =>
193
+ body.tests.some((t) => t.steps) ? http<Imported>(url, { auth, body: withoutSteps }) : Promise.reject(err),
194
+ )
195
+ const key = data?.key || data?.testExecIssue?.key
196
+ out.push({
197
+ source: "xray",
198
+ text: `Xray Test Execution ${key || "created"}${plan ? ` for ${plan}` : ""} (${list.length} tests)`,
199
+ url: key && cfg.jira ? `${cfg.jira.url}/browse/${key}` : undefined,
200
+ })
201
+ } catch (err) {
202
+ out.push({ source: "xray", text: `Xray results${plan ? ` for ${plan}` : ""} not imported`, error: (err as Error).message })
203
+ }
204
+ }
205
+ return out
206
+ }