@capagents/forgeqe 1.2.1 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HOW_TO_USE.md +40 -3
- package/README.md +1 -1
- package/docs/COST_ARCHITECTURE.md +280 -0
- package/docs/feature-test-prompts.md +95 -0
- package/docs/user-guide.html +54 -7
- package/evals/prompts/brain/route-api-scenarios.yaml +9 -0
- package/evals/prompts/brain/route-e2e.yaml +10 -0
- package/evals/prompts/brain/route-flaky.yaml +9 -0
- package/evals/prompts/brain/route-heal.yaml +9 -0
- package/evals/prompts/brain/route-regression-pack.yaml +9 -0
- package/evals/prompts/brain/route-review-update.yaml +10 -0
- package/evals/prompts/brain/route-run-cases.yaml +9 -0
- package/evals/prompts/brain/route-story-cases.yaml +12 -0
- package/forgeqe.example.yaml +15 -0
- package/package.json +1 -1
- package/prompts/README.md +30 -0
- package/prompts/brain.md +34 -49
- package/prompts/browser-checks.md +2 -0
- package/py/forgeqe_engine/checks.py +8 -2
- package/py/forgeqe_engine/llm.py +33 -5
- package/py/forgeqe_engine/memory.py +59 -2
- package/py/forgeqe_engine/observe.py +123 -0
- package/py/forgeqe_engine/replay.py +55 -2
- package/py/forgeqe_engine/service.py +155 -3
- package/src/agents.ts +52 -0
- package/src/brain/index.ts +8 -2
- package/src/capabilities/registry.ts +48 -6
- package/src/cases/ado.ts +1 -1
- package/src/cases/journey.ts +61 -0
- package/src/cases/readiness.ts +35 -6
- package/src/cases/save.ts +241 -48
- package/src/cases/types.ts +11 -0
- package/src/cli/help.ts +9 -0
- package/src/cli/index.ts +71 -1
- package/src/code/coverage.ts +22 -3
- package/src/code/decisions.ts +73 -0
- package/src/code/find.ts +3 -1
- package/src/code/graph/graph.ts +33 -2
- package/src/code/loop.ts +17 -1
- package/src/code/notes.ts +48 -1
- package/src/code/prompt.ts +31 -9
- package/src/code/protect.ts +103 -0
- package/src/code/review.ts +2 -0
- package/src/code/runtime.ts +35 -3
- package/src/code/shortlist.ts +309 -0
- package/src/code/testrun.ts +3 -1
- package/src/code/traceability.ts +2 -0
- package/src/commands.ts +6 -1
- package/src/config.ts +46 -0
- package/src/context.ts +75 -0
- package/src/engine/bridge.ts +47 -5
- package/src/mcp/captools.ts +19 -0
- package/src/mcp/casetools.ts +107 -11
- package/src/mcp/commands.ts +74 -20
- package/src/mcp/packtools.ts +187 -0
- package/src/mcp/routing.ts +145 -0
- package/src/mcp/server.ts +7 -3
- package/src/mcp/tools.ts +241 -29
- package/src/pack/collect.ts +251 -0
- package/src/pack/index.ts +134 -0
- package/src/pack/init.ts +45 -0
- package/src/pack/pack.ts +327 -0
- package/src/pack/pipeline.ts +373 -0
- package/src/pack/run.ts +155 -0
- package/src/registry/db.ts +249 -0
- package/src/registry/index.ts +72 -0
- package/src/registry/resolve.ts +57 -0
- package/src/registry/tier0.ts +81 -0
- package/src/report/data.ts +7 -0
- package/src/report/html.ts +14 -0
- package/src/report/template.html +52 -48
- package/src/report/usage.ts +113 -0
- package/src/run.ts +22 -2
- package/src/suite/run.ts +12 -5
- package/src/summary.ts +1 -0
- package/src/tokenomics/budget.ts +172 -0
- package/src/tokenomics/ledger.ts +287 -0
- package/src/tui/App.tsx +89 -2
- package/src/tui/inspector.tsx +40 -35
- package/src/types.ts +9 -2
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process"
|
|
2
|
+
import { realpathSync } from "node:fs"
|
|
3
|
+
import path from "node:path"
|
|
4
|
+
|
|
5
|
+
import type { AppConfig } from "../config.ts"
|
|
6
|
+
import { adoSettings } from "../cases/ado.ts"
|
|
7
|
+
import { htmlToText, http } from "../cases/http.ts"
|
|
8
|
+
import { resolveScope, type Scope, type WorkItem } from "../cases/readiness.ts"
|
|
9
|
+
import { changedBetween } from "../code/impact.ts"
|
|
10
|
+
import { adoBase, fetchPr, fetchPrBranches, gh, gitRemotes, parseRemote, resolvePr, type PrRef } from "../code/prreview.ts"
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* What goes into a release, from wherever it is recorded, as one list of change items: an Azure DevOps scope (a
|
|
14
|
+
* sprint, area, epic, or work items, with their pull requests and the files those changed), the commits between two
|
|
15
|
+
* refs of a repository, pull requests (named, or merged in the last days), and items ForgeQE's agent read from any
|
|
16
|
+
* other tool it is connected to (Jira, GitHub issues, a release note) and passes in as they are.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
export type ChangeItem = {
|
|
20
|
+
kind: "work_item" | "pr" | "commit" | "note"
|
|
21
|
+
/** azure_devops, github, git, or the tool the agent read it from. */
|
|
22
|
+
source: string
|
|
23
|
+
id: string
|
|
24
|
+
title: string
|
|
25
|
+
/** Description, acceptance criteria, repro steps, commit body: what changes, in words. */
|
|
26
|
+
text?: string
|
|
27
|
+
url?: string
|
|
28
|
+
/** Story, Bug, Feature… */
|
|
29
|
+
type?: string
|
|
30
|
+
/** Files the change touches, relative to the repository it is in. */
|
|
31
|
+
files?: string[]
|
|
32
|
+
/** The change is in the test repository itself: its files reach tests through the code. */
|
|
33
|
+
inTestRepo?: boolean
|
|
34
|
+
/** Work items or issues it refers to: "ado:123", "github:45", "jira:PAY-12". */
|
|
35
|
+
links?: string[]
|
|
36
|
+
/** Test cases that test it: "ADO-123". */
|
|
37
|
+
testCases?: string[]
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export type Collected = { label: string; items: ChangeItem[]; notes: string[] }
|
|
41
|
+
|
|
42
|
+
const MAX_TEXT = 2000
|
|
43
|
+
const MAX_PRS = 25
|
|
44
|
+
const MAX_COMMITS = 300
|
|
45
|
+
|
|
46
|
+
const clip = (s: string, n = MAX_TEXT) => (s.length > n ? `${s.slice(0, n)}…` : s)
|
|
47
|
+
const git = (repo: string, args: string[]) => spawnSync("git", ["-c", "core.quotepath=false", ...args], { cwd: repo, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 })
|
|
48
|
+
|
|
49
|
+
function real(p: string): string {
|
|
50
|
+
try {
|
|
51
|
+
return realpathSync(p)
|
|
52
|
+
} catch {
|
|
53
|
+
return path.resolve(p)
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Work items, issues, and keys a commit message or PR text mentions. */
|
|
58
|
+
export function linksIn(text: string, platform?: "github" | "azure_devops"): string[] {
|
|
59
|
+
const out = new Set<string>()
|
|
60
|
+
for (const m of text.matchAll(/\bAB#(\d+)\b/g)) out.add(`ado:${m[1]}`)
|
|
61
|
+
for (const m of text.matchAll(/(?<![\w#/-])#(\d+)\b/g)) out.add(platform === "azure_devops" ? `ado:${m[1]}` : `github:${m[1]}`)
|
|
62
|
+
for (const m of text.matchAll(/\b([A-Z][A-Z0-9]{1,9}-\d+)\b/g)) if (!/^(ADO|AB|SHOP|WEB)-/.test(m[1]!)) out.add(`jira:${m[1]}`)
|
|
63
|
+
return [...out]
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const PR_LINK = /PullRequestId\/([^/]+)%2F([^/]+)%2F(\d+)$/i
|
|
67
|
+
|
|
68
|
+
/** A work item as a change item: its words, its test cases, and the pull requests linked to it. */
|
|
69
|
+
function itemOf(w: WorkItem, url: string): ChangeItem & { prLinks: { project: string; repo: string; id: number }[] } {
|
|
70
|
+
const f = w.fields
|
|
71
|
+
const text = [htmlToText(String(f["System.Description"] || "")), htmlToText(String(f["Microsoft.VSTS.Common.AcceptanceCriteria"] || "")), htmlToText(String(f["Microsoft.VSTS.TCM.ReproSteps"] || ""))]
|
|
72
|
+
.map((t) => t.trim())
|
|
73
|
+
.filter(Boolean)
|
|
74
|
+
.join("\n")
|
|
75
|
+
const rels = w.relations || []
|
|
76
|
+
return {
|
|
77
|
+
kind: "work_item",
|
|
78
|
+
source: "azure_devops",
|
|
79
|
+
id: String(w.id),
|
|
80
|
+
title: String(f["System.Title"] || `Work item ${w.id}`),
|
|
81
|
+
...(text ? { text: clip(text) } : {}),
|
|
82
|
+
url,
|
|
83
|
+
type: String(f["System.WorkItemType"] || ""),
|
|
84
|
+
testCases: rels.filter((r) => r.rel === "Microsoft.VSTS.Common.TestedBy-Forward").map((r) => `ADO-${r.url.split("/").pop()}`),
|
|
85
|
+
prLinks: rels.flatMap((r) => {
|
|
86
|
+
const m = r.rel === "ArtifactLink" ? PR_LINK.exec(decodeURIComponent(r.url).replace(/\//g, "/")) || PR_LINK.exec(r.url) : null
|
|
87
|
+
return m ? [{ project: m[1]!, repo: m[2]!, id: Number(m[3]) }] : []
|
|
88
|
+
}),
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** The files an Azure DevOps pull request changed, from its last iteration. */
|
|
93
|
+
async function adoPrFiles(org: string, auth: string, project: string, repo: string, id: number): Promise<string[]> {
|
|
94
|
+
const base = `${org}/${encodeURIComponent(project)}/_apis/git/repositories/${encodeURIComponent(repo)}/pullRequests/${id}`
|
|
95
|
+
const { data: its } = await http<{ value?: { id: number }[] }>(`${base}/iterations?api-version=7.1`, { auth })
|
|
96
|
+
const last = its?.value?.at(-1)?.id
|
|
97
|
+
if (!last) return []
|
|
98
|
+
const { data } = await http<{ changeEntries?: { item?: { path?: string; isFolder?: boolean } }[] }>(`${base}/iterations/${last}/changes?$top=1000&api-version=7.1`, { auth })
|
|
99
|
+
return (data?.changeEntries || []).filter((c) => c.item?.path && !c.item.isFolder).map((c) => c.item!.path!.replace(/^\//, ""))
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* An Azure DevOps scope: its stories, and with a sprint or area its bugs too (fixed bugs are changes); each with its
|
|
104
|
+
* test cases, and the files its linked pull requests changed.
|
|
105
|
+
*/
|
|
106
|
+
export async function fromAdoScope(cfg: AppConfig, scope: Scope): Promise<Collected> {
|
|
107
|
+
const notes: string[] = []
|
|
108
|
+
const r = await resolveScope(cfg, scope)
|
|
109
|
+
const bugIds = r.filter
|
|
110
|
+
? await r.wiql(`SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @project AND [System.WorkItemType] IN GROUP 'Microsoft.BugCategory' AND [System.State] <> 'Removed' AND ${r.filter}`)
|
|
111
|
+
: []
|
|
112
|
+
const all = await r.items([...new Set([...r.storyIds, ...bugIds])])
|
|
113
|
+
const { org, auth } = adoSettings(cfg.sources)
|
|
114
|
+
const items: ChangeItem[] = []
|
|
115
|
+
let prs = 0
|
|
116
|
+
for (const w of all) {
|
|
117
|
+
const { prLinks, ...item } = itemOf(w, r.urlOf(w))
|
|
118
|
+
const files: string[] = []
|
|
119
|
+
for (const p of prLinks) {
|
|
120
|
+
if (prs >= MAX_PRS) break
|
|
121
|
+
prs++
|
|
122
|
+
try {
|
|
123
|
+
files.push(...(await adoPrFiles(org, auth, p.project, p.repo, p.id)))
|
|
124
|
+
} catch (err) {
|
|
125
|
+
notes.push(`Pull request ${p.id} of work item ${w.id} could not be read: ${(err as Error).message.slice(0, 120)}`)
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
items.push({ ...item, ...(files.length ? { files: [...new Set(files)] } : {}) })
|
|
129
|
+
}
|
|
130
|
+
if (prs >= MAX_PRS) notes.push(`Only the first ${MAX_PRS} linked pull requests were read for their files.`)
|
|
131
|
+
return { label: r.label, items, notes }
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** The commits between two refs of a repository, each with its files and the work items it mentions. */
|
|
135
|
+
export function fromGitRange(repo: string, from: string, to = "HEAD", testRepo?: string): Collected {
|
|
136
|
+
const range = `${from}..${to}`
|
|
137
|
+
const log = git(repo, ["log", "--no-merges", `--max-count=${MAX_COMMITS}`, "--format=%H%x1f%s%x1f%b%x1e", range])
|
|
138
|
+
if (log.status !== 0) throw new Error(`git log ${range} failed in ${repo}: ${log.stderr.trim().split("\n")[0]}`)
|
|
139
|
+
const inTestRepo = Boolean(testRepo) && real(repo) === real(testRepo!)
|
|
140
|
+
const platform = remotePlatform(repo)
|
|
141
|
+
const items: ChangeItem[] = log.stdout
|
|
142
|
+
.split("\x1e")
|
|
143
|
+
.map((c) => c.trim())
|
|
144
|
+
.filter(Boolean)
|
|
145
|
+
.map((c) => {
|
|
146
|
+
const [sha = "", subject = "", body = ""] = c.split("\x1f")
|
|
147
|
+
const files = git(repo, ["show", "--name-only", "--format=", sha]).stdout.split("\n").map((l) => l.trim()).filter(Boolean)
|
|
148
|
+
const links = linksIn(`${subject}\n${body}`, platform)
|
|
149
|
+
return { kind: "commit" as const, source: "git", id: sha.slice(0, 10), title: subject, ...(body.trim() ? { text: clip(body.trim()) } : {}), files, ...(inTestRepo ? { inTestRepo } : {}), ...(links.length ? { links } : {}) }
|
|
150
|
+
})
|
|
151
|
+
const notes = items.length >= MAX_COMMITS ? [`Only the newest ${MAX_COMMITS} commits of ${range} were read.`] : []
|
|
152
|
+
return { label: `commits ${range} in ${path.basename(real(repo))}`, items, notes }
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function remotePlatform(repo: string): "github" | "azure_devops" | undefined {
|
|
156
|
+
const url = git(repo, ["remote", "get-url", "origin"]).stdout.trim()
|
|
157
|
+
return /github\.com/i.test(url) ? "github" : /dev\.azure\.com|visualstudio\.com/i.test(url) ? "azure_devops" : undefined
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Pull requests by number or URL: their words, linked work items, and changed files. */
|
|
161
|
+
export async function fromPrs(cfg: AppConfig, repo: string, refs: string[], testRepo?: string): Promise<Collected> {
|
|
162
|
+
const items: ChangeItem[] = []
|
|
163
|
+
const notes: string[] = []
|
|
164
|
+
for (const input of refs.slice(0, MAX_PRS)) {
|
|
165
|
+
try {
|
|
166
|
+
const r = await resolvePr(repo, input)
|
|
167
|
+
const pr = await fetchPr(r.ref, cfg, r.root)
|
|
168
|
+
let files: string[] = []
|
|
169
|
+
try {
|
|
170
|
+
const { base, dropRefs } = await fetchPrBranches(r.root, r.remote, pr)
|
|
171
|
+
try {
|
|
172
|
+
files = changedBetween(r.root, base, pr.headSha)
|
|
173
|
+
} finally {
|
|
174
|
+
await dropRefs()
|
|
175
|
+
}
|
|
176
|
+
} catch (err) {
|
|
177
|
+
notes.push(`The files of pull request ${r.ref.number} could not be read: ${(err as Error).message.slice(0, 120)}`)
|
|
178
|
+
}
|
|
179
|
+
const links = [...(pr.workItemIds || []).map((id) => `ado:${id}`), ...linksIn(`${pr.title}\n${pr.description}`, r.ref.platform)]
|
|
180
|
+
items.push({
|
|
181
|
+
kind: "pr",
|
|
182
|
+
source: r.ref.platform,
|
|
183
|
+
id: String(r.ref.number),
|
|
184
|
+
title: pr.title,
|
|
185
|
+
text: clip([pr.description, ...pr.linked.map((l) => `${l.title}${l.text ? `: ${l.text}` : ""}`)].filter(Boolean).join("\n")),
|
|
186
|
+
url: pr.url,
|
|
187
|
+
files,
|
|
188
|
+
...(testRepo && real(r.root) === real(testRepo) ? { inTestRepo: true } : {}),
|
|
189
|
+
...(links.length ? { links: [...new Set(links)] } : {}),
|
|
190
|
+
})
|
|
191
|
+
} catch (err) {
|
|
192
|
+
notes.push(`Pull request ${input} could not be read: ${(err as Error).message.slice(0, 160)}`)
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
if (refs.length > MAX_PRS) notes.push(`Only the first ${MAX_PRS} of ${refs.length} pull requests were read.`)
|
|
196
|
+
return { label: `pull requests ${refs.slice(0, 8).join(", ")}${refs.length > 8 ? "…" : ""}`, items, notes }
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/** The pull requests merged into a repository in the last `days` days, newest first, as refs fromPrs reads. */
|
|
200
|
+
export async function recentPrs(cfg: AppConfig, repo: string, days: number): Promise<string[]> {
|
|
201
|
+
const since = new Date(Date.now() - days * 86_400_000).toISOString()
|
|
202
|
+
const remote = (await gitRemotes(repo)).find((r) => r.name === "origin") || (await gitRemotes(repo))[0]
|
|
203
|
+
const where = remote ? parseRemote(remote.url) : null
|
|
204
|
+
if (!where) throw new Error(`${repo} has no GitHub or Azure DevOps remote to list pull requests from.`)
|
|
205
|
+
if (where.platform === "github") {
|
|
206
|
+
const out = await gh(["pr", "list", "--state", "merged", "--search", `merged:>=${since.slice(0, 10)}`, "--json", "number", "--limit", "100"], repo)
|
|
207
|
+
return (JSON.parse(out || "[]") as { number: number }[]).map((p) => String(p.number))
|
|
208
|
+
}
|
|
209
|
+
const { auth } = adoSettings(cfg.sources)
|
|
210
|
+
const ref = { ...where, number: 0 } as PrRef
|
|
211
|
+
const { data } = await http<{ value?: { pullRequestId: number; closedDate?: string }[] }>(
|
|
212
|
+
`${adoBase(ref)}/pullrequests?searchCriteria.status=completed&searchCriteria.minTime=${encodeURIComponent(since)}&searchCriteria.queryTimeRangeType=closed&$top=100&api-version=7.1`,
|
|
213
|
+
{ auth },
|
|
214
|
+
)
|
|
215
|
+
return (data?.value || []).map((p) => String(p.pullRequestId))
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** Items ForgeQE's agent read from another tool (Jira, GitHub issues, a release note), checked and trimmed. */
|
|
219
|
+
export function fromGiven(raw: unknown[]): Collected {
|
|
220
|
+
const items: ChangeItem[] = []
|
|
221
|
+
for (const r of raw) {
|
|
222
|
+
if (!r || typeof r !== "object") continue
|
|
223
|
+
const o = r as Record<string, unknown>
|
|
224
|
+
const title = String(o.title || "").trim()
|
|
225
|
+
if (!title) continue
|
|
226
|
+
const list = (v: unknown) => (Array.isArray(v) ? v.map(String).map((s) => s.trim()).filter(Boolean) : [])
|
|
227
|
+
items.push({
|
|
228
|
+
kind: ["work_item", "pr", "commit", "note"].includes(String(o.kind)) ? (o.kind as ChangeItem["kind"]) : "work_item",
|
|
229
|
+
source: String(o.source || "agent"),
|
|
230
|
+
id: String(o.id || title.slice(0, 40)),
|
|
231
|
+
title,
|
|
232
|
+
...(o.text ? { text: clip(String(o.text)) } : {}),
|
|
233
|
+
...(o.url ? { url: String(o.url) } : {}),
|
|
234
|
+
...(o.type ? { type: String(o.type) } : {}),
|
|
235
|
+
...(list(o.files).length ? { files: list(o.files) } : {}),
|
|
236
|
+
...(list(o.links).length ? { links: list(o.links) } : {}),
|
|
237
|
+
...(list(o.test_cases).length ? { testCases: list(o.test_cases).map((c) => (/^\d+$/.test(c) ? `ADO-${c}` : c)) } : {}),
|
|
238
|
+
})
|
|
239
|
+
}
|
|
240
|
+
return { label: `${items.length} item${items.length === 1 ? "" : "s"} from ForgeQE's agent`, items, notes: [] }
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Work items commits or pull requests mention by id ("ado:123"), read from Azure DevOps, that are not in the list yet. */
|
|
244
|
+
export async function linkedWorkItems(cfg: AppConfig, items: ChangeItem[]): Promise<Collected | null> {
|
|
245
|
+
if (!cfg.sources.ado) return null
|
|
246
|
+
const have = new Set(items.filter((i) => i.source === "azure_devops" && i.kind === "work_item").map((i) => i.id))
|
|
247
|
+
const ids = [...new Set(items.flatMap((i) => i.links || []).filter((l) => l.startsWith("ado:")).map((l) => Number(l.slice(4))))].filter((id) => id && !have.has(String(id)))
|
|
248
|
+
if (!ids.length) return null
|
|
249
|
+
const got = await fromAdoScope(cfg, { ids: ids.slice(0, 200) })
|
|
250
|
+
return { ...got, label: `${got.items.length} linked work item${got.items.length === 1 ? "" : "s"}` }
|
|
251
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import type { Card } from "../card.ts"
|
|
2
|
+
import type { AppConfig } from "../config.ts"
|
|
3
|
+
import type { Scope } from "../cases/readiness.ts"
|
|
4
|
+
import type { TestsRun } from "../code/find.ts"
|
|
5
|
+
import { fromAdoScope, fromGitRange, fromGiven, fromPrs, linkedWorkItems, recentPrs, type ChangeItem, type Collected } from "./collect.ts"
|
|
6
|
+
import { buildPack, type Pack, type PackRun, type ReasonKind } from "./pack.ts"
|
|
7
|
+
|
|
8
|
+
export { buildPack, listPacks, loadPack, type Pack } from "./pack.ts"
|
|
9
|
+
export { runPack } from "./run.ts"
|
|
10
|
+
|
|
11
|
+
/** Where a release's changes are: any mix of these. */
|
|
12
|
+
export type PackSources = {
|
|
13
|
+
ado?: Scope
|
|
14
|
+
/** Commits between two refs of a repository (the product's, or the tests' own). */
|
|
15
|
+
range?: { repo: string; from: string; to?: string }
|
|
16
|
+
/** Pull requests by number or URL, in `prRepo` (default: the test repository). */
|
|
17
|
+
prs?: string[]
|
|
18
|
+
/** Pull requests merged in the last N days, in `prRepo`. */
|
|
19
|
+
mergedDays?: number
|
|
20
|
+
prRepo?: string
|
|
21
|
+
/** Items ForgeQE's agent read from any other tool. */
|
|
22
|
+
items?: unknown[]
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Every source's changes as one list, without repeats; work items the commits and PRs mention are read too. */
|
|
26
|
+
export async function collectChanges(cfg: AppConfig, testRepo: string, src: PackSources): Promise<Collected & { scope: string[] }> {
|
|
27
|
+
const parts: Collected[] = []
|
|
28
|
+
const notes: string[] = []
|
|
29
|
+
const tryPart = async (what: string, f: () => Promise<Collected> | Collected) => {
|
|
30
|
+
try {
|
|
31
|
+
parts.push(await f())
|
|
32
|
+
} catch (err) {
|
|
33
|
+
notes.push(`${what}: ${(err as Error).message.slice(0, 200)}`)
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
if (src.ado && Object.values(src.ado).some((v) => (Array.isArray(v) ? v.length : Boolean(v)))) await tryPart("Azure DevOps", () => fromAdoScope(cfg, src.ado!))
|
|
37
|
+
if (src.range) await tryPart(`Commits ${src.range.from}..${src.range.to || "HEAD"}`, () => fromGitRange(src.range!.repo, src.range!.from, src.range!.to || "HEAD", testRepo))
|
|
38
|
+
const prRepo = src.prRepo || testRepo
|
|
39
|
+
const prs = [...(src.prs || [])]
|
|
40
|
+
if (src.mergedDays) {
|
|
41
|
+
try {
|
|
42
|
+
prs.push(...(await recentPrs(cfg, prRepo, src.mergedDays)))
|
|
43
|
+
} catch (err) {
|
|
44
|
+
notes.push(`Pull requests merged in the last ${src.mergedDays} days: ${(err as Error).message.slice(0, 200)}`)
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (prs.length) await tryPart("Pull requests", () => fromPrs(cfg, prRepo, [...new Set(prs)], testRepo))
|
|
48
|
+
if (src.items?.length) parts.push(fromGiven(src.items))
|
|
49
|
+
const seen = new Set<string>()
|
|
50
|
+
const items: ChangeItem[] = []
|
|
51
|
+
for (const p of parts) {
|
|
52
|
+
for (const i of p.items) {
|
|
53
|
+
const key = `${i.source}:${i.kind}:${i.id}`
|
|
54
|
+
if (seen.has(key)) continue
|
|
55
|
+
seen.add(key)
|
|
56
|
+
items.push(i)
|
|
57
|
+
}
|
|
58
|
+
notes.push(...p.notes)
|
|
59
|
+
}
|
|
60
|
+
// Work items the commits and pull requests name, read for their words and their test cases.
|
|
61
|
+
try {
|
|
62
|
+
const linked = await linkedWorkItems(cfg, items)
|
|
63
|
+
if (linked?.items.length) {
|
|
64
|
+
for (const i of linked.items) if (!seen.has(`${i.source}:${i.kind}:${i.id}`)) items.push(i)
|
|
65
|
+
parts.push(linked)
|
|
66
|
+
notes.push(...linked.notes)
|
|
67
|
+
}
|
|
68
|
+
} catch (err) {
|
|
69
|
+
notes.push(`Work items the changes mention: ${(err as Error).message.slice(0, 200)}`)
|
|
70
|
+
}
|
|
71
|
+
const scope = parts.map((p) => p.label).filter(Boolean)
|
|
72
|
+
return { label: scope.join("; "), items, notes, scope }
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Collect and build in one go. */
|
|
76
|
+
export async function makePack(opts: { config: AppConfig; repo: string; name?: string; sources: PackSources; risk?: boolean; smoke?: boolean }): Promise<Pack> {
|
|
77
|
+
const c = await collectChanges(opts.config, opts.repo, opts.sources)
|
|
78
|
+
if (!c.items.length) throw new Error(`No changes found${c.notes.length ? `: ${c.notes.join(" ")}` : ". Give a sprint or work items, a commit range, pull requests, or items."}`)
|
|
79
|
+
return buildPack({ config: opts.config, repo: opts.repo, name: opts.name, scope: c.scope, items: c.items, notes: c.notes, risk: opts.risk, smoke: opts.smoke })
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const KIND_TEXT: Record<ReasonKind, string> = { linked: "linked test case", changed: "test code changed", covers: "matches a change", risk: "risk area", smoke: "smoke" }
|
|
83
|
+
|
|
84
|
+
export function packCard(p: Pack): Card {
|
|
85
|
+
const by = (k: ReasonKind) => p.tests.filter((t) => t.reasons.some((r) => r.kind === k)).length
|
|
86
|
+
const kinds = (Object.keys(KIND_TEXT) as ReasonKind[]).filter((k) => by(k)).map((k) => `${by(k)} ${KIND_TEXT[k]}`)
|
|
87
|
+
return {
|
|
88
|
+
status: p.tests.length ? "ok" : "info",
|
|
89
|
+
title: `Regression pack ${p.name}: ${p.tests.length} test${p.tests.length === 1 ? "" : "s"} for ${p.items.length} change${p.items.length === 1 ? "" : "s"}`,
|
|
90
|
+
stats: [kinds.join(", "), p.uncovered.length ? `${p.uncovered.length} not covered` : ""].filter(Boolean).join(" · "),
|
|
91
|
+
body: [
|
|
92
|
+
`From ${p.scope.join("; ") || "—"}`,
|
|
93
|
+
...p.tests.slice(0, 12).map((t) => `${t.test}${t.flaky ? " (flaky)" : ""}: ${[...new Set(t.reasons.map((r) => r.text))].slice(0, 2).join("; ").slice(0, 160)}`),
|
|
94
|
+
...(p.tests.length > 12 ? [`… and ${p.tests.length - 12} more`] : []),
|
|
95
|
+
...(p.risk.length ? ["", `Risk areas: ${p.risk.map((r) => `${r.area} (${r.level})`).join(", ")}`] : []),
|
|
96
|
+
...(p.uncovered.length ? ["", "Not covered by any test:", ...p.uncovered.slice(0, 8).map((u) => `${u.item} "${u.title}"${u.nearest ? `: closest ${u.nearest.test}${u.nearest.lacks.length ? `, which lacks ${u.nearest.lacks.slice(0, 3).join(", ")}` : ""}` : ""}`)] : []),
|
|
97
|
+
...p.notes.slice(0, 4).map((n) => `Note: ${n}`),
|
|
98
|
+
],
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export function packRunCard(p: Pack, run: TestsRun, record: PackRun, notes: string[] = []): Card {
|
|
103
|
+
const before = p.runs.at(-2)
|
|
104
|
+
const fixed = before ? before.failed.filter((f) => !record.failed.includes(f)) : []
|
|
105
|
+
const broke = before ? record.failed.filter((f) => !before.failed.includes(f)) : []
|
|
106
|
+
return {
|
|
107
|
+
status: run.status === "passed" ? "ok" : run.status === "stopped" ? "stopped" : "fail",
|
|
108
|
+
title: `Pack ${p.name} on ${record.target || record.on}: ${run.status}`,
|
|
109
|
+
stats: `${run.counts.passed} passed, ${run.counts.failed} failed${run.counts.skipped ? `, ${run.counts.skipped} skipped` : ""} · ${run.seconds}s`,
|
|
110
|
+
body: [
|
|
111
|
+
...run.tests.filter((t) => t.status === "failed").slice(0, 12).map((t) => `✗ ${t.name}${t.message ? `: ${t.message.slice(0, 140)}` : ""}`),
|
|
112
|
+
...(run.flaky?.length ? [`Flaky (passed on retry): ${run.flaky.join(", ")}`] : []),
|
|
113
|
+
...(before ? [`Since the last run (${before.at.slice(0, 16).replace("T", " ")} on ${before.target || before.on}): ${broke.length} newly failing, ${fixed.length} fixed`] : []),
|
|
114
|
+
`Command ${run.command.slice(0, 300)}`,
|
|
115
|
+
...notes.map((n) => `Note: ${n}`),
|
|
116
|
+
],
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** The pack as markdown, for a pull request, a wiki page, or a release note. */
|
|
121
|
+
export function packMarkdown(p: Pack): string {
|
|
122
|
+
return [
|
|
123
|
+
`# Regression pack ${p.name}`,
|
|
124
|
+
"",
|
|
125
|
+
`${p.tests.length} tests for ${p.items.length} changes (${p.scope.join("; ")}), built ${p.createdAt.slice(0, 16).replace("T", " ")}.`,
|
|
126
|
+
"",
|
|
127
|
+
"| Test | File | Why |",
|
|
128
|
+
"|---|---|---|",
|
|
129
|
+
...p.tests.map((t) => `| ${t.test}${t.flaky ? " ⚠ flaky" : ""} | ${t.file} | ${[...new Set(t.reasons.map((r) => r.text.replace(/\|/g, "\\|")))].join("<br>")} |`),
|
|
130
|
+
...(p.uncovered.length ? ["", "## Not covered by any test", "", ...p.uncovered.map((u) => `- **${u.item}** ${u.title}: ${u.why}${u.nearest ? `. Closest test: ${u.nearest.test}${u.nearest.lacks.length ? ` (lacks ${u.nearest.lacks.join(", ")})` : ""}` : ""}`)] : []),
|
|
131
|
+
...(p.risk.length ? ["", "## Risk areas", "", ...p.risk.map((r) => `- **${r.area}** (${r.level}): ${r.reasons.join("; ")}`)] : []),
|
|
132
|
+
...(p.runs.length ? ["", "## Runs", "", "| When | Where | Result | Passed | Failed |", "|---|---|---|---|---|", ...p.runs.map((r) => `| ${r.at.slice(0, 16).replace("T", " ")} | ${r.target || r.on} | ${r.status} | ${r.counts.passed} | ${r.counts.failed} |`)] : []),
|
|
133
|
+
].join("\n")
|
|
134
|
+
}
|
package/src/pack/init.ts
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, writeFileSync } from "node:fs"
|
|
2
|
+
import path from "node:path"
|
|
3
|
+
|
|
4
|
+
import type { AppConfig } from "../config.ts"
|
|
5
|
+
import { walk } from "../code/profile.ts"
|
|
6
|
+
import { listPacks } from "./pack.ts"
|
|
7
|
+
import { ADO_PIPELINE_FILE, adoPipeline, dotnetProject, githubWorkflow, packSelector, WORKFLOW_FILE, type Framework } from "./pipeline.ts"
|
|
8
|
+
|
|
9
|
+
/** The repository's test framework, from its newest pack, else its files. */
|
|
10
|
+
function frameworkOf(repo: string, files: string[]): { framework: Framework; dotnet?: { project: string; settings?: string } } {
|
|
11
|
+
const pack = listPacks(repo).find((p) => p.tests.length)
|
|
12
|
+
if (pack) {
|
|
13
|
+
const { framework } = packSelector(pack)
|
|
14
|
+
return { framework, ...(framework === "dotnet" ? { dotnet: dotnetProject(repo, pack, files) } : {}) }
|
|
15
|
+
}
|
|
16
|
+
const test = files.filter((f) => f.endsWith(".csproj") && /test/i.test(f)).sort((a, b) => Number(/web|ui|e2e/i.test(b)) - Number(/web|ui|e2e/i.test(a)))[0]
|
|
17
|
+
if (test) return { framework: "dotnet", dotnet: { project: test, ...(files.find((f) => /(^|\/)ci\.runsettings$/i.test(f)) ? { settings: files.find((f) => /(^|\/)ci\.runsettings$/i.test(f))! } : {}) } }
|
|
18
|
+
if (files.some((f) => /(^|\/)playwright\.config\.(ts|js|mjs|cjs)$/.test(f))) return { framework: "playwright" }
|
|
19
|
+
if (files.some((f) => f.endsWith(".py"))) return { framework: "pytest" }
|
|
20
|
+
throw new Error("ForgeQE cannot tell this repository's test framework: build a pack first.")
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** Writes the pipeline files into the test repository; never over an existing one unless asked. Returns what it did, as lines. */
|
|
24
|
+
export function writePipelineFiles(cfg: AppConfig, repo: string, which: { github: boolean; azureDevops: boolean; force?: boolean }): string[] {
|
|
25
|
+
const files = walk(repo)
|
|
26
|
+
const { framework, dotnet } = frameworkOf(repo, files)
|
|
27
|
+
const remote = cfg.browser.remotes?.testmu
|
|
28
|
+
const out: string[] = []
|
|
29
|
+
const write = (rel: string, text: string) => {
|
|
30
|
+
const file = path.join(repo, rel)
|
|
31
|
+
if (existsSync(file) && !which.force) return out.push(`Kept ${rel}: it exists (--force writes over it).`)
|
|
32
|
+
mkdirSync(path.dirname(file), { recursive: true })
|
|
33
|
+
writeFileSync(file, text)
|
|
34
|
+
out.push(`Wrote ${rel} (${framework}${dotnet ? `, ${dotnet.project}` : ""}).`)
|
|
35
|
+
}
|
|
36
|
+
if (which.github) write(path.join(".github", "workflows", WORKFLOW_FILE), githubWorkflow(framework, remote, dotnet))
|
|
37
|
+
if (which.azureDevops) write(ADO_PIPELINE_FILE, adoPipeline(framework, remote, dotnet))
|
|
38
|
+
out.push(
|
|
39
|
+
"",
|
|
40
|
+
"Next:",
|
|
41
|
+
...(which.github ? [`- GitHub: commit and push ${path.join(".github", "workflows", WORKFLOW_FILE)} to the default branch; for TestMu AI add the repository secrets LT_USERNAME and LT_ACCESS_KEY; then set packs.pipeline.github (repo: owner/name, workflow: ${WORKFLOW_FILE}) in forgeqe.yaml.`] : []),
|
|
42
|
+
...(which.azureDevops ? [`- Azure DevOps: commit ${ADO_PIPELINE_FILE}, create a pipeline from it (Pipelines > New > your repository > Existing YAML); for TestMu AI add the secret variables LT_USERNAME and LT_ACCESS_KEY; then set packs.pipeline.azure_devops (project, pipeline_id) in forgeqe.yaml. The PAT needs Build (read & execute) and Test Management (read).`] : []),
|
|
43
|
+
)
|
|
44
|
+
return out
|
|
45
|
+
}
|