@capagents/forgeqe 1.1.4 → 1.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HOW_TO_USE.md +46 -4
- package/README.md +2 -1
- package/browserstack.example.yaml +14 -2
- package/docs/user-guide.html +62 -10
- package/forgeqe.example.yaml +18 -5
- package/package.json +2 -1
- package/prompts/README.md +1 -0
- package/prompts/brain.md +7 -1
- package/prompts/defect-rca.md +43 -0
- package/prompts/test-agent.md +12 -2
- package/prompts/test-reviewer.md +5 -2
- package/prompts/test-rules.md +8 -4
- package/py/forgeqe_engine/browser.py +1 -1
- package/skills/dotnet-nunit-playwright/SKILL.md +2 -1
- package/skills/playwright-typescript/SKILL.md +2 -1
- package/src/brain/index.ts +1 -1
- package/src/capabilities/registry.ts +34 -6
- package/src/cases/bugs.ts +209 -22
- package/src/cases/defectHistory.ts +193 -0
- package/src/cases/defects.ts +255 -0
- package/src/cases/github.ts +162 -0
- package/src/cases/goal.ts +1 -1
- package/src/cases/index.ts +23 -4
- package/src/cases/predict.ts +372 -0
- package/src/cases/rca.ts +258 -0
- package/src/cases/readiness.ts +38 -1
- package/src/cases/repro.ts +202 -0
- package/src/cases/types.ts +8 -1
- package/src/cli/help.ts +8 -2
- package/src/cli/index.ts +36 -7
- package/src/code/graph/graph.ts +511 -0
- package/src/code/graph/locators.ts +398 -0
- package/src/code/graph/parse.ts +439 -0
- package/src/code/graph/tools.ts +113 -0
- package/src/code/handoff.ts +93 -18
- package/src/code/loop.ts +30 -5
- package/src/code/profile.ts +14 -3
- package/src/code/replay.ts +1 -1
- package/src/code/review.ts +22 -0
- package/src/config.ts +142 -18
- package/src/mcp/attachtools.ts +223 -0
- package/src/mcp/commands.ts +1 -1
- package/src/mcp/defecttools.ts +106 -0
- package/src/mcp/devicetools.ts +125 -0
- package/src/mcp/external.ts +42 -1
- package/src/mcp/tools.ts +92 -9
- package/src/monitor/run.ts +1 -1
- package/src/report/data.ts +14 -0
- package/src/report/html.ts +15 -3
- package/src/report/template.html +44 -43
- package/src/run.ts +4 -2
- package/src/suite/report.ts +44 -11
- package/src/suite/run.ts +163 -16
- package/src/tui/App.tsx +9 -3
- package/src/usage.ts +1 -1
- package/testmu.example.yaml +11 -0
package/src/code/handoff.ts
CHANGED
|
@@ -6,6 +6,8 @@ import YAML from "yaml"
|
|
|
6
6
|
import { caseMarkdown } from "../cases/goal.ts"
|
|
7
7
|
import type { TestCase } from "../cases/types.ts"
|
|
8
8
|
import { findPage, readPages, type MemoryElement, type MemoryPage } from "../memory.ts"
|
|
9
|
+
import { describeMatches, exampleTest, membersLocating, outline, outlines, sameElement, type CodeGraph } from "./graph/graph.ts"
|
|
10
|
+
import { fromMemory, type LocatorRef } from "./graph/locators.ts"
|
|
9
11
|
import { replayOptions, replaySection, type Replay } from "./replay.ts"
|
|
10
12
|
import { sliceFor } from "./slices.ts"
|
|
11
13
|
|
|
@@ -194,7 +196,7 @@ function stepLines(step: Step, n: number): string[] {
|
|
|
194
196
|
}
|
|
195
197
|
|
|
196
198
|
/** The steps in order, each group led by the browser agent's note from the step that performed it. */
|
|
197
|
-
function stepsSection(steps: Step[], slices: Map<number, string[]> = new Map()): string[] {
|
|
199
|
+
function stepsSection(steps: Step[], slices: Map<number, string[]> = new Map(), links: Map<number, string[]> = new Map()): string[] {
|
|
198
200
|
if (!steps.length) return ["(no recorded steps)"]
|
|
199
201
|
const lines: string[] = []
|
|
200
202
|
let last = ""
|
|
@@ -202,7 +204,7 @@ function stepsSection(steps: Step[], slices: Map<number, string[]> = new Map()):
|
|
|
202
204
|
const note = step.note || ""
|
|
203
205
|
if (note && note !== last) lines.push(`> Browser agent: ${note}`)
|
|
204
206
|
last = note
|
|
205
|
-
lines.push(...stepLines(step, i + 1), ...(slices.get(i + 1) || []))
|
|
207
|
+
lines.push(...stepLines(step, i + 1), ...(links.get(i + 1) || []), ...(slices.get(i + 1) || []))
|
|
206
208
|
})
|
|
207
209
|
return lines
|
|
208
210
|
}
|
|
@@ -263,8 +265,8 @@ function stepSlices(steps: Step[], replay: Replay | null | undefined): Map<numbe
|
|
|
263
265
|
}
|
|
264
266
|
|
|
265
267
|
/**
|
|
266
|
-
* The overlays the run closed, in order.
|
|
267
|
-
* click on one is blocked while another covers it, so the test closes them in this order from one
|
|
268
|
+
* The overlays the run closed, in order. Several often show together (a sign-in prompt over a cookie banner), and a
|
|
269
|
+
* click on one is blocked while another covers it, so the test closes them in this order from one helper.
|
|
268
270
|
*/
|
|
269
271
|
function overlaysSection(steps: Step[]): string[] {
|
|
270
272
|
const closes = steps.filter((s) => s.op === "click" && s.overlay)
|
|
@@ -275,13 +277,13 @@ function overlaysSection(steps: Step[]): string[] {
|
|
|
275
277
|
const container = o.id ? ` (container #${o.id})` : ""
|
|
276
278
|
return `${i + 1}. ${o.kind || "overlay"}${o.name ? ` "${o.name}"` : ""}${container}: close with ${best ? playwrightOf(best) : s.best_locator || s.label}`
|
|
277
279
|
})
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
]
|
|
280
|
+
// One overlay: the framework's own handler is the best practice. Several can stack, and a handler's click is then
|
|
281
|
+
// blocked by the other overlay while handlers do not run inside a handler, so one helper closes them top first.
|
|
282
|
+
const how =
|
|
283
|
+
closes.length === 1
|
|
284
|
+
? "This covered the page and the run closed it; the replay closed it the same way, as a plain step, and passed. In the test, register the framework's overlay handler for it once, in the fixture or the base class setup (Playwright 1.42 and later: page.addLocatorHandler, Page.AddLocatorHandlerAsync), with the close control below as its locator. In a framework or version without one, write a helper that closes it when visible and call it after every navigation and right before each step that opens something (a date picker, a menu)."
|
|
285
|
+
: "These covered the page and the run closed them in this order (the one on top first); the replay closed them the same way, as plain steps, and passed. They can show at once, so do not use locator handlers or combined (or) locators for them: a handler's click is blocked while another overlay covers it, and handlers do not run inside a handler. Write one helper that closes each of them that is visible, in this order, and call it after every navigation and right before each step that opens something (a date picker, a menu): an overlay can come back after a click."
|
|
286
|
+
return ["## Overlays this run closed", "", how, "", ...lines]
|
|
285
287
|
}
|
|
286
288
|
|
|
287
289
|
/** The Playwright options of the replay pass that reached the run's final page: the test uses the same, so it gets the same page. */
|
|
@@ -380,7 +382,7 @@ function checkLine(c: Check | CheckResult, n: number): string {
|
|
|
380
382
|
}
|
|
381
383
|
|
|
382
384
|
/** The browser agent's checks, with the locator the replay found for each when it ran. */
|
|
383
|
-
function checksSection(checks: Check[], replay?: Replay | null): string[] {
|
|
385
|
+
function checksSection(checks: Check[], replay?: Replay | null, links: Map<number, string[]> = new Map()): string[] {
|
|
384
386
|
if (!checks.length) return []
|
|
385
387
|
const pass = replay?.passes.find((p) => p.reached && p.checks?.length) || replay?.passes.find((p) => p.checks?.length)
|
|
386
388
|
const found = pass?.checks?.length === checks.length ? pass.checks : undefined
|
|
@@ -397,14 +399,77 @@ function checksSection(checks: Check[], replay?: Replay | null): string[] {
|
|
|
397
399
|
: "The results the goal asks to check, with the text the page showed. Assert each of them, and add no other assertions.",
|
|
398
400
|
"The text comes from one run: where it is live data (a price, a count, a date that moves), assert what the expected result says holds, not that value.",
|
|
399
401
|
"",
|
|
400
|
-
...merged.
|
|
402
|
+
...merged.flatMap((c, i) => [checkLine(c, i + 1), ...(links.get(i + 1) || [])]),
|
|
403
|
+
]
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/**
|
|
407
|
+
* "In this repository" lines: the page-object members that already locate an element, found through page memory
|
|
408
|
+
* whichever locator they use (the run's getByRole('button', { name: 'Login' }) finds LoginPage.LoginButton = #login-button).
|
|
409
|
+
* A role locator whose name comes from a parameter matches too much to name here; code_element shows those.
|
|
410
|
+
*/
|
|
411
|
+
function repoLinks(graph: CodeGraph, pages: MemoryPage[], items: { locators?: Locator[]; page?: string }[], classes: Set<string> = new Set()): Map<number, string[]> {
|
|
412
|
+
const out = new Map<number, string[]>()
|
|
413
|
+
items.forEach((item, i) => {
|
|
414
|
+
const refs = (item.locators || []).map((l) => fromMemory(l.type, l.value)).filter((r): r is LocatorRef => Boolean(r))
|
|
415
|
+
if (!refs.length) return
|
|
416
|
+
const same = sameElement(pages, refs, item.page)
|
|
417
|
+
const matches = membersLocating(graph, same.refs).filter((m) => m.how !== "param")
|
|
418
|
+
if (!matches.length) return
|
|
419
|
+
for (const m of matches.slice(0, 2)) if (m.at.cls) classes.add(m.at.cls.name)
|
|
420
|
+
const [first, ...rest] = describeMatches(graph, matches, 2)
|
|
421
|
+
out.set(i + 1, [` in this repository: ${first}`, ...rest.map((r) => ` or: ${r}`)])
|
|
422
|
+
})
|
|
423
|
+
return out
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
const MAX_EXAMPLE_LINES = 60
|
|
427
|
+
|
|
428
|
+
/**
|
|
429
|
+
* The outlines of the classes the steps and checks link to, from the code graph: each member's signature, line, and
|
|
430
|
+
* locators, as the files are now. Then the repository's test that uses the most of them, in full, as the example of
|
|
431
|
+
* its fixtures, attributes, and assertion style. Scribe works from these instead of reading files to learn them.
|
|
432
|
+
*/
|
|
433
|
+
function classesSection(graph: CodeGraph, classes: string[]): string[] {
|
|
434
|
+
const lines = [
|
|
435
|
+
"## The page objects these steps use",
|
|
436
|
+
"",
|
|
437
|
+
"Outlines of the classes named under the steps and checks, from ForgeQE's code graph of the repository as it is now: each member's signature, line, and the locators it builds. Work from these. Read a method's body with code_outline Class.member, and read a file in full only to edit it.",
|
|
438
|
+
"",
|
|
439
|
+
"```",
|
|
440
|
+
outlines(graph, classes),
|
|
441
|
+
"```",
|
|
442
|
+
]
|
|
443
|
+
const test = exampleTest(graph, classes)
|
|
444
|
+
if (!test) return lines
|
|
445
|
+
// A test outside a class (test('…'), def test_) takes its setup from the file: its imports, fixtures, and
|
|
446
|
+
// beforeEach. A short spec file is shown whole, so that setup comes with it.
|
|
447
|
+
let source = outline(graph, test.cls ? `${test.cls.name}.${test.member.name}` : test.member.name)
|
|
448
|
+
if (!test.cls) {
|
|
449
|
+
try {
|
|
450
|
+
const whole = readFileSync(path.join(graph.repo, test.file), "utf8").trimEnd()
|
|
451
|
+
if (whole.split("\n").length <= MAX_EXAMPLE_LINES) source = `${test.file}\n${whole}`
|
|
452
|
+
} catch {}
|
|
453
|
+
}
|
|
454
|
+
if (source.split("\n").length > MAX_EXAMPLE_LINES + 1) return lines
|
|
455
|
+
const header = test.cls ? `${test.cls.kind} ${test.cls.name}${test.cls.bases.length ? ` : ${test.cls.bases.join(", ")}` : ""} (${test.file}:${test.cls.line})` : test.file
|
|
456
|
+
return [
|
|
457
|
+
...lines,
|
|
458
|
+
"",
|
|
459
|
+
"## A test in this repository to follow",
|
|
460
|
+
"",
|
|
461
|
+
`The test that uses the most of these page objects, from ${header}. Follow its fixture or base class, attributes, setup, and assertion style; you need not read other tests to learn them.`,
|
|
462
|
+
"",
|
|
463
|
+
"```",
|
|
464
|
+
source,
|
|
465
|
+
"```",
|
|
401
466
|
]
|
|
402
467
|
}
|
|
403
468
|
|
|
404
469
|
/** The run found a functional defect: the test asserts the correct behaviour and is marked as an expected failure. */
|
|
405
470
|
export type KnownDefect = { reason: string; bug?: { label: string; url?: string } }
|
|
406
471
|
|
|
407
|
-
export function buildHandoff(runDir: string, memoryDir?: string, opts: { replay?: Replay | null; knownDefect?: KnownDefect } = {}): Handoff {
|
|
472
|
+
export function buildHandoff(runDir: string, memoryDir?: string, opts: { replay?: Replay | null; knownDefect?: KnownDefect; graph?: CodeGraph | null } = {}): Handoff {
|
|
408
473
|
const run = readJson<RunJson>(path.join(runDir, "run.json"))
|
|
409
474
|
if (!run) throw new Error(`No run.json in ${runDir}. Point at a ForgeQE run directory.`)
|
|
410
475
|
const workflowFile = path.join(runDir, "workflow.yaml")
|
|
@@ -500,18 +565,28 @@ export function buildHandoff(runDir: string, memoryDir?: string, opts: { replay?
|
|
|
500
565
|
"",
|
|
501
566
|
)
|
|
502
567
|
}
|
|
503
|
-
|
|
568
|
+
const elementSteps = steps.map((s) => (["click", "input", "select"].includes(s.op || "") ? { locators: s.locators, page: s.page || s.page_url } : {}))
|
|
569
|
+
const linked = new Set<string>()
|
|
570
|
+
const stepLinks = opts.graph ? repoLinks(opts.graph, pages, elementSteps, linked) : new Map<number, string[]>()
|
|
571
|
+
const checkLinks = opts.graph ? repoLinks(opts.graph, pages, checks.map((c) => ({ locators: c.element?.locators, page: c.url || run.final_url })), linked) : new Map<number, string[]>()
|
|
572
|
+
if (stepLinks.size || checkLinks.size) {
|
|
573
|
+
md.push(
|
|
574
|
+
"Under a step or a check, \"in this repository\" names the page-object member that already locates that element, found through page memory whichever locator it uses, and the methods that use it. Call that method, or use that member, instead of writing a new locator for the element. When its locator is brittle and the step has a ✓ semantic locator, change it where it is defined.",
|
|
575
|
+
"",
|
|
576
|
+
)
|
|
577
|
+
}
|
|
578
|
+
md.push(...stepsSection(steps, slices, stepLinks))
|
|
504
579
|
const overlays = overlaysSection(steps)
|
|
505
580
|
if (overlays.length) md.push("", ...overlays)
|
|
506
581
|
if (opts.replay) md.push("", ...replaySection(opts.replay))
|
|
507
582
|
const launch = launchSection(run.browser, opts.replay)
|
|
508
583
|
if (launch.length) md.push("", ...launch)
|
|
509
|
-
if (checks.length) md.push("", ...checksSection(checks, opts.replay))
|
|
584
|
+
if (checks.length) md.push("", ...checksSection(checks, opts.replay, checkLinks))
|
|
585
|
+
if (opts.graph && linked.size) md.push("", ...classesSection(opts.graph, [...linked]))
|
|
510
586
|
const snapshots = Boolean(opts.replay?.passes.some((p) => p.pages?.length))
|
|
511
587
|
// Page memory is for finding elements the handoff does not give; with every step and check verified it is noise.
|
|
512
|
-
const elementSteps = steps.filter((s) => ["click", "input", "select"].includes(s.op || ""))
|
|
513
588
|
const allVerified =
|
|
514
|
-
|
|
589
|
+
steps.filter((s) => ["click", "input", "select"].includes(s.op || "")).every((s) => verifiedLocators(s.locators).length) && checks.every((c) => c.item || verifiedLocators(c.element?.locators).length)
|
|
515
590
|
if (visited.length && !snapshots && !allVerified) {
|
|
516
591
|
md.push(
|
|
517
592
|
"",
|
package/src/code/loop.ts
CHANGED
|
@@ -9,6 +9,10 @@ import { buildHandoff, type KnownDefect } from "./handoff.ts"
|
|
|
9
9
|
import type { TestCase } from "../cases/types.ts"
|
|
10
10
|
import { buildProfile, profileSection } from "./profile.ts"
|
|
11
11
|
import { LIVE_BROWSER_SERVER, liveBrowser } from "./livebrowser.ts"
|
|
12
|
+
import { additionFacts, buildGraph, classesUsedBy, outlines, type CodeGraph } from "./graph/graph.ts"
|
|
13
|
+
import { CODE_GRAPH_SERVER, codeGraphTools } from "./graph/tools.ts"
|
|
14
|
+
import { localToolServer } from "../mcp/server.ts"
|
|
15
|
+
import type { ToolContext } from "../mcp/tools.ts"
|
|
12
16
|
import { addWrittenTest, notesSection, readNotes, saveNotes } from "./notes.ts"
|
|
13
17
|
import { canReplay, replayRun, savedReplay, type Replay } from "./replay.ts"
|
|
14
18
|
import { openSession, READ_AGENT, TEST_AGENT, type OpencodeSession, type PromptOutcome } from "./runtime.ts"
|
|
@@ -117,12 +121,20 @@ export async function runCodeAgent(opts: {
|
|
|
117
121
|
const repo = realpathSync(given)
|
|
118
122
|
const { config } = opts
|
|
119
123
|
const emit = opts.onEvent || (() => {})
|
|
120
|
-
|
|
124
|
+
// The repository's code graph, read without a model: the handoff links each element to the page object that already
|
|
125
|
+
// locates it, Scribe gets code_* tools instead of reading whole files, and the reviewer gets what the job duplicated.
|
|
126
|
+
const built = config.code.graph === false ? null : await buildGraph(repo).catch(() => null)
|
|
127
|
+
const graph: CodeGraph | null = built?.files.length ? built : null
|
|
128
|
+
let handoff = buildHandoff(opts.runDir, config.memory.dir, { knownDefect: opts.knownDefect, graph })
|
|
121
129
|
if (!handoff.success && !opts.knownDefect) throw new Error("The coding agent only takes runs that passed. This run failed.")
|
|
122
130
|
const plural = (n: number, w: string) => `${n} ${w}${n === 1 ? "" : "s"}`
|
|
123
131
|
const shown = path.relative(process.cwd(), handoff.file)
|
|
124
132
|
const handed = [plural(handoff.steps, "step"), handoff.checks ? plural(handoff.checks, "check") : "", handoff.pages ? `${plural(handoff.pages, "page")} from page memory` : ""]
|
|
125
133
|
emit({ type: "code", kind: "info", text: `Handoff written: ${shown && !shown.startsWith("..") ? shown : handoff.file} (${handed.filter(Boolean).join(", ")})` })
|
|
134
|
+
if (graph) {
|
|
135
|
+
const classes = graph.files.reduce((n, f) => n + f.classes.length, 0)
|
|
136
|
+
emit({ type: "code", kind: "info", text: `Code graph: ${graph.files.length} files, ${classes} classes (${graph.parsed} read, ${graph.reused} unchanged, ${graph.ms}ms)` })
|
|
137
|
+
}
|
|
126
138
|
const maxAttempts = Math.max(1, config.code.maxAttempts)
|
|
127
139
|
|
|
128
140
|
const result: CodeResult = {
|
|
@@ -152,7 +164,15 @@ export async function runCodeAgent(opts: {
|
|
|
152
164
|
const before = snapshot(repo, own)
|
|
153
165
|
const outside = [opts.runDir, config.memory.dir].map((d) => realpathOr(d)).filter((d) => path.relative(repo, d).startsWith(".."))
|
|
154
166
|
let session: OpencodeSession | null = null
|
|
155
|
-
const
|
|
167
|
+
const codeTools = graph
|
|
168
|
+
? localToolServer({
|
|
169
|
+
tools: codeGraphTools({ repo, ...(config.memory.enabled ? { memoryDir: config.memory.dir } : {}) }),
|
|
170
|
+
context: () => ({ config, cwd: repo }) as ToolContext,
|
|
171
|
+
timeoutMs: 120_000,
|
|
172
|
+
})
|
|
173
|
+
: null
|
|
174
|
+
const reviewer =
|
|
175
|
+
opts.reviewer || openReviewer({ config, repo, readDirs, signal: opts.signal, onEvent: emit, open: opts.open, ...(codeTools ? { mcp: { [CODE_GRAPH_SERVER]: codeTools.entry } } : {}) })
|
|
156
176
|
const runTest = opts.runTest || runTestCommand
|
|
157
177
|
let challenged = false
|
|
158
178
|
let stalls = 0
|
|
@@ -226,7 +246,7 @@ export async function runCodeAgent(opts: {
|
|
|
226
246
|
agent: { ...TEST_AGENT, steps: config.code.maxSteps ?? 50 },
|
|
227
247
|
agents: [READ_AGENT],
|
|
228
248
|
skills: loadSkills(config.skills, repo).skills,
|
|
229
|
-
...(live ? { mcp: { [LIVE_BROWSER_SERVER]: live.config } } : {}),
|
|
249
|
+
...(live || codeTools ? { mcp: { ...(live ? { [LIVE_BROWSER_SERVER]: live.config } : {}), ...(codeTools ? { [CODE_GRAPH_SERVER]: codeTools.entry } : {}) } } : {}),
|
|
230
250
|
...(resume ? { resume } : {}),
|
|
231
251
|
})
|
|
232
252
|
// Each attempt after the first runs in a new session from a brief of the job so far (nextAttemptPrompt), so the
|
|
@@ -285,7 +305,7 @@ export async function runCodeAgent(opts: {
|
|
|
285
305
|
...(r.error ? { error: r.error } : {}),
|
|
286
306
|
passes: r.passes.map((p) => ({ setup: p.setup, failed: p.steps.filter((s) => !s.ok && !s.skipped).map((s) => s.n), reached: p.reached })),
|
|
287
307
|
}
|
|
288
|
-
handoff = buildHandoff(opts.runDir, config.memory.dir, { replay, knownDefect: opts.knownDefect })
|
|
308
|
+
handoff = buildHandoff(opts.runDir, config.memory.dir, { replay, knownDefect: opts.knownDefect, graph })
|
|
289
309
|
// The replay's failure (its error, screenshot, and page snapshot) is in the handoff; the live browser waits for a repeated failure.
|
|
290
310
|
liveOn = Boolean(live) && Boolean(resumed && earlier?.liveOn)
|
|
291
311
|
} else if (resumed && earlier?.liveOn) {
|
|
@@ -463,8 +483,12 @@ export async function runCodeAgent(opts: {
|
|
|
463
483
|
const where = run.browsers.length ? run.browsers.join(", ") : `remote browser (${report.remote_browser})`
|
|
464
484
|
emit({ type: "code", kind: "info", text: `${report.test_file} passed in ${secs}s in ${where}; a reviewer checks it against the goal` })
|
|
465
485
|
const changes = jobChanges(before) ?? undefined
|
|
486
|
+
const now = graph && changes?.length ? await buildGraph(repo).catch(() => null) : null
|
|
487
|
+
const facts = graph && now && changes ? additionFacts(graph, now, changes) : []
|
|
488
|
+
const used = now && changes ? outlines(now, classesUsedBy(now, changes)) : ""
|
|
489
|
+
const settings = (repoProfile?.settings || []).map((st) => `- ${st.file}: ${Object.entries(st.values).map(([k, v]) => `${k}=${v}`).join(", ")}`)
|
|
466
490
|
// After a rejection or a tidy-up, the reviewer checks its own earlier issues instead of starting over.
|
|
467
|
-
const verdict = await reviewer.review({ handoff, report, output: run.output, browsers: run.browsers, seconds: secs, changes, before: preexisting(before), ...(result.review ? { earlier: result.review } : {}) })
|
|
491
|
+
const verdict = await reviewer.review({ handoff, report, output: run.output, browsers: run.browsers, seconds: secs, changes, before: preexisting(before), ...(facts.length ? { facts } : {}), ...(used ? { outlines: used } : {}), ...(settings.length ? { settings } : {}), ...(result.review ? { earlier: result.review } : {}) })
|
|
468
492
|
if (opts.signal?.aborted) {
|
|
469
493
|
result.status = "stopped"
|
|
470
494
|
break
|
|
@@ -584,6 +608,7 @@ export async function runCodeAgent(opts: {
|
|
|
584
608
|
await session.close()
|
|
585
609
|
}
|
|
586
610
|
await reviewer.close()
|
|
611
|
+
codeTools?.stop()
|
|
587
612
|
// With code.model the agent runtime picks the model, so there is no price for it in llms.json; the reviewer is
|
|
588
613
|
// priced at its own profile (code.review_profile). The job's cost is unknown when either part has no price.
|
|
589
614
|
const scribeCost = costOf(result.usage, config.code.model ? undefined : config.llm.price)
|
package/src/code/profile.ts
CHANGED
|
@@ -26,7 +26,8 @@ const PAGE_DIRS = /(^|\/)(pages|page-?objects|pom|components|flows|screens)(\/|$
|
|
|
26
26
|
const MAX_MEMBERS = 14
|
|
27
27
|
const MAX_CLASSES = 25
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
/** The repository's files (relative, with /), skipping build output, dependencies, and dot folders. */
|
|
30
|
+
export function walk(repo: string): string[] {
|
|
30
31
|
const out: string[] = []
|
|
31
32
|
const visit = (dir: string) => {
|
|
32
33
|
if (out.length >= MAX_FILES) return
|
|
@@ -119,13 +120,21 @@ export function buildProfile(repo: string): RepoProfile | null {
|
|
|
119
120
|
try {
|
|
120
121
|
const doc = JSON.parse(read(repo, f)) as { dependencies?: Record<string, string>; devDependencies?: Record<string, string>; scripts?: Record<string, string> }
|
|
121
122
|
const deps = { ...doc.dependencies, ...doc.devDependencies }
|
|
122
|
-
const interesting = Object.entries(deps).filter(([n]) => /playwright|cypress|webdriverio|jest|vitest|mocha/.test(n))
|
|
123
|
+
const interesting = Object.entries(deps).filter(([n]) => /playwright|cypress|webdriverio|@wdio\/|selenium|testing-library|jest|vitest|mocha/.test(n))
|
|
123
124
|
if (interesting.length) profile.stack.push(`${f}: ${interesting.map(([n, v]) => `${n} ${v}`).join(", ")}`)
|
|
124
125
|
if (doc.scripts?.test) profile.stack.push(`${f} test script: ${doc.scripts.test}`)
|
|
125
126
|
} catch {
|
|
126
127
|
/* not JSON */
|
|
127
128
|
}
|
|
128
129
|
}
|
|
130
|
+
// Java: Maven and Gradle builds.
|
|
131
|
+
for (const f of files.filter((f) => /(^|\/)(pom\.xml|build\.gradle(\.kts)?)$/.test(f))) {
|
|
132
|
+
const text = read(repo, f)
|
|
133
|
+
const maven = [...text.matchAll(/<artifactId>([^<]*(?:selenium|playwright|testng|junit|cucumber|selenide)[^<]*)<\/artifactId>\s*(?:<version>([^<]+)<\/version>)?/gi)].map((m) => (m[2] ? `${m[1]} ${m[2]}` : m[1]))
|
|
134
|
+
const gradle = [...text.matchAll(/["']([\w.-]+:[\w.-]*(?:selenium|playwright|testng|junit|cucumber|selenide)[\w.-]*(?::[\w.-]+)?)["']/gi)].map((m) => m[1])
|
|
135
|
+
const hits = [...new Set([...maven, ...gradle])]
|
|
136
|
+
if (hits.length) profile.stack.push(`${f}: ${hits.join(", ")}`)
|
|
137
|
+
}
|
|
129
138
|
const pyDeps = files.filter((f) => /(^|\/)(pyproject\.toml|requirements[^/]*\.txt)$/.test(f))
|
|
130
139
|
for (const f of pyDeps) {
|
|
131
140
|
const hits = read(repo, f).match(/(pytest-playwright|playwright|pytest|selenium)[^\s"',]*/g)
|
|
@@ -185,8 +194,10 @@ export function buildProfile(repo: string): RepoProfile | null {
|
|
|
185
194
|
}
|
|
186
195
|
if (files.some((f) => /(^|\/)playwright\.config\.(ts|js|mjs|cjs)$/.test(f))) profile.oneTest.push('npx playwright test <file> -g "<test title>"')
|
|
187
196
|
if (files.some((f) => /(^|\/)conftest\.py$/.test(f)) || pyDeps.length) profile.oneTest.push("pytest <file>::<test_name>")
|
|
197
|
+
if (files.some((f) => /(^|\/)pom\.xml$/.test(f))) profile.oneTest.push('mvn test -Dtest="<Class>#<method>"')
|
|
198
|
+
if (files.some((f) => /(^|\/)build\.gradle(\.kts)?$/.test(f))) profile.oneTest.push('./gradlew test --tests "<package>.<Class>.<method>"')
|
|
188
199
|
// Only a recognised browser test stack is enough to start from; otherwise Scribe reads the repository itself.
|
|
189
|
-
const browserStack = testProjects.length > 0 || profile.stack.some((s) => /playwright|cypress|selenium|webdriverio
|
|
200
|
+
const browserStack = testProjects.length > 0 || profile.stack.some((s) => /playwright|cypress|selenium|selenide|webdriverio|@wdio\//i.test(s))
|
|
190
201
|
return browserStack ? profile : null
|
|
191
202
|
}
|
|
192
203
|
|
package/src/code/replay.ts
CHANGED
|
@@ -735,7 +735,7 @@ export function replaySection(r: Replay): string[] {
|
|
|
735
735
|
}
|
|
736
736
|
lines.push("")
|
|
737
737
|
const port =
|
|
738
|
-
"Port its steps into the repository's framework and page objects with the same locators and waits, then add the goal's assertions (the handoff's checks, when it lists them). Steps that dealt with a pop-up or banner on this run (see the browser agent's notes) still belong in
|
|
738
|
+
"Port its steps into the repository's framework and page objects with the same locators and waits, then add the goal's assertions (the handoff's checks, when it lists them). Steps that dealt with a pop-up or banner on this run (see the browser agent's notes) still belong in the overlay handler or helper the rules describe. Steps that run the browser agent's JavaScript were how it looked around or read values, not steps to copy: where the JavaScript clicks or types, do that with a Playwright action on the same element; where it reads values, assert them with locators."
|
|
739
739
|
if (first.ok) {
|
|
740
740
|
lines.push(`The script works as it is. ${port}`)
|
|
741
741
|
return lines
|
package/src/code/review.ts
CHANGED
|
@@ -72,6 +72,12 @@ export type ReviewInput = {
|
|
|
72
72
|
before?: string[]
|
|
73
73
|
/** This reviewer's earlier verdict on this job, when the agent has since changed the test to answer it. */
|
|
74
74
|
earlier?: CodeReview
|
|
75
|
+
/** What ForgeQE's code graph found in what the job added: locators that duplicate an existing member's, brittle locators. */
|
|
76
|
+
facts?: string[]
|
|
77
|
+
/** Outlines of the classes the changed code uses, from the code graph, so the reviewer need not read their files. */
|
|
78
|
+
outlines?: string
|
|
79
|
+
/** The repository's browser settings files and their values, as ForgeQE read them. */
|
|
80
|
+
settings?: string[]
|
|
75
81
|
}
|
|
76
82
|
|
|
77
83
|
const LIST_MAX = 60
|
|
@@ -131,6 +137,19 @@ export function reviewPrompt(input: ReviewInput, repo?: string): string {
|
|
|
131
137
|
"```",
|
|
132
138
|
out,
|
|
133
139
|
"```",
|
|
140
|
+
...(input.facts?.length
|
|
141
|
+
? ["", "ForgeQE's code graph found these in what the job added (from the code, not a guess; judge each by the test rules):", ...input.facts.map((f) => `- ${f}`)]
|
|
142
|
+
: []),
|
|
143
|
+
...(input.outlines
|
|
144
|
+
? [
|
|
145
|
+
"",
|
|
146
|
+
"Outlines of the classes the changed code uses, from ForgeQE's code graph of the repository as it is now (signatures, lines, and the locators each member builds). Check reuse and calls against these; read a member's body with code_outline Class.member, not the whole file:",
|
|
147
|
+
"```",
|
|
148
|
+
input.outlines,
|
|
149
|
+
"```",
|
|
150
|
+
]
|
|
151
|
+
: []),
|
|
152
|
+
...(input.settings?.length ? ["", "Browser settings in the repository, as ForgeQE read them from its settings files:", ...input.settings] : []),
|
|
134
153
|
...(repo && changed.length ? ["", "The changed files as they are now:", "", ...changedFiles(repo, changed)] : []),
|
|
135
154
|
...(input.earlier
|
|
136
155
|
? [
|
|
@@ -153,6 +172,8 @@ export function openReviewer(opts: {
|
|
|
153
172
|
signal?: AbortSignal
|
|
154
173
|
onEvent?: (e: CodeEvent) => void
|
|
155
174
|
open?: (o: Parameters<typeof openSession>[0]) => Promise<OpencodeSession>
|
|
175
|
+
/** MCP servers for the review session: the code graph's tools. */
|
|
176
|
+
mcp?: Record<string, unknown>
|
|
156
177
|
}): Reviewer {
|
|
157
178
|
let session: OpencodeSession<ReviewReport> | null = null
|
|
158
179
|
const usage: Usage = { input: 0, output: 0 }
|
|
@@ -185,6 +206,7 @@ export function openReviewer(opts: {
|
|
|
185
206
|
skills: loadSkills(opts.config.skills, opts.repo).skills,
|
|
186
207
|
schema: REVIEW_SCHEMA as unknown as Record<string, unknown>,
|
|
187
208
|
title: "ForgeQE review",
|
|
209
|
+
...(opts.mcp ? { mcp: opts.mcp } : {}),
|
|
188
210
|
})
|
|
189
211
|
}
|
|
190
212
|
let reply = await session.prompt(reviewPrompt(input, opts.repo))
|