@capagents/forgeqe 1.2.1 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HOW_TO_USE.md +40 -3
- package/README.md +1 -1
- package/docs/COST_ARCHITECTURE.md +280 -0
- package/docs/feature-test-prompts.md +95 -0
- package/docs/user-guide.html +54 -7
- package/evals/prompts/brain/route-api-scenarios.yaml +9 -0
- package/evals/prompts/brain/route-e2e.yaml +10 -0
- package/evals/prompts/brain/route-flaky.yaml +9 -0
- package/evals/prompts/brain/route-heal.yaml +9 -0
- package/evals/prompts/brain/route-regression-pack.yaml +9 -0
- package/evals/prompts/brain/route-review-update.yaml +10 -0
- package/evals/prompts/brain/route-run-cases.yaml +9 -0
- package/evals/prompts/brain/route-story-cases.yaml +12 -0
- package/forgeqe.example.yaml +15 -0
- package/package.json +1 -1
- package/prompts/README.md +30 -0
- package/prompts/brain.md +34 -49
- package/prompts/browser-checks.md +2 -0
- package/py/forgeqe_engine/checks.py +8 -2
- package/py/forgeqe_engine/llm.py +33 -5
- package/py/forgeqe_engine/memory.py +59 -2
- package/py/forgeqe_engine/observe.py +123 -0
- package/py/forgeqe_engine/replay.py +55 -2
- package/py/forgeqe_engine/service.py +155 -3
- package/src/agents.ts +52 -0
- package/src/brain/index.ts +8 -2
- package/src/capabilities/registry.ts +48 -6
- package/src/cases/ado.ts +1 -1
- package/src/cases/journey.ts +61 -0
- package/src/cases/readiness.ts +35 -6
- package/src/cases/save.ts +241 -48
- package/src/cases/types.ts +11 -0
- package/src/cli/help.ts +9 -0
- package/src/cli/index.ts +71 -1
- package/src/code/coverage.ts +22 -3
- package/src/code/decisions.ts +73 -0
- package/src/code/find.ts +3 -1
- package/src/code/graph/graph.ts +33 -2
- package/src/code/loop.ts +17 -1
- package/src/code/notes.ts +48 -1
- package/src/code/prompt.ts +31 -9
- package/src/code/protect.ts +103 -0
- package/src/code/review.ts +2 -0
- package/src/code/runtime.ts +35 -3
- package/src/code/shortlist.ts +309 -0
- package/src/code/testrun.ts +3 -1
- package/src/code/traceability.ts +2 -0
- package/src/commands.ts +6 -1
- package/src/config.ts +46 -0
- package/src/context.ts +75 -0
- package/src/engine/bridge.ts +47 -5
- package/src/mcp/captools.ts +19 -0
- package/src/mcp/casetools.ts +107 -11
- package/src/mcp/commands.ts +74 -20
- package/src/mcp/packtools.ts +187 -0
- package/src/mcp/routing.ts +145 -0
- package/src/mcp/server.ts +7 -3
- package/src/mcp/tools.ts +241 -29
- package/src/pack/collect.ts +251 -0
- package/src/pack/index.ts +134 -0
- package/src/pack/init.ts +45 -0
- package/src/pack/pack.ts +327 -0
- package/src/pack/pipeline.ts +373 -0
- package/src/pack/run.ts +155 -0
- package/src/registry/db.ts +249 -0
- package/src/registry/index.ts +72 -0
- package/src/registry/resolve.ts +57 -0
- package/src/registry/tier0.ts +81 -0
- package/src/report/data.ts +7 -0
- package/src/report/html.ts +14 -0
- package/src/report/template.html +52 -48
- package/src/report/usage.ts +113 -0
- package/src/run.ts +22 -2
- package/src/suite/run.ts +12 -5
- package/src/summary.ts +1 -0
- package/src/tokenomics/budget.ts +172 -0
- package/src/tokenomics/ledger.ts +287 -0
- package/src/tui/App.tsx +89 -2
- package/src/tui/inspector.tsx +40 -35
- package/src/types.ts +9 -2
package/src/mcp/tools.ts
CHANGED
|
@@ -11,14 +11,18 @@ import { createAdoDashboard, type DashboardWidget } from "../cases/dashboards.ts
|
|
|
11
11
|
import { writeAdoWikiPage } from "../cases/wiki.ts"
|
|
12
12
|
import { readConfluence } from "../cases/confluence.ts"
|
|
13
13
|
import { casesFrom, type CaseSelector, type TestCase } from "../cases/index.ts"
|
|
14
|
-
import { saveAdoCases, saveJiraCases, type NewCase } from "../cases/save.ts"
|
|
14
|
+
import { saveAdoCases, saveJiraCases, type NewCase, type SaveStatus } from "../cases/save.ts"
|
|
15
15
|
import { buildHandoff } from "../code/handoff.ts"
|
|
16
16
|
import { latestRunDir, runCodeAgent } from "../code/index.ts"
|
|
17
17
|
import type { OpencodeSession } from "../code/runtime.ts"
|
|
18
18
|
import { runTestCommand } from "../code/testrun.ts"
|
|
19
|
-
import { EXISTING_TEST_FILE, findWrittenTest } from "../code/notes.ts"
|
|
19
|
+
import { EXISTING_TEST_FILE, EXTEND_TEST_FILE, findRelatedTests, findWrittenTest, sameFlow, writtenTestFor, writtenTests } from "../code/notes.ts"
|
|
20
|
+
import { buildGraph } from "../code/graph/graph.ts"
|
|
21
|
+
import { walk } from "../code/profile.ts"
|
|
22
|
+
import { indexTests, oneTestCommand, shortlistTests } from "../code/shortlist.ts"
|
|
23
|
+
import { recordDecision } from "../code/decisions.ts"
|
|
20
24
|
import { browserProvider, type AppConfig } from "../config.ts"
|
|
21
|
-
import { clearMemory, memoryFilesFor, memoryFlows, memoryOverview, memoryShow } from "../memory.ts"
|
|
25
|
+
import { clearMemory, memoryFilesFor, memoryFlows, memoryOverview, memoryShow, readPages } from "../memory.ts"
|
|
22
26
|
import { executeRun, runDirectory } from "../run.ts"
|
|
23
27
|
import { counts } from "../suite/report.ts"
|
|
24
28
|
import { deviceCases, runSuite, suiteParallel, type SuiteEvent, type SuiteResult } from "../suite/run.ts"
|
|
@@ -27,10 +31,12 @@ import { PR_TOOLS } from "./prtools.ts"
|
|
|
27
31
|
import { CASE_TOOLS, checkResult, precheck } from "./casetools.ts"
|
|
28
32
|
import { REPO_TEST_TOOLS } from "./repotests.ts"
|
|
29
33
|
import { DEFECT_TOOLS } from "./defecttools.ts"
|
|
34
|
+
import { PACK_TOOLS } from "./packtools.ts"
|
|
30
35
|
import { API_TOOLS } from "./apitools.ts"
|
|
31
36
|
import { EXPLORE_TOOLS } from "./exploretools.ts"
|
|
32
37
|
import { QUALITY_TOOLS } from "./qualitytools.ts"
|
|
33
38
|
import { CAPABILITY_TOOLS } from "./captools.ts"
|
|
39
|
+
import type { Router } from "./routing.ts"
|
|
34
40
|
import { NOTIFY_TOOLS } from "./notifytools.ts"
|
|
35
41
|
import { REPORT_TOOLS } from "./reporttools.ts"
|
|
36
42
|
import { DEVICE_TOOLS } from "./devicetools.ts"
|
|
@@ -135,6 +141,8 @@ export function queryLine(args: Record<string, any>): string {
|
|
|
135
141
|
|
|
136
142
|
export type ToolContext = {
|
|
137
143
|
config: AppConfig
|
|
144
|
+
/** ForgeQE's agent's session: which tools and playbooks it gets on demand (src/mcp/routing.ts). */
|
|
145
|
+
router?: Router
|
|
138
146
|
/** The repository or folder the agent works in. */
|
|
139
147
|
cwd: string
|
|
140
148
|
headed?: boolean
|
|
@@ -330,21 +338,42 @@ function devicesFor(args: Record<string, unknown>, ctx: ToolContext): string[] {
|
|
|
330
338
|
return [...new Set(devices)]
|
|
331
339
|
}
|
|
332
340
|
|
|
341
|
+
type ExistingFirst = { passed?: Record<string, unknown>; failed?: { testFile: string; command: string; output: string; why: string } }
|
|
342
|
+
|
|
343
|
+
function repoOf(ctx: ToolContext): string {
|
|
344
|
+
try {
|
|
345
|
+
return realpathSync(ctx.cwd)
|
|
346
|
+
} catch {
|
|
347
|
+
return ctx.cwd
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
|
|
333
351
|
/**
|
|
334
352
|
* Runs the test an earlier job wrote for this goal, if there is one. `passed`: the tool's answer, the test having
|
|
335
353
|
* passed in a browser. `failed`: the test to update, for write_test. Null when there is no such test.
|
|
336
354
|
*/
|
|
337
|
-
async function existingTestFirst(goal: string, runDir: string, ctx: ToolContext): Promise<
|
|
355
|
+
async function existingTestFirst(goal: string, runDir: string, ctx: ToolContext): Promise<ExistingFirst | null> {
|
|
338
356
|
const cfg = ctx.config
|
|
339
357
|
if (!cfg.memory.enabled) return null
|
|
340
|
-
|
|
341
|
-
try {
|
|
342
|
-
repo = realpathSync(ctx.cwd)
|
|
343
|
-
} catch {}
|
|
358
|
+
const repo = repoOf(ctx)
|
|
344
359
|
const known = findWrittenTest(cfg.memory.dir, repo, goal, (f) => existsSync(path.resolve(repo, f)))
|
|
345
360
|
if (!known) return null
|
|
361
|
+
ctx.emit?.({ type: "info", text: `This flow already has a test, ${known.test_file} (written ${known.at.slice(0, 10)}): ForgeQE runs it first` })
|
|
362
|
+
recordDecision(repo, { goal, stage: "chosen", where: "browser_run", by: "history", mode: "reuse", file: known.test_file, why: `an earlier job wrote it for the same flow (${known.at.slice(0, 10)})` })
|
|
363
|
+
return runKnownTest(known, repo, runDir, ctx, goal)
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/** A test to run as it is: one an earlier job wrote (with the goal it was written for), or one found in the repository. */
|
|
367
|
+
type RunnableTest = { test_file: string; command: string; test?: string; goal?: string; at?: string; derived?: boolean }
|
|
368
|
+
|
|
369
|
+
/**
|
|
370
|
+
* Runs an existing test as it is: passing in a browser answers the goal; failing, Scribe updates it after the run.
|
|
371
|
+
* A command ForgeQE built itself (derived) that opens no browser ran nothing: the browser runs, and nothing is updated.
|
|
372
|
+
*/
|
|
373
|
+
async function runKnownTest(known: RunnableTest, repo: string, runDir: string, ctx: ToolContext, goal: string, asCheck = false): Promise<ExistingFirst | null> {
|
|
374
|
+
const outcome = (status: string) => recordDecision(repo, { goal, stage: "outcome", where: "browser_run", mode: "reuse", file: known.test_file, ...(known.test ? { test: known.test } : {}), status, run_dir: runDir })
|
|
375
|
+
const cfg = ctx.config
|
|
346
376
|
const emit = ctx.emit || (() => {})
|
|
347
|
-
emit({ type: "info", text: `This flow already has a test, ${known.test_file} (written ${known.at.slice(0, 10)}): ForgeQE runs it first` })
|
|
348
377
|
mkdirSync(runDir, { recursive: true })
|
|
349
378
|
const run = await (ctx.runTest || runTestCommand)({
|
|
350
379
|
command: known.command,
|
|
@@ -356,33 +385,151 @@ async function existingTestFirst(goal: string, runDir: string, ctx: ToolContext)
|
|
|
356
385
|
})
|
|
357
386
|
if (ctx.signal?.aborted) return null
|
|
358
387
|
const secs = Math.round(run.ms / 1000)
|
|
388
|
+
const name = known.test ? `${known.test} (${known.test_file})` : known.test_file
|
|
359
389
|
if (run.ok && run.browsers.length) {
|
|
390
|
+
outcome("passed")
|
|
360
391
|
emit({
|
|
361
392
|
type: "card",
|
|
362
393
|
card: {
|
|
363
394
|
status: "ok",
|
|
364
395
|
title: "Existing test passed",
|
|
365
396
|
stats: `${secs}s · no browser run or new test needed`,
|
|
366
|
-
body: [`Test ${
|
|
397
|
+
body: [`Test ${name}`, `Command ${known.command}`, `Browser ${run.browsers.join(", ")}`, ...(known.at ? [`Written ${known.at.slice(0, 10)} by an earlier job`] : [])],
|
|
367
398
|
},
|
|
368
399
|
})
|
|
369
400
|
return {
|
|
370
401
|
passed: {
|
|
371
402
|
status: "passed",
|
|
372
403
|
reused_test: known.test_file,
|
|
373
|
-
|
|
404
|
+
...(known.test ? { test: known.test } : {}),
|
|
405
|
+
...(known.goal ? { written_for: known.goal } : {}),
|
|
374
406
|
command: known.command,
|
|
375
407
|
browsers: run.browsers,
|
|
376
408
|
seconds: secs,
|
|
377
|
-
note:
|
|
409
|
+
note: asCheck
|
|
410
|
+
? "The user's check is answered by this existing test: it does what the check asks and passed just now in a real browser, so no browser run was needed. Report that it passed, naming the test and the command; nothing is written for a check."
|
|
411
|
+
: `An existing test${known.goal ? " written for written_for" : ""} passed as it is, in a real browser: no browser run and no new test were needed. If the user's request asks for something that test does not do or check, call browser_run again with existing_test set to ${known.test ? "test" : "reused_test"} and existing_mode extend or sibling (the browser runs, and that test is extended, or a test is added beside it, rather than a new one written); otherwise say in one or two lines that the existing test passed.`,
|
|
378
412
|
},
|
|
379
413
|
}
|
|
380
414
|
}
|
|
415
|
+
if (run.ok && known.derived) {
|
|
416
|
+
outcome("ran nothing")
|
|
417
|
+
emit({ type: "info", text: `${known.command} ran no browser test: ForgeQE could not run ${name} on its own, so the browser runs the flow` })
|
|
418
|
+
return null
|
|
419
|
+
}
|
|
420
|
+
outcome(run.ok ? "opened no browser" : "failed")
|
|
381
421
|
const why = run.ok ? "It exits 0 but opened no browser." : `It fails now (${run.exitCode === null ? "timed out" : `exit ${run.exitCode}`}): the application or the page changed since it was written.`
|
|
382
|
-
emit({ type: "info", text: `${
|
|
422
|
+
emit({ type: "info", text: asCheck ? `${name} ${run.ok ? "opened no browser" : "fails now"}: ForgeQE runs the check in the browser` : `${name} ${run.ok ? "opened no browser" : "fails now"}: ForgeQE runs the flow in the browser, and Scribe updates that test instead of writing a new one` })
|
|
383
423
|
return { failed: { testFile: known.test_file, command: known.command, output: run.output, why } }
|
|
384
424
|
}
|
|
385
425
|
|
|
426
|
+
type ChosenMode = "reuse" | "extend" | "sibling"
|
|
427
|
+
type Chosen = { mode: ChosenMode; file: string; test?: string; runnable?: RunnableTest; repo: string }
|
|
428
|
+
|
|
429
|
+
/**
|
|
430
|
+
* The existing test ForgeQE's agent chose: a test id from existing_tests_found (Class.Method) or a file. Reusing it
|
|
431
|
+
* needs a command: the one an earlier job ran it with, else one built the way the repository runs a single test.
|
|
432
|
+
*/
|
|
433
|
+
async function chosenTest(ref: string, mode: ChosenMode, ctx: ToolContext): Promise<Chosen> {
|
|
434
|
+
const repo = repoOf(ctx)
|
|
435
|
+
const graph = await buildGraph(repo).catch(() => null)
|
|
436
|
+
const tests = graph ? indexTests(graph).map((t) => t.entry) : []
|
|
437
|
+
const clean = ref.replace(/^\.\//, "")
|
|
438
|
+
const byId = tests.filter((t) => t.test === clean || t.test.endsWith(`.${clean}`))
|
|
439
|
+
const file = byId[0]?.file || clean
|
|
440
|
+
if (!existsSync(path.resolve(repo, file))) throw new Error(`existing_test ${ref} is not a test in ${repo}. Pass a test or test_file from existing_tests_found, or fresh: true.`)
|
|
441
|
+
const inFile = tests.filter((t) => t.file === file)
|
|
442
|
+
const test = byId[0]?.test || (inFile.length === 1 ? inFile[0]!.test : undefined)
|
|
443
|
+
if (mode !== "reuse") return { mode, file, ...(test ? { test } : {}), repo }
|
|
444
|
+
const known = ctx.config.memory.enabled ? writtenTestFor(ctx.config.memory.dir, repo, file) : null
|
|
445
|
+
// An earlier job's command runs the test it wrote; for another test in that file, a command is built.
|
|
446
|
+
const method = test?.split(".").pop() || ""
|
|
447
|
+
if (known?.command && (!test || inFile.length <= 1 || known.command.includes(method))) return { mode, file, ...(test ? { test } : {}), repo, runnable: { ...known, ...(test ? { test } : {}) } }
|
|
448
|
+
if (!test && inFile.length > 1) throw new Error(`${file} has ${inFile.length} tests: pass the one to reuse as existing_test (Class.Method, from existing_tests_found).`)
|
|
449
|
+
const knownCommands = ctx.config.memory.enabled ? writtenTests(ctx.config.memory.dir, repo).map((t) => t.command) : []
|
|
450
|
+
const command = test ? oneTestCommand(repo, { test, file }, knownCommands, walk(repo)) : null
|
|
451
|
+
if (!command || !test) throw new Error(`ForgeQE cannot tell how to run ${test || file} on its own: call browser_run with existing_mode extend or sibling to work from it, or write_test with mode reuse after the run.`)
|
|
452
|
+
return { mode, file, test, repo, runnable: { test_file: file, test, command, derived: true } }
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
/** Goals the existing-tests check already stopped once in this process: asked again, the browser runs. */
|
|
456
|
+
const gated = new Set<string>()
|
|
457
|
+
/** Existing tests ForgeQE's agent has been shown: fresh: true skips the check only for tests it has seen. */
|
|
458
|
+
const shown = new Set<string>()
|
|
459
|
+
|
|
460
|
+
/**
|
|
461
|
+
* Before the browser explores anything: the repository's tests closest to this goal, ranked from the code graph and
|
|
462
|
+
* page memory without a model (shortlistTests), with the tests earlier jobs wrote for close goals. ForgeQE's agent
|
|
463
|
+
* compares them with the request and picks: reuse one, extend one, add a test beside one, or run fresh for a new test
|
|
464
|
+
* that reuses the repository's code. Null when there are none, or this goal was already stopped once.
|
|
465
|
+
*/
|
|
466
|
+
/** A request sent as a task is shown the repository's tests only when they match it this closely. */
|
|
467
|
+
const TASK_FROM = 0.4
|
|
468
|
+
|
|
469
|
+
async function existingTestsFirst(goal: string, url: string | undefined, fresh: boolean, ctx: ToolContext, asTask = false): Promise<Record<string, unknown> | null> {
|
|
470
|
+
const cfg = ctx.config
|
|
471
|
+
if (!cfg.memory.enabled) return null
|
|
472
|
+
const repo = repoOf(ctx)
|
|
473
|
+
const key = `${repo}\n${goal.replace(/\s+/g, " ").trim().toLowerCase()}`
|
|
474
|
+
if (gated.has(key)) {
|
|
475
|
+
// Shown already: the browser runs, which is the agent's choice, made or not.
|
|
476
|
+
recordDecision(repo, { goal, stage: "chosen", where: "browser_run", by: "agent", mode: "fresh", why: fresh ? (asTask ? "a task, not a test" : "none of the tests shown is this journey") : "asked again without choosing" })
|
|
477
|
+
return null
|
|
478
|
+
}
|
|
479
|
+
const exists = (f: string) => existsSync(path.resolve(repo, f))
|
|
480
|
+
// fresh: true asks for a new run of a flow that has a test; for tests only close to it, before the agent has seen
|
|
481
|
+
// them, it is a guess, not a decision: it sees them first.
|
|
482
|
+
const written = findRelatedTests(cfg.memory.dir, repo, goal, exists).filter((t) => !fresh || !sameFlow(t.goal, goal))
|
|
483
|
+
const graph = await buildGraph(repo).catch(() => null)
|
|
484
|
+
const list = graph ? shortlistTests({ graph, pages: readPages(cfg.memory.dir), request: goal, url }) : null
|
|
485
|
+
type Shown = { test?: string; test_file: string; line?: number; lines?: number; tests_in_class?: number; tags?: string[]; steps?: string[]; covers?: string[]; lacks?: string[]; why?: string[]; score?: number; written_for?: string; command?: string; written?: string }
|
|
486
|
+
const out: Shown[] = (list?.candidates || []).filter((c) => !asTask || c.score >= TASK_FROM).map((c) => ({ test: c.test, test_file: c.file, line: c.line, lines: c.lines, tests_in_class: c.siblings + 1, tags: c.tags, steps: c.steps, covers: c.covers, lacks: c.lacks, why: c.why, score: c.score }))
|
|
487
|
+
// A test an earlier job wrote: its goal and command on the entry for its file, or an entry of its own.
|
|
488
|
+
for (const w of asTask ? written.filter((t) => t.similarity >= TASK_FROM) : written) {
|
|
489
|
+
const same = out.find((o) => o.test_file === w.test_file)
|
|
490
|
+
if (same) Object.assign(same, { written_for: w.goal, command: w.command, written: w.at.slice(0, 10) })
|
|
491
|
+
else out.push({ test_file: w.test_file, written_for: w.goal, command: w.command, written: w.at.slice(0, 10), score: Number(w.similarity.toFixed(2)) })
|
|
492
|
+
}
|
|
493
|
+
const keyOf = (o: Shown) => `${repo}\n${o.test_file}#${o.test || ""}`
|
|
494
|
+
if (!out.length) return null
|
|
495
|
+
if (fresh && out.every((o) => shown.has(keyOf(o)))) {
|
|
496
|
+
recordDecision(repo, { goal, stage: "chosen", where: "browser_run", by: "agent", mode: "fresh", why: asTask ? "a task, not a test" : "none of the tests shown is this journey" })
|
|
497
|
+
return null
|
|
498
|
+
}
|
|
499
|
+
gated.add(key)
|
|
500
|
+
recordDecision(repo, { goal, stage: "shown", where: "browser_run", ...(asTask ? { why: "sent as a task" } : {}), shown: out.map((o) => ({ ...(o.test ? { test: o.test } : {}), file: o.test_file, ...(o.score !== undefined ? { score: o.score } : {}), ...(o.covers ? { covers: o.covers, lacks: o.lacks } : {}) })) })
|
|
501
|
+
for (const o of out) shown.add(keyOf(o))
|
|
502
|
+
const body = out.map((o) => {
|
|
503
|
+
const what = o.covers?.length || o.lacks?.length ? ` covers ${o.covers!.length} of ${o.covers!.length + o.lacks!.length}${o.lacks!.length ? `; lacks ${o.lacks!.slice(0, 3).join(", ")}${o.lacks!.length > 3 ? ` +${o.lacks!.length - 3}` : ""}` : ""}` : ""
|
|
504
|
+
return `${o.test || o.test_file}${o.line ? ` (${o.test_file}:${o.line}, ${o.lines} lines)` : ""}${what}${o.written_for ? ` · written for "${o.written_for.replace(/\s+/g, " ").slice(0, 80)}"` : ""}`
|
|
505
|
+
})
|
|
506
|
+
ctx.emit?.({
|
|
507
|
+
type: "card",
|
|
508
|
+
card: {
|
|
509
|
+
status: "info",
|
|
510
|
+
title: `${out.length === 1 ? "An existing test is" : `${out.length} existing tests are`} close to this flow`,
|
|
511
|
+
stats: `no browser run yet: reuse, extend, or build on them before exploring${list ? ` · ${list.tests} tests searched in ${list.ms}ms` : ""}`,
|
|
512
|
+
body: [...body, ...(list?.unlocated.length ? [`not in any page object yet: ${list.unlocated.join(", ")}`] : [])],
|
|
513
|
+
},
|
|
514
|
+
})
|
|
515
|
+
return {
|
|
516
|
+
status: "existing_tests_found",
|
|
517
|
+
browser: "not started",
|
|
518
|
+
searched: list ? `${list.tests} tests, ranked from the code graph and page memory` : "the tests earlier jobs wrote",
|
|
519
|
+
...(list?.elements.length ? { request_elements: list.elements } : {}),
|
|
520
|
+
...(list?.unlocated.length ? { not_in_page_objects: list.unlocated } : {}),
|
|
521
|
+
existing_tests: out,
|
|
522
|
+
...(asTask
|
|
523
|
+
? {
|
|
524
|
+
sent_as: "task",
|
|
525
|
+
note: "This was sent as a task, but the repository has tests close to it. When one of them does what the request asks, answer the check with it: call browser_run again with the same goal and purpose, existing_test set to that test, and existing_mode reuse. ForgeQE runs it, and the browser runs only when it fails; nothing is written. When the user asked for a test, call browser_run with purpose test and one of the choices in next_step. Only when no test does what it asks, or it needs data a passing test does not give (a value to read or report), call it again with fresh: true.",
|
|
526
|
+
}
|
|
527
|
+
: {}),
|
|
528
|
+
next_step:
|
|
529
|
+
"Before any browser run, compare each test's steps, covers, lacks, and written_for with the user's request, then call browser_run again with the same goal and one of these: existing_test (its test, or test_file) with existing_mode reuse, when it already does and checks everything the request asks (ForgeQE runs it; no browser run when it passes); existing_mode extend, when it covers the start of the journey and the request adds a few steps or checks to it; existing_mode sibling, when the request is a variant of it (another path, other data, or more than a few added steps, or the test is long already): a new test beside it in its class, reusing its setup and steps; or fresh: true, when none of them is this journey: a new test that still reuses the repository's page objects and helpers.",
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
|
|
386
533
|
export const TOOLS: McpTool[] = [
|
|
387
534
|
{
|
|
388
535
|
name: "browser_run",
|
|
@@ -419,7 +566,17 @@ export const TOOLS: McpTool[] = [
|
|
|
419
566
|
fresh: {
|
|
420
567
|
type: "boolean",
|
|
421
568
|
description:
|
|
422
|
-
"true: run in the browser even when an earlier job wrote a test for this goal. Only when the user asks for a new browser run or a new test; by default an existing test is run first and reused when it passes.",
|
|
569
|
+
"true: run in the browser even when an earlier job wrote a test for this goal or one close to it. Only when the user asks for a new browser run or a new test, or when existing_tests_found listed none that is this journey; by default an existing test is run first and reused when it passes.",
|
|
570
|
+
},
|
|
571
|
+
existing_test: {
|
|
572
|
+
type: "string",
|
|
573
|
+
description: "A test (Class.Method) or test_file from existing_tests_found, or one the user named, to work from, with existing_mode.",
|
|
574
|
+
},
|
|
575
|
+
existing_mode: {
|
|
576
|
+
type: "string",
|
|
577
|
+
enum: ["reuse", "extend", "sibling"],
|
|
578
|
+
description:
|
|
579
|
+
"With existing_test. reuse: the test already does and checks everything the request asks; ForgeQE runs it, and the browser runs only when it fails. extend: the request adds a few steps or checks to that test's journey; the browser runs the goal and write_test extends that test. sibling: the request is a variant of that test; the browser runs the goal and write_test adds a test beside it in its class, reusing its setup and steps.",
|
|
423
580
|
},
|
|
424
581
|
},
|
|
425
582
|
required: ["goal", "purpose"],
|
|
@@ -439,8 +596,30 @@ export const TOOLS: McpTool[] = [
|
|
|
439
596
|
const emit = ctx.emit || (() => {})
|
|
440
597
|
// The same flow asked again: an earlier job's test for it runs first. Passing in a browser, it answers the goal
|
|
441
598
|
// in seconds, with no browser run and no new test; failing, the run goes ahead and Scribe updates that test.
|
|
442
|
-
const
|
|
599
|
+
const mayReuse = ctx.writeTests && args.purpose !== "task" && !resumeFrom && devices.length <= 1
|
|
600
|
+
// Sent as a task, but close to the repository's tests: the agent may have taken a test of the app for a lookup.
|
|
601
|
+
// A check of the application, answered by running the existing test that does it: no browser run when it passes.
|
|
602
|
+
const checkByTest = ctx.writeTests && args.purpose === "task" && !resumeFrom && devices.length <= 1
|
|
603
|
+
if (checkByTest && !args.existing_test) {
|
|
604
|
+
const found = await existingTestsFirst(goal, url, Boolean(args.fresh), ctx, true)
|
|
605
|
+
if (found) return found
|
|
606
|
+
}
|
|
607
|
+
const mode: ChosenMode = args.existing_mode === "extend" || args.existing_mode === "sibling" ? args.existing_mode : "reuse"
|
|
608
|
+
if (checkByTest && args.existing_test && mode !== "reuse") throw new Error(`existing_mode ${mode} writes a test: call browser_run with purpose test for it, or existing_mode reuse to answer the check with that test.`)
|
|
609
|
+
const chosen = (mayReuse || checkByTest) && args.existing_test ? await chosenTest(String(args.existing_test), mode, ctx) : null
|
|
610
|
+
let reuse: ExistingFirst | null = null
|
|
611
|
+
if (chosen) recordDecision(chosen.repo, { goal, stage: "chosen", where: "browser_run", by: "agent", mode: chosen.mode, file: chosen.file, ...(chosen.test ? { test: chosen.test } : {}) })
|
|
612
|
+
if (chosen?.runnable) reuse = await runKnownTest(chosen.runnable, chosen.repo, runDir, ctx, goal, checkByTest)
|
|
613
|
+
else if (mayReuse && !chosen) {
|
|
614
|
+
if (!args.fresh) reuse = await existingTestFirst(goal, runDir, ctx)
|
|
615
|
+
// Not the same flow, but close to existing tests: decide reuse, extend, or build on one before the browser explores.
|
|
616
|
+
if (!reuse) {
|
|
617
|
+
const found = await existingTestsFirst(goal, url, Boolean(args.fresh), ctx)
|
|
618
|
+
if (found) return found
|
|
619
|
+
}
|
|
620
|
+
}
|
|
443
621
|
if (reuse?.passed) return reuse.passed
|
|
622
|
+
const extend = chosen && chosen.mode !== "reuse" ? chosen : undefined
|
|
444
623
|
const t0 = Date.now()
|
|
445
624
|
emit({ type: "run-start", goal, url })
|
|
446
625
|
try {
|
|
@@ -470,10 +649,15 @@ export const TOOLS: McpTool[] = [
|
|
|
470
649
|
}
|
|
471
650
|
if (ctx.writeTests && result.status === "complete" && result.driver !== "synthetic") {
|
|
472
651
|
if (reuse?.failed) writeFileSync(path.join(runDir, EXISTING_TEST_FILE), `${JSON.stringify(reuse.failed, null, 2)}\n`)
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
652
|
+
else if (extend) writeFileSync(path.join(runDir, EXTEND_TEST_FILE), `${JSON.stringify({ testFile: extend.file, ...(extend.test ? { test: extend.test } : {}), mode: extend.mode }, null, 2)}\n`)
|
|
653
|
+
const how = reuse?.failed
|
|
654
|
+
? ` It updates the existing test ${reuse.failed.testFile} instead of writing a new one.`
|
|
655
|
+
: extend?.mode === "sibling"
|
|
656
|
+
? ` It adds a test beside ${extend.test || "the existing test"} in ${extend.file}, reusing its setup and steps, instead of writing a new file.`
|
|
657
|
+
: extend
|
|
658
|
+
? ` It extends the existing test ${extend.file} instead of writing a new one.`
|
|
659
|
+
: ""
|
|
660
|
+
return { ...summary, next_step: `This session turns passed runs into tests: call write_test with run_dir ${runDir} now, before you reply.${how}` }
|
|
477
661
|
}
|
|
478
662
|
return summary
|
|
479
663
|
} catch (err) {
|
|
@@ -507,7 +691,7 @@ export const TOOLS: McpTool[] = [
|
|
|
507
691
|
{
|
|
508
692
|
name: "save_test_cases",
|
|
509
693
|
description:
|
|
510
|
-
"Save test cases you wrote from a story into Azure DevOps (Test Case work items with steps and expected results, linked to the story as Tests / Tested By, in the story's area and iteration) or Jira (issues of type Test by default, steps in the description, linked to the story).
|
|
694
|
+
"Save test cases you wrote from a story into Azure DevOps (Test Case work items with steps and expected results, linked to the story as Tests / Tested By, in the story's area and iteration) or Jira (issues of type Test by default, steps in the description, linked to the story). Never duplicates: each case is compared with the cases already linked to the story. Same steps (under any title): unchanged, nothing written. Same title, other steps: differs, nothing written; propose the new steps with update_test_case. A close title or mostly the same steps: similar, nothing written; update that case, or save again with the title in force_new when it is really a different case. Returns each case's status, id, and link, and shows them as a card.",
|
|
511
695
|
inputSchema: {
|
|
512
696
|
type: "object",
|
|
513
697
|
properties: {
|
|
@@ -531,6 +715,7 @@ export const TOOLS: McpTool[] = [
|
|
|
531
715
|
},
|
|
532
716
|
issue_type: { type: "string", description: "Jira only: issue type for the cases (default Test)" },
|
|
533
717
|
link_type: { type: "string", description: "Jira only: link type to the story (default Relates)" },
|
|
718
|
+
force_new: { type: "array", items: { type: "string" }, description: "Titles to create although a similar case is linked, only after the user said they are different cases" },
|
|
534
719
|
},
|
|
535
720
|
required: ["system", "story", "cases"],
|
|
536
721
|
},
|
|
@@ -539,29 +724,55 @@ export const TOOLS: McpTool[] = [
|
|
|
539
724
|
if (!cases.length) throw new Error("cases is empty.")
|
|
540
725
|
const story = String(args.story || "").trim()
|
|
541
726
|
if (!story) throw new Error("story is required.")
|
|
727
|
+
const forceNew = Array.isArray(args.force_new) ? (args.force_new as unknown[]).map(String) : undefined
|
|
542
728
|
const saved =
|
|
543
729
|
args.system === "jira"
|
|
544
|
-
? await saveJiraCases(ctx.config.sources, story, cases, { issueType: args.issue_type ? String(args.issue_type) : undefined, linkType: args.link_type ? String(args.link_type) : undefined })
|
|
545
|
-
: await saveAdoCases(ctx.config.sources, Number(story.replace(/^#/, "")), cases)
|
|
546
|
-
const
|
|
547
|
-
const
|
|
730
|
+
? await saveJiraCases(ctx.config.sources, story, cases, { issueType: args.issue_type ? String(args.issue_type) : undefined, linkType: args.link_type ? String(args.link_type) : undefined, forceNew })
|
|
731
|
+
: await saveAdoCases(ctx.config.sources, Number(story.replace(/^#/, "")), cases, { forceNew })
|
|
732
|
+
const count = (st: SaveStatus) => saved.cases.filter((c) => c.status === st).length
|
|
733
|
+
const MARKS: Record<SaveStatus, string> = { created: "+", unchanged: "=", differs: "~", similar: "?", failed: "✗" }
|
|
734
|
+
const failed = count("failed")
|
|
548
735
|
ctx.emit?.({
|
|
549
736
|
type: "card",
|
|
550
737
|
card: {
|
|
551
|
-
status: failed ? "fail" : "ok",
|
|
738
|
+
status: failed ? "fail" : count("differs") || count("similar") ? "info" : "ok",
|
|
552
739
|
title: `Test cases saved for ${saved.story.id}${saved.story.title ? ` ${saved.story.title}` : ""}`,
|
|
553
|
-
stats: [`${
|
|
740
|
+
stats: [`${count("created")} created`, count("unchanged") ? `${count("unchanged")} already linked` : "", count("differs") ? `${count("differs")} differ from the linked case` : "", count("similar") ? `${count("similar")} like a linked case` : "", failed ? `${failed} failed` : ""].filter(Boolean).join(" · "),
|
|
554
741
|
body: [
|
|
555
742
|
`Story ${saved.story.url}`,
|
|
556
743
|
"",
|
|
557
|
-
...saved.cases.flatMap((c) =>
|
|
744
|
+
...saved.cases.flatMap((c) =>
|
|
745
|
+
c.id
|
|
746
|
+
? [
|
|
747
|
+
`${MARKS[c.status]} ${c.id} ${c.title}`,
|
|
748
|
+
...(c.existing && c.existing.title !== c.title ? [` linked as "${c.existing.title}"`] : []),
|
|
749
|
+
` ${c.url}`,
|
|
750
|
+
...(c.error ? [` ${c.error}`] : []),
|
|
751
|
+
]
|
|
752
|
+
: [`✗ ${c.title}: ${c.error}`],
|
|
753
|
+
),
|
|
558
754
|
],
|
|
559
755
|
},
|
|
560
756
|
})
|
|
757
|
+
const show = (steps: NewCase["steps"]) => steps.map((st, i) => `${i + 1}. ${st.action}${st.expected ? ` → ${st.expected}` : ""}`)
|
|
758
|
+
const pending = saved.cases.filter((c) => c.status === "differs" || c.status === "similar")
|
|
561
759
|
return {
|
|
562
760
|
story: saved.story,
|
|
563
|
-
cases: saved.cases.map((c) => ({
|
|
564
|
-
|
|
761
|
+
cases: saved.cases.map((c) => ({
|
|
762
|
+
title: c.title,
|
|
763
|
+
status: c.status,
|
|
764
|
+
id: c.id || undefined,
|
|
765
|
+
url: c.url || undefined,
|
|
766
|
+
...(c.existing ? { linked_case: { id: c.existing.id, title: c.existing.title, steps: show(c.existing.steps) } } : {}),
|
|
767
|
+
error: c.error,
|
|
768
|
+
})),
|
|
769
|
+
...(saved.system === "azure_devops" && saved.cases.some((c) => c.id) ? { run_with: { source: "azure_devops", ids: [...new Set(saved.cases.filter((c) => c.id).map((c) => c.id))] } } : {}),
|
|
770
|
+
...(pending.length
|
|
771
|
+
? {
|
|
772
|
+
next_step:
|
|
773
|
+
"Nothing was written for the cases with status differs or similar. Do not say they were saved or updated. For each one, show the user how it differs from linked_case and offer update_test_case with the steps you wrote (id: linked_case.id). Save again with the title in force_new only when the user says it is a different case.",
|
|
774
|
+
}
|
|
775
|
+
: {}),
|
|
565
776
|
}
|
|
566
777
|
},
|
|
567
778
|
},
|
|
@@ -918,6 +1129,7 @@ export const TOOLS: McpTool[] = [
|
|
|
918
1129
|
...COMMAND_TOOLS,
|
|
919
1130
|
...REPO_TEST_TOOLS,
|
|
920
1131
|
...DEFECT_TOOLS,
|
|
1132
|
+
...PACK_TOOLS,
|
|
921
1133
|
...PR_TOOLS,
|
|
922
1134
|
...CASE_TOOLS,
|
|
923
1135
|
...API_TOOLS,
|