@capagents/forgeqe 1.1.3 → 1.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/HOW_TO_USE.md +46 -4
  2. package/README.md +7 -5
  3. package/browserstack.example.yaml +15 -3
  4. package/docs/user-guide.html +62 -10
  5. package/forgeqe.example.yaml +25 -18
  6. package/package.json +2 -2
  7. package/prompts/README.md +1 -0
  8. package/prompts/brain.md +7 -1
  9. package/prompts/defect-rca.md +43 -0
  10. package/prompts/test-agent.md +12 -2
  11. package/prompts/test-reviewer.md +5 -2
  12. package/prompts/test-rules.md +8 -4
  13. package/py/forgeqe_engine/browser.py +79 -12
  14. package/py/forgeqe_engine/service.py +12 -0
  15. package/skills/dotnet-nunit-playwright/SKILL.md +2 -1
  16. package/skills/playwright-typescript/SKILL.md +2 -1
  17. package/src/bi.ts +382 -302
  18. package/src/brain/index.ts +1 -1
  19. package/src/capabilities/guide.ts +3 -3
  20. package/src/capabilities/registry.ts +34 -6
  21. package/src/cases/bugs.ts +209 -22
  22. package/src/cases/defectHistory.ts +193 -0
  23. package/src/cases/defects.ts +255 -0
  24. package/src/cases/github.ts +162 -0
  25. package/src/cases/goal.ts +1 -1
  26. package/src/cases/index.ts +23 -4
  27. package/src/cases/predict.ts +372 -0
  28. package/src/cases/rca.ts +258 -0
  29. package/src/cases/readiness.ts +38 -1
  30. package/src/cases/repro.ts +202 -0
  31. package/src/cases/types.ts +8 -1
  32. package/src/cli/help.ts +16 -7
  33. package/src/cli/index.ts +56 -15
  34. package/src/code/graph/graph.ts +511 -0
  35. package/src/code/graph/locators.ts +398 -0
  36. package/src/code/graph/parse.ts +439 -0
  37. package/src/code/graph/tools.ts +113 -0
  38. package/src/code/handoff.ts +93 -18
  39. package/src/code/loop.ts +30 -5
  40. package/src/code/profile.ts +14 -3
  41. package/src/code/replay.ts +1 -1
  42. package/src/code/review.ts +22 -0
  43. package/src/config.ts +160 -56
  44. package/src/mcp/attachtools.ts +223 -0
  45. package/src/mcp/commands.ts +1 -1
  46. package/src/mcp/defecttools.ts +106 -0
  47. package/src/mcp/devicetools.ts +125 -0
  48. package/src/mcp/external.ts +42 -1
  49. package/src/mcp/tools.ts +92 -9
  50. package/src/monitor/run.ts +1 -1
  51. package/src/report/data.ts +14 -0
  52. package/src/report/html.ts +15 -3
  53. package/src/report/template.html +44 -43
  54. package/src/run.ts +9 -2
  55. package/src/suite/report.ts +44 -11
  56. package/src/suite/run.ts +163 -16
  57. package/src/tui/App.tsx +9 -3
  58. package/src/usage.ts +1 -1
  59. package/testmu.example.yaml +14 -4
  60. package/src/cache.ts +0 -92
package/src/config.ts CHANGED
@@ -38,6 +38,13 @@ export type BrowserRemoteConfig = {
38
38
  extra?: Record<string, unknown>
39
39
  }
40
40
 
41
+ /** A named remote browser (devices: in a provider file): the provider's shared remote settings with its own on top. */
42
+ export type BrowserDevice = {
43
+ name: string
44
+ provider: RemoteBrowserProvider
45
+ remote: BrowserRemoteConfig
46
+ }
47
+
41
48
  export type BrowserConfig = {
42
49
  provider: BrowserProvider
43
50
  headless: boolean
@@ -51,6 +58,8 @@ export type BrowserConfig = {
51
58
  remote?: BrowserRemoteConfig
52
59
  remotes: BrowserRemoteConfigs
53
60
  providerFiles: BrowserProviderFiles
61
+ /** Named devices from the provider files, by name; names are unique across providers. */
62
+ devices: Record<string, BrowserDevice>
54
63
  /** ForgeQE overlay in a visible browser: edge glow, a bar with the current step, the person's input locked. */
55
64
  overlay: boolean
56
65
  /** A second model call checks the finished run against the goal, and its verdict can fail a run the agent completed. */
@@ -63,20 +72,6 @@ export type MemoryConfig = {
63
72
  replay: ReplayMode
64
73
  }
65
74
 
66
- export type CacheConfig = {
67
- enabled: boolean
68
- /** Redis connection string, e.g. redis://127.0.0.1:6379. */
69
- redisUrl: string
70
- /** Prefix for every Redis key this ForgeQE workspace writes. */
71
- keyPrefix: string
72
- /** Default TTL for cached JSON payloads, in seconds. */
73
- defaultTtlSeconds: number
74
- /** TTL for BI external dataset exports, in seconds. */
75
- biTtlSeconds: number
76
- /** Socket connect timeout for Redis, in milliseconds. */
77
- connectTimeoutMs: number
78
- }
79
-
80
75
  function runtimeBin(value: unknown, base: string): string | undefined {
81
76
  if (!value) return undefined
82
77
  const s = String(value)
@@ -92,6 +87,8 @@ export type CodeConfig = {
92
87
  stallAfter?: number
93
88
  /** Give Scribe a Playwright-driven browser, set up like the run's, switched on once a step fails. */
94
89
  liveBrowser?: boolean
90
+ /** Give Scribe the code graph of the repository (code_find, code_outline, code_element, …) and link the handoff's elements to the page objects that locate them (code.graph, default true). */
91
+ graph?: boolean
95
92
  /** Before Scribe starts, replay the run's steps as a Playwright script and hand over what worked and where it failed. */
96
93
  replay?: boolean
97
94
  /** Model for ForgeQE's agent and Scribe as provider/model, instead of the run's llms.json profile. */
@@ -136,11 +133,16 @@ export type SourcesConfig = {
136
133
  qtest?: { url: string; tokenEnv: string; projectId: number; statuses: { passed: string; failed: string; error: string } }
137
134
  /** File a bug for each failed case, or comment on the open one ForgeQE filed before. */
138
135
  bugs?: BugsConfig
136
+ /**
137
+ * GitHub: issues as test cases and bugs, and GitHub's MCP server for ForgeQE's agent. `repo` (owner/name) is the default
138
+ * for issue references; the token is `tokenEnv`, else the GitHub CLI's sign-in. `toolsets`: the MCP server's tool groups.
139
+ */
140
+ github?: { repo?: string; apiUrl: string; tokenEnv: string; mcp: boolean; toolsets: string[] }
139
141
  }
140
142
 
141
143
  export type BugsConfig = {
142
- system: "jira" | "azure_devops"
143
- /** Jira project key. Azure DevOps uses this, or sources.azure_devops.project. */
144
+ system: "jira" | "azure_devops" | "github"
145
+ /** Jira project key; GitHub owner/repo (default sources.github.repo). Azure DevOps uses this, or sources.azure_devops.project. */
144
146
  project?: string
145
147
  /** Jira issue type or Azure DevOps work item type. */
146
148
  type: string
@@ -149,6 +151,11 @@ export type BugsConfig = {
149
151
  areaPath?: string
150
152
  /** Also file bugs for cases that could not run (status error). */
151
153
  onError: boolean
154
+ /**
155
+ * "defect" (the default): one bug per defect, so cases failing the same way (the same failed request, the same
156
+ * uncaught error) share a bug, each added to it. "case": one bug per failing case.
157
+ */
158
+ groupBy?: "defect" | "case"
152
159
  }
153
160
 
154
161
  /** A site the browser agent can sign in to. Values come from the environment; the agent only sees placeholders. */
@@ -321,6 +328,9 @@ function notifyConfig(raw: unknown): NotifyConfig | undefined {
321
328
  if (unknown.length) throw new Error(`${where}.on: ${unknown.join(", ")} is not an event. Events: ${NOTIFY_EVENTS.join(", ")}.`)
322
329
  return { name, ...alerts, on: on as NotifyEvent[] }
323
330
  })
331
+ // No channels set up, but a Teams webhook in the environment: a "teams" channel for messages the user asks for. It
332
+ // takes no events (on: []), so runs nobody watches never start posting to it unasked.
333
+ if (!channels.length && process.env.TEAMS_WEBHOOK_URL?.trim()) channels.push({ name: "teams", teams: { webhookEnv: "TEAMS_WEBHOOK_URL" }, on: [] })
324
334
  if (!channels.length) return undefined
325
335
  return { channels, ...(n.report_url ? { reportUrl: trimUrl(n.report_url) } : {}) }
326
336
  }
@@ -348,8 +358,6 @@ export type AppConfig = {
348
358
  configPath?: string
349
359
  /** `mcp:` in forgeqe.yaml: more MCP servers for ForgeQE's agent, by name. */
350
360
  mcp?: Record<string, ExtraMcpServer>
351
- /** Optional Redis-backed cache for deterministic external data fetches. */
352
- cache?: CacheConfig
353
361
  activeProfile: string
354
362
  profiles: Record<string, LlmProfile>
355
363
  llm: LlmProfile
@@ -361,6 +369,8 @@ export type AppConfig = {
361
369
  prompts: { stepInstruction: string; guiderails: string; dirs?: { dir: string; source: string }[] }
362
370
  /** Product knowledge (.forgeqe/knowledge): see src/knowledge.ts. */
363
371
  knowledge?: { dir: string; browserMaxChars: number }
372
+ /** Defect intelligence (defects: in forgeqe.yaml): see src/cases/defects.ts, repro.ts, rca.ts. */
373
+ defects: DefectsConfig
364
374
  export: { outDir: string; format: string }
365
375
  memory: MemoryConfig
366
376
  code: CodeConfig
@@ -410,22 +420,6 @@ function domainConfig(raw: unknown): DomainConfig | undefined {
410
420
  return { industry, regulations, ...(d.notes ? { notes: String(d.notes).trim() } : {}) }
411
421
  }
412
422
 
413
- function cacheConfig(raw: unknown): CacheConfig | undefined {
414
- const c = asRecord(raw)
415
- const asked = raw !== undefined || Boolean(process.env.FORGEQE_REDIS_URL)
416
- if (!asked || c.enabled === false) return undefined
417
- const redisUrl = String(c.redis_url || c.url || process.env.FORGEQE_REDIS_URL || "").trim()
418
- if (!redisUrl) throw new Error('cache.redis_url is required when cache is enabled. Example: cache: { redis_url: "redis://127.0.0.1:6379" }')
419
- return {
420
- enabled: true,
421
- redisUrl,
422
- keyPrefix: String(c.key_prefix || "forgeqe").trim() || "forgeqe",
423
- defaultTtlSeconds: positiveInt(c.default_ttl_seconds, 3600, "cache.default_ttl_seconds"),
424
- biTtlSeconds: positiveInt(c.bi_ttl_seconds, 900, "cache.bi_ttl_seconds"),
425
- connectTimeoutMs: positiveInt(c.connect_timeout_ms, 3000, "cache.connect_timeout_ms"),
426
- }
427
- }
428
-
429
423
  export function replayMode(raw: unknown): ReplayMode {
430
424
  const mode = String(raw || "verify").toLowerCase()
431
425
  if (mode === "trust" || mode === "off") return mode
@@ -471,12 +465,12 @@ const REMOTE_BROWSER_DEFAULTS: Record<RemoteBrowserProvider, Pick<BrowserRemoteC
471
465
  browserstack: {
472
466
  usernameEnv: "BROWSERSTACK_USERNAME",
473
467
  accessKeyEnv: "BROWSERSTACK_ACCESS_KEY",
474
- endpoint: "wss://cdp.browserstack.com/playwright",
468
+ endpoint: "wss://cdp.browserstack.com/puppeteer",
475
469
  },
476
470
  testmu: {
477
471
  usernameEnv: "LT_USERNAME",
478
472
  accessKeyEnv: "LT_ACCESS_KEY",
479
- endpoint: "wss://cdp.lambdatest.com/playwright",
473
+ endpoint: "wss://cdp.lambdatest.com/puppeteer",
480
474
  },
481
475
  }
482
476
 
@@ -545,9 +539,20 @@ function browserRemoteConfig(raw: Record<string, unknown>, provider: BrowserProv
545
539
  const defaults = REMOTE_BROWSER_DEFAULTS[provider]
546
540
  const browser = String(remote.browser || "").trim()
547
541
  if (!browser) throw new Error(`${where}.remote.browser is required when browser.provider is ${provider}.`)
542
+ // The engine drives the browser over the Chrome DevTools Protocol, so the grid must hand back a Chromium browser.
543
+ if (!/chrome|edge|chromium/i.test(browser) || /^(playwright|pw)-/i.test(browser)) {
544
+ throw new Error(`${where}.remote.browser is "${browser}": ForgeQE drives remote browsers over the Chrome DevTools Protocol, so use chrome or edge.`)
545
+ }
546
+ // Checked live: TestMu AI's CDP endpoint opens Chrome sessions only; an Edge session never opens and times out.
547
+ if (provider === "testmu" && !/chrome/i.test(browser)) {
548
+ throw new Error(`${where}.remote.browser is "${browser}": TestMu AI's CDP endpoint only serves Chrome, so use Chrome there (Edge works on BrowserStack).`)
549
+ }
548
550
  const browserVersion = String(remote.browser_version || remote.browserVersion || "latest").trim() || "latest"
549
- const os = String(remote.os || "").trim() || undefined
550
- const osVersion = String(remote.os_version || remote.osVersion || "").trim() || undefined
551
+ const rawOs = String(remote.os || "").trim() || undefined
552
+ const rawOsVersion = String(remote.os_version || remote.osVersion || "").trim() || undefined
553
+ // BrowserStack documents "Windows" and "OS X" with versions like 11 or Sequoia: what people write maps onto those.
554
+ const os = provider === "browserstack" && rawOs ? browserstackOs(rawOs) : rawOs
555
+ const osVersion = provider === "browserstack" && rawOsVersion && os === "OS X" ? titleCase(rawOsVersion) : rawOsVersion
551
556
  const platform = String(remote.platform || "").trim() || undefined
552
557
  if (provider === "browserstack" && (!os || !osVersion)) {
553
558
  throw new Error(`${where}.remote.os and ${where}.remote.os_version are required when browser.provider is browserstack.`)
@@ -555,12 +560,16 @@ function browserRemoteConfig(raw: Record<string, unknown>, provider: BrowserProv
555
560
  if (provider === "testmu" && !platform && !os) {
556
561
  throw new Error(`${where}.remote.platform is required when browser.provider is testmu, or set ${where}.remote.os (and optionally os_version).`)
557
562
  }
563
+ const endpoint = String(remote.endpoint || defaults.endpoint).trim().replace(/\/+$/, "")
564
+ if (/\/playwright$/i.test(endpoint)) {
565
+ throw new Error(`${where}.remote.endpoint ${endpoint} speaks Playwright's own protocol; ForgeQE needs the CDP endpoint (${defaults.endpoint}).`)
566
+ }
558
567
  const extra = objectMap(remote.extra ?? remote.capabilities, `${where}.remote.extra`)
559
568
  return {
560
569
  provider,
561
570
  usernameEnv: String(remote.username_env || remote.usernameEnv || defaults.usernameEnv),
562
571
  accessKeyEnv: String(remote.access_key_env || remote.accessKeyEnv || defaults.accessKeyEnv),
563
- endpoint: String(remote.endpoint || defaults.endpoint).trim().replace(/\/+$/, ""),
572
+ endpoint,
564
573
  browser,
565
574
  browserVersion,
566
575
  ...(os ? { os } : {}),
@@ -574,28 +583,68 @@ function browserRemoteConfig(raw: Record<string, unknown>, provider: BrowserProv
574
583
  }
575
584
  }
576
585
 
577
- function browserRemoteConfigs(browser: Record<string, unknown>, provider: BrowserProvider, base: string): { remotes: BrowserRemoteConfigs; providerFiles: BrowserProviderFiles } {
586
+ function browserstackOs(os: string): string {
587
+ if (/^(os ?x|macos|mac( ?os)?)$/i.test(os)) return "OS X"
588
+ if (/^win(dows)?$/i.test(os)) return "Windows"
589
+ return os
590
+ }
591
+
592
+ const titleCase = (s: string) => s.toLowerCase().replace(/\b[a-z]/g, (c) => c.toUpperCase())
593
+
594
+ /** A device name: what the user and the agent call it ("win11-chrome"), also used in folder names. */
595
+ export const DEVICE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/
596
+
597
+ function browserRemoteConfigs(
598
+ browser: Record<string, unknown>,
599
+ provider: BrowserProvider,
600
+ base: string,
601
+ ): { remotes: BrowserRemoteConfigs; providerFiles: BrowserProviderFiles; devices: Record<string, BrowserDevice> } {
578
602
  const providerFiles = browserProviderFiles(browser, base)
579
603
  const remotes: BrowserRemoteConfigs = {}
604
+ const devices: Record<string, BrowserDevice> = {}
580
605
  for (const remoteProvider of REMOTE_BROWSER_PROVIDERS) {
581
606
  const fromFile = providerFiles[remoteProvider] ? browserProviderFileSection(providerFiles[remoteProvider]!, remoteProvider) : undefined
582
607
  const inline = provider === remoteProvider ? browser : undefined
583
608
  if (!fromFile && !inline) continue
584
609
  const where = fromFile ? `browser.providers.${remoteProvider}` : "browser"
585
- remotes[remoteProvider] = browserRemoteConfig(
586
- {
587
- ...(fromFile || {}),
588
- ...(inline || {}),
589
- remote: { ...asRecord(fromFile?.remote), ...asRecord(inline?.remote) },
590
- },
591
- remoteProvider,
592
- where,
593
- )
610
+ const shared = { ...asRecord(fromFile?.remote), ...asRecord(inline?.remote) }
611
+ const named = objectMap(fromFile?.devices, `${where}.devices`) || {}
612
+ for (const [name, raw] of Object.entries(named)) {
613
+ if (!DEVICE_NAME.test(name)) throw new Error(`${where}.devices.${name}: a device name uses letters, digits, ".", "_" and "-".`)
614
+ const other = devices[name]
615
+ if (other) throw new Error(`Device ${name} is in both ${other.provider} and ${remoteProvider}; device names must be unique.`)
616
+ const remote = browserRemoteConfig({ remote: { ...shared, ...asRecord(raw) } }, remoteProvider, `${where}.devices.${name}`)!
617
+ devices[name] = { name, provider: remoteProvider, remote }
618
+ }
619
+ // A provider file that only lists devices: its first device is the provider's default browser.
620
+ const first = Object.values(devices).find((d) => d.provider === remoteProvider)
621
+ remotes[remoteProvider] =
622
+ !String(shared.browser || "").trim() && first
623
+ ? first.remote
624
+ : browserRemoteConfig({ ...(fromFile || {}), ...(inline || {}), remote: shared }, remoteProvider, where)
594
625
  }
595
- return { remotes, providerFiles }
626
+ return { remotes, providerFiles, devices }
596
627
  }
597
628
 
598
- export function browserConfigFor(config: Pick<AppConfig, "browser">, provider = config.browser.provider): Pick<BrowserConfig, "provider" | "remote"> {
629
+ /**
630
+ * The browser a run uses: a named device (which also picks its provider), else the provider's default remote,
631
+ * else the local browser. A device on another provider than the one asked for is an error, not a silent switch.
632
+ */
633
+ export function browserConfigFor(
634
+ config: Pick<AppConfig, "browser">,
635
+ provider?: BrowserProvider,
636
+ device?: string,
637
+ ): Pick<BrowserConfig, "provider" | "remote"> & { device?: string } {
638
+ if (device) {
639
+ const d = config.browser.devices?.[device]
640
+ if (!d) {
641
+ const known = Object.keys(config.browser.devices || {})
642
+ throw new Error(`No browser device named ${device}. ${known.length ? `Configured: ${known.join(", ")}.` : "None is configured yet."} Add one with browser_devices, or under devices: in the provider's YAML file.`)
643
+ }
644
+ if (provider && provider !== d.provider) throw new Error(`Device ${device} is on ${d.provider}, not ${provider}.`)
645
+ // The session name shows the device in the provider's dashboard.
646
+ return { provider: d.provider, remote: { ...d.remote, name: [d.remote.name, device].filter(Boolean).join(" · ") }, device }
647
+ }
599
648
  const chosen = provider || config.browser.provider || "local"
600
649
  if (chosen === "local") return { provider: chosen }
601
650
  const remote = config.browser.remotes?.[chosen] || ((config.browser.provider === chosen || !config.browser.provider) ? config.browser.remote : undefined)
@@ -775,11 +824,10 @@ export function loadConfig(opts?: { configPath?: string; profile?: string }): Ap
775
824
 
776
825
  const sessionsDir = configPath ? path.join(projectDir(base), "sessions") : path.join(forgeqeHome(), "sessions")
777
826
  const provider = browserProvider(browser.provider)
778
- const { remotes, providerFiles } = browserRemoteConfigs(browser, provider, base)
827
+ const { remotes, providerFiles, devices } = browserRemoteConfigs(browser, provider, base)
779
828
  const config: AppConfig = {
780
829
  configPath,
781
830
  mcp: extraMcpServers(yaml.mcp, base),
782
- cache: cacheConfig(yaml.cache),
783
831
  activeProfile,
784
832
  profiles,
785
833
  llm: profiles[activeProfile] || { ...NO_PROFILE },
@@ -796,6 +844,7 @@ export function loadConfig(opts?: { configPath?: string; profile?: string }): Ap
796
844
  remote: provider === "local" ? undefined : remotes[provider],
797
845
  remotes,
798
846
  providerFiles,
847
+ devices,
799
848
  overlay: browser.overlay !== false,
800
849
  judge: browser.judge === true,
801
850
  },
@@ -805,6 +854,7 @@ export function loadConfig(opts?: { configPath?: string; profile?: string }): Ap
805
854
  dirs: promptDirsFor(prompts, base, configPath),
806
855
  },
807
856
  knowledge: knowledgeConfig(asRecord(yaml.knowledge), base, configPath),
857
+ defects: defectsConfig(asRecord(yaml.defects), base),
808
858
  export: {
809
859
  outDir,
810
860
  format: String(exp.format || "playwright"),
@@ -824,6 +874,7 @@ export function loadConfig(opts?: { configPath?: string; profile?: string }): Ap
824
874
  testTimeoutSeconds: Number(code.test_timeout_seconds || 600),
825
875
  stallAfter: Math.max(2, Number(code.stall_after || 5)),
826
876
  liveBrowser: code.live_browser !== false,
877
+ graph: code.graph !== false,
827
878
  replay: code.replay !== false,
828
879
  model: code.model ? String(code.model) : undefined,
829
880
  ...(code.review_profile ? { reviewProfile: String(code.review_profile) } : {}),
@@ -1026,6 +1077,26 @@ function sourcesConfig(raw: Record<string, unknown>): SourcesConfig {
1026
1077
  ...optional("zephyr", zephyrConfig(raw.zephyr)),
1027
1078
  ...optional("qtest", qtestConfig(asRecord(raw.qtest))),
1028
1079
  ...optional("bugs", bugsConfig(raw.bugs)),
1080
+ ...optional("github", githubConfig(raw.github)),
1081
+ }
1082
+ }
1083
+
1084
+ /** The GitHub MCP server's tool groups ForgeQE loads unless sources.github.toolsets says otherwise. */
1085
+ export const GITHUB_TOOLSETS = ["repos", "issues", "pull_requests", "actions"]
1086
+
1087
+ function githubConfig(raw: unknown): SourcesConfig["github"] {
1088
+ if (raw === undefined || raw === null) return undefined
1089
+ if (raw === false) return { apiUrl: "https://api.github.com", tokenEnv: "GITHUB_TOKEN", mcp: false, toolsets: [] }
1090
+ const g = asRecord(raw)
1091
+ const repo = String(g.repo || "").trim()
1092
+ if (repo && !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) throw new Error(`sources.github.repo is owner/name; got "${repo}".`)
1093
+ const toolsets = Array.isArray(g.toolsets) ? g.toolsets.map(String).filter(Boolean) : GITHUB_TOOLSETS
1094
+ return {
1095
+ ...(repo ? { repo } : {}),
1096
+ apiUrl: String(g.api_url || "https://api.github.com").replace(/\/+$/, ""),
1097
+ tokenEnv: String(g.token_env || "GITHUB_TOKEN"),
1098
+ mcp: g.mcp !== false,
1099
+ toolsets,
1029
1100
  }
1030
1101
  }
1031
1102
 
@@ -1049,6 +1120,27 @@ function promptDirsFor(p: Record<string, unknown>, base: string, configPath: str
1049
1120
  return dirs.filter((d, i) => dirs.findIndex((x) => path.resolve(x.dir) === path.resolve(d.dir)) === i)
1050
1121
  }
1051
1122
 
1123
+ export type DefectsConfig = {
1124
+ /** The application's own repository: suspect commits, and the code the root-cause analysis reads. */
1125
+ productRepo?: string
1126
+ /** Run the first case of each new defect once more, to see whether it repeats. */
1127
+ reproduce: boolean
1128
+ /** A model writes a root-cause hypothesis for new defects and regressions. */
1129
+ analyze: boolean
1130
+ /** Most defects analysed by the model in one suite. */
1131
+ maxAnalyses: number
1132
+ }
1133
+
1134
+ export function defectsConfig(d: Record<string, unknown>, base: string): DefectsConfig {
1135
+ const max = Number(d.max_analyses)
1136
+ return {
1137
+ ...(d.product_repo ? { productRepo: path.resolve(base, String(d.product_repo)) } : {}),
1138
+ reproduce: d.reproduce !== false,
1139
+ analyze: d.analyze !== false,
1140
+ maxAnalyses: Number.isInteger(max) && max >= 0 && d.max_analyses !== undefined ? max : 3,
1141
+ }
1142
+ }
1143
+
1052
1144
  function knowledgeConfig(k: Record<string, unknown>, base: string, configPath: string | undefined): NonNullable<AppConfig["knowledge"]> {
1053
1145
  const dir = k.dir ? path.resolve(base, String(k.dir)) : configPath ? path.join(projectDir(base), "knowledge") : path.join(forgeqeHome(), "knowledge")
1054
1146
  const max = Number(k.browser_max_chars)
@@ -1076,9 +1168,12 @@ function apiConfig(a: Record<string, unknown>): ApiConfig {
1076
1168
  }
1077
1169
  }
1078
1170
 
1171
+ /** The tables forgeqe bi always exports; a dataset id may not reuse one. */
1172
+ export const BI_BUILTIN_TABLES = ["suites", "case_results", "step_results", "writebacks", "readiness", "readiness_stories", "readiness_cases", "external_datasets"]
1173
+
1079
1174
  function biConfig(b: Record<string, unknown>, base: string, outDir: string, configPath: string | undefined): BiConfig {
1080
1175
  const evidenceDir = path.resolve(base, String(b.evidence_dir || path.join(".", "evidence")))
1081
- const defaultDataDir = path.join(evidenceDir, "sources", "forgeqe", "data")
1176
+ const defaultDataDir = path.join(evidenceDir, "sources", "forgeqe")
1082
1177
  const title = String(b.title || "ForgeQE BI").trim() || "ForgeQE BI"
1083
1178
  const datasets = Object.entries(asRecord(b.datasets)).map(([id, value]) => biDataset(id, asRecord(value)))
1084
1179
  return {
@@ -1091,7 +1186,9 @@ function biConfig(b: Record<string, unknown>, base: string, outDir: string, conf
1091
1186
  }
1092
1187
 
1093
1188
  function biDataset(id: string, raw: Record<string, unknown>): BiDatasetConfig {
1094
- if (!/^[A-Za-z0-9_-]+$/.test(id)) throw new Error(`bi.datasets.${id}: ids may only use letters, digits, - and _.`)
1189
+ // The id is the dataset's table name in Evidence's SQL (forgeqe.<id>), so it must be a plain identifier.
1190
+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(id)) throw new Error(`bi.datasets.${id}: ids may only use letters, digits and _, and must not start with a digit.`)
1191
+ if (BI_BUILTIN_TABLES.includes(id)) throw new Error(`bi.datasets.${id}: ${id} is one of ForgeQE's own BI tables; choose another id.`)
1095
1192
  const title = String(raw.title || id.replace(/[-_]+/g, " ").trim()).trim()
1096
1193
  const kind = String(raw.kind || "").trim().toLowerCase()
1097
1194
  if (kind === "ado_wiql") {
@@ -1183,7 +1280,7 @@ function bugsConfig(raw: unknown): BugsConfig | undefined {
1183
1280
  if (raw === undefined || raw === null || raw === false) return undefined
1184
1281
  const b = asRecord(raw)
1185
1282
  const system = String(b.system || "").toLowerCase().replace(/[\s-]/g, "_")
1186
- if (system !== "jira" && system !== "azure_devops") throw new Error(`sources.bugs.system must be jira or azure_devops; got "${String(b.system ?? "")}".`)
1283
+ if (system !== "jira" && system !== "azure_devops" && system !== "github") throw new Error(`sources.bugs.system must be jira, azure_devops, or github; got "${String(b.system ?? "")}".`)
1187
1284
  if (system === "jira" && !b.project) throw new Error("sources.bugs.project is required for Jira: the project key bugs are filed in.")
1188
1285
  return {
1189
1286
  system,
@@ -1192,9 +1289,16 @@ function bugsConfig(raw: unknown): BugsConfig | undefined {
1192
1289
  labels: Array.isArray(b.labels) ? b.labels.map(String).filter(Boolean) : ["forgeqe"],
1193
1290
  ...(b.area_path ? { areaPath: String(b.area_path) } : {}),
1194
1291
  onError: b.on_error === true,
1292
+ ...(groupBy(b.group_by) === "case" ? { groupBy: "case" as const } : {}),
1195
1293
  }
1196
1294
  }
1197
1295
 
1296
+ function groupBy(raw: unknown): "defect" | "case" {
1297
+ if (raw === undefined || raw === null || raw === "defect") return "defect"
1298
+ if (raw === "case") return "case"
1299
+ throw new Error(`sources.bugs.group_by must be defect or case; got "${String(raw)}".`)
1300
+ }
1301
+
1198
1302
  function jiraMcp(raw: unknown): JiraMcp | false {
1199
1303
  if (raw === false || raw === "false" || raw === "off") return false
1200
1304
  if (raw === undefined || raw === null || raw === true || raw === "mcp-atlassian") return "mcp-atlassian"
@@ -0,0 +1,223 @@
1
+ import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from "node:fs"
2
+ import path from "node:path"
3
+
4
+ import { adoSettings } from "../cases/ado.ts"
5
+ import { http, textToAdf } from "../cases/http.ts"
6
+ import { jiraApi } from "../cases/jira.ts"
7
+ import type { AppConfig } from "../config.ts"
8
+ import type { McpTool } from "./tools.ts"
9
+
10
+ /**
11
+ * Evidence onto an existing work item or issue: files from a run (screenshots, the report, JUnit, the trace) uploaded
12
+ * as attachments, posted in a comment (images inline, text as text), or both. Only files inside the working folder or
13
+ * the reports folders can go, so a request can never send anything else from the machine.
14
+ */
15
+
16
+ const MAX_FILE = 25 * 1024 * 1024
17
+ /** Text read into a comment; longer text is attached instead. */
18
+ const MAX_COMMENT_TEXT = 20_000
19
+ const IMAGE = /\.(png|jpe?g|gif|webp)$/i
20
+ const TEXT = /\.(md|txt|log|json|xml|csv|ya?ml)$/i
21
+ const MIME: Record<string, string> = {
22
+ ".png": "image/png",
23
+ ".jpg": "image/jpeg",
24
+ ".jpeg": "image/jpeg",
25
+ ".gif": "image/gif",
26
+ ".webp": "image/webp",
27
+ ".html": "text/html",
28
+ ".json": "application/json",
29
+ ".xml": "application/xml",
30
+ }
31
+
32
+ export type Pick = "last_screenshot" | "failure_screenshot" | "report" | "junit" | "trace"
33
+ const PICKS: Pick[] = ["last_screenshot", "failure_screenshot", "report", "junit", "trace"]
34
+
35
+ type File = { path: string; name: string; data: Buffer; type: string }
36
+
37
+ /** Files a run folder holds, by what they are. A suite folder's last screenshot is its last case's. */
38
+ export function pickFiles(runDir: string, picks: Pick[]): string[] {
39
+ const out: string[] = []
40
+ const lastShot = (dir: string): string | undefined => {
41
+ const steps = path.join(dir, "steps")
42
+ const shots = existsSync(steps) ? readdirSync(steps).filter((f) => IMAGE.test(f)).sort() : []
43
+ if (shots.length) return path.join(steps, shots[shots.length - 1])
44
+ if (existsSync(path.join(dir, "failure.png"))) return path.join(dir, "failure.png")
45
+ // A suite folder: its case folders, last one first.
46
+ const cases = readdirSync(dir).filter((d) => statSync(path.join(dir, d)).isDirectory() && d !== "steps").sort().reverse()
47
+ for (const c of cases) {
48
+ const shot = lastShot(path.join(dir, c))
49
+ if (shot) return shot
50
+ }
51
+ return undefined
52
+ }
53
+ for (const pick of picks) {
54
+ const file =
55
+ pick === "last_screenshot"
56
+ ? lastShot(runDir)
57
+ : pick === "failure_screenshot"
58
+ ? [path.join(runDir, "failure.png")].find(existsSync)
59
+ : pick === "report"
60
+ ? path.join(runDir, "report.html")
61
+ : pick === "junit"
62
+ ? path.join(runDir, "junit.xml")
63
+ : path.join(runDir, "trace.md")
64
+ if (!file || !existsSync(file)) throw new Error(`${runDir} has no ${pick.replace("_", " ")}.`)
65
+ out.push(file)
66
+ }
67
+ return out
68
+ }
69
+
70
+ /** A file to send: inside the working folder or a reports folder, and not too big. */
71
+ function readAllowed(file: string, config: AppConfig, cwd: string): File {
72
+ if (!existsSync(file)) throw new Error(`No such file: ${file}`)
73
+ const real = realpathSync(file)
74
+ const roots = [cwd, config.export?.outDir, config.suite?.outDir, config.monitor?.outDir].filter(Boolean).map((r) => (existsSync(r!) ? realpathSync(r!) : path.resolve(r!)))
75
+ if (!roots.some((r) => real === r || real.startsWith(`${r}${path.sep}`))) {
76
+ throw new Error(`${file} is outside the working folder and the reports folders, so it is not sent.`)
77
+ }
78
+ const size = statSync(real).size
79
+ if (size > MAX_FILE) throw new Error(`${file} is ${(size / 1024 / 1024).toFixed(1)} MB; the limit is ${MAX_FILE / 1024 / 1024} MB.`)
80
+ return { path: real, name: path.basename(real), data: readFileSync(real), type: MIME[path.extname(real).toLowerCase()] || "application/octet-stream" }
81
+ }
82
+
83
+ const esc = (s: string) => s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;")
84
+
85
+ type Outcome = { item: string; url?: string; attached: string[]; commented: boolean; inline: string[] }
86
+
87
+ async function toAdo(config: AppConfig, id: string, files: File[], mode: string, comment: string): Promise<Outcome> {
88
+ const { auth, org } = adoSettings(config.sources)
89
+ if (!/^\d+$/.test(id)) throw new Error(`An Azure DevOps work item id is a number; got "${id}".`)
90
+ // Comments and attachments live in the work item's own project.
91
+ const { data: wi } = await http<{ fields?: Record<string, unknown>; _links?: { html?: { href?: string } } }>(`${org}/_apis/wit/workitems/${id}?fields=System.TeamProject&api-version=7.1`, { auth })
92
+ const project = encodeURIComponent(String(wi.fields?.["System.TeamProject"] || ""))
93
+ const scoped = `${org}/${project}`
94
+ const upload = async (f: File) => {
95
+ const res = await fetch(`${scoped}/_apis/wit/attachments?fileName=${encodeURIComponent(f.name)}&api-version=7.1`, {
96
+ method: "POST",
97
+ headers: { Authorization: auth, "Content-Type": "application/octet-stream", Accept: "application/json" },
98
+ body: new Uint8Array(f.data),
99
+ })
100
+ if (!res.ok) throw new Error(`Uploading ${f.name} to work item ${id} failed: ${res.status} ${res.statusText}.`)
101
+ return ((await res.json()) as { url: string }).url
102
+ }
103
+ const out: Outcome = { item: `#${id}`, url: wi._links?.html?.href, attached: [], commented: false, inline: [] }
104
+ const urls = new Map<string, string>()
105
+ if (mode === "attach" || mode === "both") {
106
+ const ops = []
107
+ for (const f of files) {
108
+ const url = await upload(f)
109
+ urls.set(f.path, url)
110
+ ops.push({ op: "add", path: "/relations/-", value: { rel: "AttachedFile", url, attributes: { comment: comment ? comment.slice(0, 250) : "Added by ForgeQE" } } })
111
+ out.attached.push(f.name)
112
+ }
113
+ await http(`${org}/_apis/wit/workitems/${id}?api-version=7.1`, { method: "PATCH", auth, body: ops, contentType: "application/json-patch+json" })
114
+ }
115
+ if (mode === "comment" || mode === "both") {
116
+ const parts = comment ? [`<p>${esc(comment).replace(/\n/g, "<br>")}</p>`] : []
117
+ for (const f of files) {
118
+ if (IMAGE.test(f.name)) {
119
+ const url = urls.get(f.path) || (await upload(f))
120
+ parts.push(`<p><b>${esc(f.name)}</b><br><img src="${url}" alt="${esc(f.name)}"></p>`)
121
+ out.inline.push(f.name)
122
+ } else if (TEXT.test(f.name) && f.data.length <= MAX_COMMENT_TEXT) {
123
+ parts.push(`<p><b>${esc(f.name)}</b></p><pre>${esc(f.data.toString("utf8"))}</pre>`)
124
+ out.inline.push(f.name)
125
+ } else {
126
+ // An HTML report or a big file cannot go in a comment: it is attached, and the comment names it.
127
+ const url = urls.get(f.path) || (await upload(f))
128
+ if (!urls.has(f.path)) {
129
+ await http(`${org}/_apis/wit/workitems/${id}?api-version=7.1`, {
130
+ method: "PATCH",
131
+ auth,
132
+ body: [{ op: "add", path: "/relations/-", value: { rel: "AttachedFile", url, attributes: { comment: "Added by ForgeQE" } } }],
133
+ contentType: "application/json-patch+json",
134
+ })
135
+ out.attached.push(f.name)
136
+ }
137
+ parts.push(`<p>Attached: <a href="${url}">${esc(f.name)}</a></p>`)
138
+ }
139
+ }
140
+ await http(`${scoped}/_apis/wit/workItems/${id}/comments?api-version=7.1-preview.4`, { auth, body: { text: parts.join("\n") } })
141
+ out.commented = true
142
+ }
143
+ return out
144
+ }
145
+
146
+ async function toJira(config: AppConfig, key: string, files: File[], mode: string, comment: string): Promise<Outcome> {
147
+ const j = jiraApi(config.sources, "Attaching to Jira")
148
+ const issue = key.trim().toUpperCase()
149
+ if (!/^[A-Z][A-Z0-9_]*-\d+$/.test(issue)) throw new Error(`A Jira issue key looks like SHOP-12; got "${key}".`)
150
+ const out: Outcome = { item: issue, url: `${j.url}/browse/${issue}`, attached: [], commented: false, inline: [] }
151
+ // Jira shows images from the issue's attachments, so files go up as attachments in either mode.
152
+ const send = files.filter((f) => mode !== "comment" || !(TEXT.test(f.name) && f.data.length <= MAX_COMMENT_TEXT))
153
+ for (const f of send) {
154
+ const form = new FormData()
155
+ form.append("file", new Blob([new Uint8Array(f.data)], { type: f.type }), f.name)
156
+ const res = await fetch(`${j.api}/issue/${encodeURIComponent(issue)}/attachments`, {
157
+ method: "POST",
158
+ headers: { Authorization: j.auth, "X-Atlassian-Token": "no-check", Accept: "application/json" },
159
+ body: form,
160
+ })
161
+ if (!res.ok) throw new Error(`Attaching ${f.name} to ${issue} failed: ${res.status} ${res.statusText}.`)
162
+ out.attached.push(f.name)
163
+ }
164
+ if (mode === "comment" || mode === "both") {
165
+ const lines = comment ? [comment, ""] : []
166
+ for (const f of files) {
167
+ if (TEXT.test(f.name) && f.data.length <= MAX_COMMENT_TEXT) {
168
+ lines.push(`${f.name}:`, f.data.toString("utf8"), "")
169
+ out.inline.push(f.name)
170
+ } else if (IMAGE.test(f.name) && !j.cloud) {
171
+ // Jira Server/Data Center wiki markup shows an attached image in the comment.
172
+ lines.push(`!${f.name}|thumbnail!`)
173
+ out.inline.push(f.name)
174
+ } else lines.push(`Attached: ${f.name}`)
175
+ }
176
+ const text = lines.join("\n").trim()
177
+ await http(`${j.api}/issue/${encodeURIComponent(issue)}/comment`, { auth: j.auth, body: { body: j.cloud ? textToAdf(text) : text } })
178
+ out.commented = true
179
+ }
180
+ return out
181
+ }
182
+
183
+ export const ATTACH_TOOLS: McpTool[] = [
184
+ {
185
+ name: "attach_to_work_item",
186
+ description:
187
+ "Put evidence on an existing Azure DevOps work item or Jira issue, when the user asks (\"add the screenshot to ADO 256\", \"attach the report to SHOP-12\", \"comment the results on the bug\"). mode attach uploads the files as attachments; comment posts a comment with images shown inline and text files as text (an HTML report cannot be shown in a comment, so it is attached and named); both does both. Files come from a run folder by what they are (pick), or by path; only files in the working folder or the reports folders can be sent. Returns the item's link and what went where.",
188
+ inputSchema: {
189
+ type: "object",
190
+ properties: {
191
+ system: { type: "string", enum: ["azure_devops", "jira"] },
192
+ item: { type: "string", description: "Work item id (azure_devops), e.g. 256, or issue key (jira), e.g. SHOP-12" },
193
+ mode: { type: "string", enum: ["attach", "comment", "both"], description: "attach: upload as attachments. comment: one comment with the files in it. both: attachments and a comment (default when the user did not say)" },
194
+ run_dir: { type: "string", description: "A run or suite folder from browser_run or run_test_cases, to pick files from" },
195
+ pick: { type: "array", items: { type: "string", enum: PICKS }, description: "What to take from run_dir: last_screenshot, failure_screenshot, report, junit, trace" },
196
+ files: { type: "array", items: { type: "string" }, description: "Other files by path, relative to the working folder" },
197
+ comment: { type: "string", description: "Text for the comment (comment/both), and the attachment note: what ran, where, the result" },
198
+ },
199
+ required: ["system", "item"],
200
+ },
201
+ async run(args, ctx) {
202
+ const mode = String(args.mode || "both")
203
+ if (!["attach", "comment", "both"].includes(mode)) throw new Error(`mode is attach, comment, or both; got "${mode}".`)
204
+ const picks = (Array.isArray(args.pick) ? args.pick.map(String) : []) as Pick[]
205
+ const bad = picks.filter((p) => !PICKS.includes(p))
206
+ if (bad.length) throw new Error(`pick takes ${PICKS.join(", ")}; got ${bad.join(", ")}.`)
207
+ if (picks.length && !args.run_dir) throw new Error("pick needs run_dir: the run folder to take the files from.")
208
+ const runDir = args.run_dir ? path.resolve(ctx.cwd, String(args.run_dir)) : ""
209
+ const paths = [...(runDir ? pickFiles(runDir, picks.length ? picks : ["last_screenshot"]) : []), ...(Array.isArray(args.files) ? args.files.map((f) => path.resolve(ctx.cwd, String(f))) : [])]
210
+ const comment = String(args.comment || "").trim()
211
+ if (!paths.length && !comment) throw new Error("Nothing to add: give run_dir (and pick), files, or a comment.")
212
+ if (!paths.length && mode === "attach") throw new Error("Nothing to attach: give run_dir (and pick) or files.")
213
+ const files = [...new Set(paths)].map((p) => readAllowed(p, ctx.config, ctx.cwd))
214
+ const out = args.system === "jira" ? await toJira(ctx.config, String(args.item), files, mode, comment) : await toAdo(ctx.config, String(args.item), files, mode, comment)
215
+ return {
216
+ item: out.item,
217
+ ...(out.url ? { link: out.url } : {}),
218
+ ...(out.attached.length ? { attached: out.attached } : {}),
219
+ ...(out.commented ? { comment: out.inline.length ? `posted, with ${out.inline.join(", ")} in it` : "posted" } : {}),
220
+ }
221
+ },
222
+ },
223
+ ]
@@ -361,7 +361,7 @@ export const COMMAND_TOOLS: McpTool[] = [
361
361
  note: "The user switches profiles with Tab or /profile in the shell, or --profile.",
362
362
  }
363
363
  case "mcp_servers": {
364
- const { servers, skipped } = externalMcp(cfg)
364
+ const { servers, skipped } = externalMcp(cfg, Bun.which, process.env, { cwd: ctx.cwd })
365
365
  return {
366
366
  servers: servers.map((s) => {
367
367
  const c = s.config as { type: string; command?: string[]; url?: string }