@yasserkhanorg/e2e-agents 1.8.4 → 1.9.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (259) hide show
  1. package/README.md +95 -8
  2. package/dist/adapters/cypress.d.ts +10 -0
  3. package/dist/adapters/cypress.d.ts.map +1 -0
  4. package/dist/adapters/cypress.js +86 -0
  5. package/dist/adapters/framework_adapter.d.ts +41 -0
  6. package/dist/adapters/framework_adapter.d.ts.map +1 -0
  7. package/dist/adapters/framework_adapter.js +152 -0
  8. package/dist/adapters/playwright.d.ts +10 -0
  9. package/dist/adapters/playwright.d.ts.map +1 -0
  10. package/dist/adapters/playwright.js +86 -0
  11. package/dist/adapters/pytest.d.ts +10 -0
  12. package/dist/adapters/pytest.d.ts.map +1 -0
  13. package/dist/adapters/pytest.js +96 -0
  14. package/dist/adapters/supertest.d.ts +12 -0
  15. package/dist/adapters/supertest.d.ts.map +1 -0
  16. package/dist/adapters/supertest.js +85 -0
  17. package/dist/agent/config.d.ts +1 -1
  18. package/dist/agent/config.d.ts.map +1 -1
  19. package/dist/agent/git.d.ts +1 -0
  20. package/dist/agent/git.d.ts.map +1 -1
  21. package/dist/agent/git.js +3 -0
  22. package/dist/agentic/fix_loop.d.ts.map +1 -1
  23. package/dist/agentic/fix_loop.js +5 -4
  24. package/dist/agentic/runner.d.ts +2 -0
  25. package/dist/agentic/runner.d.ts.map +1 -1
  26. package/dist/agentic/runner.js +15 -12
  27. package/dist/agents/cross-impact.d.ts.map +1 -1
  28. package/dist/agents/cross-impact.js +6 -1
  29. package/dist/agents/executor.d.ts.map +1 -1
  30. package/dist/agents/executor.js +6 -1
  31. package/dist/agents/strategist.d.ts.map +1 -1
  32. package/dist/agents/strategist.js +6 -1
  33. package/dist/agents/test-designer.d.ts.map +1 -1
  34. package/dist/agents/test-designer.js +6 -1
  35. package/dist/anthropic_provider.d.ts.map +1 -1
  36. package/dist/anthropic_provider.js +1 -0
  37. package/dist/base_provider.d.ts +56 -0
  38. package/dist/base_provider.d.ts.map +1 -1
  39. package/dist/base_provider.js +123 -1
  40. package/dist/budget_ledger.d.ts +28 -0
  41. package/dist/budget_ledger.d.ts.map +1 -0
  42. package/dist/budget_ledger.js +62 -0
  43. package/dist/cache/cached_provider.d.ts +45 -0
  44. package/dist/cache/cached_provider.d.ts.map +1 -0
  45. package/dist/cache/cached_provider.js +88 -0
  46. package/dist/cache/response_cache.d.ts +79 -0
  47. package/dist/cache/response_cache.d.ts.map +1 -0
  48. package/dist/cache/response_cache.js +177 -0
  49. package/dist/cli/commands/bootstrap.d.ts +3 -0
  50. package/dist/cli/commands/bootstrap.d.ts.map +1 -0
  51. package/dist/cli/commands/bootstrap.js +109 -0
  52. package/dist/cli/commands/cost_report.d.ts +3 -0
  53. package/dist/cli/commands/cost_report.d.ts.map +1 -0
  54. package/dist/cli/commands/cost_report.js +115 -0
  55. package/dist/cli/commands/crew.d.ts.map +1 -1
  56. package/dist/cli/commands/crew.js +118 -1
  57. package/dist/cli/commands/gate.d.ts +3 -0
  58. package/dist/cli/commands/gate.d.ts.map +1 -0
  59. package/dist/cli/commands/gate.js +86 -0
  60. package/dist/cli/commands/init.d.ts.map +1 -1
  61. package/dist/cli/commands/init.js +7 -62
  62. package/dist/cli/commands/plan_crew.d.ts.map +1 -1
  63. package/dist/cli/commands/plan_crew.js +33 -21
  64. package/dist/cli/commands/train.d.ts.map +1 -1
  65. package/dist/cli/commands/train.js +16 -21
  66. package/dist/cli/defaults.d.ts +35 -0
  67. package/dist/cli/defaults.d.ts.map +1 -0
  68. package/dist/cli/defaults.js +125 -0
  69. package/dist/cli/errors.d.ts +27 -0
  70. package/dist/cli/errors.d.ts.map +1 -0
  71. package/dist/cli/errors.js +57 -0
  72. package/dist/cli/parse_args.d.ts.map +1 -1
  73. package/dist/cli/parse_args.js +24 -2
  74. package/dist/cli/types.d.ts +7 -1
  75. package/dist/cli/types.d.ts.map +1 -1
  76. package/dist/cli.js +47 -2
  77. package/dist/crew/context.d.ts +15 -0
  78. package/dist/crew/context.d.ts.map +1 -1
  79. package/dist/crew/orchestrator.d.ts +14 -0
  80. package/dist/crew/orchestrator.d.ts.map +1 -1
  81. package/dist/crew/orchestrator.js +162 -4
  82. package/dist/crew/protocol.d.ts +13 -0
  83. package/dist/crew/protocol.d.ts.map +1 -1
  84. package/dist/crew/provider.d.ts +15 -1
  85. package/dist/crew/provider.d.ts.map +1 -1
  86. package/dist/crew/provider.js +24 -4
  87. package/dist/custom_provider.d.ts.map +1 -1
  88. package/dist/custom_provider.js +1 -0
  89. package/dist/engine/diff_loader.d.ts.map +1 -1
  90. package/dist/engine/diff_loader.js +3 -14
  91. package/dist/engine/impact_engine.d.ts.map +1 -1
  92. package/dist/engine/impact_engine.js +9 -23
  93. package/dist/esm/adapters/cypress.js +49 -0
  94. package/dist/esm/adapters/framework_adapter.js +114 -0
  95. package/dist/esm/adapters/playwright.js +49 -0
  96. package/dist/esm/adapters/pytest.js +59 -0
  97. package/dist/esm/adapters/supertest.js +48 -0
  98. package/dist/esm/agent/git.js +3 -1
  99. package/dist/esm/agentic/fix_loop.js +5 -4
  100. package/dist/esm/agentic/runner.js +15 -12
  101. package/dist/esm/agents/cross-impact.js +6 -1
  102. package/dist/esm/agents/executor.js +6 -1
  103. package/dist/esm/agents/strategist.js +6 -1
  104. package/dist/esm/agents/test-designer.js +6 -1
  105. package/dist/esm/anthropic_provider.js +1 -0
  106. package/dist/esm/base_provider.js +121 -0
  107. package/dist/esm/budget_ledger.js +58 -0
  108. package/dist/esm/cache/cached_provider.js +82 -0
  109. package/dist/esm/cache/response_cache.js +140 -0
  110. package/dist/esm/cli/commands/bootstrap.js +106 -0
  111. package/dist/esm/cli/commands/cost_report.js +112 -0
  112. package/dist/esm/cli/commands/crew.js +118 -1
  113. package/dist/esm/cli/commands/gate.js +83 -0
  114. package/dist/esm/cli/commands/init.js +3 -58
  115. package/dist/esm/cli/commands/plan_crew.js +33 -21
  116. package/dist/esm/cli/commands/train.js +16 -21
  117. package/dist/esm/cli/defaults.js +118 -0
  118. package/dist/esm/cli/errors.js +52 -0
  119. package/dist/esm/cli/parse_args.js +24 -2
  120. package/dist/esm/cli.js +47 -2
  121. package/dist/esm/crew/orchestrator.js +162 -4
  122. package/dist/esm/crew/provider.js +24 -4
  123. package/dist/esm/custom_provider.js +1 -0
  124. package/dist/esm/engine/diff_loader.js +1 -12
  125. package/dist/esm/engine/impact_engine.js +9 -23
  126. package/dist/esm/index.js +21 -0
  127. package/dist/esm/knowledge/cluster_utils.js +60 -0
  128. package/dist/esm/knowledge/kg_bridge.js +381 -0
  129. package/dist/esm/knowledge/kg_types.js +3 -0
  130. package/dist/esm/knowledge/route_families.js +89 -0
  131. package/dist/esm/mcp-server.js +2 -4
  132. package/dist/esm/metrics/prometheus.js +149 -0
  133. package/dist/esm/model_router.js +59 -0
  134. package/dist/esm/ollama_provider.js +1 -0
  135. package/dist/esm/openai_provider.js +1 -0
  136. package/dist/esm/pipeline/orchestrator.js +6 -12
  137. package/dist/esm/pipeline/stage0_preprocess.js +12 -19
  138. package/dist/esm/pipeline/stage2_coverage.js +1 -0
  139. package/dist/esm/pipeline/stage3_generation.js +1 -0
  140. package/dist/esm/progress.js +112 -0
  141. package/dist/esm/prompts/coverage.js +7 -24
  142. package/dist/esm/prompts/cross-impact.js +3 -21
  143. package/dist/esm/prompts/generation.js +158 -36
  144. package/dist/esm/prompts/generation_profile.js +147 -0
  145. package/dist/esm/prompts/heal.js +33 -15
  146. package/dist/esm/prompts/impact.js +3 -22
  147. package/dist/esm/prompts/json_extract.js +36 -0
  148. package/dist/esm/prompts/strategist.js +2 -20
  149. package/dist/esm/prompts/test-designer.js +6 -21
  150. package/dist/esm/provider_factory.js +6 -4
  151. package/dist/esm/reporters/junit.js +86 -0
  152. package/dist/esm/reporters/reporter.js +3 -0
  153. package/dist/esm/reporters/sarif.js +131 -0
  154. package/dist/esm/resilience/circuit_breaker.js +78 -0
  155. package/dist/esm/resilience/retry.js +56 -0
  156. package/dist/esm/sanitize.js +66 -0
  157. package/dist/esm/training/kg_scanner.js +115 -0
  158. package/dist/esm/training/scanner.js +27 -34
  159. package/dist/esm/version.js +33 -0
  160. package/dist/index.d.ts +21 -1
  161. package/dist/index.d.ts.map +1 -1
  162. package/dist/index.js +45 -1
  163. package/dist/knowledge/cluster_utils.d.ts +28 -0
  164. package/dist/knowledge/cluster_utils.d.ts.map +1 -0
  165. package/dist/knowledge/cluster_utils.js +67 -0
  166. package/dist/knowledge/kg_bridge.d.ts +31 -0
  167. package/dist/knowledge/kg_bridge.d.ts.map +1 -0
  168. package/dist/knowledge/kg_bridge.js +388 -0
  169. package/dist/knowledge/kg_types.d.ts +75 -0
  170. package/dist/knowledge/kg_types.d.ts.map +1 -0
  171. package/dist/knowledge/kg_types.js +4 -0
  172. package/dist/knowledge/route_families.d.ts +18 -0
  173. package/dist/knowledge/route_families.d.ts.map +1 -1
  174. package/dist/knowledge/route_families.js +91 -0
  175. package/dist/mcp-server.d.ts.map +1 -1
  176. package/dist/mcp-server.js +2 -4
  177. package/dist/metrics/prometheus.d.ts +37 -0
  178. package/dist/metrics/prometheus.d.ts.map +1 -0
  179. package/dist/metrics/prometheus.js +153 -0
  180. package/dist/model_router.d.ts +28 -0
  181. package/dist/model_router.d.ts.map +1 -0
  182. package/dist/model_router.js +63 -0
  183. package/dist/ollama_provider.d.ts.map +1 -1
  184. package/dist/ollama_provider.js +1 -0
  185. package/dist/openai_provider.d.ts.map +1 -1
  186. package/dist/openai_provider.js +1 -0
  187. package/dist/pipeline/orchestrator.d.ts +2 -0
  188. package/dist/pipeline/orchestrator.d.ts.map +1 -1
  189. package/dist/pipeline/orchestrator.js +6 -12
  190. package/dist/pipeline/stage0_preprocess.d.ts.map +1 -1
  191. package/dist/pipeline/stage0_preprocess.js +11 -18
  192. package/dist/pipeline/stage2_coverage.d.ts +2 -0
  193. package/dist/pipeline/stage2_coverage.d.ts.map +1 -1
  194. package/dist/pipeline/stage2_coverage.js +1 -0
  195. package/dist/pipeline/stage3_generation.d.ts +2 -0
  196. package/dist/pipeline/stage3_generation.d.ts.map +1 -1
  197. package/dist/pipeline/stage3_generation.js +1 -0
  198. package/dist/pipeline/stage4_heal.d.ts +2 -0
  199. package/dist/pipeline/stage4_heal.d.ts.map +1 -1
  200. package/dist/progress.d.ts +22 -0
  201. package/dist/progress.d.ts.map +1 -0
  202. package/dist/progress.js +116 -0
  203. package/dist/prompts/coverage.d.ts +2 -0
  204. package/dist/prompts/coverage.d.ts.map +1 -1
  205. package/dist/prompts/coverage.js +7 -24
  206. package/dist/prompts/cross-impact.d.ts +1 -0
  207. package/dist/prompts/cross-impact.d.ts.map +1 -1
  208. package/dist/prompts/cross-impact.js +3 -21
  209. package/dist/prompts/generation.d.ts +3 -1
  210. package/dist/prompts/generation.d.ts.map +1 -1
  211. package/dist/prompts/generation.js +158 -36
  212. package/dist/prompts/generation_profile.d.ts +29 -0
  213. package/dist/prompts/generation_profile.d.ts.map +1 -0
  214. package/dist/prompts/generation_profile.js +151 -0
  215. package/dist/prompts/heal.d.ts +3 -1
  216. package/dist/prompts/heal.d.ts.map +1 -1
  217. package/dist/prompts/heal.js +33 -15
  218. package/dist/prompts/impact.d.ts +1 -0
  219. package/dist/prompts/impact.d.ts.map +1 -1
  220. package/dist/prompts/impact.js +3 -22
  221. package/dist/prompts/json_extract.d.ts +14 -0
  222. package/dist/prompts/json_extract.d.ts.map +1 -0
  223. package/dist/prompts/json_extract.js +39 -0
  224. package/dist/prompts/strategist.d.ts.map +1 -1
  225. package/dist/prompts/strategist.js +2 -20
  226. package/dist/prompts/test-designer.d.ts +2 -0
  227. package/dist/prompts/test-designer.d.ts.map +1 -1
  228. package/dist/prompts/test-designer.js +6 -21
  229. package/dist/provider_factory.d.ts.map +1 -1
  230. package/dist/provider_factory.js +6 -4
  231. package/dist/reporters/junit.d.ts +6 -0
  232. package/dist/reporters/junit.d.ts.map +1 -0
  233. package/dist/reporters/junit.js +89 -0
  234. package/dist/reporters/reporter.d.ts +42 -0
  235. package/dist/reporters/reporter.d.ts.map +1 -0
  236. package/dist/reporters/reporter.js +4 -0
  237. package/dist/reporters/sarif.d.ts +7 -0
  238. package/dist/reporters/sarif.d.ts.map +1 -0
  239. package/dist/reporters/sarif.js +134 -0
  240. package/dist/resilience/circuit_breaker.d.ts +36 -0
  241. package/dist/resilience/circuit_breaker.d.ts.map +1 -0
  242. package/dist/resilience/circuit_breaker.js +82 -0
  243. package/dist/resilience/retry.d.ts +11 -0
  244. package/dist/resilience/retry.d.ts.map +1 -0
  245. package/dist/resilience/retry.js +59 -0
  246. package/dist/sanitize.d.ts +15 -0
  247. package/dist/sanitize.d.ts.map +1 -0
  248. package/dist/sanitize.js +71 -0
  249. package/dist/training/kg_scanner.d.ts +13 -0
  250. package/dist/training/kg_scanner.d.ts.map +1 -0
  251. package/dist/training/kg_scanner.js +118 -0
  252. package/dist/training/scanner.d.ts +7 -2
  253. package/dist/training/scanner.d.ts.map +1 -1
  254. package/dist/training/scanner.js +27 -34
  255. package/dist/version.d.ts +6 -0
  256. package/dist/version.d.ts.map +1 -0
  257. package/dist/version.js +36 -0
  258. package/package.json +7 -2
  259. package/schemas/route-families.schema.json +31 -1
@@ -0,0 +1,114 @@
1
+ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
2
+ // See LICENSE.txt for license information.
3
+ /**
4
+ * Framework Adapter Interface — abstracts test-framework-specific logic
5
+ * behind a uniform contract so the rest of the pipeline is framework-agnostic.
6
+ */
7
+ import * as fs from 'node:fs';
8
+ import * as path from 'node:path';
9
+ import { PlaywrightAdapter } from './playwright.js';
10
+ import { CypressAdapter } from './cypress.js';
11
+ import { SupertestAdapter } from './supertest.js';
12
+ import { PytestAdapter } from './pytest.js';
13
+ /** Shared framework name lists used for test-mode detection across the codebase. */
14
+ export const UI_FRAMEWORKS = ['playwright', '@playwright/test', 'cypress', 'selenium'];
15
+ export const API_FRAMEWORKS = ['supertest', 'pytest', 'requests', 'vitest', 'jest'];
16
+ /**
17
+ * Auto-detect which test framework a project uses by inspecting its
18
+ * package.json dependencies. Falls back to Playwright when detection
19
+ * is inconclusive.
20
+ */
21
+ export function detectFramework(projectRoot) {
22
+ const pkgPath = path.join(projectRoot, 'package.json');
23
+ if (fs.existsSync(pkgPath)) {
24
+ try {
25
+ const raw = fs.readFileSync(pkgPath, 'utf-8');
26
+ const pkg = JSON.parse(raw);
27
+ const allDeps = {
28
+ ...pkg.dependencies,
29
+ ...pkg.devDependencies,
30
+ };
31
+ if ('@playwright/test' in allDeps) {
32
+ return new PlaywrightAdapter();
33
+ }
34
+ if ('cypress' in allDeps) {
35
+ return new CypressAdapter();
36
+ }
37
+ // Backend API test frameworks
38
+ if ('supertest' in allDeps) {
39
+ const runner = 'vitest' in allDeps ? 'vitest' : 'jest';
40
+ return new SupertestAdapter(runner);
41
+ }
42
+ }
43
+ catch {
44
+ // Malformed package.json — fall through to default.
45
+ }
46
+ }
47
+ // Check for Python project
48
+ const pyprojectPath = path.join(projectRoot, 'pyproject.toml');
49
+ if (fs.existsSync(pyprojectPath)) {
50
+ try {
51
+ const content = fs.readFileSync(pyprojectPath, 'utf-8');
52
+ if (content.includes('pytest')) {
53
+ return new PytestAdapter();
54
+ }
55
+ }
56
+ catch {
57
+ // fall through
58
+ }
59
+ }
60
+ // Default to Playwright when we cannot determine the framework.
61
+ return new PlaywrightAdapter();
62
+ }
63
+ /**
64
+ * Detect the test mode for a project: UI testing, API testing, or both.
65
+ * Uses package.json / pyproject.toml dependencies and optional KG metadata.
66
+ */
67
+ export function detectTestMode(projectRoot, kg) {
68
+ // If KG provides framework hints, use them
69
+ if (kg) {
70
+ const frameworks = kg.project.frameworks.map((f) => f.toLowerCase());
71
+ const uiSet = new Set(UI_FRAMEWORKS);
72
+ const apiSet = new Set(API_FRAMEWORKS);
73
+ const hasUi = frameworks.some((f) => uiSet.has(f));
74
+ const hasApi = frameworks.some((f) => apiSet.has(f));
75
+ if (hasUi && hasApi)
76
+ return 'both';
77
+ if (hasApi)
78
+ return 'api';
79
+ if (hasUi)
80
+ return 'ui';
81
+ }
82
+ // Fall back to package.json inspection
83
+ const pkgPath = path.join(projectRoot, 'package.json');
84
+ if (fs.existsSync(pkgPath)) {
85
+ try {
86
+ const raw = fs.readFileSync(pkgPath, 'utf-8');
87
+ const pkg = JSON.parse(raw);
88
+ const allDeps = { ...pkg.dependencies, ...pkg.devDependencies };
89
+ const hasUi = '@playwright/test' in allDeps || 'cypress' in allDeps;
90
+ const hasApi = 'supertest' in allDeps;
91
+ if (hasUi && hasApi)
92
+ return 'both';
93
+ if (hasApi)
94
+ return 'api'; // supertest alone means API-only when no UI framework is present
95
+ }
96
+ catch {
97
+ // fall through
98
+ }
99
+ }
100
+ // Check for Python API testing
101
+ const pyprojectPath = path.join(projectRoot, 'pyproject.toml');
102
+ if (fs.existsSync(pyprojectPath)) {
103
+ try {
104
+ const content = fs.readFileSync(pyprojectPath, 'utf-8');
105
+ if (content.includes('pytest') && !fs.existsSync(path.join(projectRoot, 'package.json'))) {
106
+ return 'api';
107
+ }
108
+ }
109
+ catch {
110
+ // fall through
111
+ }
112
+ }
113
+ return 'ui';
114
+ }
@@ -0,0 +1,49 @@
1
+ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
2
+ // See LICENSE.txt for license information.
3
+ /**
4
+ * Playwright Adapter — FrameworkAdapter implementation for @playwright/test.
5
+ */
6
+ import * as fs from 'node:fs';
7
+ import * as path from 'node:path';
8
+ export class PlaywrightAdapter {
9
+ constructor() {
10
+ this.name = 'playwright';
11
+ this.specGlob = '**/*.spec.{ts,js}';
12
+ this.extractTestPattern = /\btest(?:\.describe)?\s*\(/g;
13
+ this.configFileNames = ['playwright.config.ts', 'playwright.config.js'];
14
+ }
15
+ detect(projectRoot) {
16
+ const pkgPath = path.join(projectRoot, 'package.json');
17
+ if (!fs.existsSync(pkgPath)) {
18
+ return false;
19
+ }
20
+ try {
21
+ const raw = fs.readFileSync(pkgPath, 'utf-8');
22
+ const pkg = JSON.parse(raw);
23
+ const allDeps = {
24
+ ...pkg.dependencies,
25
+ ...pkg.devDependencies,
26
+ };
27
+ return '@playwright/test' in allDeps;
28
+ }
29
+ catch {
30
+ return false;
31
+ }
32
+ }
33
+ buildRunCommand(specPath, options) {
34
+ const args = ['playwright', 'test', specPath];
35
+ if (options?.headed) {
36
+ args.push('--headed');
37
+ }
38
+ if (options?.browser) {
39
+ args.push('--browser', options.browser);
40
+ }
41
+ if (options?.project) {
42
+ args.push('--project', options.project);
43
+ }
44
+ if (options?.timeout != null) {
45
+ args.push('--timeout', String(options.timeout));
46
+ }
47
+ return { executable: 'npx', args };
48
+ }
49
+ }
@@ -0,0 +1,59 @@
1
+ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
2
+ // See LICENSE.txt for license information.
3
+ /**
4
+ * Pytest adapter for Python API testing.
5
+ */
6
+ import * as fs from 'node:fs';
7
+ import * as path from 'node:path';
8
+ export class PytestAdapter {
9
+ constructor() {
10
+ this.name = 'pytest';
11
+ this.specGlob = '**/test_*.py';
12
+ this.extractTestPattern = /def\s+(test_\w+)/g;
13
+ this.configFileNames = ['pytest.ini', 'pyproject.toml', 'setup.cfg', 'conftest.py'];
14
+ }
15
+ detect(projectRoot) {
16
+ // Check for common pytest indicator files
17
+ const indicators = ['pyproject.toml', 'pytest.ini', 'conftest.py', 'setup.cfg'];
18
+ for (const file of indicators) {
19
+ const filePath = path.join(projectRoot, file);
20
+ if (!fs.existsSync(filePath))
21
+ continue;
22
+ // For setup.cfg, only match if it contains a [tool:pytest] or [pytest] section
23
+ if (file === 'setup.cfg') {
24
+ try {
25
+ const content = fs.readFileSync(filePath, 'utf-8');
26
+ if (content.includes('[tool:pytest]') || content.includes('[pytest]')) {
27
+ return true;
28
+ }
29
+ }
30
+ catch {
31
+ continue;
32
+ }
33
+ }
34
+ else if (file === 'pyproject.toml') {
35
+ try {
36
+ const content = fs.readFileSync(filePath, 'utf-8');
37
+ if (content.includes('pytest')) {
38
+ return true;
39
+ }
40
+ }
41
+ catch {
42
+ continue;
43
+ }
44
+ }
45
+ else {
46
+ // pytest.ini or conftest.py existence is sufficient
47
+ return true;
48
+ }
49
+ }
50
+ return false;
51
+ }
52
+ buildRunCommand(specPath, options) {
53
+ const args = ['-m', 'pytest', specPath, '-v'];
54
+ if (options?.timeout) {
55
+ args.push(`--timeout=${Math.ceil(options.timeout / 1000)}`);
56
+ }
57
+ return { executable: 'python', args };
58
+ }
59
+ }
@@ -0,0 +1,48 @@
1
+ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
2
+ // See LICENSE.txt for license information.
3
+ /**
4
+ * Supertest + Vitest/Jest adapter for Node.js API testing.
5
+ */
6
+ import * as fs from 'node:fs';
7
+ import * as path from 'node:path';
8
+ export class SupertestAdapter {
9
+ constructor(runner = 'vitest') {
10
+ this.name = 'supertest';
11
+ this.specGlob = '**/*.{test,spec}.{ts,js}';
12
+ this.extractTestPattern = /(?:it|test)\s*\(\s*(['"`])(.*?)\1/g;
13
+ this.configFileNames = ['vitest.config.ts', 'vitest.config.js', 'jest.config.ts', 'jest.config.js'];
14
+ this.runner = runner;
15
+ }
16
+ detect(projectRoot) {
17
+ const pkgPath = path.join(projectRoot, 'package.json');
18
+ if (!fs.existsSync(pkgPath)) {
19
+ return false;
20
+ }
21
+ try {
22
+ const raw = fs.readFileSync(pkgPath, 'utf-8');
23
+ const pkg = JSON.parse(raw);
24
+ const allDeps = {
25
+ ...pkg.dependencies,
26
+ ...pkg.devDependencies,
27
+ };
28
+ return 'supertest' in allDeps;
29
+ }
30
+ catch {
31
+ return false;
32
+ }
33
+ }
34
+ buildRunCommand(specPath, options) {
35
+ if (this.runner === 'jest') {
36
+ const args = ['jest', specPath];
37
+ if (options?.timeout) {
38
+ args.push(`--testTimeout=${options.timeout}`);
39
+ }
40
+ return { executable: 'npx', args };
41
+ }
42
+ const args = ['vitest', 'run', specPath];
43
+ if (options?.timeout) {
44
+ args.push(`--testTimeout=${options.timeout}`);
45
+ }
46
+ return { executable: 'npx', args };
47
+ }
48
+ }
@@ -98,7 +98,7 @@ function isRelevantFile(file) {
98
98
  }
99
99
  return true;
100
100
  }
101
- function runGitRaw(args, cwd) {
101
+ export function runGitRaw(args, cwd) {
102
102
  const result = spawnSync('git', args, {
103
103
  cwd,
104
104
  encoding: 'utf-8',
@@ -182,8 +182,10 @@ function isCommentOnlyDiff(file, repoRoot, baseRef) {
182
182
  export function isTestFile(file) {
183
183
  const normalized = file.replace(/\\/g, '/');
184
184
  return /\.(spec|test)\.(ts|tsx|js|jsx)$/.test(normalized) ||
185
+ /\.snap$/.test(normalized) ||
185
186
  /_test\.go$/.test(normalized) ||
186
187
  normalized.includes('__tests__/') ||
188
+ normalized.includes('__snapshots__/') ||
187
189
  normalized.includes('/tests/') ||
188
190
  normalized.includes('/test/');
189
191
  }
@@ -1,17 +1,18 @@
1
1
  // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
2
2
  // See LICENSE.txt for license information.
3
+ import { sanitizeForPrompt } from '../crew/sanitize.js';
3
4
  export function buildFixPrompt(ctx) {
4
5
  const isCompileError = ctx.failures.some((f) => f.testTitle === '(compile)');
5
6
  const failuresBlock = ctx.failures.map((f) => {
6
- const lines = [` Test: ${f.testTitle}`, ` Error: ${f.error}`];
7
+ const lines = [` Test: ${sanitizeForPrompt(f.testTitle)}`, ` Error: ${sanitizeForPrompt(f.error)}`];
7
8
  if (f.stack)
8
- lines.push(` Stack: ${f.stack}`);
9
+ lines.push(` Stack: ${sanitizeForPrompt(f.stack)}`);
9
10
  if (f.line)
10
11
  lines.push(` Line: ${f.line}`);
11
12
  if (f.expected)
12
- lines.push(` Expected: ${f.expected}`);
13
+ lines.push(` Expected: ${sanitizeForPrompt(f.expected)}`);
13
14
  if (f.actual)
14
- lines.push(` Actual: ${f.actual}`);
15
+ lines.push(` Actual: ${sanitizeForPrompt(f.actual)}`);
15
16
  return lines.join('\n');
16
17
  }).join('\n\n');
17
18
  const errorType = isCompileError ? 'COMPILE ERROR' : 'TEST FAILURE';
@@ -6,17 +6,20 @@ import { runPlaywrightSpec } from './playwright_runner.js';
6
6
  import { generateFix } from './fix_loop.js';
7
7
  import { parseGenerationResponse } from '../prompts/generation.js';
8
8
  import { formatApiSurfaceForPrompt } from '../knowledge/api_surface.js';
9
- function buildGeneratePrompt(scenario, apiSurfaceHint) {
9
+ import { sanitizeForPrompt } from '../crew/sanitize.js';
10
+ function buildGeneratePrompt(scenario, apiSurfaceHint, profile) {
11
+ const projectName = profile?.projectName || 'Mattermost';
12
+ const importSource = profile?.importStatement || '@mattermost/playwright-lib';
10
13
  const scenariosBlock = scenario.scenarios
11
- .map((s, i) => ` ${i + 1}. ${s}`)
14
+ .map((s, i) => ` ${i + 1}. ${sanitizeForPrompt(s)}`)
12
15
  .join('\n');
13
16
  return [
14
- 'Generate a Mattermost Playwright E2E test file.',
17
+ `Generate a ${projectName} Playwright E2E test file.`,
15
18
  '',
16
- `FLOW: ${scenario.name}`,
19
+ `FLOW: ${sanitizeForPrompt(scenario.name)}`,
17
20
  `Route Family: ${scenario.routeFamily}`,
18
21
  `Priority: ${scenario.priority}`,
19
- scenario.evidence ? `Evidence: ${scenario.evidence}` : '',
22
+ scenario.evidence ? `Evidence: ${sanitizeForPrompt(scenario.evidence)}` : '',
20
23
  '',
21
24
  'SCENARIOS TO IMPLEMENT:',
22
25
  scenariosBlock,
@@ -25,14 +28,14 @@ function buildGeneratePrompt(scenario, apiSurfaceHint) {
25
28
  apiSurfaceHint || 'Use page.getByRole() or page.getByTestId() for selectors.',
26
29
  '',
27
30
  'MANDATORY RULES:',
28
- '1. Import ONLY from "@mattermost/playwright-lib" — no other test framework imports.',
31
+ `1. Import ONLY from "${importSource}" — no other test framework imports.`,
29
32
  '2. Every test must call `await pw.initSetup()` first.',
30
33
  '3. Use `await pw.testBrowser.login(user)` to log in — never hardcode credentials.',
31
34
  '4. Use ONLY page object methods listed above. Do NOT invent methods.',
32
35
  '5. If a method is not available, use `page.getByRole()` or `page.getByTestId()`.',
33
36
  `6. Tag every test: {tag: '@${scenario.routeFamily}'}`,
34
37
  '7. Write one test per scenario with a descriptive name.',
35
- '8. Use `expect` from "@mattermost/playwright-lib".',
38
+ `8. Use \`expect\` from "${importSource}".`,
36
39
  '9. Include the copyright header.',
37
40
  '10. NEVER fabricate test IDs (MM-TXXXX). Use descriptive names only.',
38
41
  '',
@@ -41,7 +44,7 @@ function buildGeneratePrompt(scenario, apiSurfaceHint) {
41
44
  '// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.',
42
45
  '// See LICENSE.txt for license information.',
43
46
  '',
44
- "import {expect, test} from '@mattermost/playwright-lib';",
47
+ `import {expect, test} from '${importSource}';`,
45
48
  '',
46
49
  'test(',
47
50
  " 'user can post a message in channel',",
@@ -80,13 +83,13 @@ function resolveSpecPath(scenario, testsRoot) {
80
83
  }
81
84
  return specPath;
82
85
  }
83
- async function generateInitialSpec(provider, scenario, specPath, apiSurfaceHint) {
84
- const prompt = buildGeneratePrompt(scenario, apiSurfaceHint);
86
+ async function generateInitialSpec(provider, scenario, specPath, apiSurfaceHint, profile) {
87
+ const prompt = buildGeneratePrompt(scenario, apiSurfaceHint, profile);
85
88
  const response = await provider.generateText(prompt, {
86
89
  maxTokens: 8000,
87
90
  temperature: 0.1,
88
91
  timeout: 60000,
89
- systemPrompt: 'You are an expert Playwright test writer for Mattermost. Return only TypeScript code.',
92
+ systemPrompt: `You are an expert Playwright test writer for ${profile?.projectName || 'Mattermost'}. Return only TypeScript code.`,
90
93
  });
91
94
  // Reuse existing parsing logic from prompts/generation.ts
92
95
  const parsed = parseGenerationResponse(response.text, specPath, 'create_spec', scenario.id);
@@ -105,7 +108,7 @@ async function runSingleScenario(scenario, options) {
105
108
  // Step 1: Generate initial spec
106
109
  let specCode;
107
110
  try {
108
- specCode = await generateInitialSpec(provider, scenario, specPath, apiHint);
111
+ specCode = await generateInitialSpec(provider, scenario, specPath, apiHint, options.generationProfile);
109
112
  }
110
113
  catch (error) {
111
114
  const msg = error instanceof Error ? error.message : String(error);
@@ -30,7 +30,12 @@ export class CrossImpactAgent {
30
30
  ctx.crossImpacts.push(...deterministicCrossImpacts);
31
31
  // Then: LLM-enriched analysis for semantic cross-impacts
32
32
  try {
33
- const provider = await getCrewProvider(ctx.providerOverride);
33
+ const provider = await getCrewProvider(ctx.providerOverride, ctx.budgetUSD, {
34
+ agentRole: 'cross-impact',
35
+ modelRoutingProviderType: ctx.modelRoutingProviderType,
36
+ modelRoutingOverrides: ctx.modelRoutingOverrides,
37
+ budgetLedger: ctx.budgetLedger,
38
+ });
34
39
  const prompt = buildCrossImpactPrompt({
35
40
  changedFiles: ctx.changedFiles,
36
41
  families: ctx.routeFamilies,
@@ -35,7 +35,12 @@ export class ExecutorAgent {
35
35
  };
36
36
  });
37
37
  try {
38
- const provider = await getCrewProvider(ctx.providerOverride);
38
+ const provider = await getCrewProvider(ctx.providerOverride, ctx.budgetUSD, {
39
+ agentRole: 'executor',
40
+ modelRoutingProviderType: ctx.modelRoutingProviderType,
41
+ modelRoutingOverrides: ctx.modelRoutingOverrides,
42
+ budgetLedger: ctx.budgetLedger,
43
+ });
39
44
  const summary = await runAgenticGeneration({
40
45
  scenarios,
41
46
  config: {
@@ -28,7 +28,12 @@ export class StrategistAgent {
28
28
  regressionRisks: ctx.regressionRisks,
29
29
  });
30
30
  try {
31
- const provider = await getCrewProvider(ctx.providerOverride);
31
+ const provider = await getCrewProvider(ctx.providerOverride, ctx.budgetUSD, {
32
+ agentRole: 'strategist',
33
+ modelRoutingProviderType: ctx.modelRoutingProviderType,
34
+ modelRoutingOverrides: ctx.modelRoutingOverrides,
35
+ budgetLedger: ctx.budgetLedger,
36
+ });
32
37
  const response = await provider.generateText(prompt, {
33
38
  maxTokens: 4000,
34
39
  temperature: 0,
@@ -30,7 +30,12 @@ export class TestDesignerAgent {
30
30
  }
31
31
  let provider;
32
32
  try {
33
- provider = await getCrewProvider(ctx.providerOverride);
33
+ provider = await getCrewProvider(ctx.providerOverride, ctx.budgetUSD, {
34
+ agentRole: 'test-designer',
35
+ modelRoutingProviderType: ctx.modelRoutingProviderType,
36
+ modelRoutingOverrides: ctx.modelRoutingOverrides,
37
+ budgetLedger: ctx.budgetLedger,
38
+ });
34
39
  }
35
40
  catch (error) {
36
41
  const message = error instanceof Error ? error.message : String(error);
@@ -68,6 +68,7 @@ export class AnthropicProvider extends BaseProvider {
68
68
  this.model = config.model || 'claude-sonnet-4-5-20250929';
69
69
  }
70
70
  async generateText(prompt, options) {
71
+ this.checkBudget();
71
72
  const startTime = Date.now();
72
73
  try {
73
74
  // SECURITY: Validate prompt length to prevent resource exhaustion
@@ -1,14 +1,110 @@
1
1
  // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
2
2
  // See LICENSE.txt for license information.
3
+ import { withRetry } from './resilience/retry.js';
4
+ import { CircuitBreaker } from './resilience/circuit_breaker.js';
3
5
  /**
4
6
  * Abstract base class for all LLM providers
5
7
  * Eliminates 240+ lines of duplicate stats management code
6
8
  * Provides common functionality for token tracking, cost calculation, and stats management
7
9
  */
10
+ export class BudgetExceededError extends Error {
11
+ constructor(currentCost, budgetUSD) {
12
+ super(`Budget exceeded: $${currentCost.toFixed(4)} >= $${budgetUSD} limit`);
13
+ this.currentCost = currentCost;
14
+ this.budgetUSD = budgetUSD;
15
+ this.name = 'BudgetExceededError';
16
+ }
17
+ }
8
18
  export class BaseProvider {
9
19
  constructor() {
20
+ /** Tracks the current in-flight budget reservation for this provider instance. */
21
+ this._activeReservation = 0;
10
22
  this.initializeStats();
11
23
  }
24
+ /** Lazily get-or-create a circuit breaker shared across all instances of this provider type. */
25
+ get circuitBreaker() {
26
+ let cb = BaseProvider._sharedBreakers.get(this.name);
27
+ if (!cb) {
28
+ cb = new CircuitBreaker({
29
+ shouldCount: (error) => {
30
+ if (error instanceof BudgetExceededError)
31
+ return false;
32
+ if (!(error instanceof Error))
33
+ return true;
34
+ const msg = error.message.toLowerCase();
35
+ return msg.includes('429') || msg.includes('rate limit') ||
36
+ msg.includes('500') || msg.includes('502') || msg.includes('503') || msg.includes('504') ||
37
+ msg.includes('econnreset') || msg.includes('econnrefused') || msg.includes('etimedout') ||
38
+ msg.includes('overloaded') || msg.includes('socket hang up') || msg.includes('network error');
39
+ },
40
+ });
41
+ BaseProvider._sharedBreakers.set(this.name, cb);
42
+ }
43
+ return cb;
44
+ }
45
+ /**
46
+ * Set a hard budget limit. Once totalCost reaches this value,
47
+ * subsequent calls will throw BudgetExceededError.
48
+ */
49
+ setBudget(usd) {
50
+ this._budgetUSD = usd;
51
+ }
52
+ get budgetUSD() {
53
+ return this._budgetUSD;
54
+ }
55
+ /**
56
+ * Attach a shared budget ledger so aggregate cost across all providers
57
+ * in a crew run is checked before each LLM call.
58
+ */
59
+ setBudgetLedger(ledger) {
60
+ this._ledger = ledger;
61
+ }
62
+ /**
63
+ * Check budget and pre-reserve estimated cost for the upcoming LLM call.
64
+ *
65
+ * When a shared ledger exists, reserves an estimate derived from the provider's
66
+ * output token cost × maxTokens (default 4096). This blocks parallel agents from
67
+ * spending into the same headroom — like a credit card authorization hold.
68
+ *
69
+ * Self-healing: if a prior call failed without reaching updateStats(), the stale
70
+ * reservation is released here before placing the new one.
71
+ */
72
+ checkBudget() {
73
+ if (this._ledger) {
74
+ // Release stale reservation from a prior failed call that never hit updateStats
75
+ if (this._activeReservation > 0) {
76
+ this._ledger.release(this._activeReservation);
77
+ this._activeReservation = 0;
78
+ }
79
+ // Reserve estimated cost for the upcoming call
80
+ const estimate = this.estimateCallCost();
81
+ this._ledger.reserve(estimate);
82
+ this._activeReservation = estimate;
83
+ try {
84
+ this._ledger.check();
85
+ }
86
+ catch (err) {
87
+ // Budget exceeded — release reservation immediately so it doesn't leak
88
+ this._ledger.release(estimate);
89
+ this._activeReservation = 0;
90
+ throw err;
91
+ }
92
+ return;
93
+ }
94
+ if (this._budgetUSD !== undefined && this.stats.totalCost >= this._budgetUSD) {
95
+ throw new BudgetExceededError(this.stats.totalCost, this._budgetUSD);
96
+ }
97
+ }
98
+ /**
99
+ * Conservative cost estimate for the upcoming call.
100
+ * Uses maxTokens (or 4096 default) × output cost rate.
101
+ * Overestimating is safe — the reservation is replaced with actual cost in updateStats.
102
+ */
103
+ estimateCallCost() {
104
+ const outputTokenEstimate = 4096;
105
+ const costRate = this.capabilities?.costPer1MOutputTokens ?? 15; // default to ~Sonnet
106
+ return (outputTokenEstimate / 1000000) * costRate;
107
+ }
12
108
  /**
13
109
  * Initialize stats object with default values
14
110
  */
@@ -35,6 +131,14 @@ export class BaseProvider {
35
131
  this.stats.totalOutputTokens += usage.outputTokens;
36
132
  this.stats.totalTokens += usage.totalTokens;
37
133
  this.stats.totalCost += cost;
134
+ if (this._ledger) {
135
+ // Settle: release the estimate, record actual
136
+ if (this._activeReservation > 0) {
137
+ this._ledger.release(this._activeReservation);
138
+ this._activeReservation = 0;
139
+ }
140
+ this._ledger.record(cost);
141
+ }
38
142
  // Update rolling average response time
39
143
  const totalRequests = this.stats.requestCount;
40
144
  this.stats.averageResponseTimeMs =
@@ -53,6 +157,17 @@ export class BaseProvider {
53
157
  resetUsageStats() {
54
158
  this.initializeStats();
55
159
  }
160
+ /**
161
+ * Wrap an async call with circuit breaker + retry logic.
162
+ * Circuit breaker protects against cascading failures from a down provider;
163
+ * retry handles transient failures within a healthy circuit.
164
+ *
165
+ * Non-transient errors (budget, auth, validation) are thrown directly and
166
+ * bypass the circuit breaker so they don't incorrectly trip it.
167
+ */
168
+ retryCall(fn) {
169
+ return this.circuitBreaker.call(() => withRetry(fn, { maxRetries: 2, baseDelayMs: 1000, maxDelayMs: 10000, jitter: true }), () => { throw new Error(`${this.name} provider circuit open — too many consecutive failures`); });
170
+ }
56
171
  /**
57
172
  * Calculate cost for token usage, accounting for prompt caching discounts
58
173
  * Cached tokens cost 90% less than regular tokens
@@ -75,3 +190,9 @@ export class BaseProvider {
75
190
  return inputCost + outputCost;
76
191
  }
77
192
  }
193
+ /**
194
+ * Shared circuit breakers keyed by provider name (e.g., "anthropic", "openai").
195
+ * All instances of the same provider type share one breaker, so if Anthropic is
196
+ * down, ALL agents discover it after 3 total failures instead of 3 × N.
197
+ */
198
+ BaseProvider._sharedBreakers = new Map();
@@ -0,0 +1,58 @@
1
+ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
2
+ // See LICENSE.txt for license information.
3
+ /**
4
+ * Shared budget ledger — tracks aggregate cost across all provider instances
5
+ * in a single crew run. Prevents parallel agents from each seeing only 1/N
6
+ * of actual spend and overshooting the budget by N×limit.
7
+ *
8
+ * Usage: create one BudgetLedger per crew run, pass it to getCrewProvider(),
9
+ * which attaches it to each provider via setBudgetLedger().
10
+ */
11
+ import { BudgetExceededError } from './base_provider.js';
12
+ export class BudgetLedger {
13
+ constructor(limitUSD) {
14
+ this._totalCost = 0;
15
+ this._reserved = 0;
16
+ this._limitUSD = limitUSD;
17
+ }
18
+ get totalCost() {
19
+ return this._totalCost;
20
+ }
21
+ get limitUSD() {
22
+ return this._limitUSD;
23
+ }
24
+ /**
25
+ * Record actual cost from a completed LLM call.
26
+ */
27
+ record(cost) {
28
+ if (!Number.isFinite(cost) || cost < 0)
29
+ return;
30
+ this._totalCost += cost;
31
+ }
32
+ /**
33
+ * Pre-reserve estimated cost before an LLM call begins.
34
+ * Blocks parallel agents from spending into the same headroom.
35
+ * Like a credit card authorization hold.
36
+ */
37
+ reserve(estimate) {
38
+ if (!Number.isFinite(estimate) || estimate <= 0)
39
+ return;
40
+ this._reserved += estimate;
41
+ }
42
+ /**
43
+ * Release a prior reservation (after API response or on error).
44
+ */
45
+ release(estimate) {
46
+ this._reserved = Math.max(0, this._reserved - estimate);
47
+ }
48
+ /**
49
+ * Throws BudgetExceededError if committed cost + in-flight reservations
50
+ * have reached the limit.
51
+ */
52
+ check() {
53
+ const effective = this._totalCost + this._reserved;
54
+ if (effective >= this._limitUSD) {
55
+ throw new BudgetExceededError(effective, this._limitUSD);
56
+ }
57
+ }
58
+ }