@yasserkhanorg/e2e-agents 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/LICENSE +168 -0
  2. package/README.md +620 -0
  3. package/dist/agent/analysis.d.ts +62 -0
  4. package/dist/agent/analysis.d.ts.map +1 -0
  5. package/dist/agent/analysis.js +292 -0
  6. package/dist/agent/blast_radius.d.ts +4 -0
  7. package/dist/agent/blast_radius.d.ts.map +1 -0
  8. package/dist/agent/blast_radius.js +37 -0
  9. package/dist/agent/cache_utils.d.ts +38 -0
  10. package/dist/agent/cache_utils.d.ts.map +1 -0
  11. package/dist/agent/cache_utils.js +67 -0
  12. package/dist/agent/config.d.ts +148 -0
  13. package/dist/agent/config.d.ts.map +1 -0
  14. package/dist/agent/config.js +640 -0
  15. package/dist/agent/dependency_graph.d.ts +14 -0
  16. package/dist/agent/dependency_graph.d.ts.map +1 -0
  17. package/dist/agent/dependency_graph.js +227 -0
  18. package/dist/agent/feedback.d.ts +55 -0
  19. package/dist/agent/feedback.d.ts.map +1 -0
  20. package/dist/agent/feedback.js +257 -0
  21. package/dist/agent/flags.d.ts +23 -0
  22. package/dist/agent/flags.d.ts.map +1 -0
  23. package/dist/agent/flags.js +171 -0
  24. package/dist/agent/flow_catalog.d.ts +25 -0
  25. package/dist/agent/flow_catalog.d.ts.map +1 -0
  26. package/dist/agent/flow_catalog.js +106 -0
  27. package/dist/agent/flow_mapping.d.ts +10 -0
  28. package/dist/agent/flow_mapping.d.ts.map +1 -0
  29. package/dist/agent/flow_mapping.js +84 -0
  30. package/dist/agent/framework.d.ts +13 -0
  31. package/dist/agent/framework.d.ts.map +1 -0
  32. package/dist/agent/framework.js +149 -0
  33. package/dist/agent/gap_suggestions.d.ts +14 -0
  34. package/dist/agent/gap_suggestions.d.ts.map +1 -0
  35. package/dist/agent/gap_suggestions.js +101 -0
  36. package/dist/agent/generator.d.ts +10 -0
  37. package/dist/agent/generator.d.ts.map +1 -0
  38. package/dist/agent/generator.js +115 -0
  39. package/dist/agent/git.d.ts +11 -0
  40. package/dist/agent/git.d.ts.map +1 -0
  41. package/dist/agent/git.js +90 -0
  42. package/dist/agent/handoff.d.ts +22 -0
  43. package/dist/agent/handoff.d.ts.map +1 -0
  44. package/dist/agent/handoff.js +180 -0
  45. package/dist/agent/impact-analyzer.d.ts +114 -0
  46. package/dist/agent/impact-analyzer.d.ts.map +1 -0
  47. package/dist/agent/impact-analyzer.js +557 -0
  48. package/dist/agent/index.d.ts +21 -0
  49. package/dist/agent/index.d.ts.map +1 -0
  50. package/dist/agent/index.js +38 -0
  51. package/dist/agent/model-router.d.ts +57 -0
  52. package/dist/agent/model-router.d.ts.map +1 -0
  53. package/dist/agent/model-router.js +154 -0
  54. package/dist/agent/operational_insights.d.ts +41 -0
  55. package/dist/agent/operational_insights.d.ts.map +1 -0
  56. package/dist/agent/operational_insights.js +126 -0
  57. package/dist/agent/pipeline.d.ts +23 -0
  58. package/dist/agent/pipeline.d.ts.map +1 -0
  59. package/dist/agent/pipeline.js +609 -0
  60. package/dist/agent/plan.d.ts +91 -0
  61. package/dist/agent/plan.d.ts.map +1 -0
  62. package/dist/agent/plan.js +331 -0
  63. package/dist/agent/playwright_report.d.ts +8 -0
  64. package/dist/agent/playwright_report.d.ts.map +1 -0
  65. package/dist/agent/playwright_report.js +126 -0
  66. package/dist/agent/report-generator.d.ts +24 -0
  67. package/dist/agent/report-generator.d.ts.map +1 -0
  68. package/dist/agent/report-generator.js +250 -0
  69. package/dist/agent/report.d.ts +81 -0
  70. package/dist/agent/report.d.ts.map +1 -0
  71. package/dist/agent/report.js +147 -0
  72. package/dist/agent/runner.d.ts +7 -0
  73. package/dist/agent/runner.d.ts.map +1 -0
  74. package/dist/agent/runner.js +576 -0
  75. package/dist/agent/selectors.d.ts +10 -0
  76. package/dist/agent/selectors.d.ts.map +1 -0
  77. package/dist/agent/selectors.js +75 -0
  78. package/dist/agent/spec-bridge.d.ts +101 -0
  79. package/dist/agent/spec-bridge.d.ts.map +1 -0
  80. package/dist/agent/spec-bridge.js +273 -0
  81. package/dist/agent/spec-builder.d.ts +102 -0
  82. package/dist/agent/spec-builder.d.ts.map +1 -0
  83. package/dist/agent/spec-builder.js +273 -0
  84. package/dist/agent/subsystem_risk.d.ts +23 -0
  85. package/dist/agent/subsystem_risk.d.ts.map +1 -0
  86. package/dist/agent/subsystem_risk.js +207 -0
  87. package/dist/agent/telemetry.d.ts +84 -0
  88. package/dist/agent/telemetry.d.ts.map +1 -0
  89. package/dist/agent/telemetry.js +220 -0
  90. package/dist/agent/test_path.d.ts +2 -0
  91. package/dist/agent/test_path.d.ts.map +1 -0
  92. package/dist/agent/test_path.js +23 -0
  93. package/dist/agent/tests.d.ts +18 -0
  94. package/dist/agent/tests.d.ts.map +1 -0
  95. package/dist/agent/tests.js +106 -0
  96. package/dist/agent/traceability.d.ts +22 -0
  97. package/dist/agent/traceability.d.ts.map +1 -0
  98. package/dist/agent/traceability.js +183 -0
  99. package/dist/agent/traceability_capture.d.ts +18 -0
  100. package/dist/agent/traceability_capture.d.ts.map +1 -0
  101. package/dist/agent/traceability_capture.js +313 -0
  102. package/dist/agent/traceability_ingest.d.ts +21 -0
  103. package/dist/agent/traceability_ingest.d.ts.map +1 -0
  104. package/dist/agent/traceability_ingest.js +237 -0
  105. package/dist/agent/utils.d.ts +13 -0
  106. package/dist/agent/utils.d.ts.map +1 -0
  107. package/dist/agent/utils.js +152 -0
  108. package/dist/agent/validators/selector-validator.d.ts +74 -0
  109. package/dist/agent/validators/selector-validator.d.ts.map +1 -0
  110. package/dist/agent/validators/selector-validator.js +165 -0
  111. package/dist/anthropic_provider.d.ts +65 -0
  112. package/dist/anthropic_provider.d.ts.map +1 -0
  113. package/dist/anthropic_provider.js +332 -0
  114. package/dist/api.d.ts +48 -0
  115. package/dist/api.d.ts.map +1 -0
  116. package/dist/api.js +113 -0
  117. package/dist/base_provider.d.ts +53 -0
  118. package/dist/base_provider.d.ts.map +1 -0
  119. package/dist/base_provider.js +81 -0
  120. package/dist/cli.d.ts +3 -0
  121. package/dist/cli.d.ts.map +1 -0
  122. package/dist/cli.js +843 -0
  123. package/dist/custom_provider.d.ts +20 -0
  124. package/dist/custom_provider.d.ts.map +1 -0
  125. package/dist/custom_provider.js +276 -0
  126. package/dist/e2e-test-gen/index.d.ts +51 -0
  127. package/dist/e2e-test-gen/index.d.ts.map +1 -0
  128. package/dist/e2e-test-gen/index.js +57 -0
  129. package/dist/e2e-test-gen/spec_parser.d.ts +142 -0
  130. package/dist/e2e-test-gen/spec_parser.d.ts.map +1 -0
  131. package/dist/e2e-test-gen/spec_parser.js +786 -0
  132. package/dist/e2e-test-gen/types.d.ts +185 -0
  133. package/dist/e2e-test-gen/types.d.ts.map +1 -0
  134. package/dist/e2e-test-gen/types.js +4 -0
  135. package/dist/esm/agent/analysis.js +287 -0
  136. package/dist/esm/agent/blast_radius.js +34 -0
  137. package/dist/esm/agent/cache_utils.js +63 -0
  138. package/dist/esm/agent/config.js +637 -0
  139. package/dist/esm/agent/dependency_graph.js +224 -0
  140. package/dist/esm/agent/feedback.js +253 -0
  141. package/dist/esm/agent/flags.js +160 -0
  142. package/dist/esm/agent/flow_catalog.js +103 -0
  143. package/dist/esm/agent/flow_mapping.js +81 -0
  144. package/dist/esm/agent/framework.js +145 -0
  145. package/dist/esm/agent/gap_suggestions.js +98 -0
  146. package/dist/esm/agent/generator.js +112 -0
  147. package/dist/esm/agent/git.js +87 -0
  148. package/dist/esm/agent/handoff.js +177 -0
  149. package/dist/esm/agent/impact-analyzer.js +548 -0
  150. package/dist/esm/agent/index.js +22 -0
  151. package/dist/esm/agent/model-router.js +150 -0
  152. package/dist/esm/agent/operational_insights.js +123 -0
  153. package/dist/esm/agent/pipeline.js +605 -0
  154. package/dist/esm/agent/plan.js +324 -0
  155. package/dist/esm/agent/playwright_report.js +123 -0
  156. package/dist/esm/agent/report-generator.js +247 -0
  157. package/dist/esm/agent/report.js +144 -0
  158. package/dist/esm/agent/runner.js +572 -0
  159. package/dist/esm/agent/selectors.js +71 -0
  160. package/dist/esm/agent/spec-bridge.js +267 -0
  161. package/dist/esm/agent/spec-builder.js +267 -0
  162. package/dist/esm/agent/subsystem_risk.js +204 -0
  163. package/dist/esm/agent/telemetry.js +216 -0
  164. package/dist/esm/agent/test_path.js +20 -0
  165. package/dist/esm/agent/tests.js +101 -0
  166. package/dist/esm/agent/traceability.js +180 -0
  167. package/dist/esm/agent/traceability_capture.js +310 -0
  168. package/dist/esm/agent/traceability_ingest.js +234 -0
  169. package/dist/esm/agent/utils.js +138 -0
  170. package/dist/esm/agent/validators/selector-validator.js +160 -0
  171. package/dist/esm/anthropic_provider.js +324 -0
  172. package/dist/esm/api.js +105 -0
  173. package/dist/esm/base_provider.js +77 -0
  174. package/dist/esm/cli.js +841 -0
  175. package/dist/esm/custom_provider.js +272 -0
  176. package/dist/esm/e2e-test-gen/index.js +50 -0
  177. package/dist/esm/e2e-test-gen/spec_parser.js +782 -0
  178. package/dist/esm/e2e-test-gen/types.js +3 -0
  179. package/dist/esm/index.js +16 -0
  180. package/dist/esm/logger.js +89 -0
  181. package/dist/esm/mcp-server.js +465 -0
  182. package/dist/esm/ollama_provider.js +300 -0
  183. package/dist/esm/openai_provider.js +242 -0
  184. package/dist/esm/package.json +3 -0
  185. package/dist/esm/plan-and-test-constants.js +126 -0
  186. package/dist/esm/provider_factory.js +336 -0
  187. package/dist/esm/provider_interface.js +23 -0
  188. package/dist/esm/provider_utils.js +96 -0
  189. package/dist/index.d.ts +31 -0
  190. package/dist/index.d.ts.map +1 -0
  191. package/dist/index.js +41 -0
  192. package/dist/logger.d.ts +23 -0
  193. package/dist/logger.d.ts.map +1 -0
  194. package/dist/logger.js +93 -0
  195. package/dist/mcp-server.d.ts +35 -0
  196. package/dist/mcp-server.d.ts.map +1 -0
  197. package/dist/mcp-server.js +469 -0
  198. package/dist/ollama_provider.d.ts +65 -0
  199. package/dist/ollama_provider.d.ts.map +1 -0
  200. package/dist/ollama_provider.js +308 -0
  201. package/dist/openai_provider.d.ts +23 -0
  202. package/dist/openai_provider.d.ts.map +1 -0
  203. package/dist/openai_provider.js +250 -0
  204. package/dist/plan-and-test-constants.d.ts +110 -0
  205. package/dist/plan-and-test-constants.d.ts.map +1 -0
  206. package/dist/plan-and-test-constants.js +132 -0
  207. package/dist/provider_factory.d.ts +99 -0
  208. package/dist/provider_factory.d.ts.map +1 -0
  209. package/dist/provider_factory.js +341 -0
  210. package/dist/provider_interface.d.ts +358 -0
  211. package/dist/provider_interface.d.ts.map +1 -0
  212. package/dist/provider_interface.js +28 -0
  213. package/dist/provider_utils.d.ts +39 -0
  214. package/dist/provider_utils.d.ts.map +1 -0
  215. package/dist/provider_utils.js +103 -0
  216. package/package.json +101 -0
  217. package/schemas/gap.schema.json +18 -0
  218. package/schemas/impact.schema.json +418 -0
  219. package/schemas/plan.schema.json +285 -0
  220. package/schemas/subsystem-risk-map.schema.json +62 -0
  221. package/schemas/traceability-input.schema.json +122 -0
package/README.md ADDED
@@ -0,0 +1,620 @@
1
+ # @yasserkhanorg/e2e-agents
2
+
3
+ Framework-agnostic LLM provider library with MCP server for autonomous E2E testing.
4
+
5
+ [![npm](https://img.shields.io/npm/v/%40yasserkhanorg%2Fe2e-agents)](https://www.npmjs.com/package/@yasserkhanorg/e2e-agents)
6
+ [![License](https://img.shields.io/badge/license-Apache%202.0-blue)](LICENSE)
7
+ [![GitHub](https://img.shields.io/badge/github-yasserfaraazkhan%2Fe2e--agents-blue?logo=github)](https://github.com/yasserfaraazkhan/e2e-agents)
8
+
9
+ ## Overview
10
+
11
+ Pluggable LLM provider abstraction for test automation with:
12
+ - **Anthropic Claude** — Advanced reasoning, vision support
13
+ - **OpenAI GPT** — Official OpenAI API integration
14
+ - **Ollama** — Free, local, privacy-first
15
+ - **MCP Server** — 6 tools for test discovery, generation, and healing
16
+ - **Custom Providers** — Extend with any OpenAI-compatible API
17
+
18
+ ## Installation
19
+
20
+ ```bash
21
+ npm install @yasserkhanorg/e2e-agents
22
+ ```
23
+
24
+ ## Module Formats (CJS + ESM)
25
+
26
+ This package ships both CommonJS and ESM builds:
27
+ - `require('@yasserkhanorg/e2e-agents')` loads the CommonJS build from `dist/index.js`.
28
+ - `import ... from '@yasserkhanorg/e2e-agents'` loads the ESM build from `dist/esm/index.js`.
29
+ - `./mcp` follows the same pattern (`dist/mcp-server.js` for CJS, `dist/esm/mcp-server.js` for ESM).
30
+
31
+ Node.js >= 20 is required.
32
+
33
+ ## Quick Links
34
+
35
+ 📖 **[Comprehensive Guide](E2E_AI_TESTING.md)** - In-depth documentation including:
36
+ - How to use e2e-ai-agents in your projects
37
+ - Real-world examples for Playwright, Cypress, Selenium
38
+ - How Mattermost uses this package
39
+ - Cost optimization and best practices
40
+
41
+ ## Quick Start
42
+
43
+ ### Use Claude
44
+
45
+ ```typescript
46
+ import { AnthropicProvider } from '@yasserkhanorg/e2e-agents';
47
+
48
+ const claude = new AnthropicProvider({
49
+ apiKey: process.env.ANTHROPIC_API_KEY
50
+ });
51
+
52
+ const response = await claude.generateText('Analyze test failure');
53
+ console.log(response.text);
54
+ console.log(`Cost: $${response.cost.toFixed(4)}`);
55
+ ```
56
+
57
+ ### Use OpenAI
58
+
59
+ ```typescript
60
+ import { OpenAIProvider } from '@yasserkhanorg/e2e-agents';
61
+
62
+ const openai = new OpenAIProvider({
63
+ apiKey: process.env.OPENAI_API_KEY,
64
+ model: 'gpt-4'
65
+ });
66
+
67
+ const response = await openai.generateText('Summarize test failure');
68
+ console.log(response.text);
69
+ ```
70
+
71
+ Tip: for accurate OpenAI cost tracking, set `costPer1MInputTokens` and `costPer1MOutputTokens` in the `OpenAIProvider` config.
72
+
73
+ ### Use Ollama (Free)
74
+
75
+ ```typescript
76
+ import { OllamaProvider } from '@yasserkhanorg/e2e-agents';
77
+
78
+ const ollama = new OllamaProvider({
79
+ model: 'deepseek-r1:7b'
80
+ });
81
+
82
+ const response = await ollama.generateText('Generate test case');
83
+ console.log(response.text); // Free!
84
+ ```
85
+
86
+ ### Use Custom Provider (OpenAI-compatible endpoint)
87
+
88
+ ```typescript
89
+ import { CustomProvider } from '@yasserkhanorg/e2e-agents';
90
+
91
+ const custom = new CustomProvider({
92
+ baseUrl: 'https://your-llm-gateway.example.com/v1',
93
+ auth: { Authorization: `Bearer ${process.env.CUSTOM_API_KEY}` },
94
+ model: 'your-model-name',
95
+ requestFormat: 'openai'
96
+ });
97
+
98
+ const response = await custom.generateText('Generate test case');
99
+ console.log(response.text);
100
+ ```
101
+
102
+ `requestFormat` can be `'openai'`, `'anthropic'`, or `'custom'` (with `transformRequest`/`transformResponse`).
103
+
104
+ ### Factory Pattern
105
+
106
+ ```typescript
107
+ import { LLMProviderFactory } from '@yasserkhanorg/e2e-agents';
108
+
109
+ // Auto-detect from environment
110
+ const provider = LLMProviderFactory.create({
111
+ type: 'anthropic',
112
+ config: { apiKey: process.env.ANTHROPIC_API_KEY }
113
+ });
114
+ ```
115
+
116
+ ### Hybrid Mode (Free + Premium)
117
+
118
+ ```typescript
119
+ const provider = LLMProviderFactory.createHybrid({
120
+ primary: { type: 'ollama', config: { model: 'deepseek-r1:7b' } },
121
+ fallback: { type: 'anthropic', config: { apiKey: process.env.ANTHROPIC_API_KEY } },
122
+ useFallbackFor: ['vision'] // Only use Claude for vision
123
+ });
124
+
125
+ await provider.generateText('Analyze code'); // Uses Ollama (free)
126
+ await provider.analyzeImage([...], 'Compare screenshots'); // Uses Claude (vision)
127
+ ```
128
+
129
+ ## CLI: Impact and Gap Analysis
130
+
131
+ Run AI-driven impact analysis or gap analysis on any frontend repo.
132
+
133
+ ```bash
134
+ npx e2e-ai-agents impact --path /path/to/webapp
135
+ npx e2e-ai-agents gap --path /path/to/webapp
136
+ npx e2e-ai-agents suggest --path /path/to/webapp
137
+ npx e2e-ai-agents approve-and-generate --path /path/to/webapp
138
+ npx e2e-ai-agents finalize-generated-tests --path /path/to/webapp
139
+ npx e2e-ai-agents feedback --path /path/to/webapp --feedback-input ./feedback.json
140
+ npx e2e-ai-agents traceability-capture --path /path/to/webapp --traceability-report ./playwright-report.json
141
+ npx e2e-ai-agents traceability-ingest --path /path/to/webapp --traceability-input ./traceability-input.json
142
+ ```
143
+
144
+ If tests live outside the app root:
145
+
146
+ ```bash
147
+ npx e2e-ai-agents impact --path /path/to/webapp --tests-root /path/to/e2e-tests
148
+ ```
149
+
150
+ Optional config file `e2e-ai-agents.config.json` (JSON):
151
+
152
+ ```json
153
+ {
154
+ "path": ".",
155
+ "testsRoot": ".",
156
+ "flowCatalogPath": ".e2e-ai-agents/flows.json",
157
+ "mode": "impact",
158
+ "framework": "auto",
159
+ "timeLimitMinutes": 10,
160
+ "budget": { "maxUSD": 2, "maxTokens": 20000 },
161
+ "artifacts": { "mode": "commit", "specsDir": ".e2e-ai-agents/reports" },
162
+ "selectors": { "patchOnApply": true },
163
+ "testDiscovery": { "patterns": ["tests/**/*.spec.ts"] },
164
+ "flowDiscovery": {
165
+ "patterns": ["channels/src/components/**/*.{tsx,jsx}"],
166
+ "exclude": ["**/components/**/stories/**"]
167
+ },
168
+ "catalogScoring": {
169
+ "priorityScores": { "P0": 10, "P1": 6, "P2": 3 },
170
+ "fileMatchWeight": 1
171
+ },
172
+ "impact": {
173
+ "allowFallback": false,
174
+ "dependencyGraph": {
175
+ "enabled": true,
176
+ "maxDepth": 3,
177
+ "maxExpandedFiles": 1000,
178
+ "filePatterns": ["**/*.{ts,tsx,js,jsx}"],
179
+ "excludePatterns": ["**/node_modules/**", "**/.git/**", "**/dist/**", "**/build/**"],
180
+ "aliasRoots": ["src", "channels/src"],
181
+ "pathAliases": {
182
+ "@app/*": ["src/*"],
183
+ "@channels/*": ["channels/src/*"]
184
+ }
185
+ },
186
+ "traceability": {
187
+ "enabled": true,
188
+ "manifestPath": ".e2e-ai-agents/traceability.json",
189
+ "minSignalsPerTest": 1
190
+ },
191
+ "subsystemRisk": {
192
+ "enabled": false,
193
+ "mapPath": ".e2e-ai-agents/subsystem-risk-map.json",
194
+ "maxRulesPerFile": 4
195
+ }
196
+ },
197
+ "pipeline": {
198
+ "enabled": false,
199
+ "scenarios": 3,
200
+ "outputDir": "specs/functional/ai-assisted",
201
+ "heal": true,
202
+ "mcp": false
203
+ },
204
+ "llm": { "provider": "anthropic", "fallback": "ollama" },
205
+ "policy": {
206
+ "minConfidenceForTargeted": 60,
207
+ "safeMergeMinConfidence": 85,
208
+ "forceFullOnWarningsAtOrAbove": 2,
209
+ "forceFullOnP0WithGaps": true,
210
+ "forceFullOnRiskyFiles": true,
211
+ "riskyFilePatterns": ["**/auth/**", "**/permissions/**", "**/security/**", "**/*.sql"]
212
+ },
213
+ "flags": { "defaultState": "on" },
214
+ "audience": { "defaultRoles": ["member"] },
215
+ "blastRadius": {
216
+ "memberBonus": 1,
217
+ "guestBonus": 1,
218
+ "adminOnlyPenalty": -1,
219
+ "flagOffPenalty": -2
220
+ }
221
+ }
222
+ ```
223
+
224
+ Notes:
225
+ - If no framework config is found, provide `testDiscovery.patterns` or `--patterns`.
226
+ - Use `flowDiscovery.patterns` or `--flow-patterns` to customize flow scanning.
227
+ - Use `testsRoot` when tests live outside the app root.
228
+ - Use `flowCatalogPath` or `--flow-catalog` to provide a flow catalog for deterministic P0/P1 mapping.
229
+ - Impact mode expects a git diff; use `--since` or add `"impact": { "allowFallback": true }` to fall back to scanning.
230
+ - Impact analysis now uses static reverse dependency graph expansion (configurable via `impact.dependencyGraph`) to propagate changed-file impact, including alias imports via `aliasRoots` and `pathAliases`.
231
+ - Impact analysis can use coverage-style traceability manifests (`impact.traceability`) for file->test mapping with heuristic fallback for uncovered flows.
232
+ - Impact analysis can apply subsystem-aware risk boosts and priority floors from a map (`impact.subsystemRisk`) to capture known high-blast-radius areas.
233
+ - Diffing is computed from `merge-base(<since>, HEAD)` when available, which is the standard PR-impact baseline.
234
+ - Reports are written under `testsRoot/.e2e-ai-agents/reports` (or app root if `testsRoot` is not set).
235
+ - Use `approve-and-generate` for explicit approval before patching `data-testid` and generating/healing tests.
236
+ - `--apply` remains available as a legacy shortcut for direct `gap` execution.
237
+ - Use `--pipeline` to run the Playwright generation pipeline.
238
+ - If `e2e-test-gen-cli.ts` exists in `testsRoot`, it is used as the advanced runner.
239
+ - If it is absent, `@yasserkhanorg/e2e-agents` falls back to package-native generation with strategy-based templates, quality guardrails (`no test.describe`, single tag), and iterative heal attempts.
240
+ - Use `--pipeline-mcp` (or `"pipeline": { "mcp": true }`) to run exploration/healing via Playwright MCP.
241
+ - `suggest` writes `.e2e-ai-agents/plan.json` with `runSet` (`smoke|targeted|full`) and confidence.
242
+ - `suggest` also writes `.e2e-ai-agents/ci-summary.md` with CI status: `run-now`, `must-add-tests`, or `safe-to-merge`.
243
+ - CLI policy overrides: `--policy-min-confidence`, `--policy-safe-merge-confidence`, `--policy-force-full-on-warnings`, `--policy-risky-patterns`.
244
+ - GitHub Actions output wiring: `--github-output $GITHUB_OUTPUT`.
245
+ - Optional merge gating: `--fail-on-must-add-tests` exits non-zero when uncovered P0/P1 gaps are detected. Leave this flag unset for advisory-only mode.
246
+ - `impact/gap` now include actionable `testSuggestions` with linked source files and skeleton test code.
247
+ - `impact/gap` now include `impactModel` metadata (`flowMapping`, `testMapping`, `confidenceClass`, traceability stats, dependency graph stats).
248
+ - `impact/gap` now include optional `impactModel.subsystemRisk` stats (map status, matched files/rules, boosted flows).
249
+ - `feedback` appends outcomes to `.e2e-ai-agents/feedback.json` and recomputes `.e2e-ai-agents/calibration.json`.
250
+ - `feedback` also computes intelligent flaky scores into `.e2e-ai-agents/flaky-tests.json`.
251
+ - `traceability-capture` converts Playwright JSON execution report + optional coverage map into `.e2e-ai-agents/traceability-input.json`.
252
+ - `traceability-ingest` merges CI execution mappings into `.e2e-ai-agents/traceability.json` and persists rolling counts in `.e2e-ai-agents/traceability-state.json`.
253
+ - Traceability capture flags: `--traceability-report`, `--traceability-capture-output`, `--traceability-coverage-map`, `--traceability-changed-files`.
254
+ - Traceability ingest tuning flags: `--traceability-min-hits`, `--traceability-max-files-per-test`, `--traceability-max-age-days`.
255
+ - Optional ownership routing for flaky alerts: `.e2e-ai-agents/subsystem-owners.json`.
256
+ - `suggest` automatically consumes optional operational manifests:
257
+ - `.e2e-ai-agents/flaky-tests.json`
258
+ - `.e2e-ai-agents/quality-gates.json`
259
+ - `plan.json` includes `nextActions` commands for run/approve-and-generate/heal/finalize/PR handoff.
260
+ - `finalize-generated-tests` stages generated artifacts from `gap.json`, commits, and can open a PR with `--create-pr`.
261
+ - Generated Mattermost Playwright tests use standalone `test(...)` style (no `test.describe`) and a single tag string.
262
+
263
+ Programmatic API:
264
+
265
+ ```typescript
266
+ import {analyzeImpact, findGaps, recommendTests, captureTraceability, ingestTraceability} from '@yasserkhanorg/e2e-agents';
267
+
268
+ await analyzeImpact({path: '/path/to/webapp'});
269
+ await findGaps({path: '/path/to/webapp'});
270
+ const suggestion = await recommendTests({path: '/path/to/webapp'});
271
+ console.log(suggestion.plan.runSet);
272
+
273
+ const captured = captureTraceability({
274
+ path: '/path/to/webapp',
275
+ testsRoot: '/path/to/e2e-tests/playwright',
276
+ reportPath: '/path/to/playwright-report.json',
277
+ });
278
+
279
+ ingestTraceability({
280
+ path: '/path/to/webapp',
281
+ testsRoot: '/path/to/e2e-tests/playwright',
282
+ payload: JSON.parse(require('fs').readFileSync(captured.outputPath, 'utf8')),
283
+ });
284
+ ```
285
+
286
+ Feedback API:
287
+
288
+ ```typescript
289
+ import {appendFeedbackAndRecompute} from '@yasserkhanorg/e2e-agents';
290
+
291
+ appendFeedbackAndRecompute('/path/to/webapp', {
292
+ timestamp: new Date().toISOString(),
293
+ runSet: 'targeted',
294
+ recommendedTests: ['specs/channels/realtime.spec.ts'],
295
+ executedTests: ['specs/channels/realtime.spec.ts'],
296
+ failedTests: ['specs/channels/realtime.spec.ts'],
297
+ escapedFailures: []
298
+ });
299
+ ```
300
+
301
+ Traceability ingest API:
302
+
303
+ ```typescript
304
+ import {ingestTraceability} from '@yasserkhanorg/e2e-agents';
305
+
306
+ ingestTraceability({
307
+ path: '/path/to/webapp',
308
+ testsRoot: '/path/to/e2e-tests/playwright',
309
+ payload: {
310
+ runs: [
311
+ {
312
+ test: 'specs/channels/channels.switch.spec.ts',
313
+ touchedFiles: ['channels/src/components/channel_switcher/channel_switcher.tsx']
314
+ }
315
+ ]
316
+ },
317
+ options: {minHits: 2}
318
+ });
319
+ ```
320
+
321
+ Automation API:
322
+
323
+ ```typescript
324
+ import {handoffGeneratedTests} from '@yasserkhanorg/e2e-agents';
325
+
326
+ handoffGeneratedTests({
327
+ appPath: '/path/to/webapp',
328
+ testsRoot: '/path/to/e2e-tests/playwright',
329
+ createPr: true,
330
+ });
331
+ ```
332
+
333
+ CI integration template:
334
+
335
+ - [GitHub Actions example](examples/github-actions/pr-impact.yml)
336
+ - The example uses Node 22 (`actions/setup-node@v4` with `node-version: 22`).
337
+ - The example captures Playwright JSON output via `traceability-capture` and ingests it with `traceability-ingest`.
338
+ - Feedback payload example: [examples/feedback.sample.json](examples/feedback.sample.json)
339
+ - Subsystem owners example: [examples/subsystem-owners.sample.json](examples/subsystem-owners.sample.json)
340
+ - Traceability ingest payload schema: [schemas/traceability-input.schema.json](schemas/traceability-input.schema.json)
341
+ - Traceability ingest payload example: [examples/traceability-input.sample.json](examples/traceability-input.sample.json)
342
+ - Traceability manifest example: [examples/traceability.sample.json](examples/traceability.sample.json)
343
+ - Subsystem risk map schema: [schemas/subsystem-risk-map.schema.json](schemas/subsystem-risk-map.schema.json)
344
+ - Subsystem risk map example: [examples/subsystem-risk-map.sample.json](examples/subsystem-risk-map.sample.json)
345
+ - End-to-end verification steps: [examples/verification/README.md](examples/verification/README.md)
346
+
347
+ Traceability manifest example (`.e2e-ai-agents/traceability.json`):
348
+
349
+ ```json
350
+ {
351
+ "schemaVersion": "1.0.0",
352
+ "tests": [
353
+ {
354
+ "test": "specs/channels/channels.switch.spec.ts",
355
+ "touchedFiles": ["channels/src/components/channel_switcher/channel_switcher.tsx"]
356
+ }
357
+ ]
358
+ }
359
+ ```
360
+
361
+ Traceability ingest input example (`traceability-input.json`):
362
+
363
+ ```json
364
+ {
365
+ "runs": [
366
+ {
367
+ "test": "specs/channels/channels.switch.spec.ts",
368
+ "touchedFiles": ["channels/src/components/channel_switcher/channel_switcher.tsx"]
369
+ }
370
+ ]
371
+ }
372
+ ```
373
+
374
+ Flow catalog entries can also include optional audience and flag metadata:
375
+
376
+ ```json
377
+ {
378
+ "id": "messaging.realtime",
379
+ "priority": "P0",
380
+ "audience": ["member", "guest"],
381
+ "flags": [
382
+ "EnableSomething",
383
+ { "name": "EnableEnterpriseOnly", "source": "config", "defaultState": "off" }
384
+ ],
385
+ "tests": ["specs/functional/channels/realtime.spec.ts"]
386
+ }
387
+ ```
388
+
389
+ ## Extending with Custom Frameworks
390
+
391
+ ### 1. Create Custom Provider
392
+
393
+ ```typescript
394
+ import { LLMProvider } from '@yasserkhanorg/e2e-agents';
395
+
396
+ export class MyCustomProvider implements LLMProvider {
397
+ async generateText(prompt: string) {
398
+ // Your API call here
399
+ return {
400
+ text: '...',
401
+ cost: 0.001,
402
+ tokens: { input: 100, output: 50 }
403
+ };
404
+ }
405
+
406
+ async analyzeImage(images, prompt) {
407
+ throw new Error('Vision not supported');
408
+ }
409
+
410
+ async streamText(prompt) {
411
+ // Generator implementation
412
+ yield 'chunk1';
413
+ yield 'chunk2';
414
+ }
415
+
416
+ getUsageStats() {
417
+ return { /* ... */ };
418
+ }
419
+ }
420
+ ```
421
+
422
+ ### 2. Register with Factory
423
+
424
+ ```typescript
425
+ import { LLMProviderFactory } from '@yasserkhanorg/e2e-agents';
426
+
427
+ LLMProviderFactory.register('my-provider', (config) => {
428
+ return new MyCustomProvider(config);
429
+ });
430
+
431
+ // Use it
432
+ const provider = LLMProviderFactory.create({
433
+ type: 'my-provider',
434
+ config: { apiKey: '...' }
435
+ });
436
+ ```
437
+
438
+ ### 3. Integrate with Test Framework
439
+
440
+ ```typescript
441
+ // Playwright example
442
+ import { test } from '@playwright/test';
443
+ import { LLMProviderFactory } from '@yasserkhanorg/e2e-agents';
444
+
445
+ const llm = LLMProviderFactory.create({
446
+ type: 'anthropic',
447
+ config: { apiKey: process.env.ANTHROPIC_API_KEY }
448
+ });
449
+
450
+ test('use LLM to verify UI', async ({ page }) => {
451
+ await page.goto('https://example.com');
452
+ const screenshot = await page.screenshot();
453
+
454
+ const analysis = await llm.analyzeImage(
455
+ [{ data: screenshot.toString('base64'), mimeType: 'image/png' }],
456
+ 'Is the login button visible and correctly styled?'
457
+ );
458
+
459
+ console.log(analysis.text);
460
+ });
461
+ ```
462
+
463
+ ## MCP Server Integration
464
+
465
+ For Playwright test agents (v1.56+):
466
+
467
+ ```typescript
468
+ import { E2EAgentsMCPServer } from '@yasserkhanorg/e2e-agents/mcp';
469
+
470
+ const server = new E2EAgentsMCPServer();
471
+ const tools = server.getTools();
472
+
473
+ // Available tools:
474
+ // - discover_tests: Find tests needed for code changes
475
+ // - read_file: Read repository files
476
+ // - write_file: Create/update test files
477
+ // - run_tests: Execute tests
478
+ // - get_git_changes: Detect changed files
479
+ // - get_repository_context: Gather metadata
480
+ ```
481
+
482
+ ## Configuration
483
+
484
+ ### Environment Variables
485
+
486
+ ```bash
487
+ ANTHROPIC_API_KEY=sk-ant-...
488
+ ANTHROPIC_MODEL=claude-sonnet-4-5-20250929
489
+
490
+ OPENAI_API_KEY=sk-...
491
+ OPENAI_MODEL=gpt-4
492
+ OPENAI_BASE_URL=https://api.openai.com/v1
493
+ OPENAI_ORG_ID=org_...
494
+
495
+ OLLAMA_BASE_URL=http://localhost:11434
496
+ OLLAMA_MODEL=deepseek-r1:7b
497
+ ```
498
+
499
+ Note: If `OLLAMA_BASE_URL` points to the root host (for example, `http://localhost:11434`), it will be normalized to `/v1`.
500
+
501
+ ### Setup
502
+
503
+ **Claude:**
504
+ 1. Get key: https://console.anthropic.com
505
+ 2. Export: `export ANTHROPIC_API_KEY=sk-ant-...`
506
+
507
+ **OpenAI:**
508
+ 1. Get key: https://platform.openai.com
509
+ 2. Export: `export OPENAI_API_KEY=sk-...`
510
+
511
+ **Ollama:**
512
+ 1. Install: `curl -fsSL https://ollama.com/install.sh | sh`
513
+ 2. Pull: `ollama pull deepseek-r1:7b`
514
+ 3. Run: `ollama serve`
515
+
516
+ ## Error Handling
517
+
518
+ ```typescript
519
+ import { LLMProviderError, UnsupportedCapabilityError } from '@yasserkhanorg/e2e-agents';
520
+
521
+ try {
522
+ await provider.analyzeImage([...], 'Analyze');
523
+ } catch (error) {
524
+ if (error instanceof UnsupportedCapabilityError) {
525
+ console.log(`Not supported by: ${error.provider}`);
526
+ } else if (error instanceof LLMProviderError) {
527
+ console.log(`API error: ${error.message}`);
528
+ }
529
+ }
530
+ ```
531
+
532
+ ## Performance Comparison
533
+
534
+ | Feature | Claude | OpenAI | Ollama |
535
+ |---------|--------|--------|--------|
536
+ | Vision | ✅ | ✅ (model dependent) | ❌ |
537
+ | Cost | $3-15/1M tokens | Model dependent | Free |
538
+ | Speed | ~800ms | ~1000ms | ~3000ms |
539
+ | Streaming | ✅ | ✅ | ✅ |
540
+ | Local | ❌ | ❌ | ✅ |
541
+
542
+ ## Cost Optimization
543
+
544
+ ```typescript
545
+ const stats = provider.getUsageStats();
546
+ console.log(`Tokens: ${stats.totalTokens.toLocaleString()}`);
547
+ console.log(`Cost: $${stats.totalCost.toFixed(2)}`);
548
+ console.log(`Avg speed: ${stats.averageResponseTimeMs.toFixed(0)}ms`);
549
+ ```
550
+
551
+ ## Performance & Optimization (v0.3.0+)
552
+
553
+ ### Logging Configuration
554
+
555
+ Control logging verbosity with the `LOG_LEVEL` environment variable:
556
+
557
+ ```bash
558
+ # Production: errors only
559
+ LOG_LEVEL=ERROR npm start
560
+
561
+ # Development: all messages
562
+ LOG_LEVEL=DEBUG npm start
563
+ ```
564
+
565
+ Supported levels: `ERROR`, `WARN`, `INFO`, `DEBUG` (default: `INFO`)
566
+
567
+ ### Caching
568
+
569
+ The library includes a simple TTL cache for repository context:
570
+
571
+ ```typescript
572
+ import { SimpleCache } from '@yasserkhanorg/e2e-agents/agent/cache_utils';
573
+
574
+ // Create a 10-minute cache
575
+ const cache = new SimpleCache(10 * 60 * 1000);
576
+
577
+ // Store and retrieve
578
+ cache.set('key', {data: 'value'});
579
+ const value = cache.get('key');
580
+
581
+ // Check stats
582
+ const {size, entries} = cache.stats();
583
+ ```
584
+
585
+ ### Performance Metrics (v0.3.0)
586
+
587
+ Improvements from code quality refactoring:
588
+
589
+ - **40% faster** stats calculation (incremental updates)
590
+ - **30% faster** API key validation (pre-compiled patterns)
591
+ - **90% faster** repository context (cache hits)
592
+ - **15% smaller** bundle size (code deduplication)
593
+ - **44 comprehensive tests** (80%+ coverage)
594
+
595
+ See [CHANGELOG.md](CHANGELOG.md) for detailed improvements.
596
+
597
+ ## Learn More
598
+
599
+ For comprehensive documentation on:
600
+ - Real-world usage examples
601
+ - Integration with different frameworks
602
+ - How Mattermost uses e2e-ai-agents in production
603
+ - Cost optimization strategies
604
+ - Security features and best practices
605
+
606
+ 👉 **See [E2E_AI_TESTING.md](E2E_AI_TESTING.md)**
607
+
608
+ ## Production Usage
609
+
610
+ This package is used in production by Mattermost for:
611
+ - ✅ Automated test generation
612
+ - ✅ Test validation and healing
613
+ - ✅ UI screenshot analysis
614
+ - ✅ Test data generation
615
+
616
+ See the [Mattermost e2e-test-gen implementation](https://github.com/mattermost/mattermost/tree/master/e2e-tests/playwright) for a complete example.
617
+
618
+ ## License
619
+
620
+ Apache 2.0
@@ -0,0 +1,62 @@
1
+ import type { AgentConfig, AudienceRole } from './config.js';
2
+ import { type BlastRadius, type FlagHit } from './flags.js';
3
+ export type FlowPriority = 'P0' | 'P1' | 'P2';
4
+ export interface FileAnalysis {
5
+ relativePath: string;
6
+ extension: string;
7
+ exists: boolean;
8
+ content: string | null;
9
+ isUI: boolean;
10
+ isScreen: boolean;
11
+ isComponent: boolean;
12
+ isState: boolean;
13
+ isStyle: boolean;
14
+ hasInteractions: boolean;
15
+ keywords: string[];
16
+ flowId: string;
17
+ flowName: string;
18
+ flowKind: 'screen' | 'flow';
19
+ audience: AudienceRole[];
20
+ flags: FlagHit[];
21
+ subsystemRisk?: {
22
+ rules: string[];
23
+ scoreDelta: number;
24
+ priorityFloor?: FlowPriority;
25
+ reasons: string[];
26
+ };
27
+ }
28
+ export interface FlowImpact {
29
+ id: string;
30
+ name: string;
31
+ kind: 'screen' | 'flow';
32
+ score: number;
33
+ priority: FlowPriority;
34
+ reasons: string[];
35
+ keywords: string[];
36
+ files: string[];
37
+ audience?: AudienceRole[];
38
+ flags?: FlagHit[];
39
+ blastRadius?: BlastRadius;
40
+ priorityFloor?: FlowPriority;
41
+ subsystemRiskBoost?: number;
42
+ subsystemRiskRules?: string[];
43
+ }
44
+ export interface ImpactAnalysisResult {
45
+ files: FileAnalysis[];
46
+ flows: FlowImpact[];
47
+ warnings: string[];
48
+ subsystemRisk: {
49
+ source: 'map';
50
+ enabled: boolean;
51
+ mapPath: string;
52
+ mapFound: boolean;
53
+ rulesLoaded: number;
54
+ filesMatched: number;
55
+ ruleMatches: number;
56
+ boostedFlows: number;
57
+ };
58
+ }
59
+ export declare function analyzeFiles(appRoot: string, relativePaths: string[], config: AgentConfig): ImpactAnalysisResult;
60
+ export declare function isTestFilePath(relativePath: string): boolean;
61
+ export declare function scanRepositoryFlows(appRoot: string, limit?: number, patterns?: string[], exclude?: string[]): string[];
62
+ //# sourceMappingURL=analysis.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"analysis.d.ts","sourceRoot":"","sources":["../../src/agent/analysis.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAC,WAAW,EAAE,YAAY,EAAa,MAAM,aAAa,CAAC;AACvE,OAAO,EAAqE,KAAK,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,YAAY,CAAC;AAY9H,MAAM,MAAM,YAAY,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;AAE9C,MAAM,WAAW,YAAY;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,IAAI,EAAE,OAAO,CAAC;IACd,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,EAAE,OAAO,CAAC;IACrB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,eAAe,EAAE,OAAO,CAAC;IACzB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,GAAG,MAAM,CAAC;IAC5B,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,KAAK,EAAE,OAAO,EAAE,CAAC;IACjB,aAAa,CAAC,EAAE;QACZ,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC;CACL;AAED,MAAM,WAAW,UAAU;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,GAAG,MAAM,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,YAAY,CAAC;IACvB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,CAAC,EAAE,YAAY,EAAE,CAAC;IAC1B,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC;IAClB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,aAAa,CAAC,EAAE,YAAY,CAAC;IAC7B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACjC,KAAK,EAAE,YAAY,EAAE,CAAC;IACtB,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,aAAa,EAAE;QACX,MAAM,EAAE,KAAK,CAAC;QACd,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,OAAO,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;KACxB,CAAC;CACL;AA6HD,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,WAAW,GAAG,oBAAoB,CA+IhH;AAED,wBAAgB,cAAc,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAE5D;AAED,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,SAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAsCnH"}