@yasserkhanorg/e2e-agents 0.3.2 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/README.md +22 -18
  2. package/dist/agent/config.d.ts +1 -0
  3. package/dist/agent/config.d.ts.map +1 -1
  4. package/dist/agent/config.js +10 -0
  5. package/dist/agent/pipeline.d.ts +6 -1
  6. package/dist/agent/pipeline.d.ts.map +1 -1
  7. package/dist/agent/pipeline.js +627 -27
  8. package/dist/agent/report.d.ts +5 -0
  9. package/dist/agent/report.d.ts.map +1 -1
  10. package/dist/agent/report.js +3 -0
  11. package/dist/agent/runner.d.ts.map +1 -1
  12. package/dist/agent/runner.js +25 -6
  13. package/dist/agent/tests.d.ts.map +1 -1
  14. package/dist/agent/tests.js +12 -2
  15. package/dist/cli.js +73 -5
  16. package/dist/esm/agent/config.js +10 -0
  17. package/dist/esm/agent/pipeline.js +627 -27
  18. package/dist/esm/agent/report.js +3 -0
  19. package/dist/esm/agent/runner.js +25 -6
  20. package/dist/esm/agent/tests.js +12 -2
  21. package/dist/esm/cli.js +73 -5
  22. package/package.json +1 -1
  23. package/dist/agent/cache_utils.d.ts +0 -38
  24. package/dist/agent/cache_utils.d.ts.map +0 -1
  25. package/dist/agent/cache_utils.js +0 -67
  26. package/dist/agent/impact-analyzer.d.ts +0 -114
  27. package/dist/agent/impact-analyzer.d.ts.map +0 -1
  28. package/dist/agent/impact-analyzer.js +0 -557
  29. package/dist/agent/index.d.ts +0 -21
  30. package/dist/agent/index.d.ts.map +0 -1
  31. package/dist/agent/index.js +0 -38
  32. package/dist/agent/model-router.d.ts +0 -57
  33. package/dist/agent/model-router.d.ts.map +0 -1
  34. package/dist/agent/model-router.js +0 -154
  35. package/dist/agent/report-generator.d.ts +0 -24
  36. package/dist/agent/report-generator.d.ts.map +0 -1
  37. package/dist/agent/report-generator.js +0 -250
  38. package/dist/agent/spec-bridge.d.ts +0 -101
  39. package/dist/agent/spec-bridge.d.ts.map +0 -1
  40. package/dist/agent/spec-bridge.js +0 -273
  41. package/dist/agent/spec-builder.d.ts +0 -102
  42. package/dist/agent/spec-builder.d.ts.map +0 -1
  43. package/dist/agent/spec-builder.js +0 -273
  44. package/dist/agent/telemetry.d.ts +0 -84
  45. package/dist/agent/telemetry.d.ts.map +0 -1
  46. package/dist/agent/telemetry.js +0 -220
  47. package/dist/agent/validators/selector-validator.d.ts +0 -74
  48. package/dist/agent/validators/selector-validator.d.ts.map +0 -1
  49. package/dist/agent/validators/selector-validator.js +0 -165
  50. package/dist/e2e-test-gen/index.d.ts +0 -51
  51. package/dist/e2e-test-gen/index.d.ts.map +0 -1
  52. package/dist/e2e-test-gen/index.js +0 -57
  53. package/dist/e2e-test-gen/spec_parser.d.ts +0 -142
  54. package/dist/e2e-test-gen/spec_parser.d.ts.map +0 -1
  55. package/dist/e2e-test-gen/spec_parser.js +0 -786
  56. package/dist/e2e-test-gen/types.d.ts +0 -185
  57. package/dist/e2e-test-gen/types.d.ts.map +0 -1
  58. package/dist/e2e-test-gen/types.js +0 -4
  59. package/dist/esm/agent/cache_utils.js +0 -63
  60. package/dist/esm/agent/impact-analyzer.js +0 -548
  61. package/dist/esm/agent/index.js +0 -22
  62. package/dist/esm/agent/model-router.js +0 -150
  63. package/dist/esm/agent/report-generator.js +0 -247
  64. package/dist/esm/agent/spec-bridge.js +0 -267
  65. package/dist/esm/agent/spec-builder.js +0 -267
  66. package/dist/esm/agent/telemetry.js +0 -216
  67. package/dist/esm/agent/validators/selector-validator.js +0 -160
  68. package/dist/esm/e2e-test-gen/index.js +0 -50
  69. package/dist/esm/e2e-test-gen/spec_parser.js +0 -782
  70. package/dist/esm/e2e-test-gen/types.js +0 -3
  71. package/dist/esm/plan-and-test-constants.js +0 -126
  72. package/dist/plan-and-test-constants.d.ts +0 -110
  73. package/dist/plan-and-test-constants.d.ts.map +0 -1
  74. package/dist/plan-and-test-constants.js +0 -132
@@ -1,267 +0,0 @@
1
- // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
2
- // See LICENSE.txt for license information.
3
- /**
4
- * Spec Bridge - Bridge between PDF specs and Playwright Agent workflow
5
- *
6
- * This module converts specification documents (PDF, Markdown, JSON) into
7
- * Playwright-compatible markdown files that can be consumed by Playwright's
8
- * native agents (Planner, Generator, Healer).
9
- *
10
- * Flow: PDF → SpecParser → Markdown specs → Playwright Planner
11
- *
12
- * The Playwright agents (available in Playwright 1.56+) provide production-ready:
13
- * - Test planning and exploration
14
- * - Test code generation
15
- * - Automatic test healing
16
- *
17
- * Usage:
18
- * ```typescript
19
- * const bridge = new SpecBridge({type: 'anthropic', apiKey: process.env.ANTHROPIC_API_KEY});
20
- * const specPaths = await bridge.convertToPlaywrightSpecs('spec.pdf', 'specs/');
21
- * // Now use Playwright agents: @planner, @generator, @healer
22
- * ```
23
- */
24
- import { writeFileSync, mkdirSync, existsSync } from 'fs';
25
- import { join, basename } from 'path';
26
- import { LLMProviderFactory } from '../provider_factory.js';
27
- import { SpecificationParser } from '../e2e-test-gen/spec_parser.js';
28
- /**
29
- * Bridge between PDF/JSON specs and Playwright Agent workflow
30
- *
31
- * Converts specification documents into Playwright-compatible markdown
32
- * that can be consumed by Playwright's native agents.
33
- */
34
- export class SpecBridge {
35
- constructor(config) {
36
- // Handle both single provider and hybrid config
37
- let llmProvider;
38
- if ('primary' in config.llmConfig) {
39
- // It's a HybridConfig
40
- llmProvider = LLMProviderFactory.createHybrid(config.llmConfig);
41
- }
42
- else {
43
- // It's a ProviderConfig
44
- llmProvider = LLMProviderFactory.create(config.llmConfig);
45
- }
46
- this.parser = new SpecificationParser(llmProvider);
47
- this.outputDir = config.outputDir || 'specs';
48
- this.overwrite = config.overwrite !== false;
49
- }
50
- /**
51
- * Convert a specification file to Playwright-compatible markdown specs
52
- *
53
- * @param inputPath - Path to the input specification (PDF, MD, or JSON)
54
- * @param outputDir - Optional override for output directory
55
- * @returns Conversion result with paths and metadata
56
- */
57
- async convertToPlaywrightSpecs(inputPath, outputDir) {
58
- /* eslint-disable no-console */
59
- const targetDir = outputDir || this.outputDir;
60
- const warnings = [];
61
- console.log(`Converting specification: ${inputPath}`);
62
- // Parse the specification
63
- const specs = await this.parser.parse(inputPath, 'file');
64
- if (specs.length === 0) {
65
- warnings.push('No features extracted from specification');
66
- return {
67
- specPaths: [],
68
- features: [],
69
- totalScenarios: 0,
70
- warnings,
71
- };
72
- }
73
- // Create output directory
74
- mkdirSync(targetDir, { recursive: true });
75
- const specPaths = [];
76
- let totalScenarios = 0;
77
- for (const spec of specs) {
78
- // Validate the spec
79
- const validation = this.parser.validateSpec(spec);
80
- if (!validation.valid) {
81
- warnings.push(`Feature "${spec.name}": ${validation.errors.join(', ')}`);
82
- }
83
- // Convert to Playwright markdown format
84
- const markdown = this.toPlaywrightMarkdown(spec);
85
- // Generate output filename
86
- const filename = `${spec.id}.md`;
87
- const outputPath = join(targetDir, filename);
88
- // Check for existing file
89
- if (existsSync(outputPath) && !this.overwrite) {
90
- warnings.push(`Skipping existing file: ${outputPath}`);
91
- continue;
92
- }
93
- // Write the spec file
94
- writeFileSync(outputPath, markdown, 'utf-8');
95
- specPaths.push(outputPath);
96
- totalScenarios += spec.scenarios.length;
97
- console.log(` Created: ${outputPath} (${spec.scenarios.length} scenarios)`);
98
- }
99
- console.log(`Converted ${specs.length} feature(s) with ${totalScenarios} scenario(s)`);
100
- /* eslint-enable no-console */
101
- return {
102
- specPaths,
103
- features: specs,
104
- totalScenarios,
105
- warnings,
106
- };
107
- }
108
- /**
109
- * Convert a FeatureSpecification to Playwright Agent markdown format
110
- *
111
- * The format is designed to be consumed by Playwright's Planner and Generator agents.
112
- * It includes structured test scenarios in Given-When-Then format.
113
- */
114
- toPlaywrightMarkdown(spec) {
115
- const lines = [];
116
- // Header
117
- lines.push(`# ${spec.name}`);
118
- lines.push('');
119
- // Metadata
120
- lines.push(`**Priority**: ${spec.priority}`);
121
- if (spec.targetUrls.length > 0) {
122
- lines.push(`**Target URLs**: ${spec.targetUrls.join(', ')}`);
123
- }
124
- lines.push('');
125
- // Description
126
- if (spec.description) {
127
- lines.push('## Description');
128
- lines.push('');
129
- lines.push(spec.description);
130
- lines.push('');
131
- }
132
- // Test Scenarios
133
- if (spec.scenarios.length > 0) {
134
- lines.push('## Test Scenarios');
135
- lines.push('');
136
- for (const scenario of spec.scenarios) {
137
- lines.push(this.formatScenario(scenario));
138
- lines.push('');
139
- }
140
- }
141
- // Acceptance Criteria
142
- if (spec.acceptanceCriteria.length > 0) {
143
- lines.push('## Acceptance Criteria');
144
- lines.push('');
145
- for (const criterion of spec.acceptanceCriteria) {
146
- lines.push(`- ${criterion}`);
147
- }
148
- lines.push('');
149
- }
150
- // Screenshots reference (if any)
151
- if (spec.screenshots.length > 0) {
152
- lines.push('## Reference Screenshots');
153
- lines.push('');
154
- for (const screenshot of spec.screenshots) {
155
- lines.push(`- ${screenshot.description}: \`${screenshot.path}\``);
156
- }
157
- lines.push('');
158
- }
159
- // Metadata footer
160
- lines.push('---');
161
- lines.push(`*Generated from: ${basename(spec.sourcePath)}*`);
162
- lines.push(`*Feature ID: ${spec.id}*`);
163
- return lines.join('\n');
164
- }
165
- /**
166
- * Format a business scenario in Playwright-friendly markdown
167
- */
168
- formatScenario(scenario) {
169
- const lines = [];
170
- lines.push(`### ${scenario.name}`);
171
- lines.push('');
172
- lines.push(`**Priority**: ${scenario.priority}`);
173
- lines.push('');
174
- if (scenario.given) {
175
- lines.push(`**Given**: ${scenario.given}`);
176
- }
177
- if (scenario.when) {
178
- lines.push(`**When**: ${scenario.when}`);
179
- }
180
- if (scenario.then) {
181
- lines.push(`**Then**: ${scenario.then}`);
182
- }
183
- return lines.join('\n');
184
- }
185
- /**
186
- * Parse a spec file without writing output
187
- * Useful for validation or inspection
188
- */
189
- async parseSpec(inputPath) {
190
- return this.parser.parse(inputPath, 'file');
191
- }
192
- /**
193
- * Validate a specification file
194
- */
195
- async validateSpec(inputPath) {
196
- const specs = await this.parser.parse(inputPath, 'file');
197
- const allErrors = [];
198
- let totalScenarios = 0;
199
- for (const spec of specs) {
200
- const validation = this.parser.validateSpec(spec);
201
- if (!validation.valid) {
202
- allErrors.push(...validation.errors.map((e) => `${spec.name}: ${e}`));
203
- }
204
- totalScenarios += spec.scenarios.length;
205
- }
206
- return {
207
- valid: allErrors.length === 0,
208
- errors: allErrors,
209
- features: specs.length,
210
- scenarios: totalScenarios,
211
- };
212
- }
213
- /**
214
- * Get a summary of the specification
215
- */
216
- async getSpecSummary(inputPath) {
217
- const specs = await this.parser.parse(inputPath, 'file');
218
- const lines = [];
219
- lines.push(`Specification Summary: ${basename(inputPath)}`);
220
- lines.push('='.repeat(50));
221
- lines.push('');
222
- for (const spec of specs) {
223
- const summary = this.parser.getSpecSummary(spec);
224
- lines.push(`Feature: ${summary.name}`);
225
- lines.push(` Priority: ${summary.priority}`);
226
- lines.push(` Scenarios: ${summary.scenarioCount}`);
227
- lines.push(` - Must-have: ${summary.mustHaveScenarios}`);
228
- lines.push(` - Should-have: ${summary.shouldHaveScenarios}`);
229
- lines.push(` - Nice-to-have: ${summary.niceToHaveScenarios}`);
230
- lines.push(` Acceptance Criteria: ${summary.acceptanceCriteriaCount}`);
231
- lines.push(` Has Screenshots: ${summary.hasScreenshots ? 'Yes' : 'No'}`);
232
- lines.push('');
233
- }
234
- return lines.join('\n');
235
- }
236
- }
237
- /**
238
- * Create a SpecBridge with Anthropic provider
239
- * Convenience function for common use case
240
- */
241
- export function createAnthropicBridge(apiKey, outputDir) {
242
- return new SpecBridge({
243
- llmConfig: {
244
- type: 'anthropic',
245
- config: {
246
- apiKey: apiKey || process.env.ANTHROPIC_API_KEY || '',
247
- },
248
- },
249
- outputDir,
250
- });
251
- }
252
- /**
253
- * Create a SpecBridge with Ollama provider (free, local)
254
- * Convenience function for local development
255
- */
256
- export function createOllamaBridge(model, outputDir) {
257
- return new SpecBridge({
258
- llmConfig: {
259
- type: 'ollama',
260
- config: {
261
- model: model || 'deepseek-r1:7b',
262
- baseUrl: process.env.OLLAMA_BASE_URL || 'http://localhost:11434',
263
- },
264
- },
265
- outputDir,
266
- });
267
- }
@@ -1,216 +0,0 @@
1
- // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
2
- // See LICENSE.txt for license information.
3
- /**
4
- * Telemetry Collection System (Phase A2)
5
- *
6
- * Tracks costs, performance, and success metrics for test generation operations.
7
- * Provides visibility into:
8
- * - Cost per operation (input/output tokens * model rate)
9
- * - Model usage breakdown (Haiku, Sonnet, Opus)
10
- * - Success rate by operation
11
- * - Performance metrics (duration, tokens used)
12
- *
13
- * Data stored in: `.e2e-ai-agents/metrics/YYYY-MM-DD.json`
14
- */
15
- import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
16
- import { join } from 'path';
17
- import { randomUUID } from 'crypto';
18
- const MODEL_RATES = {
19
- 'claude-haiku-4-0-20250430': 0.25 / 1000000, // per input token
20
- 'claude-sonnet-4-5-20250929': 3 / 1000000,
21
- 'claude-opus-4-6-20250820': 15 / 1000000,
22
- };
23
- export class TelemetryCollector {
24
- constructor(metricsDir = '.e2e-ai-agents/metrics') {
25
- this.metrics = new Map();
26
- this.metricsDir = metricsDir;
27
- this.ensureMetricsDir();
28
- this.loadTodayMetrics();
29
- }
30
- /**
31
- * Ensure metrics directory exists
32
- */
33
- ensureMetricsDir() {
34
- if (!existsSync(this.metricsDir)) {
35
- mkdirSync(this.metricsDir, { recursive: true });
36
- }
37
- }
38
- /**
39
- * Get today's metrics file path
40
- */
41
- getTodayPath() {
42
- const now = new Date();
43
- const date = now.toISOString().split('T')[0]; // YYYY-MM-DD
44
- return join(this.metricsDir, `${date}.json`);
45
- }
46
- /**
47
- * Load metrics from disk
48
- */
49
- loadTodayMetrics() {
50
- const path = this.getTodayPath();
51
- if (existsSync(path)) {
52
- try {
53
- const data = JSON.parse(readFileSync(path, 'utf-8'));
54
- this.metrics.set(path, data);
55
- }
56
- catch (error) {
57
- console.error(`Failed to load metrics from ${path}:`, error);
58
- }
59
- }
60
- }
61
- /**
62
- * Track a metric
63
- */
64
- track(metric) {
65
- const fullMetric = {
66
- id: randomUUID().substring(0, 8),
67
- ...metric,
68
- };
69
- const path = this.getTodayPath();
70
- const metrics = this.metrics.get(path) || [];
71
- metrics.push(fullMetric);
72
- this.metrics.set(path, metrics);
73
- // Persist to disk
74
- this.saveMetrics();
75
- }
76
- /**
77
- * Save metrics to disk
78
- */
79
- saveMetrics() {
80
- this.metrics.forEach((metrics, path) => {
81
- writeFileSync(path, JSON.stringify(metrics, null, 2), 'utf-8');
82
- });
83
- }
84
- /**
85
- * Calculate cost for a metric
86
- */
87
- static calculateCost(model, tokensInput, tokensOutput) {
88
- const inputRate = MODEL_RATES[model] || 0.003 / 1000000; // Default estimate
89
- const outputRate = inputRate * 3; // Output usually 3x input cost
90
- return tokensInput * inputRate + tokensOutput * outputRate;
91
- }
92
- /**
93
- * Generate report for a date range
94
- */
95
- generateReport(since, until) {
96
- const start = since || new Date(new Date().setDate(new Date().getDate() - 7)); // Default: last 7 days
97
- const end = until || new Date();
98
- // Collect all metrics in date range
99
- const allMetrics = [];
100
- this.metrics.forEach((metrics) => {
101
- metrics.forEach((m) => {
102
- const metricDate = new Date(m.timestamp);
103
- if (metricDate >= start && metricDate <= end) {
104
- allMetrics.push(m);
105
- }
106
- });
107
- });
108
- // Calculate summary
109
- const successCount = allMetrics.filter((m) => m.success).length;
110
- const failureCount = allMetrics.length - successCount;
111
- const totalCost = allMetrics.reduce((sum, m) => sum + m.costUsd, 0);
112
- const avgCost = allMetrics.length > 0 ? totalCost / allMetrics.length : 0;
113
- const totalTokens = allMetrics.reduce((sum, m) => sum + (m.tokensInput + m.tokensOutput), 0);
114
- const avgDuration = allMetrics.length > 0 ? allMetrics.reduce((sum, m) => sum + m.durationMs, 0) / allMetrics.length / 1000 : 0;
115
- // By model
116
- const byModel = {};
117
- allMetrics.forEach((m) => {
118
- if (!byModel[m.model]) {
119
- byModel[m.model] = { count: 0, totalCost: 0, successCount: 0 };
120
- }
121
- byModel[m.model].count += 1;
122
- byModel[m.model].totalCost += m.costUsd;
123
- if (m.success)
124
- byModel[m.model].successCount += 1;
125
- });
126
- Object.keys(byModel).forEach((model) => {
127
- const data = byModel[model];
128
- byModel[model] = {
129
- count: data.count,
130
- totalCost: data.totalCost,
131
- avgCost: data.totalCost / data.count,
132
- successRate: (data.successCount / data.count) * 100,
133
- };
134
- });
135
- // By operation
136
- const byOperation = {};
137
- allMetrics.forEach((m) => {
138
- if (!byOperation[m.operation]) {
139
- byOperation[m.operation] = { count: 0, totalCost: 0, totalDuration: 0, successCount: 0 };
140
- }
141
- byOperation[m.operation].count += 1;
142
- byOperation[m.operation].totalCost += m.costUsd;
143
- byOperation[m.operation].totalDuration += m.durationMs;
144
- if (m.success)
145
- byOperation[m.operation].successCount += 1;
146
- });
147
- Object.keys(byOperation).forEach((op) => {
148
- const data = byOperation[op];
149
- byOperation[op] = {
150
- count: data.count,
151
- totalCost: data.totalCost,
152
- avgDuration: data.totalDuration / data.count / 1000,
153
- successRate: (data.successCount / data.count) * 100,
154
- };
155
- });
156
- return {
157
- period: {
158
- start: start.toISOString().split('T')[0],
159
- end: end.toISOString().split('T')[0],
160
- },
161
- summary: {
162
- totalOperations: allMetrics.length,
163
- successCount,
164
- failureCount,
165
- successRate: allMetrics.length > 0 ? (successCount / allMetrics.length) * 100 : 0,
166
- totalCost,
167
- avgCost,
168
- totalTokens,
169
- avgDuration,
170
- },
171
- byModel,
172
- byOperation,
173
- };
174
- }
175
- /**
176
- * Format report for console output
177
- */
178
- static formatReport(report) {
179
- const lines = [
180
- '',
181
- '📊 Test Generation Metrics',
182
- `Period: ${report.period.start} to ${report.period.end}`,
183
- '═'.repeat(50),
184
- '',
185
- `Total Operations: ${report.summary.totalOperations}`,
186
- `Success Rate: ${report.summary.successRate.toFixed(1)}% (${report.summary.successCount}/${report.summary.totalOperations})`,
187
- `Total Cost: $${report.summary.totalCost.toFixed(2)}`,
188
- `Avg Cost/Op: $${report.summary.avgCost.toFixed(4)}`,
189
- `Avg Duration: ${report.summary.avgDuration.toFixed(1)}s`,
190
- `Total Tokens: ${report.summary.totalTokens.toLocaleString()}`,
191
- '',
192
- 'Model Usage:',
193
- ];
194
- Object.entries(report.byModel).forEach(([model, data]) => {
195
- const shortName = model.includes('haiku') ? 'Haiku' : model.includes('sonnet') ? 'Sonnet' : 'Opus';
196
- lines.push(` ${shortName}: ${data.count} ops - $${data.totalCost.toFixed(2)} (avg $${data.avgCost.toFixed(4)}, ${data.successRate.toFixed(0)}% success)`);
197
- });
198
- lines.push('', 'By Operation:');
199
- Object.entries(report.byOperation).forEach(([op, data]) => {
200
- lines.push(` ${op}: ${data.count} ops - $${data.totalCost.toFixed(2)} (${data.avgDuration.toFixed(1)}s avg, ${data.successRate.toFixed(0)}% success)`);
201
- });
202
- lines.push('');
203
- return lines.join('\n');
204
- }
205
- /**
206
- * Export metrics as JSON
207
- */
208
- exportJson(filepath) {
209
- const allMetrics = [];
210
- this.metrics.forEach((metrics) => {
211
- allMetrics.push(...metrics);
212
- });
213
- writeFileSync(filepath, JSON.stringify(allMetrics, null, 2), 'utf-8');
214
- console.log(` ✓ Exported ${allMetrics.length} metrics to ${filepath}`);
215
- }
216
- }
@@ -1,160 +0,0 @@
1
- // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
2
- // See LICENSE.txt for license information.
3
- /**
4
- * SelectorValidator: Enforces whitelist matching on generated selectors
5
- * Comments out unobserved selectors instead of letting tests fail randomly
6
- */
7
- export class SelectorValidator {
8
- constructor(globalSelectors, minConfidence = 50) {
9
- this.whitelist = new Set();
10
- this.semanticWhitelist = new Map();
11
- this.minConfidence = minConfidence;
12
- // Build flat whitelist from semantic map
13
- for (const [semantic, elements] of Object.entries(globalSelectors)) {
14
- for (const elem of elements) {
15
- if (elem.confidence >= minConfidence) {
16
- this.whitelist.add(elem.selector);
17
- }
18
- }
19
- this.semanticWhitelist.set(semantic, elements.map((e) => e.selector));
20
- }
21
- }
22
- /**
23
- * Validate a selector against the whitelist
24
- */
25
- validateSelector(selector) {
26
- if (this.whitelist.has(selector)) {
27
- return {
28
- isValid: true,
29
- confidence: 100,
30
- reason: 'Found in whitelist',
31
- };
32
- }
33
- // Check for similar selectors (lenient matching)
34
- const normalized = this.normalizeSelector(selector);
35
- for (const whitelisted of this.whitelist) {
36
- if (this.normalizeSelector(whitelisted).includes(normalized)) {
37
- return {
38
- isValid: true,
39
- confidence: 75,
40
- reason: 'Found similar whitelisted selector',
41
- };
42
- }
43
- }
44
- return {
45
- isValid: false,
46
- confidence: 0,
47
- reason: 'Not found in whitelist',
48
- suggestedComment: `// UNOBSERVED SELECTOR - Not found in UI map. Use test.fixme() if needed.`,
49
- };
50
- }
51
- /**
52
- * Validate and comment out invalid selectors in generated test code
53
- */
54
- validateTestCode(code) {
55
- const results = [];
56
- const selectorRegex = /page\.(getByTestId|getByLabel|getByRole|locator)\(['"`]([^'"`]+)['"`]\)/g;
57
- let match;
58
- while ((match = selectorRegex.exec(code)) !== null) {
59
- const [fullMatch, , selector] = match;
60
- const validation = this.validateSelector(selector);
61
- results.push({
62
- selector,
63
- isWhitelisted: validation.isValid,
64
- confidence: validation.confidence,
65
- originalCode: fullMatch,
66
- validatedCode: validation.isValid
67
- ? fullMatch
68
- : `// ${validation.suggestedComment}\n // ${fullMatch}`,
69
- });
70
- }
71
- return results;
72
- }
73
- /**
74
- * Apply validation to test code, commenting out unwhitelisted selectors
75
- */
76
- applyValidation(code) {
77
- let validated = code;
78
- const results = this.validateTestCode(code);
79
- // Apply in reverse order to preserve indices
80
- for (const result of results.reverse()) {
81
- if (!result.isWhitelisted) {
82
- validated = validated.replace(result.originalCode, result.validatedCode);
83
- }
84
- }
85
- return validated;
86
- }
87
- /**
88
- * Get validation summary
89
- */
90
- getSummary(code) {
91
- const results = this.validateTestCode(code);
92
- const unobserved = results.filter((r) => !r.isWhitelisted).map((r) => r.selector);
93
- return {
94
- total: results.length,
95
- whitelisted: results.filter((r) => r.isWhitelisted).length,
96
- coverage: results.length > 0
97
- ? Math.round((results.filter((r) => r.isWhitelisted).length / results.length) * 100)
98
- : 100,
99
- unobserved,
100
- };
101
- }
102
- normalizeSelector(selector) {
103
- // Normalize selector for lenient matching
104
- return selector.toLowerCase().replace(/[^a-z0-9-_]/g, '');
105
- }
106
- }
107
- /**
108
- * APIFallbackResolver: Provides fallback strategies when UI selectors fail
109
- * Converts test methods to API calls when UI elements aren't available
110
- */
111
- export class APIFallbackResolver {
112
- constructor() {
113
- this.apiMapping = new Map([
114
- // UI action -> API endpoint mapping
115
- ['click.*button.*submit', 'POST /api/v4/posts'],
116
- ['fill.*search', 'GET /api/v4/users'],
117
- ['click.*profile', 'GET /api/v4/users/me'],
118
- ['navigate.*channel', 'GET /api/v4/channels'],
119
- ['click.*settings', 'PATCH /api/v4/users/me'],
120
- ]);
121
- }
122
- /**
123
- * Check if a test should fall back to API testing
124
- */
125
- shouldFallback(selector, confidence) {
126
- return confidence < 50;
127
- }
128
- /**
129
- * Generate API-based fallback for unobserved selector
130
- */
131
- generateAPIFallback(selector, action) {
132
- // Find matching API endpoint
133
- let endpoint = 'GET /api/v4/';
134
- for (const [pattern, api] of this.apiMapping) {
135
- if (new RegExp(pattern, 'i').test(action)) {
136
- endpoint = api;
137
- break;
138
- }
139
- }
140
- return `
141
- // UI selector not found - falling back to API
142
- const response = await fetch(\`\${baseUrl}${endpoint.split(' ')[1]}\`, {
143
- method: '${endpoint.split(' ')[0]}',
144
- headers: {'Authorization': \`Bearer \${token}\`},
145
- });
146
- expect(response.ok).toBe(true);`;
147
- }
148
- /**
149
- * Wrap unobserved test in try-catch with API fallback
150
- */
151
- wrapWithFallback(testCode) {
152
- return `
153
- try {
154
- ${testCode}
155
- } catch (error) {
156
- // UI element not found - using API fallback
157
- ${this.generateAPIFallback('unknown', testCode)}
158
- }`;
159
- }
160
- }
@@ -1,50 +0,0 @@
1
- // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
2
- // See LICENSE.txt for license information.
3
- /**
4
- * Autonomous E2E Testing System
5
- *
6
- * A specification-driven testing system that bridges PDF/Markdown specs
7
- * with Playwright's native agents for test planning, generation, and healing.
8
- *
9
- * Quick Start:
10
- * ```typescript
11
- * import {SpecBridge, createAnthropicBridge} from '@mattermost/playwright-lib/autonomous';
12
- *
13
- * // Convert a specification to Playwright-compatible markdown
14
- * const bridge = createAnthropicBridge(process.env.ANTHROPIC_API_KEY);
15
- * const result = await bridge.convertToPlaywrightSpecs('spec.pdf', 'specs/');
16
- *
17
- * // Then use Playwright agents:
18
- * // @planner explore http://localhost:8065
19
- * // @generator create tests from specs/
20
- * // @healer fix failing tests
21
- * ```
22
- *
23
- * Architecture:
24
- * - SpecificationParser: Parses PDF/MD/JSON specs into structured format
25
- * - SpecBridge: Converts specs to Playwright Agent-compatible markdown
26
- * - LLM Providers: Pluggable AI providers (Anthropic, Ollama, OpenAI)
27
- *
28
- * The heavy lifting (test generation, execution, healing) is delegated to
29
- * Playwright's built-in agents which are production-ready and maintained
30
- * by the Playwright team.
31
- */
32
- // Core Components
33
- export { SpecificationParser } from './spec_parser.js';
34
- // LLM Providers (re-exported from parent package)
35
- export { LLMProviderFactory } from '../provider_factory.js';
36
- export { OllamaProvider } from '../ollama_provider.js';
37
- export { AnthropicProvider } from '../anthropic_provider.js';
38
- /**
39
- * Version info
40
- */
41
- export const VERSION = '2.0.0';
42
- export const SUPPORTED_PLAYWRIGHT_VERSION = '1.56.0';
43
- /**
44
- * Feature flags
45
- */
46
- export const FEATURES = {
47
- LLM_AGNOSTIC: true,
48
- SPECIFICATION_DRIVEN: true,
49
- PLAYWRIGHT_AGENTS: true,
50
- };