@yasserkhanorg/e2e-agents 0.3.2 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -18
- package/dist/agent/config.d.ts +1 -0
- package/dist/agent/config.d.ts.map +1 -1
- package/dist/agent/config.js +10 -0
- package/dist/agent/pipeline.d.ts +6 -1
- package/dist/agent/pipeline.d.ts.map +1 -1
- package/dist/agent/pipeline.js +627 -27
- package/dist/agent/report.d.ts +5 -0
- package/dist/agent/report.d.ts.map +1 -1
- package/dist/agent/report.js +3 -0
- package/dist/agent/runner.d.ts.map +1 -1
- package/dist/agent/runner.js +25 -6
- package/dist/agent/tests.d.ts.map +1 -1
- package/dist/agent/tests.js +12 -2
- package/dist/cli.js +73 -5
- package/dist/esm/agent/config.js +10 -0
- package/dist/esm/agent/pipeline.js +627 -27
- package/dist/esm/agent/report.js +3 -0
- package/dist/esm/agent/runner.js +25 -6
- package/dist/esm/agent/tests.js +12 -2
- package/dist/esm/cli.js +73 -5
- package/package.json +1 -1
- package/dist/agent/cache_utils.d.ts +0 -38
- package/dist/agent/cache_utils.d.ts.map +0 -1
- package/dist/agent/cache_utils.js +0 -67
- package/dist/agent/impact-analyzer.d.ts +0 -114
- package/dist/agent/impact-analyzer.d.ts.map +0 -1
- package/dist/agent/impact-analyzer.js +0 -557
- package/dist/agent/index.d.ts +0 -21
- package/dist/agent/index.d.ts.map +0 -1
- package/dist/agent/index.js +0 -38
- package/dist/agent/model-router.d.ts +0 -57
- package/dist/agent/model-router.d.ts.map +0 -1
- package/dist/agent/model-router.js +0 -154
- package/dist/agent/report-generator.d.ts +0 -24
- package/dist/agent/report-generator.d.ts.map +0 -1
- package/dist/agent/report-generator.js +0 -250
- package/dist/agent/spec-bridge.d.ts +0 -101
- package/dist/agent/spec-bridge.d.ts.map +0 -1
- package/dist/agent/spec-bridge.js +0 -273
- package/dist/agent/spec-builder.d.ts +0 -102
- package/dist/agent/spec-builder.d.ts.map +0 -1
- package/dist/agent/spec-builder.js +0 -273
- package/dist/agent/telemetry.d.ts +0 -84
- package/dist/agent/telemetry.d.ts.map +0 -1
- package/dist/agent/telemetry.js +0 -220
- package/dist/agent/validators/selector-validator.d.ts +0 -74
- package/dist/agent/validators/selector-validator.d.ts.map +0 -1
- package/dist/agent/validators/selector-validator.js +0 -165
- package/dist/e2e-test-gen/index.d.ts +0 -51
- package/dist/e2e-test-gen/index.d.ts.map +0 -1
- package/dist/e2e-test-gen/index.js +0 -57
- package/dist/e2e-test-gen/spec_parser.d.ts +0 -142
- package/dist/e2e-test-gen/spec_parser.d.ts.map +0 -1
- package/dist/e2e-test-gen/spec_parser.js +0 -786
- package/dist/e2e-test-gen/types.d.ts +0 -185
- package/dist/e2e-test-gen/types.d.ts.map +0 -1
- package/dist/e2e-test-gen/types.js +0 -4
- package/dist/esm/agent/cache_utils.js +0 -63
- package/dist/esm/agent/impact-analyzer.js +0 -548
- package/dist/esm/agent/index.js +0 -22
- package/dist/esm/agent/model-router.js +0 -150
- package/dist/esm/agent/report-generator.js +0 -247
- package/dist/esm/agent/spec-bridge.js +0 -267
- package/dist/esm/agent/spec-builder.js +0 -267
- package/dist/esm/agent/telemetry.js +0 -216
- package/dist/esm/agent/validators/selector-validator.js +0 -160
- package/dist/esm/e2e-test-gen/index.js +0 -50
- package/dist/esm/e2e-test-gen/spec_parser.js +0 -782
- package/dist/esm/e2e-test-gen/types.js +0 -3
- package/dist/esm/plan-and-test-constants.js +0 -126
- package/dist/plan-and-test-constants.d.ts +0 -110
- package/dist/plan-and-test-constants.d.ts.map +0 -1
- package/dist/plan-and-test-constants.js +0 -132
|
@@ -1,267 +0,0 @@
|
|
|
1
|
-
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
-
// See LICENSE.txt for license information.
|
|
3
|
-
/**
|
|
4
|
-
* Spec Bridge - Bridge between PDF specs and Playwright Agent workflow
|
|
5
|
-
*
|
|
6
|
-
* This module converts specification documents (PDF, Markdown, JSON) into
|
|
7
|
-
* Playwright-compatible markdown files that can be consumed by Playwright's
|
|
8
|
-
* native agents (Planner, Generator, Healer).
|
|
9
|
-
*
|
|
10
|
-
* Flow: PDF → SpecParser → Markdown specs → Playwright Planner
|
|
11
|
-
*
|
|
12
|
-
* The Playwright agents (available in Playwright 1.56+) provide production-ready:
|
|
13
|
-
* - Test planning and exploration
|
|
14
|
-
* - Test code generation
|
|
15
|
-
* - Automatic test healing
|
|
16
|
-
*
|
|
17
|
-
* Usage:
|
|
18
|
-
* ```typescript
|
|
19
|
-
* const bridge = new SpecBridge({type: 'anthropic', apiKey: process.env.ANTHROPIC_API_KEY});
|
|
20
|
-
* const specPaths = await bridge.convertToPlaywrightSpecs('spec.pdf', 'specs/');
|
|
21
|
-
* // Now use Playwright agents: @planner, @generator, @healer
|
|
22
|
-
* ```
|
|
23
|
-
*/
|
|
24
|
-
import { writeFileSync, mkdirSync, existsSync } from 'fs';
|
|
25
|
-
import { join, basename } from 'path';
|
|
26
|
-
import { LLMProviderFactory } from '../provider_factory.js';
|
|
27
|
-
import { SpecificationParser } from '../e2e-test-gen/spec_parser.js';
|
|
28
|
-
/**
|
|
29
|
-
* Bridge between PDF/JSON specs and Playwright Agent workflow
|
|
30
|
-
*
|
|
31
|
-
* Converts specification documents into Playwright-compatible markdown
|
|
32
|
-
* that can be consumed by Playwright's native agents.
|
|
33
|
-
*/
|
|
34
|
-
export class SpecBridge {
|
|
35
|
-
constructor(config) {
|
|
36
|
-
// Handle both single provider and hybrid config
|
|
37
|
-
let llmProvider;
|
|
38
|
-
if ('primary' in config.llmConfig) {
|
|
39
|
-
// It's a HybridConfig
|
|
40
|
-
llmProvider = LLMProviderFactory.createHybrid(config.llmConfig);
|
|
41
|
-
}
|
|
42
|
-
else {
|
|
43
|
-
// It's a ProviderConfig
|
|
44
|
-
llmProvider = LLMProviderFactory.create(config.llmConfig);
|
|
45
|
-
}
|
|
46
|
-
this.parser = new SpecificationParser(llmProvider);
|
|
47
|
-
this.outputDir = config.outputDir || 'specs';
|
|
48
|
-
this.overwrite = config.overwrite !== false;
|
|
49
|
-
}
|
|
50
|
-
/**
|
|
51
|
-
* Convert a specification file to Playwright-compatible markdown specs
|
|
52
|
-
*
|
|
53
|
-
* @param inputPath - Path to the input specification (PDF, MD, or JSON)
|
|
54
|
-
* @param outputDir - Optional override for output directory
|
|
55
|
-
* @returns Conversion result with paths and metadata
|
|
56
|
-
*/
|
|
57
|
-
async convertToPlaywrightSpecs(inputPath, outputDir) {
|
|
58
|
-
/* eslint-disable no-console */
|
|
59
|
-
const targetDir = outputDir || this.outputDir;
|
|
60
|
-
const warnings = [];
|
|
61
|
-
console.log(`Converting specification: ${inputPath}`);
|
|
62
|
-
// Parse the specification
|
|
63
|
-
const specs = await this.parser.parse(inputPath, 'file');
|
|
64
|
-
if (specs.length === 0) {
|
|
65
|
-
warnings.push('No features extracted from specification');
|
|
66
|
-
return {
|
|
67
|
-
specPaths: [],
|
|
68
|
-
features: [],
|
|
69
|
-
totalScenarios: 0,
|
|
70
|
-
warnings,
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
|
-
// Create output directory
|
|
74
|
-
mkdirSync(targetDir, { recursive: true });
|
|
75
|
-
const specPaths = [];
|
|
76
|
-
let totalScenarios = 0;
|
|
77
|
-
for (const spec of specs) {
|
|
78
|
-
// Validate the spec
|
|
79
|
-
const validation = this.parser.validateSpec(spec);
|
|
80
|
-
if (!validation.valid) {
|
|
81
|
-
warnings.push(`Feature "${spec.name}": ${validation.errors.join(', ')}`);
|
|
82
|
-
}
|
|
83
|
-
// Convert to Playwright markdown format
|
|
84
|
-
const markdown = this.toPlaywrightMarkdown(spec);
|
|
85
|
-
// Generate output filename
|
|
86
|
-
const filename = `${spec.id}.md`;
|
|
87
|
-
const outputPath = join(targetDir, filename);
|
|
88
|
-
// Check for existing file
|
|
89
|
-
if (existsSync(outputPath) && !this.overwrite) {
|
|
90
|
-
warnings.push(`Skipping existing file: ${outputPath}`);
|
|
91
|
-
continue;
|
|
92
|
-
}
|
|
93
|
-
// Write the spec file
|
|
94
|
-
writeFileSync(outputPath, markdown, 'utf-8');
|
|
95
|
-
specPaths.push(outputPath);
|
|
96
|
-
totalScenarios += spec.scenarios.length;
|
|
97
|
-
console.log(` Created: ${outputPath} (${spec.scenarios.length} scenarios)`);
|
|
98
|
-
}
|
|
99
|
-
console.log(`Converted ${specs.length} feature(s) with ${totalScenarios} scenario(s)`);
|
|
100
|
-
/* eslint-enable no-console */
|
|
101
|
-
return {
|
|
102
|
-
specPaths,
|
|
103
|
-
features: specs,
|
|
104
|
-
totalScenarios,
|
|
105
|
-
warnings,
|
|
106
|
-
};
|
|
107
|
-
}
|
|
108
|
-
/**
|
|
109
|
-
* Convert a FeatureSpecification to Playwright Agent markdown format
|
|
110
|
-
*
|
|
111
|
-
* The format is designed to be consumed by Playwright's Planner and Generator agents.
|
|
112
|
-
* It includes structured test scenarios in Given-When-Then format.
|
|
113
|
-
*/
|
|
114
|
-
toPlaywrightMarkdown(spec) {
|
|
115
|
-
const lines = [];
|
|
116
|
-
// Header
|
|
117
|
-
lines.push(`# ${spec.name}`);
|
|
118
|
-
lines.push('');
|
|
119
|
-
// Metadata
|
|
120
|
-
lines.push(`**Priority**: ${spec.priority}`);
|
|
121
|
-
if (spec.targetUrls.length > 0) {
|
|
122
|
-
lines.push(`**Target URLs**: ${spec.targetUrls.join(', ')}`);
|
|
123
|
-
}
|
|
124
|
-
lines.push('');
|
|
125
|
-
// Description
|
|
126
|
-
if (spec.description) {
|
|
127
|
-
lines.push('## Description');
|
|
128
|
-
lines.push('');
|
|
129
|
-
lines.push(spec.description);
|
|
130
|
-
lines.push('');
|
|
131
|
-
}
|
|
132
|
-
// Test Scenarios
|
|
133
|
-
if (spec.scenarios.length > 0) {
|
|
134
|
-
lines.push('## Test Scenarios');
|
|
135
|
-
lines.push('');
|
|
136
|
-
for (const scenario of spec.scenarios) {
|
|
137
|
-
lines.push(this.formatScenario(scenario));
|
|
138
|
-
lines.push('');
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
// Acceptance Criteria
|
|
142
|
-
if (spec.acceptanceCriteria.length > 0) {
|
|
143
|
-
lines.push('## Acceptance Criteria');
|
|
144
|
-
lines.push('');
|
|
145
|
-
for (const criterion of spec.acceptanceCriteria) {
|
|
146
|
-
lines.push(`- ${criterion}`);
|
|
147
|
-
}
|
|
148
|
-
lines.push('');
|
|
149
|
-
}
|
|
150
|
-
// Screenshots reference (if any)
|
|
151
|
-
if (spec.screenshots.length > 0) {
|
|
152
|
-
lines.push('## Reference Screenshots');
|
|
153
|
-
lines.push('');
|
|
154
|
-
for (const screenshot of spec.screenshots) {
|
|
155
|
-
lines.push(`- ${screenshot.description}: \`${screenshot.path}\``);
|
|
156
|
-
}
|
|
157
|
-
lines.push('');
|
|
158
|
-
}
|
|
159
|
-
// Metadata footer
|
|
160
|
-
lines.push('---');
|
|
161
|
-
lines.push(`*Generated from: ${basename(spec.sourcePath)}*`);
|
|
162
|
-
lines.push(`*Feature ID: ${spec.id}*`);
|
|
163
|
-
return lines.join('\n');
|
|
164
|
-
}
|
|
165
|
-
/**
|
|
166
|
-
* Format a business scenario in Playwright-friendly markdown
|
|
167
|
-
*/
|
|
168
|
-
formatScenario(scenario) {
|
|
169
|
-
const lines = [];
|
|
170
|
-
lines.push(`### ${scenario.name}`);
|
|
171
|
-
lines.push('');
|
|
172
|
-
lines.push(`**Priority**: ${scenario.priority}`);
|
|
173
|
-
lines.push('');
|
|
174
|
-
if (scenario.given) {
|
|
175
|
-
lines.push(`**Given**: ${scenario.given}`);
|
|
176
|
-
}
|
|
177
|
-
if (scenario.when) {
|
|
178
|
-
lines.push(`**When**: ${scenario.when}`);
|
|
179
|
-
}
|
|
180
|
-
if (scenario.then) {
|
|
181
|
-
lines.push(`**Then**: ${scenario.then}`);
|
|
182
|
-
}
|
|
183
|
-
return lines.join('\n');
|
|
184
|
-
}
|
|
185
|
-
/**
|
|
186
|
-
* Parse a spec file without writing output
|
|
187
|
-
* Useful for validation or inspection
|
|
188
|
-
*/
|
|
189
|
-
async parseSpec(inputPath) {
|
|
190
|
-
return this.parser.parse(inputPath, 'file');
|
|
191
|
-
}
|
|
192
|
-
/**
|
|
193
|
-
* Validate a specification file
|
|
194
|
-
*/
|
|
195
|
-
async validateSpec(inputPath) {
|
|
196
|
-
const specs = await this.parser.parse(inputPath, 'file');
|
|
197
|
-
const allErrors = [];
|
|
198
|
-
let totalScenarios = 0;
|
|
199
|
-
for (const spec of specs) {
|
|
200
|
-
const validation = this.parser.validateSpec(spec);
|
|
201
|
-
if (!validation.valid) {
|
|
202
|
-
allErrors.push(...validation.errors.map((e) => `${spec.name}: ${e}`));
|
|
203
|
-
}
|
|
204
|
-
totalScenarios += spec.scenarios.length;
|
|
205
|
-
}
|
|
206
|
-
return {
|
|
207
|
-
valid: allErrors.length === 0,
|
|
208
|
-
errors: allErrors,
|
|
209
|
-
features: specs.length,
|
|
210
|
-
scenarios: totalScenarios,
|
|
211
|
-
};
|
|
212
|
-
}
|
|
213
|
-
/**
|
|
214
|
-
* Get a summary of the specification
|
|
215
|
-
*/
|
|
216
|
-
async getSpecSummary(inputPath) {
|
|
217
|
-
const specs = await this.parser.parse(inputPath, 'file');
|
|
218
|
-
const lines = [];
|
|
219
|
-
lines.push(`Specification Summary: ${basename(inputPath)}`);
|
|
220
|
-
lines.push('='.repeat(50));
|
|
221
|
-
lines.push('');
|
|
222
|
-
for (const spec of specs) {
|
|
223
|
-
const summary = this.parser.getSpecSummary(spec);
|
|
224
|
-
lines.push(`Feature: ${summary.name}`);
|
|
225
|
-
lines.push(` Priority: ${summary.priority}`);
|
|
226
|
-
lines.push(` Scenarios: ${summary.scenarioCount}`);
|
|
227
|
-
lines.push(` - Must-have: ${summary.mustHaveScenarios}`);
|
|
228
|
-
lines.push(` - Should-have: ${summary.shouldHaveScenarios}`);
|
|
229
|
-
lines.push(` - Nice-to-have: ${summary.niceToHaveScenarios}`);
|
|
230
|
-
lines.push(` Acceptance Criteria: ${summary.acceptanceCriteriaCount}`);
|
|
231
|
-
lines.push(` Has Screenshots: ${summary.hasScreenshots ? 'Yes' : 'No'}`);
|
|
232
|
-
lines.push('');
|
|
233
|
-
}
|
|
234
|
-
return lines.join('\n');
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
|
-
/**
|
|
238
|
-
* Create a SpecBridge with Anthropic provider
|
|
239
|
-
* Convenience function for common use case
|
|
240
|
-
*/
|
|
241
|
-
export function createAnthropicBridge(apiKey, outputDir) {
|
|
242
|
-
return new SpecBridge({
|
|
243
|
-
llmConfig: {
|
|
244
|
-
type: 'anthropic',
|
|
245
|
-
config: {
|
|
246
|
-
apiKey: apiKey || process.env.ANTHROPIC_API_KEY || '',
|
|
247
|
-
},
|
|
248
|
-
},
|
|
249
|
-
outputDir,
|
|
250
|
-
});
|
|
251
|
-
}
|
|
252
|
-
/**
|
|
253
|
-
* Create a SpecBridge with Ollama provider (free, local)
|
|
254
|
-
* Convenience function for local development
|
|
255
|
-
*/
|
|
256
|
-
export function createOllamaBridge(model, outputDir) {
|
|
257
|
-
return new SpecBridge({
|
|
258
|
-
llmConfig: {
|
|
259
|
-
type: 'ollama',
|
|
260
|
-
config: {
|
|
261
|
-
model: model || 'deepseek-r1:7b',
|
|
262
|
-
baseUrl: process.env.OLLAMA_BASE_URL || 'http://localhost:11434',
|
|
263
|
-
},
|
|
264
|
-
},
|
|
265
|
-
outputDir,
|
|
266
|
-
});
|
|
267
|
-
}
|
|
@@ -1,216 +0,0 @@
|
|
|
1
|
-
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
-
// See LICENSE.txt for license information.
|
|
3
|
-
/**
|
|
4
|
-
* Telemetry Collection System (Phase A2)
|
|
5
|
-
*
|
|
6
|
-
* Tracks costs, performance, and success metrics for test generation operations.
|
|
7
|
-
* Provides visibility into:
|
|
8
|
-
* - Cost per operation (input/output tokens * model rate)
|
|
9
|
-
* - Model usage breakdown (Haiku, Sonnet, Opus)
|
|
10
|
-
* - Success rate by operation
|
|
11
|
-
* - Performance metrics (duration, tokens used)
|
|
12
|
-
*
|
|
13
|
-
* Data stored in: `.e2e-ai-agents/metrics/YYYY-MM-DD.json`
|
|
14
|
-
*/
|
|
15
|
-
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
|
|
16
|
-
import { join } from 'path';
|
|
17
|
-
import { randomUUID } from 'crypto';
|
|
18
|
-
const MODEL_RATES = {
|
|
19
|
-
'claude-haiku-4-0-20250430': 0.25 / 1000000, // per input token
|
|
20
|
-
'claude-sonnet-4-5-20250929': 3 / 1000000,
|
|
21
|
-
'claude-opus-4-6-20250820': 15 / 1000000,
|
|
22
|
-
};
|
|
23
|
-
export class TelemetryCollector {
|
|
24
|
-
constructor(metricsDir = '.e2e-ai-agents/metrics') {
|
|
25
|
-
this.metrics = new Map();
|
|
26
|
-
this.metricsDir = metricsDir;
|
|
27
|
-
this.ensureMetricsDir();
|
|
28
|
-
this.loadTodayMetrics();
|
|
29
|
-
}
|
|
30
|
-
/**
|
|
31
|
-
* Ensure metrics directory exists
|
|
32
|
-
*/
|
|
33
|
-
ensureMetricsDir() {
|
|
34
|
-
if (!existsSync(this.metricsDir)) {
|
|
35
|
-
mkdirSync(this.metricsDir, { recursive: true });
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
/**
|
|
39
|
-
* Get today's metrics file path
|
|
40
|
-
*/
|
|
41
|
-
getTodayPath() {
|
|
42
|
-
const now = new Date();
|
|
43
|
-
const date = now.toISOString().split('T')[0]; // YYYY-MM-DD
|
|
44
|
-
return join(this.metricsDir, `${date}.json`);
|
|
45
|
-
}
|
|
46
|
-
/**
|
|
47
|
-
* Load metrics from disk
|
|
48
|
-
*/
|
|
49
|
-
loadTodayMetrics() {
|
|
50
|
-
const path = this.getTodayPath();
|
|
51
|
-
if (existsSync(path)) {
|
|
52
|
-
try {
|
|
53
|
-
const data = JSON.parse(readFileSync(path, 'utf-8'));
|
|
54
|
-
this.metrics.set(path, data);
|
|
55
|
-
}
|
|
56
|
-
catch (error) {
|
|
57
|
-
console.error(`Failed to load metrics from ${path}:`, error);
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* Track a metric
|
|
63
|
-
*/
|
|
64
|
-
track(metric) {
|
|
65
|
-
const fullMetric = {
|
|
66
|
-
id: randomUUID().substring(0, 8),
|
|
67
|
-
...metric,
|
|
68
|
-
};
|
|
69
|
-
const path = this.getTodayPath();
|
|
70
|
-
const metrics = this.metrics.get(path) || [];
|
|
71
|
-
metrics.push(fullMetric);
|
|
72
|
-
this.metrics.set(path, metrics);
|
|
73
|
-
// Persist to disk
|
|
74
|
-
this.saveMetrics();
|
|
75
|
-
}
|
|
76
|
-
/**
|
|
77
|
-
* Save metrics to disk
|
|
78
|
-
*/
|
|
79
|
-
saveMetrics() {
|
|
80
|
-
this.metrics.forEach((metrics, path) => {
|
|
81
|
-
writeFileSync(path, JSON.stringify(metrics, null, 2), 'utf-8');
|
|
82
|
-
});
|
|
83
|
-
}
|
|
84
|
-
/**
|
|
85
|
-
* Calculate cost for a metric
|
|
86
|
-
*/
|
|
87
|
-
static calculateCost(model, tokensInput, tokensOutput) {
|
|
88
|
-
const inputRate = MODEL_RATES[model] || 0.003 / 1000000; // Default estimate
|
|
89
|
-
const outputRate = inputRate * 3; // Output usually 3x input cost
|
|
90
|
-
return tokensInput * inputRate + tokensOutput * outputRate;
|
|
91
|
-
}
|
|
92
|
-
/**
|
|
93
|
-
* Generate report for a date range
|
|
94
|
-
*/
|
|
95
|
-
generateReport(since, until) {
|
|
96
|
-
const start = since || new Date(new Date().setDate(new Date().getDate() - 7)); // Default: last 7 days
|
|
97
|
-
const end = until || new Date();
|
|
98
|
-
// Collect all metrics in date range
|
|
99
|
-
const allMetrics = [];
|
|
100
|
-
this.metrics.forEach((metrics) => {
|
|
101
|
-
metrics.forEach((m) => {
|
|
102
|
-
const metricDate = new Date(m.timestamp);
|
|
103
|
-
if (metricDate >= start && metricDate <= end) {
|
|
104
|
-
allMetrics.push(m);
|
|
105
|
-
}
|
|
106
|
-
});
|
|
107
|
-
});
|
|
108
|
-
// Calculate summary
|
|
109
|
-
const successCount = allMetrics.filter((m) => m.success).length;
|
|
110
|
-
const failureCount = allMetrics.length - successCount;
|
|
111
|
-
const totalCost = allMetrics.reduce((sum, m) => sum + m.costUsd, 0);
|
|
112
|
-
const avgCost = allMetrics.length > 0 ? totalCost / allMetrics.length : 0;
|
|
113
|
-
const totalTokens = allMetrics.reduce((sum, m) => sum + (m.tokensInput + m.tokensOutput), 0);
|
|
114
|
-
const avgDuration = allMetrics.length > 0 ? allMetrics.reduce((sum, m) => sum + m.durationMs, 0) / allMetrics.length / 1000 : 0;
|
|
115
|
-
// By model
|
|
116
|
-
const byModel = {};
|
|
117
|
-
allMetrics.forEach((m) => {
|
|
118
|
-
if (!byModel[m.model]) {
|
|
119
|
-
byModel[m.model] = { count: 0, totalCost: 0, successCount: 0 };
|
|
120
|
-
}
|
|
121
|
-
byModel[m.model].count += 1;
|
|
122
|
-
byModel[m.model].totalCost += m.costUsd;
|
|
123
|
-
if (m.success)
|
|
124
|
-
byModel[m.model].successCount += 1;
|
|
125
|
-
});
|
|
126
|
-
Object.keys(byModel).forEach((model) => {
|
|
127
|
-
const data = byModel[model];
|
|
128
|
-
byModel[model] = {
|
|
129
|
-
count: data.count,
|
|
130
|
-
totalCost: data.totalCost,
|
|
131
|
-
avgCost: data.totalCost / data.count,
|
|
132
|
-
successRate: (data.successCount / data.count) * 100,
|
|
133
|
-
};
|
|
134
|
-
});
|
|
135
|
-
// By operation
|
|
136
|
-
const byOperation = {};
|
|
137
|
-
allMetrics.forEach((m) => {
|
|
138
|
-
if (!byOperation[m.operation]) {
|
|
139
|
-
byOperation[m.operation] = { count: 0, totalCost: 0, totalDuration: 0, successCount: 0 };
|
|
140
|
-
}
|
|
141
|
-
byOperation[m.operation].count += 1;
|
|
142
|
-
byOperation[m.operation].totalCost += m.costUsd;
|
|
143
|
-
byOperation[m.operation].totalDuration += m.durationMs;
|
|
144
|
-
if (m.success)
|
|
145
|
-
byOperation[m.operation].successCount += 1;
|
|
146
|
-
});
|
|
147
|
-
Object.keys(byOperation).forEach((op) => {
|
|
148
|
-
const data = byOperation[op];
|
|
149
|
-
byOperation[op] = {
|
|
150
|
-
count: data.count,
|
|
151
|
-
totalCost: data.totalCost,
|
|
152
|
-
avgDuration: data.totalDuration / data.count / 1000,
|
|
153
|
-
successRate: (data.successCount / data.count) * 100,
|
|
154
|
-
};
|
|
155
|
-
});
|
|
156
|
-
return {
|
|
157
|
-
period: {
|
|
158
|
-
start: start.toISOString().split('T')[0],
|
|
159
|
-
end: end.toISOString().split('T')[0],
|
|
160
|
-
},
|
|
161
|
-
summary: {
|
|
162
|
-
totalOperations: allMetrics.length,
|
|
163
|
-
successCount,
|
|
164
|
-
failureCount,
|
|
165
|
-
successRate: allMetrics.length > 0 ? (successCount / allMetrics.length) * 100 : 0,
|
|
166
|
-
totalCost,
|
|
167
|
-
avgCost,
|
|
168
|
-
totalTokens,
|
|
169
|
-
avgDuration,
|
|
170
|
-
},
|
|
171
|
-
byModel,
|
|
172
|
-
byOperation,
|
|
173
|
-
};
|
|
174
|
-
}
|
|
175
|
-
/**
|
|
176
|
-
* Format report for console output
|
|
177
|
-
*/
|
|
178
|
-
static formatReport(report) {
|
|
179
|
-
const lines = [
|
|
180
|
-
'',
|
|
181
|
-
'📊 Test Generation Metrics',
|
|
182
|
-
`Period: ${report.period.start} to ${report.period.end}`,
|
|
183
|
-
'═'.repeat(50),
|
|
184
|
-
'',
|
|
185
|
-
`Total Operations: ${report.summary.totalOperations}`,
|
|
186
|
-
`Success Rate: ${report.summary.successRate.toFixed(1)}% (${report.summary.successCount}/${report.summary.totalOperations})`,
|
|
187
|
-
`Total Cost: $${report.summary.totalCost.toFixed(2)}`,
|
|
188
|
-
`Avg Cost/Op: $${report.summary.avgCost.toFixed(4)}`,
|
|
189
|
-
`Avg Duration: ${report.summary.avgDuration.toFixed(1)}s`,
|
|
190
|
-
`Total Tokens: ${report.summary.totalTokens.toLocaleString()}`,
|
|
191
|
-
'',
|
|
192
|
-
'Model Usage:',
|
|
193
|
-
];
|
|
194
|
-
Object.entries(report.byModel).forEach(([model, data]) => {
|
|
195
|
-
const shortName = model.includes('haiku') ? 'Haiku' : model.includes('sonnet') ? 'Sonnet' : 'Opus';
|
|
196
|
-
lines.push(` ${shortName}: ${data.count} ops - $${data.totalCost.toFixed(2)} (avg $${data.avgCost.toFixed(4)}, ${data.successRate.toFixed(0)}% success)`);
|
|
197
|
-
});
|
|
198
|
-
lines.push('', 'By Operation:');
|
|
199
|
-
Object.entries(report.byOperation).forEach(([op, data]) => {
|
|
200
|
-
lines.push(` ${op}: ${data.count} ops - $${data.totalCost.toFixed(2)} (${data.avgDuration.toFixed(1)}s avg, ${data.successRate.toFixed(0)}% success)`);
|
|
201
|
-
});
|
|
202
|
-
lines.push('');
|
|
203
|
-
return lines.join('\n');
|
|
204
|
-
}
|
|
205
|
-
/**
|
|
206
|
-
* Export metrics as JSON
|
|
207
|
-
*/
|
|
208
|
-
exportJson(filepath) {
|
|
209
|
-
const allMetrics = [];
|
|
210
|
-
this.metrics.forEach((metrics) => {
|
|
211
|
-
allMetrics.push(...metrics);
|
|
212
|
-
});
|
|
213
|
-
writeFileSync(filepath, JSON.stringify(allMetrics, null, 2), 'utf-8');
|
|
214
|
-
console.log(` ✓ Exported ${allMetrics.length} metrics to ${filepath}`);
|
|
215
|
-
}
|
|
216
|
-
}
|
|
@@ -1,160 +0,0 @@
|
|
|
1
|
-
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
-
// See LICENSE.txt for license information.
|
|
3
|
-
/**
|
|
4
|
-
* SelectorValidator: Enforces whitelist matching on generated selectors
|
|
5
|
-
* Comments out unobserved selectors instead of letting tests fail randomly
|
|
6
|
-
*/
|
|
7
|
-
export class SelectorValidator {
|
|
8
|
-
constructor(globalSelectors, minConfidence = 50) {
|
|
9
|
-
this.whitelist = new Set();
|
|
10
|
-
this.semanticWhitelist = new Map();
|
|
11
|
-
this.minConfidence = minConfidence;
|
|
12
|
-
// Build flat whitelist from semantic map
|
|
13
|
-
for (const [semantic, elements] of Object.entries(globalSelectors)) {
|
|
14
|
-
for (const elem of elements) {
|
|
15
|
-
if (elem.confidence >= minConfidence) {
|
|
16
|
-
this.whitelist.add(elem.selector);
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
this.semanticWhitelist.set(semantic, elements.map((e) => e.selector));
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
/**
|
|
23
|
-
* Validate a selector against the whitelist
|
|
24
|
-
*/
|
|
25
|
-
validateSelector(selector) {
|
|
26
|
-
if (this.whitelist.has(selector)) {
|
|
27
|
-
return {
|
|
28
|
-
isValid: true,
|
|
29
|
-
confidence: 100,
|
|
30
|
-
reason: 'Found in whitelist',
|
|
31
|
-
};
|
|
32
|
-
}
|
|
33
|
-
// Check for similar selectors (lenient matching)
|
|
34
|
-
const normalized = this.normalizeSelector(selector);
|
|
35
|
-
for (const whitelisted of this.whitelist) {
|
|
36
|
-
if (this.normalizeSelector(whitelisted).includes(normalized)) {
|
|
37
|
-
return {
|
|
38
|
-
isValid: true,
|
|
39
|
-
confidence: 75,
|
|
40
|
-
reason: 'Found similar whitelisted selector',
|
|
41
|
-
};
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
return {
|
|
45
|
-
isValid: false,
|
|
46
|
-
confidence: 0,
|
|
47
|
-
reason: 'Not found in whitelist',
|
|
48
|
-
suggestedComment: `// UNOBSERVED SELECTOR - Not found in UI map. Use test.fixme() if needed.`,
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
/**
|
|
52
|
-
* Validate and comment out invalid selectors in generated test code
|
|
53
|
-
*/
|
|
54
|
-
validateTestCode(code) {
|
|
55
|
-
const results = [];
|
|
56
|
-
const selectorRegex = /page\.(getByTestId|getByLabel|getByRole|locator)\(['"`]([^'"`]+)['"`]\)/g;
|
|
57
|
-
let match;
|
|
58
|
-
while ((match = selectorRegex.exec(code)) !== null) {
|
|
59
|
-
const [fullMatch, , selector] = match;
|
|
60
|
-
const validation = this.validateSelector(selector);
|
|
61
|
-
results.push({
|
|
62
|
-
selector,
|
|
63
|
-
isWhitelisted: validation.isValid,
|
|
64
|
-
confidence: validation.confidence,
|
|
65
|
-
originalCode: fullMatch,
|
|
66
|
-
validatedCode: validation.isValid
|
|
67
|
-
? fullMatch
|
|
68
|
-
: `// ${validation.suggestedComment}\n // ${fullMatch}`,
|
|
69
|
-
});
|
|
70
|
-
}
|
|
71
|
-
return results;
|
|
72
|
-
}
|
|
73
|
-
/**
|
|
74
|
-
* Apply validation to test code, commenting out unwhitelisted selectors
|
|
75
|
-
*/
|
|
76
|
-
applyValidation(code) {
|
|
77
|
-
let validated = code;
|
|
78
|
-
const results = this.validateTestCode(code);
|
|
79
|
-
// Apply in reverse order to preserve indices
|
|
80
|
-
for (const result of results.reverse()) {
|
|
81
|
-
if (!result.isWhitelisted) {
|
|
82
|
-
validated = validated.replace(result.originalCode, result.validatedCode);
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
return validated;
|
|
86
|
-
}
|
|
87
|
-
/**
|
|
88
|
-
* Get validation summary
|
|
89
|
-
*/
|
|
90
|
-
getSummary(code) {
|
|
91
|
-
const results = this.validateTestCode(code);
|
|
92
|
-
const unobserved = results.filter((r) => !r.isWhitelisted).map((r) => r.selector);
|
|
93
|
-
return {
|
|
94
|
-
total: results.length,
|
|
95
|
-
whitelisted: results.filter((r) => r.isWhitelisted).length,
|
|
96
|
-
coverage: results.length > 0
|
|
97
|
-
? Math.round((results.filter((r) => r.isWhitelisted).length / results.length) * 100)
|
|
98
|
-
: 100,
|
|
99
|
-
unobserved,
|
|
100
|
-
};
|
|
101
|
-
}
|
|
102
|
-
normalizeSelector(selector) {
|
|
103
|
-
// Normalize selector for lenient matching
|
|
104
|
-
return selector.toLowerCase().replace(/[^a-z0-9-_]/g, '');
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
/**
|
|
108
|
-
* APIFallbackResolver: Provides fallback strategies when UI selectors fail
|
|
109
|
-
* Converts test methods to API calls when UI elements aren't available
|
|
110
|
-
*/
|
|
111
|
-
export class APIFallbackResolver {
|
|
112
|
-
constructor() {
|
|
113
|
-
this.apiMapping = new Map([
|
|
114
|
-
// UI action -> API endpoint mapping
|
|
115
|
-
['click.*button.*submit', 'POST /api/v4/posts'],
|
|
116
|
-
['fill.*search', 'GET /api/v4/users'],
|
|
117
|
-
['click.*profile', 'GET /api/v4/users/me'],
|
|
118
|
-
['navigate.*channel', 'GET /api/v4/channels'],
|
|
119
|
-
['click.*settings', 'PATCH /api/v4/users/me'],
|
|
120
|
-
]);
|
|
121
|
-
}
|
|
122
|
-
/**
|
|
123
|
-
* Check if a test should fall back to API testing
|
|
124
|
-
*/
|
|
125
|
-
shouldFallback(selector, confidence) {
|
|
126
|
-
return confidence < 50;
|
|
127
|
-
}
|
|
128
|
-
/**
|
|
129
|
-
* Generate API-based fallback for unobserved selector
|
|
130
|
-
*/
|
|
131
|
-
generateAPIFallback(selector, action) {
|
|
132
|
-
// Find matching API endpoint
|
|
133
|
-
let endpoint = 'GET /api/v4/';
|
|
134
|
-
for (const [pattern, api] of this.apiMapping) {
|
|
135
|
-
if (new RegExp(pattern, 'i').test(action)) {
|
|
136
|
-
endpoint = api;
|
|
137
|
-
break;
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
return `
|
|
141
|
-
// UI selector not found - falling back to API
|
|
142
|
-
const response = await fetch(\`\${baseUrl}${endpoint.split(' ')[1]}\`, {
|
|
143
|
-
method: '${endpoint.split(' ')[0]}',
|
|
144
|
-
headers: {'Authorization': \`Bearer \${token}\`},
|
|
145
|
-
});
|
|
146
|
-
expect(response.ok).toBe(true);`;
|
|
147
|
-
}
|
|
148
|
-
/**
|
|
149
|
-
* Wrap unobserved test in try-catch with API fallback
|
|
150
|
-
*/
|
|
151
|
-
wrapWithFallback(testCode) {
|
|
152
|
-
return `
|
|
153
|
-
try {
|
|
154
|
-
${testCode}
|
|
155
|
-
} catch (error) {
|
|
156
|
-
// UI element not found - using API fallback
|
|
157
|
-
${this.generateAPIFallback('unknown', testCode)}
|
|
158
|
-
}`;
|
|
159
|
-
}
|
|
160
|
-
}
|
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
-
// See LICENSE.txt for license information.
|
|
3
|
-
/**
|
|
4
|
-
* Autonomous E2E Testing System
|
|
5
|
-
*
|
|
6
|
-
* A specification-driven testing system that bridges PDF/Markdown specs
|
|
7
|
-
* with Playwright's native agents for test planning, generation, and healing.
|
|
8
|
-
*
|
|
9
|
-
* Quick Start:
|
|
10
|
-
* ```typescript
|
|
11
|
-
* import {SpecBridge, createAnthropicBridge} from '@mattermost/playwright-lib/autonomous';
|
|
12
|
-
*
|
|
13
|
-
* // Convert a specification to Playwright-compatible markdown
|
|
14
|
-
* const bridge = createAnthropicBridge(process.env.ANTHROPIC_API_KEY);
|
|
15
|
-
* const result = await bridge.convertToPlaywrightSpecs('spec.pdf', 'specs/');
|
|
16
|
-
*
|
|
17
|
-
* // Then use Playwright agents:
|
|
18
|
-
* // @planner explore http://localhost:8065
|
|
19
|
-
* // @generator create tests from specs/
|
|
20
|
-
* // @healer fix failing tests
|
|
21
|
-
* ```
|
|
22
|
-
*
|
|
23
|
-
* Architecture:
|
|
24
|
-
* - SpecificationParser: Parses PDF/MD/JSON specs into structured format
|
|
25
|
-
* - SpecBridge: Converts specs to Playwright Agent-compatible markdown
|
|
26
|
-
* - LLM Providers: Pluggable AI providers (Anthropic, Ollama, OpenAI)
|
|
27
|
-
*
|
|
28
|
-
* The heavy lifting (test generation, execution, healing) is delegated to
|
|
29
|
-
* Playwright's built-in agents which are production-ready and maintained
|
|
30
|
-
* by the Playwright team.
|
|
31
|
-
*/
|
|
32
|
-
// Core Components
|
|
33
|
-
export { SpecificationParser } from './spec_parser.js';
|
|
34
|
-
// LLM Providers (re-exported from parent package)
|
|
35
|
-
export { LLMProviderFactory } from '../provider_factory.js';
|
|
36
|
-
export { OllamaProvider } from '../ollama_provider.js';
|
|
37
|
-
export { AnthropicProvider } from '../anthropic_provider.js';
|
|
38
|
-
/**
|
|
39
|
-
* Version info
|
|
40
|
-
*/
|
|
41
|
-
export const VERSION = '2.0.0';
|
|
42
|
-
export const SUPPORTED_PLAYWRIGHT_VERSION = '1.56.0';
|
|
43
|
-
/**
|
|
44
|
-
* Feature flags
|
|
45
|
-
*/
|
|
46
|
-
export const FEATURES = {
|
|
47
|
-
LLM_AGNOSTIC: true,
|
|
48
|
-
SPECIFICATION_DRIVEN: true,
|
|
49
|
-
PLAYWRIGHT_AGENTS: true,
|
|
50
|
-
};
|