@yasserkhanorg/e2e-agents 0.3.2 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -20
- package/dist/agent/config.d.ts +3 -0
- package/dist/agent/config.d.ts.map +1 -1
- package/dist/agent/config.js +38 -0
- package/dist/agent/operational_insights.d.ts +1 -1
- package/dist/agent/operational_insights.d.ts.map +1 -1
- package/dist/agent/operational_insights.js +2 -1
- package/dist/agent/pipeline.d.ts +8 -1
- package/dist/agent/pipeline.d.ts.map +1 -1
- package/dist/agent/pipeline.js +844 -33
- package/dist/agent/plan.d.ts +39 -0
- package/dist/agent/plan.d.ts.map +1 -1
- package/dist/agent/plan.js +146 -0
- package/dist/agent/report.d.ts +16 -0
- package/dist/agent/report.d.ts.map +1 -1
- package/dist/agent/report.js +12 -0
- package/dist/agent/runner.d.ts.map +1 -1
- package/dist/agent/runner.js +66 -11
- package/dist/agent/tests.d.ts.map +1 -1
- package/dist/agent/tests.js +12 -2
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +1 -0
- package/dist/cli.js +111 -7
- package/dist/esm/agent/config.js +38 -0
- package/dist/esm/agent/operational_insights.js +2 -1
- package/dist/esm/agent/pipeline.js +844 -33
- package/dist/esm/agent/plan.js +145 -1
- package/dist/esm/agent/report.js +12 -0
- package/dist/esm/agent/runner.js +66 -11
- package/dist/esm/agent/tests.js +12 -2
- package/dist/esm/api.js +2 -1
- package/dist/esm/cli.js +112 -8
- package/package.json +1 -1
- package/schemas/impact.schema.json +37 -0
- package/schemas/plan.schema.json +48 -0
- package/dist/agent/cache_utils.d.ts +0 -38
- package/dist/agent/cache_utils.d.ts.map +0 -1
- package/dist/agent/cache_utils.js +0 -67
- package/dist/agent/impact-analyzer.d.ts +0 -114
- package/dist/agent/impact-analyzer.d.ts.map +0 -1
- package/dist/agent/impact-analyzer.js +0 -557
- package/dist/agent/index.d.ts +0 -21
- package/dist/agent/index.d.ts.map +0 -1
- package/dist/agent/index.js +0 -38
- package/dist/agent/model-router.d.ts +0 -57
- package/dist/agent/model-router.d.ts.map +0 -1
- package/dist/agent/model-router.js +0 -154
- package/dist/agent/report-generator.d.ts +0 -24
- package/dist/agent/report-generator.d.ts.map +0 -1
- package/dist/agent/report-generator.js +0 -250
- package/dist/agent/spec-bridge.d.ts +0 -101
- package/dist/agent/spec-bridge.d.ts.map +0 -1
- package/dist/agent/spec-bridge.js +0 -273
- package/dist/agent/spec-builder.d.ts +0 -102
- package/dist/agent/spec-builder.d.ts.map +0 -1
- package/dist/agent/spec-builder.js +0 -273
- package/dist/agent/telemetry.d.ts +0 -84
- package/dist/agent/telemetry.d.ts.map +0 -1
- package/dist/agent/telemetry.js +0 -220
- package/dist/agent/validators/selector-validator.d.ts +0 -74
- package/dist/agent/validators/selector-validator.d.ts.map +0 -1
- package/dist/agent/validators/selector-validator.js +0 -165
- package/dist/e2e-test-gen/index.d.ts +0 -51
- package/dist/e2e-test-gen/index.d.ts.map +0 -1
- package/dist/e2e-test-gen/index.js +0 -57
- package/dist/e2e-test-gen/spec_parser.d.ts +0 -142
- package/dist/e2e-test-gen/spec_parser.d.ts.map +0 -1
- package/dist/e2e-test-gen/spec_parser.js +0 -786
- package/dist/e2e-test-gen/types.d.ts +0 -185
- package/dist/e2e-test-gen/types.d.ts.map +0 -1
- package/dist/e2e-test-gen/types.js +0 -4
- package/dist/esm/agent/cache_utils.js +0 -63
- package/dist/esm/agent/impact-analyzer.js +0 -548
- package/dist/esm/agent/index.js +0 -22
- package/dist/esm/agent/model-router.js +0 -150
- package/dist/esm/agent/report-generator.js +0 -247
- package/dist/esm/agent/spec-bridge.js +0 -267
- package/dist/esm/agent/spec-builder.js +0 -267
- package/dist/esm/agent/telemetry.js +0 -216
- package/dist/esm/agent/validators/selector-validator.js +0 -160
- package/dist/esm/e2e-test-gen/index.js +0 -50
- package/dist/esm/e2e-test-gen/spec_parser.js +0 -782
- package/dist/esm/e2e-test-gen/types.js +0 -3
- package/dist/esm/plan-and-test-constants.js +0 -126
- package/dist/plan-and-test-constants.d.ts +0 -110
- package/dist/plan-and-test-constants.d.ts.map +0 -1
- package/dist/plan-and-test-constants.js +0 -132
|
@@ -1,150 +0,0 @@
|
|
|
1
|
-
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
-
// See LICENSE.txt for license information.
|
|
3
|
-
const MODEL_RATES = {
|
|
4
|
-
'claude-haiku-4-0-20250430': 0.25 / 1000000,
|
|
5
|
-
'claude-sonnet-4-5-20250929': 3 / 1000000,
|
|
6
|
-
'claude-opus-4-6-20250820': 15 / 1000000,
|
|
7
|
-
};
|
|
8
|
-
export class ModelRouter {
|
|
9
|
-
constructor(config = {}) {
|
|
10
|
-
this.modelConfig = {
|
|
11
|
-
simpleModel: config.simpleModel || 'claude-haiku-4-0-20250430',
|
|
12
|
-
moderateModel: config.moderateModel || 'claude-sonnet-4-5-20250929',
|
|
13
|
-
complexModel: config.complexModel || 'claude-opus-4-6-20250820',
|
|
14
|
-
criticalModel: config.criticalModel || 'claude-opus-4-6-20250820',
|
|
15
|
-
};
|
|
16
|
-
}
|
|
17
|
-
/**
|
|
18
|
-
* Classify task complexity based on operation, attempt number, and context
|
|
19
|
-
*/
|
|
20
|
-
classifyTask(context) {
|
|
21
|
-
// Healing: Haiku for attempts 1-2, Sonnet for attempt 3
|
|
22
|
-
if (context.operation === 'heal') {
|
|
23
|
-
if (context.attemptNumber && context.attemptNumber <= 2) {
|
|
24
|
-
// First two healing attempts use simple classification
|
|
25
|
-
return {
|
|
26
|
-
type: 'simple',
|
|
27
|
-
confidence: 95,
|
|
28
|
-
reasoning: `Healing attempt ${context.attemptNumber}/3 - re-exploration with targeted fixes`,
|
|
29
|
-
};
|
|
30
|
-
}
|
|
31
|
-
// Final healing attempt is more complex
|
|
32
|
-
return {
|
|
33
|
-
type: 'moderate',
|
|
34
|
-
confidence: 90,
|
|
35
|
-
reasoning: 'Final healing attempt - may need comprehensive refactoring',
|
|
36
|
-
};
|
|
37
|
-
}
|
|
38
|
-
// Validation: Always simple (use Haiku)
|
|
39
|
-
if (context.operation === 'validate') {
|
|
40
|
-
return {
|
|
41
|
-
type: 'simple',
|
|
42
|
-
confidence: 100,
|
|
43
|
-
reasoning: 'Selector/code validation is lightweight',
|
|
44
|
-
};
|
|
45
|
-
}
|
|
46
|
-
// Scoring: Simple (quick analysis)
|
|
47
|
-
if (context.operation === 'score') {
|
|
48
|
-
return {
|
|
49
|
-
type: 'simple',
|
|
50
|
-
confidence: 95,
|
|
51
|
-
reasoning: 'Test quality scoring via static analysis',
|
|
52
|
-
};
|
|
53
|
-
}
|
|
54
|
-
// Exploration: Simple (just navigation and snapshot)
|
|
55
|
-
if (context.operation === 'explore') {
|
|
56
|
-
return {
|
|
57
|
-
type: 'simple',
|
|
58
|
-
confidence: 90,
|
|
59
|
-
reasoning: 'UI exploration is mostly navigation',
|
|
60
|
-
};
|
|
61
|
-
}
|
|
62
|
-
// Generation: Varies by UI map coverage
|
|
63
|
-
if (context.operation === 'generate') {
|
|
64
|
-
// Strong signal: Use Sonnet but can optimize
|
|
65
|
-
if (context.uiMapCoverage && context.uiMapCoverage >= 75) {
|
|
66
|
-
return {
|
|
67
|
-
type: 'moderate',
|
|
68
|
-
confidence: 90,
|
|
69
|
-
reasoning: `Strong UI signal (${context.uiMapCoverage}% coverage) - moderate complexity generation`,
|
|
70
|
-
};
|
|
71
|
-
}
|
|
72
|
-
// Moderate signal: Use Sonnet
|
|
73
|
-
if (context.uiMapCoverage && context.uiMapCoverage >= 50) {
|
|
74
|
-
return {
|
|
75
|
-
type: 'moderate',
|
|
76
|
-
confidence: 75,
|
|
77
|
-
reasoning: `Moderate UI signal (${context.uiMapCoverage}% coverage) - standard generation`,
|
|
78
|
-
};
|
|
79
|
-
}
|
|
80
|
-
// Weak signal: Complex task (need better reasoning)
|
|
81
|
-
if (context.uiMapCoverage && context.uiMapCoverage < 50) {
|
|
82
|
-
return {
|
|
83
|
-
type: 'complex',
|
|
84
|
-
confidence: 70,
|
|
85
|
-
reasoning: `Weak UI signal (${context.uiMapCoverage}% coverage) - requires advanced reasoning`,
|
|
86
|
-
};
|
|
87
|
-
}
|
|
88
|
-
// Unknown coverage: Assume moderate
|
|
89
|
-
return {
|
|
90
|
-
type: 'moderate',
|
|
91
|
-
confidence: 50,
|
|
92
|
-
reasoning: 'Generation with unknown UI coverage - use standard model',
|
|
93
|
-
};
|
|
94
|
-
}
|
|
95
|
-
// Default to moderate
|
|
96
|
-
return {
|
|
97
|
-
type: 'moderate',
|
|
98
|
-
confidence: 50,
|
|
99
|
-
reasoning: 'Unknown operation - defaulting to moderate complexity',
|
|
100
|
-
};
|
|
101
|
-
}
|
|
102
|
-
/**
|
|
103
|
-
* Select appropriate model based on task complexity
|
|
104
|
-
*/
|
|
105
|
-
selectModel(complexity) {
|
|
106
|
-
switch (complexity.type) {
|
|
107
|
-
case 'simple':
|
|
108
|
-
return this.modelConfig.simpleModel;
|
|
109
|
-
case 'moderate':
|
|
110
|
-
return this.modelConfig.moderateModel;
|
|
111
|
-
case 'complex':
|
|
112
|
-
return this.modelConfig.complexModel;
|
|
113
|
-
case 'critical':
|
|
114
|
-
return this.modelConfig.criticalModel;
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
/**
|
|
118
|
-
* Get estimated cost for a task
|
|
119
|
-
*/
|
|
120
|
-
estimateCost(complexity, estimatedTokens = 5000) {
|
|
121
|
-
const model = this.selectModel(complexity);
|
|
122
|
-
const rate = MODEL_RATES[model] || 0.003 / 1000000;
|
|
123
|
-
return estimatedTokens * rate;
|
|
124
|
-
}
|
|
125
|
-
/**
|
|
126
|
-
* Get cost savings vs always using Sonnet
|
|
127
|
-
*/
|
|
128
|
-
estimateSavings(complexity, estimatedTokens = 5000) {
|
|
129
|
-
const selectedModel = this.selectModel(complexity);
|
|
130
|
-
const selectedRate = MODEL_RATES[selectedModel] || 0.003 / 1000000;
|
|
131
|
-
const sonnetRate = MODEL_RATES['claude-sonnet-4-5-20250929'];
|
|
132
|
-
const selectedCost = estimatedTokens * selectedRate;
|
|
133
|
-
const sonnetCost = estimatedTokens * sonnetRate;
|
|
134
|
-
const savedCost = sonnetCost - selectedCost;
|
|
135
|
-
const savingsPercent = (savedCost / sonnetCost) * 100;
|
|
136
|
-
return { savedCost, savingsPercent };
|
|
137
|
-
}
|
|
138
|
-
/**
|
|
139
|
-
* Format complexity for logging
|
|
140
|
-
*/
|
|
141
|
-
formatComplexity(complexity, tokensUsed) {
|
|
142
|
-
const model = this.selectModel(complexity);
|
|
143
|
-
const modelShort = model.includes('haiku') ? 'Haiku' : model.includes('sonnet') ? 'Sonnet' : 'Opus';
|
|
144
|
-
const confidence = `${complexity.confidence}%`;
|
|
145
|
-
const cost = tokensUsed
|
|
146
|
-
? ` ($${(tokensUsed * (MODEL_RATES[model] || 0)).toFixed(4)})`
|
|
147
|
-
: '';
|
|
148
|
-
return `${complexity.type.toUpperCase()}/${modelShort}/${confidence}${cost} - ${complexity.reasoning}`;
|
|
149
|
-
}
|
|
150
|
-
}
|
|
@@ -1,247 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
3
|
-
* See LICENSE.txt for license information.
|
|
4
|
-
*
|
|
5
|
-
* Report Generation Engine
|
|
6
|
-
*
|
|
7
|
-
* Generates console, markdown, and JSON reports from impact analysis results.
|
|
8
|
-
*/
|
|
9
|
-
import { writeFileSync, mkdirSync } from 'fs';
|
|
10
|
-
import { join } from 'path';
|
|
11
|
-
/**
|
|
12
|
-
* Generate all requested report formats
|
|
13
|
-
*/
|
|
14
|
-
export async function generateReports(analysis, options = {}) {
|
|
15
|
-
const { console: consoleReport = true, markdown = true, json = true, outputDir = './.e2e-ai-agents/reports', } = options;
|
|
16
|
-
if (consoleReport) {
|
|
17
|
-
printConsoleReport(analysis);
|
|
18
|
-
}
|
|
19
|
-
if (markdown || json) {
|
|
20
|
-
mkdirSync(outputDir, { recursive: true });
|
|
21
|
-
}
|
|
22
|
-
if (markdown) {
|
|
23
|
-
const mdPath = join(outputDir, `impact-${Date.now()}.md`);
|
|
24
|
-
const markdownReport = generateMarkdownReport(analysis);
|
|
25
|
-
writeFileSync(mdPath, markdownReport);
|
|
26
|
-
console.log(`\n📄 Markdown report: ${mdPath}`);
|
|
27
|
-
}
|
|
28
|
-
if (json) {
|
|
29
|
-
const jsonPath = join(outputDir, `impact-${Date.now()}.json`);
|
|
30
|
-
writeFileSync(jsonPath, JSON.stringify(analysis, null, 2));
|
|
31
|
-
console.log(`📊 JSON report: ${jsonPath}`);
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
// =============================================================================
|
|
35
|
-
// CONSOLE REPORT
|
|
36
|
-
// =============================================================================
|
|
37
|
-
function printConsoleReport(analysis) {
|
|
38
|
-
console.log('\n═══════════════════════════════════════════════════════════════════');
|
|
39
|
-
console.log('📊 CODE IMPACT ANALYSIS REPORT');
|
|
40
|
-
console.log('═══════════════════════════════════════════════════════════════════\n');
|
|
41
|
-
// Summary
|
|
42
|
-
console.log('📈 SUMMARY');
|
|
43
|
-
console.log(` Changed files: ${analysis.totalChanges}`);
|
|
44
|
-
console.log(` Affected flows: ${analysis.affectedFlows.length}`);
|
|
45
|
-
console.log(` Priority breakdown: P0=${analysis.priorityBreakdown.p0}, P1=${analysis.priorityBreakdown.p1}, P2=${analysis.priorityBreakdown.p2}`);
|
|
46
|
-
console.log(` Test coverage: ${analysis.testCoverage.covered}/${analysis.testCoverage.total} flows have tests (${analysis.testCoverage.gaps} gaps)\n`);
|
|
47
|
-
// P0 Flows (Critical)
|
|
48
|
-
const p0Flows = analysis.affectedFlows.filter((f) => f.flow.priority === 'P0');
|
|
49
|
-
if (p0Flows.length > 0) {
|
|
50
|
-
console.log('🔴 CRITICAL (P0) FLOWS AFFECTED:');
|
|
51
|
-
p0Flows.forEach((impact) => {
|
|
52
|
-
console.log(` • ${impact.flow.name} (${impact.flow.id})`);
|
|
53
|
-
console.log(` Confidence: ${impact.confidence}% | Match: ${impact.matchType}`);
|
|
54
|
-
console.log(` Existing tests: ${impact.existingTests.length} ${impact.existingTests.length > 0 ? '✓' : '✗'}`);
|
|
55
|
-
if (impact.testGaps.length > 0) {
|
|
56
|
-
console.log(` ⚠️ Test gaps: ${impact.testGaps.slice(0, 2).join(', ')}${impact.testGaps.length > 2 ? '...' : ''}`);
|
|
57
|
-
}
|
|
58
|
-
console.log(` Affected files: ${impact.affectedFiles.length}`);
|
|
59
|
-
});
|
|
60
|
-
console.log('');
|
|
61
|
-
}
|
|
62
|
-
// P1 Flows (High Priority)
|
|
63
|
-
const p1Flows = analysis.affectedFlows.filter((f) => f.flow.priority === 'P1');
|
|
64
|
-
if (p1Flows.length > 0) {
|
|
65
|
-
console.log('🟡 HIGH PRIORITY (P1) FLOWS AFFECTED:');
|
|
66
|
-
p1Flows.slice(0, 5).forEach((impact) => {
|
|
67
|
-
console.log(` • ${impact.flow.name} (${impact.flow.id})`);
|
|
68
|
-
console.log(` Tests: ${impact.existingTests.length}/${impact.affectedFiles.length} files, Gaps: ${impact.testGaps.length}`);
|
|
69
|
-
});
|
|
70
|
-
if (p1Flows.length > 5) {
|
|
71
|
-
console.log(` ... and ${p1Flows.length - 5} more P1 flows`);
|
|
72
|
-
}
|
|
73
|
-
console.log('');
|
|
74
|
-
}
|
|
75
|
-
// Recommendations
|
|
76
|
-
if (analysis.recommendations.length > 0) {
|
|
77
|
-
console.log('💡 RECOMMENDATIONS:');
|
|
78
|
-
analysis.recommendations.forEach((rec, i) => {
|
|
79
|
-
console.log(` ${i + 1}. ${rec}`);
|
|
80
|
-
});
|
|
81
|
-
console.log('');
|
|
82
|
-
}
|
|
83
|
-
// Action Items
|
|
84
|
-
console.log('🎯 SUGGESTED ACTIONS:');
|
|
85
|
-
if (analysis.hasP0Impact) {
|
|
86
|
-
console.log(' 1. Run P0 flow tests immediately:');
|
|
87
|
-
const p0Tests = p0Flows.flatMap((f) => f.existingTests).filter((t) => t);
|
|
88
|
-
if (p0Tests.length > 0) {
|
|
89
|
-
console.log(` npx playwright test ${p0Tests.slice(0, 3).join(' ')}`);
|
|
90
|
-
if (p0Tests.length > 3) {
|
|
91
|
-
console.log(` ... and ${p0Tests.length - 3} more P0 tests`);
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
else {
|
|
95
|
-
console.log(' ⚠️ No existing P0 tests found - generate tests first:');
|
|
96
|
-
console.log(' npx e2e-ai-agents approve-and-generate --path <app-root> --tests-root <tests-root> --pipeline --pipeline-scenarios 3');
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
if (analysis.testCoverage.gaps > 0) {
|
|
100
|
-
console.log(` 2. Address ${analysis.testCoverage.gaps} test coverage gap(s):`);
|
|
101
|
-
const gapFlows = analysis.affectedFlows.filter((f) => f.testGaps.length > 0).slice(0, 2);
|
|
102
|
-
gapFlows.forEach(() => {
|
|
103
|
-
console.log(' npx e2e-ai-agents approve-and-generate --path <app-root> --tests-root <tests-root> --pipeline --pipeline-scenarios 3');
|
|
104
|
-
});
|
|
105
|
-
if (analysis.testCoverage.gaps > 2) {
|
|
106
|
-
console.log(` # ... and ${analysis.testCoverage.gaps - 2} more flows with gaps`);
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
console.log('\n═══════════════════════════════════════════════════════════════════\n');
|
|
110
|
-
}
|
|
111
|
-
// =============================================================================
|
|
112
|
-
// MARKDOWN REPORT
|
|
113
|
-
// =============================================================================
|
|
114
|
-
function generateMarkdownReport(analysis) {
|
|
115
|
-
const lines = [];
|
|
116
|
-
lines.push('# Code Impact Analysis Report');
|
|
117
|
-
lines.push('');
|
|
118
|
-
lines.push(`**Generated**: ${new Date(analysis.timestamp).toLocaleString()}`);
|
|
119
|
-
lines.push(`**Git Reference**: ${analysis.gitRef}`);
|
|
120
|
-
lines.push('');
|
|
121
|
-
// Risk Level Badge
|
|
122
|
-
const riskLevel = analysis.hasP0Impact
|
|
123
|
-
? '🔴 **HIGH** (P0 flows affected)'
|
|
124
|
-
: analysis.priorityBreakdown.p1 > 0
|
|
125
|
-
? '🟡 **MEDIUM** (P1 flows affected)'
|
|
126
|
-
: '🟢 **LOW** (Only P2 flows affected)';
|
|
127
|
-
// Executive Summary
|
|
128
|
-
lines.push('## Executive Summary');
|
|
129
|
-
lines.push('');
|
|
130
|
-
lines.push(`- **Changed files**: ${analysis.totalChanges}`);
|
|
131
|
-
lines.push(`- **Affected flows**: ${analysis.affectedFlows.length}`);
|
|
132
|
-
lines.push(`- **Priority**: P0=${analysis.priorityBreakdown.p0}, P1=${analysis.priorityBreakdown.p1}, P2=${analysis.priorityBreakdown.p2}`);
|
|
133
|
-
lines.push(`- **Test coverage**: ${analysis.testCoverage.covered}/${analysis.testCoverage.total} flows have tests (${analysis.testCoverage.gaps} gaps)`);
|
|
134
|
-
lines.push(`- **Risk level**: ${riskLevel}`);
|
|
135
|
-
lines.push('');
|
|
136
|
-
// Critical Flows
|
|
137
|
-
const p0Flows = analysis.affectedFlows.filter((f) => f.flow.priority === 'P0');
|
|
138
|
-
if (p0Flows.length > 0) {
|
|
139
|
-
lines.push('## 🔴 Critical (P0) Flows Affected');
|
|
140
|
-
lines.push('');
|
|
141
|
-
p0Flows.forEach((impact) => {
|
|
142
|
-
lines.push(`### ${impact.flow.name} (\`${impact.flow.id}\`)`);
|
|
143
|
-
lines.push('');
|
|
144
|
-
lines.push(`**Confidence**: ${impact.confidence}%`);
|
|
145
|
-
lines.push(`**Match type**: ${impact.matchType}`);
|
|
146
|
-
lines.push(`**Affected files**: ${impact.affectedFiles.length} file${impact.affectedFiles.length !== 1 ? 's' : ''}`);
|
|
147
|
-
lines.push('');
|
|
148
|
-
lines.push(`**Existing tests**: ${impact.existingTests.length > 0 ? '✓ ' + impact.existingTests.length : '✗ None'}`);
|
|
149
|
-
if (impact.existingTests.length > 0) {
|
|
150
|
-
lines.push('```');
|
|
151
|
-
impact.existingTests.slice(0, 3).forEach((test) => {
|
|
152
|
-
lines.push(test);
|
|
153
|
-
});
|
|
154
|
-
if (impact.existingTests.length > 3) {
|
|
155
|
-
lines.push(`... and ${impact.existingTests.length - 3} more`);
|
|
156
|
-
}
|
|
157
|
-
lines.push('```');
|
|
158
|
-
}
|
|
159
|
-
if (impact.testGaps.length > 0) {
|
|
160
|
-
lines.push('');
|
|
161
|
-
lines.push(`**Test gaps**:`);
|
|
162
|
-
impact.testGaps.slice(0, 3).forEach((gap) => {
|
|
163
|
-
lines.push(`- ${gap}`);
|
|
164
|
-
});
|
|
165
|
-
if (impact.testGaps.length > 3) {
|
|
166
|
-
lines.push(`- ... and ${impact.testGaps.length - 3} more`);
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
lines.push('');
|
|
170
|
-
});
|
|
171
|
-
}
|
|
172
|
-
// High Priority Flows
|
|
173
|
-
const p1Flows = analysis.affectedFlows.filter((f) => f.flow.priority === 'P1');
|
|
174
|
-
if (p1Flows.length > 0) {
|
|
175
|
-
lines.push('## 🟡 High Priority (P1) Flows Affected');
|
|
176
|
-
lines.push('');
|
|
177
|
-
lines.push('| Flow | ID | Files | Tests | Gaps |');
|
|
178
|
-
lines.push('|------|----| ------|-------|------|');
|
|
179
|
-
p1Flows.slice(0, 10).forEach((impact) => {
|
|
180
|
-
lines.push(`| ${impact.flow.name} | \`${impact.flow.id}\` | ${impact.affectedFiles.length} | ${impact.existingTests.length} | ${impact.testGaps.length} |`);
|
|
181
|
-
});
|
|
182
|
-
if (p1Flows.length > 10) {
|
|
183
|
-
lines.push(`| ... ${p1Flows.length - 10} more | ... | ... | ... | ... |`);
|
|
184
|
-
}
|
|
185
|
-
lines.push('');
|
|
186
|
-
}
|
|
187
|
-
// Recommendations
|
|
188
|
-
if (analysis.recommendations.length > 0) {
|
|
189
|
-
lines.push('## 💡 Recommendations');
|
|
190
|
-
lines.push('');
|
|
191
|
-
analysis.recommendations.forEach((rec, i) => {
|
|
192
|
-
lines.push(`${i + 1}. ${rec}`);
|
|
193
|
-
});
|
|
194
|
-
lines.push('');
|
|
195
|
-
}
|
|
196
|
-
// Action Items
|
|
197
|
-
lines.push('## 🎯 Action Items');
|
|
198
|
-
lines.push('');
|
|
199
|
-
if (analysis.hasP0Impact) {
|
|
200
|
-
lines.push('### Immediate: Run P0 Tests');
|
|
201
|
-
lines.push('');
|
|
202
|
-
const p0Tests = p0Flows.flatMap((f) => f.existingTests).filter((t) => t);
|
|
203
|
-
if (p0Tests.length > 0) {
|
|
204
|
-
lines.push('```bash');
|
|
205
|
-
lines.push(`npx playwright test ${p0Tests.slice(0, 3).join(' ')}`);
|
|
206
|
-
if (p0Tests.length > 3) {
|
|
207
|
-
lines.push(`# ... and ${p0Tests.length - 3} more P0 tests`);
|
|
208
|
-
}
|
|
209
|
-
lines.push('```');
|
|
210
|
-
}
|
|
211
|
-
else {
|
|
212
|
-
lines.push('No existing P0 tests found. Generate tests:');
|
|
213
|
-
lines.push('');
|
|
214
|
-
lines.push('```bash');
|
|
215
|
-
lines.push('npx e2e-ai-agents approve-and-generate --path <app-root> --tests-root <tests-root> --pipeline --pipeline-scenarios 3');
|
|
216
|
-
lines.push('```');
|
|
217
|
-
}
|
|
218
|
-
lines.push('');
|
|
219
|
-
}
|
|
220
|
-
if (analysis.testCoverage.gaps > 0) {
|
|
221
|
-
lines.push('### Generate Missing Tests');
|
|
222
|
-
lines.push('');
|
|
223
|
-
const gapFlows = analysis.affectedFlows.filter((f) => f.testGaps.length > 0).slice(0, 3);
|
|
224
|
-
if (gapFlows.length > 0) {
|
|
225
|
-
lines.push('```bash');
|
|
226
|
-
gapFlows.forEach(() => {
|
|
227
|
-
lines.push('npx e2e-ai-agents approve-and-generate --path <app-root> --tests-root <tests-root> --pipeline --pipeline-scenarios 3');
|
|
228
|
-
});
|
|
229
|
-
if (analysis.testCoverage.gaps > 3) {
|
|
230
|
-
lines.push(`# ... and ${analysis.testCoverage.gaps - 3} more gaps to address`);
|
|
231
|
-
}
|
|
232
|
-
lines.push('```');
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
|
-
lines.push('');
|
|
236
|
-
lines.push('---');
|
|
237
|
-
lines.push('');
|
|
238
|
-
lines.push('_This report was generated by the E2E Impact Analysis Agent. To regenerate, run: `npx e2e-ai-agents impact --path <app-root> --tests-root <tests-root>`_');
|
|
239
|
-
return lines.join('\n');
|
|
240
|
-
}
|
|
241
|
-
// =============================================================================
|
|
242
|
-
// JSON REPORT
|
|
243
|
-
// =============================================================================
|
|
244
|
-
/**
|
|
245
|
-
* JSON report is generated directly in generateReports()
|
|
246
|
-
* No additional formatting needed
|
|
247
|
-
*/
|
|
@@ -1,267 +0,0 @@
|
|
|
1
|
-
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
-
// See LICENSE.txt for license information.
|
|
3
|
-
/**
|
|
4
|
-
* Spec Bridge - Bridge between PDF specs and Playwright Agent workflow
|
|
5
|
-
*
|
|
6
|
-
* This module converts specification documents (PDF, Markdown, JSON) into
|
|
7
|
-
* Playwright-compatible markdown files that can be consumed by Playwright's
|
|
8
|
-
* native agents (Planner, Generator, Healer).
|
|
9
|
-
*
|
|
10
|
-
* Flow: PDF → SpecParser → Markdown specs → Playwright Planner
|
|
11
|
-
*
|
|
12
|
-
* The Playwright agents (available in Playwright 1.56+) provide production-ready:
|
|
13
|
-
* - Test planning and exploration
|
|
14
|
-
* - Test code generation
|
|
15
|
-
* - Automatic test healing
|
|
16
|
-
*
|
|
17
|
-
* Usage:
|
|
18
|
-
* ```typescript
|
|
19
|
-
* const bridge = new SpecBridge({type: 'anthropic', apiKey: process.env.ANTHROPIC_API_KEY});
|
|
20
|
-
* const specPaths = await bridge.convertToPlaywrightSpecs('spec.pdf', 'specs/');
|
|
21
|
-
* // Now use Playwright agents: @planner, @generator, @healer
|
|
22
|
-
* ```
|
|
23
|
-
*/
|
|
24
|
-
import { writeFileSync, mkdirSync, existsSync } from 'fs';
|
|
25
|
-
import { join, basename } from 'path';
|
|
26
|
-
import { LLMProviderFactory } from '../index.js';
|
|
27
|
-
import { SpecificationParser } from '../e2e-test-gen/spec_parser';
|
|
28
|
-
/**
|
|
29
|
-
* Bridge between PDF/JSON specs and Playwright Agent workflow
|
|
30
|
-
*
|
|
31
|
-
* Converts specification documents into Playwright-compatible markdown
|
|
32
|
-
* that can be consumed by Playwright's native agents.
|
|
33
|
-
*/
|
|
34
|
-
export class SpecBridge {
|
|
35
|
-
constructor(config) {
|
|
36
|
-
// Handle both single provider and hybrid config
|
|
37
|
-
let llmProvider;
|
|
38
|
-
if ('primary' in config.llmConfig) {
|
|
39
|
-
// It's a HybridConfig
|
|
40
|
-
llmProvider = LLMProviderFactory.createHybrid(config.llmConfig);
|
|
41
|
-
}
|
|
42
|
-
else {
|
|
43
|
-
// It's a ProviderConfig
|
|
44
|
-
llmProvider = LLMProviderFactory.create(config.llmConfig);
|
|
45
|
-
}
|
|
46
|
-
this.parser = new SpecificationParser(llmProvider);
|
|
47
|
-
this.outputDir = config.outputDir || 'specs';
|
|
48
|
-
this.overwrite = config.overwrite !== false;
|
|
49
|
-
}
|
|
50
|
-
/**
|
|
51
|
-
* Convert a specification file to Playwright-compatible markdown specs
|
|
52
|
-
*
|
|
53
|
-
* @param inputPath - Path to the input specification (PDF, MD, or JSON)
|
|
54
|
-
* @param outputDir - Optional override for output directory
|
|
55
|
-
* @returns Conversion result with paths and metadata
|
|
56
|
-
*/
|
|
57
|
-
async convertToPlaywrightSpecs(inputPath, outputDir) {
|
|
58
|
-
/* eslint-disable no-console */
|
|
59
|
-
const targetDir = outputDir || this.outputDir;
|
|
60
|
-
const warnings = [];
|
|
61
|
-
console.log(`Converting specification: ${inputPath}`);
|
|
62
|
-
// Parse the specification
|
|
63
|
-
const specs = await this.parser.parse(inputPath, 'file');
|
|
64
|
-
if (specs.length === 0) {
|
|
65
|
-
warnings.push('No features extracted from specification');
|
|
66
|
-
return {
|
|
67
|
-
specPaths: [],
|
|
68
|
-
features: [],
|
|
69
|
-
totalScenarios: 0,
|
|
70
|
-
warnings,
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
|
-
// Create output directory
|
|
74
|
-
mkdirSync(targetDir, { recursive: true });
|
|
75
|
-
const specPaths = [];
|
|
76
|
-
let totalScenarios = 0;
|
|
77
|
-
for (const spec of specs) {
|
|
78
|
-
// Validate the spec
|
|
79
|
-
const validation = this.parser.validateSpec(spec);
|
|
80
|
-
if (!validation.valid) {
|
|
81
|
-
warnings.push(`Feature "${spec.name}": ${validation.errors.join(', ')}`);
|
|
82
|
-
}
|
|
83
|
-
// Convert to Playwright markdown format
|
|
84
|
-
const markdown = this.toPlaywrightMarkdown(spec);
|
|
85
|
-
// Generate output filename
|
|
86
|
-
const filename = `${spec.id}.md`;
|
|
87
|
-
const outputPath = join(targetDir, filename);
|
|
88
|
-
// Check for existing file
|
|
89
|
-
if (existsSync(outputPath) && !this.overwrite) {
|
|
90
|
-
warnings.push(`Skipping existing file: ${outputPath}`);
|
|
91
|
-
continue;
|
|
92
|
-
}
|
|
93
|
-
// Write the spec file
|
|
94
|
-
writeFileSync(outputPath, markdown, 'utf-8');
|
|
95
|
-
specPaths.push(outputPath);
|
|
96
|
-
totalScenarios += spec.scenarios.length;
|
|
97
|
-
console.log(` Created: ${outputPath} (${spec.scenarios.length} scenarios)`);
|
|
98
|
-
}
|
|
99
|
-
console.log(`Converted ${specs.length} feature(s) with ${totalScenarios} scenario(s)`);
|
|
100
|
-
/* eslint-enable no-console */
|
|
101
|
-
return {
|
|
102
|
-
specPaths,
|
|
103
|
-
features: specs,
|
|
104
|
-
totalScenarios,
|
|
105
|
-
warnings,
|
|
106
|
-
};
|
|
107
|
-
}
|
|
108
|
-
/**
|
|
109
|
-
* Convert a FeatureSpecification to Playwright Agent markdown format
|
|
110
|
-
*
|
|
111
|
-
* The format is designed to be consumed by Playwright's Planner and Generator agents.
|
|
112
|
-
* It includes structured test scenarios in Given-When-Then format.
|
|
113
|
-
*/
|
|
114
|
-
toPlaywrightMarkdown(spec) {
|
|
115
|
-
const lines = [];
|
|
116
|
-
// Header
|
|
117
|
-
lines.push(`# ${spec.name}`);
|
|
118
|
-
lines.push('');
|
|
119
|
-
// Metadata
|
|
120
|
-
lines.push(`**Priority**: ${spec.priority}`);
|
|
121
|
-
if (spec.targetUrls.length > 0) {
|
|
122
|
-
lines.push(`**Target URLs**: ${spec.targetUrls.join(', ')}`);
|
|
123
|
-
}
|
|
124
|
-
lines.push('');
|
|
125
|
-
// Description
|
|
126
|
-
if (spec.description) {
|
|
127
|
-
lines.push('## Description');
|
|
128
|
-
lines.push('');
|
|
129
|
-
lines.push(spec.description);
|
|
130
|
-
lines.push('');
|
|
131
|
-
}
|
|
132
|
-
// Test Scenarios
|
|
133
|
-
if (spec.scenarios.length > 0) {
|
|
134
|
-
lines.push('## Test Scenarios');
|
|
135
|
-
lines.push('');
|
|
136
|
-
for (const scenario of spec.scenarios) {
|
|
137
|
-
lines.push(this.formatScenario(scenario));
|
|
138
|
-
lines.push('');
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
// Acceptance Criteria
|
|
142
|
-
if (spec.acceptanceCriteria.length > 0) {
|
|
143
|
-
lines.push('## Acceptance Criteria');
|
|
144
|
-
lines.push('');
|
|
145
|
-
for (const criterion of spec.acceptanceCriteria) {
|
|
146
|
-
lines.push(`- ${criterion}`);
|
|
147
|
-
}
|
|
148
|
-
lines.push('');
|
|
149
|
-
}
|
|
150
|
-
// Screenshots reference (if any)
|
|
151
|
-
if (spec.screenshots.length > 0) {
|
|
152
|
-
lines.push('## Reference Screenshots');
|
|
153
|
-
lines.push('');
|
|
154
|
-
for (const screenshot of spec.screenshots) {
|
|
155
|
-
lines.push(`- ${screenshot.description}: \`${screenshot.path}\``);
|
|
156
|
-
}
|
|
157
|
-
lines.push('');
|
|
158
|
-
}
|
|
159
|
-
// Metadata footer
|
|
160
|
-
lines.push('---');
|
|
161
|
-
lines.push(`*Generated from: ${basename(spec.sourcePath)}*`);
|
|
162
|
-
lines.push(`*Feature ID: ${spec.id}*`);
|
|
163
|
-
return lines.join('\n');
|
|
164
|
-
}
|
|
165
|
-
/**
|
|
166
|
-
* Format a business scenario in Playwright-friendly markdown
|
|
167
|
-
*/
|
|
168
|
-
formatScenario(scenario) {
|
|
169
|
-
const lines = [];
|
|
170
|
-
lines.push(`### ${scenario.name}`);
|
|
171
|
-
lines.push('');
|
|
172
|
-
lines.push(`**Priority**: ${scenario.priority}`);
|
|
173
|
-
lines.push('');
|
|
174
|
-
if (scenario.given) {
|
|
175
|
-
lines.push(`**Given**: ${scenario.given}`);
|
|
176
|
-
}
|
|
177
|
-
if (scenario.when) {
|
|
178
|
-
lines.push(`**When**: ${scenario.when}`);
|
|
179
|
-
}
|
|
180
|
-
if (scenario.then) {
|
|
181
|
-
lines.push(`**Then**: ${scenario.then}`);
|
|
182
|
-
}
|
|
183
|
-
return lines.join('\n');
|
|
184
|
-
}
|
|
185
|
-
/**
|
|
186
|
-
* Parse a spec file without writing output
|
|
187
|
-
* Useful for validation or inspection
|
|
188
|
-
*/
|
|
189
|
-
async parseSpec(inputPath) {
|
|
190
|
-
return this.parser.parse(inputPath, 'file');
|
|
191
|
-
}
|
|
192
|
-
/**
|
|
193
|
-
* Validate a specification file
|
|
194
|
-
*/
|
|
195
|
-
async validateSpec(inputPath) {
|
|
196
|
-
const specs = await this.parser.parse(inputPath, 'file');
|
|
197
|
-
const allErrors = [];
|
|
198
|
-
let totalScenarios = 0;
|
|
199
|
-
for (const spec of specs) {
|
|
200
|
-
const validation = this.parser.validateSpec(spec);
|
|
201
|
-
if (!validation.valid) {
|
|
202
|
-
allErrors.push(...validation.errors.map((e) => `${spec.name}: ${e}`));
|
|
203
|
-
}
|
|
204
|
-
totalScenarios += spec.scenarios.length;
|
|
205
|
-
}
|
|
206
|
-
return {
|
|
207
|
-
valid: allErrors.length === 0,
|
|
208
|
-
errors: allErrors,
|
|
209
|
-
features: specs.length,
|
|
210
|
-
scenarios: totalScenarios,
|
|
211
|
-
};
|
|
212
|
-
}
|
|
213
|
-
/**
|
|
214
|
-
* Get a summary of the specification
|
|
215
|
-
*/
|
|
216
|
-
async getSpecSummary(inputPath) {
|
|
217
|
-
const specs = await this.parser.parse(inputPath, 'file');
|
|
218
|
-
const lines = [];
|
|
219
|
-
lines.push(`Specification Summary: ${basename(inputPath)}`);
|
|
220
|
-
lines.push('='.repeat(50));
|
|
221
|
-
lines.push('');
|
|
222
|
-
for (const spec of specs) {
|
|
223
|
-
const summary = this.parser.getSpecSummary(spec);
|
|
224
|
-
lines.push(`Feature: ${summary.name}`);
|
|
225
|
-
lines.push(` Priority: ${summary.priority}`);
|
|
226
|
-
lines.push(` Scenarios: ${summary.scenarioCount}`);
|
|
227
|
-
lines.push(` - Must-have: ${summary.mustHaveScenarios}`);
|
|
228
|
-
lines.push(` - Should-have: ${summary.shouldHaveScenarios}`);
|
|
229
|
-
lines.push(` - Nice-to-have: ${summary.niceToHaveScenarios}`);
|
|
230
|
-
lines.push(` Acceptance Criteria: ${summary.acceptanceCriteriaCount}`);
|
|
231
|
-
lines.push(` Has Screenshots: ${summary.hasScreenshots ? 'Yes' : 'No'}`);
|
|
232
|
-
lines.push('');
|
|
233
|
-
}
|
|
234
|
-
return lines.join('\n');
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
|
-
/**
|
|
238
|
-
* Create a SpecBridge with Anthropic provider
|
|
239
|
-
* Convenience function for common use case
|
|
240
|
-
*/
|
|
241
|
-
export function createAnthropicBridge(apiKey, outputDir) {
|
|
242
|
-
return new SpecBridge({
|
|
243
|
-
llmConfig: {
|
|
244
|
-
type: 'anthropic',
|
|
245
|
-
config: {
|
|
246
|
-
apiKey: apiKey || process.env.ANTHROPIC_API_KEY || '',
|
|
247
|
-
},
|
|
248
|
-
},
|
|
249
|
-
outputDir,
|
|
250
|
-
});
|
|
251
|
-
}
|
|
252
|
-
/**
|
|
253
|
-
* Create a SpecBridge with Ollama provider (free, local)
|
|
254
|
-
* Convenience function for local development
|
|
255
|
-
*/
|
|
256
|
-
export function createOllamaBridge(model, outputDir) {
|
|
257
|
-
return new SpecBridge({
|
|
258
|
-
llmConfig: {
|
|
259
|
-
type: 'ollama',
|
|
260
|
-
config: {
|
|
261
|
-
model: model || 'deepseek-r1:7b',
|
|
262
|
-
baseUrl: process.env.OLLAMA_BASE_URL || 'http://localhost:11434',
|
|
263
|
-
},
|
|
264
|
-
},
|
|
265
|
-
outputDir,
|
|
266
|
-
});
|
|
267
|
-
}
|