@yasserkhanorg/e2e-agents 0.3.2 → 0.3.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/README.md +29 -20
  2. package/dist/agent/config.d.ts +3 -0
  3. package/dist/agent/config.d.ts.map +1 -1
  4. package/dist/agent/config.js +38 -0
  5. package/dist/agent/operational_insights.d.ts +1 -1
  6. package/dist/agent/operational_insights.d.ts.map +1 -1
  7. package/dist/agent/operational_insights.js +2 -1
  8. package/dist/agent/pipeline.d.ts +8 -1
  9. package/dist/agent/pipeline.d.ts.map +1 -1
  10. package/dist/agent/pipeline.js +844 -33
  11. package/dist/agent/plan.d.ts +39 -0
  12. package/dist/agent/plan.d.ts.map +1 -1
  13. package/dist/agent/plan.js +146 -0
  14. package/dist/agent/report.d.ts +16 -0
  15. package/dist/agent/report.d.ts.map +1 -1
  16. package/dist/agent/report.js +12 -0
  17. package/dist/agent/runner.d.ts.map +1 -1
  18. package/dist/agent/runner.js +66 -11
  19. package/dist/agent/tests.d.ts.map +1 -1
  20. package/dist/agent/tests.js +12 -2
  21. package/dist/api.d.ts.map +1 -1
  22. package/dist/api.js +1 -0
  23. package/dist/cli.js +111 -7
  24. package/dist/esm/agent/config.js +38 -0
  25. package/dist/esm/agent/operational_insights.js +2 -1
  26. package/dist/esm/agent/pipeline.js +844 -33
  27. package/dist/esm/agent/plan.js +145 -1
  28. package/dist/esm/agent/report.js +12 -0
  29. package/dist/esm/agent/runner.js +66 -11
  30. package/dist/esm/agent/tests.js +12 -2
  31. package/dist/esm/api.js +2 -1
  32. package/dist/esm/cli.js +112 -8
  33. package/package.json +1 -1
  34. package/schemas/impact.schema.json +37 -0
  35. package/schemas/plan.schema.json +48 -0
  36. package/dist/agent/cache_utils.d.ts +0 -38
  37. package/dist/agent/cache_utils.d.ts.map +0 -1
  38. package/dist/agent/cache_utils.js +0 -67
  39. package/dist/agent/impact-analyzer.d.ts +0 -114
  40. package/dist/agent/impact-analyzer.d.ts.map +0 -1
  41. package/dist/agent/impact-analyzer.js +0 -557
  42. package/dist/agent/index.d.ts +0 -21
  43. package/dist/agent/index.d.ts.map +0 -1
  44. package/dist/agent/index.js +0 -38
  45. package/dist/agent/model-router.d.ts +0 -57
  46. package/dist/agent/model-router.d.ts.map +0 -1
  47. package/dist/agent/model-router.js +0 -154
  48. package/dist/agent/report-generator.d.ts +0 -24
  49. package/dist/agent/report-generator.d.ts.map +0 -1
  50. package/dist/agent/report-generator.js +0 -250
  51. package/dist/agent/spec-bridge.d.ts +0 -101
  52. package/dist/agent/spec-bridge.d.ts.map +0 -1
  53. package/dist/agent/spec-bridge.js +0 -273
  54. package/dist/agent/spec-builder.d.ts +0 -102
  55. package/dist/agent/spec-builder.d.ts.map +0 -1
  56. package/dist/agent/spec-builder.js +0 -273
  57. package/dist/agent/telemetry.d.ts +0 -84
  58. package/dist/agent/telemetry.d.ts.map +0 -1
  59. package/dist/agent/telemetry.js +0 -220
  60. package/dist/agent/validators/selector-validator.d.ts +0 -74
  61. package/dist/agent/validators/selector-validator.d.ts.map +0 -1
  62. package/dist/agent/validators/selector-validator.js +0 -165
  63. package/dist/e2e-test-gen/index.d.ts +0 -51
  64. package/dist/e2e-test-gen/index.d.ts.map +0 -1
  65. package/dist/e2e-test-gen/index.js +0 -57
  66. package/dist/e2e-test-gen/spec_parser.d.ts +0 -142
  67. package/dist/e2e-test-gen/spec_parser.d.ts.map +0 -1
  68. package/dist/e2e-test-gen/spec_parser.js +0 -786
  69. package/dist/e2e-test-gen/types.d.ts +0 -185
  70. package/dist/e2e-test-gen/types.d.ts.map +0 -1
  71. package/dist/e2e-test-gen/types.js +0 -4
  72. package/dist/esm/agent/cache_utils.js +0 -63
  73. package/dist/esm/agent/impact-analyzer.js +0 -548
  74. package/dist/esm/agent/index.js +0 -22
  75. package/dist/esm/agent/model-router.js +0 -150
  76. package/dist/esm/agent/report-generator.js +0 -247
  77. package/dist/esm/agent/spec-bridge.js +0 -267
  78. package/dist/esm/agent/spec-builder.js +0 -267
  79. package/dist/esm/agent/telemetry.js +0 -216
  80. package/dist/esm/agent/validators/selector-validator.js +0 -160
  81. package/dist/esm/e2e-test-gen/index.js +0 -50
  82. package/dist/esm/e2e-test-gen/spec_parser.js +0 -782
  83. package/dist/esm/e2e-test-gen/types.js +0 -3
  84. package/dist/esm/plan-and-test-constants.js +0 -126
  85. package/dist/plan-and-test-constants.d.ts +0 -110
  86. package/dist/plan-and-test-constants.d.ts.map +0 -1
  87. package/dist/plan-and-test-constants.js +0 -132
@@ -1,84 +0,0 @@
1
- export interface GenerationMetric {
2
- id: string;
3
- timestamp: string;
4
- operation: 'explore' | 'generate' | 'heal' | 'validate' | 'score' | 'pdf-parse';
5
- model: string;
6
- tokensInput: number;
7
- tokensOutput: number;
8
- costUsd: number;
9
- durationMs: number;
10
- success: boolean;
11
- errorType?: string;
12
- errorMessage?: string;
13
- metadata?: Record<string, unknown>;
14
- }
15
- export interface TelemetryReport {
16
- period: {
17
- start: string;
18
- end: string;
19
- };
20
- summary: {
21
- totalOperations: number;
22
- successCount: number;
23
- failureCount: number;
24
- successRate: number;
25
- totalCost: number;
26
- avgCost: number;
27
- totalTokens: number;
28
- avgDuration: number;
29
- };
30
- byModel: Record<string, {
31
- count: number;
32
- totalCost: number;
33
- avgCost: number;
34
- successRate: number;
35
- }>;
36
- byOperation: Record<string, {
37
- count: number;
38
- totalCost: number;
39
- avgDuration: number;
40
- successRate: number;
41
- }>;
42
- }
43
- export declare class TelemetryCollector {
44
- private metricsDir;
45
- private metrics;
46
- constructor(metricsDir?: string);
47
- /**
48
- * Ensure metrics directory exists
49
- */
50
- private ensureMetricsDir;
51
- /**
52
- * Get today's metrics file path
53
- */
54
- private getTodayPath;
55
- /**
56
- * Load metrics from disk
57
- */
58
- private loadTodayMetrics;
59
- /**
60
- * Track a metric
61
- */
62
- track(metric: Omit<GenerationMetric, 'id'>): void;
63
- /**
64
- * Save metrics to disk
65
- */
66
- private saveMetrics;
67
- /**
68
- * Calculate cost for a metric
69
- */
70
- static calculateCost(model: string, tokensInput: number, tokensOutput: number): number;
71
- /**
72
- * Generate report for a date range
73
- */
74
- generateReport(since?: Date, until?: Date): TelemetryReport;
75
- /**
76
- * Format report for console output
77
- */
78
- static formatReport(report: TelemetryReport): string;
79
- /**
80
- * Export metrics as JSON
81
- */
82
- exportJson(filepath: string): void;
83
- }
84
- //# sourceMappingURL=telemetry.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../src/agent/telemetry.ts"],"names":[],"mappings":"AAoBA,MAAM,WAAW,gBAAgB;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,SAAS,GAAG,UAAU,GAAG,MAAM,GAAG,UAAU,GAAG,OAAO,GAAG,WAAW,CAAC;IAChF,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,eAAe;IAC5B,MAAM,EAAE;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAC,CAAC;IACrC,OAAO,EAAE;QACL,eAAe,EAAE,MAAM,CAAC;QACxB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,OAAO,EAAE,MAAM,CACX,MAAM,EACN;QACI,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,WAAW,EAAE,MAAM,CAAC;KACvB,CACJ,CAAC;IACF,WAAW,EAAE,MAAM,CACf,MAAM,EACN;QACI,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;KACvB,CACJ,CAAC;CACL;AAQD,qBAAa,kBAAkB;IAC3B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,OAAO,CAA8C;gBAEjD,UAAU,GAAE,MAAiC;IAMzD;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;OAEG;IACH,OAAO,CAAC,YAAY;IAMpB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAYxB;;OAEG;IACH,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,GAAG,IAAI;IAejD;;OAEG;IACH,OAAO,CAAC,WAAW;IAMnB;;OAEG;IACH,MAAM,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM;IAMtF;;OAEG;IACH,cAAc,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,IAAI,GAAG,eAAe;IAuF3D;;OAEG;IACH,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM;IAmCpD;;OAEG;IACH,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;CAQrC"}
@@ -1,220 +0,0 @@
1
- "use strict";
2
- // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
3
- // See LICENSE.txt for license information.
4
- Object.defineProperty(exports, "__esModule", { value: true });
5
- exports.TelemetryCollector = void 0;
6
- /**
7
- * Telemetry Collection System (Phase A2)
8
- *
9
- * Tracks costs, performance, and success metrics for test generation operations.
10
- * Provides visibility into:
11
- * - Cost per operation (input/output tokens * model rate)
12
- * - Model usage breakdown (Haiku, Sonnet, Opus)
13
- * - Success rate by operation
14
- * - Performance metrics (duration, tokens used)
15
- *
16
- * Data stored in: `.e2e-ai-agents/metrics/YYYY-MM-DD.json`
17
- */
18
- const fs_1 = require("fs");
19
- const path_1 = require("path");
20
- const crypto_1 = require("crypto");
21
- const MODEL_RATES = {
22
- 'claude-haiku-4-0-20250430': 0.25 / 1000000, // per input token
23
- 'claude-sonnet-4-5-20250929': 3 / 1000000,
24
- 'claude-opus-4-6-20250820': 15 / 1000000,
25
- };
26
- class TelemetryCollector {
27
- constructor(metricsDir = '.e2e-ai-agents/metrics') {
28
- this.metrics = new Map();
29
- this.metricsDir = metricsDir;
30
- this.ensureMetricsDir();
31
- this.loadTodayMetrics();
32
- }
33
- /**
34
- * Ensure metrics directory exists
35
- */
36
- ensureMetricsDir() {
37
- if (!(0, fs_1.existsSync)(this.metricsDir)) {
38
- (0, fs_1.mkdirSync)(this.metricsDir, { recursive: true });
39
- }
40
- }
41
- /**
42
- * Get today's metrics file path
43
- */
44
- getTodayPath() {
45
- const now = new Date();
46
- const date = now.toISOString().split('T')[0]; // YYYY-MM-DD
47
- return (0, path_1.join)(this.metricsDir, `${date}.json`);
48
- }
49
- /**
50
- * Load metrics from disk
51
- */
52
- loadTodayMetrics() {
53
- const path = this.getTodayPath();
54
- if ((0, fs_1.existsSync)(path)) {
55
- try {
56
- const data = JSON.parse((0, fs_1.readFileSync)(path, 'utf-8'));
57
- this.metrics.set(path, data);
58
- }
59
- catch (error) {
60
- console.error(`Failed to load metrics from ${path}:`, error);
61
- }
62
- }
63
- }
64
- /**
65
- * Track a metric
66
- */
67
- track(metric) {
68
- const fullMetric = {
69
- id: (0, crypto_1.randomUUID)().substring(0, 8),
70
- ...metric,
71
- };
72
- const path = this.getTodayPath();
73
- const metrics = this.metrics.get(path) || [];
74
- metrics.push(fullMetric);
75
- this.metrics.set(path, metrics);
76
- // Persist to disk
77
- this.saveMetrics();
78
- }
79
- /**
80
- * Save metrics to disk
81
- */
82
- saveMetrics() {
83
- this.metrics.forEach((metrics, path) => {
84
- (0, fs_1.writeFileSync)(path, JSON.stringify(metrics, null, 2), 'utf-8');
85
- });
86
- }
87
- /**
88
- * Calculate cost for a metric
89
- */
90
- static calculateCost(model, tokensInput, tokensOutput) {
91
- const inputRate = MODEL_RATES[model] || 0.003 / 1000000; // Default estimate
92
- const outputRate = inputRate * 3; // Output usually 3x input cost
93
- return tokensInput * inputRate + tokensOutput * outputRate;
94
- }
95
- /**
96
- * Generate report for a date range
97
- */
98
- generateReport(since, until) {
99
- const start = since || new Date(new Date().setDate(new Date().getDate() - 7)); // Default: last 7 days
100
- const end = until || new Date();
101
- // Collect all metrics in date range
102
- const allMetrics = [];
103
- this.metrics.forEach((metrics) => {
104
- metrics.forEach((m) => {
105
- const metricDate = new Date(m.timestamp);
106
- if (metricDate >= start && metricDate <= end) {
107
- allMetrics.push(m);
108
- }
109
- });
110
- });
111
- // Calculate summary
112
- const successCount = allMetrics.filter((m) => m.success).length;
113
- const failureCount = allMetrics.length - successCount;
114
- const totalCost = allMetrics.reduce((sum, m) => sum + m.costUsd, 0);
115
- const avgCost = allMetrics.length > 0 ? totalCost / allMetrics.length : 0;
116
- const totalTokens = allMetrics.reduce((sum, m) => sum + (m.tokensInput + m.tokensOutput), 0);
117
- const avgDuration = allMetrics.length > 0 ? allMetrics.reduce((sum, m) => sum + m.durationMs, 0) / allMetrics.length / 1000 : 0;
118
- // By model
119
- const byModel = {};
120
- allMetrics.forEach((m) => {
121
- if (!byModel[m.model]) {
122
- byModel[m.model] = { count: 0, totalCost: 0, successCount: 0 };
123
- }
124
- byModel[m.model].count += 1;
125
- byModel[m.model].totalCost += m.costUsd;
126
- if (m.success)
127
- byModel[m.model].successCount += 1;
128
- });
129
- Object.keys(byModel).forEach((model) => {
130
- const data = byModel[model];
131
- byModel[model] = {
132
- count: data.count,
133
- totalCost: data.totalCost,
134
- avgCost: data.totalCost / data.count,
135
- successRate: (data.successCount / data.count) * 100,
136
- };
137
- });
138
- // By operation
139
- const byOperation = {};
140
- allMetrics.forEach((m) => {
141
- if (!byOperation[m.operation]) {
142
- byOperation[m.operation] = { count: 0, totalCost: 0, totalDuration: 0, successCount: 0 };
143
- }
144
- byOperation[m.operation].count += 1;
145
- byOperation[m.operation].totalCost += m.costUsd;
146
- byOperation[m.operation].totalDuration += m.durationMs;
147
- if (m.success)
148
- byOperation[m.operation].successCount += 1;
149
- });
150
- Object.keys(byOperation).forEach((op) => {
151
- const data = byOperation[op];
152
- byOperation[op] = {
153
- count: data.count,
154
- totalCost: data.totalCost,
155
- avgDuration: data.totalDuration / data.count / 1000,
156
- successRate: (data.successCount / data.count) * 100,
157
- };
158
- });
159
- return {
160
- period: {
161
- start: start.toISOString().split('T')[0],
162
- end: end.toISOString().split('T')[0],
163
- },
164
- summary: {
165
- totalOperations: allMetrics.length,
166
- successCount,
167
- failureCount,
168
- successRate: allMetrics.length > 0 ? (successCount / allMetrics.length) * 100 : 0,
169
- totalCost,
170
- avgCost,
171
- totalTokens,
172
- avgDuration,
173
- },
174
- byModel,
175
- byOperation,
176
- };
177
- }
178
- /**
179
- * Format report for console output
180
- */
181
- static formatReport(report) {
182
- const lines = [
183
- '',
184
- '📊 Test Generation Metrics',
185
- `Period: ${report.period.start} to ${report.period.end}`,
186
- '═'.repeat(50),
187
- '',
188
- `Total Operations: ${report.summary.totalOperations}`,
189
- `Success Rate: ${report.summary.successRate.toFixed(1)}% (${report.summary.successCount}/${report.summary.totalOperations})`,
190
- `Total Cost: $${report.summary.totalCost.toFixed(2)}`,
191
- `Avg Cost/Op: $${report.summary.avgCost.toFixed(4)}`,
192
- `Avg Duration: ${report.summary.avgDuration.toFixed(1)}s`,
193
- `Total Tokens: ${report.summary.totalTokens.toLocaleString()}`,
194
- '',
195
- 'Model Usage:',
196
- ];
197
- Object.entries(report.byModel).forEach(([model, data]) => {
198
- const shortName = model.includes('haiku') ? 'Haiku' : model.includes('sonnet') ? 'Sonnet' : 'Opus';
199
- lines.push(` ${shortName}: ${data.count} ops - $${data.totalCost.toFixed(2)} (avg $${data.avgCost.toFixed(4)}, ${data.successRate.toFixed(0)}% success)`);
200
- });
201
- lines.push('', 'By Operation:');
202
- Object.entries(report.byOperation).forEach(([op, data]) => {
203
- lines.push(` ${op}: ${data.count} ops - $${data.totalCost.toFixed(2)} (${data.avgDuration.toFixed(1)}s avg, ${data.successRate.toFixed(0)}% success)`);
204
- });
205
- lines.push('');
206
- return lines.join('\n');
207
- }
208
- /**
209
- * Export metrics as JSON
210
- */
211
- exportJson(filepath) {
212
- const allMetrics = [];
213
- this.metrics.forEach((metrics) => {
214
- allMetrics.push(...metrics);
215
- });
216
- (0, fs_1.writeFileSync)(filepath, JSON.stringify(allMetrics, null, 2), 'utf-8');
217
- console.log(` ✓ Exported ${allMetrics.length} metrics to ${filepath}`);
218
- }
219
- }
220
- exports.TelemetryCollector = TelemetryCollector;
@@ -1,74 +0,0 @@
1
- /**
2
- * Phase 3: Selector Enforcement
3
- * Validates selectors against whitelist and comments out invalid ones
4
- */
5
- export interface ValidationResult {
6
- isValid: boolean;
7
- confidence: number;
8
- reason?: string;
9
- suggestedComment?: string;
10
- }
11
- export interface ValidatedSelector {
12
- selector: string;
13
- isWhitelisted: boolean;
14
- confidence: number;
15
- originalCode: string;
16
- validatedCode: string;
17
- }
18
- /**
19
- * SelectorValidator: Enforces whitelist matching on generated selectors
20
- * Comments out unobserved selectors instead of letting tests fail randomly
21
- */
22
- export declare class SelectorValidator {
23
- private whitelist;
24
- private semanticWhitelist;
25
- private minConfidence;
26
- constructor(globalSelectors: {
27
- [semantic: string]: Array<{
28
- selector: string;
29
- confidence: number;
30
- }>;
31
- }, minConfidence?: number);
32
- /**
33
- * Validate a selector against the whitelist
34
- */
35
- validateSelector(selector: string): ValidationResult;
36
- /**
37
- * Validate and comment out invalid selectors in generated test code
38
- */
39
- validateTestCode(code: string): ValidatedSelector[];
40
- /**
41
- * Apply validation to test code, commenting out unwhitelisted selectors
42
- */
43
- applyValidation(code: string): string;
44
- /**
45
- * Get validation summary
46
- */
47
- getSummary(code: string): {
48
- total: number;
49
- whitelisted: number;
50
- coverage: number;
51
- unobserved: string[];
52
- };
53
- private normalizeSelector;
54
- }
55
- /**
56
- * APIFallbackResolver: Provides fallback strategies when UI selectors fail
57
- * Converts test methods to API calls when UI elements aren't available
58
- */
59
- export declare class APIFallbackResolver {
60
- private apiMapping;
61
- /**
62
- * Check if a test should fall back to API testing
63
- */
64
- shouldFallback(selector: string, confidence: number): boolean;
65
- /**
66
- * Generate API-based fallback for unobserved selector
67
- */
68
- generateAPIFallback(selector: string, action: string): string;
69
- /**
70
- * Wrap unobserved test in try-catch with API fallback
71
- */
72
- wrapWithFallback(testCode: string): string;
73
- }
74
- //# sourceMappingURL=selector-validator.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"selector-validator.d.ts","sourceRoot":"","sources":["../../../src/agent/validators/selector-validator.ts"],"names":[],"mappings":"AAGA;;;GAGG;AAEH,MAAM,WAAW,gBAAgB;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,iBAAiB;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,OAAO,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACzB;AAED;;;GAGG;AACH,qBAAa,iBAAiB;IAC1B,OAAO,CAAC,SAAS,CAA0B;IAC3C,OAAO,CAAC,iBAAiB,CAAoC;IAC7D,OAAO,CAAC,aAAa,CAAS;gBAG1B,eAAe,EAAE;QAAC,CAAC,QAAQ,EAAE,MAAM,GAAG,KAAK,CAAC;YAAC,QAAQ,EAAE,MAAM,CAAC;YAAC,UAAU,EAAE,MAAM,CAAA;SAAC,CAAC,CAAA;KAAC,EACpF,aAAa,GAAE,MAAW;IAkB9B;;OAEG;IACH,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,gBAAgB;IA6BpD;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE;IAuBnD;;OAEG;IACH,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAcrC;;OAEG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG;QACtB,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,EAAE,CAAC;KACxB;IAeD,OAAO,CAAC,iBAAiB;CAI5B;AAED;;;GAGG;AACH,qBAAa,mBAAmB;IAC5B,OAAO,CAAC,UAAU,CAOf;IAEH;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO;IAI7D;;OAEG;IACH,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAmB7D;;OAEG;IACH,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;CAS7C"}
@@ -1,165 +0,0 @@
1
- "use strict";
2
- // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
3
- // See LICENSE.txt for license information.
4
- Object.defineProperty(exports, "__esModule", { value: true });
5
- exports.APIFallbackResolver = exports.SelectorValidator = void 0;
6
- /**
7
- * SelectorValidator: Enforces whitelist matching on generated selectors
8
- * Comments out unobserved selectors instead of letting tests fail randomly
9
- */
10
- class SelectorValidator {
11
- constructor(globalSelectors, minConfidence = 50) {
12
- this.whitelist = new Set();
13
- this.semanticWhitelist = new Map();
14
- this.minConfidence = minConfidence;
15
- // Build flat whitelist from semantic map
16
- for (const [semantic, elements] of Object.entries(globalSelectors)) {
17
- for (const elem of elements) {
18
- if (elem.confidence >= minConfidence) {
19
- this.whitelist.add(elem.selector);
20
- }
21
- }
22
- this.semanticWhitelist.set(semantic, elements.map((e) => e.selector));
23
- }
24
- }
25
- /**
26
- * Validate a selector against the whitelist
27
- */
28
- validateSelector(selector) {
29
- if (this.whitelist.has(selector)) {
30
- return {
31
- isValid: true,
32
- confidence: 100,
33
- reason: 'Found in whitelist',
34
- };
35
- }
36
- // Check for similar selectors (lenient matching)
37
- const normalized = this.normalizeSelector(selector);
38
- for (const whitelisted of this.whitelist) {
39
- if (this.normalizeSelector(whitelisted).includes(normalized)) {
40
- return {
41
- isValid: true,
42
- confidence: 75,
43
- reason: 'Found similar whitelisted selector',
44
- };
45
- }
46
- }
47
- return {
48
- isValid: false,
49
- confidence: 0,
50
- reason: 'Not found in whitelist',
51
- suggestedComment: `// UNOBSERVED SELECTOR - Not found in UI map. Use test.fixme() if needed.`,
52
- };
53
- }
54
- /**
55
- * Validate and comment out invalid selectors in generated test code
56
- */
57
- validateTestCode(code) {
58
- const results = [];
59
- const selectorRegex = /page\.(getByTestId|getByLabel|getByRole|locator)\(['"`]([^'"`]+)['"`]\)/g;
60
- let match;
61
- while ((match = selectorRegex.exec(code)) !== null) {
62
- const [fullMatch, , selector] = match;
63
- const validation = this.validateSelector(selector);
64
- results.push({
65
- selector,
66
- isWhitelisted: validation.isValid,
67
- confidence: validation.confidence,
68
- originalCode: fullMatch,
69
- validatedCode: validation.isValid
70
- ? fullMatch
71
- : `// ${validation.suggestedComment}\n // ${fullMatch}`,
72
- });
73
- }
74
- return results;
75
- }
76
- /**
77
- * Apply validation to test code, commenting out unwhitelisted selectors
78
- */
79
- applyValidation(code) {
80
- let validated = code;
81
- const results = this.validateTestCode(code);
82
- // Apply in reverse order to preserve indices
83
- for (const result of results.reverse()) {
84
- if (!result.isWhitelisted) {
85
- validated = validated.replace(result.originalCode, result.validatedCode);
86
- }
87
- }
88
- return validated;
89
- }
90
- /**
91
- * Get validation summary
92
- */
93
- getSummary(code) {
94
- const results = this.validateTestCode(code);
95
- const unobserved = results.filter((r) => !r.isWhitelisted).map((r) => r.selector);
96
- return {
97
- total: results.length,
98
- whitelisted: results.filter((r) => r.isWhitelisted).length,
99
- coverage: results.length > 0
100
- ? Math.round((results.filter((r) => r.isWhitelisted).length / results.length) * 100)
101
- : 100,
102
- unobserved,
103
- };
104
- }
105
- normalizeSelector(selector) {
106
- // Normalize selector for lenient matching
107
- return selector.toLowerCase().replace(/[^a-z0-9-_]/g, '');
108
- }
109
- }
110
- exports.SelectorValidator = SelectorValidator;
111
- /**
112
- * APIFallbackResolver: Provides fallback strategies when UI selectors fail
113
- * Converts test methods to API calls when UI elements aren't available
114
- */
115
- class APIFallbackResolver {
116
- constructor() {
117
- this.apiMapping = new Map([
118
- // UI action -> API endpoint mapping
119
- ['click.*button.*submit', 'POST /api/v4/posts'],
120
- ['fill.*search', 'GET /api/v4/users'],
121
- ['click.*profile', 'GET /api/v4/users/me'],
122
- ['navigate.*channel', 'GET /api/v4/channels'],
123
- ['click.*settings', 'PATCH /api/v4/users/me'],
124
- ]);
125
- }
126
- /**
127
- * Check if a test should fall back to API testing
128
- */
129
- shouldFallback(selector, confidence) {
130
- return confidence < 50;
131
- }
132
- /**
133
- * Generate API-based fallback for unobserved selector
134
- */
135
- generateAPIFallback(selector, action) {
136
- // Find matching API endpoint
137
- let endpoint = 'GET /api/v4/';
138
- for (const [pattern, api] of this.apiMapping) {
139
- if (new RegExp(pattern, 'i').test(action)) {
140
- endpoint = api;
141
- break;
142
- }
143
- }
144
- return `
145
- // UI selector not found - falling back to API
146
- const response = await fetch(\`\${baseUrl}${endpoint.split(' ')[1]}\`, {
147
- method: '${endpoint.split(' ')[0]}',
148
- headers: {'Authorization': \`Bearer \${token}\`},
149
- });
150
- expect(response.ok).toBe(true);`;
151
- }
152
- /**
153
- * Wrap unobserved test in try-catch with API fallback
154
- */
155
- wrapWithFallback(testCode) {
156
- return `
157
- try {
158
- ${testCode}
159
- } catch (error) {
160
- // UI element not found - using API fallback
161
- ${this.generateAPIFallback('unknown', testCode)}
162
- }`;
163
- }
164
- }
165
- exports.APIFallbackResolver = APIFallbackResolver;
@@ -1,51 +0,0 @@
1
- /**
2
- * Autonomous E2E Testing System
3
- *
4
- * A specification-driven testing system that bridges PDF/Markdown specs
5
- * with Playwright's native agents for test planning, generation, and healing.
6
- *
7
- * Quick Start:
8
- * ```typescript
9
- * import {SpecBridge, createAnthropicBridge} from '@mattermost/playwright-lib/autonomous';
10
- *
11
- * // Convert a specification to Playwright-compatible markdown
12
- * const bridge = createAnthropicBridge(process.env.ANTHROPIC_API_KEY);
13
- * const result = await bridge.convertToPlaywrightSpecs('spec.pdf', 'specs/');
14
- *
15
- * // Then use Playwright agents:
16
- * // @planner explore http://localhost:8065
17
- * // @generator create tests from specs/
18
- * // @healer fix failing tests
19
- * ```
20
- *
21
- * Architecture:
22
- * - SpecificationParser: Parses PDF/MD/JSON specs into structured format
23
- * - SpecBridge: Converts specs to Playwright Agent-compatible markdown
24
- * - LLM Providers: Pluggable AI providers (Anthropic, Ollama, OpenAI)
25
- *
26
- * The heavy lifting (test generation, execution, healing) is delegated to
27
- * Playwright's built-in agents which are production-ready and maintained
28
- * by the Playwright team.
29
- */
30
- export { SpecificationParser } from './spec_parser.js';
31
- export type { SpecSummary, SpecificationCache } from './spec_parser.js';
32
- export { LLMProviderFactory } from '../provider_factory.js';
33
- export { OllamaProvider } from '../ollama_provider.js';
34
- export { AnthropicProvider } from '../anthropic_provider.js';
35
- export type { LLMProvider, LLMResponse, GenerateOptions, ImageInput, ProviderCapabilities, ProviderUsageStats, ProviderConfig, OllamaConfig, AnthropicConfig, } from '../provider_interface.js';
36
- export type { HybridConfig } from '../provider_factory.js';
37
- export type { FeatureSpecification, BusinessScenario, SpecScreenshot, GeneratedTest, } from './types.js';
38
- /**
39
- * Version info
40
- */
41
- export declare const VERSION = "2.0.0";
42
- export declare const SUPPORTED_PLAYWRIGHT_VERSION = "1.56.0";
43
- /**
44
- * Feature flags
45
- */
46
- export declare const FEATURES: {
47
- readonly LLM_AGNOSTIC: true;
48
- readonly SPECIFICATION_DRIVEN: true;
49
- readonly PLAYWRIGHT_AGENTS: true;
50
- };
51
- //# sourceMappingURL=index.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/e2e-test-gen/index.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAGH,OAAO,EAAC,mBAAmB,EAAC,MAAM,kBAAkB,CAAC;AACrD,YAAY,EAAC,WAAW,EAAE,kBAAkB,EAAC,MAAM,kBAAkB,CAAC;AAGtE,OAAO,EAAC,kBAAkB,EAAC,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAC,cAAc,EAAC,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAE3D,YAAY,EACR,WAAW,EACX,WAAW,EACX,eAAe,EACf,UAAU,EACV,oBAAoB,EACpB,kBAAkB,EAClB,cAAc,EACd,YAAY,EACZ,eAAe,GAClB,MAAM,0BAA0B,CAAC;AAClC,YAAY,EAAC,YAAY,EAAC,MAAM,wBAAwB,CAAC;AAGzD,YAAY,EAER,oBAAoB,EACpB,gBAAgB,EAChB,cAAc,EAGd,aAAa,GAChB,MAAM,YAAY,CAAC;AAEpB;;GAEG;AACH,eAAO,MAAM,OAAO,UAAU,CAAC;AAC/B,eAAO,MAAM,4BAA4B,WAAW,CAAC;AAErD;;GAEG;AACH,eAAO,MAAM,QAAQ;;;;CAIX,CAAC"}
@@ -1,57 +0,0 @@
1
- "use strict";
2
- // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
3
- // See LICENSE.txt for license information.
4
- Object.defineProperty(exports, "__esModule", { value: true });
5
- exports.FEATURES = exports.SUPPORTED_PLAYWRIGHT_VERSION = exports.VERSION = exports.AnthropicProvider = exports.OllamaProvider = exports.LLMProviderFactory = exports.SpecificationParser = void 0;
6
- /**
7
- * Autonomous E2E Testing System
8
- *
9
- * A specification-driven testing system that bridges PDF/Markdown specs
10
- * with Playwright's native agents for test planning, generation, and healing.
11
- *
12
- * Quick Start:
13
- * ```typescript
14
- * import {SpecBridge, createAnthropicBridge} from '@mattermost/playwright-lib/autonomous';
15
- *
16
- * // Convert a specification to Playwright-compatible markdown
17
- * const bridge = createAnthropicBridge(process.env.ANTHROPIC_API_KEY);
18
- * const result = await bridge.convertToPlaywrightSpecs('spec.pdf', 'specs/');
19
- *
20
- * // Then use Playwright agents:
21
- * // @planner explore http://localhost:8065
22
- * // @generator create tests from specs/
23
- * // @healer fix failing tests
24
- * ```
25
- *
26
- * Architecture:
27
- * - SpecificationParser: Parses PDF/MD/JSON specs into structured format
28
- * - SpecBridge: Converts specs to Playwright Agent-compatible markdown
29
- * - LLM Providers: Pluggable AI providers (Anthropic, Ollama, OpenAI)
30
- *
31
- * The heavy lifting (test generation, execution, healing) is delegated to
32
- * Playwright's built-in agents which are production-ready and maintained
33
- * by the Playwright team.
34
- */
35
- // Core Components
36
- var spec_parser_js_1 = require("./spec_parser.js");
37
- Object.defineProperty(exports, "SpecificationParser", { enumerable: true, get: function () { return spec_parser_js_1.SpecificationParser; } });
38
- // LLM Providers (re-exported from parent package)
39
- var provider_factory_js_1 = require("../provider_factory.js");
40
- Object.defineProperty(exports, "LLMProviderFactory", { enumerable: true, get: function () { return provider_factory_js_1.LLMProviderFactory; } });
41
- var ollama_provider_js_1 = require("../ollama_provider.js");
42
- Object.defineProperty(exports, "OllamaProvider", { enumerable: true, get: function () { return ollama_provider_js_1.OllamaProvider; } });
43
- var anthropic_provider_js_1 = require("../anthropic_provider.js");
44
- Object.defineProperty(exports, "AnthropicProvider", { enumerable: true, get: function () { return anthropic_provider_js_1.AnthropicProvider; } });
45
- /**
46
- * Version info
47
- */
48
- exports.VERSION = '2.0.0';
49
- exports.SUPPORTED_PLAYWRIGHT_VERSION = '1.56.0';
50
- /**
51
- * Feature flags
52
- */
53
- exports.FEATURES = {
54
- LLM_AGNOSTIC: true,
55
- SPECIFICATION_DRIVEN: true,
56
- PLAYWRIGHT_AGENTS: true,
57
- };