@testivai/witness-playwright 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -3
- package/dist/cli/init.js +2 -2
- package/dist/reporter.js +10 -14
- package/dist/snapshot.js +89 -2
- package/dist/types.d.ts +66 -0
- package/package.json +2 -1
- package/progress.md +88 -3
- package/src/cli/init.ts +2 -2
- package/src/reporter.ts +11 -19
- package/src/snapshot.ts +98 -3
- package/src/types.ts +69 -0
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @testivai/witness-playwright
|
|
2
2
|
|
|
3
|
-
**Status**: ✅ Production Ready | **Last Updated**: January
|
|
3
|
+
**Status**: ✅ Production Ready | **Last Updated**: January 10, 2026
|
|
4
4
|
|
|
5
5
|
**Project:** @testivai/witness-playwright (MVP - 1 Month Plan)
|
|
6
6
|
|
|
@@ -25,9 +25,10 @@ To use the reporter, you need to configure it in your `playwright.config.ts` fil
|
|
|
25
25
|
```bash
|
|
26
26
|
# Create .env file
|
|
27
27
|
echo "TESTIVAI_API_KEY=tstvai-your-key-here" > .env
|
|
28
|
-
echo "TESTIVAI_API_URL=https://core-api-147980626268.us-central1.run.app" >> .env
|
|
29
28
|
```
|
|
30
29
|
|
|
30
|
+
**Note:** The SDK automatically uses the production API URL. You only need to set `TESTIVAI_API_KEY`.
|
|
31
|
+
|
|
31
32
|
```typescript
|
|
32
33
|
// playwright.config.ts
|
|
33
34
|
import { defineConfig } from '@playwright/test';
|
|
@@ -72,6 +73,74 @@ test('my example test', async ({ page }, testInfo) => {
|
|
|
72
73
|
npm install @testivai/witness-playwright
|
|
73
74
|
```
|
|
74
75
|
|
|
76
|
+
## Performance Metrics
|
|
77
|
+
|
|
78
|
+
**NEW**: Automatically capture page performance metrics and Core Web Vitals!
|
|
79
|
+
|
|
80
|
+
### Basic Performance Capture (Enabled by Default)
|
|
81
|
+
|
|
82
|
+
Every snapshot automatically captures:
|
|
83
|
+
- **Core Web Vitals**: LCP, FCP, CLS
|
|
84
|
+
- **Page Load Metrics**: DOM Content Loaded, Load Complete
|
|
85
|
+
- **Navigation Timing**: Navigation start and timing data
|
|
86
|
+
|
|
87
|
+
No configuration needed - it just works!
|
|
88
|
+
|
|
89
|
+
### Optional Lighthouse Integration
|
|
90
|
+
|
|
91
|
+
For comprehensive performance audits, enable Lighthouse:
|
|
92
|
+
|
|
93
|
+
```typescript
|
|
94
|
+
// testivai.config.ts
|
|
95
|
+
export default {
|
|
96
|
+
performance: {
|
|
97
|
+
captureTimings: true, // Basic timing (default: ON)
|
|
98
|
+
enableLighthouse: false, // Lighthouse audit (default: OFF)
|
|
99
|
+
lighthouseThresholds: {
|
|
100
|
+
performance: 80,
|
|
101
|
+
accessibility: 90,
|
|
102
|
+
bestPractices: 80,
|
|
103
|
+
seo: 80
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
### Per-Test Configuration
|
|
110
|
+
|
|
111
|
+
Enable Lighthouse for specific tests:
|
|
112
|
+
|
|
113
|
+
```typescript
|
|
114
|
+
test('performance critical page', async ({ page }, testInfo) => {
|
|
115
|
+
await page.goto('https://example.com');
|
|
116
|
+
|
|
117
|
+
await testivai.witness(page, testInfo, 'homepage', {
|
|
118
|
+
performance: {
|
|
119
|
+
enableLighthouse: true,
|
|
120
|
+
lighthouseThresholds: {
|
|
121
|
+
performance: 90
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
});
|
|
125
|
+
});
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
### What Gets Captured
|
|
129
|
+
|
|
130
|
+
**Basic Timing (Always Captured)**:
|
|
131
|
+
- First Contentful Paint (FCP)
|
|
132
|
+
- Largest Contentful Paint (LCP)
|
|
133
|
+
- Cumulative Layout Shift (CLS)
|
|
134
|
+
- DOM Content Loaded
|
|
135
|
+
- Load Complete
|
|
136
|
+
|
|
137
|
+
**Lighthouse (When Enabled)**:
|
|
138
|
+
- Performance Score (0-100)
|
|
139
|
+
- Accessibility Score (0-100)
|
|
140
|
+
- Best Practices Score (0-100)
|
|
141
|
+
- SEO Score (0-100)
|
|
142
|
+
- Detailed Core Web Vitals
|
|
143
|
+
|
|
75
144
|
## 📊 Progress
|
|
76
145
|
|
|
77
|
-
See [
|
|
146
|
+
See [progress.md](./progress.md) for detailed development progress.
|
package/dist/cli/init.js
CHANGED
|
@@ -126,9 +126,8 @@ async function createConfigFile() {
|
|
|
126
126
|
console.log('📁 Config file:', configPath);
|
|
127
127
|
console.log('');
|
|
128
128
|
console.log('📖 Next steps:');
|
|
129
|
-
console.log(' 1. Set up
|
|
129
|
+
console.log(' 1. Set up your API key:');
|
|
130
130
|
console.log(' TESTIVAI_API_KEY=tstvai-your-key-here');
|
|
131
|
-
console.log(' TESTIVAI_API_URL=https://core-api-147980626268.us-central1.run.app');
|
|
132
131
|
console.log('');
|
|
133
132
|
console.log(' 2. Update your playwright.config.ts to use TestivAI reporter:');
|
|
134
133
|
console.log(' reporter: [[\'@testivai/witness-playwright/reporter\']]');
|
|
@@ -137,6 +136,7 @@ async function createConfigFile() {
|
|
|
137
136
|
console.log(' 4. Run your tests: npx playwright test');
|
|
138
137
|
console.log('');
|
|
139
138
|
console.log('💡 Get your API key from: https://dashboard-147980626268.us-central1.run.app');
|
|
139
|
+
console.log('💡 The SDK automatically connects to the production API - no URL configuration needed!');
|
|
140
140
|
}
|
|
141
141
|
catch (error) {
|
|
142
142
|
console.error('❌ Failed to create configuration file:', error);
|
package/dist/reporter.js
CHANGED
|
@@ -49,13 +49,14 @@ class TestivAIPlaywrightReporter {
|
|
|
49
49
|
this.runId = null;
|
|
50
50
|
this.tempDir = path.join(process.cwd(), '.testivai', 'temp');
|
|
51
51
|
this.options = {
|
|
52
|
-
apiUrl: options.apiUrl || process.env.TESTIVAI_API_URL,
|
|
52
|
+
apiUrl: options.apiUrl || process.env.TESTIVAI_API_URL || 'https://core-api-147980626268.us-central1.run.app',
|
|
53
53
|
apiKey: options.apiKey || process.env.TESTIVAI_API_KEY,
|
|
54
54
|
};
|
|
55
55
|
}
|
|
56
56
|
async onBegin(config, suite) {
|
|
57
|
-
if (!this.options.
|
|
58
|
-
console.error('Testivai Reporter: API
|
|
57
|
+
if (!this.options.apiKey) {
|
|
58
|
+
console.error('Testivai Reporter: API Key is not configured. Disabling reporter.');
|
|
59
|
+
console.error('Set TESTIVAI_API_KEY environment variable or pass apiKey in reporter options.');
|
|
59
60
|
this.options.apiUrl = undefined; // Disable reporter
|
|
60
61
|
return;
|
|
61
62
|
}
|
|
@@ -105,7 +106,6 @@ class TestivAIPlaywrightReporter {
|
|
|
105
106
|
return;
|
|
106
107
|
}
|
|
107
108
|
const snapshots = [];
|
|
108
|
-
const filesToUpload = [];
|
|
109
109
|
for (const jsonFile of jsonFiles) {
|
|
110
110
|
const metadataPath = path.join(this.tempDir, jsonFile);
|
|
111
111
|
const metadata = await fs.readJson(metadataPath);
|
|
@@ -119,6 +119,9 @@ class TestivAIPlaywrightReporter {
|
|
|
119
119
|
}
|
|
120
120
|
const firstSelector = layoutKeys[0];
|
|
121
121
|
const layoutData = metadata.layout[firstSelector];
|
|
122
|
+
// Read screenshot and encode to base64
|
|
123
|
+
const screenshotBuffer = await fs.readFile(screenshotPath);
|
|
124
|
+
const screenshotBase64 = screenshotBuffer.toString('base64');
|
|
122
125
|
const snapshotPayload = {
|
|
123
126
|
...metadata,
|
|
124
127
|
dom: { html: await fs.readFile(domPath, 'utf-8') },
|
|
@@ -128,10 +131,10 @@ class TestivAIPlaywrightReporter {
|
|
|
128
131
|
width: layoutData.width,
|
|
129
132
|
height: layoutData.height
|
|
130
133
|
},
|
|
131
|
-
testivaiConfig: metadata.testivaiConfig
|
|
134
|
+
testivaiConfig: metadata.testivaiConfig,
|
|
135
|
+
screenshotData: screenshotBase64 // Include base64-encoded screenshot
|
|
132
136
|
};
|
|
133
137
|
snapshots.push(snapshotPayload);
|
|
134
|
-
filesToUpload.push({ filePath: screenshotPath, contentType: 'image/png' });
|
|
135
138
|
}
|
|
136
139
|
const batchPayload = {
|
|
137
140
|
git: this.gitInfo,
|
|
@@ -140,20 +143,13 @@ class TestivAIPlaywrightReporter {
|
|
|
140
143
|
timestamp: Date.now(),
|
|
141
144
|
runId: this.runId,
|
|
142
145
|
};
|
|
143
|
-
// Start batch
|
|
146
|
+
// Start batch - screenshots are now included in the payload
|
|
144
147
|
const startBatchResponse = await axios_1.default.post(`${this.options.apiUrl}/api/v1/ingest/start-batch`, batchPayload, {
|
|
145
148
|
headers: { 'X-API-KEY': this.options.apiKey },
|
|
146
149
|
});
|
|
147
150
|
console.log('Testivai Reporter: API Response:', JSON.stringify(startBatchResponse.data, null, 2));
|
|
148
151
|
// Handle both snake_case and camelCase response formats
|
|
149
152
|
const batchId = startBatchResponse.data.batch_id || startBatchResponse.data.batchId;
|
|
150
|
-
const uploadInstructions = startBatchResponse.data.upload_instructions || startBatchResponse.data.uploadInstructions;
|
|
151
|
-
// Upload files
|
|
152
|
-
const uploadPromises = filesToUpload.map((file, index) => {
|
|
153
|
-
const instruction = uploadInstructions[index];
|
|
154
|
-
return fs.readFile(file.filePath).then(buffer => axios_1.default.put(instruction.url, buffer, { headers: { 'Content-Type': file.contentType } }));
|
|
155
|
-
});
|
|
156
|
-
await Promise.all(uploadPromises);
|
|
157
153
|
// Finalize batch
|
|
158
154
|
await axios_1.default.post(`${this.options.apiUrl}/api/v1/ingest/finish-batch/${batchId}`, {}, {
|
|
159
155
|
headers: { 'X-API-KEY': this.options.apiKey },
|
package/dist/snapshot.js
CHANGED
|
@@ -100,7 +100,90 @@ async function snapshot(page, testInfo, name, config) {
|
|
|
100
100
|
};
|
|
101
101
|
}
|
|
102
102
|
}
|
|
103
|
-
// 4.
|
|
103
|
+
// 4. Capture performance metrics (if enabled)
|
|
104
|
+
let performanceTimings;
|
|
105
|
+
let lighthouseResults;
|
|
106
|
+
const captureTimings = effectiveConfig.performance?.captureTimings ?? true; // Default: enabled
|
|
107
|
+
const enableLighthouse = effectiveConfig.performance?.enableLighthouse ?? false; // Default: disabled
|
|
108
|
+
// Capture basic timing metrics
|
|
109
|
+
if (captureTimings) {
|
|
110
|
+
try {
|
|
111
|
+
performanceTimings = await page.evaluate(() => {
|
|
112
|
+
// @ts-ignore - window is available in browser context
|
|
113
|
+
const perfData = window.performance;
|
|
114
|
+
const navigation = perfData.timing;
|
|
115
|
+
const paintEntries = perfData.getEntriesByType('paint');
|
|
116
|
+
const lcpEntries = perfData.getEntriesByType('largest-contentful-paint');
|
|
117
|
+
// Get First Contentful Paint
|
|
118
|
+
const fcpEntry = paintEntries.find((entry) => entry.name === 'first-contentful-paint');
|
|
119
|
+
// Get Largest Contentful Paint
|
|
120
|
+
const lcpEntry = lcpEntries.length > 0 ? lcpEntries[lcpEntries.length - 1] : null;
|
|
121
|
+
// Calculate Cumulative Layout Shift
|
|
122
|
+
let cls = 0;
|
|
123
|
+
const layoutShiftEntries = perfData.getEntriesByType('layout-shift');
|
|
124
|
+
layoutShiftEntries.forEach((entry) => {
|
|
125
|
+
if (!entry.hadRecentInput) {
|
|
126
|
+
cls += entry.value;
|
|
127
|
+
}
|
|
128
|
+
});
|
|
129
|
+
return {
|
|
130
|
+
navigationStart: navigation.navigationStart,
|
|
131
|
+
domContentLoaded: navigation.domContentLoadedEventEnd - navigation.navigationStart,
|
|
132
|
+
loadComplete: navigation.loadEventEnd - navigation.navigationStart,
|
|
133
|
+
firstContentfulPaint: fcpEntry ? fcpEntry.startTime : undefined,
|
|
134
|
+
largestContentfulPaint: lcpEntry ? lcpEntry.startTime : undefined,
|
|
135
|
+
cumulativeLayoutShift: cls,
|
|
136
|
+
};
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
catch (err) {
|
|
140
|
+
console.warn('Failed to capture performance metrics:', err);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
// Capture Lighthouse audit (if enabled)
|
|
144
|
+
if (enableLighthouse) {
|
|
145
|
+
try {
|
|
146
|
+
const { playAudit } = await Promise.resolve().then(() => __importStar(require('playwright-lighthouse')));
|
|
147
|
+
const thresholds = effectiveConfig.performance?.lighthouseThresholds || {};
|
|
148
|
+
const auditResult = await playAudit({
|
|
149
|
+
page,
|
|
150
|
+
port: 9222, // Required by playwright-lighthouse
|
|
151
|
+
thresholds: {
|
|
152
|
+
performance: thresholds.performance || 0,
|
|
153
|
+
accessibility: thresholds.accessibility || 0,
|
|
154
|
+
'best-practices': thresholds.bestPractices || 0,
|
|
155
|
+
seo: thresholds.seo || 0,
|
|
156
|
+
},
|
|
157
|
+
reports: {
|
|
158
|
+
formats: {
|
|
159
|
+
json: false, // Don't save report files
|
|
160
|
+
},
|
|
161
|
+
},
|
|
162
|
+
});
|
|
163
|
+
// Extract key metrics from Lighthouse result
|
|
164
|
+
if (auditResult) {
|
|
165
|
+
const lhr = auditResult.lhr;
|
|
166
|
+
if (lhr) {
|
|
167
|
+
lighthouseResults = {
|
|
168
|
+
performance: lhr.categories?.performance?.score ? Math.round(lhr.categories.performance.score * 100) : undefined,
|
|
169
|
+
accessibility: lhr.categories?.accessibility?.score ? Math.round(lhr.categories.accessibility.score * 100) : undefined,
|
|
170
|
+
bestPractices: lhr.categories?.['best-practices']?.score ? Math.round(lhr.categories['best-practices'].score * 100) : undefined,
|
|
171
|
+
seo: lhr.categories?.seo?.score ? Math.round(lhr.categories.seo.score * 100) : undefined,
|
|
172
|
+
coreWebVitals: {
|
|
173
|
+
lcp: lhr.audits?.['largest-contentful-paint']?.numericValue,
|
|
174
|
+
fid: lhr.audits?.['max-potential-fid']?.numericValue,
|
|
175
|
+
cls: lhr.audits?.['cumulative-layout-shift']?.numericValue,
|
|
176
|
+
},
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
catch (err) {
|
|
182
|
+
console.warn('Failed to run Lighthouse audit:', err);
|
|
183
|
+
console.warn('Make sure playwright-lighthouse is installed: npm install playwright-lighthouse');
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
// 5. Save metadata with configuration and performance data
|
|
104
187
|
const metadataPath = path.join(outputDir, `${baseFilename}.json`);
|
|
105
188
|
const metadata = {
|
|
106
189
|
snapshotName,
|
|
@@ -117,6 +200,10 @@ async function snapshot(page, testInfo, name, config) {
|
|
|
117
200
|
},
|
|
118
201
|
layout,
|
|
119
202
|
// Store the effective configuration for the reporter
|
|
120
|
-
testivaiConfig: effectiveConfig
|
|
203
|
+
testivaiConfig: effectiveConfig,
|
|
204
|
+
// Store performance metrics if captured
|
|
205
|
+
performanceTimings,
|
|
206
|
+
// Store Lighthouse results if captured
|
|
207
|
+
lighthouseResults
|
|
121
208
|
});
|
|
122
209
|
}
|
package/dist/types.d.ts
CHANGED
|
@@ -32,6 +32,62 @@ export interface AIConfig {
|
|
|
32
32
|
/** Include AI reasoning in results (optional) */
|
|
33
33
|
enableReasoning?: boolean;
|
|
34
34
|
}
|
|
35
|
+
/**
|
|
36
|
+
* Performance metrics configuration
|
|
37
|
+
*/
|
|
38
|
+
export interface PerformanceConfig {
|
|
39
|
+
/** Enable basic timing capture (default: true) */
|
|
40
|
+
captureTimings?: boolean;
|
|
41
|
+
/** Enable Lighthouse performance audit (default: false) */
|
|
42
|
+
enableLighthouse?: boolean;
|
|
43
|
+
/** Lighthouse performance thresholds (optional) */
|
|
44
|
+
lighthouseThresholds?: {
|
|
45
|
+
performance?: number;
|
|
46
|
+
accessibility?: number;
|
|
47
|
+
bestPractices?: number;
|
|
48
|
+
seo?: number;
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Performance timing metrics
|
|
53
|
+
*/
|
|
54
|
+
export interface PerformanceTimings {
|
|
55
|
+
/** Navigation start time */
|
|
56
|
+
navigationStart?: number;
|
|
57
|
+
/** DOM content loaded time */
|
|
58
|
+
domContentLoaded?: number;
|
|
59
|
+
/** Page load complete time */
|
|
60
|
+
loadComplete?: number;
|
|
61
|
+
/** First contentful paint */
|
|
62
|
+
firstContentfulPaint?: number;
|
|
63
|
+
/** Largest contentful paint */
|
|
64
|
+
largestContentfulPaint?: number;
|
|
65
|
+
/** Time to interactive */
|
|
66
|
+
timeToInteractive?: number;
|
|
67
|
+
/** Total blocking time */
|
|
68
|
+
totalBlockingTime?: number;
|
|
69
|
+
/** Cumulative layout shift */
|
|
70
|
+
cumulativeLayoutShift?: number;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Lighthouse performance results
|
|
74
|
+
*/
|
|
75
|
+
export interface LighthouseResults {
|
|
76
|
+
/** Performance score (0-100) */
|
|
77
|
+
performance?: number;
|
|
78
|
+
/** Accessibility score (0-100) */
|
|
79
|
+
accessibility?: number;
|
|
80
|
+
/** Best practices score (0-100) */
|
|
81
|
+
bestPractices?: number;
|
|
82
|
+
/** SEO score (0-100) */
|
|
83
|
+
seo?: number;
|
|
84
|
+
/** Core Web Vitals */
|
|
85
|
+
coreWebVitals?: {
|
|
86
|
+
lcp?: number;
|
|
87
|
+
fid?: number;
|
|
88
|
+
cls?: number;
|
|
89
|
+
};
|
|
90
|
+
}
|
|
35
91
|
/**
|
|
36
92
|
* Environment-specific configuration overrides
|
|
37
93
|
*/
|
|
@@ -60,6 +116,8 @@ export interface TestivAIProjectConfig {
|
|
|
60
116
|
layout: LayoutConfig;
|
|
61
117
|
/** AI analysis settings */
|
|
62
118
|
ai: AIConfig;
|
|
119
|
+
/** Performance metrics settings (optional) */
|
|
120
|
+
performance?: PerformanceConfig;
|
|
63
121
|
/** Environment-specific overrides (optional) */
|
|
64
122
|
environments?: EnvironmentConfig;
|
|
65
123
|
}
|
|
@@ -71,6 +129,8 @@ export interface TestivAIConfig {
|
|
|
71
129
|
layout?: Partial<LayoutConfig>;
|
|
72
130
|
/** AI settings (optional - overrides project defaults) */
|
|
73
131
|
ai?: Partial<AIConfig>;
|
|
132
|
+
/** Performance settings (optional - overrides project defaults) */
|
|
133
|
+
performance?: Partial<PerformanceConfig>;
|
|
74
134
|
/** Element selectors to capture (existing option) */
|
|
75
135
|
selectors?: string[];
|
|
76
136
|
}
|
|
@@ -127,6 +187,12 @@ export interface SnapshotPayload {
|
|
|
127
187
|
};
|
|
128
188
|
/** TestivAI configuration for this snapshot */
|
|
129
189
|
testivaiConfig?: TestivAIConfig;
|
|
190
|
+
/** Base64-encoded screenshot data (PNG) */
|
|
191
|
+
screenshotData?: string;
|
|
192
|
+
/** Performance timing metrics (optional) */
|
|
193
|
+
performanceTimings?: PerformanceTimings;
|
|
194
|
+
/** Lighthouse results (optional) */
|
|
195
|
+
lighthouseResults?: LighthouseResults;
|
|
130
196
|
}
|
|
131
197
|
/**
|
|
132
198
|
* Git information for batch context
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@testivai/witness-playwright",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.3",
|
|
4
4
|
"description": "Playwright sensor for Testivai Visual Regression Test system",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -32,6 +32,7 @@
|
|
|
32
32
|
"axios": "^1.6.0",
|
|
33
33
|
"cross-fetch": "^4.0.0",
|
|
34
34
|
"fs-extra": "^11.2.0",
|
|
35
|
+
"playwright-lighthouse": "^4.0.0",
|
|
35
36
|
"simple-git": "^3.21.0"
|
|
36
37
|
},
|
|
37
38
|
"devDependencies": {
|
package/progress.md
CHANGED
|
@@ -661,11 +661,96 @@ The Playwright SDK provides two main components that are production-ready:
|
|
|
661
661
|
|
|
662
662
|
---
|
|
663
663
|
|
|
664
|
-
|
|
664
|
+
## Performance Metrics Integration (January 10, 2026)
|
|
665
|
+
|
|
666
|
+
### Performance Monitoring ✅ COMPLETE
|
|
667
|
+
|
|
668
|
+
**Goal**: Automatically capture page performance metrics and Core Web Vitals with optional Lighthouse integration.
|
|
669
|
+
|
|
670
|
+
#### Changes Implemented
|
|
671
|
+
|
|
672
|
+
1. **Basic Performance Capture** ✅
|
|
673
|
+
- Automatically captures performance timing metrics on every snapshot
|
|
674
|
+
- Enabled by default (`captureTimings: true`)
|
|
675
|
+
- Zero configuration needed for basic metrics
|
|
676
|
+
- Metrics captured:
|
|
677
|
+
- Navigation Start
|
|
678
|
+
- DOM Content Loaded
|
|
679
|
+
- Load Complete
|
|
680
|
+
- First Contentful Paint (FCP)
|
|
681
|
+
- Largest Contentful Paint (LCP)
|
|
682
|
+
- Cumulative Layout Shift (CLS)
|
|
683
|
+
|
|
684
|
+
2. **Lighthouse Integration** ✅
|
|
685
|
+
- Added `playwright-lighthouse@4.0.0` dependency
|
|
686
|
+
- Optional full Lighthouse audit (OFF by default)
|
|
687
|
+
- Captures all 4 Lighthouse categories:
|
|
688
|
+
- Performance Score (0-100)
|
|
689
|
+
- Accessibility Score (0-100)
|
|
690
|
+
- Best Practices Score (0-100)
|
|
691
|
+
- SEO Score (0-100)
|
|
692
|
+
- Extracts Core Web Vitals from Lighthouse
|
|
693
|
+
|
|
694
|
+
3. **Configuration** ✅
|
|
695
|
+
```typescript
|
|
696
|
+
// In testivai.config.ts
|
|
697
|
+
export default {
|
|
698
|
+
performance: {
|
|
699
|
+
captureTimings: true, // Basic timing (default: ON)
|
|
700
|
+
enableLighthouse: false, // Lighthouse audit (default: OFF)
|
|
701
|
+
lighthouseThresholds: {
|
|
702
|
+
performance: 80,
|
|
703
|
+
accessibility: 90,
|
|
704
|
+
bestPractices: 80,
|
|
705
|
+
seo: 80
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
// Per-test override
|
|
711
|
+
await testivai.witness(page, testInfo, 'test-name', {
|
|
712
|
+
performance: {
|
|
713
|
+
enableLighthouse: true
|
|
714
|
+
}
|
|
715
|
+
});
|
|
716
|
+
```
|
|
717
|
+
|
|
718
|
+
4. **Data Storage** ✅
|
|
719
|
+
- Performance data stored in `testivai_config.performanceTimings`
|
|
720
|
+
- Lighthouse results stored in `testivai_config.lighthouseResults`
|
|
721
|
+
- Flows through existing ingestion pipeline
|
|
722
|
+
- No database schema changes needed
|
|
723
|
+
|
|
724
|
+
5. **Dashboard Display** ✅
|
|
725
|
+
- Core Web Vitals displayed with color coding:
|
|
726
|
+
- Green: Good (LCP < 2.5s, FCP < 1.8s, CLS < 0.1)
|
|
727
|
+
- Yellow: Needs Improvement
|
|
728
|
+
- Red: Poor
|
|
729
|
+
- Page Load metrics table
|
|
730
|
+
- Beautiful card-based UI
|
|
731
|
+
|
|
732
|
+
#### Files Modified
|
|
733
|
+
- `src/types.ts` - Added PerformanceConfig, PerformanceTimings, LighthouseResults
|
|
734
|
+
- `src/snapshot.ts` - Implemented performance capture and Lighthouse integration
|
|
735
|
+
- `package.json` - Added playwright-lighthouse dependency
|
|
736
|
+
- `apps/dashboard/src/services/api.ts` - Added testivai_config to TestRun interface
|
|
737
|
+
- `apps/dashboard/src/pages/TestRunDetailPage.tsx` - Performance metrics display
|
|
738
|
+
|
|
739
|
+
#### Benefits
|
|
740
|
+
- **Automatic Tracking**: Every test run captures performance metrics
|
|
741
|
+
- **Core Web Vitals**: Track LCP, FCP, CLS automatically
|
|
742
|
+
- **Optional Lighthouse**: Full audit when needed (OFF by default)
|
|
743
|
+
- **Performance Regression Detection**: Compare metrics across test runs
|
|
744
|
+
- **Zero Configuration**: Works out of the box with sensible defaults
|
|
745
|
+
|
|
746
|
+
---
|
|
747
|
+
|
|
748
|
+
**Last Updated**: January 10, 2026
|
|
665
749
|
**Status**: 🎉 PUBLISHED TO NPM ✅ - Publicly available
|
|
666
|
-
**NPM Package**: @testivai/witness-playwright@0.1.
|
|
667
|
-
**Core Features**: Evidence capture
|
|
750
|
+
**NPM Package**: @testivai/witness-playwright@0.1.2
|
|
751
|
+
**Core Features**: Evidence capture, batch upload, and performance monitoring fully functional
|
|
668
752
|
**Configuration**: ✅ COMPLETE - End-to-end flow working
|
|
753
|
+
**Performance Metrics**: ✅ COMPLETE - Basic timing + optional Lighthouse
|
|
669
754
|
**API Key Format**: tstvai-{secure-random-string}
|
|
670
755
|
**Known Issues**: Minor UX improvements (retry logic, progress reporting)
|
|
671
756
|
**Blocker**: None - All critical features implemented
|
package/src/cli/init.ts
CHANGED
|
@@ -96,9 +96,8 @@ async function createConfigFile(): Promise<void> {
|
|
|
96
96
|
console.log('📁 Config file:', configPath);
|
|
97
97
|
console.log('');
|
|
98
98
|
console.log('📖 Next steps:');
|
|
99
|
-
console.log(' 1. Set up
|
|
99
|
+
console.log(' 1. Set up your API key:');
|
|
100
100
|
console.log(' TESTIVAI_API_KEY=tstvai-your-key-here');
|
|
101
|
-
console.log(' TESTIVAI_API_URL=https://core-api-147980626268.us-central1.run.app');
|
|
102
101
|
console.log('');
|
|
103
102
|
console.log(' 2. Update your playwright.config.ts to use TestivAI reporter:');
|
|
104
103
|
console.log(' reporter: [[\'@testivai/witness-playwright/reporter\']]');
|
|
@@ -107,6 +106,7 @@ async function createConfigFile(): Promise<void> {
|
|
|
107
106
|
console.log(' 4. Run your tests: npx playwright test');
|
|
108
107
|
console.log('');
|
|
109
108
|
console.log('💡 Get your API key from: https://dashboard-147980626268.us-central1.run.app');
|
|
109
|
+
console.log('💡 The SDK automatically connects to the production API - no URL configuration needed!');
|
|
110
110
|
|
|
111
111
|
} catch (error) {
|
|
112
112
|
console.error('❌ Failed to create configuration file:', error);
|
package/src/reporter.ts
CHANGED
|
@@ -20,14 +20,15 @@ export class TestivAIPlaywrightReporter implements Reporter {
|
|
|
20
20
|
|
|
21
21
|
constructor(options: TestivaiReporterOptions = {}) {
|
|
22
22
|
this.options = {
|
|
23
|
-
apiUrl: options.apiUrl || process.env.TESTIVAI_API_URL,
|
|
23
|
+
apiUrl: options.apiUrl || process.env.TESTIVAI_API_URL || 'https://core-api-147980626268.us-central1.run.app',
|
|
24
24
|
apiKey: options.apiKey || process.env.TESTIVAI_API_KEY,
|
|
25
25
|
};
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
async onBegin(config: FullConfig, suite: Suite): Promise<void> {
|
|
29
|
-
if (!this.options.
|
|
30
|
-
console.error('Testivai Reporter: API
|
|
29
|
+
if (!this.options.apiKey) {
|
|
30
|
+
console.error('Testivai Reporter: API Key is not configured. Disabling reporter.');
|
|
31
|
+
console.error('Set TESTIVAI_API_KEY environment variable or pass apiKey in reporter options.');
|
|
31
32
|
this.options.apiUrl = undefined; // Disable reporter
|
|
32
33
|
return;
|
|
33
34
|
}
|
|
@@ -85,7 +86,6 @@ export class TestivAIPlaywrightReporter implements Reporter {
|
|
|
85
86
|
}
|
|
86
87
|
|
|
87
88
|
const snapshots: SnapshotPayload[] = [];
|
|
88
|
-
const filesToUpload: { filePath: string, contentType: string }[] = [];
|
|
89
89
|
|
|
90
90
|
for (const jsonFile of jsonFiles) {
|
|
91
91
|
const metadataPath = path.join(this.tempDir, jsonFile);
|
|
@@ -103,6 +103,10 @@ export class TestivAIPlaywrightReporter implements Reporter {
|
|
|
103
103
|
const firstSelector = layoutKeys[0];
|
|
104
104
|
const layoutData = metadata.layout[firstSelector];
|
|
105
105
|
|
|
106
|
+
// Read screenshot and encode to base64
|
|
107
|
+
const screenshotBuffer = await fs.readFile(screenshotPath);
|
|
108
|
+
const screenshotBase64 = screenshotBuffer.toString('base64');
|
|
109
|
+
|
|
106
110
|
const snapshotPayload: SnapshotPayload = {
|
|
107
111
|
...metadata,
|
|
108
112
|
dom: { html: await fs.readFile(domPath, 'utf-8') },
|
|
@@ -112,11 +116,10 @@ export class TestivAIPlaywrightReporter implements Reporter {
|
|
|
112
116
|
width: layoutData.width,
|
|
113
117
|
height: layoutData.height
|
|
114
118
|
},
|
|
115
|
-
testivaiConfig: metadata.testivaiConfig
|
|
119
|
+
testivaiConfig: metadata.testivaiConfig,
|
|
120
|
+
screenshotData: screenshotBase64 // Include base64-encoded screenshot
|
|
116
121
|
};
|
|
117
122
|
snapshots.push(snapshotPayload);
|
|
118
|
-
|
|
119
|
-
filesToUpload.push({ filePath: screenshotPath, contentType: 'image/png' });
|
|
120
123
|
}
|
|
121
124
|
|
|
122
125
|
const batchPayload: Omit<BatchPayload, 'batchId'> = {
|
|
@@ -127,7 +130,7 @@ export class TestivAIPlaywrightReporter implements Reporter {
|
|
|
127
130
|
runId: this.runId,
|
|
128
131
|
};
|
|
129
132
|
|
|
130
|
-
// Start batch
|
|
133
|
+
// Start batch - screenshots are now included in the payload
|
|
131
134
|
const startBatchResponse = await axios.post(`${this.options.apiUrl}/api/v1/ingest/start-batch`, batchPayload, {
|
|
132
135
|
headers: { 'X-API-KEY': this.options.apiKey },
|
|
133
136
|
});
|
|
@@ -136,17 +139,6 @@ export class TestivAIPlaywrightReporter implements Reporter {
|
|
|
136
139
|
|
|
137
140
|
// Handle both snake_case and camelCase response formats
|
|
138
141
|
const batchId = startBatchResponse.data.batch_id || startBatchResponse.data.batchId;
|
|
139
|
-
const uploadInstructions = startBatchResponse.data.upload_instructions || startBatchResponse.data.uploadInstructions;
|
|
140
|
-
|
|
141
|
-
// Upload files
|
|
142
|
-
const uploadPromises = filesToUpload.map((file, index) => {
|
|
143
|
-
const instruction = uploadInstructions[index];
|
|
144
|
-
return fs.readFile(file.filePath).then(buffer =>
|
|
145
|
-
axios.put(instruction.url, buffer, { headers: { 'Content-Type': file.contentType } })
|
|
146
|
-
);
|
|
147
|
-
});
|
|
148
|
-
|
|
149
|
-
await Promise.all(uploadPromises);
|
|
150
142
|
|
|
151
143
|
// Finalize batch
|
|
152
144
|
await axios.post(`${this.options.apiUrl}/api/v1/ingest/finish-batch/${batchId}`, {}, {
|
package/src/snapshot.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { Page, TestInfo } from '@playwright/test';
|
|
|
2
2
|
import * as fs from 'fs-extra';
|
|
3
3
|
import * as path from 'path';
|
|
4
4
|
import { URL } from 'url';
|
|
5
|
-
import { SnapshotPayload, LayoutData, TestivAIConfig } from './types';
|
|
5
|
+
import { SnapshotPayload, LayoutData, TestivAIConfig, PerformanceTimings, LighthouseResults } from './types';
|
|
6
6
|
import { loadConfig, mergeTestConfig } from './config/loader';
|
|
7
7
|
|
|
8
8
|
/**
|
|
@@ -80,7 +80,98 @@ export async function snapshot(
|
|
|
80
80
|
}
|
|
81
81
|
}
|
|
82
82
|
|
|
83
|
-
// 4.
|
|
83
|
+
// 4. Capture performance metrics (if enabled)
|
|
84
|
+
let performanceTimings: PerformanceTimings | undefined;
|
|
85
|
+
let lighthouseResults: LighthouseResults | undefined;
|
|
86
|
+
|
|
87
|
+
const captureTimings = effectiveConfig.performance?.captureTimings ?? true; // Default: enabled
|
|
88
|
+
const enableLighthouse = effectiveConfig.performance?.enableLighthouse ?? false; // Default: disabled
|
|
89
|
+
|
|
90
|
+
// Capture basic timing metrics
|
|
91
|
+
if (captureTimings) {
|
|
92
|
+
try {
|
|
93
|
+
performanceTimings = await page.evaluate(() => {
|
|
94
|
+
// @ts-ignore - window is available in browser context
|
|
95
|
+
const perfData = window.performance;
|
|
96
|
+
const navigation = perfData.timing;
|
|
97
|
+
const paintEntries = perfData.getEntriesByType('paint');
|
|
98
|
+
const lcpEntries = perfData.getEntriesByType('largest-contentful-paint');
|
|
99
|
+
|
|
100
|
+
// Get First Contentful Paint
|
|
101
|
+
const fcpEntry = paintEntries.find((entry: any) => entry.name === 'first-contentful-paint');
|
|
102
|
+
|
|
103
|
+
// Get Largest Contentful Paint
|
|
104
|
+
const lcpEntry = lcpEntries.length > 0 ? lcpEntries[lcpEntries.length - 1] : null;
|
|
105
|
+
|
|
106
|
+
// Calculate Cumulative Layout Shift
|
|
107
|
+
let cls = 0;
|
|
108
|
+
const layoutShiftEntries = perfData.getEntriesByType('layout-shift') as any[];
|
|
109
|
+
layoutShiftEntries.forEach((entry: any) => {
|
|
110
|
+
if (!entry.hadRecentInput) {
|
|
111
|
+
cls += entry.value;
|
|
112
|
+
}
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
return {
|
|
116
|
+
navigationStart: navigation.navigationStart,
|
|
117
|
+
domContentLoaded: navigation.domContentLoadedEventEnd - navigation.navigationStart,
|
|
118
|
+
loadComplete: navigation.loadEventEnd - navigation.navigationStart,
|
|
119
|
+
firstContentfulPaint: fcpEntry ? fcpEntry.startTime : undefined,
|
|
120
|
+
largestContentfulPaint: lcpEntry ? (lcpEntry as any).startTime : undefined,
|
|
121
|
+
cumulativeLayoutShift: cls,
|
|
122
|
+
};
|
|
123
|
+
});
|
|
124
|
+
} catch (err) {
|
|
125
|
+
console.warn('Failed to capture performance metrics:', err);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// Capture Lighthouse audit (if enabled)
|
|
130
|
+
if (enableLighthouse) {
|
|
131
|
+
try {
|
|
132
|
+
const { playAudit } = await import('playwright-lighthouse');
|
|
133
|
+
const thresholds = effectiveConfig.performance?.lighthouseThresholds || {};
|
|
134
|
+
|
|
135
|
+
const auditResult = await playAudit({
|
|
136
|
+
page,
|
|
137
|
+
port: 9222, // Required by playwright-lighthouse
|
|
138
|
+
thresholds: {
|
|
139
|
+
performance: thresholds.performance || 0,
|
|
140
|
+
accessibility: thresholds.accessibility || 0,
|
|
141
|
+
'best-practices': thresholds.bestPractices || 0,
|
|
142
|
+
seo: thresholds.seo || 0,
|
|
143
|
+
},
|
|
144
|
+
reports: {
|
|
145
|
+
formats: {
|
|
146
|
+
json: false, // Don't save report files
|
|
147
|
+
},
|
|
148
|
+
},
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
// Extract key metrics from Lighthouse result
|
|
152
|
+
if (auditResult) {
|
|
153
|
+
const lhr = (auditResult as any).lhr;
|
|
154
|
+
if (lhr) {
|
|
155
|
+
lighthouseResults = {
|
|
156
|
+
performance: lhr.categories?.performance?.score ? Math.round(lhr.categories.performance.score * 100) : undefined,
|
|
157
|
+
accessibility: lhr.categories?.accessibility?.score ? Math.round(lhr.categories.accessibility.score * 100) : undefined,
|
|
158
|
+
bestPractices: lhr.categories?.['best-practices']?.score ? Math.round(lhr.categories['best-practices'].score * 100) : undefined,
|
|
159
|
+
seo: lhr.categories?.seo?.score ? Math.round(lhr.categories.seo.score * 100) : undefined,
|
|
160
|
+
coreWebVitals: {
|
|
161
|
+
lcp: lhr.audits?.['largest-contentful-paint']?.numericValue,
|
|
162
|
+
fid: lhr.audits?.['max-potential-fid']?.numericValue,
|
|
163
|
+
cls: lhr.audits?.['cumulative-layout-shift']?.numericValue,
|
|
164
|
+
},
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
} catch (err) {
|
|
169
|
+
console.warn('Failed to run Lighthouse audit:', err);
|
|
170
|
+
console.warn('Make sure playwright-lighthouse is installed: npm install playwright-lighthouse');
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// 5. Save metadata with configuration and performance data
|
|
84
175
|
const metadataPath = path.join(outputDir, `${baseFilename}.json`);
|
|
85
176
|
const metadata: Partial<SnapshotPayload> = {
|
|
86
177
|
snapshotName,
|
|
@@ -98,6 +189,10 @@ export async function snapshot(
|
|
|
98
189
|
},
|
|
99
190
|
layout,
|
|
100
191
|
// Store the effective configuration for the reporter
|
|
101
|
-
testivaiConfig: effectiveConfig
|
|
192
|
+
testivaiConfig: effectiveConfig,
|
|
193
|
+
// Store performance metrics if captured
|
|
194
|
+
performanceTimings,
|
|
195
|
+
// Store Lighthouse results if captured
|
|
196
|
+
lighthouseResults
|
|
102
197
|
});
|
|
103
198
|
}
|
package/src/types.ts
CHANGED
|
@@ -35,6 +35,65 @@ export interface AIConfig {
|
|
|
35
35
|
enableReasoning?: boolean;
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
/**
|
|
39
|
+
* Performance metrics configuration
|
|
40
|
+
*/
|
|
41
|
+
export interface PerformanceConfig {
|
|
42
|
+
/** Enable basic timing capture (default: true) */
|
|
43
|
+
captureTimings?: boolean;
|
|
44
|
+
/** Enable Lighthouse performance audit (default: false) */
|
|
45
|
+
enableLighthouse?: boolean;
|
|
46
|
+
/** Lighthouse performance thresholds (optional) */
|
|
47
|
+
lighthouseThresholds?: {
|
|
48
|
+
performance?: number;
|
|
49
|
+
accessibility?: number;
|
|
50
|
+
bestPractices?: number;
|
|
51
|
+
seo?: number;
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Performance timing metrics
|
|
57
|
+
*/
|
|
58
|
+
export interface PerformanceTimings {
|
|
59
|
+
/** Navigation start time */
|
|
60
|
+
navigationStart?: number;
|
|
61
|
+
/** DOM content loaded time */
|
|
62
|
+
domContentLoaded?: number;
|
|
63
|
+
/** Page load complete time */
|
|
64
|
+
loadComplete?: number;
|
|
65
|
+
/** First contentful paint */
|
|
66
|
+
firstContentfulPaint?: number;
|
|
67
|
+
/** Largest contentful paint */
|
|
68
|
+
largestContentfulPaint?: number;
|
|
69
|
+
/** Time to interactive */
|
|
70
|
+
timeToInteractive?: number;
|
|
71
|
+
/** Total blocking time */
|
|
72
|
+
totalBlockingTime?: number;
|
|
73
|
+
/** Cumulative layout shift */
|
|
74
|
+
cumulativeLayoutShift?: number;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Lighthouse performance results
|
|
79
|
+
*/
|
|
80
|
+
export interface LighthouseResults {
|
|
81
|
+
/** Performance score (0-100) */
|
|
82
|
+
performance?: number;
|
|
83
|
+
/** Accessibility score (0-100) */
|
|
84
|
+
accessibility?: number;
|
|
85
|
+
/** Best practices score (0-100) */
|
|
86
|
+
bestPractices?: number;
|
|
87
|
+
/** SEO score (0-100) */
|
|
88
|
+
seo?: number;
|
|
89
|
+
/** Core Web Vitals */
|
|
90
|
+
coreWebVitals?: {
|
|
91
|
+
lcp?: number;
|
|
92
|
+
fid?: number;
|
|
93
|
+
cls?: number;
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
|
|
38
97
|
/**
|
|
39
98
|
* Environment-specific configuration overrides
|
|
40
99
|
*/
|
|
@@ -64,6 +123,8 @@ export interface TestivAIProjectConfig {
|
|
|
64
123
|
layout: LayoutConfig;
|
|
65
124
|
/** AI analysis settings */
|
|
66
125
|
ai: AIConfig;
|
|
126
|
+
/** Performance metrics settings (optional) */
|
|
127
|
+
performance?: PerformanceConfig;
|
|
67
128
|
/** Environment-specific overrides (optional) */
|
|
68
129
|
environments?: EnvironmentConfig;
|
|
69
130
|
}
|
|
@@ -76,6 +137,8 @@ export interface TestivAIConfig {
|
|
|
76
137
|
layout?: Partial<LayoutConfig>;
|
|
77
138
|
/** AI settings (optional - overrides project defaults) */
|
|
78
139
|
ai?: Partial<AIConfig>;
|
|
140
|
+
/** Performance settings (optional - overrides project defaults) */
|
|
141
|
+
performance?: Partial<PerformanceConfig>;
|
|
79
142
|
/** Element selectors to capture (existing option) */
|
|
80
143
|
selectors?: string[];
|
|
81
144
|
}
|
|
@@ -135,6 +198,12 @@ export interface SnapshotPayload {
|
|
|
135
198
|
};
|
|
136
199
|
/** TestivAI configuration for this snapshot */
|
|
137
200
|
testivaiConfig?: TestivAIConfig;
|
|
201
|
+
/** Base64-encoded screenshot data (PNG) */
|
|
202
|
+
screenshotData?: string;
|
|
203
|
+
/** Performance timing metrics (optional) */
|
|
204
|
+
performanceTimings?: PerformanceTimings;
|
|
205
|
+
/** Lighthouse results (optional) */
|
|
206
|
+
lighthouseResults?: LighthouseResults;
|
|
138
207
|
}
|
|
139
208
|
|
|
140
209
|
/**
|