cbrowser 6.0.0 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +150 -0
- package/dist/browser.d.ts +93 -1
- package/dist/browser.d.ts.map +1 -1
- package/dist/browser.js +941 -0
- package/dist/browser.js.map +1 -1
- package/dist/cli.js +337 -2
- package/dist/cli.js.map +1 -1
- package/dist/types.d.ts +127 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
package/dist/browser.js
CHANGED
|
@@ -16,6 +16,15 @@ exports.applyChaos = applyChaos;
|
|
|
16
16
|
exports.runChaosTest = runChaosTest;
|
|
17
17
|
exports.comparePersonas = comparePersonas;
|
|
18
18
|
exports.formatComparisonReport = formatComparisonReport;
|
|
19
|
+
exports.parseNLInstruction = parseNLInstruction;
|
|
20
|
+
exports.parseNLTestSuite = parseNLTestSuite;
|
|
21
|
+
exports.runNLTestSuite = runNLTestSuite;
|
|
22
|
+
exports.formatNLTestReport = formatNLTestReport;
|
|
23
|
+
exports.runNLTestFile = runNLTestFile;
|
|
24
|
+
exports.repairTest = repairTest;
|
|
25
|
+
exports.repairTestSuite = repairTestSuite;
|
|
26
|
+
exports.formatRepairReport = formatRepairReport;
|
|
27
|
+
exports.exportRepairedTest = exportRepairedTest;
|
|
19
28
|
const playwright_1 = require("playwright");
|
|
20
29
|
const fs_1 = require("fs");
|
|
21
30
|
const path_1 = require("path");
|
|
@@ -2877,4 +2886,936 @@ function formatComparisonReport(comparison) {
|
|
|
2877
2886
|
lines.push("");
|
|
2878
2887
|
return lines.join("\n");
|
|
2879
2888
|
}
|
|
2889
|
+
// =========================================================================
|
|
2890
|
+
// Natural Language Test Suites
|
|
2891
|
+
// =========================================================================
|
|
2892
|
+
/**
|
|
2893
|
+
* Parse a single natural language instruction into an NLTestStep.
|
|
2894
|
+
*
|
|
2895
|
+
* Supported patterns:
|
|
2896
|
+
* - "go to https://..." / "navigate to https://..." / "open https://..."
|
|
2897
|
+
* - "click [the] <target>" / "press <target>"
|
|
2898
|
+
* - "type '<value>' in[to] <target>" / "fill <target> with '<value>'"
|
|
2899
|
+
* - "select '<option>' from <dropdown>"
|
|
2900
|
+
* - "scroll down/up"
|
|
2901
|
+
* - "wait [for] <seconds> seconds"
|
|
2902
|
+
* - "verify <assertion>" / "assert <assertion>" / "check <assertion>"
|
|
2903
|
+
* - "take screenshot"
|
|
2904
|
+
*/
|
|
2905
|
+
function parseNLInstruction(instruction) {
|
|
2906
|
+
const lower = instruction.toLowerCase().trim();
|
|
2907
|
+
// Navigate patterns
|
|
2908
|
+
const navigateMatch = lower.match(/^(?:go to|navigate to|open|visit)\s+(.+)$/i);
|
|
2909
|
+
if (navigateMatch) {
|
|
2910
|
+
return {
|
|
2911
|
+
instruction,
|
|
2912
|
+
action: "navigate",
|
|
2913
|
+
target: navigateMatch[1].trim(),
|
|
2914
|
+
};
|
|
2915
|
+
}
|
|
2916
|
+
// Click patterns
|
|
2917
|
+
const clickMatch = lower.match(/^(?:click|tap|press)\s+(?:on\s+)?(?:the\s+)?(.+)$/i);
|
|
2918
|
+
if (clickMatch) {
|
|
2919
|
+
return {
|
|
2920
|
+
instruction,
|
|
2921
|
+
action: "click",
|
|
2922
|
+
target: clickMatch[1].trim(),
|
|
2923
|
+
};
|
|
2924
|
+
}
|
|
2925
|
+
// Fill patterns: "type 'value' in target" or "fill target with 'value'"
|
|
2926
|
+
const typeMatch = lower.match(/^(?:type|enter)\s+['"](.+?)['"]\s+(?:in|into)\s+(?:the\s+)?(.+)$/i);
|
|
2927
|
+
if (typeMatch) {
|
|
2928
|
+
return {
|
|
2929
|
+
instruction,
|
|
2930
|
+
action: "fill",
|
|
2931
|
+
value: typeMatch[1],
|
|
2932
|
+
target: typeMatch[2].trim(),
|
|
2933
|
+
};
|
|
2934
|
+
}
|
|
2935
|
+
const fillMatch = lower.match(/^fill\s+(?:the\s+)?(.+?)\s+with\s+['"](.+?)['"]$/i);
|
|
2936
|
+
if (fillMatch) {
|
|
2937
|
+
return {
|
|
2938
|
+
instruction,
|
|
2939
|
+
action: "fill",
|
|
2940
|
+
target: fillMatch[1].trim(),
|
|
2941
|
+
value: fillMatch[2],
|
|
2942
|
+
};
|
|
2943
|
+
}
|
|
2944
|
+
// Select patterns
|
|
2945
|
+
const selectMatch = lower.match(/^select\s+['"](.+?)['"]\s+(?:from|in)\s+(?:the\s+)?(.+)$/i);
|
|
2946
|
+
if (selectMatch) {
|
|
2947
|
+
return {
|
|
2948
|
+
instruction,
|
|
2949
|
+
action: "select",
|
|
2950
|
+
value: selectMatch[1],
|
|
2951
|
+
target: selectMatch[2].trim(),
|
|
2952
|
+
};
|
|
2953
|
+
}
|
|
2954
|
+
// Scroll patterns
|
|
2955
|
+
const scrollMatch = lower.match(/^scroll\s+(up|down|left|right)(?:\s+(\d+)\s+(?:times|pixels))?$/i);
|
|
2956
|
+
if (scrollMatch) {
|
|
2957
|
+
return {
|
|
2958
|
+
instruction,
|
|
2959
|
+
action: "scroll",
|
|
2960
|
+
target: scrollMatch[1],
|
|
2961
|
+
value: scrollMatch[2] || "3",
|
|
2962
|
+
};
|
|
2963
|
+
}
|
|
2964
|
+
// Wait patterns
|
|
2965
|
+
const waitMatch = lower.match(/^wait\s+(?:for\s+)?(\d+(?:\.\d+)?)\s*(?:seconds?|s)$/i);
|
|
2966
|
+
if (waitMatch) {
|
|
2967
|
+
return {
|
|
2968
|
+
instruction,
|
|
2969
|
+
action: "wait",
|
|
2970
|
+
value: waitMatch[1],
|
|
2971
|
+
};
|
|
2972
|
+
}
|
|
2973
|
+
// Wait for text pattern
|
|
2974
|
+
const waitForMatch = lower.match(/^wait\s+(?:for|until)\s+['"](.+?)['"]\s+(?:appears?|is visible|shows?)$/i);
|
|
2975
|
+
if (waitForMatch) {
|
|
2976
|
+
return {
|
|
2977
|
+
instruction,
|
|
2978
|
+
action: "wait",
|
|
2979
|
+
target: waitForMatch[1],
|
|
2980
|
+
};
|
|
2981
|
+
}
|
|
2982
|
+
// Assert/verify patterns
|
|
2983
|
+
const assertPatterns = [
|
|
2984
|
+
// Title assertions
|
|
2985
|
+
{ pattern: /^(?:verify|assert|check)\s+(?:that\s+)?(?:page\s+)?title\s+(?:contains?|has)\s+['"](.+?)['"]$/i, type: "title", assertType: "contains" },
|
|
2986
|
+
{ pattern: /^(?:verify|assert|check)\s+(?:that\s+)?(?:page\s+)?title\s+(?:is|equals?)\s+['"](.+?)['"]$/i, type: "title", assertType: "equals" },
|
|
2987
|
+
// URL assertions
|
|
2988
|
+
{ pattern: /^(?:verify|assert|check)\s+(?:that\s+)?url\s+(?:contains?|has)\s+['"](.+?)['"]$/i, type: "url", assertType: "contains" },
|
|
2989
|
+
{ pattern: /^(?:verify|assert|check)\s+(?:that\s+)?url\s+(?:is|equals?)\s+['"](.+?)['"]$/i, type: "url", assertType: "equals" },
|
|
2990
|
+
// Content assertions
|
|
2991
|
+
{ pattern: /^(?:verify|assert|check)\s+(?:that\s+)?(?:page\s+)?(?:contains?|has|shows?)\s+['"](.+?)['"]$/i, type: "content", assertType: "contains" },
|
|
2992
|
+
{ pattern: /^(?:verify|assert|check)\s+(?:that\s+)?['"](.+?)['"]\s+(?:is\s+)?(?:visible|displayed|shown)$/i, type: "content", assertType: "contains" },
|
|
2993
|
+
// Element exists
|
|
2994
|
+
{ pattern: /^(?:verify|assert|check)\s+(?:that\s+)?['"](.+?)['"]\s+exists?$/i, type: "element", assertType: "exists" },
|
|
2995
|
+
{ pattern: /^(?:verify|assert|check)\s+(?:that\s+)?(?:there\s+is\s+)?(?:a|an)\s+['"](.+?)['"]$/i, type: "element", assertType: "exists" },
|
|
2996
|
+
// Count assertions
|
|
2997
|
+
{ pattern: /^(?:verify|assert|check)\s+(?:that\s+)?(?:there\s+are\s+)?(\d+)\s+(.+?)$/i, type: "count", assertType: "count" },
|
|
2998
|
+
];
|
|
2999
|
+
for (const { pattern, type, assertType } of assertPatterns) {
|
|
3000
|
+
const match = lower.match(pattern);
|
|
3001
|
+
if (match) {
|
|
3002
|
+
return {
|
|
3003
|
+
instruction,
|
|
3004
|
+
action: "assert",
|
|
3005
|
+
target: type === "count" ? match[2] : match[1],
|
|
3006
|
+
value: type === "count" ? match[1] : undefined,
|
|
3007
|
+
assertionType: assertType,
|
|
3008
|
+
};
|
|
3009
|
+
}
|
|
3010
|
+
}
|
|
3011
|
+
// Screenshot pattern
|
|
3012
|
+
if (/^(?:take\s+(?:a\s+)?screenshot|screenshot|capture\s+(?:the\s+)?(?:page|screen))$/i.test(lower)) {
|
|
3013
|
+
return {
|
|
3014
|
+
instruction,
|
|
3015
|
+
action: "screenshot",
|
|
3016
|
+
};
|
|
3017
|
+
}
|
|
3018
|
+
// Unknown - return as-is for AI-powered interpretation later
|
|
3019
|
+
return {
|
|
3020
|
+
instruction,
|
|
3021
|
+
action: "unknown",
|
|
3022
|
+
target: instruction,
|
|
3023
|
+
};
|
|
3024
|
+
}
|
|
3025
|
+
/**
|
|
3026
|
+
* Parse a natural language test suite from text.
|
|
3027
|
+
*
|
|
3028
|
+
* Format:
|
|
3029
|
+
* ```
|
|
3030
|
+
* # Test: Login Flow
|
|
3031
|
+
* go to https://example.com
|
|
3032
|
+
* click the login button
|
|
3033
|
+
* type "user@example.com" in email field
|
|
3034
|
+
* type "password123" in password field
|
|
3035
|
+
* click submit
|
|
3036
|
+
* verify url contains "/dashboard"
|
|
3037
|
+
*
|
|
3038
|
+
* # Test: Search Functionality
|
|
3039
|
+
* go to https://example.com
|
|
3040
|
+
* type "test query" in search box
|
|
3041
|
+
* click search button
|
|
3042
|
+
* verify page contains "results"
|
|
3043
|
+
* ```
|
|
3044
|
+
*/
|
|
3045
|
+
function parseNLTestSuite(text, suiteName = "Unnamed Suite") {
|
|
3046
|
+
const lines = text.split("\n").map(l => l.trim()).filter(l => l && !l.startsWith("//"));
|
|
3047
|
+
const tests = [];
|
|
3048
|
+
let currentTest = null;
|
|
3049
|
+
for (const line of lines) {
|
|
3050
|
+
// Check for test header: "# Test: Name" or "## Name" or "Test: Name"
|
|
3051
|
+
const testHeaderMatch = line.match(/^(?:#\s*)?(?:test:\s*)?(.+)$/i);
|
|
3052
|
+
if (line.startsWith("#") || line.toLowerCase().startsWith("test:")) {
|
|
3053
|
+
// Save previous test if exists
|
|
3054
|
+
if (currentTest && currentTest.steps.length > 0) {
|
|
3055
|
+
tests.push(currentTest);
|
|
3056
|
+
}
|
|
3057
|
+
const name = testHeaderMatch?.[1]?.replace(/^#+\s*/, "").replace(/^test:\s*/i, "").trim() || "Unnamed Test";
|
|
3058
|
+
currentTest = {
|
|
3059
|
+
name,
|
|
3060
|
+
steps: [],
|
|
3061
|
+
};
|
|
3062
|
+
}
|
|
3063
|
+
else if (line.length > 0) {
|
|
3064
|
+
// Parse as instruction
|
|
3065
|
+
if (!currentTest) {
|
|
3066
|
+
// Create default test if no header found
|
|
3067
|
+
currentTest = {
|
|
3068
|
+
name: "Default Test",
|
|
3069
|
+
steps: [],
|
|
3070
|
+
};
|
|
3071
|
+
}
|
|
3072
|
+
const step = parseNLInstruction(line);
|
|
3073
|
+
currentTest.steps.push(step);
|
|
3074
|
+
}
|
|
3075
|
+
}
|
|
3076
|
+
// Save final test
|
|
3077
|
+
if (currentTest && currentTest.steps.length > 0) {
|
|
3078
|
+
tests.push(currentTest);
|
|
3079
|
+
}
|
|
3080
|
+
return { name: suiteName, tests };
|
|
3081
|
+
}
|
|
3082
|
+
/**
|
|
3083
|
+
* Run a natural language test suite.
|
|
3084
|
+
*/
|
|
3085
|
+
async function runNLTestSuite(suite, options = {}) {
|
|
3086
|
+
const { stepTimeout = 30000, continueOnFailure = true, screenshotOnFailure = true, headless = true, } = options;
|
|
3087
|
+
const startTime = Date.now();
|
|
3088
|
+
const testResults = [];
|
|
3089
|
+
console.log(`\n🧪 Running Test Suite: ${suite.name}`);
|
|
3090
|
+
console.log(` Tests: ${suite.tests.length}`);
|
|
3091
|
+
console.log(` Continue on failure: ${continueOnFailure}`);
|
|
3092
|
+
console.log("");
|
|
3093
|
+
const browser = new CBrowser({
|
|
3094
|
+
headless,
|
|
3095
|
+
});
|
|
3096
|
+
try {
|
|
3097
|
+
await browser.launch();
|
|
3098
|
+
for (const test of suite.tests) {
|
|
3099
|
+
console.log(`\n📋 Test: ${test.name}`);
|
|
3100
|
+
const testStartTime = Date.now();
|
|
3101
|
+
const stepResults = [];
|
|
3102
|
+
let testPassed = true;
|
|
3103
|
+
let testError;
|
|
3104
|
+
for (const step of test.steps) {
|
|
3105
|
+
console.log(` → ${step.instruction}`);
|
|
3106
|
+
const stepStartTime = Date.now();
|
|
3107
|
+
let stepPassed = true;
|
|
3108
|
+
let stepError;
|
|
3109
|
+
let screenshot;
|
|
3110
|
+
let actualValue;
|
|
3111
|
+
try {
|
|
3112
|
+
// Execute the step based on action type
|
|
3113
|
+
switch (step.action) {
|
|
3114
|
+
case "navigate": {
|
|
3115
|
+
await browser.navigate(step.target || "");
|
|
3116
|
+
break;
|
|
3117
|
+
}
|
|
3118
|
+
case "click": {
|
|
3119
|
+
const result = await browser.smartClick(step.target || "");
|
|
3120
|
+
if (!result.success) {
|
|
3121
|
+
throw new Error(`Failed to click: ${step.target}`);
|
|
3122
|
+
}
|
|
3123
|
+
break;
|
|
3124
|
+
}
|
|
3125
|
+
case "fill": {
|
|
3126
|
+
await browser.fill(step.target || "", step.value || "");
|
|
3127
|
+
break;
|
|
3128
|
+
}
|
|
3129
|
+
case "select": {
|
|
3130
|
+
await browser.fill(step.target || "", step.value || "");
|
|
3131
|
+
break;
|
|
3132
|
+
}
|
|
3133
|
+
case "scroll": {
|
|
3134
|
+
const direction = step.target?.toLowerCase() === "up" ? -500 : 500;
|
|
3135
|
+
// Use private page access through cast
|
|
3136
|
+
const page = browser.page;
|
|
3137
|
+
if (page) {
|
|
3138
|
+
await page.evaluate((d) => window.scrollBy(0, d), direction);
|
|
3139
|
+
}
|
|
3140
|
+
break;
|
|
3141
|
+
}
|
|
3142
|
+
case "wait": {
|
|
3143
|
+
if (step.target) {
|
|
3144
|
+
// Wait for text to appear - use private page access
|
|
3145
|
+
const page = browser.page;
|
|
3146
|
+
if (page) {
|
|
3147
|
+
await page.waitForSelector(`text=${step.target}`, { timeout: stepTimeout });
|
|
3148
|
+
}
|
|
3149
|
+
}
|
|
3150
|
+
else {
|
|
3151
|
+
// Wait for duration
|
|
3152
|
+
const ms = parseFloat(step.value || "1") * 1000;
|
|
3153
|
+
await new Promise(r => setTimeout(r, ms));
|
|
3154
|
+
}
|
|
3155
|
+
break;
|
|
3156
|
+
}
|
|
3157
|
+
case "assert": {
|
|
3158
|
+
const assertResult = await browser.assert(step.instruction);
|
|
3159
|
+
stepPassed = assertResult.passed;
|
|
3160
|
+
actualValue = String(assertResult.actual);
|
|
3161
|
+
if (!assertResult.passed) {
|
|
3162
|
+
stepError = assertResult.message;
|
|
3163
|
+
}
|
|
3164
|
+
break;
|
|
3165
|
+
}
|
|
3166
|
+
case "screenshot": {
|
|
3167
|
+
screenshot = await browser.screenshot();
|
|
3168
|
+
break;
|
|
3169
|
+
}
|
|
3170
|
+
case "unknown": {
|
|
3171
|
+
// Try to interpret as a click or fill
|
|
3172
|
+
console.log(` ⚠️ Unknown instruction, attempting smart interpretation...`);
|
|
3173
|
+
const result = await browser.smartClick(step.target || step.instruction);
|
|
3174
|
+
if (!result.success) {
|
|
3175
|
+
throw new Error(`Could not interpret: ${step.instruction}`);
|
|
3176
|
+
}
|
|
3177
|
+
break;
|
|
3178
|
+
}
|
|
3179
|
+
}
|
|
3180
|
+
console.log(` ✓ Passed (${Date.now() - stepStartTime}ms)`);
|
|
3181
|
+
}
|
|
3182
|
+
catch (e) {
|
|
3183
|
+
stepPassed = false;
|
|
3184
|
+
stepError = e.message;
|
|
3185
|
+
testPassed = false;
|
|
3186
|
+
testError = testError || e.message;
|
|
3187
|
+
console.log(` ✗ Failed: ${e.message}`);
|
|
3188
|
+
if (screenshotOnFailure) {
|
|
3189
|
+
try {
|
|
3190
|
+
screenshot = await browser.screenshot();
|
|
3191
|
+
}
|
|
3192
|
+
catch { }
|
|
3193
|
+
}
|
|
3194
|
+
}
|
|
3195
|
+
stepResults.push({
|
|
3196
|
+
instruction: step.instruction,
|
|
3197
|
+
action: step.action,
|
|
3198
|
+
passed: stepPassed,
|
|
3199
|
+
duration: Date.now() - stepStartTime,
|
|
3200
|
+
error: stepError,
|
|
3201
|
+
screenshot,
|
|
3202
|
+
actualValue,
|
|
3203
|
+
});
|
|
3204
|
+
// Stop test if step failed and not continuing on failure
|
|
3205
|
+
if (!stepPassed && !continueOnFailure) {
|
|
3206
|
+
break;
|
|
3207
|
+
}
|
|
3208
|
+
}
|
|
3209
|
+
testResults.push({
|
|
3210
|
+
name: test.name,
|
|
3211
|
+
passed: testPassed,
|
|
3212
|
+
duration: Date.now() - testStartTime,
|
|
3213
|
+
stepResults,
|
|
3214
|
+
error: testError,
|
|
3215
|
+
});
|
|
3216
|
+
console.log(` ${testPassed ? "✅" : "❌"} ${test.name}: ${testPassed ? "PASSED" : "FAILED"}`);
|
|
3217
|
+
}
|
|
3218
|
+
}
|
|
3219
|
+
finally {
|
|
3220
|
+
await browser.close();
|
|
3221
|
+
}
|
|
3222
|
+
const passed = testResults.filter(t => t.passed).length;
|
|
3223
|
+
const failed = testResults.filter(t => !t.passed).length;
|
|
3224
|
+
const result = {
|
|
3225
|
+
name: suite.name,
|
|
3226
|
+
timestamp: new Date().toISOString(),
|
|
3227
|
+
duration: Date.now() - startTime,
|
|
3228
|
+
testResults,
|
|
3229
|
+
summary: {
|
|
3230
|
+
total: suite.tests.length,
|
|
3231
|
+
passed,
|
|
3232
|
+
failed,
|
|
3233
|
+
skipped: 0,
|
|
3234
|
+
passRate: suite.tests.length > 0 ? (passed / suite.tests.length) * 100 : 0,
|
|
3235
|
+
},
|
|
3236
|
+
};
|
|
3237
|
+
return result;
|
|
3238
|
+
}
|
|
3239
|
+
/**
|
|
3240
|
+
* Format a test suite result as a report.
|
|
3241
|
+
*/
|
|
3242
|
+
function formatNLTestReport(result) {
|
|
3243
|
+
const lines = [];
|
|
3244
|
+
lines.push("");
|
|
3245
|
+
lines.push("╔══════════════════════════════════════════════════════════════════════════════╗");
|
|
3246
|
+
lines.push("║ NATURAL LANGUAGE TEST REPORT ║");
|
|
3247
|
+
lines.push("╚══════════════════════════════════════════════════════════════════════════════╝");
|
|
3248
|
+
lines.push("");
|
|
3249
|
+
lines.push(`📋 Suite: ${result.name}`);
|
|
3250
|
+
lines.push(`⏱️ Duration: ${(result.duration / 1000).toFixed(1)}s`);
|
|
3251
|
+
lines.push(`📅 Timestamp: ${result.timestamp}`);
|
|
3252
|
+
lines.push("");
|
|
3253
|
+
// Summary stats
|
|
3254
|
+
const passEmoji = result.summary.passRate === 100 ? "🎉" : result.summary.passRate >= 80 ? "✅" : "⚠️";
|
|
3255
|
+
lines.push(`${passEmoji} Pass Rate: ${result.summary.passed}/${result.summary.total} (${result.summary.passRate.toFixed(0)}%)`);
|
|
3256
|
+
lines.push("");
|
|
3257
|
+
// Results table
|
|
3258
|
+
lines.push("┌───────────────────────────────────────┬──────────┬──────────┬────────────────────┐");
|
|
3259
|
+
lines.push("│ Test │ Status │ Duration │ Error │");
|
|
3260
|
+
lines.push("├───────────────────────────────────────┼──────────┼──────────┼────────────────────┤");
|
|
3261
|
+
for (const test of result.testResults) {
|
|
3262
|
+
const name = test.name.padEnd(37).slice(0, 37);
|
|
3263
|
+
const status = test.passed ? "✓ PASS".padEnd(8) : "✗ FAIL".padEnd(8);
|
|
3264
|
+
const duration = `${(test.duration / 1000).toFixed(1)}s`.padEnd(8);
|
|
3265
|
+
const error = (test.error || "-").slice(0, 18).padEnd(18);
|
|
3266
|
+
lines.push(`│ ${name} │ ${status} │ ${duration} │ ${error} │`);
|
|
3267
|
+
}
|
|
3268
|
+
lines.push("└───────────────────────────────────────┴──────────┴──────────┴────────────────────┘");
|
|
3269
|
+
lines.push("");
|
|
3270
|
+
// Failed test details
|
|
3271
|
+
const failedTests = result.testResults.filter(t => !t.passed);
|
|
3272
|
+
if (failedTests.length > 0) {
|
|
3273
|
+
lines.push("❌ FAILED TESTS");
|
|
3274
|
+
lines.push("─".repeat(60));
|
|
3275
|
+
for (const test of failedTests) {
|
|
3276
|
+
lines.push(`\n 📋 ${test.name}`);
|
|
3277
|
+
const failedSteps = test.stepResults.filter(s => !s.passed);
|
|
3278
|
+
for (const step of failedSteps) {
|
|
3279
|
+
lines.push(` ✗ ${step.instruction}`);
|
|
3280
|
+
if (step.error) {
|
|
3281
|
+
lines.push(` Error: ${step.error}`);
|
|
3282
|
+
}
|
|
3283
|
+
if (step.screenshot) {
|
|
3284
|
+
lines.push(` Screenshot: ${step.screenshot}`);
|
|
3285
|
+
}
|
|
3286
|
+
}
|
|
3287
|
+
}
|
|
3288
|
+
lines.push("");
|
|
3289
|
+
}
|
|
3290
|
+
return lines.join("\n");
|
|
3291
|
+
}
|
|
3292
|
+
/**
|
|
3293
|
+
* Run a natural language test suite from a file.
|
|
3294
|
+
*/
|
|
3295
|
+
async function runNLTestFile(filepath, options = {}) {
|
|
3296
|
+
if (!(0, fs_1.existsSync)(filepath)) {
|
|
3297
|
+
throw new Error(`Test file not found: ${filepath}`);
|
|
3298
|
+
}
|
|
3299
|
+
const content = (0, fs_1.readFileSync)(filepath, "utf-8");
|
|
3300
|
+
const suiteName = filepath.split("/").pop()?.replace(/\.[^.]+$/, "") || "Test Suite";
|
|
3301
|
+
const suite = parseNLTestSuite(content, suiteName);
|
|
3302
|
+
return runNLTestSuite(suite, options);
|
|
3303
|
+
}
|
|
3304
|
+
// =========================================================================
|
|
3305
|
+
// AI Test Repair (v6.2.0)
|
|
3306
|
+
// =========================================================================
|
|
3307
|
+
/**
|
|
3308
|
+
* Classify the type of failure from an error message.
|
|
3309
|
+
*/
|
|
3310
|
+
function classifyFailure(error, step) {
|
|
3311
|
+
const lowerError = error.toLowerCase();
|
|
3312
|
+
if (lowerError.includes("not found") || lowerError.includes("no element") || lowerError.includes("failed to click")) {
|
|
3313
|
+
return "selector_not_found";
|
|
3314
|
+
}
|
|
3315
|
+
if (lowerError.includes("assertion") || lowerError.includes("verify") || lowerError.includes("expected")) {
|
|
3316
|
+
return "assertion_failed";
|
|
3317
|
+
}
|
|
3318
|
+
if (lowerError.includes("timeout") || lowerError.includes("timed out")) {
|
|
3319
|
+
return "timeout";
|
|
3320
|
+
}
|
|
3321
|
+
if (lowerError.includes("navigation") || lowerError.includes("navigate") || lowerError.includes("url")) {
|
|
3322
|
+
return "navigation_failed";
|
|
3323
|
+
}
|
|
3324
|
+
if (lowerError.includes("not interactable") || lowerError.includes("disabled") || lowerError.includes("hidden")) {
|
|
3325
|
+
return "element_not_interactable";
|
|
3326
|
+
}
|
|
3327
|
+
return "unknown";
|
|
3328
|
+
}
|
|
3329
|
+
/**
|
|
3330
|
+
* Find alternative selectors for a target on the current page.
|
|
3331
|
+
*/
|
|
3332
|
+
async function findAlternatives(browser, originalTarget) {
|
|
3333
|
+
const page = browser.page;
|
|
3334
|
+
if (!page)
|
|
3335
|
+
return [];
|
|
3336
|
+
try {
|
|
3337
|
+
const alternatives = await page.evaluate((target) => {
|
|
3338
|
+
const results = [];
|
|
3339
|
+
const lowerTarget = target.toLowerCase();
|
|
3340
|
+
// Find buttons with similar text
|
|
3341
|
+
document.querySelectorAll("button, [role='button'], input[type='submit'], input[type='button']").forEach((el) => {
|
|
3342
|
+
const text = el.textContent?.trim() || el.value || "";
|
|
3343
|
+
if (text && (text.toLowerCase().includes(lowerTarget) || lowerTarget.includes(text.toLowerCase()))) {
|
|
3344
|
+
results.push(`button: "${text}"`);
|
|
3345
|
+
}
|
|
3346
|
+
});
|
|
3347
|
+
// Find links with similar text
|
|
3348
|
+
document.querySelectorAll("a").forEach((el) => {
|
|
3349
|
+
const text = el.textContent?.trim() || "";
|
|
3350
|
+
if (text && (text.toLowerCase().includes(lowerTarget) || lowerTarget.includes(text.toLowerCase()))) {
|
|
3351
|
+
results.push(`link: "${text}"`);
|
|
3352
|
+
}
|
|
3353
|
+
});
|
|
3354
|
+
// Find inputs with similar labels/placeholders
|
|
3355
|
+
document.querySelectorAll("input, textarea, select").forEach((el) => {
|
|
3356
|
+
const input = el;
|
|
3357
|
+
const placeholder = input.placeholder || "";
|
|
3358
|
+
const label = document.querySelector(`label[for="${input.id}"]`)?.textContent?.trim() || "";
|
|
3359
|
+
const name = input.name || "";
|
|
3360
|
+
if (placeholder.toLowerCase().includes(lowerTarget) || lowerTarget.includes(placeholder.toLowerCase())) {
|
|
3361
|
+
results.push(`input with placeholder "${placeholder}"`);
|
|
3362
|
+
}
|
|
3363
|
+
if (label.toLowerCase().includes(lowerTarget) || lowerTarget.includes(label.toLowerCase())) {
|
|
3364
|
+
results.push(`input labeled "${label}"`);
|
|
3365
|
+
}
|
|
3366
|
+
if (name.toLowerCase().includes(lowerTarget)) {
|
|
3367
|
+
results.push(`input named "${name}"`);
|
|
3368
|
+
}
|
|
3369
|
+
});
|
|
3370
|
+
// Find elements by aria-label
|
|
3371
|
+
document.querySelectorAll("[aria-label]").forEach((el) => {
|
|
3372
|
+
const label = el.getAttribute("aria-label") || "";
|
|
3373
|
+
if (label.toLowerCase().includes(lowerTarget) || lowerTarget.includes(label.toLowerCase())) {
|
|
3374
|
+
results.push(`aria:${el.tagName.toLowerCase()}/"${label}"`);
|
|
3375
|
+
}
|
|
3376
|
+
});
|
|
3377
|
+
return [...new Set(results)].slice(0, 10);
|
|
3378
|
+
}, originalTarget);
|
|
3379
|
+
return alternatives;
|
|
3380
|
+
}
|
|
3381
|
+
catch {
|
|
3382
|
+
return [];
|
|
3383
|
+
}
|
|
3384
|
+
}
|
|
3385
|
+
/**
|
|
3386
|
+
* Get page context for failure analysis.
|
|
3387
|
+
*/
|
|
3388
|
+
async function getPageContext(browser) {
|
|
3389
|
+
const page = browser.page;
|
|
3390
|
+
if (!page)
|
|
3391
|
+
return { url: "", title: "", visibleText: [] };
|
|
3392
|
+
try {
|
|
3393
|
+
const context = await page.evaluate(() => {
|
|
3394
|
+
const visibleText = [];
|
|
3395
|
+
// Get visible button/link text
|
|
3396
|
+
document.querySelectorAll("button, a, [role='button']").forEach((el) => {
|
|
3397
|
+
const text = el.textContent?.trim();
|
|
3398
|
+
if (text && text.length < 50) {
|
|
3399
|
+
visibleText.push(text);
|
|
3400
|
+
}
|
|
3401
|
+
});
|
|
3402
|
+
return {
|
|
3403
|
+
url: window.location.href,
|
|
3404
|
+
title: document.title,
|
|
3405
|
+
visibleText: [...new Set(visibleText)].slice(0, 20),
|
|
3406
|
+
};
|
|
3407
|
+
});
|
|
3408
|
+
return context;
|
|
3409
|
+
}
|
|
3410
|
+
catch {
|
|
3411
|
+
return { url: "", title: "", visibleText: [] };
|
|
3412
|
+
}
|
|
3413
|
+
}
|
|
3414
|
+
/**
|
|
3415
|
+
* Generate repair suggestions for a failed step.
|
|
3416
|
+
*/
|
|
3417
|
+
async function generateRepairSuggestions(browser, step, error, failureType, alternatives, pageContext) {
|
|
3418
|
+
const suggestions = [];
|
|
3419
|
+
switch (failureType) {
|
|
3420
|
+
case "selector_not_found": {
|
|
3421
|
+
// Suggest alternative selectors
|
|
3422
|
+
for (const alt of alternatives.slice(0, 3)) {
|
|
3423
|
+
const newInstruction = step.instruction.replace(step.target || "", alt.replace(/^(button|link|input|aria):\s*/, "").replace(/"/g, "'"));
|
|
3424
|
+
suggestions.push({
|
|
3425
|
+
type: "selector_update",
|
|
3426
|
+
confidence: 0.7,
|
|
3427
|
+
description: `Update selector to "${alt}"`,
|
|
3428
|
+
originalInstruction: step.instruction,
|
|
3429
|
+
suggestedInstruction: `click ${alt.replace(/^(button|link|input|aria):\s*/, "")}`,
|
|
3430
|
+
reasoning: `Found similar element on page: ${alt}`,
|
|
3431
|
+
});
|
|
3432
|
+
}
|
|
3433
|
+
// Suggest adding a wait
|
|
3434
|
+
if (suggestions.length === 0) {
|
|
3435
|
+
suggestions.push({
|
|
3436
|
+
type: "add_wait",
|
|
3437
|
+
confidence: 0.5,
|
|
3438
|
+
description: "Add wait before this step",
|
|
3439
|
+
originalInstruction: step.instruction,
|
|
3440
|
+
suggestedInstruction: `wait 2 seconds\n${step.instruction}`,
|
|
3441
|
+
reasoning: "Element might not be loaded yet - adding wait may help",
|
|
3442
|
+
});
|
|
3443
|
+
}
|
|
3444
|
+
break;
|
|
3445
|
+
}
|
|
3446
|
+
case "assertion_failed": {
|
|
3447
|
+
// Suggest updating the assertion based on page content
|
|
3448
|
+
if (step.action === "assert" && pageContext.visibleText.length > 0) {
|
|
3449
|
+
const possibleText = pageContext.visibleText.find((t) => t.length > 3 && t.length < 30);
|
|
3450
|
+
if (possibleText) {
|
|
3451
|
+
suggestions.push({
|
|
3452
|
+
type: "assertion_update",
|
|
3453
|
+
confidence: 0.6,
|
|
3454
|
+
description: `Update assertion to check for visible text`,
|
|
3455
|
+
originalInstruction: step.instruction,
|
|
3456
|
+
suggestedInstruction: `verify page contains "${possibleText}"`,
|
|
3457
|
+
reasoning: `Page contains "${possibleText}" which might be the intended check`,
|
|
3458
|
+
});
|
|
3459
|
+
}
|
|
3460
|
+
}
|
|
3461
|
+
// Suggest checking URL instead
|
|
3462
|
+
if (pageContext.url) {
|
|
3463
|
+
const urlPath = new URL(pageContext.url).pathname;
|
|
3464
|
+
suggestions.push({
|
|
3465
|
+
type: "assertion_update",
|
|
3466
|
+
confidence: 0.5,
|
|
3467
|
+
description: "Assert URL instead of content",
|
|
3468
|
+
originalInstruction: step.instruction,
|
|
3469
|
+
suggestedInstruction: `verify url contains "${urlPath}"`,
|
|
3470
|
+
reasoning: `Current URL is ${pageContext.url}`,
|
|
3471
|
+
});
|
|
3472
|
+
}
|
|
3473
|
+
break;
|
|
3474
|
+
}
|
|
3475
|
+
case "timeout": {
|
|
3476
|
+
suggestions.push({
|
|
3477
|
+
type: "add_wait",
|
|
3478
|
+
confidence: 0.7,
|
|
3479
|
+
description: "Increase wait time",
|
|
3480
|
+
originalInstruction: step.instruction,
|
|
3481
|
+
suggestedInstruction: `wait 5 seconds\n${step.instruction}`,
|
|
3482
|
+
reasoning: "Operation timed out - page may need more time to load",
|
|
3483
|
+
});
|
|
3484
|
+
break;
|
|
3485
|
+
}
|
|
3486
|
+
case "element_not_interactable": {
|
|
3487
|
+
suggestions.push({
|
|
3488
|
+
type: "add_wait",
|
|
3489
|
+
confidence: 0.6,
|
|
3490
|
+
description: "Wait for element to become interactive",
|
|
3491
|
+
originalInstruction: step.instruction,
|
|
3492
|
+
suggestedInstruction: `wait 2 seconds\n${step.instruction}`,
|
|
3493
|
+
reasoning: "Element exists but is not interactable - may need to wait",
|
|
3494
|
+
});
|
|
3495
|
+
// Suggest scrolling
|
|
3496
|
+
suggestions.push({
|
|
3497
|
+
type: "change_action",
|
|
3498
|
+
confidence: 0.5,
|
|
3499
|
+
description: "Scroll element into view first",
|
|
3500
|
+
originalInstruction: step.instruction,
|
|
3501
|
+
suggestedInstruction: `scroll down\n${step.instruction}`,
|
|
3502
|
+
reasoning: "Element might be outside viewport",
|
|
3503
|
+
});
|
|
3504
|
+
break;
|
|
3505
|
+
}
|
|
3506
|
+
default: {
|
|
3507
|
+
// Generic suggestion to skip
|
|
3508
|
+
suggestions.push({
|
|
3509
|
+
type: "skip_step",
|
|
3510
|
+
confidence: 0.3,
|
|
3511
|
+
description: "Skip this step",
|
|
3512
|
+
originalInstruction: step.instruction,
|
|
3513
|
+
suggestedInstruction: `// SKIPPED: ${step.instruction}`,
|
|
3514
|
+
reasoning: "Unable to determine a fix - consider removing this step",
|
|
3515
|
+
});
|
|
3516
|
+
}
|
|
3517
|
+
}
|
|
3518
|
+
// Sort by confidence
|
|
3519
|
+
return suggestions.sort((a, b) => b.confidence - a.confidence);
|
|
3520
|
+
}
|
|
3521
|
+
/**
|
|
3522
|
+
* Analyze a failed test step and suggest repairs.
|
|
3523
|
+
*/
|
|
3524
|
+
async function analyzeFailure(browser, step, error) {
|
|
3525
|
+
const failureType = classifyFailure(error, step);
|
|
3526
|
+
const alternatives = step.target ? await findAlternatives(browser, step.target) : [];
|
|
3527
|
+
const pageContext = await getPageContext(browser);
|
|
3528
|
+
const suggestions = await generateRepairSuggestions(browser, step, error, failureType, alternatives, pageContext);
|
|
3529
|
+
return {
|
|
3530
|
+
step,
|
|
3531
|
+
error,
|
|
3532
|
+
failureType,
|
|
3533
|
+
targetSelector: step.target,
|
|
3534
|
+
alternativeSelectors: alternatives,
|
|
3535
|
+
pageContext,
|
|
3536
|
+
suggestions,
|
|
3537
|
+
};
|
|
3538
|
+
}
|
|
3539
|
+
/**
|
|
3540
|
+
* Run a test, analyze failures, and suggest/apply repairs.
|
|
3541
|
+
*/
|
|
3542
|
+
async function repairTest(test, options = {}) {
|
|
3543
|
+
const { headless = true, autoApply = false, verifyRepairs = true, maxRetries = 3, } = options;
|
|
3544
|
+
const browser = new CBrowser({ headless });
|
|
3545
|
+
const failureAnalyses = [];
|
|
3546
|
+
const repairedSteps = [];
|
|
3547
|
+
let failedSteps = 0;
|
|
3548
|
+
console.log(`\n🔧 Analyzing test: ${test.name}`);
|
|
3549
|
+
console.log(` Steps: ${test.steps.length}`);
|
|
3550
|
+
console.log(` Auto-apply: ${autoApply}`);
|
|
3551
|
+
console.log("");
|
|
3552
|
+
try {
|
|
3553
|
+
await browser.launch();
|
|
3554
|
+
for (const step of test.steps) {
|
|
3555
|
+
console.log(` → ${step.instruction}`);
|
|
3556
|
+
let stepPassed = false;
|
|
3557
|
+
let lastError = "";
|
|
3558
|
+
let attempts = 0;
|
|
3559
|
+
while (!stepPassed && attempts < maxRetries) {
|
|
3560
|
+
attempts++;
|
|
3561
|
+
try {
|
|
3562
|
+
// Execute the step
|
|
3563
|
+
switch (step.action) {
|
|
3564
|
+
case "navigate":
|
|
3565
|
+
await browser.navigate(step.target || "");
|
|
3566
|
+
stepPassed = true;
|
|
3567
|
+
break;
|
|
3568
|
+
case "click":
|
|
3569
|
+
const clickResult = await browser.smartClick(step.target || "");
|
|
3570
|
+
stepPassed = clickResult.success;
|
|
3571
|
+
if (!stepPassed)
|
|
3572
|
+
lastError = `Failed to click: ${step.target}`;
|
|
3573
|
+
break;
|
|
3574
|
+
case "fill":
|
|
3575
|
+
await browser.fill(step.target || "", step.value || "");
|
|
3576
|
+
stepPassed = true;
|
|
3577
|
+
break;
|
|
3578
|
+
case "assert":
|
|
3579
|
+
const assertResult = await browser.assert(step.instruction);
|
|
3580
|
+
stepPassed = assertResult.passed;
|
|
3581
|
+
if (!stepPassed)
|
|
3582
|
+
lastError = assertResult.message;
|
|
3583
|
+
break;
|
|
3584
|
+
case "wait":
|
|
3585
|
+
if (step.target) {
|
|
3586
|
+
const page = browser.page;
|
|
3587
|
+
if (page) {
|
|
3588
|
+
await page.waitForSelector(`text=${step.target}`, { timeout: 10000 });
|
|
3589
|
+
}
|
|
3590
|
+
}
|
|
3591
|
+
else {
|
|
3592
|
+
const ms = parseFloat(step.value || "1") * 1000;
|
|
3593
|
+
await new Promise((r) => setTimeout(r, ms));
|
|
3594
|
+
}
|
|
3595
|
+
stepPassed = true;
|
|
3596
|
+
break;
|
|
3597
|
+
case "scroll":
|
|
3598
|
+
const direction = step.target?.toLowerCase() === "up" ? -500 : 500;
|
|
3599
|
+
const page = browser.page;
|
|
3600
|
+
if (page) {
|
|
3601
|
+
await page.evaluate((d) => window.scrollBy(0, d), direction);
|
|
3602
|
+
}
|
|
3603
|
+
stepPassed = true;
|
|
3604
|
+
break;
|
|
3605
|
+
case "screenshot":
|
|
3606
|
+
await browser.screenshot();
|
|
3607
|
+
stepPassed = true;
|
|
3608
|
+
break;
|
|
3609
|
+
default:
|
|
3610
|
+
// Try as click
|
|
3611
|
+
const unknownResult = await browser.smartClick(step.target || step.instruction);
|
|
3612
|
+
stepPassed = unknownResult.success;
|
|
3613
|
+
if (!stepPassed)
|
|
3614
|
+
lastError = `Could not interpret: ${step.instruction}`;
|
|
3615
|
+
}
|
|
3616
|
+
}
|
|
3617
|
+
catch (e) {
|
|
3618
|
+
lastError = e.message;
|
|
3619
|
+
stepPassed = false;
|
|
3620
|
+
}
|
|
3621
|
+
if (!stepPassed && attempts < maxRetries) {
|
|
3622
|
+
console.log(` ⚠️ Attempt ${attempts} failed, retrying...`);
|
|
3623
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
3624
|
+
}
|
|
3625
|
+
}
|
|
3626
|
+
if (stepPassed) {
|
|
3627
|
+
console.log(` ✓ Passed`);
|
|
3628
|
+
repairedSteps.push(step);
|
|
3629
|
+
}
|
|
3630
|
+
else {
|
|
3631
|
+
console.log(` ✗ Failed: ${lastError}`);
|
|
3632
|
+
failedSteps++;
|
|
3633
|
+
// Analyze the failure
|
|
3634
|
+
const analysis = await analyzeFailure(browser, step, lastError);
|
|
3635
|
+
failureAnalyses.push(analysis);
|
|
3636
|
+
if (autoApply && analysis.suggestions.length > 0) {
|
|
3637
|
+
const bestSuggestion = analysis.suggestions[0];
|
|
3638
|
+
console.log(` 🔧 Auto-applying: ${bestSuggestion.description}`);
|
|
3639
|
+
// Parse the suggested instruction into a step
|
|
3640
|
+
const repairedStep = parseNLInstruction(bestSuggestion.suggestedInstruction.split("\n").pop() || "");
|
|
3641
|
+
repairedSteps.push(repairedStep);
|
|
3642
|
+
}
|
|
3643
|
+
else {
|
|
3644
|
+
// Keep original step
|
|
3645
|
+
repairedSteps.push(step);
|
|
3646
|
+
if (analysis.suggestions.length > 0) {
|
|
3647
|
+
console.log(` 💡 Suggestions:`);
|
|
3648
|
+
for (const suggestion of analysis.suggestions.slice(0, 2)) {
|
|
3649
|
+
console.log(` - ${suggestion.description} (${Math.round(suggestion.confidence * 100)}%)`);
|
|
3650
|
+
console.log(` → ${suggestion.suggestedInstruction}`);
|
|
3651
|
+
}
|
|
3652
|
+
}
|
|
3653
|
+
}
|
|
3654
|
+
}
|
|
3655
|
+
}
|
|
3656
|
+
// Create repaired test
|
|
3657
|
+
const repairedTest = {
|
|
3658
|
+
name: test.name,
|
|
3659
|
+
description: test.description,
|
|
3660
|
+
steps: repairedSteps,
|
|
3661
|
+
};
|
|
3662
|
+
// Optionally verify the repaired test
|
|
3663
|
+
let repairedTestPasses;
|
|
3664
|
+
if (verifyRepairs && autoApply && failedSteps > 0) {
|
|
3665
|
+
console.log(`\n 🔄 Verifying repaired test...`);
|
|
3666
|
+
await browser.close();
|
|
3667
|
+
const verifyBrowser = new CBrowser({ headless });
|
|
3668
|
+
try {
|
|
3669
|
+
await verifyBrowser.launch();
|
|
3670
|
+
let allPassed = true;
|
|
3671
|
+
for (const step of repairedSteps) {
|
|
3672
|
+
try {
|
|
3673
|
+
switch (step.action) {
|
|
3674
|
+
case "navigate":
|
|
3675
|
+
await verifyBrowser.navigate(step.target || "");
|
|
3676
|
+
break;
|
|
3677
|
+
case "click":
|
|
3678
|
+
const result = await verifyBrowser.smartClick(step.target || "");
|
|
3679
|
+
if (!result.success)
|
|
3680
|
+
allPassed = false;
|
|
3681
|
+
break;
|
|
3682
|
+
case "fill":
|
|
3683
|
+
await verifyBrowser.fill(step.target || "", step.value || "");
|
|
3684
|
+
break;
|
|
3685
|
+
case "assert":
|
|
3686
|
+
const assertResult = await verifyBrowser.assert(step.instruction);
|
|
3687
|
+
if (!assertResult.passed)
|
|
3688
|
+
allPassed = false;
|
|
3689
|
+
break;
|
|
3690
|
+
case "wait":
|
|
3691
|
+
if (!step.target) {
|
|
3692
|
+
const ms = parseFloat(step.value || "1") * 1000;
|
|
3693
|
+
await new Promise((r) => setTimeout(r, ms));
|
|
3694
|
+
}
|
|
3695
|
+
break;
|
|
3696
|
+
}
|
|
3697
|
+
}
|
|
3698
|
+
catch {
|
|
3699
|
+
allPassed = false;
|
|
3700
|
+
}
|
|
3701
|
+
}
|
|
3702
|
+
repairedTestPasses = allPassed;
|
|
3703
|
+
console.log(` ${allPassed ? "✅" : "❌"} Repaired test ${allPassed ? "PASSES" : "still FAILS"}`);
|
|
3704
|
+
}
|
|
3705
|
+
finally {
|
|
3706
|
+
await verifyBrowser.close();
|
|
3707
|
+
}
|
|
3708
|
+
}
|
|
3709
|
+
return {
|
|
3710
|
+
originalTest: test,
|
|
3711
|
+
repairedTest: failedSteps > 0 ? repairedTest : undefined,
|
|
3712
|
+
failedSteps,
|
|
3713
|
+
repairedSteps: autoApply ? failedSteps : 0,
|
|
3714
|
+
failureAnalyses,
|
|
3715
|
+
repairedTestPasses,
|
|
3716
|
+
};
|
|
3717
|
+
}
|
|
3718
|
+
finally {
|
|
3719
|
+
await browser.close();
|
|
3720
|
+
}
|
|
3721
|
+
}
|
|
3722
|
+
/**
|
|
3723
|
+
* Run test repair on a full suite.
|
|
3724
|
+
*/
|
|
3725
|
+
async function repairTestSuite(suite, options = {}) {
|
|
3726
|
+
const startTime = Date.now();
|
|
3727
|
+
const testResults = [];
|
|
3728
|
+
console.log(`\n🔧 Repairing Test Suite: ${suite.name}`);
|
|
3729
|
+
console.log(` Tests: ${suite.tests.length}`);
|
|
3730
|
+
console.log("");
|
|
3731
|
+
for (const test of suite.tests) {
|
|
3732
|
+
const result = await repairTest(test, options);
|
|
3733
|
+
testResults.push(result);
|
|
3734
|
+
}
|
|
3735
|
+
const testsWithFailures = testResults.filter((r) => r.failedSteps > 0).length;
|
|
3736
|
+
const testsRepaired = testResults.filter((r) => r.repairedSteps > 0).length;
|
|
3737
|
+
const totalFailedSteps = testResults.reduce((sum, r) => sum + r.failedSteps, 0);
|
|
3738
|
+
const totalRepairedSteps = testResults.reduce((sum, r) => sum + r.repairedSteps, 0);
|
|
3739
|
+
return {
|
|
3740
|
+
suiteName: suite.name,
|
|
3741
|
+
timestamp: new Date().toISOString(),
|
|
3742
|
+
duration: Date.now() - startTime,
|
|
3743
|
+
testResults,
|
|
3744
|
+
summary: {
|
|
3745
|
+
totalTests: suite.tests.length,
|
|
3746
|
+
testsWithFailures,
|
|
3747
|
+
testsRepaired,
|
|
3748
|
+
totalFailedSteps,
|
|
3749
|
+
totalRepairedSteps,
|
|
3750
|
+
repairSuccessRate: totalFailedSteps > 0 ? (totalRepairedSteps / totalFailedSteps) * 100 : 100,
|
|
3751
|
+
},
|
|
3752
|
+
};
|
|
3753
|
+
}
|
|
3754
|
+
/**
|
|
3755
|
+
* Format a repair result as a report.
|
|
3756
|
+
*/
|
|
3757
|
+
function formatRepairReport(result) {
|
|
3758
|
+
const lines = [];
|
|
3759
|
+
lines.push("");
|
|
3760
|
+
lines.push("╔══════════════════════════════════════════════════════════════════════════════╗");
|
|
3761
|
+
lines.push("║ AI TEST REPAIR REPORT ║");
|
|
3762
|
+
lines.push("╚══════════════════════════════════════════════════════════════════════════════╝");
|
|
3763
|
+
lines.push("");
|
|
3764
|
+
lines.push(`📋 Suite: ${result.suiteName}`);
|
|
3765
|
+
lines.push(`⏱️ Duration: ${(result.duration / 1000).toFixed(1)}s`);
|
|
3766
|
+
lines.push(`📅 Timestamp: ${result.timestamp}`);
|
|
3767
|
+
lines.push("");
|
|
3768
|
+
// Summary
|
|
3769
|
+
lines.push("📊 SUMMARY");
|
|
3770
|
+
lines.push("─".repeat(60));
|
|
3771
|
+
lines.push(` Total Tests: ${result.summary.totalTests}`);
|
|
3772
|
+
lines.push(` Tests with Failures: ${result.summary.testsWithFailures}`);
|
|
3773
|
+
lines.push(` Tests Repaired: ${result.summary.testsRepaired}`);
|
|
3774
|
+
lines.push(` Total Failed Steps: ${result.summary.totalFailedSteps}`);
|
|
3775
|
+
lines.push(` Total Repaired Steps: ${result.summary.totalRepairedSteps}`);
|
|
3776
|
+
lines.push(` Repair Success Rate: ${result.summary.repairSuccessRate.toFixed(0)}%`);
|
|
3777
|
+
lines.push("");
|
|
3778
|
+
// Per-test details
|
|
3779
|
+
for (const testResult of result.testResults) {
|
|
3780
|
+
if (testResult.failedSteps === 0)
|
|
3781
|
+
continue;
|
|
3782
|
+
lines.push(`\n🔧 ${testResult.originalTest.name}`);
|
|
3783
|
+
lines.push(` Failed Steps: ${testResult.failedSteps}`);
|
|
3784
|
+
lines.push(` Repaired: ${testResult.repairedSteps}`);
|
|
3785
|
+
for (const analysis of testResult.failureAnalyses) {
|
|
3786
|
+
lines.push(`\n ❌ ${analysis.step.instruction}`);
|
|
3787
|
+
lines.push(` Error: ${analysis.error}`);
|
|
3788
|
+
lines.push(` Type: ${analysis.failureType}`);
|
|
3789
|
+
if (analysis.suggestions.length > 0) {
|
|
3790
|
+
lines.push(` 💡 Suggestions:`);
|
|
3791
|
+
for (const s of analysis.suggestions.slice(0, 2)) {
|
|
3792
|
+
lines.push(` - ${s.description} (${Math.round(s.confidence * 100)}%)`);
|
|
3793
|
+
lines.push(` → ${s.suggestedInstruction}`);
|
|
3794
|
+
}
|
|
3795
|
+
}
|
|
3796
|
+
}
|
|
3797
|
+
if (testResult.repairedTestPasses !== undefined) {
|
|
3798
|
+
lines.push(`\n ${testResult.repairedTestPasses ? "✅" : "❌"} Repaired test ${testResult.repairedTestPasses ? "PASSES" : "still FAILS"}`);
|
|
3799
|
+
}
|
|
3800
|
+
}
|
|
3801
|
+
lines.push("");
|
|
3802
|
+
return lines.join("\n");
|
|
3803
|
+
}
|
|
3804
|
+
/**
|
|
3805
|
+
* Export repaired test to file.
|
|
3806
|
+
*/
|
|
3807
|
+
function exportRepairedTest(result) {
|
|
3808
|
+
if (!result.repairedTest) {
|
|
3809
|
+
return `# Test: ${result.originalTest.name}\n# No repairs needed\n${result.originalTest.steps.map((s) => s.instruction).join("\n")}`;
|
|
3810
|
+
}
|
|
3811
|
+
const lines = [];
|
|
3812
|
+
lines.push(`# Test: ${result.repairedTest.name} (Repaired)`);
|
|
3813
|
+
lines.push(`# Original failures: ${result.failedSteps}`);
|
|
3814
|
+
lines.push(`# Repairs applied: ${result.repairedSteps}`);
|
|
3815
|
+
lines.push("");
|
|
3816
|
+
for (const step of result.repairedTest.steps) {
|
|
3817
|
+
lines.push(step.instruction);
|
|
3818
|
+
}
|
|
3819
|
+
return lines.join("\n");
|
|
3820
|
+
}
|
|
2880
3821
|
//# sourceMappingURL=browser.js.map
|