@canaryai/cli 0.1.1-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -0
- package/dist/fsevents-72LCIACT.node +0 -0
- package/dist/index.d.ts +53 -0
- package/dist/index.js +916 -0
- package/dist/index.js.map +1 -0
- package/dist/runner/preload.d.ts +2 -0
- package/dist/runner/preload.js +1491 -0
- package/dist/runner/preload.js.map +1 -0
- package/dist/test.d.ts +1167 -0
- package/dist/test.js +149276 -0
- package/dist/test.js.map +1 -0
- package/package.json +57 -0
|
@@ -0,0 +1,1491 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
|
|
3
|
+
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
|
|
4
|
+
}) : x)(function(x) {
|
|
5
|
+
if (typeof require !== "undefined") return require.apply(this, arguments);
|
|
6
|
+
throw Error('Dynamic require of "' + x + '" is not supported');
|
|
7
|
+
});
|
|
8
|
+
|
|
9
|
+
// src/runner/instrumentation.ts
|
|
10
|
+
import Module from "module";
|
|
11
|
+
import { createRequire } from "module";
|
|
12
|
+
|
|
13
|
+
// src/runner/config.ts
|
|
14
|
+
import path2 from "path";
|
|
15
|
+
|
|
16
|
+
// src/runner/env.ts
|
|
17
|
+
import fs from "fs";
|
|
18
|
+
import path from "path";
|
|
19
|
+
var loaded = false;
|
|
20
|
+
function loadCanaryEnv() {
|
|
21
|
+
if (loaded) return;
|
|
22
|
+
loaded = true;
|
|
23
|
+
const explicitPath = process.env.CANARY_ENV_FILE;
|
|
24
|
+
const defaultPath = path.join(process.cwd(), ".env");
|
|
25
|
+
const envPath = explicitPath || defaultPath;
|
|
26
|
+
if (fs.existsSync(envPath)) {
|
|
27
|
+
loadFile(envPath);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
function loadFile(filePath) {
|
|
31
|
+
try {
|
|
32
|
+
const raw = fs.readFileSync(filePath, "utf-8");
|
|
33
|
+
for (const line of raw.split("\n")) {
|
|
34
|
+
const trimmed = line.trim();
|
|
35
|
+
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
36
|
+
const idx = trimmed.indexOf("=");
|
|
37
|
+
if (idx === -1) continue;
|
|
38
|
+
const key = trimmed.slice(0, idx).trim();
|
|
39
|
+
const value = trimmed.slice(idx + 1).trim();
|
|
40
|
+
if (!key) continue;
|
|
41
|
+
if (process.env[key] === void 0) {
|
|
42
|
+
process.env[key] = stripQuotes(value);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
} catch {
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
function stripQuotes(value) {
|
|
49
|
+
if (value.startsWith('"') && value.endsWith('"') || value.startsWith("'") && value.endsWith("'")) {
|
|
50
|
+
return value.slice(1, -1);
|
|
51
|
+
}
|
|
52
|
+
return value;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// src/runner/config.ts
|
|
56
|
+
function loadCanaryConfig() {
|
|
57
|
+
loadCanaryEnv();
|
|
58
|
+
const eventLogPath = (() => {
|
|
59
|
+
const userPath = process.env.CANARY_EVENT_LOG;
|
|
60
|
+
if (userPath) return userPath;
|
|
61
|
+
const worker = process.env.TEST_WORKER_INDEX ?? "runner";
|
|
62
|
+
return path2.join(process.cwd(), "test-results", "ai-healer", `events-worker-${worker}.jsonl`);
|
|
63
|
+
})();
|
|
64
|
+
const baseConfig = {
|
|
65
|
+
enabled: true,
|
|
66
|
+
allowedPlaywrightVersion: process.env.CANARY_PLAYWRIGHT_VERSION,
|
|
67
|
+
aiProvider: process.env.AI_PROVIDER,
|
|
68
|
+
aiModel: process.env.AI_MODEL,
|
|
69
|
+
apiKey: process.env.AI_API_KEY,
|
|
70
|
+
healTimeoutMs: Number(process.env.AI_TIMEOUT_MS ?? 12e4),
|
|
71
|
+
// 2 minutes for agentic healing
|
|
72
|
+
maxActions: Number(process.env.CANARY_MAX_ACTIONS ?? 50),
|
|
73
|
+
// Generous step limit for agentic healing
|
|
74
|
+
dryRun: process.env.CANARY_DRY_RUN === "1",
|
|
75
|
+
warnOnly: process.env.CANARY_WARN_ONLY === "1",
|
|
76
|
+
visionEnabled: process.env.CANARY_VISION === "1" || process.env.AI_VISION === "1",
|
|
77
|
+
debug: process.env.CANARY_DEBUG === "1",
|
|
78
|
+
readOnly: process.env.CANARY_READ_ONLY === "1",
|
|
79
|
+
allowRunCode: process.env.CANARY_ALLOW_RUN_CODE === "1",
|
|
80
|
+
allowEvaluate: process.env.CANARY_ALLOW_EVALUATE !== "0",
|
|
81
|
+
maxPayloadBytes: Number(process.env.CANARY_MAX_PAYLOAD_BYTES ?? 6e4),
|
|
82
|
+
// cap snapshots/screenshots/text
|
|
83
|
+
eventLogPath,
|
|
84
|
+
eventLoggingEnabled: process.env.CANARY_EVENT_LOG !== "0"
|
|
85
|
+
};
|
|
86
|
+
const disabled = process.env.CANARY_ENABLED === "0" || process.env.CANARY_DISABLED === "1" || process.env.AI_HEALING === "0";
|
|
87
|
+
return {
|
|
88
|
+
...baseConfig,
|
|
89
|
+
enabled: !disabled && baseConfig.enabled
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// src/runner/healer.ts
|
|
94
|
+
import { stepCountIs, streamText, tool } from "ai";
|
|
95
|
+
import { z } from "zod";
|
|
96
|
+
|
|
97
|
+
// src/runner/ai-client.ts
|
|
98
|
+
import { createOpenAI } from "@ai-sdk/openai";
|
|
99
|
+
import { createAnthropic } from "@ai-sdk/anthropic";
|
|
100
|
+
import { createAzure } from "@ai-sdk/azure";
|
|
101
|
+
function resolveHealerModel(config = loadCanaryConfig()) {
|
|
102
|
+
if (!config.enabled) {
|
|
103
|
+
return { model: null, reason: "healing_disabled" };
|
|
104
|
+
}
|
|
105
|
+
if (!config.apiKey) {
|
|
106
|
+
return { model: null, reason: "missing_api_key" };
|
|
107
|
+
}
|
|
108
|
+
const provider = (config.aiProvider ?? "openai").toLowerCase();
|
|
109
|
+
const modelId = config.aiModel || (provider === "anthropic" ? "claude-3-5-haiku-20241022" : "gpt-4o-mini");
|
|
110
|
+
try {
|
|
111
|
+
switch (provider) {
|
|
112
|
+
case "openai": {
|
|
113
|
+
const client = createOpenAI({ apiKey: config.apiKey, baseURL: process.env.AI_BASE_URL });
|
|
114
|
+
return { model: client(modelId), modelId };
|
|
115
|
+
}
|
|
116
|
+
case "anthropic": {
|
|
117
|
+
const client = createAnthropic({ apiKey: config.apiKey, baseURL: process.env.AI_BASE_URL });
|
|
118
|
+
return { model: client(modelId), modelId };
|
|
119
|
+
}
|
|
120
|
+
case "azure": {
|
|
121
|
+
const resourceName = process.env.AZURE_OPENAI_RESOURCE_NAME;
|
|
122
|
+
const apiVersion = process.env.AZURE_OPENAI_API_VERSION;
|
|
123
|
+
if (!resourceName) {
|
|
124
|
+
return { model: null, reason: "missing_azure_resource" };
|
|
125
|
+
}
|
|
126
|
+
const client = createAzure({
|
|
127
|
+
apiKey: config.apiKey,
|
|
128
|
+
resourceName,
|
|
129
|
+
apiVersion
|
|
130
|
+
});
|
|
131
|
+
return { model: client(modelId), modelId };
|
|
132
|
+
}
|
|
133
|
+
default:
|
|
134
|
+
return { model: null, reason: "unsupported_provider" };
|
|
135
|
+
}
|
|
136
|
+
} catch (error) {
|
|
137
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
138
|
+
return { model: null, reason: `model_error:${message}` };
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// src/runner/healer.ts
|
|
143
|
+
var REDACTION_PATTERNS = [
|
|
144
|
+
{ regex: /bearer\s+[a-z0-9._-]+/gi, replacement: "[REDACTED_BEARER]" },
|
|
145
|
+
{ regex: /api[_-]?key[:\s"']+[a-z0-9._-]+/gi, replacement: "[REDACTED_API_KEY]" },
|
|
146
|
+
{ regex: /secret[:\s"']+[a-z0-9._-]+/gi, replacement: "[REDACTED_SECRET]" },
|
|
147
|
+
{ regex: /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g, replacement: "[REDACTED_EMAIL]" },
|
|
148
|
+
{ regex: /[0-9]{12,}/g, replacement: "[REDACTED_NUMBER]" }
|
|
149
|
+
];
|
|
150
|
+
var CODE_DENY_PATTERNS = [/process\.env/i, /child_process/i, /\brequire\s*\(/i, /\bimport\s*\(/i];
|
|
151
|
+
function classifyFailure(context) {
|
|
152
|
+
const message = (context.errorMessage ?? "").toLowerCase();
|
|
153
|
+
if (message.includes("closed") || message.includes("target page, context or browser has been closed")) {
|
|
154
|
+
return { healable: false, reason: "context_closed" };
|
|
155
|
+
}
|
|
156
|
+
if (message.includes("navigation failed") && message.includes("net::")) {
|
|
157
|
+
return { healable: false, reason: "navigation_failed" };
|
|
158
|
+
}
|
|
159
|
+
return { healable: true, reason: "agent_healing" };
|
|
160
|
+
}
|
|
161
|
+
async function executeHealActions(decision, failure, execCtx) {
|
|
162
|
+
const config = loadCanaryConfig();
|
|
163
|
+
const started = Date.now();
|
|
164
|
+
const mode = resolveMode(config);
|
|
165
|
+
const actionsRun = [];
|
|
166
|
+
const errors = [];
|
|
167
|
+
const logTools = config.debug || process.env.CANARY_TOOL_LOG === "1";
|
|
168
|
+
const initialContext = await buildInitialPageContext(execCtx.page, config);
|
|
169
|
+
if (!decision.healable) {
|
|
170
|
+
return baseOutcome({
|
|
171
|
+
mode,
|
|
172
|
+
started,
|
|
173
|
+
healed: false,
|
|
174
|
+
shouldRetryOriginal: false,
|
|
175
|
+
reason: decision.reason,
|
|
176
|
+
actionsRun,
|
|
177
|
+
errors
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
const { model, modelId, reason: modelReason } = resolveHealerModel(config);
|
|
181
|
+
if (!model) {
|
|
182
|
+
return baseOutcome({
|
|
183
|
+
mode,
|
|
184
|
+
started,
|
|
185
|
+
healed: false,
|
|
186
|
+
shouldRetryOriginal: false,
|
|
187
|
+
reason: modelReason ?? "no_model",
|
|
188
|
+
actionsRun,
|
|
189
|
+
errors
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
const testContext = {
|
|
193
|
+
testFile: execCtx.testContext?.testFile,
|
|
194
|
+
testTitle: execCtx.testContext?.testTitle,
|
|
195
|
+
testSource: execCtx.testContext?.testSource ? sanitizeString(execCtx.testContext.testSource) : void 0,
|
|
196
|
+
currentStep: `${failure.action ?? "unknown"}(${failure.target ?? "unknown"})`,
|
|
197
|
+
expectedAfter: execCtx.testContext?.expectedAfter,
|
|
198
|
+
action: failure.action ?? "unknown",
|
|
199
|
+
target: failure.target,
|
|
200
|
+
errorMessage: failure.errorMessage
|
|
201
|
+
};
|
|
202
|
+
const toolset = createAgenticTools({
|
|
203
|
+
page: execCtx.page,
|
|
204
|
+
config,
|
|
205
|
+
actionsRun,
|
|
206
|
+
debug: config.debug
|
|
207
|
+
});
|
|
208
|
+
let completionReason;
|
|
209
|
+
try {
|
|
210
|
+
const abort = AbortSignal.timeout(config.healTimeoutMs);
|
|
211
|
+
const result = await streamText({
|
|
212
|
+
model,
|
|
213
|
+
system: buildSystemPrompt(testContext, config),
|
|
214
|
+
messages: buildInitialMessages(testContext, failure, initialContext),
|
|
215
|
+
tools: toolset.tools,
|
|
216
|
+
stopWhen: stepCountIs(Math.max(1, config.maxActions)),
|
|
217
|
+
abortSignal: abort,
|
|
218
|
+
maxRetries: 0,
|
|
219
|
+
onStepFinish: (step) => {
|
|
220
|
+
if (config.debug) {
|
|
221
|
+
console.log(
|
|
222
|
+
`[canary][debug] step finish: finishReason=${step.finishReason}; toolCalls=${step.toolCalls?.length ?? 0}; toolResults=${step.toolResults?.length ?? 0}`
|
|
223
|
+
);
|
|
224
|
+
}
|
|
225
|
+
if (logTools) {
|
|
226
|
+
logToolStep(step);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
});
|
|
230
|
+
for await (const _ of result.fullStream) {
|
|
231
|
+
if (toolset.isComplete()) {
|
|
232
|
+
completionReason = toolset.getCompletionReason();
|
|
233
|
+
if (config.debug) {
|
|
234
|
+
console.log(`[canary][debug] agent marked complete: ${completionReason}`);
|
|
235
|
+
}
|
|
236
|
+
break;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
} catch (error) {
|
|
240
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
241
|
+
errors.push(message);
|
|
242
|
+
if (config.debug) {
|
|
243
|
+
console.log(`[canary][debug] agent error: ${message}`);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
const healed = toolset.isComplete();
|
|
247
|
+
return {
|
|
248
|
+
healed,
|
|
249
|
+
shouldRetryOriginal: false,
|
|
250
|
+
// Agent-controlled completion, no retry needed
|
|
251
|
+
actionsRun,
|
|
252
|
+
timedOut: Date.now() - started > config.healTimeoutMs,
|
|
253
|
+
durationMs: Date.now() - started,
|
|
254
|
+
errors,
|
|
255
|
+
mode,
|
|
256
|
+
reason: completionReason ?? (healed ? "agent_healed" : "agent_incomplete"),
|
|
257
|
+
modelId,
|
|
258
|
+
summary: toolset.getSummary()
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
function createAgenticTools(params) {
|
|
262
|
+
const { page, config, actionsRun, debug } = params;
|
|
263
|
+
let complete = false;
|
|
264
|
+
let completionReason;
|
|
265
|
+
let completionSummary;
|
|
266
|
+
const maxPayloadBytes = Math.max(1e3, config.maxPayloadBytes || 6e4);
|
|
267
|
+
const log = (msg) => {
|
|
268
|
+
if (debug) {
|
|
269
|
+
console.log(`[canary][debug] ${msg}`);
|
|
270
|
+
}
|
|
271
|
+
};
|
|
272
|
+
const track = (action) => {
|
|
273
|
+
actionsRun.push(action);
|
|
274
|
+
};
|
|
275
|
+
const tools = {
|
|
276
|
+
// === Core Action Tools ===
|
|
277
|
+
click: tool({
|
|
278
|
+
description: "Click an element by its visible text",
|
|
279
|
+
inputSchema: z.object({ text: z.string().describe("The visible text to click") }),
|
|
280
|
+
execute: async ({ text }) => {
|
|
281
|
+
track(`click:${text}`);
|
|
282
|
+
log(`click called with text="${text}"`);
|
|
283
|
+
if (config.readOnly) return { clicked: "", error: "read_only_mode" };
|
|
284
|
+
if (config.dryRun) return { clicked: text, mode: "dry-run" };
|
|
285
|
+
if (!page?.getByText) return { clicked: "", error: "Page unavailable" };
|
|
286
|
+
try {
|
|
287
|
+
const el = page.getByText(text);
|
|
288
|
+
if (el?.scrollIntoViewIfNeeded) {
|
|
289
|
+
await el.scrollIntoViewIfNeeded({ timeout: 2e3 });
|
|
290
|
+
}
|
|
291
|
+
const clickable = el;
|
|
292
|
+
if (clickable?.click) {
|
|
293
|
+
await clickable.click({ timeout: 5e3 });
|
|
294
|
+
}
|
|
295
|
+
log(`click succeeded for "${text}"`);
|
|
296
|
+
return { clicked: text };
|
|
297
|
+
} catch (err) {
|
|
298
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
299
|
+
log(`click error: ${msg}`);
|
|
300
|
+
return { clicked: "", error: msg };
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
}),
|
|
304
|
+
click_selector: tool({
|
|
305
|
+
description: "Click an element by CSS or Playwright selector",
|
|
306
|
+
inputSchema: z.object({ selector: z.string().describe("CSS or Playwright selector") }),
|
|
307
|
+
execute: async ({ selector }) => {
|
|
308
|
+
track(`click_selector:${selector}`);
|
|
309
|
+
log(`click_selector called with selector="${selector}"`);
|
|
310
|
+
if (config.readOnly) return { clicked: "", error: "read_only_mode" };
|
|
311
|
+
if (config.dryRun) return { clicked: selector, mode: "dry-run" };
|
|
312
|
+
if (!page?.locator) return { clicked: "", error: "Page unavailable" };
|
|
313
|
+
try {
|
|
314
|
+
const el = page.locator(selector);
|
|
315
|
+
if (el?.scrollIntoViewIfNeeded) {
|
|
316
|
+
await el.scrollIntoViewIfNeeded({ timeout: 2e3 });
|
|
317
|
+
}
|
|
318
|
+
if (el?.click) {
|
|
319
|
+
await el.click({ timeout: 5e3 });
|
|
320
|
+
}
|
|
321
|
+
log(`click_selector succeeded for "${selector}"`);
|
|
322
|
+
return { clicked: selector };
|
|
323
|
+
} catch (err) {
|
|
324
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
325
|
+
log(`click_selector error: ${msg}`);
|
|
326
|
+
return { clicked: "", error: msg };
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
}),
|
|
330
|
+
fill: tool({
|
|
331
|
+
description: "Fill an input field with a value using CSS selector (clears existing content)",
|
|
332
|
+
inputSchema: z.object({
|
|
333
|
+
selector: z.string().describe('CSS selector for the input (e.g. input[placeholder="..."])'),
|
|
334
|
+
value: z.string().describe("Value to fill")
|
|
335
|
+
}),
|
|
336
|
+
execute: async ({ selector, value }) => {
|
|
337
|
+
track(`fill:${selector}`);
|
|
338
|
+
log(`fill called with selector="${selector}" value="${value}"`);
|
|
339
|
+
if (config.readOnly) return { filled: "", error: "read_only_mode" };
|
|
340
|
+
if (config.dryRun) return { filled: selector, mode: "dry-run" };
|
|
341
|
+
if (!page?.locator) return { filled: "", error: "Page unavailable" };
|
|
342
|
+
try {
|
|
343
|
+
const el = page.locator(selector);
|
|
344
|
+
if (el?.fill) {
|
|
345
|
+
await el.fill(value, { timeout: 5e3 });
|
|
346
|
+
}
|
|
347
|
+
log(`fill succeeded`);
|
|
348
|
+
return { filled: selector, value };
|
|
349
|
+
} catch (err) {
|
|
350
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
351
|
+
log(`fill error: ${msg}`);
|
|
352
|
+
return { filled: "", error: msg };
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
}),
|
|
356
|
+
fill_by_name: tool({
|
|
357
|
+
description: 'Fill a textbox by its accessible name (shown in snapshot as textbox "NAME"). This is the PREFERRED way to fill inputs - use the name from the snapshot directly.',
|
|
358
|
+
inputSchema: z.object({
|
|
359
|
+
name: z.string().describe('The accessible name of the textbox (from snapshot, e.g. "Enter 6-digit code")'),
|
|
360
|
+
value: z.string().describe("Value to fill")
|
|
361
|
+
}),
|
|
362
|
+
execute: async ({ name, value }) => {
|
|
363
|
+
track(`fill_by_name:${name}`);
|
|
364
|
+
log(`fill_by_name called with name="${name}" value="${value}"`);
|
|
365
|
+
if (config.readOnly) return { filled: "", error: "read_only_mode" };
|
|
366
|
+
if (config.dryRun) return { filled: name, mode: "dry-run" };
|
|
367
|
+
if (!page?.locator) return { filled: "", error: "Page unavailable" };
|
|
368
|
+
try {
|
|
369
|
+
const el = page.locator(`role=textbox[name="${name}"]`);
|
|
370
|
+
if (el?.fill) {
|
|
371
|
+
await el.fill(value, { timeout: 5e3 });
|
|
372
|
+
}
|
|
373
|
+
log(`fill_by_name succeeded`);
|
|
374
|
+
return { filled: name, value };
|
|
375
|
+
} catch (err) {
|
|
376
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
377
|
+
log(`fill_by_name error: ${msg}`);
|
|
378
|
+
return { filled: "", error: msg };
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}),
|
|
382
|
+
type: tool({
|
|
383
|
+
description: "Type text character by character (triggers key events)",
|
|
384
|
+
inputSchema: z.object({
|
|
385
|
+
selector: z.string().describe("CSS or Playwright selector"),
|
|
386
|
+
text: z.string().describe("Text to type")
|
|
387
|
+
}),
|
|
388
|
+
execute: async ({ selector, text }) => {
|
|
389
|
+
track(`type:${selector}`);
|
|
390
|
+
log(`type called with selector="${selector}" text="${text}"`);
|
|
391
|
+
if (config.readOnly) return { typed: "", error: "read_only_mode" };
|
|
392
|
+
if (config.dryRun) return { typed: text, mode: "dry-run" };
|
|
393
|
+
if (!page?.locator) return { typed: "", error: "Page unavailable" };
|
|
394
|
+
try {
|
|
395
|
+
const el = page.locator(selector);
|
|
396
|
+
if (el?.pressSequentially) {
|
|
397
|
+
await el.pressSequentially(text);
|
|
398
|
+
}
|
|
399
|
+
log(`type succeeded`);
|
|
400
|
+
return { typed: text, into: selector };
|
|
401
|
+
} catch (err) {
|
|
402
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
403
|
+
log(`type error: ${msg}`);
|
|
404
|
+
return { typed: "", error: msg };
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
}),
|
|
408
|
+
press_key: tool({
|
|
409
|
+
description: "Press a keyboard key (Enter, Tab, Escape, ArrowDown, etc.)",
|
|
410
|
+
inputSchema: z.object({ key: z.string().describe("Key to press") }),
|
|
411
|
+
execute: async ({ key }) => {
|
|
412
|
+
track(`press_key:${key}`);
|
|
413
|
+
log(`press_key called with key="${key}"`);
|
|
414
|
+
if (config.readOnly) return { pressed: "", error: "read_only_mode" };
|
|
415
|
+
if (config.dryRun) return { pressed: key, mode: "dry-run" };
|
|
416
|
+
if (!page?.keyboard?.press) return { pressed: "", error: "Page unavailable" };
|
|
417
|
+
try {
|
|
418
|
+
await page.keyboard.press(key);
|
|
419
|
+
log(`press_key succeeded`);
|
|
420
|
+
return { pressed: key };
|
|
421
|
+
} catch (err) {
|
|
422
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
423
|
+
log(`press_key error: ${msg}`);
|
|
424
|
+
return { pressed: "", error: msg };
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
}),
|
|
428
|
+
hover: tool({
|
|
429
|
+
description: "Hover over an element",
|
|
430
|
+
inputSchema: z.object({ selector: z.string().describe("CSS or Playwright selector") }),
|
|
431
|
+
execute: async ({ selector }) => {
|
|
432
|
+
track(`hover:${selector}`);
|
|
433
|
+
log(`hover called with selector="${selector}"`);
|
|
434
|
+
if (config.readOnly) return { hovered: "", error: "read_only_mode" };
|
|
435
|
+
if (config.dryRun) return { hovered: selector, mode: "dry-run" };
|
|
436
|
+
if (!page?.locator) return { hovered: "", error: "Page unavailable" };
|
|
437
|
+
try {
|
|
438
|
+
const el = page.locator(selector);
|
|
439
|
+
if (el?.hover) {
|
|
440
|
+
await el.hover();
|
|
441
|
+
}
|
|
442
|
+
log(`hover succeeded`);
|
|
443
|
+
return { hovered: selector };
|
|
444
|
+
} catch (err) {
|
|
445
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
446
|
+
log(`hover error: ${msg}`);
|
|
447
|
+
return { hovered: "", error: msg };
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
}),
|
|
451
|
+
select_option: tool({
|
|
452
|
+
description: "Select an option from a dropdown",
|
|
453
|
+
inputSchema: z.object({
|
|
454
|
+
selector: z.string().describe("CSS or Playwright selector for the select element"),
|
|
455
|
+
value: z.string().describe("Value or label to select")
|
|
456
|
+
}),
|
|
457
|
+
execute: async ({ selector, value }) => {
|
|
458
|
+
track(`select_option:${selector}`);
|
|
459
|
+
log(`select_option called with selector="${selector}" value="${value}"`);
|
|
460
|
+
if (config.readOnly) return { selected: "", error: "read_only_mode" };
|
|
461
|
+
if (config.dryRun) return { selected: value, mode: "dry-run" };
|
|
462
|
+
if (!page?.locator) return { selected: "", error: "Page unavailable" };
|
|
463
|
+
try {
|
|
464
|
+
const el = page.locator(selector);
|
|
465
|
+
if (el?.selectOption) {
|
|
466
|
+
await el.selectOption(value);
|
|
467
|
+
}
|
|
468
|
+
log(`select_option succeeded`);
|
|
469
|
+
return { selected: value, from: selector };
|
|
470
|
+
} catch (err) {
|
|
471
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
472
|
+
log(`select_option error: ${msg}`);
|
|
473
|
+
return { selected: "", error: msg };
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
}),
|
|
477
|
+
// === Navigation & Waiting ===
|
|
478
|
+
wait: tool({
|
|
479
|
+
description: "Wait for a duration in milliseconds",
|
|
480
|
+
inputSchema: z.object({ ms: z.number().min(100).max(3e4).describe("Milliseconds to wait") }),
|
|
481
|
+
execute: async ({ ms }) => {
|
|
482
|
+
track(`wait:${ms}`);
|
|
483
|
+
log(`wait called with ms=${ms}`);
|
|
484
|
+
if (config.dryRun) return { waited: ms, mode: "dry-run" };
|
|
485
|
+
if (page?.waitForTimeout) {
|
|
486
|
+
await page.waitForTimeout(ms);
|
|
487
|
+
} else {
|
|
488
|
+
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
489
|
+
}
|
|
490
|
+
log(`wait completed`);
|
|
491
|
+
return { waited: ms };
|
|
492
|
+
}
|
|
493
|
+
}),
|
|
494
|
+
wait_for_selector: tool({
|
|
495
|
+
description: "Wait for an element to reach a state (visible, hidden, attached, detached)",
|
|
496
|
+
inputSchema: z.object({
|
|
497
|
+
selector: z.string().describe("CSS or Playwright selector"),
|
|
498
|
+
state: z.enum(["visible", "hidden", "attached", "detached"]).optional().describe("State to wait for")
|
|
499
|
+
}),
|
|
500
|
+
execute: async ({ selector, state }) => {
|
|
501
|
+
track(`wait_for_selector:${selector}`);
|
|
502
|
+
log(`wait_for_selector called with selector="${selector}" state="${state ?? "visible"}"`);
|
|
503
|
+
if (config.dryRun) return { found: selector, mode: "dry-run" };
|
|
504
|
+
if (!page?.locator) return { found: "", error: "Page unavailable" };
|
|
505
|
+
try {
|
|
506
|
+
const el = page.locator(selector);
|
|
507
|
+
if (el?.waitFor) {
|
|
508
|
+
await el.waitFor({ state: state ?? "visible" });
|
|
509
|
+
}
|
|
510
|
+
log(`wait_for_selector succeeded`);
|
|
511
|
+
return { found: selector, state: state ?? "visible" };
|
|
512
|
+
} catch (err) {
|
|
513
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
514
|
+
log(`wait_for_selector error: ${msg}`);
|
|
515
|
+
return { found: "", error: msg };
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
}),
|
|
519
|
+
navigate: tool({
|
|
520
|
+
description: "Navigate to a URL",
|
|
521
|
+
inputSchema: z.object({ url: z.string().describe("URL to navigate to") }),
|
|
522
|
+
execute: async ({ url }) => {
|
|
523
|
+
track(`navigate:${url}`);
|
|
524
|
+
log(`navigate called with url="${url}"`);
|
|
525
|
+
if (config.readOnly) return { navigated: "", error: "read_only_mode" };
|
|
526
|
+
if (config.dryRun) return { navigated: url, mode: "dry-run" };
|
|
527
|
+
if (!page?.goto) return { navigated: "", error: "Page unavailable" };
|
|
528
|
+
try {
|
|
529
|
+
await page.goto(url);
|
|
530
|
+
log(`navigate succeeded`);
|
|
531
|
+
return { navigated: url };
|
|
532
|
+
} catch (err) {
|
|
533
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
534
|
+
log(`navigate error: ${msg}`);
|
|
535
|
+
return { navigated: "", error: msg };
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
}),
|
|
539
|
+
go_back: tool({
|
|
540
|
+
description: "Go back in browser history",
|
|
541
|
+
inputSchema: z.object({}),
|
|
542
|
+
execute: async () => {
|
|
543
|
+
track("go_back");
|
|
544
|
+
log(`go_back called`);
|
|
545
|
+
if (config.readOnly) return { action: "", error: "read_only_mode" };
|
|
546
|
+
if (config.dryRun) return { action: "back", mode: "dry-run" };
|
|
547
|
+
if (!page?.goBack) return { action: "", error: "Page unavailable" };
|
|
548
|
+
try {
|
|
549
|
+
await page.goBack();
|
|
550
|
+
log(`go_back succeeded`);
|
|
551
|
+
return { action: "back" };
|
|
552
|
+
} catch (err) {
|
|
553
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
554
|
+
log(`go_back error: ${msg}`);
|
|
555
|
+
return { action: "", error: msg };
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
}),
|
|
559
|
+
reload: tool({
|
|
560
|
+
description: "Reload the current page",
|
|
561
|
+
inputSchema: z.object({}),
|
|
562
|
+
execute: async () => {
|
|
563
|
+
track("reload");
|
|
564
|
+
log(`reload called`);
|
|
565
|
+
if (config.readOnly) return { action: "", error: "read_only_mode" };
|
|
566
|
+
if (config.dryRun) return { action: "reload", mode: "dry-run" };
|
|
567
|
+
if (!page?.reload) return { action: "", error: "Page unavailable" };
|
|
568
|
+
try {
|
|
569
|
+
await page.reload();
|
|
570
|
+
log(`reload succeeded`);
|
|
571
|
+
return { action: "reload" };
|
|
572
|
+
} catch (err) {
|
|
573
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
574
|
+
log(`reload error: ${msg}`);
|
|
575
|
+
return { action: "", error: msg };
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
}),
|
|
579
|
+
// === Inspection Tools ===
|
|
580
|
+
snapshot: tool({
|
|
581
|
+
description: "Get the accessibility tree of the current page to understand its structure. ALWAYS call this first!",
|
|
582
|
+
inputSchema: z.object({}),
|
|
583
|
+
execute: async () => {
|
|
584
|
+
track("snapshot");
|
|
585
|
+
log(`snapshot called`);
|
|
586
|
+
if (config.dryRun) return { tree: null, mode: "dry-run" };
|
|
587
|
+
if (!page) return { error: "Page unavailable" };
|
|
588
|
+
let tree;
|
|
589
|
+
let html;
|
|
590
|
+
try {
|
|
591
|
+
if (page.accessibility && typeof page.accessibility.snapshot === "function") {
|
|
592
|
+
tree = await page.accessibility.snapshot();
|
|
593
|
+
}
|
|
594
|
+
} catch (err) {
|
|
595
|
+
log(`accessibility snapshot error: ${err instanceof Error ? err.message : String(err)}`);
|
|
596
|
+
}
|
|
597
|
+
if (!tree && page.evaluate) {
|
|
598
|
+
try {
|
|
599
|
+
html = await page.evaluate("document.body.innerHTML");
|
|
600
|
+
if (html) {
|
|
601
|
+
log(`snapshot using HTML fallback`);
|
|
602
|
+
}
|
|
603
|
+
} catch (err) {
|
|
604
|
+
log(`HTML fallback error: ${err instanceof Error ? err.message : String(err)}`);
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
if (!tree && !html) {
|
|
608
|
+
return { error: "Could not get page content" };
|
|
609
|
+
}
|
|
610
|
+
log(`snapshot succeeded`);
|
|
611
|
+
const sanitizedTree = sanitizeUnknown(tree, maxPayloadBytes);
|
|
612
|
+
const sanitizedHtml = html ? truncate(sanitizeString(html), maxPayloadBytes) : void 0;
|
|
613
|
+
return { tree: sanitizedTree, html: sanitizedHtml };
|
|
614
|
+
}
|
|
615
|
+
}),
|
|
616
|
+
screenshot: tool({
|
|
617
|
+
description: "Capture a screenshot of the current page",
|
|
618
|
+
inputSchema: z.object({
|
|
619
|
+
fullPage: z.boolean().optional().describe("Whether to capture the full page")
|
|
620
|
+
}),
|
|
621
|
+
execute: async ({ fullPage }) => {
|
|
622
|
+
track("screenshot");
|
|
623
|
+
log(`screenshot called fullPage=${fullPage}`);
|
|
624
|
+
if (config.dryRun) return { screenshot: void 0, mode: "dry-run" };
|
|
625
|
+
if (!page?.screenshot) return { error: "Page unavailable" };
|
|
626
|
+
try {
|
|
627
|
+
const buffer = await page.screenshot({ fullPage: fullPage ?? false });
|
|
628
|
+
const b = typeof buffer === "string" ? Buffer.from(buffer) : Buffer.from(buffer);
|
|
629
|
+
const base64 = b.toString("base64");
|
|
630
|
+
if (b.byteLength > maxPayloadBytes || base64.length > maxPayloadBytes) {
|
|
631
|
+
return { error: "screenshot_too_large" };
|
|
632
|
+
}
|
|
633
|
+
const dataUrl = `data:image/png;base64,${truncate(base64, maxPayloadBytes)}`;
|
|
634
|
+
log(`screenshot succeeded`);
|
|
635
|
+
return { screenshot: dataUrl };
|
|
636
|
+
} catch (err) {
|
|
637
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
638
|
+
log(`screenshot error: ${msg}`);
|
|
639
|
+
return { error: msg };
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
}),
|
|
643
|
+
get_text: tool({
|
|
644
|
+
description: "Get the text content of an element",
|
|
645
|
+
inputSchema: z.object({ selector: z.string().describe("CSS or Playwright selector") }),
|
|
646
|
+
execute: async ({ selector }) => {
|
|
647
|
+
track(`get_text:${selector}`);
|
|
648
|
+
log(`get_text called with selector="${selector}"`);
|
|
649
|
+
if (config.dryRun) return { text: null, mode: "dry-run" };
|
|
650
|
+
if (!page?.locator) return { error: "Page unavailable" };
|
|
651
|
+
try {
|
|
652
|
+
const el = page.locator(selector);
|
|
653
|
+
const text = el?.textContent ? await el.textContent() : null;
|
|
654
|
+
const sanitized = text ? truncate(sanitizeString(text), maxPayloadBytes) : text;
|
|
655
|
+
log(`get_text succeeded: "${sanitized}"`);
|
|
656
|
+
return { text: sanitized };
|
|
657
|
+
} catch (err) {
|
|
658
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
659
|
+
log(`get_text error: ${msg}`);
|
|
660
|
+
return { error: msg };
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
}),
|
|
664
|
+
is_visible: tool({
|
|
665
|
+
description: "Check if an element is visible",
|
|
666
|
+
inputSchema: z.object({ selector: z.string().describe("CSS or Playwright selector") }),
|
|
667
|
+
execute: async ({ selector }) => {
|
|
668
|
+
track(`is_visible:${selector}`);
|
|
669
|
+
log(`is_visible called with selector="${selector}"`);
|
|
670
|
+
if (config.dryRun) return { visible: false, mode: "dry-run" };
|
|
671
|
+
if (!page?.locator) return { error: "Page unavailable" };
|
|
672
|
+
try {
|
|
673
|
+
const el = page.locator(selector);
|
|
674
|
+
const visible = el?.isVisible ? await el.isVisible() : false;
|
|
675
|
+
log(`is_visible succeeded: ${visible}`);
|
|
676
|
+
return { visible };
|
|
677
|
+
} catch (err) {
|
|
678
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
679
|
+
log(`is_visible error: ${msg}`);
|
|
680
|
+
return { error: msg };
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
}),
|
|
684
|
+
evaluate: tool({
|
|
685
|
+
description: "Run JavaScript in the page context",
|
|
686
|
+
inputSchema: z.object({ script: z.string().describe("JavaScript code to execute") }),
|
|
687
|
+
execute: async ({ script }) => {
|
|
688
|
+
track("evaluate");
|
|
689
|
+
log(`evaluate called`);
|
|
690
|
+
if (!config.allowEvaluate) return { error: "evaluate_disabled" };
|
|
691
|
+
if (violatesCodeDenylist(script)) return { error: "evaluate_blocked" };
|
|
692
|
+
if (config.dryRun) return { result: null, mode: "dry-run" };
|
|
693
|
+
if (!page?.evaluate) return { error: "Page unavailable" };
|
|
694
|
+
try {
|
|
695
|
+
const scriptSafe = truncate(script, maxPayloadBytes);
|
|
696
|
+
const result = await page.evaluate(scriptSafe);
|
|
697
|
+
log(`evaluate succeeded`);
|
|
698
|
+
return { result: sanitizeUnknown(result, maxPayloadBytes) };
|
|
699
|
+
} catch (err) {
|
|
700
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
701
|
+
log(`evaluate error: ${msg}`);
|
|
702
|
+
return { error: msg };
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
}),
|
|
706
|
+
// === Power Tool: Arbitrary Playwright Code ===
|
|
707
|
+
run_playwright_code: tool({
|
|
708
|
+
description: 'Execute arbitrary Playwright code. Has access to `page` object. Use for complex operations not covered by other tools. Example: `await page.getByRole("button", { name: "Submit" }).click();`',
|
|
709
|
+
inputSchema: z.object({
|
|
710
|
+
code: z.string().describe("Playwright code to execute (has access to `page` object)")
|
|
711
|
+
}),
|
|
712
|
+
execute: async ({ code }) => {
|
|
713
|
+
track("run_playwright_code");
|
|
714
|
+
log(`run_playwright_code called with code:
|
|
715
|
+
${code}`);
|
|
716
|
+
if (!config.allowRunCode) return { executed: false, error: "run_code_disabled" };
|
|
717
|
+
if (violatesCodeDenylist(code)) return { executed: false, error: "run_code_blocked" };
|
|
718
|
+
if (config.dryRun) return { executed: false, mode: "dry-run" };
|
|
719
|
+
if (!page) return { executed: false, error: "Page unavailable" };
|
|
720
|
+
try {
|
|
721
|
+
const boundedCode = truncate(code, maxPayloadBytes);
|
|
722
|
+
const fn = new Function("page", `return (async () => { ${boundedCode} })();`);
|
|
723
|
+
const result = await fn(page);
|
|
724
|
+
log(`run_playwright_code succeeded`);
|
|
725
|
+
return { executed: true, result: sanitizeUnknown(result, maxPayloadBytes) };
|
|
726
|
+
} catch (err) {
|
|
727
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
728
|
+
log(`run_playwright_code error: ${msg}`);
|
|
729
|
+
return { executed: false, error: msg };
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
}),
|
|
733
|
+
// === Completion Tool ===
|
|
734
|
+
mark_complete: tool({
|
|
735
|
+
description: "Call this when the browser is ready for the test to continue. You MUST call this when done healing!",
|
|
736
|
+
inputSchema: z.object({
|
|
737
|
+
reason: z.string().describe("Brief technical description of what was done"),
|
|
738
|
+
summary: z.string().describe(`One-line human-readable summary for the test report, e.g. "Clicked 'Send OTP' instead of 'Submit'" or "Filled email field by accessible name"`)
|
|
739
|
+
}),
|
|
740
|
+
execute: async ({ reason, summary }) => {
|
|
741
|
+
track("mark_complete");
|
|
742
|
+
log(`mark_complete called with reason="${reason}" summary="${summary}"`);
|
|
743
|
+
if (!config.dryRun) {
|
|
744
|
+
complete = true;
|
|
745
|
+
completionReason = reason;
|
|
746
|
+
completionSummary = summary;
|
|
747
|
+
}
|
|
748
|
+
return { complete: !config.dryRun, reason, summary, mode: config.dryRun ? "dry-run" : void 0 };
|
|
749
|
+
}
|
|
750
|
+
})
|
|
751
|
+
};
|
|
752
|
+
return {
|
|
753
|
+
tools,
|
|
754
|
+
isComplete: () => complete,
|
|
755
|
+
getCompletionReason: () => completionReason,
|
|
756
|
+
getSummary: () => completionSummary
|
|
757
|
+
};
|
|
758
|
+
}
|
|
759
|
+
function buildSystemPrompt(testContext, config) {
|
|
760
|
+
const testInfo = testContext.testFile ? `
|
|
761
|
+
## Test Context
|
|
762
|
+
File: ${testContext.testFile}
|
|
763
|
+
Test: "${testContext.testTitle ?? "unknown"}"
|
|
764
|
+
` : "";
|
|
765
|
+
const testSource = testContext.testSource ? `
|
|
766
|
+
### Full Test Code:
|
|
767
|
+
\`\`\`typescript
|
|
768
|
+
${testContext.testSource}
|
|
769
|
+
\`\`\`
|
|
770
|
+
` : "";
|
|
771
|
+
const expectedAfter = testContext.expectedAfter ? `
|
|
772
|
+
### What the Test Expects After This Step:
|
|
773
|
+
${testContext.expectedAfter}
|
|
774
|
+
` : "";
|
|
775
|
+
return `You are an AI test healer. A Playwright test step failed. Your job is to make the browser ready for the next step in the test **without violating the intent of the test**. Heal only the failing step; do not advance beyond it.
|
|
776
|
+
|
|
777
|
+
## Your Goal
|
|
778
|
+
Get the browser into the state the test expects, then call mark_complete.
|
|
779
|
+
${testInfo}${testSource}
|
|
780
|
+
### Current Step That Failed:
|
|
781
|
+
${testContext.currentStep}
|
|
782
|
+
${expectedAfter}
|
|
783
|
+
## The Failure
|
|
784
|
+
Action attempted: ${testContext.action}
|
|
785
|
+
Target: ${testContext.target ?? "unknown"}
|
|
786
|
+
Error: ${testContext.errorMessage ?? "unknown"}
|
|
787
|
+
|
|
788
|
+
## Test Intent and Discipline
|
|
789
|
+
- Adhere to the spirit of the test. If advancing would bypass a required validation or skip a missing field, you must not proceed\u2014report failure instead.
|
|
790
|
+
- Heal only this step. Avoid extra navigation or state changes unrelated to the intended action. If you take a detour, undo it before completion.
|
|
791
|
+
- If you cannot confidently achieve the expected postcondition for this step, do not mark complete; let the test fail.
|
|
792
|
+
|
|
793
|
+
## Tools Available
|
|
794
|
+
|
|
795
|
+
### Inspection (use first!)
|
|
796
|
+
- snapshot() - Get accessibility tree to see page structure
|
|
797
|
+
- screenshot() - Capture visual screenshot
|
|
798
|
+
- get_text(selector) - Get text content of an element
|
|
799
|
+
- is_visible(selector) - Check if element is visible
|
|
800
|
+
|
|
801
|
+
### Actions
|
|
802
|
+
- click(text) - Click element by visible text
|
|
803
|
+
- click_selector(selector) - Click by CSS/Playwright selector
|
|
804
|
+
- fill_by_name(name, value) - PREFERRED: Fill textbox by accessible name from snapshot (e.g. textbox "Enter 6-digit code")
|
|
805
|
+
- fill(selector, value) - Fill input by CSS selector (use placeholder attribute, NOT aria-label)
|
|
806
|
+
- type(selector, text) - Type text character by character
|
|
807
|
+
- press_key(key) - Press keyboard key (Enter, Tab, etc.)
|
|
808
|
+
- hover(selector) - Hover over element
|
|
809
|
+
- select_option(selector, value) - Select from dropdown
|
|
810
|
+
|
|
811
|
+
### Navigation & Waiting
|
|
812
|
+
- wait(ms) - Wait for a duration
|
|
813
|
+
- wait_for_selector(selector, state?) - Wait for element state
|
|
814
|
+
- navigate(url) - Navigate to URL
|
|
815
|
+
- go_back() - Go back in history
|
|
816
|
+
- reload() - Reload page
|
|
817
|
+
|
|
818
|
+
### Power Tool
|
|
819
|
+
- run_playwright_code(code) - Execute any Playwright code with \`page\` object
|
|
820
|
+
Example: \`await page.getByRole('button', { name: 'Submit' }).click();\`
|
|
821
|
+
|
|
822
|
+
### Completion (REQUIRED)
|
|
823
|
+
- mark_complete(reason, summary) - Call when browser is ready for test to continue
|
|
824
|
+
- reason: technical description of what was done
|
|
825
|
+
- summary: one-line human-readable summary for test report (e.g. "Clicked 'Send OTP' instead of 'Submit'")
|
|
826
|
+
|
|
827
|
+
## CRITICAL RULES - READ CAREFULLY
|
|
828
|
+
|
|
829
|
+
### Rule 1: YOU MUST PERFORM THE ACTION YOURSELF
|
|
830
|
+
The test tried to ${testContext.action}("${testContext.target ?? ""}") but it failed.
|
|
831
|
+
YOU must perform the equivalent action on the correct element.
|
|
832
|
+
DO NOT leave it for "the next step" - there is no next step until YOU complete the action.
|
|
833
|
+
|
|
834
|
+
### Rule 2: The workflow is ALWAYS:
|
|
835
|
+
1. snapshot() - understand the page
|
|
836
|
+
2. click() / fill() / type() - PERFORM THE ACTION on the correct element
|
|
837
|
+
3. mark_complete() - report what you did
|
|
838
|
+
|
|
839
|
+
### Rule 3: NEVER call mark_complete without performing an action
|
|
840
|
+
If mark_complete is called without a click/fill/type, the test WILL FAIL.
|
|
841
|
+
|
|
842
|
+
## Example 1: Test tried click("Submit") but button says "Send OTP"
|
|
843
|
+
CORRECT:
|
|
844
|
+
1. snapshot() \u2192 see "Send OTP" button exists
|
|
845
|
+
2. click("Send OTP") \u2192 CLICK THE BUTTON
|
|
846
|
+
3. mark_complete({reason: "clicked Send OTP button", summary: "Clicked 'Send OTP' instead of 'Submit'"})
|
|
847
|
+
|
|
848
|
+
WRONG (this will cause test failure):
|
|
849
|
+
1. snapshot() \u2192 see "Send OTP" button exists
|
|
850
|
+
2. mark_complete(...) \u2190 WRONG! No click performed!
|
|
851
|
+
|
|
852
|
+
## Example 2: Test tried fill with wrong placeholder
|
|
853
|
+
If snapshot shows: textbox "Enter 6-digit code" [ref=e37]
|
|
854
|
+
CORRECT:
|
|
855
|
+
1. snapshot() \u2192 see textbox "Enter 6-digit code"
|
|
856
|
+
2. fill_by_name("Enter 6-digit code", "111222") \u2192 USE THE NAME FROM SNAPSHOT
|
|
857
|
+
3. mark_complete({reason: "filled by accessible name", summary: "Filled 'Enter 6-digit code' field"})
|
|
858
|
+
|
|
859
|
+
## If healing would break test intent
|
|
860
|
+
- If the page is missing required UI (e.g., a field not present) or advancing would skip validation, do NOT hack around it. Report failure by not calling mark_complete.
|
|
861
|
+
- If you temporarily navigate or change state, undo it before completion so the test can continue from the intended point.
|
|
862
|
+
|
|
863
|
+
## Start now
|
|
864
|
+
1. Call snapshot()
|
|
865
|
+
2. Identify the correct element for the intended action
|
|
866
|
+
3. PERFORM THE ACTION (click, fill, type, etc.)
|
|
867
|
+
4. Call mark_complete with what you did
|
|
868
|
+
|
|
869
|
+
Mode: ${resolveMode(config)}.`;
|
|
870
|
+
}
|
|
871
|
+
function resolveMode(config) {
|
|
872
|
+
if (config.dryRun) return "dry-run";
|
|
873
|
+
if (config.warnOnly) return "warn";
|
|
874
|
+
return "full";
|
|
875
|
+
}
|
|
876
|
+
function baseOutcome({
|
|
877
|
+
started,
|
|
878
|
+
healed,
|
|
879
|
+
shouldRetryOriginal,
|
|
880
|
+
mode,
|
|
881
|
+
reason,
|
|
882
|
+
actionsRun = [],
|
|
883
|
+
errors = []
|
|
884
|
+
}) {
|
|
885
|
+
return {
|
|
886
|
+
healed,
|
|
887
|
+
shouldRetryOriginal,
|
|
888
|
+
actionsRun,
|
|
889
|
+
timedOut: false,
|
|
890
|
+
durationMs: Date.now() - started,
|
|
891
|
+
errors,
|
|
892
|
+
mode,
|
|
893
|
+
reason
|
|
894
|
+
};
|
|
895
|
+
}
|
|
896
|
+
function logToolStep(step) {
|
|
897
|
+
const color = {
|
|
898
|
+
magenta: "\x1B[35m",
|
|
899
|
+
cyan: "\x1B[36m",
|
|
900
|
+
yellow: "\x1B[33m",
|
|
901
|
+
green: "\x1B[32m",
|
|
902
|
+
reset: "\x1B[0m"
|
|
903
|
+
};
|
|
904
|
+
const calls = step.toolCalls ?? [];
|
|
905
|
+
const results = step.toolResults ?? [];
|
|
906
|
+
if (calls.length === 0 && results.length === 0) {
|
|
907
|
+
console.log(
|
|
908
|
+
`${color.magenta}[canary][tool] step${color.reset} (no tool calls/results; finish=${step.finishReason ?? "unknown"})`
|
|
909
|
+
);
|
|
910
|
+
}
|
|
911
|
+
for (const call of calls) {
|
|
912
|
+
const name = call.toolName ?? "unknown_tool";
|
|
913
|
+
const args = call.args ? truncate(safeJson(call.args), 300) : "";
|
|
914
|
+
console.log(
|
|
915
|
+
`${color.magenta}[canary][tool] call${color.reset} ${color.cyan}${name}${color.reset}${args ? ` args=${args}` : ""}`
|
|
916
|
+
);
|
|
917
|
+
}
|
|
918
|
+
for (const result of results) {
|
|
919
|
+
const name = result.toolName ?? "unknown_tool";
|
|
920
|
+
const res = result.result ? truncate(safeJson(result.result), 300) : void 0;
|
|
921
|
+
const err = result.error ? truncate(safeJson(result.error), 300) : void 0;
|
|
922
|
+
const statusColor = err ? color.yellow : color.green;
|
|
923
|
+
console.log(
|
|
924
|
+
`${color.magenta}[canary][tool] result${color.reset} ${color.cyan}${name}${color.reset}` + (res ? ` ${statusColor}->${color.reset} ${res}` : "") + (err ? ` ${color.yellow}error=${err}${color.reset}` : "")
|
|
925
|
+
);
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
function buildInitialMessages(testContext, failure, initial) {
|
|
929
|
+
const parts = [];
|
|
930
|
+
parts.push(
|
|
931
|
+
`Please heal this test step. Start by reviewing the snapshot below. Failing step: ${failure.action ?? "unknown"}(${failure.target ?? "unknown"}). Error: ${failure.errorMessage ?? "unknown"}.`
|
|
932
|
+
);
|
|
933
|
+
if (testContext.testSource) {
|
|
934
|
+
parts.push(
|
|
935
|
+
`Full test source (truncated/redacted):
|
|
936
|
+
\`\`\`typescript
|
|
937
|
+
${truncate(testContext.testSource, 15e3)}
|
|
938
|
+
\`\`\``
|
|
939
|
+
);
|
|
940
|
+
}
|
|
941
|
+
parts.push(`Test file: ${testContext.testFile ?? "unknown"}`);
|
|
942
|
+
parts.push(`Test title: ${testContext.testTitle ?? "unknown"}`);
|
|
943
|
+
if (initial.snapshot) {
|
|
944
|
+
parts.push(`Initial accessibility snapshot (sanitized):
|
|
945
|
+
\`\`\`json
|
|
946
|
+
${truncate(safeJson(initial.snapshot), 8e3)}
|
|
947
|
+
\`\`\``);
|
|
948
|
+
}
|
|
949
|
+
if (initial.html) {
|
|
950
|
+
parts.push(`HTML fallback (sanitized):
|
|
951
|
+
\`\`\`html
|
|
952
|
+
${truncate(initial.html, 4e3)}
|
|
953
|
+
\`\`\``);
|
|
954
|
+
}
|
|
955
|
+
if (initial.screenshot) {
|
|
956
|
+
parts.push(`Screenshot (base64 data URL, truncated): ${truncate(initial.screenshot, 12e3)}`);
|
|
957
|
+
}
|
|
958
|
+
parts.push(`Remember: perform only the intended step and mark_complete only when the postcondition for this step is truly met.`);
|
|
959
|
+
return [{ role: "user", content: parts.join("\n\n") }];
|
|
960
|
+
}
|
|
961
|
+
async function buildInitialPageContext(page, config) {
|
|
962
|
+
const maxPayloadBytes = Math.max(1e3, config.maxPayloadBytes || 6e4);
|
|
963
|
+
const result = {};
|
|
964
|
+
if (!page) return result;
|
|
965
|
+
try {
|
|
966
|
+
if (page.accessibility?.snapshot) {
|
|
967
|
+
const tree = await page.accessibility.snapshot();
|
|
968
|
+
if (tree) {
|
|
969
|
+
result.snapshot = sanitizeUnknown(tree, maxPayloadBytes);
|
|
970
|
+
}
|
|
971
|
+
}
|
|
972
|
+
} catch {
|
|
973
|
+
}
|
|
974
|
+
if (!result.snapshot && page.evaluate) {
|
|
975
|
+
try {
|
|
976
|
+
const html = await page.evaluate("document.body.innerHTML");
|
|
977
|
+
if (html) {
|
|
978
|
+
result.html = truncate(sanitizeString(html), maxPayloadBytes);
|
|
979
|
+
}
|
|
980
|
+
} catch {
|
|
981
|
+
}
|
|
982
|
+
}
|
|
983
|
+
if (config.visionEnabled && page.screenshot) {
|
|
984
|
+
try {
|
|
985
|
+
const buffer = await page.screenshot({ fullPage: false });
|
|
986
|
+
const b = typeof buffer === "string" ? Buffer.from(buffer) : Buffer.from(buffer);
|
|
987
|
+
const base64 = b.toString("base64");
|
|
988
|
+
if (b.byteLength <= maxPayloadBytes && base64.length <= maxPayloadBytes) {
|
|
989
|
+
result.screenshot = `data:image/png;base64,${truncate(base64, maxPayloadBytes)}`;
|
|
990
|
+
}
|
|
991
|
+
} catch {
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
return result;
|
|
995
|
+
}
|
|
996
|
+
function truncate(value, max) {
|
|
997
|
+
if (value.length <= max) return value;
|
|
998
|
+
return value.slice(0, max) + "...";
|
|
999
|
+
}
|
|
1000
|
+
function safeJson(value) {
|
|
1001
|
+
try {
|
|
1002
|
+
return JSON.stringify(value);
|
|
1003
|
+
} catch {
|
|
1004
|
+
return String(value);
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
function sanitizeString(value) {
|
|
1008
|
+
let result = value;
|
|
1009
|
+
for (const { regex, replacement } of REDACTION_PATTERNS) {
|
|
1010
|
+
result = result.replace(regex, replacement);
|
|
1011
|
+
}
|
|
1012
|
+
return result;
|
|
1013
|
+
}
|
|
1014
|
+
function sanitizeUnknown(value, maxPayloadBytes, seen = /* @__PURE__ */ new WeakSet()) {
|
|
1015
|
+
if (value === null || value === void 0) return value;
|
|
1016
|
+
if (typeof value === "string") {
|
|
1017
|
+
return truncate(sanitizeString(value), maxPayloadBytes);
|
|
1018
|
+
}
|
|
1019
|
+
if (typeof value === "number" || typeof value === "boolean") return value;
|
|
1020
|
+
if (typeof value === "object") {
|
|
1021
|
+
if (seen.has(value)) return "[REDACTED_CYCLE]";
|
|
1022
|
+
seen.add(value);
|
|
1023
|
+
if (Array.isArray(value)) {
|
|
1024
|
+
return value.slice(0, 50).map((v) => sanitizeUnknown(v, maxPayloadBytes, seen));
|
|
1025
|
+
}
|
|
1026
|
+
const out = {};
|
|
1027
|
+
const entries = Object.entries(value).slice(0, 50);
|
|
1028
|
+
for (const [k, v] of entries) {
|
|
1029
|
+
out[k] = sanitizeUnknown(v, maxPayloadBytes, seen);
|
|
1030
|
+
}
|
|
1031
|
+
return out;
|
|
1032
|
+
}
|
|
1033
|
+
return "[REDACTED_UNKNOWN]";
|
|
1034
|
+
}
|
|
1035
|
+
function violatesCodeDenylist(code) {
|
|
1036
|
+
return CODE_DENY_PATTERNS.some((p) => p.test(code));
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
// src/runner/state.ts
|
|
1040
|
+
import fs2 from "fs";
|
|
1041
|
+
import path3 from "path";
|
|
1042
|
+
function getEventLog() {
|
|
1043
|
+
if (!globalThis.__CANARY_EVENTS) {
|
|
1044
|
+
globalThis.__CANARY_EVENTS = [];
|
|
1045
|
+
}
|
|
1046
|
+
return globalThis.__CANARY_EVENTS;
|
|
1047
|
+
}
|
|
1048
|
+
function setEventLogPath(path4) {
|
|
1049
|
+
globalThis.__CANARY_EVENT_LOG_PATH = path4;
|
|
1050
|
+
}
|
|
1051
|
+
function getEventLogPath() {
|
|
1052
|
+
return globalThis.__CANARY_EVENT_LOG_PATH;
|
|
1053
|
+
}
|
|
1054
|
+
function recordHealingEvent(event) {
|
|
1055
|
+
loadCanaryEnv();
|
|
1056
|
+
const log = getEventLog();
|
|
1057
|
+
const entry = {
|
|
1058
|
+
...event,
|
|
1059
|
+
workerId: process.env.TEST_WORKER_INDEX,
|
|
1060
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1061
|
+
};
|
|
1062
|
+
log.push(entry);
|
|
1063
|
+
appendEvent(entry);
|
|
1064
|
+
}
|
|
1065
|
+
function markPatched() {
|
|
1066
|
+
globalThis.__CANARY_PATCHED = true;
|
|
1067
|
+
}
|
|
1068
|
+
function alreadyPatched() {
|
|
1069
|
+
return globalThis.__CANARY_PATCHED === true;
|
|
1070
|
+
}
|
|
1071
|
+
function appendEvent(event) {
|
|
1072
|
+
const logPath = getEventLogPath();
|
|
1073
|
+
if (!logPath) return;
|
|
1074
|
+
const config = loadCanaryConfig();
|
|
1075
|
+
if (!config.eventLoggingEnabled) return;
|
|
1076
|
+
try {
|
|
1077
|
+
fs2.mkdirSync(path3.dirname(logPath), { recursive: true });
|
|
1078
|
+
fs2.appendFileSync(logPath, JSON.stringify(event) + "\n");
|
|
1079
|
+
} catch {
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
// src/runner/instrumentation.ts
|
|
1084
|
+
var DEBUG = false;
|
|
1085
|
+
var requireFn = typeof __require !== "undefined" ? __require : createRequire(import.meta.url);
|
|
1086
|
+
var LOCATOR_ACTIONS = /* @__PURE__ */ new Set([
|
|
1087
|
+
"click",
|
|
1088
|
+
"dblclick",
|
|
1089
|
+
"fill",
|
|
1090
|
+
"check",
|
|
1091
|
+
"uncheck",
|
|
1092
|
+
"hover",
|
|
1093
|
+
"press",
|
|
1094
|
+
"type",
|
|
1095
|
+
"selectOption",
|
|
1096
|
+
"tap"
|
|
1097
|
+
]);
|
|
1098
|
+
var PAGE_ACTIONS = /* @__PURE__ */ new Set([
|
|
1099
|
+
"goto",
|
|
1100
|
+
"click",
|
|
1101
|
+
"dblclick",
|
|
1102
|
+
"fill",
|
|
1103
|
+
"check",
|
|
1104
|
+
"uncheck",
|
|
1105
|
+
"hover",
|
|
1106
|
+
"press",
|
|
1107
|
+
"type",
|
|
1108
|
+
"selectOption",
|
|
1109
|
+
"tap",
|
|
1110
|
+
"waitForSelector"
|
|
1111
|
+
]);
|
|
1112
|
+
var LOCATOR_FACTORIES = /* @__PURE__ */ new Set([
|
|
1113
|
+
"locator",
|
|
1114
|
+
"getByRole",
|
|
1115
|
+
"getByText",
|
|
1116
|
+
"getByLabel",
|
|
1117
|
+
"getByPlaceholder",
|
|
1118
|
+
"getByAltText",
|
|
1119
|
+
"getByTitle",
|
|
1120
|
+
"getByTestId",
|
|
1121
|
+
"frameLocator"
|
|
1122
|
+
]);
|
|
1123
|
+
var LOCATOR_CHAIN_METHODS = /* @__PURE__ */ new Set([
|
|
1124
|
+
"locator",
|
|
1125
|
+
"first",
|
|
1126
|
+
"last",
|
|
1127
|
+
"nth",
|
|
1128
|
+
"filter",
|
|
1129
|
+
"getByRole",
|
|
1130
|
+
"getByText",
|
|
1131
|
+
"getByLabel",
|
|
1132
|
+
"getByPlaceholder",
|
|
1133
|
+
"getByAltText",
|
|
1134
|
+
"getByTitle",
|
|
1135
|
+
"getByTestId"
|
|
1136
|
+
]);
|
|
1137
|
+
function installInstrumentation() {
|
|
1138
|
+
if (alreadyPatched()) return;
|
|
1139
|
+
const config = loadCanaryConfig();
|
|
1140
|
+
DEBUG = !!config.debug;
|
|
1141
|
+
if (!config.enabled) {
|
|
1142
|
+
markPatched();
|
|
1143
|
+
return;
|
|
1144
|
+
}
|
|
1145
|
+
setEventLogPath(config.eventLogPath);
|
|
1146
|
+
if (!ensurePlaywrightVersion(config.allowedPlaywrightVersion)) {
|
|
1147
|
+
recordHealingEvent({
|
|
1148
|
+
kind: "unknown",
|
|
1149
|
+
action: "playwright_version_mismatch",
|
|
1150
|
+
errorMessage: "Playwright version does not match CANARY_PLAYWRIGHT_VERSION",
|
|
1151
|
+
healed: false
|
|
1152
|
+
});
|
|
1153
|
+
markPatched();
|
|
1154
|
+
return;
|
|
1155
|
+
}
|
|
1156
|
+
hookPlaywrightModuleLoad();
|
|
1157
|
+
if (DEBUG) {
|
|
1158
|
+
console.log(`[canary][debug] instrumentation installed (eventLog=${config.eventLogPath})`);
|
|
1159
|
+
}
|
|
1160
|
+
recordHealingEvent({
|
|
1161
|
+
kind: "unknown",
|
|
1162
|
+
action: "instrumentation_ready",
|
|
1163
|
+
healed: false
|
|
1164
|
+
});
|
|
1165
|
+
markPatched();
|
|
1166
|
+
}
|
|
1167
|
+
function hookPlaywrightModuleLoad() {
|
|
1168
|
+
const mod = Module;
|
|
1169
|
+
const originalLoad = mod._canaryOriginalLoad ?? mod._load;
|
|
1170
|
+
if (!mod._canaryOriginalLoad) {
|
|
1171
|
+
mod._canaryOriginalLoad = originalLoad;
|
|
1172
|
+
}
|
|
1173
|
+
mod._load = function patchedLoad(request, parent, isMain) {
|
|
1174
|
+
if (request === "@playwright/test" || request === "playwright/test") {
|
|
1175
|
+
if (DEBUG) {
|
|
1176
|
+
console.log(`[canary][debug] patching playwright module (request=${request})`);
|
|
1177
|
+
}
|
|
1178
|
+
const real = originalLoad.call(this, request, parent, isMain);
|
|
1179
|
+
return createPatchedPlaywright(real);
|
|
1180
|
+
}
|
|
1181
|
+
return originalLoad.call(this, request, parent, isMain);
|
|
1182
|
+
};
|
|
1183
|
+
}
|
|
1184
|
+
function createPatchedPlaywright(real) {
|
|
1185
|
+
if (real.__canaryPatched) return real;
|
|
1186
|
+
const extended = real.test.extend({
|
|
1187
|
+
page: async ({ page }, use) => {
|
|
1188
|
+
recordHealingEvent({
|
|
1189
|
+
kind: "page",
|
|
1190
|
+
action: "fixture_initialized",
|
|
1191
|
+
healed: false
|
|
1192
|
+
});
|
|
1193
|
+
const wrappedPage = wrapPage(page);
|
|
1194
|
+
await use(wrappedPage);
|
|
1195
|
+
}
|
|
1196
|
+
});
|
|
1197
|
+
const patched = {
|
|
1198
|
+
...real,
|
|
1199
|
+
test: extended,
|
|
1200
|
+
expect: wrapExpect(real.expect)
|
|
1201
|
+
};
|
|
1202
|
+
patched.__canaryPatched = true;
|
|
1203
|
+
if (DEBUG) {
|
|
1204
|
+
console.log(`[canary][debug] playwright patched`);
|
|
1205
|
+
}
|
|
1206
|
+
return patched;
|
|
1207
|
+
}
|
|
1208
|
+
function ensurePlaywrightVersion(expected) {
|
|
1209
|
+
if (!expected) return true;
|
|
1210
|
+
try {
|
|
1211
|
+
const pkg = requireFn("playwright/package.json");
|
|
1212
|
+
if (pkg.version && pkg.version.startsWith(expected)) {
|
|
1213
|
+
return true;
|
|
1214
|
+
}
|
|
1215
|
+
console.warn(`[canary] Playwright version mismatch. Expected ${expected}, found ${pkg.version ?? "unknown"}`);
|
|
1216
|
+
return false;
|
|
1217
|
+
} catch {
|
|
1218
|
+
return true;
|
|
1219
|
+
}
|
|
1220
|
+
}
|
|
1221
|
+
function wrapExpect(expectImpl) {
|
|
1222
|
+
const wrapMatchers = (expectation, mode) => {
|
|
1223
|
+
return new Proxy(expectation ?? {}, {
|
|
1224
|
+
get(target, prop, receiver) {
|
|
1225
|
+
const value = Reflect.get(target, prop, receiver);
|
|
1226
|
+
if (typeof prop === "string" && typeof value === "function") {
|
|
1227
|
+
return (...args) => runMatcherWithHealing({ matcher: value, matcherName: prop, args, expectTarget: target, mode });
|
|
1228
|
+
}
|
|
1229
|
+
return value;
|
|
1230
|
+
}
|
|
1231
|
+
});
|
|
1232
|
+
};
|
|
1233
|
+
const proxy = new Proxy(expectImpl, {
|
|
1234
|
+
apply(target, thisArg, argArray) {
|
|
1235
|
+
const expectation = target.apply(thisArg, argArray);
|
|
1236
|
+
return wrapMatchers(expectation, "hard");
|
|
1237
|
+
},
|
|
1238
|
+
get(target, prop, receiver) {
|
|
1239
|
+
const value = Reflect.get(target, prop, receiver);
|
|
1240
|
+
if (prop === "soft" && typeof value === "function") {
|
|
1241
|
+
return (...args) => wrapMatchers(value.apply(target, args), "soft");
|
|
1242
|
+
}
|
|
1243
|
+
return value;
|
|
1244
|
+
}
|
|
1245
|
+
});
|
|
1246
|
+
return proxy;
|
|
1247
|
+
}
|
|
1248
|
+
function wrapPage(page) {
|
|
1249
|
+
return new Proxy(page, {
|
|
1250
|
+
get(target, prop, receiver) {
|
|
1251
|
+
const value = Reflect.get(target, prop, receiver);
|
|
1252
|
+
if (typeof prop === "string") {
|
|
1253
|
+
if (LOCATOR_FACTORIES.has(prop) && typeof value === "function") {
|
|
1254
|
+
return (...args) => {
|
|
1255
|
+
const locator = value.apply(target, args);
|
|
1256
|
+
return wrapLocator(locator);
|
|
1257
|
+
};
|
|
1258
|
+
}
|
|
1259
|
+
if (PAGE_ACTIONS.has(prop) && typeof value === "function") {
|
|
1260
|
+
return async (...args) => {
|
|
1261
|
+
return attemptWithHealing({
|
|
1262
|
+
kind: "page",
|
|
1263
|
+
action: prop,
|
|
1264
|
+
target: safeTargetString(target),
|
|
1265
|
+
locator: void 0,
|
|
1266
|
+
page: target,
|
|
1267
|
+
invoke: () => value.apply(target, args)
|
|
1268
|
+
});
|
|
1269
|
+
};
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
1272
|
+
return value;
|
|
1273
|
+
}
|
|
1274
|
+
});
|
|
1275
|
+
}
|
|
1276
|
+
function wrapLocator(locator) {
|
|
1277
|
+
if (!locator || typeof locator !== "object") return locator;
|
|
1278
|
+
return new Proxy(locator, {
|
|
1279
|
+
get(target, prop, receiver) {
|
|
1280
|
+
const value = Reflect.get(target, prop, receiver);
|
|
1281
|
+
if (typeof prop === "string" && LOCATOR_ACTIONS.has(prop) && typeof value === "function") {
|
|
1282
|
+
return async (...args) => {
|
|
1283
|
+
return attemptWithHealing({
|
|
1284
|
+
kind: "locator",
|
|
1285
|
+
action: prop,
|
|
1286
|
+
target: safeTargetString(target),
|
|
1287
|
+
locator: target,
|
|
1288
|
+
page: void 0,
|
|
1289
|
+
invoke: () => value.apply(target, args)
|
|
1290
|
+
});
|
|
1291
|
+
};
|
|
1292
|
+
}
|
|
1293
|
+
if (typeof prop === "string" && LOCATOR_CHAIN_METHODS.has(prop) && typeof value === "function") {
|
|
1294
|
+
return (...args) => wrapLocator(value.apply(target, args));
|
|
1295
|
+
}
|
|
1296
|
+
return value;
|
|
1297
|
+
}
|
|
1298
|
+
});
|
|
1299
|
+
}
|
|
1300
|
+
async function attemptWithHealing(ctx) {
|
|
1301
|
+
try {
|
|
1302
|
+
return await Promise.resolve(ctx.invoke());
|
|
1303
|
+
} catch (error) {
|
|
1304
|
+
const failure = buildFailureContext(ctx.kind, ctx.action, ctx.target, error);
|
|
1305
|
+
recordHealingEvent({ ...failure, healed: false });
|
|
1306
|
+
const decision = classifyFailure(failure);
|
|
1307
|
+
if (DEBUG) {
|
|
1308
|
+
console.log(`[canary][debug] failure intercepted: kind=${ctx.kind} action=${ctx.action} reason=${decision.healable ? decision.reason ?? "healable" : decision.reason}`);
|
|
1309
|
+
}
|
|
1310
|
+
if (!decision.healable) {
|
|
1311
|
+
throw error;
|
|
1312
|
+
}
|
|
1313
|
+
const outcome = await executeHealActions(decision, failure, {
|
|
1314
|
+
kind: ctx.kind,
|
|
1315
|
+
action: ctx.action,
|
|
1316
|
+
target: ctx.target,
|
|
1317
|
+
locator: isLocatorLike(ctx.locator) ? ctx.locator : void 0,
|
|
1318
|
+
page: isPageLike(ctx.page) ? ctx.page : void 0,
|
|
1319
|
+
testContext: {
|
|
1320
|
+
testFile: testContext?.testFile,
|
|
1321
|
+
testTitle: testContext?.testTitle,
|
|
1322
|
+
testSource: loadTestSource(testContext?.testFile)
|
|
1323
|
+
}
|
|
1324
|
+
});
|
|
1325
|
+
const testContext = getTestContext();
|
|
1326
|
+
const summaryBase = {
|
|
1327
|
+
...failure,
|
|
1328
|
+
healed: outcome.healed,
|
|
1329
|
+
strategy: "agentic",
|
|
1330
|
+
reason: outcome.reason ?? decision.reason,
|
|
1331
|
+
actions: actionsToEventItems(outcome.actionsRun),
|
|
1332
|
+
durationMs: outcome.durationMs,
|
|
1333
|
+
mode: outcome.mode,
|
|
1334
|
+
decision: decision.reason,
|
|
1335
|
+
modelId: outcome.modelId,
|
|
1336
|
+
summary: outcome.summary,
|
|
1337
|
+
testFile: testContext?.testFile,
|
|
1338
|
+
testTitle: testContext?.testTitle
|
|
1339
|
+
};
|
|
1340
|
+
if (outcome.healed && !outcome.shouldRetryOriginal) {
|
|
1341
|
+
recordHealingEvent({ ...summaryBase, healed: true });
|
|
1342
|
+
return void 0;
|
|
1343
|
+
}
|
|
1344
|
+
if (outcome.shouldRetryOriginal) {
|
|
1345
|
+
try {
|
|
1346
|
+
const retried = await Promise.resolve(ctx.invoke());
|
|
1347
|
+
if (DEBUG) {
|
|
1348
|
+
console.log(`[canary][debug] retry_original succeeded for action=${ctx.action}`);
|
|
1349
|
+
}
|
|
1350
|
+
recordHealingEvent({ ...summaryBase, healed: true });
|
|
1351
|
+
return retried;
|
|
1352
|
+
} catch (retryError) {
|
|
1353
|
+
const retryInfo = errorInfo(retryError);
|
|
1354
|
+
recordHealingEvent({ ...summaryBase, healed: false, errorMessage: retryInfo.message });
|
|
1355
|
+
if (DEBUG) {
|
|
1356
|
+
console.log(`[canary][debug] retry_original failed for action=${ctx.action}: ${retryInfo.message ?? retryError}`);
|
|
1357
|
+
}
|
|
1358
|
+
throw retryError;
|
|
1359
|
+
}
|
|
1360
|
+
}
|
|
1361
|
+
recordHealingEvent(summaryBase);
|
|
1362
|
+
throw error;
|
|
1363
|
+
}
|
|
1364
|
+
}
|
|
1365
|
+
async function runMatcherWithHealing(params) {
|
|
1366
|
+
const { matcher, matcherName, args, expectTarget, mode } = params;
|
|
1367
|
+
const invoke = () => matcher.apply(expectTarget, args);
|
|
1368
|
+
if (mode === "soft") {
|
|
1369
|
+
return Promise.resolve(invoke());
|
|
1370
|
+
}
|
|
1371
|
+
try {
|
|
1372
|
+
return await Promise.resolve(invoke());
|
|
1373
|
+
} catch (error) {
|
|
1374
|
+
const target = stringifyTarget(args?.[0]);
|
|
1375
|
+
const failure = buildFailureContext("expect", matcherName, target, error);
|
|
1376
|
+
recordHealingEvent({ ...failure, healed: false });
|
|
1377
|
+
const decision = classifyFailure(failure);
|
|
1378
|
+
if (!decision.healable) {
|
|
1379
|
+
throw error;
|
|
1380
|
+
}
|
|
1381
|
+
const outcome = await executeHealActions(decision, failure, {
|
|
1382
|
+
kind: "expect",
|
|
1383
|
+
action: matcherName,
|
|
1384
|
+
target,
|
|
1385
|
+
locator: isLocatorLike(args?.[0]) ? args[0] : void 0
|
|
1386
|
+
});
|
|
1387
|
+
const testContext = getTestContext();
|
|
1388
|
+
const summaryBase = {
|
|
1389
|
+
...failure,
|
|
1390
|
+
healed: false,
|
|
1391
|
+
strategy: "agentic",
|
|
1392
|
+
reason: outcome.reason ?? decision.reason,
|
|
1393
|
+
actions: actionsToEventItems(outcome.actionsRun),
|
|
1394
|
+
durationMs: outcome.durationMs,
|
|
1395
|
+
mode: outcome.mode,
|
|
1396
|
+
decision: decision.reason,
|
|
1397
|
+
modelId: outcome.modelId,
|
|
1398
|
+
summary: outcome.summary,
|
|
1399
|
+
testFile: testContext?.testFile,
|
|
1400
|
+
testTitle: testContext?.testTitle
|
|
1401
|
+
};
|
|
1402
|
+
if (outcome.shouldRetryOriginal) {
|
|
1403
|
+
try {
|
|
1404
|
+
const retried = await Promise.resolve(invoke());
|
|
1405
|
+
recordHealingEvent({ ...summaryBase, healed: true });
|
|
1406
|
+
return retried;
|
|
1407
|
+
} catch (retryError) {
|
|
1408
|
+
const retryInfo = errorInfo(retryError);
|
|
1409
|
+
recordHealingEvent({ ...summaryBase, healed: false, errorMessage: retryInfo.message });
|
|
1410
|
+
throw retryError;
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
recordHealingEvent(summaryBase);
|
|
1414
|
+
throw error;
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
function buildFailureContext(kind, action, target, error) {
|
|
1418
|
+
const info = errorInfo(error);
|
|
1419
|
+
return {
|
|
1420
|
+
kind,
|
|
1421
|
+
action,
|
|
1422
|
+
target,
|
|
1423
|
+
errorMessage: info.message,
|
|
1424
|
+
errorName: info.name,
|
|
1425
|
+
stack: info.stack
|
|
1426
|
+
};
|
|
1427
|
+
}
|
|
1428
|
+
function actionsToEventItems(actions) {
|
|
1429
|
+
return actions.map((action) => ({ type: action }));
|
|
1430
|
+
}
|
|
1431
|
+
function errorInfo(error) {
|
|
1432
|
+
if (error instanceof Error) {
|
|
1433
|
+
return { message: error.message, name: error.name, stack: error.stack };
|
|
1434
|
+
}
|
|
1435
|
+
return { message: typeof error === "string" ? error : JSON.stringify(error) };
|
|
1436
|
+
}
|
|
1437
|
+
function isLocatorLike(candidate) {
|
|
1438
|
+
return Boolean(candidate && typeof candidate === "object" && "scrollIntoViewIfNeeded" in candidate);
|
|
1439
|
+
}
|
|
1440
|
+
function isPageLike(candidate) {
|
|
1441
|
+
return Boolean(candidate && typeof candidate === "object" && "waitForTimeout" in candidate);
|
|
1442
|
+
}
|
|
1443
|
+
function safeTargetString(target) {
|
|
1444
|
+
try {
|
|
1445
|
+
if (typeof target === "object" && target !== null && "toString" in target) {
|
|
1446
|
+
const s = String(target.toString());
|
|
1447
|
+
if (s && s !== "[object Object]") return s;
|
|
1448
|
+
}
|
|
1449
|
+
} catch {
|
|
1450
|
+
}
|
|
1451
|
+
return void 0;
|
|
1452
|
+
}
|
|
1453
|
+
function loadTestSource(filePath) {
|
|
1454
|
+
if (!filePath) return void 0;
|
|
1455
|
+
try {
|
|
1456
|
+
return requireFn("fs").readFileSync(filePath, "utf-8");
|
|
1457
|
+
} catch {
|
|
1458
|
+
return void 0;
|
|
1459
|
+
}
|
|
1460
|
+
}
|
|
1461
|
+
function stringifyTarget(candidate) {
|
|
1462
|
+
if (!candidate) return void 0;
|
|
1463
|
+
if (typeof candidate === "string") return candidate;
|
|
1464
|
+
if (typeof candidate === "object") {
|
|
1465
|
+
if ("selector" in candidate && typeof candidate.selector === "string") {
|
|
1466
|
+
return String(candidate.selector);
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
return safeTargetString(candidate);
|
|
1470
|
+
}
|
|
1471
|
+
function getTestContext() {
|
|
1472
|
+
try {
|
|
1473
|
+
const playwright = requireFn("@playwright/test");
|
|
1474
|
+
if (playwright?.test?.info) {
|
|
1475
|
+
const info = playwright.test.info();
|
|
1476
|
+
if (info) {
|
|
1477
|
+
return {
|
|
1478
|
+
testFile: info.file,
|
|
1479
|
+
testTitle: info.title
|
|
1480
|
+
};
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1483
|
+
} catch {
|
|
1484
|
+
}
|
|
1485
|
+
return void 0;
|
|
1486
|
+
}
|
|
1487
|
+
|
|
1488
|
+
// src/runner/preload.ts
|
|
1489
|
+
getEventLog();
|
|
1490
|
+
installInstrumentation();
|
|
1491
|
+
//# sourceMappingURL=preload.js.map
|