@stablyai/playwright-base 0.1.7 → 0.1.8-next.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +506 -201
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.mts +6 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.mjs +506 -201
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -67,7 +67,7 @@ var isObject = (value) => {
|
|
|
67
67
|
// src/ai/metadata.ts
|
|
68
68
|
var SDK_METADATA_HEADERS = {
|
|
69
69
|
"X-Client-Name": "stably-playwright-sdk-js",
|
|
70
|
-
"X-Client-Version": "0.1.
|
|
70
|
+
"X-Client-Version": "0.1.8-next.2"
|
|
71
71
|
};
|
|
72
72
|
|
|
73
73
|
// src/ai/extract.ts
|
|
@@ -167,143 +167,6 @@ function createExtract(pageOrLocator) {
|
|
|
167
167
|
var createLocatorExtract = (locator) => createExtract(locator);
|
|
168
168
|
var createPageExtract = (page) => createExtract(page);
|
|
169
169
|
|
|
170
|
-
// src/playwright-augment/methods/agent.ts
|
|
171
|
-
function createAgentStub() {
|
|
172
|
-
return async (prompt, options) => {
|
|
173
|
-
requireApiKey();
|
|
174
|
-
void prompt;
|
|
175
|
-
void options;
|
|
176
|
-
return { success: true };
|
|
177
|
-
};
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
// src/playwright-augment/augment.ts
|
|
181
|
-
var LOCATOR_PATCHED = Symbol.for("stably.playwright.locatorPatched");
|
|
182
|
-
var LOCATOR_DESCRIBE_WRAPPED = Symbol.for(
|
|
183
|
-
"stably.playwright.locatorDescribeWrapped"
|
|
184
|
-
);
|
|
185
|
-
var PAGE_PATCHED = Symbol.for("stably.playwright.pagePatched");
|
|
186
|
-
var CONTEXT_PATCHED = Symbol.for("stably.playwright.contextPatched");
|
|
187
|
-
var BROWSER_PATCHED = Symbol.for("stably.playwright.browserPatched");
|
|
188
|
-
var BROWSER_TYPE_PATCHED = Symbol.for("stably.playwright.browserTypePatched");
|
|
189
|
-
function defineHiddenProperty(target, key, value) {
|
|
190
|
-
Object.defineProperty(target, key, {
|
|
191
|
-
value,
|
|
192
|
-
enumerable: false,
|
|
193
|
-
configurable: true,
|
|
194
|
-
writable: true
|
|
195
|
-
});
|
|
196
|
-
}
|
|
197
|
-
function augmentLocator(locator) {
|
|
198
|
-
if (locator[LOCATOR_PATCHED]) {
|
|
199
|
-
return locator;
|
|
200
|
-
}
|
|
201
|
-
defineHiddenProperty(locator, "extract", createLocatorExtract(locator));
|
|
202
|
-
const markerTarget = locator;
|
|
203
|
-
if (typeof locator.describe === "function" && !markerTarget[LOCATOR_DESCRIBE_WRAPPED]) {
|
|
204
|
-
const originalDescribe = locator.describe.bind(locator);
|
|
205
|
-
locator.describe = (description, options) => {
|
|
206
|
-
void options;
|
|
207
|
-
const result = originalDescribe(description);
|
|
208
|
-
return result ? augmentLocator(result) : result;
|
|
209
|
-
};
|
|
210
|
-
defineHiddenProperty(locator, LOCATOR_DESCRIBE_WRAPPED, true);
|
|
211
|
-
}
|
|
212
|
-
defineHiddenProperty(locator, LOCATOR_PATCHED, true);
|
|
213
|
-
return locator;
|
|
214
|
-
}
|
|
215
|
-
function augmentPage(page) {
|
|
216
|
-
if (page[PAGE_PATCHED]) {
|
|
217
|
-
return page;
|
|
218
|
-
}
|
|
219
|
-
const originalLocator = page.locator.bind(page);
|
|
220
|
-
page.locator = (...args) => {
|
|
221
|
-
const locator = originalLocator(...args);
|
|
222
|
-
return augmentLocator(locator);
|
|
223
|
-
};
|
|
224
|
-
defineHiddenProperty(page, "extract", createPageExtract(page));
|
|
225
|
-
defineHiddenProperty(page, PAGE_PATCHED, true);
|
|
226
|
-
return page;
|
|
227
|
-
}
|
|
228
|
-
function augmentBrowserContext(context) {
|
|
229
|
-
if (context[CONTEXT_PATCHED]) {
|
|
230
|
-
return context;
|
|
231
|
-
}
|
|
232
|
-
const originalNewPage = context.newPage.bind(context);
|
|
233
|
-
context.newPage = async (...args) => {
|
|
234
|
-
const page = await originalNewPage(...args);
|
|
235
|
-
return augmentPage(page);
|
|
236
|
-
};
|
|
237
|
-
const originalPages = context.pages?.bind(context);
|
|
238
|
-
if (originalPages) {
|
|
239
|
-
context.pages = () => originalPages().map(
|
|
240
|
-
(page) => augmentPage(page)
|
|
241
|
-
);
|
|
242
|
-
}
|
|
243
|
-
if (!context.agent) {
|
|
244
|
-
defineHiddenProperty(context, "agent", createAgentStub());
|
|
245
|
-
}
|
|
246
|
-
defineHiddenProperty(context, CONTEXT_PATCHED, true);
|
|
247
|
-
return context;
|
|
248
|
-
}
|
|
249
|
-
function augmentBrowser(browser) {
|
|
250
|
-
if (browser[BROWSER_PATCHED]) {
|
|
251
|
-
return browser;
|
|
252
|
-
}
|
|
253
|
-
const originalNewContext = browser.newContext.bind(browser);
|
|
254
|
-
browser.newContext = async (...args) => {
|
|
255
|
-
const context = await originalNewContext(...args);
|
|
256
|
-
return augmentBrowserContext(context);
|
|
257
|
-
};
|
|
258
|
-
const originalNewPage = browser.newPage.bind(browser);
|
|
259
|
-
browser.newPage = async (...args) => {
|
|
260
|
-
const page = await originalNewPage(...args);
|
|
261
|
-
return augmentPage(page);
|
|
262
|
-
};
|
|
263
|
-
const originalContexts = browser.contexts.bind(browser);
|
|
264
|
-
browser.contexts = () => originalContexts().map(
|
|
265
|
-
(context) => augmentBrowserContext(context)
|
|
266
|
-
);
|
|
267
|
-
if (!browser.agent) {
|
|
268
|
-
defineHiddenProperty(browser, "agent", createAgentStub());
|
|
269
|
-
}
|
|
270
|
-
defineHiddenProperty(browser, BROWSER_PATCHED, true);
|
|
271
|
-
return browser;
|
|
272
|
-
}
|
|
273
|
-
function augmentBrowserType(browserType) {
|
|
274
|
-
if (browserType[BROWSER_TYPE_PATCHED]) {
|
|
275
|
-
return browserType;
|
|
276
|
-
}
|
|
277
|
-
const originalLaunch = browserType.launch.bind(browserType);
|
|
278
|
-
browserType.launch = async (...args) => {
|
|
279
|
-
const browser = await originalLaunch(...args);
|
|
280
|
-
return augmentBrowser(browser);
|
|
281
|
-
};
|
|
282
|
-
const originalConnect = browserType.connect?.bind(browserType);
|
|
283
|
-
if (originalConnect) {
|
|
284
|
-
browserType.connect = async (...args) => {
|
|
285
|
-
const browser = await originalConnect(...args);
|
|
286
|
-
return augmentBrowser(browser);
|
|
287
|
-
};
|
|
288
|
-
}
|
|
289
|
-
const originalConnectOverCDP = browserType.connectOverCDP?.bind(browserType);
|
|
290
|
-
if (originalConnectOverCDP) {
|
|
291
|
-
browserType.connectOverCDP = async (...args) => {
|
|
292
|
-
const browser = await originalConnectOverCDP(...args);
|
|
293
|
-
return augmentBrowser(browser);
|
|
294
|
-
};
|
|
295
|
-
}
|
|
296
|
-
const originalLaunchPersistentContext = browserType.launchPersistentContext?.bind(browserType);
|
|
297
|
-
if (originalLaunchPersistentContext) {
|
|
298
|
-
browserType.launchPersistentContext = async (...args) => {
|
|
299
|
-
const context = await originalLaunchPersistentContext(...args);
|
|
300
|
-
return augmentBrowserContext(context);
|
|
301
|
-
};
|
|
302
|
-
}
|
|
303
|
-
defineHiddenProperty(browserType, BROWSER_TYPE_PATCHED, true);
|
|
304
|
-
return browserType;
|
|
305
|
-
}
|
|
306
|
-
|
|
307
170
|
// src/playwright-type-predicates.ts
|
|
308
171
|
function isPage(candidate) {
|
|
309
172
|
return typeof candidate === "object" && candidate !== null && typeof candidate.screenshot === "function" && typeof candidate.goto === "function";
|
|
@@ -312,63 +175,6 @@ function isLocator(candidate) {
|
|
|
312
175
|
return typeof candidate === "object" && candidate !== null && typeof candidate.screenshot === "function" && typeof candidate.nth === "function";
|
|
313
176
|
}
|
|
314
177
|
|
|
315
|
-
// src/ai/verify-prompt.ts
|
|
316
|
-
var PROMPT_ASSERTION_ENDPOINT = "https://api.stably.ai/internal/v1/assert";
|
|
317
|
-
var parseSuccessResponse = (value) => {
|
|
318
|
-
if (!isObject(value)) {
|
|
319
|
-
throw new Error("Verify prompt returned unexpected response shape");
|
|
320
|
-
}
|
|
321
|
-
const { success, reason } = value;
|
|
322
|
-
if (typeof success !== "boolean") {
|
|
323
|
-
throw new Error("Verify prompt returned unexpected response shape");
|
|
324
|
-
}
|
|
325
|
-
if (reason !== void 0 && typeof reason !== "string") {
|
|
326
|
-
throw new Error("Verify prompt returned unexpected response shape");
|
|
327
|
-
}
|
|
328
|
-
return {
|
|
329
|
-
success,
|
|
330
|
-
reason
|
|
331
|
-
};
|
|
332
|
-
};
|
|
333
|
-
var parseErrorResponse = (value) => {
|
|
334
|
-
if (!isObject(value)) {
|
|
335
|
-
return void 0;
|
|
336
|
-
}
|
|
337
|
-
const { error } = value;
|
|
338
|
-
return typeof error !== "string" ? void 0 : { error };
|
|
339
|
-
};
|
|
340
|
-
async function verifyPrompt({
|
|
341
|
-
prompt,
|
|
342
|
-
screenshot
|
|
343
|
-
}) {
|
|
344
|
-
const apiKey = requireApiKey();
|
|
345
|
-
const form = new FormData();
|
|
346
|
-
form.append("prompt", prompt);
|
|
347
|
-
const u8 = Uint8Array.from(screenshot);
|
|
348
|
-
const blob = new Blob([u8], { type: "image/png" });
|
|
349
|
-
form.append("image", blob, "screenshot.png");
|
|
350
|
-
const response = await fetch(PROMPT_ASSERTION_ENDPOINT, {
|
|
351
|
-
method: "POST",
|
|
352
|
-
headers: {
|
|
353
|
-
...SDK_METADATA_HEADERS,
|
|
354
|
-
Authorization: `Bearer ${apiKey}`
|
|
355
|
-
},
|
|
356
|
-
body: form
|
|
357
|
-
});
|
|
358
|
-
const parsed = await response.json().catch(() => void 0);
|
|
359
|
-
if (response.ok) {
|
|
360
|
-
const { success, reason } = parseSuccessResponse(parsed);
|
|
361
|
-
return {
|
|
362
|
-
pass: success,
|
|
363
|
-
reason
|
|
364
|
-
};
|
|
365
|
-
}
|
|
366
|
-
const err = parseErrorResponse(parsed);
|
|
367
|
-
throw new Error(
|
|
368
|
-
`Verify prompt failed (${response.status})${err ? `: ${err.error}` : ""}`
|
|
369
|
-
);
|
|
370
|
-
}
|
|
371
|
-
|
|
372
178
|
// ../../node_modules/.pnpm/pixelmatch@7.1.0/node_modules/pixelmatch/index.js
|
|
373
179
|
function pixelmatch(img1, img2, output, width, height, options = {}) {
|
|
374
180
|
const {
|
|
@@ -584,6 +390,17 @@ async function takeStableScreenshot(target, options) {
|
|
|
584
390
|
let previous;
|
|
585
391
|
const pollIntervals = [0, 100, 250, 500];
|
|
586
392
|
let isFirstIteration = true;
|
|
393
|
+
const safeScreenshot = async () => {
|
|
394
|
+
for (let i = 0; i < 3; i++) {
|
|
395
|
+
try {
|
|
396
|
+
return await target.screenshot(options);
|
|
397
|
+
} catch {
|
|
398
|
+
await page.waitForTimeout(250);
|
|
399
|
+
continue;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
return await target.screenshot(options);
|
|
403
|
+
};
|
|
587
404
|
while (true) {
|
|
588
405
|
if (Date.now() >= deadline) break;
|
|
589
406
|
const delay = pollIntervals.length ? pollIntervals.shift() : 1e3;
|
|
@@ -591,7 +408,7 @@ async function takeStableScreenshot(target, options) {
|
|
|
591
408
|
await page.waitForTimeout(delay);
|
|
592
409
|
}
|
|
593
410
|
previous = actual;
|
|
594
|
-
actual = await
|
|
411
|
+
actual = await safeScreenshot();
|
|
595
412
|
if (!isFirstIteration && actual && previous && imagesAreSimilar({
|
|
596
413
|
image1: previous,
|
|
597
414
|
image2: actual,
|
|
@@ -601,11 +418,475 @@ async function takeStableScreenshot(target, options) {
|
|
|
601
418
|
}
|
|
602
419
|
isFirstIteration = false;
|
|
603
420
|
}
|
|
604
|
-
return actual ?? await
|
|
421
|
+
return actual ?? await safeScreenshot();
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
// src/playwright-augment/methods/agent/construct-payload.ts
|
|
425
|
+
function constructAgentPayload({
|
|
426
|
+
sessionId,
|
|
427
|
+
message,
|
|
428
|
+
isError,
|
|
429
|
+
screenshot,
|
|
430
|
+
tabManager,
|
|
431
|
+
activePage,
|
|
432
|
+
additionalContext
|
|
433
|
+
}) {
|
|
434
|
+
const form = new FormData();
|
|
435
|
+
form.append("session_id", sessionId);
|
|
436
|
+
form.append("message", message);
|
|
437
|
+
if (isError) {
|
|
438
|
+
form.append("is_error", JSON.stringify(isError));
|
|
439
|
+
}
|
|
440
|
+
if (additionalContext) {
|
|
441
|
+
form.append("additional_context", JSON.stringify(additionalContext));
|
|
442
|
+
}
|
|
443
|
+
const viewportSize = activePage.viewportSize();
|
|
444
|
+
if (viewportSize) {
|
|
445
|
+
form.append("page_dimensions", JSON.stringify(viewportSize));
|
|
446
|
+
}
|
|
447
|
+
const screenshotBytes = Uint8Array.from(screenshot);
|
|
448
|
+
const screenshotBlob = new Blob([screenshotBytes], { type: "image/png" });
|
|
449
|
+
form.append("screenshot", screenshotBlob, "screenshot.png");
|
|
450
|
+
const tabs = Array.from(tabManager.entries()).map(([page, alias]) => ({
|
|
451
|
+
alias,
|
|
452
|
+
url: page.url()
|
|
453
|
+
}));
|
|
454
|
+
form.append("all_pages", JSON.stringify(tabs));
|
|
455
|
+
const activePageAlias = tabManager.get(activePage);
|
|
456
|
+
if (activePageAlias) {
|
|
457
|
+
form.append("active_page_alias", activePageAlias);
|
|
458
|
+
}
|
|
459
|
+
return form;
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
// src/playwright-augment/methods/agent.ts
|
|
463
|
+
var AGENT_PATH = "internal/v1/agent";
|
|
464
|
+
var STABLY_API_URL = process.env.STABLY_API_URL || "https://api.stably.ai";
|
|
465
|
+
var AGENT_ENDPOINT = new URL(AGENT_PATH, STABLY_API_URL).toString();
|
|
466
|
+
function createAgentStub() {
|
|
467
|
+
return async (prompt, options) => {
|
|
468
|
+
const apiKey = requireApiKey();
|
|
469
|
+
const maxCycles = options.maxCycles ?? 30;
|
|
470
|
+
const browserContext = options.page.context();
|
|
471
|
+
const tabManager = /* @__PURE__ */ new Map();
|
|
472
|
+
browserContext.pages().forEach((page, index) => {
|
|
473
|
+
tabManager.set(page, `page${index + 1}`);
|
|
474
|
+
});
|
|
475
|
+
let activePage = options.page;
|
|
476
|
+
let agentMessage = {
|
|
477
|
+
message: prompt,
|
|
478
|
+
isError: false,
|
|
479
|
+
shouldTerminate: false
|
|
480
|
+
};
|
|
481
|
+
let finalSuccess;
|
|
482
|
+
let newPageOpenedMsg;
|
|
483
|
+
const sessionId = crypto.randomUUID();
|
|
484
|
+
const onNewPage = async (page) => {
|
|
485
|
+
await page.waitForLoadState("domcontentloaded");
|
|
486
|
+
if (!tabManager.has(page)) {
|
|
487
|
+
const alias2 = `page${tabManager.size + 1}`;
|
|
488
|
+
tabManager.set(page, alias2);
|
|
489
|
+
}
|
|
490
|
+
await page.bringToFront();
|
|
491
|
+
activePage = page;
|
|
492
|
+
const alias = tabManager.get(page);
|
|
493
|
+
newPageOpenedMsg = `opened new tab ${alias} (${page.url()})`;
|
|
494
|
+
};
|
|
495
|
+
browserContext.on("page", onNewPage);
|
|
496
|
+
try {
|
|
497
|
+
for (let i = 0; i < maxCycles; i++) {
|
|
498
|
+
if (agentMessage.shouldTerminate) {
|
|
499
|
+
break;
|
|
500
|
+
}
|
|
501
|
+
const screenshot = await takeStableScreenshot(activePage);
|
|
502
|
+
const response = await fetch(AGENT_ENDPOINT, {
|
|
503
|
+
method: "POST",
|
|
504
|
+
headers: {
|
|
505
|
+
...SDK_METADATA_HEADERS,
|
|
506
|
+
Authorization: `Bearer ${apiKey}`
|
|
507
|
+
},
|
|
508
|
+
body: constructAgentPayload({
|
|
509
|
+
sessionId,
|
|
510
|
+
message: agentMessage.message,
|
|
511
|
+
isError: agentMessage.isError,
|
|
512
|
+
screenshot,
|
|
513
|
+
tabManager,
|
|
514
|
+
activePage,
|
|
515
|
+
additionalContext: newPageOpenedMsg ? { newPageMessage: newPageOpenedMsg } : void 0
|
|
516
|
+
})
|
|
517
|
+
});
|
|
518
|
+
newPageOpenedMsg = void 0;
|
|
519
|
+
const responseJson = await response.json();
|
|
520
|
+
if (!response.ok) {
|
|
521
|
+
throw new Error(`Agent call failed: ${JSON.stringify(responseJson)}`);
|
|
522
|
+
}
|
|
523
|
+
const agentResponse = responseJson;
|
|
524
|
+
agentMessage = await (async () => {
|
|
525
|
+
try {
|
|
526
|
+
switch (agentResponse.action) {
|
|
527
|
+
case "key": {
|
|
528
|
+
const { text } = agentResponse;
|
|
529
|
+
if (text) {
|
|
530
|
+
await activePage.keyboard.press(text);
|
|
531
|
+
return { message: `pressed "${text}"` };
|
|
532
|
+
}
|
|
533
|
+
return { message: "pressed key" };
|
|
534
|
+
}
|
|
535
|
+
case "type": {
|
|
536
|
+
const { text } = agentResponse;
|
|
537
|
+
await activePage.keyboard.type(text);
|
|
538
|
+
return { message: `typed "${text}"` };
|
|
539
|
+
}
|
|
540
|
+
case "mouse_move": {
|
|
541
|
+
const [x, y] = agentResponse.coordinate;
|
|
542
|
+
await activePage.mouse.move(x, y);
|
|
543
|
+
return { message: `mouse moved to [${x}, ${y}]` };
|
|
544
|
+
}
|
|
545
|
+
case "left_click": {
|
|
546
|
+
const [x, y] = agentResponse.coordinate;
|
|
547
|
+
await activePage.mouse.click(x, y);
|
|
548
|
+
return { message: `left click at [${x}, ${y}]` };
|
|
549
|
+
}
|
|
550
|
+
case "right_click": {
|
|
551
|
+
const [x, y] = agentResponse.coordinate;
|
|
552
|
+
await activePage.mouse.click(x, y, { button: "right" });
|
|
553
|
+
return { message: `right click at [${x}, ${y}]` };
|
|
554
|
+
}
|
|
555
|
+
case "double_click": {
|
|
556
|
+
const [x, y] = agentResponse.coordinate;
|
|
557
|
+
await activePage.mouse.dblclick(x, y);
|
|
558
|
+
return { message: `double click at [${x}, ${y}]` };
|
|
559
|
+
}
|
|
560
|
+
case "triple_click": {
|
|
561
|
+
const [x, y] = agentResponse.coordinate;
|
|
562
|
+
await activePage.mouse.click(x, y, { clickCount: 3 });
|
|
563
|
+
return { message: `triple click at [${x}, ${y}]` };
|
|
564
|
+
}
|
|
565
|
+
case "left_click_drag": {
|
|
566
|
+
const [startX, startY] = agentResponse.start_coordinate;
|
|
567
|
+
const [endX, endY] = agentResponse.coordinate;
|
|
568
|
+
await activePage.mouse.move(startX, startY);
|
|
569
|
+
await activePage.mouse.down();
|
|
570
|
+
await activePage.mouse.move(endX, endY);
|
|
571
|
+
await activePage.mouse.up();
|
|
572
|
+
return {
|
|
573
|
+
message: `dragged from [${startX}, ${startY}] to [${endX}, ${endY}]`
|
|
574
|
+
};
|
|
575
|
+
}
|
|
576
|
+
case "screenshot": {
|
|
577
|
+
await takeStableScreenshot(activePage);
|
|
578
|
+
return { message: "captured screenshot" };
|
|
579
|
+
}
|
|
580
|
+
case "wait": {
|
|
581
|
+
const waitMs = agentResponse.milliseconds ?? 3e3;
|
|
582
|
+
await activePage.waitForTimeout(waitMs);
|
|
583
|
+
return { message: `waited ${waitMs}ms` };
|
|
584
|
+
}
|
|
585
|
+
case "navigate_to_url": {
|
|
586
|
+
await activePage.goto(agentResponse.url);
|
|
587
|
+
return { message: `navigated to "${agentResponse.url}"` };
|
|
588
|
+
}
|
|
589
|
+
case "new_tab_url": {
|
|
590
|
+
const newPage = await browserContext.newPage();
|
|
591
|
+
await newPage.goto(agentResponse.url);
|
|
592
|
+
await newPage.waitForLoadState("domcontentloaded");
|
|
593
|
+
return { message: "opened new tab" };
|
|
594
|
+
}
|
|
595
|
+
case "switch_tab": {
|
|
596
|
+
const entry = Array.from(tabManager.entries()).find(
|
|
597
|
+
([, alias]) => alias === agentResponse.tab_alias
|
|
598
|
+
);
|
|
599
|
+
const page = entry?.[0];
|
|
600
|
+
if (!page) {
|
|
601
|
+
throw new Error(
|
|
602
|
+
`Tab with alias ${agentResponse.tab_alias} not found`
|
|
603
|
+
);
|
|
604
|
+
}
|
|
605
|
+
await page.bringToFront();
|
|
606
|
+
activePage = page;
|
|
607
|
+
return { message: `switched to "${agentResponse.tab_alias}"` };
|
|
608
|
+
}
|
|
609
|
+
case "scroll": {
|
|
610
|
+
const [x, y] = agentResponse.coordinate;
|
|
611
|
+
await activePage.mouse.move(x, y);
|
|
612
|
+
let deltaX = 0;
|
|
613
|
+
let deltaY = 0;
|
|
614
|
+
switch (agentResponse.scroll_direction) {
|
|
615
|
+
case "up":
|
|
616
|
+
deltaY = -agentResponse.scroll_amount;
|
|
617
|
+
break;
|
|
618
|
+
case "down":
|
|
619
|
+
deltaY = agentResponse.scroll_amount;
|
|
620
|
+
break;
|
|
621
|
+
case "left":
|
|
622
|
+
deltaX = -agentResponse.scroll_amount;
|
|
623
|
+
break;
|
|
624
|
+
case "right":
|
|
625
|
+
deltaX = agentResponse.scroll_amount;
|
|
626
|
+
break;
|
|
627
|
+
}
|
|
628
|
+
await activePage.mouse.wheel(deltaX, deltaY);
|
|
629
|
+
return {
|
|
630
|
+
message: `scrolled ${agentResponse.scroll_direction}`
|
|
631
|
+
};
|
|
632
|
+
}
|
|
633
|
+
case "navigate_back": {
|
|
634
|
+
const res = await activePage.goBack();
|
|
635
|
+
if (!res)
|
|
636
|
+
throw new Error("navigate_back failed: no history entry");
|
|
637
|
+
return { message: "navigated back" };
|
|
638
|
+
}
|
|
639
|
+
case "terminate_test": {
|
|
640
|
+
const { success, reason } = agentResponse;
|
|
641
|
+
finalSuccess = success;
|
|
642
|
+
return { message: reason, shouldTerminate: true };
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
} catch (error) {
|
|
646
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
647
|
+
return { message, isError: true };
|
|
648
|
+
}
|
|
649
|
+
})();
|
|
650
|
+
}
|
|
651
|
+
} finally {
|
|
652
|
+
browserContext.off("page", onNewPage);
|
|
653
|
+
}
|
|
654
|
+
return { success: finalSuccess ?? false };
|
|
655
|
+
};
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
// src/playwright-augment/augment.ts
|
|
659
|
+
var LOCATOR_PATCHED = Symbol.for("stably.playwright.locatorPatched");
|
|
660
|
+
var LOCATOR_DESCRIBE_WRAPPED = Symbol.for(
|
|
661
|
+
"stably.playwright.locatorDescribeWrapped"
|
|
662
|
+
);
|
|
663
|
+
var PAGE_PATCHED = Symbol.for("stably.playwright.pagePatched");
|
|
664
|
+
var CONTEXT_PATCHED = Symbol.for("stably.playwright.contextPatched");
|
|
665
|
+
var BROWSER_PATCHED = Symbol.for("stably.playwright.browserPatched");
|
|
666
|
+
var BROWSER_TYPE_PATCHED = Symbol.for("stably.playwright.browserTypePatched");
|
|
667
|
+
function defineHiddenProperty(target, key, value) {
|
|
668
|
+
Object.defineProperty(target, key, {
|
|
669
|
+
value,
|
|
670
|
+
enumerable: false,
|
|
671
|
+
configurable: true,
|
|
672
|
+
writable: true
|
|
673
|
+
});
|
|
674
|
+
}
|
|
675
|
+
function augmentLocator(locator) {
|
|
676
|
+
if (locator[LOCATOR_PATCHED]) {
|
|
677
|
+
return locator;
|
|
678
|
+
}
|
|
679
|
+
defineHiddenProperty(locator, "extract", createLocatorExtract(locator));
|
|
680
|
+
const markerTarget = locator;
|
|
681
|
+
if (typeof locator.describe === "function" && !markerTarget[LOCATOR_DESCRIBE_WRAPPED]) {
|
|
682
|
+
const originalDescribe = locator.describe.bind(locator);
|
|
683
|
+
locator.describe = (description, options) => {
|
|
684
|
+
void options;
|
|
685
|
+
const result = originalDescribe(description);
|
|
686
|
+
return result ? augmentLocator(result) : result;
|
|
687
|
+
};
|
|
688
|
+
defineHiddenProperty(locator, LOCATOR_DESCRIBE_WRAPPED, true);
|
|
689
|
+
}
|
|
690
|
+
defineHiddenProperty(locator, LOCATOR_PATCHED, true);
|
|
691
|
+
return locator;
|
|
692
|
+
}
|
|
693
|
+
function augmentPage(page) {
|
|
694
|
+
if (page[PAGE_PATCHED]) {
|
|
695
|
+
return page;
|
|
696
|
+
}
|
|
697
|
+
const originalLocator = page.locator.bind(page);
|
|
698
|
+
page.locator = (...args) => {
|
|
699
|
+
const locator = originalLocator(...args);
|
|
700
|
+
return augmentLocator(locator);
|
|
701
|
+
};
|
|
702
|
+
defineHiddenProperty(page, "extract", createPageExtract(page));
|
|
703
|
+
defineHiddenProperty(page, PAGE_PATCHED, true);
|
|
704
|
+
return page;
|
|
705
|
+
}
|
|
706
|
+
function augmentBrowserContext(context) {
|
|
707
|
+
if (context[CONTEXT_PATCHED]) {
|
|
708
|
+
return context;
|
|
709
|
+
}
|
|
710
|
+
const originalNewPage = context.newPage.bind(context);
|
|
711
|
+
context.newPage = async (...args) => {
|
|
712
|
+
const page = await originalNewPage(...args);
|
|
713
|
+
return augmentPage(page);
|
|
714
|
+
};
|
|
715
|
+
const originalPages = context.pages?.bind(context);
|
|
716
|
+
if (originalPages) {
|
|
717
|
+
context.pages = () => originalPages().map(
|
|
718
|
+
(page) => augmentPage(page)
|
|
719
|
+
);
|
|
720
|
+
}
|
|
721
|
+
if (!context.agent) {
|
|
722
|
+
defineHiddenProperty(context, "agent", createAgentStub());
|
|
723
|
+
}
|
|
724
|
+
defineHiddenProperty(context, CONTEXT_PATCHED, true);
|
|
725
|
+
return context;
|
|
726
|
+
}
|
|
727
|
+
function augmentBrowser(browser) {
|
|
728
|
+
if (browser[BROWSER_PATCHED]) {
|
|
729
|
+
return browser;
|
|
730
|
+
}
|
|
731
|
+
const originalNewContext = browser.newContext.bind(browser);
|
|
732
|
+
browser.newContext = async (...args) => {
|
|
733
|
+
const context = await originalNewContext(...args);
|
|
734
|
+
return augmentBrowserContext(context);
|
|
735
|
+
};
|
|
736
|
+
const originalNewPage = browser.newPage.bind(browser);
|
|
737
|
+
browser.newPage = async (...args) => {
|
|
738
|
+
const page = await originalNewPage(...args);
|
|
739
|
+
return augmentPage(page);
|
|
740
|
+
};
|
|
741
|
+
const originalContexts = browser.contexts.bind(browser);
|
|
742
|
+
browser.contexts = () => originalContexts().map(
|
|
743
|
+
(context) => augmentBrowserContext(context)
|
|
744
|
+
);
|
|
745
|
+
if (!browser.agent) {
|
|
746
|
+
defineHiddenProperty(browser, "agent", createAgentStub());
|
|
747
|
+
}
|
|
748
|
+
defineHiddenProperty(browser, BROWSER_PATCHED, true);
|
|
749
|
+
return browser;
|
|
750
|
+
}
|
|
751
|
+
function augmentBrowserType(browserType) {
|
|
752
|
+
if (browserType[BROWSER_TYPE_PATCHED]) {
|
|
753
|
+
return browserType;
|
|
754
|
+
}
|
|
755
|
+
const originalLaunch = browserType.launch.bind(browserType);
|
|
756
|
+
browserType.launch = async (...args) => {
|
|
757
|
+
const browser = await originalLaunch(...args);
|
|
758
|
+
return augmentBrowser(browser);
|
|
759
|
+
};
|
|
760
|
+
const originalConnect = browserType.connect?.bind(browserType);
|
|
761
|
+
if (originalConnect) {
|
|
762
|
+
browserType.connect = async (...args) => {
|
|
763
|
+
const browser = await originalConnect(...args);
|
|
764
|
+
return augmentBrowser(browser);
|
|
765
|
+
};
|
|
766
|
+
}
|
|
767
|
+
const originalConnectOverCDP = browserType.connectOverCDP?.bind(browserType);
|
|
768
|
+
if (originalConnectOverCDP) {
|
|
769
|
+
browserType.connectOverCDP = async (...args) => {
|
|
770
|
+
const browser = await originalConnectOverCDP(...args);
|
|
771
|
+
return augmentBrowser(browser);
|
|
772
|
+
};
|
|
773
|
+
}
|
|
774
|
+
const originalLaunchPersistentContext = browserType.launchPersistentContext?.bind(browserType);
|
|
775
|
+
if (originalLaunchPersistentContext) {
|
|
776
|
+
browserType.launchPersistentContext = async (...args) => {
|
|
777
|
+
const context = await originalLaunchPersistentContext(...args);
|
|
778
|
+
return augmentBrowserContext(context);
|
|
779
|
+
};
|
|
780
|
+
}
|
|
781
|
+
defineHiddenProperty(browserType, BROWSER_TYPE_PATCHED, true);
|
|
782
|
+
return browserType;
|
|
605
783
|
}
|
|
606
784
|
|
|
607
|
-
// src/
|
|
608
|
-
|
|
785
|
+
// src/ai/verify-prompt.ts
|
|
786
|
+
var PROMPT_ASSERTION_ENDPOINT = "https://api.stably.ai/internal/v1/assert";
|
|
787
|
+
var parseSuccessResponse = (value) => {
|
|
788
|
+
if (!isObject(value)) {
|
|
789
|
+
throw new Error("Verify prompt returned unexpected response shape");
|
|
790
|
+
}
|
|
791
|
+
const { success, reason } = value;
|
|
792
|
+
if (typeof success !== "boolean") {
|
|
793
|
+
throw new Error("Verify prompt returned unexpected response shape");
|
|
794
|
+
}
|
|
795
|
+
if (reason !== void 0 && typeof reason !== "string") {
|
|
796
|
+
throw new Error("Verify prompt returned unexpected response shape");
|
|
797
|
+
}
|
|
798
|
+
return {
|
|
799
|
+
success,
|
|
800
|
+
reason
|
|
801
|
+
};
|
|
802
|
+
};
|
|
803
|
+
var parseErrorResponse = (value) => {
|
|
804
|
+
if (!isObject(value)) {
|
|
805
|
+
return void 0;
|
|
806
|
+
}
|
|
807
|
+
const { error } = value;
|
|
808
|
+
return typeof error !== "string" ? void 0 : { error };
|
|
809
|
+
};
|
|
810
|
+
async function verifyPrompt({
|
|
811
|
+
prompt,
|
|
812
|
+
screenshot
|
|
813
|
+
}) {
|
|
814
|
+
const apiKey = requireApiKey();
|
|
815
|
+
const form = new FormData();
|
|
816
|
+
form.append("prompt", prompt);
|
|
817
|
+
const u8 = Uint8Array.from(screenshot);
|
|
818
|
+
const blob = new Blob([u8], { type: "image/png" });
|
|
819
|
+
form.append("image", blob, "screenshot.png");
|
|
820
|
+
const response = await fetch(PROMPT_ASSERTION_ENDPOINT, {
|
|
821
|
+
method: "POST",
|
|
822
|
+
headers: {
|
|
823
|
+
...SDK_METADATA_HEADERS,
|
|
824
|
+
Authorization: `Bearer ${apiKey}`
|
|
825
|
+
},
|
|
826
|
+
body: form
|
|
827
|
+
});
|
|
828
|
+
const parsed = await response.json().catch(() => void 0);
|
|
829
|
+
if (response.ok) {
|
|
830
|
+
const { success, reason } = parseSuccessResponse(parsed);
|
|
831
|
+
return {
|
|
832
|
+
pass: success,
|
|
833
|
+
reason
|
|
834
|
+
};
|
|
835
|
+
}
|
|
836
|
+
const err = parseErrorResponse(parsed);
|
|
837
|
+
throw new Error(
|
|
838
|
+
`Verify prompt failed (${response.status})${err ? `: ${err.error}` : ""}`
|
|
839
|
+
);
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
// src/expect/snapshotCache.ts
|
|
843
|
+
var import_path = __toESM(require("path"));
|
|
844
|
+
var import_promises = require("fs/promises");
|
|
845
|
+
var import_path2 = require("path");
|
|
846
|
+
var import_fs = require("fs");
|
|
847
|
+
var import_internal_playwright_test = require("@stablyai/internal-playwright-test");
|
|
848
|
+
async function getCacheEntryPath(snapshotKey) {
|
|
849
|
+
const testDir = import_path.default.dirname(import_internal_playwright_test.test.info().file);
|
|
850
|
+
return import_path.default.join(testDir, snapshotKey);
|
|
851
|
+
}
|
|
852
|
+
async function tryUseCachedSnapshot({
|
|
853
|
+
currentSnapshot,
|
|
854
|
+
cachedSnapshotKey,
|
|
855
|
+
snapshotSimilarityThreshold
|
|
856
|
+
}) {
|
|
857
|
+
const cacheEntryPath = await getCacheEntryPath(cachedSnapshotKey);
|
|
858
|
+
try {
|
|
859
|
+
await (0, import_promises.access)(cacheEntryPath, import_fs.constants.F_OK);
|
|
860
|
+
} catch {
|
|
861
|
+
return { success: false, usedCachedSnapshot: false };
|
|
862
|
+
}
|
|
863
|
+
const cacheEntry = await (0, import_promises.readFile)(cacheEntryPath);
|
|
864
|
+
return {
|
|
865
|
+
success: imagesAreSimilar({
|
|
866
|
+
image1: cacheEntry,
|
|
867
|
+
image2: currentSnapshot,
|
|
868
|
+
threshold: snapshotSimilarityThreshold
|
|
869
|
+
}),
|
|
870
|
+
usedCachedSnapshot: true
|
|
871
|
+
};
|
|
872
|
+
}
|
|
873
|
+
async function updateCachedSnapshot({
|
|
874
|
+
snapshotKey,
|
|
875
|
+
snapshot
|
|
876
|
+
}) {
|
|
877
|
+
const cacheEntryPath = await getCacheEntryPath(snapshotKey);
|
|
878
|
+
try {
|
|
879
|
+
await (0, import_promises.mkdir)((0, import_path2.dirname)(cacheEntryPath), { recursive: true });
|
|
880
|
+
await (0, import_promises.writeFile)(cacheEntryPath, snapshot);
|
|
881
|
+
} catch (err) {
|
|
882
|
+
console.error("Error writing snapshot", err);
|
|
883
|
+
throw err;
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
// src/expect/expect.ts
|
|
888
|
+
var DEFAULT_SNAPSHOT_SIMILARITY_THRESHOLD = 0.02;
|
|
889
|
+
function createResultMessage({
|
|
609
890
|
targetType,
|
|
610
891
|
condition,
|
|
611
892
|
didPass,
|
|
@@ -632,14 +913,38 @@ var stablyPlaywrightMatchers = {
|
|
|
632
913
|
}
|
|
633
914
|
const targetType = isPage(target) ? "page" : "locator";
|
|
634
915
|
const screenshot = await takeStableScreenshot(target, options);
|
|
916
|
+
const useCachedSnapshotResult = options?.cache ? await tryUseCachedSnapshot({
|
|
917
|
+
currentSnapshot: screenshot,
|
|
918
|
+
cachedSnapshotKey: options.cache.snapshotKey,
|
|
919
|
+
snapshotSimilarityThreshold: options?.threshold ?? DEFAULT_SNAPSHOT_SIMILARITY_THRESHOLD
|
|
920
|
+
}) : void 0;
|
|
921
|
+
if (useCachedSnapshotResult?.success) {
|
|
922
|
+
return {
|
|
923
|
+
pass: true,
|
|
924
|
+
message: () => createResultMessage({
|
|
925
|
+
targetType,
|
|
926
|
+
condition,
|
|
927
|
+
didPass: true,
|
|
928
|
+
reason: "Used cached snapshot to verify condition",
|
|
929
|
+
isNot: this.isNot
|
|
930
|
+
})
|
|
931
|
+
};
|
|
932
|
+
}
|
|
635
933
|
const verifyResult = await verifyPrompt({ prompt: condition, screenshot });
|
|
934
|
+
if (options?.cache && verifyResult.pass) {
|
|
935
|
+
await updateCachedSnapshot({
|
|
936
|
+
snapshotKey: options.cache.snapshotKey,
|
|
937
|
+
snapshot: screenshot
|
|
938
|
+
});
|
|
939
|
+
}
|
|
940
|
+
const snapshotUsageMessage = useCachedSnapshotResult ? `Attempted to use the cached snapshot to verify the condition, but ${useCachedSnapshotResult.usedCachedSnapshot ? "the snapshot did not match the current snapshot" : "no snapshot was found"}.` : void 0;
|
|
636
941
|
return {
|
|
637
942
|
pass: verifyResult.pass,
|
|
638
|
-
message: () =>
|
|
943
|
+
message: () => createResultMessage({
|
|
639
944
|
targetType,
|
|
640
945
|
condition,
|
|
641
946
|
didPass: verifyResult.pass,
|
|
642
|
-
reason: verifyResult.reason,
|
|
947
|
+
reason: snapshotUsageMessage ? `${snapshotUsageMessage} ${verifyResult.reason}` : verifyResult.reason,
|
|
643
948
|
isNot: this.isNot
|
|
644
949
|
})
|
|
645
950
|
};
|