@stablyai/playwright-base 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai/extract.d.ts +2 -2
- package/dist/ai/extract.js +16 -13
- package/dist/ai/verify-prompt.js +6 -6
- package/dist/expect.d.ts +2 -2
- package/dist/expect.js +10 -52
- package/dist/image-compare.d.ts +5 -0
- package/dist/image-compare.js +81 -0
- package/dist/index.d.ts +53 -11
- package/dist/playwright-augment/augment.d.ts +1 -1
- package/dist/playwright-augment/augment.js +15 -17
- package/dist/playwright-augment/methods/agent.d.ts +1 -1
- package/dist/playwright-augment/methods/auto-heal.d.ts +16 -0
- package/dist/playwright-augment/methods/auto-heal.js +7 -0
- package/dist/playwright-augment/methods/extract.d.ts +2 -2
- package/dist/playwright-augment/methods/extract.js +1 -1
- package/dist/playwright-augment/methods/test-info.d.ts +16 -0
- package/dist/playwright-augment/methods/test-info.js +96 -0
- package/dist/playwright-type-predicates.d.ts +1 -1
- package/dist/playwright-type-predicates.js +6 -6
- package/dist/runtime.js +1 -1
- package/dist/screenshot.d.ts +3 -0
- package/dist/screenshot.js +41 -0
- package/package.json +15 -6
- package/src/ai/extract.ts +0 -97
- package/src/ai/verify-prompt.ts +0 -57
- package/src/expect.ts +0 -134
- package/src/index.ts +0 -69
- package/src/playwright-augment/augment.ts +0 -207
- package/src/playwright-augment/methods/agent.ts +0 -19
- package/src/playwright-augment/methods/extract.ts +0 -48
- package/src/playwright-type-predicates.ts +0 -19
- package/src/runtime.ts +0 -19
- package/tsconfig.build.json +0 -6
- package/tsconfig.json +0 -9
package/dist/ai/extract.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { Locator, Page } from
|
|
2
|
-
import * as z4 from
|
|
1
|
+
import type { Locator, Page } from "@stablyai/internal-playwright-test";
|
|
2
|
+
import * as z4 from "zod/v4/core";
|
|
3
3
|
export type ExtractSchema = z4.$ZodType;
|
|
4
4
|
export type SchemaOutput<T extends ExtractSchema> = z4.output<T>;
|
|
5
5
|
type ExtractSubject = Page | Locator;
|
package/dist/ai/extract.js
CHANGED
|
@@ -37,20 +37,21 @@ exports.extract = extract;
|
|
|
37
37
|
const z4 = __importStar(require("zod/v4/core"));
|
|
38
38
|
const zod_1 = require("zod");
|
|
39
39
|
const runtime_1 = require("../runtime");
|
|
40
|
-
const EXTRACT_ENDPOINT =
|
|
40
|
+
const EXTRACT_ENDPOINT = "https://api.stably.ai/internal/v1/extract";
|
|
41
41
|
const zSuccess = zod_1.z.object({ value: zod_1.z.unknown() });
|
|
42
42
|
const zError = zod_1.z.object({ error: zod_1.z.string() });
|
|
43
43
|
class ExtractValidationError extends Error {
|
|
44
|
+
issues;
|
|
44
45
|
constructor(message, issues) {
|
|
45
46
|
super(message);
|
|
46
47
|
this.issues = issues;
|
|
47
|
-
this.name =
|
|
48
|
+
this.name = "ExtractValidationError";
|
|
48
49
|
}
|
|
49
50
|
}
|
|
50
51
|
async function validateWithSchema(schema, value) {
|
|
51
52
|
const result = await z4.safeParseAsync(schema, value);
|
|
52
53
|
if (!result.success) {
|
|
53
|
-
throw new ExtractValidationError(
|
|
54
|
+
throw new ExtractValidationError("Validation failed", result.error.issues);
|
|
54
55
|
}
|
|
55
56
|
return result.data;
|
|
56
57
|
}
|
|
@@ -58,16 +59,16 @@ async function extract({ prompt, pageOrLocator, schema, }) {
|
|
|
58
59
|
const jsonSchema = schema ? z4.toJSONSchema(schema) : undefined;
|
|
59
60
|
const apiKey = (0, runtime_1.requireApiKey)();
|
|
60
61
|
const form = new FormData();
|
|
61
|
-
form.append(
|
|
62
|
+
form.append("prompt", prompt);
|
|
62
63
|
if (jsonSchema) {
|
|
63
|
-
form.append(
|
|
64
|
+
form.append("jsonSchema", JSON.stringify(jsonSchema));
|
|
64
65
|
}
|
|
65
|
-
const pngBuffer = await pageOrLocator.screenshot({ type:
|
|
66
|
+
const pngBuffer = await pageOrLocator.screenshot({ type: "png" }); // Buffer
|
|
66
67
|
const u8 = Uint8Array.from(pngBuffer); // strips Buffer type → plain Uint8Array
|
|
67
|
-
const blob = new Blob([u8], { type:
|
|
68
|
-
form.append(
|
|
68
|
+
const blob = new Blob([u8], { type: "image/png" });
|
|
69
|
+
form.append("image", blob, "screenshot.png");
|
|
69
70
|
const response = await fetch(EXTRACT_ENDPOINT, {
|
|
70
|
-
method:
|
|
71
|
+
method: "POST",
|
|
71
72
|
headers: {
|
|
72
73
|
Authorization: `Bearer ${apiKey}`,
|
|
73
74
|
},
|
|
@@ -76,10 +77,12 @@ async function extract({ prompt, pageOrLocator, schema, }) {
|
|
|
76
77
|
const parsed = await response.json().catch(() => undefined);
|
|
77
78
|
if (response.ok) {
|
|
78
79
|
const { value } = zSuccess.parse(parsed);
|
|
79
|
-
return
|
|
80
|
-
|
|
81
|
-
|
|
80
|
+
return schema
|
|
81
|
+
? await validateWithSchema(schema, value)
|
|
82
|
+
: typeof value === "string"
|
|
83
|
+
? value
|
|
84
|
+
: JSON.stringify(value);
|
|
82
85
|
}
|
|
83
86
|
const err = zError.safeParse(parsed);
|
|
84
|
-
throw new Error(`Extract failed (${response.status})${err.success ? `: ${err.data.error}` :
|
|
87
|
+
throw new Error(`Extract failed (${response.status})${err.success ? `: ${err.data.error}` : ""}`);
|
|
85
88
|
}
|
package/dist/ai/verify-prompt.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.verifyPrompt = verifyPrompt;
|
|
4
|
-
const PROMPT_ASSERTION_ENDPOINT =
|
|
4
|
+
const PROMPT_ASSERTION_ENDPOINT = "https://api.stably.ai/internal/v1/assert";
|
|
5
5
|
const zod_1 = require("zod");
|
|
6
6
|
const runtime_1 = require("../runtime");
|
|
7
7
|
const zSuccess = zod_1.z.object({
|
|
@@ -14,12 +14,12 @@ const zError = zod_1.z.object({
|
|
|
14
14
|
async function verifyPrompt({ prompt, screenshot, }) {
|
|
15
15
|
const apiKey = (0, runtime_1.requireApiKey)();
|
|
16
16
|
const form = new FormData();
|
|
17
|
-
form.append(
|
|
17
|
+
form.append("prompt", prompt);
|
|
18
18
|
const u8 = Uint8Array.from(screenshot);
|
|
19
|
-
const blob = new Blob([u8], { type:
|
|
20
|
-
form.append(
|
|
19
|
+
const blob = new Blob([u8], { type: "image/png" });
|
|
20
|
+
form.append("image", blob, "screenshot.png");
|
|
21
21
|
const response = await fetch(PROMPT_ASSERTION_ENDPOINT, {
|
|
22
|
-
method:
|
|
22
|
+
method: "POST",
|
|
23
23
|
headers: {
|
|
24
24
|
Authorization: `Bearer ${apiKey}`,
|
|
25
25
|
},
|
|
@@ -34,5 +34,5 @@ async function verifyPrompt({ prompt, screenshot, }) {
|
|
|
34
34
|
};
|
|
35
35
|
}
|
|
36
36
|
const err = zError.safeParse(parsed);
|
|
37
|
-
throw new Error(`Verify prompt failed (${response.status})${err.success ? `: ${err.data.error}` :
|
|
37
|
+
throw new Error(`Verify prompt failed (${response.status})${err.success ? `: ${err.data.error}` : ""}`);
|
|
38
38
|
}
|
package/dist/expect.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { Locator, Page } from
|
|
2
|
-
import type { ScreenshotPromptOptions } from
|
|
1
|
+
import type { Locator, Page } from "@stablyai/internal-playwright-test";
|
|
2
|
+
import type { ScreenshotPromptOptions } from "./index";
|
|
3
3
|
type MatcherContext = {
|
|
4
4
|
isNot: boolean;
|
|
5
5
|
message?: () => string;
|
package/dist/expect.js
CHANGED
|
@@ -3,72 +3,30 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.stablyPlaywrightMatchers = void 0;
|
|
4
4
|
const playwright_type_predicates_1 = require("./playwright-type-predicates");
|
|
5
5
|
const verify_prompt_1 = require("./ai/verify-prompt");
|
|
6
|
+
const screenshot_1 = require("./screenshot");
|
|
6
7
|
function createFailureMessage({ targetType, condition, didPass, isNot, reason, }) {
|
|
7
|
-
const expectation = isNot ?
|
|
8
|
-
const result = didPass ?
|
|
8
|
+
const expectation = isNot ? "not to satisfy" : "to satisfy";
|
|
9
|
+
const result = didPass ? "it did" : "it did not";
|
|
9
10
|
let message = `Expected ${targetType} ${expectation} ${JSON.stringify(condition)}, but ${result}.`;
|
|
10
11
|
if (reason) {
|
|
11
12
|
message += `\n\nReason: ${reason}`;
|
|
12
13
|
}
|
|
13
14
|
return message;
|
|
14
15
|
}
|
|
15
|
-
function areScreenshotsEqual(a, b) {
|
|
16
|
-
if (a.byteLength !== b.byteLength) {
|
|
17
|
-
return false;
|
|
18
|
-
}
|
|
19
|
-
for (let index = 0; index < a.byteLength; index += 1) {
|
|
20
|
-
if (a[index] !== b[index]) {
|
|
21
|
-
return false;
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
return true;
|
|
25
|
-
}
|
|
26
|
-
async function takeStableScreenshot(target, options) {
|
|
27
|
-
var _a;
|
|
28
|
-
const page = (0, playwright_type_predicates_1.isPage)(target) ? target : target.page();
|
|
29
|
-
// Use a small budget for stabilization within the overall assertion timeout.
|
|
30
|
-
// We allocate up to 25% of the total timeout (bounded between 300ms and 2000ms).
|
|
31
|
-
const totalTimeout = (_a = options === null || options === void 0 ? void 0 : options.timeout) !== null && _a !== void 0 ? _a : 5000;
|
|
32
|
-
// Budget is 25% of the total timeout
|
|
33
|
-
const stabilizationBudgetMs = Math.floor(totalTimeout * 0.25);
|
|
34
|
-
const stabilityBudgetMs = Math.min(2000, Math.max(300, stabilizationBudgetMs));
|
|
35
|
-
const deadline = Date.now() + stabilityBudgetMs;
|
|
36
|
-
let actual;
|
|
37
|
-
let previous;
|
|
38
|
-
const pollIntervals = [0, 100, 250, 500];
|
|
39
|
-
let isFirstIteration = true;
|
|
40
|
-
while (true) {
|
|
41
|
-
if (Date.now() >= deadline)
|
|
42
|
-
break;
|
|
43
|
-
const delay = pollIntervals.length ? pollIntervals.shift() : 1000;
|
|
44
|
-
if (delay) {
|
|
45
|
-
await page.waitForTimeout(delay);
|
|
46
|
-
}
|
|
47
|
-
previous = actual;
|
|
48
|
-
const rawScreenshot = await target.screenshot(options);
|
|
49
|
-
actual = Uint8Array.from(rawScreenshot);
|
|
50
|
-
if (!isFirstIteration &&
|
|
51
|
-
actual &&
|
|
52
|
-
previous &&
|
|
53
|
-
areScreenshotsEqual(actual, previous)) {
|
|
54
|
-
return actual;
|
|
55
|
-
}
|
|
56
|
-
isFirstIteration = false;
|
|
57
|
-
}
|
|
58
|
-
return actual !== null && actual !== void 0 ? actual : Uint8Array.from(await target.screenshot(options));
|
|
59
|
-
}
|
|
60
16
|
exports.stablyPlaywrightMatchers = {
|
|
61
17
|
async toMatchScreenshotPrompt(received, condition, options) {
|
|
62
|
-
const target = (0, playwright_type_predicates_1.isPage)(received)
|
|
63
|
-
|
|
18
|
+
const target = (0, playwright_type_predicates_1.isPage)(received)
|
|
19
|
+
? received
|
|
20
|
+
: (0, playwright_type_predicates_1.isLocator)(received)
|
|
21
|
+
? received
|
|
64
22
|
: undefined;
|
|
65
23
|
if (!target) {
|
|
66
24
|
// Should never happen
|
|
67
|
-
throw new Error(
|
|
25
|
+
throw new Error("toMatchScreenshotPrompt only supports Playwright Page and Locator instances.");
|
|
68
26
|
}
|
|
69
|
-
const targetType = (0, playwright_type_predicates_1.isPage)(target) ?
|
|
27
|
+
const targetType = (0, playwright_type_predicates_1.isPage)(target) ? "page" : "locator";
|
|
70
28
|
// Wait for two consecutive identical screenshots before sending to AI
|
|
71
|
-
const screenshot = await takeStableScreenshot(target, options);
|
|
29
|
+
const screenshot = await (0, screenshot_1.takeStableScreenshot)(target, options);
|
|
72
30
|
const verifyResult = await (0, verify_prompt_1.verifyPrompt)({ prompt: condition, screenshot });
|
|
73
31
|
return {
|
|
74
32
|
pass: verifyResult.pass,
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
+
};
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.imagesAreSimilar = void 0;
|
|
40
|
+
const pixelmatch_1 = __importDefault(require("pixelmatch"));
|
|
41
|
+
const pngjs_1 = require("pngjs");
|
|
42
|
+
const jpeg = __importStar(require("jpeg-js"));
|
|
43
|
+
const isPng = (buffer) => {
|
|
44
|
+
return (buffer.length >= 8 &&
|
|
45
|
+
buffer[0] === 0x89 &&
|
|
46
|
+
buffer[1] === 0x50 &&
|
|
47
|
+
buffer[2] === 0x4e &&
|
|
48
|
+
buffer[3] === 0x47 &&
|
|
49
|
+
buffer[4] === 0x0d &&
|
|
50
|
+
buffer[5] === 0x0a &&
|
|
51
|
+
buffer[6] === 0x1a &&
|
|
52
|
+
buffer[7] === 0x0a);
|
|
53
|
+
};
|
|
54
|
+
const isJpeg = (buffer) => {
|
|
55
|
+
return buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xd8;
|
|
56
|
+
};
|
|
57
|
+
const decodeImage = (buffer) => {
|
|
58
|
+
if (isPng(buffer)) {
|
|
59
|
+
const png = pngjs_1.PNG.sync.read(buffer);
|
|
60
|
+
return { data: png.data, width: png.width, height: png.height };
|
|
61
|
+
}
|
|
62
|
+
if (isJpeg(buffer)) {
|
|
63
|
+
const img = jpeg.decode(buffer, { maxMemoryUsageInMB: 1024 });
|
|
64
|
+
return { data: img.data, width: img.width, height: img.height };
|
|
65
|
+
}
|
|
66
|
+
// Default to PNG decode; if it fails upstream, treat as different sizes
|
|
67
|
+
const png = pngjs_1.PNG.sync.read(buffer);
|
|
68
|
+
return { data: png.data, width: png.width, height: png.height };
|
|
69
|
+
};
|
|
70
|
+
const imagesAreSimilar = ({ image1, image2, threshold, }) => {
|
|
71
|
+
const decodedImage1 = decodeImage(image1);
|
|
72
|
+
const decodedImage2 = decodeImage(image2);
|
|
73
|
+
if (decodedImage1.width !== decodedImage2.width ||
|
|
74
|
+
decodedImage1.height !== decodedImage2.height) {
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
const diffRgbaData = new Uint8Array(decodedImage1.width * decodedImage1.height * 4);
|
|
78
|
+
const numDiffPixels = (0, pixelmatch_1.default)(decodedImage1.data, decodedImage2.data, diffRgbaData, decodedImage1.width, decodedImage1.height, { threshold });
|
|
79
|
+
return numDiffPixels === 0;
|
|
80
|
+
};
|
|
81
|
+
exports.imagesAreSimilar = imagesAreSimilar;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,27 +1,69 @@
|
|
|
1
|
-
import type { Page } from
|
|
2
|
-
import type { LocatorDescribeOptions } from
|
|
3
|
-
import type { ExtractSchema, SchemaOutput } from
|
|
4
|
-
import { augmentBrowser, augmentBrowserContext, augmentBrowserType, augmentLocator, augmentPage } from
|
|
5
|
-
import { stablyPlaywrightMatchers } from
|
|
6
|
-
import { requireApiKey } from
|
|
7
|
-
export { setApiKey } from
|
|
8
|
-
export type { LocatorDescribeOptions } from
|
|
9
|
-
export type { ExtractSchema, SchemaOutput } from
|
|
10
|
-
export type ScreenshotPromptOptions = import(
|
|
1
|
+
import type { Page } from "@stablyai/internal-playwright-test";
|
|
2
|
+
import type { LocatorDescribeOptions } from "./playwright-augment/augment";
|
|
3
|
+
import type { ExtractSchema, SchemaOutput } from "./ai/extract";
|
|
4
|
+
import { augmentBrowser, augmentBrowserContext, augmentBrowserType, augmentLocator, augmentPage } from "./playwright-augment/augment";
|
|
5
|
+
import { stablyPlaywrightMatchers } from "./expect";
|
|
6
|
+
import { requireApiKey } from "./runtime";
|
|
7
|
+
export { setApiKey } from "./runtime";
|
|
8
|
+
export type { LocatorDescribeOptions } from "./playwright-augment/augment";
|
|
9
|
+
export type { ExtractSchema, SchemaOutput } from "./ai/extract";
|
|
10
|
+
export type ScreenshotPromptOptions = import("@stablyai/internal-playwright-test").PageAssertionsToHaveScreenshotOptions;
|
|
11
11
|
export { augmentBrowser, augmentBrowserContext, augmentBrowserType, augmentLocator, augmentPage, stablyPlaywrightMatchers, requireApiKey, };
|
|
12
12
|
export interface Expect<T = Page> {
|
|
13
13
|
toMatchScreenshotPrompt(condition: string, options?: ScreenshotPromptOptions): Promise<void>;
|
|
14
14
|
}
|
|
15
|
-
declare module
|
|
15
|
+
declare module "@stablyai/internal-playwright-test" {
|
|
16
16
|
interface Locator {
|
|
17
|
+
/**
|
|
18
|
+
* Extracts information from this locator using Stably AI.
|
|
19
|
+
*
|
|
20
|
+
* Takes a screenshot of the locator and uses AI to extract information based on the
|
|
21
|
+
* provided prompt. When a schema is provided, the extracted data is validated and
|
|
22
|
+
* typed according to the schema.
|
|
23
|
+
*
|
|
24
|
+
* @param prompt - A natural language description of what information to extract
|
|
25
|
+
* @returns A string containing the extracted information
|
|
26
|
+
*/
|
|
17
27
|
extract(prompt: string): Promise<string>;
|
|
28
|
+
/**
|
|
29
|
+
* Extracts information from this locator using Stably AI.
|
|
30
|
+
*
|
|
31
|
+
* Takes a screenshot of the locator and uses AI to extract information based on the
|
|
32
|
+
* provided prompt. The extracted data is validated and typed according to the schema.
|
|
33
|
+
*
|
|
34
|
+
* @param prompt - A natural language description of what information to extract
|
|
35
|
+
* @param options - Configuration object containing the Zod schema for validation
|
|
36
|
+
* @param options.schema - Zod schema to validate and type the extracted data
|
|
37
|
+
* @returns Typed data matching the provided schema
|
|
38
|
+
*/
|
|
18
39
|
extract<T extends ExtractSchema>(prompt: string, options: {
|
|
19
40
|
schema: T;
|
|
20
41
|
}): Promise<SchemaOutput<T>>;
|
|
21
42
|
describe(description: string, options?: LocatorDescribeOptions): Locator;
|
|
22
43
|
}
|
|
23
44
|
interface Page {
|
|
45
|
+
/**
|
|
46
|
+
* Extracts information from this page using Stably AI.
|
|
47
|
+
*
|
|
48
|
+
* Takes a screenshot of the page and uses AI to extract information based on the
|
|
49
|
+
* provided prompt. When a schema is provided, the extracted data is validated and
|
|
50
|
+
* typed according to the schema.
|
|
51
|
+
*
|
|
52
|
+
* @param prompt - A natural language description of what information to extract
|
|
53
|
+
* @returns A string containing the extracted information
|
|
54
|
+
*/
|
|
24
55
|
extract(prompt: string): Promise<string>;
|
|
56
|
+
/**
|
|
57
|
+
* Extracts information from this page using Stably AI.
|
|
58
|
+
*
|
|
59
|
+
* Takes a screenshot of the page and uses AI to extract information based on the
|
|
60
|
+
* provided prompt. The extracted data is validated and typed according to the schema.
|
|
61
|
+
*
|
|
62
|
+
* @param prompt - A natural language description of what information to extract
|
|
63
|
+
* @param options - Configuration object containing the Zod schema for validation
|
|
64
|
+
* @param options.schema - Zod schema to validate and type the extracted data
|
|
65
|
+
* @returns Typed data matching the provided schema
|
|
66
|
+
*/
|
|
25
67
|
extract<T extends ExtractSchema>(prompt: string, options: {
|
|
26
68
|
schema: T;
|
|
27
69
|
}): Promise<SchemaOutput<T>>;
|
|
@@ -7,12 +7,12 @@ exports.augmentBrowser = augmentBrowser;
|
|
|
7
7
|
exports.augmentBrowserType = augmentBrowserType;
|
|
8
8
|
const extract_1 = require("./methods/extract");
|
|
9
9
|
const agent_1 = require("./methods/agent");
|
|
10
|
-
const LOCATOR_PATCHED = Symbol.for(
|
|
11
|
-
const LOCATOR_DESCRIBE_WRAPPED = Symbol.for(
|
|
12
|
-
const PAGE_PATCHED = Symbol.for(
|
|
13
|
-
const CONTEXT_PATCHED = Symbol.for(
|
|
14
|
-
const BROWSER_PATCHED = Symbol.for(
|
|
15
|
-
const BROWSER_TYPE_PATCHED = Symbol.for(
|
|
10
|
+
const LOCATOR_PATCHED = Symbol.for("stably.playwright.locatorPatched");
|
|
11
|
+
const LOCATOR_DESCRIBE_WRAPPED = Symbol.for("stably.playwright.locatorDescribeWrapped");
|
|
12
|
+
const PAGE_PATCHED = Symbol.for("stably.playwright.pagePatched");
|
|
13
|
+
const CONTEXT_PATCHED = Symbol.for("stably.playwright.contextPatched");
|
|
14
|
+
const BROWSER_PATCHED = Symbol.for("stably.playwright.browserPatched");
|
|
15
|
+
const BROWSER_TYPE_PATCHED = Symbol.for("stably.playwright.browserTypePatched");
|
|
16
16
|
function defineHiddenProperty(target, key, value) {
|
|
17
17
|
Object.defineProperty(target, key, {
|
|
18
18
|
value,
|
|
@@ -25,9 +25,9 @@ function augmentLocator(locator) {
|
|
|
25
25
|
if (locator[LOCATOR_PATCHED]) {
|
|
26
26
|
return locator;
|
|
27
27
|
}
|
|
28
|
-
defineHiddenProperty(locator,
|
|
28
|
+
defineHiddenProperty(locator, "extract", (0, extract_1.createLocatorExtract)(locator));
|
|
29
29
|
const markerTarget = locator;
|
|
30
|
-
if (typeof locator.describe ===
|
|
30
|
+
if (typeof locator.describe === "function" &&
|
|
31
31
|
!markerTarget[LOCATOR_DESCRIBE_WRAPPED]) {
|
|
32
32
|
const originalDescribe = locator.describe.bind(locator);
|
|
33
33
|
locator.describe = ((description, options) => {
|
|
@@ -49,12 +49,11 @@ function augmentPage(page) {
|
|
|
49
49
|
const locator = originalLocator(...args);
|
|
50
50
|
return augmentLocator(locator);
|
|
51
51
|
});
|
|
52
|
-
defineHiddenProperty(page,
|
|
52
|
+
defineHiddenProperty(page, "extract", (0, extract_1.createPageExtract)(page));
|
|
53
53
|
defineHiddenProperty(page, PAGE_PATCHED, true);
|
|
54
54
|
return page;
|
|
55
55
|
}
|
|
56
56
|
function augmentBrowserContext(context) {
|
|
57
|
-
var _a;
|
|
58
57
|
if (context[CONTEXT_PATCHED]) {
|
|
59
58
|
return context;
|
|
60
59
|
}
|
|
@@ -63,12 +62,12 @@ function augmentBrowserContext(context) {
|
|
|
63
62
|
const page = await originalNewPage(...args);
|
|
64
63
|
return augmentPage(page);
|
|
65
64
|
});
|
|
66
|
-
const originalPages =
|
|
65
|
+
const originalPages = context.pages?.bind(context);
|
|
67
66
|
if (originalPages) {
|
|
68
67
|
context.pages = (() => originalPages().map((page) => augmentPage(page)));
|
|
69
68
|
}
|
|
70
69
|
if (!context.agent) {
|
|
71
|
-
defineHiddenProperty(context,
|
|
70
|
+
defineHiddenProperty(context, "agent", (0, agent_1.createAgentStub)());
|
|
72
71
|
}
|
|
73
72
|
defineHiddenProperty(context, CONTEXT_PATCHED, true);
|
|
74
73
|
return context;
|
|
@@ -90,13 +89,12 @@ function augmentBrowser(browser) {
|
|
|
90
89
|
const originalContexts = browser.contexts.bind(browser);
|
|
91
90
|
browser.contexts = (() => originalContexts().map((context) => augmentBrowserContext(context)));
|
|
92
91
|
if (!browser.agent) {
|
|
93
|
-
defineHiddenProperty(browser,
|
|
92
|
+
defineHiddenProperty(browser, "agent", (0, agent_1.createAgentStub)());
|
|
94
93
|
}
|
|
95
94
|
defineHiddenProperty(browser, BROWSER_PATCHED, true);
|
|
96
95
|
return browser;
|
|
97
96
|
}
|
|
98
97
|
function augmentBrowserType(browserType) {
|
|
99
|
-
var _a, _b, _c;
|
|
100
98
|
if (browserType[BROWSER_TYPE_PATCHED]) {
|
|
101
99
|
return browserType;
|
|
102
100
|
}
|
|
@@ -105,21 +103,21 @@ function augmentBrowserType(browserType) {
|
|
|
105
103
|
const browser = await originalLaunch(...args);
|
|
106
104
|
return augmentBrowser(browser);
|
|
107
105
|
});
|
|
108
|
-
const originalConnect =
|
|
106
|
+
const originalConnect = browserType.connect?.bind(browserType);
|
|
109
107
|
if (originalConnect) {
|
|
110
108
|
browserType.connect = (async (...args) => {
|
|
111
109
|
const browser = await originalConnect(...args);
|
|
112
110
|
return augmentBrowser(browser);
|
|
113
111
|
});
|
|
114
112
|
}
|
|
115
|
-
const originalConnectOverCDP =
|
|
113
|
+
const originalConnectOverCDP = browserType.connectOverCDP?.bind(browserType);
|
|
116
114
|
if (originalConnectOverCDP) {
|
|
117
115
|
browserType.connectOverCDP = (async (...args) => {
|
|
118
116
|
const browser = await originalConnectOverCDP(...args);
|
|
119
117
|
return augmentBrowser(browser);
|
|
120
118
|
});
|
|
121
119
|
}
|
|
122
|
-
const originalLaunchPersistentContext =
|
|
120
|
+
const originalLaunchPersistentContext = browserType.launchPersistentContext?.bind(browserType);
|
|
123
121
|
if (originalLaunchPersistentContext) {
|
|
124
122
|
browserType.launchPersistentContext = (async (...args) => {
|
|
125
123
|
const context = await originalLaunchPersistentContext(...args);
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { Locator, Page } from "playwright";
|
|
2
|
+
export type LocatorAutoHealContext = {
|
|
3
|
+
description: string;
|
|
4
|
+
error: unknown;
|
|
5
|
+
locator: Locator;
|
|
6
|
+
method: string;
|
|
7
|
+
args: unknown[];
|
|
8
|
+
page: Page;
|
|
9
|
+
};
|
|
10
|
+
export type LocatorAutoHealResult = {
|
|
11
|
+
locator?: Locator;
|
|
12
|
+
retry?: boolean;
|
|
13
|
+
} | void;
|
|
14
|
+
export type LocatorAutoHealHandler = (context: LocatorAutoHealContext) => LocatorAutoHealResult | Promise<LocatorAutoHealResult>;
|
|
15
|
+
export declare function setLocatorAutoHealHandler(handler: LocatorAutoHealHandler | undefined): void;
|
|
16
|
+
export declare function getLocatorAutoHealHandler(): LocatorAutoHealHandler | undefined;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { Locator, Page } from
|
|
2
|
-
import { type ExtractSchema, type SchemaOutput } from
|
|
1
|
+
import type { Locator, Page } from "@stablyai/internal-playwright-test";
|
|
2
|
+
import { type ExtractSchema, type SchemaOutput } from "../../ai/extract";
|
|
3
3
|
type ExtractOptions<T extends ExtractSchema> = {
|
|
4
4
|
schema: T;
|
|
5
5
|
};
|
|
@@ -4,7 +4,7 @@ exports.createPageExtract = exports.createLocatorExtract = void 0;
|
|
|
4
4
|
const extract_1 = require("../../ai/extract");
|
|
5
5
|
function createExtract(pageOrLocator) {
|
|
6
6
|
const impl = (async (prompt, options) => {
|
|
7
|
-
if (options
|
|
7
|
+
if (options?.schema) {
|
|
8
8
|
return (0, extract_1.extract)({
|
|
9
9
|
prompt,
|
|
10
10
|
schema: options.schema,
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { BrowserContext, Page } from "playwright";
|
|
2
|
+
import type { TestInfo } from "@playwright/test";
|
|
3
|
+
export declare function setContextTestInfo(context: BrowserContext, testInfo: TestInfo | undefined): void;
|
|
4
|
+
export declare function getContextTestInfo(context: BrowserContext): TestInfo | undefined;
|
|
5
|
+
export declare function setPageTestInfo(page: Page, testInfo: TestInfo | undefined): void;
|
|
6
|
+
export declare function getPageTestInfo(page: Page): TestInfo | undefined;
|
|
7
|
+
export declare function inheritTestInfoFromContext(page: Page): void;
|
|
8
|
+
export type AutoHealEventOutcome = "fallback" | "retry" | "failed" | "handler-error";
|
|
9
|
+
export interface AutoHealEvent {
|
|
10
|
+
description: string;
|
|
11
|
+
method: string;
|
|
12
|
+
outcome: AutoHealEventOutcome;
|
|
13
|
+
details?: string;
|
|
14
|
+
error?: unknown;
|
|
15
|
+
}
|
|
16
|
+
export declare function recordAutoHealEvent(page: Page, event: AutoHealEvent): void;
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
const TEST_INFO_SYMBOL = Symbol.for("stably.playwright.testInfo");
|
|
2
|
+
function defineHiddenProperty(target, key, value) {
|
|
3
|
+
Object.defineProperty(target, key, {
|
|
4
|
+
value,
|
|
5
|
+
enumerable: false,
|
|
6
|
+
configurable: true,
|
|
7
|
+
writable: true,
|
|
8
|
+
});
|
|
9
|
+
}
|
|
10
|
+
function getHiddenProperty(target, key) {
|
|
11
|
+
return target[key];
|
|
12
|
+
}
|
|
13
|
+
export function setContextTestInfo(context, testInfo) {
|
|
14
|
+
defineHiddenProperty(context, TEST_INFO_SYMBOL, testInfo);
|
|
15
|
+
}
|
|
16
|
+
export function getContextTestInfo(context) {
|
|
17
|
+
return getHiddenProperty(context, TEST_INFO_SYMBOL);
|
|
18
|
+
}
|
|
19
|
+
export function setPageTestInfo(page, testInfo) {
|
|
20
|
+
defineHiddenProperty(page, TEST_INFO_SYMBOL, testInfo);
|
|
21
|
+
}
|
|
22
|
+
export function getPageTestInfo(page) {
|
|
23
|
+
return getHiddenProperty(page, TEST_INFO_SYMBOL);
|
|
24
|
+
}
|
|
25
|
+
function safeGetContext(page) {
|
|
26
|
+
try {
|
|
27
|
+
return page.context();
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
return undefined;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
export function inheritTestInfoFromContext(page) {
|
|
34
|
+
if (getPageTestInfo(page)) {
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
const context = safeGetContext(page);
|
|
38
|
+
if (!context) {
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
const testInfo = getContextTestInfo(context);
|
|
42
|
+
if (testInfo) {
|
|
43
|
+
setPageTestInfo(page, testInfo);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
function resolveTestInfo(page) {
|
|
47
|
+
const pageInfo = getPageTestInfo(page);
|
|
48
|
+
if (pageInfo) {
|
|
49
|
+
return pageInfo;
|
|
50
|
+
}
|
|
51
|
+
const context = safeGetContext(page);
|
|
52
|
+
if (!context) {
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
return getContextTestInfo(context);
|
|
56
|
+
}
|
|
57
|
+
function toErrorMessage(error) {
|
|
58
|
+
if (!error)
|
|
59
|
+
return undefined;
|
|
60
|
+
if (error instanceof Error) {
|
|
61
|
+
return error.stack ?? error.message ?? error.name;
|
|
62
|
+
}
|
|
63
|
+
if (typeof error === "string") {
|
|
64
|
+
return error;
|
|
65
|
+
}
|
|
66
|
+
try {
|
|
67
|
+
return JSON.stringify(error);
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return String(error);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
export function recordAutoHealEvent(page, event) {
|
|
74
|
+
const testInfo = resolveTestInfo(page);
|
|
75
|
+
if (!testInfo || !Array.isArray(testInfo.attachments)) {
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
const lines = [
|
|
79
|
+
`Auto-heal description: ${event.description}`,
|
|
80
|
+
`Locator method: ${event.method}`,
|
|
81
|
+
`Outcome: ${event.outcome}`,
|
|
82
|
+
];
|
|
83
|
+
if (event.details) {
|
|
84
|
+
lines.push(`Details: ${event.details}`);
|
|
85
|
+
}
|
|
86
|
+
const errorMessage = toErrorMessage(event.error);
|
|
87
|
+
if (errorMessage) {
|
|
88
|
+
lines.push(`Error: ${errorMessage}`);
|
|
89
|
+
}
|
|
90
|
+
const body = Buffer.from(lines.join("\n"), "utf-8");
|
|
91
|
+
testInfo.attachments.push({
|
|
92
|
+
name: `Stably Auto-heal (${event.method})`,
|
|
93
|
+
contentType: "text/plain",
|
|
94
|
+
body,
|
|
95
|
+
});
|
|
96
|
+
}
|