@empiricalrun/test-gen 0.42.4 → 0.42.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/agent/codegen/repo-edit.js +1 -1
- package/dist/agent/master/action-tool-calls.d.ts +35 -0
- package/dist/agent/master/action-tool-calls.d.ts.map +1 -0
- package/dist/agent/master/action-tool-calls.js +70 -0
- package/dist/agent/master/element-annotation.d.ts +7 -2
- package/dist/agent/master/element-annotation.d.ts.map +1 -1
- package/dist/agent/master/element-annotation.js +2 -1
- package/dist/agent/master/next-action.d.ts +3 -10
- package/dist/agent/master/next-action.d.ts.map +1 -1
- package/dist/agent/master/next-action.js +4 -53
- package/dist/agent/master/run.d.ts.map +1 -1
- package/dist/agent/master/run.js +58 -41
- package/dist/agent/master/with-hints.d.ts.map +1 -1
- package/dist/agent/master/with-hints.js +4 -1
- package/dist/browser-injected-scripts/annotate-elements.js +84 -43
- package/dist/browser-injected-scripts/annotate-elements.spec.d.ts +2 -0
- package/dist/browser-injected-scripts/annotate-elements.spec.d.ts.map +1 -0
- package/dist/browser-injected-scripts/annotate-elements.spec.js +201 -0
- package/dist/browser-injected-scripts/annotate-elements.spec.ts +50 -9
- package/dist/evals/master-agent.evals.d.ts.map +1 -1
- package/dist/evals/master-agent.evals.js +2 -0
- package/package.json +4 -4
- package/playwright.config.ts +1 -1
- package/browser-injected-scripts/annotate-elements.js +0 -491
- package/browser-injected-scripts/annotate-elements.spec.ts +0 -277
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
/* eslint-disable no-
|
|
1
|
+
/* eslint-disable autofix/no-unused-vars */
|
|
2
2
|
/**
|
|
3
3
|
* Annotates all clickable elements on the page with unique hint markers.
|
|
4
4
|
* Returns an object containing annotations and methods to enable/disable them.
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* @param {string} options.markerClass - CSS class to apply to hint markers.
|
|
10
10
|
* @returns {Object} An object containing annotations map and enable/disable methods.
|
|
11
11
|
*/
|
|
12
|
-
function annotateClickableElements({ options = {}, preference } = {}) {
|
|
12
|
+
function annotateClickableElements({ options = {}, preference = {} } = {}) {
|
|
13
13
|
const {
|
|
14
14
|
hintCharacterSet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ", // Default set of characters for hints
|
|
15
15
|
maxHints = 1000, // Maximum number of hints to generate
|
|
@@ -35,14 +35,16 @@ function annotateClickableElements({ options = {}, preference } = {}) {
|
|
|
35
35
|
if (
|
|
36
36
|
centerX < 0 ||
|
|
37
37
|
centerY < 0 ||
|
|
38
|
-
centerX >
|
|
39
|
-
|
|
38
|
+
centerX >
|
|
39
|
+
(windowToAnnotate.innerWidth || document.documentElement.clientWidth) ||
|
|
40
|
+
centerY >
|
|
41
|
+
(windowToAnnotate.innerHeight || document.documentElement.clientHeight)
|
|
40
42
|
) {
|
|
41
43
|
// Determine the viewport dimensions
|
|
42
44
|
const viewportWidth =
|
|
43
|
-
|
|
45
|
+
windowToAnnotate.innerWidth || document.documentElement.clientWidth;
|
|
44
46
|
const viewportHeight =
|
|
45
|
-
|
|
47
|
+
windowToAnnotate.innerHeight || document.documentElement.clientHeight;
|
|
46
48
|
|
|
47
49
|
// Calculate the new scroll positions to bring the element into the center of the viewport
|
|
48
50
|
const newScrollX = centerX - viewportWidth / 2;
|
|
@@ -60,13 +62,15 @@ function annotateClickableElements({ options = {}, preference } = {}) {
|
|
|
60
62
|
const topElement = document.elementFromPoint(newCenterX, newCenterY);
|
|
61
63
|
const doesElementContainTopElement = element.contains(topElement);
|
|
62
64
|
|
|
63
|
-
|
|
64
65
|
// Restore the original scroll positions
|
|
65
66
|
windowToAnnotate.scrollTo(originalScrollX, originalScrollY);
|
|
66
67
|
return doesElementContainTopElement;
|
|
67
68
|
}
|
|
68
69
|
|
|
69
|
-
const topElement = windowToAnnotate.document.elementFromPoint(
|
|
70
|
+
const topElement = windowToAnnotate.document.elementFromPoint(
|
|
71
|
+
centerX,
|
|
72
|
+
centerY,
|
|
73
|
+
);
|
|
70
74
|
|
|
71
75
|
// Check if the topmost element is the target element or one of its descendants
|
|
72
76
|
return element.contains(topElement);
|
|
@@ -98,7 +102,7 @@ function annotateClickableElements({ options = {}, preference } = {}) {
|
|
|
98
102
|
acc
|
|
99
103
|
.map((a) => curr.map((b) => a.concat([b])))
|
|
100
104
|
.reduce((a, b) => a.concat(b), []),
|
|
101
|
-
[[]]
|
|
105
|
+
[[]],
|
|
102
106
|
);
|
|
103
107
|
}
|
|
104
108
|
|
|
@@ -121,7 +125,7 @@ function annotateClickableElements({ options = {}, preference } = {}) {
|
|
|
121
125
|
style.display === "none" ||
|
|
122
126
|
style.visibility === "hidden" ||
|
|
123
127
|
// This is done for cases where opacity is undefined
|
|
124
|
-
// parseFloat(style.opacity) === 0
|
|
128
|
+
// parseFloat(style.opacity) === 0
|
|
125
129
|
style.pointerEvents === "none"
|
|
126
130
|
) {
|
|
127
131
|
return false;
|
|
@@ -173,11 +177,11 @@ function annotateClickableElements({ options = {}, preference } = {}) {
|
|
|
173
177
|
// Check for onclick attribute or listener
|
|
174
178
|
if (
|
|
175
179
|
(element.hasAttribute("onclick") ||
|
|
176
|
-
|
|
180
|
+
typeof element.onclick === "function") &&
|
|
177
181
|
// check for parent element having same on click
|
|
178
|
-
|
|
182
|
+
element.parentNode.onclick !== element.onclick &&
|
|
179
183
|
// parent element should not be a focusable tag like button
|
|
180
|
-
|
|
184
|
+
focusableTags.indexOf(element.parentNode.tagName.toLowerCase()) === -1
|
|
181
185
|
) {
|
|
182
186
|
isClickable = true;
|
|
183
187
|
}
|
|
@@ -227,7 +231,6 @@ function annotateClickableElements({ options = {}, preference } = {}) {
|
|
|
227
231
|
}
|
|
228
232
|
}
|
|
229
233
|
|
|
230
|
-
|
|
231
234
|
if (focusableTags.includes(tagName)) {
|
|
232
235
|
switch (tagName) {
|
|
233
236
|
case "a":
|
|
@@ -321,7 +324,7 @@ function annotateClickableElements({ options = {}, preference } = {}) {
|
|
|
321
324
|
// Create a hint marker
|
|
322
325
|
function createHintMarker(el, hint, parentElement, windowToAnnotate) {
|
|
323
326
|
const rect = el.getBoundingClientRect();
|
|
324
|
-
|
|
327
|
+
|
|
325
328
|
// Create the marker element
|
|
326
329
|
const marker = document.createElement("div");
|
|
327
330
|
marker.textContent = hint;
|
|
@@ -350,13 +353,12 @@ function annotateClickableElements({ options = {}, preference } = {}) {
|
|
|
350
353
|
fontWeight: "bold",
|
|
351
354
|
textShadow: "rgba(255, 255, 255, 0.6) 0px 1px 0px",
|
|
352
355
|
});
|
|
353
|
-
|
|
356
|
+
|
|
354
357
|
// Attach the marker to the specified parent element
|
|
355
358
|
parentElement.appendChild(marker);
|
|
356
359
|
parentElements.push(parentElement);
|
|
357
360
|
return marker;
|
|
358
361
|
}
|
|
359
|
-
|
|
360
362
|
|
|
361
363
|
// Clear existing annotations
|
|
362
364
|
//TODO: Handle clearing annotations
|
|
@@ -371,51 +373,85 @@ function annotateClickableElements({ options = {}, preference } = {}) {
|
|
|
371
373
|
function isInputElement(el) {
|
|
372
374
|
// Check if it's an input with a text-like type
|
|
373
375
|
if (el instanceof HTMLInputElement) {
|
|
374
|
-
const textTypes = [
|
|
376
|
+
const textTypes = [
|
|
377
|
+
"text",
|
|
378
|
+
"email",
|
|
379
|
+
"number",
|
|
380
|
+
"password",
|
|
381
|
+
"search",
|
|
382
|
+
"tel",
|
|
383
|
+
"url",
|
|
384
|
+
"checkbox",
|
|
385
|
+
];
|
|
375
386
|
return textTypes.includes(el.type.toLowerCase());
|
|
376
387
|
}
|
|
377
|
-
|
|
388
|
+
|
|
378
389
|
// Check if it's a textarea
|
|
379
390
|
if (el instanceof HTMLTextAreaElement) {
|
|
380
391
|
return true;
|
|
381
392
|
}
|
|
382
|
-
|
|
393
|
+
|
|
383
394
|
// Check if it's contentEditable
|
|
384
395
|
return el.isContentEditable;
|
|
385
396
|
}
|
|
386
397
|
|
|
398
|
+
function isRequiredTextPresent(el, assertionText) {
|
|
399
|
+
assertionText = assertionText?.trim().toLowerCase();
|
|
400
|
+
if (assertionText) {
|
|
401
|
+
let elementText = el.innerText?.trim().toLowerCase();
|
|
402
|
+
let placeHolderText = el.placeholder?.trim().toLowerCase();
|
|
403
|
+
if (elementText === assertionText || placeHolderText === assertionText) {
|
|
404
|
+
//Check if the text for parent and child is same or not, if its same don't annotate again
|
|
405
|
+
if (el.parentNode.innerText?.toLowerCase() !== elementText) {
|
|
406
|
+
return true;
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
return false;
|
|
410
|
+
}
|
|
411
|
+
return true;
|
|
412
|
+
}
|
|
413
|
+
|
|
387
414
|
// Initialize annotations for a given window (including iframes)
|
|
388
415
|
function initializeAnnotations(windowToAnnotate, parentHints, depth) {
|
|
389
416
|
const container =
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
417
|
+
parentHints?.nodeName === "IFRAME"
|
|
418
|
+
? parentHints.contentWindow.document.body
|
|
419
|
+
: annotationsContainer;
|
|
393
420
|
|
|
394
421
|
// Ensure the container exists
|
|
395
422
|
if (!container) return;
|
|
396
|
-
|
|
397
|
-
// Filter for clickable elements
|
|
398
|
-
const clickableElements = Array.from(windowToAnnotate.document.querySelectorAll("*")).filter((el) => {
|
|
399
423
|
|
|
424
|
+
// Filter for clickable elements
|
|
425
|
+
const clickableElements = Array.from(
|
|
426
|
+
windowToAnnotate.document.querySelectorAll("*"),
|
|
427
|
+
).filter((el) => {
|
|
400
428
|
//If preference is fill then it should only annotate input elements
|
|
401
|
-
if(preference === "fill"){
|
|
429
|
+
if (preference.actionType === "fill") {
|
|
402
430
|
if (!isInputElement(el)) {
|
|
403
431
|
return false;
|
|
404
432
|
}
|
|
405
433
|
}
|
|
406
434
|
|
|
435
|
+
//If preference is assert text then only text with the text to be asserted should be highlighted.
|
|
436
|
+
if (preference.actionType === "assert_text") {
|
|
437
|
+
return isRequiredTextPresent(el, preference.assertionText);
|
|
438
|
+
}
|
|
439
|
+
|
|
407
440
|
const isClickable = isElementClickable(el, windowToAnnotate);
|
|
408
441
|
const isClickNotBlocked = isElementClickNotBlocked(el, windowToAnnotate);
|
|
409
442
|
return isClickable && isClickNotBlocked;
|
|
410
|
-
});
|
|
443
|
+
});
|
|
411
444
|
// Generate hint strings for the clickable elements
|
|
412
|
-
const hints = generateHintStrings(
|
|
413
|
-
|
|
445
|
+
const hints = generateHintStrings(
|
|
446
|
+
hintCharacterSet,
|
|
447
|
+
Math.min(maxHints, clickableElements.length),
|
|
448
|
+
);
|
|
449
|
+
|
|
414
450
|
// Create markers for the elements
|
|
415
451
|
clickableElements.slice(0, maxHints).forEach((el, index) => {
|
|
416
452
|
const hint = hints[index];
|
|
417
453
|
const rect = el.getBoundingClientRect();
|
|
418
|
-
|
|
454
|
+
|
|
419
455
|
// Use createHintMarker with the specified container
|
|
420
456
|
createHintMarker(el, hint, container, windowToAnnotate);
|
|
421
457
|
el.style.boxShadow = `inset 0 0 0px 2px red`;
|
|
@@ -429,23 +465,28 @@ function annotateClickableElements({ options = {}, preference } = {}) {
|
|
|
429
465
|
width: rect.width,
|
|
430
466
|
height: rect.height,
|
|
431
467
|
},
|
|
432
|
-
depth: [...depth
|
|
468
|
+
depth: [...depth],
|
|
433
469
|
};
|
|
434
470
|
});
|
|
435
|
-
|
|
471
|
+
|
|
436
472
|
// Process iframes recursively
|
|
437
|
-
Array.from(windowToAnnotate.document.querySelectorAll("iframe")).forEach(
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
473
|
+
Array.from(windowToAnnotate.document.querySelectorAll("iframe")).forEach(
|
|
474
|
+
(iframe) => {
|
|
475
|
+
try {
|
|
476
|
+
const frameWindow = iframe.contentWindow;
|
|
477
|
+
if (
|
|
478
|
+
frameWindow &&
|
|
479
|
+
iframe.offsetWidth > 0 &&
|
|
480
|
+
iframe.offsetHeight > 0
|
|
481
|
+
) {
|
|
482
|
+
initializeAnnotations(frameWindow, iframe, [...depth, iframe]);
|
|
483
|
+
}
|
|
484
|
+
} catch (e) {
|
|
485
|
+
console.warn("Cannot access iframe:", e);
|
|
442
486
|
}
|
|
443
|
-
}
|
|
444
|
-
|
|
445
|
-
}
|
|
446
|
-
});
|
|
487
|
+
},
|
|
488
|
+
);
|
|
447
489
|
}
|
|
448
|
-
|
|
449
490
|
|
|
450
491
|
// Initialize and enable annotations
|
|
451
492
|
function enable() {
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"annotate-elements.spec.d.ts","sourceRoot":"","sources":["../../src/browser-injected-scripts/annotate-elements.spec.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// @ts-nocheck
|
|
3
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
4
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
5
|
+
};
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
const test_1 = require("@playwright/test");
|
|
8
|
+
const path_1 = __importDefault(require("path"));
|
|
9
|
+
const action_tool_calls_1 = require("../agent/master/action-tool-calls");
|
|
10
|
+
(0, test_1.test)("should annotate all links on empirical landing page", async ({ page, }) => {
|
|
11
|
+
await page.goto("https://assets-test.empirical.run/selector-hints-testing/dom-1.html");
|
|
12
|
+
await page.addScriptTag({
|
|
13
|
+
path: path_1.default.resolve(__dirname, "./annotate-elements.js"),
|
|
14
|
+
});
|
|
15
|
+
const annotations = await page.evaluate(() => {
|
|
16
|
+
// eslint-disable-next-line no-undef
|
|
17
|
+
const { annotations } = annotateClickableElements();
|
|
18
|
+
return Object.entries(annotations).map(([, config]) => ({
|
|
19
|
+
innerText: config.node.innerText,
|
|
20
|
+
tagName: config.node.tagName,
|
|
21
|
+
href: config.node.href,
|
|
22
|
+
}));
|
|
23
|
+
});
|
|
24
|
+
test_1.test.expect(annotations).toEqual([
|
|
25
|
+
{
|
|
26
|
+
innerText: "Empirical",
|
|
27
|
+
tagName: "A",
|
|
28
|
+
href: "https://assets-test.empirical.run/",
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
innerText: "Blog",
|
|
32
|
+
tagName: "A",
|
|
33
|
+
href: "https://assets-test.empirical.run/blog",
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
innerText: "Contact us",
|
|
37
|
+
tagName: "A",
|
|
38
|
+
href: "https://assets-test.empirical.run/contact",
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
href: "https://dash.empirical.run/",
|
|
42
|
+
innerText: "Login ↗\n(opens in a new tab)",
|
|
43
|
+
tagName: "A",
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
innerText: "Get early access",
|
|
47
|
+
tagName: "A",
|
|
48
|
+
href: "https://assets-test.empirical.run/contact",
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
innerText: "Playwright\n(opens in a new tab)",
|
|
52
|
+
tagName: "A",
|
|
53
|
+
href: "https://github.com/microsoft/playwright",
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
innerText: "Meet with us",
|
|
57
|
+
tagName: "A",
|
|
58
|
+
href: "https://assets-test.empirical.run/contact",
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
innerText: "Privacy Policy",
|
|
62
|
+
tagName: "A",
|
|
63
|
+
href: "https://assets-test.empirical.run/privacy.html",
|
|
64
|
+
},
|
|
65
|
+
]);
|
|
66
|
+
});
|
|
67
|
+
(0, test_1.test)("should annotate all important items on quizizz page", async ({ page, }) => {
|
|
68
|
+
await page.goto("https://assets-test.empirical.run/selector-hints-testing/dom-2/index.html");
|
|
69
|
+
await page.addScriptTag({
|
|
70
|
+
path: path_1.default.resolve(__dirname, "./annotate-elements.js"),
|
|
71
|
+
});
|
|
72
|
+
const annotations = await page.evaluate(() => {
|
|
73
|
+
// eslint-disable-next-line no-undef
|
|
74
|
+
const { annotations } = annotateClickableElements();
|
|
75
|
+
return Object.entries(annotations).map(([hint, config]) => ({
|
|
76
|
+
hint,
|
|
77
|
+
innerText: config.node.innerText.toLowerCase().trim(),
|
|
78
|
+
tagName: config.node.tagName,
|
|
79
|
+
testId: config.node.getAttribute("data-testid"),
|
|
80
|
+
href: config.node.href,
|
|
81
|
+
}));
|
|
82
|
+
});
|
|
83
|
+
test_1.test
|
|
84
|
+
.expect(annotations.find((item) => item.innerText.includes("enter code")))
|
|
85
|
+
.toBeTruthy();
|
|
86
|
+
test_1.test
|
|
87
|
+
.expect(annotations.find((item) => item.innerText.includes("get help")))
|
|
88
|
+
.toBeTruthy();
|
|
89
|
+
test_1.test
|
|
90
|
+
.expect(annotations.find((item) => item.innerText.includes("create") &&
|
|
91
|
+
item.testId === "create-content-button"))
|
|
92
|
+
.toBeTruthy();
|
|
93
|
+
test_1.test
|
|
94
|
+
.expect(annotations.find((item) => item.innerText.includes("explore") &&
|
|
95
|
+
item.href === "https://quizizz.com/admin"))
|
|
96
|
+
.toBeTruthy();
|
|
97
|
+
test_1.test
|
|
98
|
+
.expect(annotations.find((item) => item.innerText.includes("library") &&
|
|
99
|
+
item.href === "https://quizizz.com/admin/my-library/createdByMe"))
|
|
100
|
+
.toBeTruthy();
|
|
101
|
+
test_1.test
|
|
102
|
+
.expect(annotations.find((item) => item.innerText.includes("reports") &&
|
|
103
|
+
item.href === "https://quizizz.com/admin/reports"))
|
|
104
|
+
.toBeTruthy();
|
|
105
|
+
test_1.test
|
|
106
|
+
.expect(annotations.find((item) => item.innerText.includes("classes") &&
|
|
107
|
+
item.href === "https://quizizz.com/admin/classes"))
|
|
108
|
+
.toBeTruthy();
|
|
109
|
+
test_1.test
|
|
110
|
+
.expect(annotations.find((item) => item.innerText.includes("accommodations") &&
|
|
111
|
+
item.href ===
|
|
112
|
+
"https://quizizz.com/admin/differentiation/accommodations"))
|
|
113
|
+
.toBeTruthy();
|
|
114
|
+
test_1.test
|
|
115
|
+
.expect(annotations.find((item) => item.innerText.includes("quizizz ai") &&
|
|
116
|
+
item.href === "https://quizizz.com/admin/quizizz-ai"))
|
|
117
|
+
.toBeTruthy();
|
|
118
|
+
test_1.test
|
|
119
|
+
.expect(annotations.find((item) => item.innerText.includes("start your free trial") &&
|
|
120
|
+
item.href === "https://quizizz.com/super-pricing"))
|
|
121
|
+
.toBeTruthy();
|
|
122
|
+
test_1.test
|
|
123
|
+
.expect(annotations.find((item) => item.innerText.includes("upgrade") &&
|
|
124
|
+
item.href === "https://quizizz.com/super-pricing?backto=/admin"))
|
|
125
|
+
.toBeTruthy();
|
|
126
|
+
test_1.test
|
|
127
|
+
.expect(annotations.find((item) => item.tagName === "INPUT" &&
|
|
128
|
+
item.testId === "emphasized-search-bar-input"))
|
|
129
|
+
.toBeTruthy();
|
|
130
|
+
test_1.test
|
|
131
|
+
.expect(annotations.find((item) => item.tagName === "BUTTON" &&
|
|
132
|
+
item.innerText.includes("verify details") &&
|
|
133
|
+
item.testId === "verify-profile-cta"))
|
|
134
|
+
.toBeTruthy();
|
|
135
|
+
test_1.test
|
|
136
|
+
.expect(annotations.find((item) => item.tagName === "BUTTON" && item.innerText.includes("for you")))
|
|
137
|
+
.toBeTruthy();
|
|
138
|
+
test_1.test
|
|
139
|
+
.expect(annotations.find((item) => item.tagName === "BUTTON" && item.innerText.includes("assessments")))
|
|
140
|
+
.toBeTruthy();
|
|
141
|
+
test_1.test
|
|
142
|
+
.expect(annotations.find((item) => item.tagName === "BUTTON" && item.innerText.includes("lessons")))
|
|
143
|
+
.toBeTruthy();
|
|
144
|
+
test_1.test
|
|
145
|
+
.expect(annotations.find((item) => item.tagName === "BUTTON" &&
|
|
146
|
+
item.innerText.includes("interactive videos")))
|
|
147
|
+
.toBeTruthy();
|
|
148
|
+
test_1.test
|
|
149
|
+
.expect(annotations.find((item) => item.tagName === "BUTTON" && item.innerText.includes("passages")))
|
|
150
|
+
.toBeTruthy();
|
|
151
|
+
});
|
|
152
|
+
(0, test_1.test)("should only annotate input fields on quizizz page", async ({ page }) => {
|
|
153
|
+
await page.goto("https://assets-test.empirical.run/selector-hints-testing/dom-2/index.html");
|
|
154
|
+
await page.addScriptTag({
|
|
155
|
+
path: path_1.default.resolve(__dirname, "./annotate-elements.js"),
|
|
156
|
+
});
|
|
157
|
+
const preference = {
|
|
158
|
+
actionType: action_tool_calls_1.ActionType.FILL,
|
|
159
|
+
};
|
|
160
|
+
const annotations = await page.evaluate((preference) => {
|
|
161
|
+
// eslint-disable-next-line no-undef
|
|
162
|
+
const { annotations } = annotateClickableElements({
|
|
163
|
+
preference: preference,
|
|
164
|
+
});
|
|
165
|
+
return Object.entries(annotations).map(([hint, config]) => ({
|
|
166
|
+
hint,
|
|
167
|
+
innerText: config.node.innerText.toLowerCase().trim(),
|
|
168
|
+
tagName: config.node.tagName,
|
|
169
|
+
testId: config.node.getAttribute("data-testid"),
|
|
170
|
+
href: config.node.href,
|
|
171
|
+
}));
|
|
172
|
+
}, preference);
|
|
173
|
+
test_1.test.expect(annotations.length).toBe(1);
|
|
174
|
+
test_1.test.expect(annotations[0].testId).toBe("emphasized-search-bar-input");
|
|
175
|
+
});
|
|
176
|
+
(0, test_1.test)("should only annotate given text on quizziz page", async ({ page }) => {
|
|
177
|
+
await page.goto("https://assets-test.empirical.run/selector-hints-testing/dom-2/index.html");
|
|
178
|
+
await page.addScriptTag({
|
|
179
|
+
path: path_1.default.resolve(__dirname, "./annotate-elements.js"),
|
|
180
|
+
});
|
|
181
|
+
const preference = {
|
|
182
|
+
actionType: action_tool_calls_1.ActionType.ASSERT_TEXT,
|
|
183
|
+
assertionText: "What are you teaching today?",
|
|
184
|
+
};
|
|
185
|
+
const annotations = await page.evaluate((preference) => {
|
|
186
|
+
// eslint-disable-next-line no-undef
|
|
187
|
+
const { annotations } = annotateClickableElements({
|
|
188
|
+
preference: preference,
|
|
189
|
+
});
|
|
190
|
+
return Object.entries(annotations).map(([hint, config]) => ({
|
|
191
|
+
hint,
|
|
192
|
+
innerText: config.node.innerText?.toLowerCase().trim(),
|
|
193
|
+
tagName: config.node.tagName,
|
|
194
|
+
testId: config.node.getAttribute("data-testid"),
|
|
195
|
+
href: config.node.href,
|
|
196
|
+
}));
|
|
197
|
+
}, preference);
|
|
198
|
+
console.log("Annotations:", annotations);
|
|
199
|
+
test_1.test.expect(annotations.length).toBe(1);
|
|
200
|
+
test_1.test.expect(annotations[0]?.tagName).toBe("DIV");
|
|
201
|
+
});
|
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
import { test } from "@playwright/test";
|
|
4
4
|
import path from "path";
|
|
5
5
|
|
|
6
|
+
import { ActionType } from "../agent/master/action-tool-calls";
|
|
7
|
+
import { AnnotationPreference } from "../agent/master/element-annotation";
|
|
8
|
+
|
|
6
9
|
test("should annotate all links on empirical landing page", async ({
|
|
7
10
|
page,
|
|
8
11
|
}) => {
|
|
@@ -15,9 +18,10 @@ test("should annotate all links on empirical landing page", async ({
|
|
|
15
18
|
});
|
|
16
19
|
|
|
17
20
|
const annotations = await page.evaluate(() => {
|
|
21
|
+
// eslint-disable-next-line no-undef
|
|
18
22
|
const { annotations } = annotateClickableElements();
|
|
19
23
|
|
|
20
|
-
return Object.entries(annotations).map(([
|
|
24
|
+
return Object.entries(annotations).map(([, config]) => ({
|
|
21
25
|
innerText: config.node.innerText,
|
|
22
26
|
tagName: config.node.tagName,
|
|
23
27
|
href: config.node.href,
|
|
@@ -80,6 +84,7 @@ test("should annotate all important items on quizizz page", async ({
|
|
|
80
84
|
});
|
|
81
85
|
|
|
82
86
|
const annotations = await page.evaluate(() => {
|
|
87
|
+
// eslint-disable-next-line no-undef
|
|
83
88
|
const { annotations } = annotateClickableElements();
|
|
84
89
|
|
|
85
90
|
return Object.entries(annotations).map(([hint, config]) => ({
|
|
@@ -148,7 +153,7 @@ test("should annotate all important items on quizizz page", async ({
|
|
|
148
153
|
(item) =>
|
|
149
154
|
item.innerText.includes("accommodations") &&
|
|
150
155
|
item.href ===
|
|
151
|
-
|
|
156
|
+
"https://quizizz.com/admin/differentiation/accommodations",
|
|
152
157
|
),
|
|
153
158
|
)
|
|
154
159
|
.toBeTruthy();
|
|
@@ -248,9 +253,7 @@ test("should annotate all important items on quizizz page", async ({
|
|
|
248
253
|
.toBeTruthy();
|
|
249
254
|
});
|
|
250
255
|
|
|
251
|
-
test("should only annotate input fields on quizizz page", async ({
|
|
252
|
-
page,
|
|
253
|
-
}) => {
|
|
256
|
+
test("should only annotate input fields on quizizz page", async ({ page }) => {
|
|
254
257
|
await page.goto(
|
|
255
258
|
"https://assets-test.empirical.run/selector-hints-testing/dom-2/index.html",
|
|
256
259
|
);
|
|
@@ -259,9 +262,14 @@ test("should only annotate input fields on quizizz page", async ({
|
|
|
259
262
|
path: path.resolve(__dirname, "./annotate-elements.js"),
|
|
260
263
|
});
|
|
261
264
|
|
|
262
|
-
const preference =
|
|
265
|
+
const preference = {
|
|
266
|
+
actionType: ActionType.FILL,
|
|
267
|
+
};
|
|
263
268
|
const annotations = await page.evaluate((preference) => {
|
|
264
|
-
|
|
269
|
+
// eslint-disable-next-line no-undef
|
|
270
|
+
const { annotations } = annotateClickableElements({
|
|
271
|
+
preference: preference,
|
|
272
|
+
});
|
|
265
273
|
|
|
266
274
|
return Object.entries(annotations).map(([hint, config]) => ({
|
|
267
275
|
hint,
|
|
@@ -270,8 +278,41 @@ test("should only annotate input fields on quizizz page", async ({
|
|
|
270
278
|
testId: config.node.getAttribute("data-testid"),
|
|
271
279
|
href: config.node.href,
|
|
272
280
|
}));
|
|
273
|
-
},preference);
|
|
281
|
+
}, preference);
|
|
282
|
+
|
|
283
|
+
test.expect(annotations.length).toBe(1);
|
|
284
|
+
test.expect(annotations[0].testId).toBe("emphasized-search-bar-input");
|
|
285
|
+
});
|
|
286
|
+
|
|
287
|
+
test("should only annotate given text on quizziz page", async ({ page }) => {
|
|
288
|
+
await page.goto(
|
|
289
|
+
"https://assets-test.empirical.run/selector-hints-testing/dom-2/index.html",
|
|
290
|
+
);
|
|
291
|
+
|
|
292
|
+
await page.addScriptTag({
|
|
293
|
+
path: path.resolve(__dirname, "./annotate-elements.js"),
|
|
294
|
+
});
|
|
295
|
+
|
|
296
|
+
const preference: AnnotationPreference = {
|
|
297
|
+
actionType: ActionType.ASSERT_TEXT,
|
|
298
|
+
assertionText: "What are you teaching today?",
|
|
299
|
+
};
|
|
300
|
+
const annotations = await page.evaluate((preference) => {
|
|
301
|
+
// eslint-disable-next-line no-undef
|
|
302
|
+
const { annotations } = annotateClickableElements({
|
|
303
|
+
preference: preference,
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
return Object.entries(annotations).map(([hint, config]) => ({
|
|
307
|
+
hint,
|
|
308
|
+
innerText: config.node.innerText?.toLowerCase().trim(),
|
|
309
|
+
tagName: config.node.tagName,
|
|
310
|
+
testId: config.node.getAttribute("data-testid"),
|
|
311
|
+
href: config.node.href,
|
|
312
|
+
}));
|
|
313
|
+
}, preference);
|
|
274
314
|
|
|
315
|
+
console.log("Annotations:", annotations);
|
|
275
316
|
test.expect(annotations.length).toBe(1);
|
|
276
|
-
test.expect(annotations[0]
|
|
317
|
+
test.expect(annotations[0]?.tagName).toBe("DIV");
|
|
277
318
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"master-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/master-agent.evals.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"master-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/master-agent.evals.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,4BAA4B,EAAE,UA2C1C,CAAC;AAEF,eAAe,4BAA4B,CAAC"}
|
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.masterGetNextActionEvaluator = void 0;
|
|
4
4
|
const actions_1 = require("../actions");
|
|
5
5
|
const skill_1 = require("../actions/skill");
|
|
6
|
+
const action_tool_calls_1 = require("../agent/master/action-tool-calls");
|
|
6
7
|
const next_action_1 = require("../agent/master/next-action");
|
|
7
8
|
const page_1 = require("../page");
|
|
8
9
|
const masterGetNextActionEvaluator = async ({ item, trace, }) => {
|
|
@@ -20,6 +21,7 @@ const masterGetNextActionEvaluator = async ({ item, trace, }) => {
|
|
|
20
21
|
pageScreenshot,
|
|
21
22
|
annotatedPageScreenshot,
|
|
22
23
|
actions,
|
|
24
|
+
actionTypes: action_tool_calls_1.ActionType,
|
|
23
25
|
disableSkills,
|
|
24
26
|
annotations,
|
|
25
27
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@empiricalrun/test-gen",
|
|
3
|
-
"version": "0.42.
|
|
3
|
+
"version": "0.42.8",
|
|
4
4
|
"publishConfig": {
|
|
5
5
|
"registry": "https://registry.npmjs.org/",
|
|
6
6
|
"access": "public"
|
|
@@ -71,9 +71,9 @@
|
|
|
71
71
|
"ts-morph": "^24.0.0",
|
|
72
72
|
"tsx": "^4.16.2",
|
|
73
73
|
"typescript": "^5.3.3",
|
|
74
|
+
"@empiricalrun/llm": "^0.9.30",
|
|
74
75
|
"@empiricalrun/r2-uploader": "^0.3.8",
|
|
75
|
-
"@empiricalrun/reporter": "^0.22.1"
|
|
76
|
-
"@empiricalrun/llm": "^0.9.29"
|
|
76
|
+
"@empiricalrun/reporter": "^0.22.1"
|
|
77
77
|
},
|
|
78
78
|
"devDependencies": {
|
|
79
79
|
"@playwright/test": "1.47.1",
|
|
@@ -88,7 +88,7 @@
|
|
|
88
88
|
},
|
|
89
89
|
"scripts": {
|
|
90
90
|
"dev": "tsc --build --watch",
|
|
91
|
-
"build": "tsc --build && cp -r browser-injected-scripts dist",
|
|
91
|
+
"build": "tsc --build && cp -r src/browser-injected-scripts dist",
|
|
92
92
|
"clean": "tsc --build --clean",
|
|
93
93
|
"lint": "eslint .",
|
|
94
94
|
"test": "vitest run",
|
package/playwright.config.ts
CHANGED