@empiricalrun/test-gen 0.42.3 → 0.42.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. package/CHANGELOG.md +36 -0
  2. package/dist/agent/codegen/repo-edit.js +1 -1
  3. package/dist/agent/master/action-tool-calls.d.ts +35 -0
  4. package/dist/agent/master/action-tool-calls.d.ts.map +1 -0
  5. package/dist/agent/master/action-tool-calls.js +70 -0
  6. package/dist/agent/master/element-annotation.d.ts +7 -2
  7. package/dist/agent/master/element-annotation.d.ts.map +1 -1
  8. package/dist/agent/master/element-annotation.js +2 -1
  9. package/dist/agent/master/next-action.d.ts +3 -10
  10. package/dist/agent/master/next-action.d.ts.map +1 -1
  11. package/dist/agent/master/next-action.js +4 -53
  12. package/dist/agent/master/run.d.ts.map +1 -1
  13. package/dist/agent/master/run.js +58 -41
  14. package/dist/agent/master/with-hints.d.ts.map +1 -1
  15. package/dist/agent/master/with-hints.js +4 -1
  16. package/dist/bin/utils/platform/web/index.d.ts.map +1 -1
  17. package/dist/bin/utils/platform/web/index.js +2 -0
  18. package/dist/browser-injected-scripts/annotate-elements.js +84 -43
  19. package/dist/browser-injected-scripts/annotate-elements.spec.d.ts +2 -0
  20. package/dist/browser-injected-scripts/annotate-elements.spec.d.ts.map +1 -0
  21. package/dist/browser-injected-scripts/annotate-elements.spec.js +201 -0
  22. package/dist/browser-injected-scripts/annotate-elements.spec.ts +50 -9
  23. package/dist/evals/master-agent.evals.d.ts.map +1 -1
  24. package/dist/evals/master-agent.evals.js +2 -0
  25. package/package.json +3 -3
  26. package/playwright.config.ts +1 -1
  27. package/browser-injected-scripts/annotate-elements.js +0 -491
  28. package/browser-injected-scripts/annotate-elements.spec.ts +0 -277
@@ -1,4 +1,4 @@
1
- /* eslint-disable no-undef */
1
+ /* eslint-disable autofix/no-unused-vars */
2
2
  /**
3
3
  * Annotates all clickable elements on the page with unique hint markers.
4
4
  * Returns an object containing annotations and methods to enable/disable them.
@@ -9,7 +9,7 @@
9
9
  * @param {string} options.markerClass - CSS class to apply to hint markers.
10
10
  * @returns {Object} An object containing annotations map and enable/disable methods.
11
11
  */
12
- function annotateClickableElements({ options = {}, preference } = {}) {
12
+ function annotateClickableElements({ options = {}, preference = {} } = {}) {
13
13
  const {
14
14
  hintCharacterSet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ", // Default set of characters for hints
15
15
  maxHints = 1000, // Maximum number of hints to generate
@@ -35,14 +35,16 @@ function annotateClickableElements({ options = {}, preference } = {}) {
35
35
  if (
36
36
  centerX < 0 ||
37
37
  centerY < 0 ||
38
- centerX > (windowToAnnotate.innerWidth || document.documentElement.clientWidth) ||
39
- centerY > (windowToAnnotate.innerHeight || document.documentElement.clientHeight)
38
+ centerX >
39
+ (windowToAnnotate.innerWidth || document.documentElement.clientWidth) ||
40
+ centerY >
41
+ (windowToAnnotate.innerHeight || document.documentElement.clientHeight)
40
42
  ) {
41
43
  // Determine the viewport dimensions
42
44
  const viewportWidth =
43
- windowToAnnotate.innerWidth || document.documentElement.clientWidth;
45
+ windowToAnnotate.innerWidth || document.documentElement.clientWidth;
44
46
  const viewportHeight =
45
- windowToAnnotate.innerHeight || document.documentElement.clientHeight;
47
+ windowToAnnotate.innerHeight || document.documentElement.clientHeight;
46
48
 
47
49
  // Calculate the new scroll positions to bring the element into the center of the viewport
48
50
  const newScrollX = centerX - viewportWidth / 2;
@@ -60,13 +62,15 @@ function annotateClickableElements({ options = {}, preference } = {}) {
60
62
  const topElement = document.elementFromPoint(newCenterX, newCenterY);
61
63
  const doesElementContainTopElement = element.contains(topElement);
62
64
 
63
-
64
65
  // Restore the original scroll positions
65
66
  windowToAnnotate.scrollTo(originalScrollX, originalScrollY);
66
67
  return doesElementContainTopElement;
67
68
  }
68
69
 
69
- const topElement = windowToAnnotate.document.elementFromPoint(centerX, centerY);
70
+ const topElement = windowToAnnotate.document.elementFromPoint(
71
+ centerX,
72
+ centerY,
73
+ );
70
74
 
71
75
  // Check if the topmost element is the target element or one of its descendants
72
76
  return element.contains(topElement);
@@ -98,7 +102,7 @@ function annotateClickableElements({ options = {}, preference } = {}) {
98
102
  acc
99
103
  .map((a) => curr.map((b) => a.concat([b])))
100
104
  .reduce((a, b) => a.concat(b), []),
101
- [[]]
105
+ [[]],
102
106
  );
103
107
  }
104
108
 
@@ -121,7 +125,7 @@ function annotateClickableElements({ options = {}, preference } = {}) {
121
125
  style.display === "none" ||
122
126
  style.visibility === "hidden" ||
123
127
  // This is done for cases where opacity is undefined
124
- // parseFloat(style.opacity) === 0
128
+ // parseFloat(style.opacity) === 0
125
129
  style.pointerEvents === "none"
126
130
  ) {
127
131
  return false;
@@ -173,11 +177,11 @@ function annotateClickableElements({ options = {}, preference } = {}) {
173
177
  // Check for onclick attribute or listener
174
178
  if (
175
179
  (element.hasAttribute("onclick") ||
176
- typeof element.onclick === "function")
180
+ typeof element.onclick === "function") &&
177
181
  // check for parent element having same on click
178
- && element.parentNode.onclick !== element.onclick
182
+ element.parentNode.onclick !== element.onclick &&
179
183
  // parent element should not be a focusable tag like button
180
- && focusableTags.indexOf(element.parentNode.tagName.toLowerCase()) === -1
184
+ focusableTags.indexOf(element.parentNode.tagName.toLowerCase()) === -1
181
185
  ) {
182
186
  isClickable = true;
183
187
  }
@@ -227,7 +231,6 @@ function annotateClickableElements({ options = {}, preference } = {}) {
227
231
  }
228
232
  }
229
233
 
230
-
231
234
  if (focusableTags.includes(tagName)) {
232
235
  switch (tagName) {
233
236
  case "a":
@@ -321,7 +324,7 @@ function annotateClickableElements({ options = {}, preference } = {}) {
321
324
  // Create a hint marker
322
325
  function createHintMarker(el, hint, parentElement, windowToAnnotate) {
323
326
  const rect = el.getBoundingClientRect();
324
-
327
+
325
328
  // Create the marker element
326
329
  const marker = document.createElement("div");
327
330
  marker.textContent = hint;
@@ -350,13 +353,12 @@ function annotateClickableElements({ options = {}, preference } = {}) {
350
353
  fontWeight: "bold",
351
354
  textShadow: "rgba(255, 255, 255, 0.6) 0px 1px 0px",
352
355
  });
353
-
356
+
354
357
  // Attach the marker to the specified parent element
355
358
  parentElement.appendChild(marker);
356
359
  parentElements.push(parentElement);
357
360
  return marker;
358
361
  }
359
-
360
362
 
361
363
  // Clear existing annotations
362
364
  //TODO: Handle clearing annotations
@@ -371,51 +373,85 @@ function annotateClickableElements({ options = {}, preference } = {}) {
371
373
  function isInputElement(el) {
372
374
  // Check if it's an input with a text-like type
373
375
  if (el instanceof HTMLInputElement) {
374
- const textTypes = ['text', 'email', 'number', 'password', 'search', 'tel', 'url', 'checkbox'];
376
+ const textTypes = [
377
+ "text",
378
+ "email",
379
+ "number",
380
+ "password",
381
+ "search",
382
+ "tel",
383
+ "url",
384
+ "checkbox",
385
+ ];
375
386
  return textTypes.includes(el.type.toLowerCase());
376
387
  }
377
-
388
+
378
389
  // Check if it's a textarea
379
390
  if (el instanceof HTMLTextAreaElement) {
380
391
  return true;
381
392
  }
382
-
393
+
383
394
  // Check if it's contentEditable
384
395
  return el.isContentEditable;
385
396
  }
386
397
 
398
+ function isRequiredTextPresent(el, assertionText) {
399
+ assertionText = assertionText?.trim().toLowerCase();
400
+ if (assertionText) {
401
+ let elementText = el.innerText?.trim().toLowerCase();
402
+ let placeHolderText = el.placeholder?.trim().toLowerCase();
403
+ if (elementText === assertionText || placeHolderText === assertionText) {
404
+ //Check if the text for parent and child is same or not, if its same don't annotate again
405
+ if (el.parentNode.innerText?.toLowerCase() !== elementText) {
406
+ return true;
407
+ }
408
+ }
409
+ return false;
410
+ }
411
+ return true;
412
+ }
413
+
387
414
  // Initialize annotations for a given window (including iframes)
388
415
  function initializeAnnotations(windowToAnnotate, parentHints, depth) {
389
416
  const container =
390
- parentHints?.nodeName === "IFRAME"
391
- ? parentHints.contentWindow.document.body
392
- : annotationsContainer;
417
+ parentHints?.nodeName === "IFRAME"
418
+ ? parentHints.contentWindow.document.body
419
+ : annotationsContainer;
393
420
 
394
421
  // Ensure the container exists
395
422
  if (!container) return;
396
-
397
- // Filter for clickable elements
398
- const clickableElements = Array.from(windowToAnnotate.document.querySelectorAll("*")).filter((el) => {
399
423
 
424
+ // Filter for clickable elements
425
+ const clickableElements = Array.from(
426
+ windowToAnnotate.document.querySelectorAll("*"),
427
+ ).filter((el) => {
400
428
  //If preference is fill then it should only annotate input elements
401
- if(preference === "fill"){
429
+ if (preference.actionType === "fill") {
402
430
  if (!isInputElement(el)) {
403
431
  return false;
404
432
  }
405
433
  }
406
434
 
435
+ //If preference is assert text then only text with the text to be asserted should be highlighted.
436
+ if (preference.actionType === "assert_text") {
437
+ return isRequiredTextPresent(el, preference.assertionText);
438
+ }
439
+
407
440
  const isClickable = isElementClickable(el, windowToAnnotate);
408
441
  const isClickNotBlocked = isElementClickNotBlocked(el, windowToAnnotate);
409
442
  return isClickable && isClickNotBlocked;
410
- });
443
+ });
411
444
  // Generate hint strings for the clickable elements
412
- const hints = generateHintStrings(hintCharacterSet, Math.min(maxHints, clickableElements.length));
413
-
445
+ const hints = generateHintStrings(
446
+ hintCharacterSet,
447
+ Math.min(maxHints, clickableElements.length),
448
+ );
449
+
414
450
  // Create markers for the elements
415
451
  clickableElements.slice(0, maxHints).forEach((el, index) => {
416
452
  const hint = hints[index];
417
453
  const rect = el.getBoundingClientRect();
418
-
454
+
419
455
  // Use createHintMarker with the specified container
420
456
  createHintMarker(el, hint, container, windowToAnnotate);
421
457
  el.style.boxShadow = `inset 0 0 0px 2px red`;
@@ -429,23 +465,28 @@ function annotateClickableElements({ options = {}, preference } = {}) {
429
465
  width: rect.width,
430
466
  height: rect.height,
431
467
  },
432
- depth: [...depth ]
468
+ depth: [...depth],
433
469
  };
434
470
  });
435
-
471
+
436
472
  // Process iframes recursively
437
- Array.from(windowToAnnotate.document.querySelectorAll("iframe")).forEach((iframe) => {
438
- try {
439
- const frameWindow = iframe.contentWindow;
440
- if (frameWindow && iframe.offsetWidth > 0 && iframe.offsetHeight > 0) {
441
- initializeAnnotations(frameWindow, iframe, [...depth, iframe]);
473
+ Array.from(windowToAnnotate.document.querySelectorAll("iframe")).forEach(
474
+ (iframe) => {
475
+ try {
476
+ const frameWindow = iframe.contentWindow;
477
+ if (
478
+ frameWindow &&
479
+ iframe.offsetWidth > 0 &&
480
+ iframe.offsetHeight > 0
481
+ ) {
482
+ initializeAnnotations(frameWindow, iframe, [...depth, iframe]);
483
+ }
484
+ } catch (e) {
485
+ console.warn("Cannot access iframe:", e);
442
486
  }
443
- } catch (e) {
444
- console.warn("Cannot access iframe:", e);
445
- }
446
- });
487
+ },
488
+ );
447
489
  }
448
-
449
490
 
450
491
  // Initialize and enable annotations
451
492
  function enable() {
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=annotate-elements.spec.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"annotate-elements.spec.d.ts","sourceRoot":"","sources":["../../src/browser-injected-scripts/annotate-elements.spec.ts"],"names":[],"mappings":""}
@@ -0,0 +1,201 @@
1
+ "use strict";
2
+ // @ts-nocheck
3
+ var __importDefault = (this && this.__importDefault) || function (mod) {
4
+ return (mod && mod.__esModule) ? mod : { "default": mod };
5
+ };
6
+ Object.defineProperty(exports, "__esModule", { value: true });
7
+ const test_1 = require("@playwright/test");
8
+ const path_1 = __importDefault(require("path"));
9
+ const action_tool_calls_1 = require("../agent/master/action-tool-calls");
10
+ (0, test_1.test)("should annotate all links on empirical landing page", async ({ page, }) => {
11
+ await page.goto("https://assets-test.empirical.run/selector-hints-testing/dom-1.html");
12
+ await page.addScriptTag({
13
+ path: path_1.default.resolve(__dirname, "./annotate-elements.js"),
14
+ });
15
+ const annotations = await page.evaluate(() => {
16
+ // eslint-disable-next-line no-undef
17
+ const { annotations } = annotateClickableElements();
18
+ return Object.entries(annotations).map(([, config]) => ({
19
+ innerText: config.node.innerText,
20
+ tagName: config.node.tagName,
21
+ href: config.node.href,
22
+ }));
23
+ });
24
+ test_1.test.expect(annotations).toEqual([
25
+ {
26
+ innerText: "Empirical",
27
+ tagName: "A",
28
+ href: "https://assets-test.empirical.run/",
29
+ },
30
+ {
31
+ innerText: "Blog",
32
+ tagName: "A",
33
+ href: "https://assets-test.empirical.run/blog",
34
+ },
35
+ {
36
+ innerText: "Contact us",
37
+ tagName: "A",
38
+ href: "https://assets-test.empirical.run/contact",
39
+ },
40
+ {
41
+ href: "https://dash.empirical.run/",
42
+ innerText: "Login ↗\n(opens in a new tab)",
43
+ tagName: "A",
44
+ },
45
+ {
46
+ innerText: "Get early access",
47
+ tagName: "A",
48
+ href: "https://assets-test.empirical.run/contact",
49
+ },
50
+ {
51
+ innerText: "Playwright\n(opens in a new tab)",
52
+ tagName: "A",
53
+ href: "https://github.com/microsoft/playwright",
54
+ },
55
+ {
56
+ innerText: "Meet with us",
57
+ tagName: "A",
58
+ href: "https://assets-test.empirical.run/contact",
59
+ },
60
+ {
61
+ innerText: "Privacy Policy",
62
+ tagName: "A",
63
+ href: "https://assets-test.empirical.run/privacy.html",
64
+ },
65
+ ]);
66
+ });
67
+ (0, test_1.test)("should annotate all important items on quizizz page", async ({ page, }) => {
68
+ await page.goto("https://assets-test.empirical.run/selector-hints-testing/dom-2/index.html");
69
+ await page.addScriptTag({
70
+ path: path_1.default.resolve(__dirname, "./annotate-elements.js"),
71
+ });
72
+ const annotations = await page.evaluate(() => {
73
+ // eslint-disable-next-line no-undef
74
+ const { annotations } = annotateClickableElements();
75
+ return Object.entries(annotations).map(([hint, config]) => ({
76
+ hint,
77
+ innerText: config.node.innerText.toLowerCase().trim(),
78
+ tagName: config.node.tagName,
79
+ testId: config.node.getAttribute("data-testid"),
80
+ href: config.node.href,
81
+ }));
82
+ });
83
+ test_1.test
84
+ .expect(annotations.find((item) => item.innerText.includes("enter code")))
85
+ .toBeTruthy();
86
+ test_1.test
87
+ .expect(annotations.find((item) => item.innerText.includes("get help")))
88
+ .toBeTruthy();
89
+ test_1.test
90
+ .expect(annotations.find((item) => item.innerText.includes("create") &&
91
+ item.testId === "create-content-button"))
92
+ .toBeTruthy();
93
+ test_1.test
94
+ .expect(annotations.find((item) => item.innerText.includes("explore") &&
95
+ item.href === "https://quizizz.com/admin"))
96
+ .toBeTruthy();
97
+ test_1.test
98
+ .expect(annotations.find((item) => item.innerText.includes("library") &&
99
+ item.href === "https://quizizz.com/admin/my-library/createdByMe"))
100
+ .toBeTruthy();
101
+ test_1.test
102
+ .expect(annotations.find((item) => item.innerText.includes("reports") &&
103
+ item.href === "https://quizizz.com/admin/reports"))
104
+ .toBeTruthy();
105
+ test_1.test
106
+ .expect(annotations.find((item) => item.innerText.includes("classes") &&
107
+ item.href === "https://quizizz.com/admin/classes"))
108
+ .toBeTruthy();
109
+ test_1.test
110
+ .expect(annotations.find((item) => item.innerText.includes("accommodations") &&
111
+ item.href ===
112
+ "https://quizizz.com/admin/differentiation/accommodations"))
113
+ .toBeTruthy();
114
+ test_1.test
115
+ .expect(annotations.find((item) => item.innerText.includes("quizizz ai") &&
116
+ item.href === "https://quizizz.com/admin/quizizz-ai"))
117
+ .toBeTruthy();
118
+ test_1.test
119
+ .expect(annotations.find((item) => item.innerText.includes("start your free trial") &&
120
+ item.href === "https://quizizz.com/super-pricing"))
121
+ .toBeTruthy();
122
+ test_1.test
123
+ .expect(annotations.find((item) => item.innerText.includes("upgrade") &&
124
+ item.href === "https://quizizz.com/super-pricing?backto=/admin"))
125
+ .toBeTruthy();
126
+ test_1.test
127
+ .expect(annotations.find((item) => item.tagName === "INPUT" &&
128
+ item.testId === "emphasized-search-bar-input"))
129
+ .toBeTruthy();
130
+ test_1.test
131
+ .expect(annotations.find((item) => item.tagName === "BUTTON" &&
132
+ item.innerText.includes("verify details") &&
133
+ item.testId === "verify-profile-cta"))
134
+ .toBeTruthy();
135
+ test_1.test
136
+ .expect(annotations.find((item) => item.tagName === "BUTTON" && item.innerText.includes("for you")))
137
+ .toBeTruthy();
138
+ test_1.test
139
+ .expect(annotations.find((item) => item.tagName === "BUTTON" && item.innerText.includes("assessments")))
140
+ .toBeTruthy();
141
+ test_1.test
142
+ .expect(annotations.find((item) => item.tagName === "BUTTON" && item.innerText.includes("lessons")))
143
+ .toBeTruthy();
144
+ test_1.test
145
+ .expect(annotations.find((item) => item.tagName === "BUTTON" &&
146
+ item.innerText.includes("interactive videos")))
147
+ .toBeTruthy();
148
+ test_1.test
149
+ .expect(annotations.find((item) => item.tagName === "BUTTON" && item.innerText.includes("passages")))
150
+ .toBeTruthy();
151
+ });
152
+ (0, test_1.test)("should only annotate input fields on quizizz page", async ({ page }) => {
153
+ await page.goto("https://assets-test.empirical.run/selector-hints-testing/dom-2/index.html");
154
+ await page.addScriptTag({
155
+ path: path_1.default.resolve(__dirname, "./annotate-elements.js"),
156
+ });
157
+ const preference = {
158
+ actionType: action_tool_calls_1.ActionType.FILL,
159
+ };
160
+ const annotations = await page.evaluate((preference) => {
161
+ // eslint-disable-next-line no-undef
162
+ const { annotations } = annotateClickableElements({
163
+ preference: preference,
164
+ });
165
+ return Object.entries(annotations).map(([hint, config]) => ({
166
+ hint,
167
+ innerText: config.node.innerText.toLowerCase().trim(),
168
+ tagName: config.node.tagName,
169
+ testId: config.node.getAttribute("data-testid"),
170
+ href: config.node.href,
171
+ }));
172
+ }, preference);
173
+ test_1.test.expect(annotations.length).toBe(1);
174
+ test_1.test.expect(annotations[0].testId).toBe("emphasized-search-bar-input");
175
+ });
176
+ (0, test_1.test)("should only annotate given text on quizziz page", async ({ page }) => {
177
+ await page.goto("https://assets-test.empirical.run/selector-hints-testing/dom-2/index.html");
178
+ await page.addScriptTag({
179
+ path: path_1.default.resolve(__dirname, "./annotate-elements.js"),
180
+ });
181
+ const preference = {
182
+ actionType: action_tool_calls_1.ActionType.ASSERT_TEXT,
183
+ assertionText: "What are you teaching today?",
184
+ };
185
+ const annotations = await page.evaluate((preference) => {
186
+ // eslint-disable-next-line no-undef
187
+ const { annotations } = annotateClickableElements({
188
+ preference: preference,
189
+ });
190
+ return Object.entries(annotations).map(([hint, config]) => ({
191
+ hint,
192
+ innerText: config.node.innerText?.toLowerCase().trim(),
193
+ tagName: config.node.tagName,
194
+ testId: config.node.getAttribute("data-testid"),
195
+ href: config.node.href,
196
+ }));
197
+ }, preference);
198
+ console.log("Annotations:", annotations);
199
+ test_1.test.expect(annotations.length).toBe(1);
200
+ test_1.test.expect(annotations[0]?.tagName).toBe("DIV");
201
+ });
@@ -3,6 +3,9 @@
3
3
  import { test } from "@playwright/test";
4
4
  import path from "path";
5
5
 
6
+ import { ActionType } from "../agent/master/action-tool-calls";
7
+ import { AnnotationPreference } from "../agent/master/element-annotation";
8
+
6
9
  test("should annotate all links on empirical landing page", async ({
7
10
  page,
8
11
  }) => {
@@ -15,9 +18,10 @@ test("should annotate all links on empirical landing page", async ({
15
18
  });
16
19
 
17
20
  const annotations = await page.evaluate(() => {
21
+ // eslint-disable-next-line no-undef
18
22
  const { annotations } = annotateClickableElements();
19
23
 
20
- return Object.entries(annotations).map(([hint, config]) => ({
24
+ return Object.entries(annotations).map(([, config]) => ({
21
25
  innerText: config.node.innerText,
22
26
  tagName: config.node.tagName,
23
27
  href: config.node.href,
@@ -80,6 +84,7 @@ test("should annotate all important items on quizizz page", async ({
80
84
  });
81
85
 
82
86
  const annotations = await page.evaluate(() => {
87
+ // eslint-disable-next-line no-undef
83
88
  const { annotations } = annotateClickableElements();
84
89
 
85
90
  return Object.entries(annotations).map(([hint, config]) => ({
@@ -148,7 +153,7 @@ test("should annotate all important items on quizizz page", async ({
148
153
  (item) =>
149
154
  item.innerText.includes("accommodations") &&
150
155
  item.href ===
151
- "https://quizizz.com/admin/differentiation/accommodations",
156
+ "https://quizizz.com/admin/differentiation/accommodations",
152
157
  ),
153
158
  )
154
159
  .toBeTruthy();
@@ -248,9 +253,7 @@ test("should annotate all important items on quizizz page", async ({
248
253
  .toBeTruthy();
249
254
  });
250
255
 
251
- test("should only annotate input fields on quizizz page", async ({
252
- page,
253
- }) => {
256
+ test("should only annotate input fields on quizizz page", async ({ page }) => {
254
257
  await page.goto(
255
258
  "https://assets-test.empirical.run/selector-hints-testing/dom-2/index.html",
256
259
  );
@@ -259,9 +262,14 @@ test("should only annotate input fields on quizizz page", async ({
259
262
  path: path.resolve(__dirname, "./annotate-elements.js"),
260
263
  });
261
264
 
262
- const preference = "fill";
265
+ const preference = {
266
+ actionType: ActionType.FILL,
267
+ };
263
268
  const annotations = await page.evaluate((preference) => {
264
- const { annotations } = annotateClickableElements({preference: preference});
269
+ // eslint-disable-next-line no-undef
270
+ const { annotations } = annotateClickableElements({
271
+ preference: preference,
272
+ });
265
273
 
266
274
  return Object.entries(annotations).map(([hint, config]) => ({
267
275
  hint,
@@ -270,8 +278,41 @@ test("should only annotate input fields on quizizz page", async ({
270
278
  testId: config.node.getAttribute("data-testid"),
271
279
  href: config.node.href,
272
280
  }));
273
- },preference);
281
+ }, preference);
282
+
283
+ test.expect(annotations.length).toBe(1);
284
+ test.expect(annotations[0].testId).toBe("emphasized-search-bar-input");
285
+ });
286
+
287
+ test("should only annotate given text on quizziz page", async ({ page }) => {
288
+ await page.goto(
289
+ "https://assets-test.empirical.run/selector-hints-testing/dom-2/index.html",
290
+ );
291
+
292
+ await page.addScriptTag({
293
+ path: path.resolve(__dirname, "./annotate-elements.js"),
294
+ });
295
+
296
+ const preference: AnnotationPreference = {
297
+ actionType: ActionType.ASSERT_TEXT,
298
+ assertionText: "What are you teaching today?",
299
+ };
300
+ const annotations = await page.evaluate((preference) => {
301
+ // eslint-disable-next-line no-undef
302
+ const { annotations } = annotateClickableElements({
303
+ preference: preference,
304
+ });
305
+
306
+ return Object.entries(annotations).map(([hint, config]) => ({
307
+ hint,
308
+ innerText: config.node.innerText?.toLowerCase().trim(),
309
+ tagName: config.node.tagName,
310
+ testId: config.node.getAttribute("data-testid"),
311
+ href: config.node.href,
312
+ }));
313
+ }, preference);
274
314
 
315
+ console.log("Annotations:", annotations);
275
316
  test.expect(annotations.length).toBe(1);
276
- test.expect(annotations[0].testId).toBe('emphasized-search-bar-input');
317
+ test.expect(annotations[0]?.tagName).toBe("DIV");
277
318
  });
@@ -1 +1 @@
1
- {"version":3,"file":"master-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/master-agent.evals.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,4BAA4B,EAAE,UA0C1C,CAAC;AAEF,eAAe,4BAA4B,CAAC"}
1
+ {"version":3,"file":"master-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/master-agent.evals.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,4BAA4B,EAAE,UA2C1C,CAAC;AAEF,eAAe,4BAA4B,CAAC"}
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.masterGetNextActionEvaluator = void 0;
4
4
  const actions_1 = require("../actions");
5
5
  const skill_1 = require("../actions/skill");
6
+ const action_tool_calls_1 = require("../agent/master/action-tool-calls");
6
7
  const next_action_1 = require("../agent/master/next-action");
7
8
  const page_1 = require("../page");
8
9
  const masterGetNextActionEvaluator = async ({ item, trace, }) => {
@@ -20,6 +21,7 @@ const masterGetNextActionEvaluator = async ({ item, trace, }) => {
20
21
  pageScreenshot,
21
22
  annotatedPageScreenshot,
22
23
  actions,
24
+ actionTypes: action_tool_calls_1.ActionType,
23
25
  disableSkills,
24
26
  annotations,
25
27
  });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@empiricalrun/test-gen",
3
- "version": "0.42.3",
3
+ "version": "0.42.8",
4
4
  "publishConfig": {
5
5
  "registry": "https://registry.npmjs.org/",
6
6
  "access": "public"
@@ -71,7 +71,7 @@
71
71
  "ts-morph": "^24.0.0",
72
72
  "tsx": "^4.16.2",
73
73
  "typescript": "^5.3.3",
74
- "@empiricalrun/llm": "^0.9.29",
74
+ "@empiricalrun/llm": "^0.9.30",
75
75
  "@empiricalrun/r2-uploader": "^0.3.8",
76
76
  "@empiricalrun/reporter": "^0.22.1"
77
77
  },
@@ -88,7 +88,7 @@
88
88
  },
89
89
  "scripts": {
90
90
  "dev": "tsc --build --watch",
91
- "build": "tsc --build && cp -r browser-injected-scripts dist",
91
+ "build": "tsc --build && cp -r src/browser-injected-scripts dist",
92
92
  "clean": "tsc --build --clean",
93
93
  "lint": "eslint .",
94
94
  "test": "vitest run",
@@ -1,5 +1,5 @@
1
1
  import { defineConfig } from "@playwright/test";
2
2
 
3
3
  export default defineConfig({
4
- testDir: "./browser-injected-scripts",
4
+ testDir: "./src/browser-injected-scripts",
5
5
  });