@empiricalrun/test-gen 0.42.24 → 0.42.26

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @empiricalrun/test-gen
 
+## 0.42.26
+
+### Patch Changes
+
+- 070c16a: feat: config for master agent sanity test suite
+
+## 0.42.25
+
+### Patch Changes
+
+- Updated dependencies [2babfe0]
+  - @empiricalrun/llm@0.9.34
+
 ## 0.42.24
 
 ### Patch Changes

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAe,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAI7E,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAsBvD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA8FD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CA0DjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAuHxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QAIjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAe,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAI7E,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAsBvD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA8FD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CA0DjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBA2HxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QAIjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}

package/dist/agent/browsing/utils.js

@@ -164,7 +164,11 @@ function newContentsWithTestOnly(existingContents, originalTestBlock, updatedTes
     }
 }
 async function injectPwLocatorGenerator(page) {
-    const pathToInstalledTestGen = require.resolve(".").split("dist")[0];
+    let pathToInstalledTestGen = require.resolve(".").split("dist")[0];
+    // eslint-disable-next-line turbo/no-undeclared-env-vars
+    if (process.env.RUNNING_BROWSER_TESTS_FOR_TEST_GEN) {
+        pathToInstalledTestGen = process.cwd();
+    }
     const annotateElementPath = path_1.default.join(pathToInstalledTestGen, "dist", "browser-injected-scripts", "annotate-elements.js");
     if (!fs_extra_1.default.existsSync(annotateElementPath)) {
         throw new Error(`annotate-elements.js not found at path: ${annotateElementPath}`);

package/dist/agent/master/browser-tests/index.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=index.spec.d.ts.map

package/dist/agent/master/browser-tests/index.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.spec.d.ts","sourceRoot":"","sources":["../../../../src/agent/master/browser-tests/index.spec.ts"],"names":[],"mappings":""}

package/dist/agent/master/browser-tests/index.spec.js

@@ -0,0 +1,65 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const test_1 = require("@playwright/test");
+const http_server_1 = __importDefault(require("http-server"));
+const path_1 = __importDefault(require("path"));
+const run_1 = require("../run");
+let server;
+let PORT = 2345;
+test_1.test.beforeAll(async () => {
+    server = http_server_1.default.createServer({
+        root: path_1.default.join(process.cwd(), "test-data"),
+    });
+    // Start the server
+    await new Promise((resolve) => {
+        server.listen(PORT, () => {
+            console.log(`Server running at http://localhost:${PORT}`);
+            resolve(true);
+        });
+    });
+});
+test_1.test.afterAll(() => {
+    if (server) {
+        server.close();
+    }
+});
+(0, test_1.test)("able to scroll and interact with elements", async ({ page }) => {
+    await page.goto(`http://localhost:${PORT}/blog-page.html`);
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        task: `fill test@test.com into the email field and click the submit`,
+        page,
+        options: {},
+    });
+    console.log(response);
+    (0, test_1.expect)(response.importPaths.length).toBe(0);
+    (0, test_1.expect)(response.code).toContain("await page.getByPlaceholder('Enter your email').fill(\"test@test.com\")");
+    (0, test_1.expect)(response.code).toContain("await page.getByRole('button', { name: 'Subscribe' }).click()");
+});
+test_1.test.skip("scroll when element does not exist", async ({ page }) => {
+    await page.goto(`http://localhost:${PORT}/blog-page.html`);
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        task: `click search button`,
+        page,
+        options: {},
+    });
+    console.log(response);
+    (0, test_1.expect)(response.importPaths.length).toBe(0);
+    (0, test_1.expect)(response.code.length).toBe(0);
+});
+(0, test_1.test)("agent can click icons accurately", async ({ page }) => {
+    await page.goto(`http://localhost:${PORT}/icons-navbar.html`);
+    await (0, test_1.expect)(page.getByText("select an icon")).toBeVisible();
+    const response = await (0, run_1.createTestUsingMasterAgent)({
+        task: `click on the gear icon`,
+        page,
+        options: {},
+    });
+    await (0, test_1.expect)(page.getByText("you clicked Gear")).toBeVisible();
+    console.log(response);
+    (0, test_1.expect)(response.importPaths.length).toBe(0);
+    (0, test_1.expect)(response.code).toContain("page.getByRole");
+    (0, test_1.expect)(response.code).toContain("click()");
+});

package/dist/browser-injected-scripts/annotate-elements.spec.js

@@ -180,7 +180,6 @@ const action_tool_calls_1 = require("../agent/master/action-tool-calls");
             href: config.node.href,
         }));
     }, preference);
-    console.log("Annotations:", annotations);
     test_1.test.expect(annotations.length).toBe(1);
     test_1.test.expect(annotations[0]?.tagName).toBe("DIV");
 });

package/dist/browser-injected-scripts/annotate-elements.spec.ts

@@ -296,8 +296,6 @@ test("should only annotate given text on quizziz page", async ({ page }) => {
       href: config.node.href,
     }));
   }, preference);
-
-  console.log("Annotations:", annotations);
   test.expect(annotations.length).toBe(1);
   test.expect(annotations[0]?.tagName).toBe("DIV");
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.42.24",
+  "version": "0.42.26",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -71,7 +71,7 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.9.33",
+    "@empiricalrun/llm": "^0.9.34",
     "@empiricalrun/r2-uploader": "^0.3.8",
     "@empiricalrun/reporter": "^0.23.1"
   },
@@ -80,9 +80,11 @@
     "@types/detect-port": "^1.3.5",
     "@types/express": "^4.17.21",
     "@types/fs-extra": "^11.0.4",
+    "@types/http-server": "^0.12.4",
     "@types/js-levenshtein": "^1.1.3",
     "@types/lodash.isequal": "^4.5.8",
     "@types/md5": "^2.3.5",
+    "http-server": "^14.1.1",
     "js-levenshtein": "^1.1.6",
     "playwright": "1.47.1",
     "ts-patch": "^3.3.0",
@@ -94,7 +96,7 @@
     "clean": "tspc --build --clean",
     "lint": "eslint .",
     "test": "vitest run",
-    "test-browser": "npx playwright test",
+    "test-browser": "pnpm build && RUNNING_BROWSER_TESTS_FOR_TEST_GEN=1 npx playwright test",
     "test:watch": "vitest",
     "test:watch-files": "vitest $0 --watch"
   }

package/playwright.config.ts

@@ -1,5 +1,12 @@
 import { defineConfig } from "@playwright/test";
 
+// To run master agent sanity tests, we want our typescript
+// transformer to be used while compiling the tests.
+// Hence, we bypass Playwright's default TS compiler, and have
+// it run compiled test files from /dist/
 export default defineConfig({
-  testDir: "./src/browser-injected-scripts",
+  testDir: "./dist/.",
+  testMatch: "**/*.spec.js",
+  // Setting large timeout as master agent takes time to run.
+  timeout: 120_000,
 });