@empiricalrun/test-gen 0.42.8 → 0.42.10

package/dist/browser-injected-scripts/annotate-elements.spec.ts

@@ -19,7 +19,7 @@ test("should annotate all links on empirical landing page", async ({
 
   const annotations = await page.evaluate(() => {
     // eslint-disable-next-line no-undef
-    const { annotations } = annotateClickableElements();
+    const { annotations } = annotateElementsWithPreference();
 
     return Object.entries(annotations).map(([, config]) => ({
       innerText: config.node.innerText,
@@ -54,21 +54,6 @@ test("should annotate all links on empirical landing page", async ({
       tagName: "A",
       href: "https://assets-test.empirical.run/contact",
     },
-    {
-      innerText: "Playwright\n(opens in a new tab)",
-      tagName: "A",
-      href: "https://github.com/microsoft/playwright",
-    },
-    {
-      innerText: "Meet with us",
-      tagName: "A",
-      href: "https://assets-test.empirical.run/contact",
-    },
-    {
-      innerText: "Privacy Policy",
-      tagName: "A",
-      href: "https://assets-test.empirical.run/privacy.html",
-    },
   ]);
 });
 
@@ -85,7 +70,7 @@ test("should annotate all important items on quizizz page", async ({
 
   const annotations = await page.evaluate(() => {
     // eslint-disable-next-line no-undef
-    const { annotations } = annotateClickableElements();
+    const { annotations } = annotateElementsWithPreference();
 
     return Object.entries(annotations).map(([hint, config]) => ({
       hint,
@@ -267,7 +252,7 @@ test("should only annotate input fields on quizizz page", async ({ page }) => {
   };
   const annotations = await page.evaluate((preference) => {
     // eslint-disable-next-line no-undef
-    const { annotations } = annotateClickableElements({
+    const { annotations } = annotateElementsWithPreference({
       preference: preference,
     });
 
@@ -299,7 +284,7 @@ test("should only annotate given text on quizziz page", async ({ page }) => {
   };
   const annotations = await page.evaluate((preference) => {
     // eslint-disable-next-line no-undef
-    const { annotations } = annotateClickableElements({
+    const { annotations } = annotateElementsWithPreference({
       preference: preference,
     });
 

package/dist/evals/master-agent.evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"master-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/master-agent.evals.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,4BAA4B,EAAE,UA2C1C,CAAC;AAEF,eAAe,4BAA4B,CAAC"}
+{"version":3,"file":"master-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/master-agent.evals.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,4BAA4B,EAAE,UAwC1C,CAAC;AAEF,eAAe,4BAA4B,CAAC"}

package/dist/evals/master-agent.evals.js

@@ -7,7 +7,7 @@ const action_tool_calls_1 = require("../agent/master/action-tool-calls");
 const next_action_1 = require("../agent/master/next-action");
 const page_1 = require("../page");
 const masterGetNextActionEvaluator = async ({ item, trace, }) => {
-    const { task, executedActions, failedActions, pageUrl, options, pageScreenshot, annotatedPageScreenshot, disableSkills, skills = [], annotations, } = item.input;
+    const { task, executedActions, failedActions, pageUrl, options, pageScreenshot, disableSkills, skills = [], } = item.input;
     const page = {};
     skill_1.testCaseSkills.updateSkills(skills);
     const actions = new actions_1.PlaywrightActions(new page_1.TestGenPage(page, "page"));
@@ -19,17 +19,16 @@ const masterGetNextActionEvaluator = async ({ item, trace, }) => {
         trace,
         options,
         pageScreenshot,
-        annotatedPageScreenshot,
         actions,
         actionTypes: action_tool_calls_1.ActionType,
         disableSkills,
-        annotations,
+        page,
     });
     return {
         scores: [
             {
                 name: "action_correctness",
-                value: item.expectedOutput.function.name === output?.function.name ? 1 : 0,
+                value: item.expectedOutput.function.name === output?.actionType ? 1 : 0,
             },
         ],
         output,

package/dist/prompts/lib/ts-transformer.d.ts

@@ -0,0 +1,4 @@
+import * as ts from "typescript";
+declare function createHandlebarsTransformer(): ts.TransformerFactory<ts.SourceFile>;
+export default createHandlebarsTransformer;
+//# sourceMappingURL=ts-transformer.d.ts.map

package/dist/prompts/lib/ts-transformer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ts-transformer.d.ts","sourceRoot":"","sources":["../../../src/prompts/lib/ts-transformer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,YAAY,CAAC;AAEjC,iBAAS,2BAA2B,IAAI,EAAE,CAAC,kBAAkB,CAAC,EAAE,CAAC,UAAU,CAAC,CAkG3E;AAED,eAAe,2BAA2B,CAAC"}

package/dist/prompts/lib/ts-transformer.js

@@ -0,0 +1,90 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const ts = __importStar(require("typescript"));
+function createHandlebarsTransformer() {
+    return (context) => {
+        // Map to store the mapping between original import identifiers and generated variable names
+        const importToVarMap = new Map();
+        const visitor = (node) => {
+            // Handle references to the imported identifiers
+            if (ts.isIdentifier(node) && importToVarMap.has(node.text)) {
+                return ts.factory.createIdentifier(importToVarMap.get(node.text));
+            }
+            return ts.visitEachChild(node, visitor, context);
+        };
+        return (sourceFile) => {
+            // Reset the import map for each source file
+            importToVarMap.clear();
+            // Keep track of generated variable names to avoid duplicates
+            let templateCounter = 0;
+            // Create a new statements array by iterating over the existing ones
+            const newStatements = sourceFile.statements.flatMap((stmt) => {
+                // Check if the statement is an import declaration for a .handlebars file
+                if (ts.isImportDeclaration(stmt) &&
+                    ts.isStringLiteral(stmt.moduleSpecifier) &&
+                    stmt.moduleSpecifier.text.endsWith(".handlebars")) {
+                    // Ensure we have a default import (an identifier)
+                    if (stmt.importClause &&
+                        stmt.importClause.name &&
+                        ts.isIdentifier(stmt.importClause.name)) {
+                        const localIdentifier = stmt.importClause.name;
+                        // Generate a unique variable name
+                        const generatedVarName = `promptTemplate_${templateCounter++}`;
+                        // Store the mapping between import identifier and generated variable name
+                        importToVarMap.set(localIdentifier.text, generatedVarName);
+                        // Resolve the path to the handlebars file
+                        const containingDir = path.dirname(sourceFile.fileName);
+                        const templatePath = path.resolve(containingDir, stmt.moduleSpecifier.text);
+                        try {
+                            // Read the handlebars file content
+                            const content = fs.readFileSync(templatePath, "utf-8");
+                            // Create a variable declaration that binds the generated name to the content
+                            const variableStmt = ts.factory.createVariableStatement(undefined, ts.factory.createVariableDeclarationList([
+                                ts.factory.createVariableDeclaration(ts.factory.createIdentifier(generatedVarName), undefined, undefined, ts.factory.createStringLiteral(content)),
+                            ], ts.NodeFlags.Const));
+                            // Replace the import declaration with the variable statement
+                            return [variableStmt];
+                        }
+                        catch (error) {
+                            console.error(`Error reading handlebars file ${templatePath}:`, error);
+                            // If reading the file fails, keep the original import
+                            return [stmt];
+                        }
+                    }
+                }
+                // For all other statements, keep them as is
+                return [stmt];
+            });
+            // Create updated source file with the new statements
+            const updatedSourceFile = ts.factory.updateSourceFile(sourceFile, newStatements);
+            // Visit the entire source file to transform references
+            return ts.visitNode(updatedSourceFile, visitor);
+        };
+    };
+}
+exports.default = createHandlebarsTransformer;

package/dist/prompts/lib/vitest-plugin.d.ts

@@ -0,0 +1,8 @@
+export declare const handlebarsLoader: {
+    name: string;
+    transform(_: any, id: string): Promise<{
+        code: string;
+        map: null;
+    } | null>;
+};
+//# sourceMappingURL=vitest-plugin.d.ts.map

package/dist/prompts/lib/vitest-plugin.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vitest-plugin.d.ts","sourceRoot":"","sources":["../../../src/prompts/lib/vitest-plugin.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,gBAAgB;;iBAER,GAAG,MAAM,MAAM;;;;CAUnC,CAAC"}

package/dist/prompts/lib/vitest-plugin.js

@@ -0,0 +1,20 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handlebarsLoader = void 0;
+const fs_1 = __importDefault(require("fs"));
+exports.handlebarsLoader = {
+    name: "handlebars-loader",
+    async transform(_, id) {
+        if (id.endsWith(".handlebars") || id.endsWith(".hbs")) {
+            const content = fs_1.default.readFileSync(id, "utf-8");
+            return {
+                code: `export default ${JSON.stringify(content)};`,
+                map: null,
+            };
+        }
+        return null;
+    },
+};

package/dist/session/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session/index.ts"],"names":[],"mappings":"AAkBA,iBAAS,iBAAiB;;;;;EAOzB;AAED,wBAAgB,iBAAiB,CAAC,EAChC,SAAS,EACT,YAAY,EACZ,UAAU,EACV,eAAe,GAChB,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;CACzB,QAIA;AAED,wBAAsB,iBAAiB,qBAItC;AAED,wBAAsB,eAAe,0DA0BpC;AAED,wBAAsB,UAAU,kBAqB/B;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC;AAE7B,wBAAsB,mBAAmB,CACvC,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE;IACP,MAAM,EAAE,MAAM,CAAC;CAChB,iBAmBF"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session/index.ts"],"names":[],"mappings":"AAkBA,iBAAS,iBAAiB;;;;;EAOzB;AAED,wBAAgB,iBAAiB,CAAC,EAChC,SAAS,EACT,YAAY,EACZ,UAAU,EACV,eAAe,GAChB,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;CACzB,QAIA;AAED,wBAAsB,iBAAiB,qBAItC;AAED,wBAAsB,eAAe,0DA0BpC;AAED,wBAAsB,UAAU,kBAyB/B;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC;AAE7B,wBAAsB,mBAAmB,CACvC,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE;IACP,MAAM,EAAE,MAAM,CAAC;CAChB,iBAmBF"}

package/dist/session/index.js

@@ -58,6 +58,10 @@ async function endSession() {
     if (!DASHBOARD_DOMAIN) {
         return;
     }
+    if (!sessionDetails.sessionId || !sessionDetails.generationId) {
+        console.warn("Session details missing, skipping session end");
+        return;
+    }
     const apiPath = `${DASHBOARD_DOMAIN}/api/sessions/${sessionDetails.sessionId}/generations/${sessionDetails.generationId}/state`;
     try {
         await fetch(apiPath, {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.42.8",
+  "version": "0.42.10",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -56,6 +56,7 @@
     "fs-extra": "^11.2.0",
     "google-auth-library": "^9.10.0",
     "google-spreadsheet": "^4.1.2",
+    "handlebars": "^4.7.8",
     "ignore": "^5.3.1",
     "lodash.isequal": "^4.5.0",
     "md5": "^2.3.0",
@@ -68,12 +69,12 @@
     "remove-markdown": "^0.5.5",
     "sanitize-html": "^2.13.0",
     "slugify": "^1.6.6",
-    "ts-morph": "^24.0.0",
+    "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.9.30",
-    "@empiricalrun/r2-uploader": "^0.3.8",
-    "@empiricalrun/reporter": "^0.22.1"
+    "@empiricalrun/llm": "^0.9.31",
+    "@empiricalrun/reporter": "^0.22.1",
+    "@empiricalrun/r2-uploader": "^0.3.8"
   },
   "devDependencies": {
     "@playwright/test": "1.47.1",
@@ -84,12 +85,13 @@
     "@types/lodash.isequal": "^4.5.8",
     "@types/md5": "^2.3.5",
     "js-levenshtein": "^1.1.6",
-    "playwright": "1.47.1"
+    "playwright": "1.47.1",
+    "ts-patch": "^3.3.0"
   },
   "scripts": {
-    "dev": "tsc --build --watch",
-    "build": "tsc --build && cp -r src/browser-injected-scripts dist",
-    "clean": "tsc --build --clean",
+    "dev": "tspc --build --watch",
+    "build": "tspc --build && cp -r src/browser-injected-scripts dist",
+    "clean": "tspc --build --clean",
     "lint": "eslint .",
     "test": "vitest run",
     "test-browser": "npx playwright test",

package/vitest.config.ts

@@ -1,11 +1,16 @@
 import { config as dotenvConfig } from "dotenv";
 import { defineConfig } from "vitest/config";
 
+import { handlebarsLoader } from "./src/prompts/lib/vitest-plugin";
+
 dotenvConfig({ path: [".env.local", ".env"] });
 
 export default defineConfig({
   test: {
     setupFiles: ["dotenv/config"],
     include: ["src/**/*.test.ts"],
+    globals: true,
   },
+  assetsInclude: ["**/*.handlebars"],
+  plugins: [handlebarsLoader],
 });