@empiricalrun/test-gen 0.34.3 → 0.34.5

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @empiricalrun/test-gen
 
+## 0.34.5
+
+### Patch Changes
+
+- dc5718a: feat: add support for evals
+- Updated dependencies [06cf0d8]
+  - @empiricalrun/llm@0.9.20
+
+## 0.34.4
+
+### Patch Changes
+
+- Updated dependencies [2dafa69]
+  - @empiricalrun/llm@0.9.19
+
 ## 0.34.3
 
 ### Patch Changes

package/dist/agent/browsing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAIhD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAMnD,MAAM,MAAM,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IACjE,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,KAAK,EACL,MAAM,EACN,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,OAAO,GACR,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,WAAW,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,oBAAoB,CAAC;IAC9B,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,iBAAiB,CAAC;CAC5B,iBAyIA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAIhD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAMnD,MAAM,MAAM,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC,GAAG;IACjE,YAAY,CAAC,EAAE;QACb,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;CACH,CAAC;AAEF,wBAAsB,6BAA6B,CAAC,EAClD,KAAK,EACL,MAAM,EACN,MAAM,EACN,IAAI,EACJ,OAAO,EACP,GAAG,EACH,OAAO,GACR,EAAE;IACD,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,WAAW,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,oBAAoB,CAAC;IAC9B,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,iBAAiB,CAAC;CAC5B,iBAwIA"}

package/dist/agent/browsing/index.js

@@ -38,7 +38,6 @@ async function executeTaskUsingBrowsingAgent({ trace, action, logger, page, opti
             .map((a) => a.action);
         if (successfulActions.length > 0) {
             const verificationAgentResp = await (0, verification_1.verificationAgent)({
-                llm,
                 trace: browsingAgentSpan,
                 task: action,
                 conversation: ["Successfully executed actions", ...successfulActions],

package/dist/agent/master/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAQrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,IAAI,EACJ,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,GACjB,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,oBAAoB,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;CACnB,2FAgEA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,GACR,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;CAC/B;;;GA8QA"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,GAAG,EACH,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AAQrB,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,IAAI,EACJ,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,uBAAuB,EACvB,OAAO,EACP,aAAa,EACb,QAAgB,GACjB,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,OAAO,EAAE,oBAAoB,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;CACnB,2FAgEA;AAGD,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,GACR,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,oBAAoB,CAAC;CAC/B;;;GA6QA"}

package/dist/agent/master/run.js

@@ -132,7 +132,6 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, }) {
         });
         if (masterAgentActions.length > 0) {
             const verificationAgentResp = await (0, verification_1.verificationAgent)({
-                llm,
                 trace: masterAgentSpan,
                 task,
                 conversation: ["Successfully executed actions", ...masterAgentActions],

package/dist/agent/verification/index.d.ts

@@ -1,9 +1,8 @@
-import { LLM, TraceClient } from "@empiricalrun/llm";
+import { TraceClient } from "@empiricalrun/llm";
 /**
  * This agent is used to verify whether the task is done basis the conversation history
  */
-export declare function verificationAgent({ llm, trace, task, conversation, }: {
-    llm: LLM;
+export declare function verificationAgent({ trace, task, conversation, }: {
     trace?: TraceClient;
     conversation: string[];
     task: string;

package/dist/agent/verification/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/verification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhE;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,GAAG,EACH,KAAK,EACL,IAAI,EACJ,YAAY,GACb,EAAE;IACD,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd;;;GAkDA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/verification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhE;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,KAAK,EACL,IAAI,EACJ,YAAY,GACb,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd;;;GA+EA"}

package/dist/agent/verification/index.js

@@ -5,13 +5,21 @@ const llm_1 = require("@empiricalrun/llm");
 /**
  * This agent is used to verify whether the task is done basis the conversation history
  */
-async function verificationAgent({ llm, trace, task, conversation, }) {
+async function verificationAgent({ trace, task, conversation, }) {
+    const verificationAgentSpan = trace?.span({
+        name: "verification-agent",
+        input: {
+            task,
+            conversation,
+        },
+    });
     const messages = await (0, llm_1.getPrompt)("agent-steps-verification", {
         task,
         conversation: conversation.join("\n"),
-    });
+    }, 4);
+    const llm = new llm_1.LLM({ provider: "openai" });
     const response = await llm.createChatCompletion({
-        trace,
+        trace: verificationAgentSpan,
         traceName: "verification-agent-llm",
         model: "gpt-4o",
         messages,
@@ -24,13 +32,21 @@ async function verificationAgent({ llm, trace, task, conversation, }) {
                     parameters: {
                         type: "object",
                         properties: {
-                            isDone: {
-                                type: "boolean",
-                                description: "whether the task is done",
+                            actions: {
+                                type: "string",
+                                description: "actions extracted from task",
+                            },
+                            successful_actions: {
+                                type: "string",
+                                description: "successful actions mentioned in the conversation",
                             },
                             reason: {
                                 type: "string",
-                                description: "reason for declaring the task is complete",
+                                description: "reasoning for identification of task status",
+                            },
+                            isDone: {
+                                type: "boolean",
+                                description: "whether the task is done",
                             },
                         },
                         required: ["isDone", "reason"],
@@ -40,19 +56,28 @@ async function verificationAgent({ llm, trace, task, conversation, }) {
         ],
         modelParameters: {
             tool_choice: "required",
+            temperature: 0.5,
         },
     });
     const toolCallResp = (response?.tool_calls || [])[0];
     if (toolCallResp) {
         const toolCall = JSON.parse(toolCallResp.function.arguments);
-        return {
+        const output = {
             isDone: toolCall.isDone,
             reason: toolCall.reason,
         };
+        verificationAgentSpan?.end({
+            output,
+        });
+        return output;
     }
-    return {
+    const output = {
         isDone: false,
         reason: "LLM failed to generate a valid response",
     };
+    verificationAgentSpan?.end({
+        output,
+    });
+    return output;
 }
 exports.verificationAgent = verificationAgent;

package/dist/evals/infer-master-or-code-agent.evals.d.ts

@@ -0,0 +1,4 @@
+import { EvaluateFn } from "./type";
+export declare const inferMasterOrCodeAgentEvaluate: EvaluateFn;
+export default inferMasterOrCodeAgentEvaluate;
+//# sourceMappingURL=infer-master-or-code-agent.evals.d.ts.map

package/dist/evals/infer-master-or-code-agent.evals.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"infer-master-or-code-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/infer-master-or-code-agent.evals.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,8BAA8B,EAAE,UAkB5C,CAAC;AAEF,eAAe,8BAA8B,CAAC"}

package/dist/evals/infer-master-or-code-agent.evals.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.inferMasterOrCodeAgentEvaluate = void 0;
+const infer_agent_1 = require("../agent/infer-agent");
+const inferMasterOrCodeAgentEvaluate = async ({ item, trace, }) => {
+    const { task } = item.input;
+    const { response } = await (0, infer_agent_1.inferAgentBasedTask)({
+        task,
+        trace,
+    });
+    return {
+        scores: [
+            {
+                name: "equality",
+                value: item.expectedOutput === response ? 1 : 0,
+            },
+        ],
+        output: response,
+    };
+};
+exports.inferMasterOrCodeAgentEvaluate = inferMasterOrCodeAgentEvaluate;
+exports.default = exports.inferMasterOrCodeAgentEvaluate;

package/dist/evals/type.d.ts

@@ -0,0 +1,12 @@
+import { TraceClient } from "@empiricalrun/llm";
+export type EvaluateFn = ({ trace, item, }: {
+    trace: TraceClient;
+    item: any;
+}) => Promise<{
+    output: any;
+    scores: {
+        name: string;
+        value: number;
+    }[];
+}>;
+//# sourceMappingURL=type.d.ts.map

package/dist/evals/type.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"type.d.ts","sourceRoot":"","sources":["../../src/evals/type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,MAAM,UAAU,GAAG,CAAC,EACxB,KAAK,EACL,IAAI,GACL,EAAE;IACD,KAAK,EAAE,WAAW,CAAC;IACnB,IAAI,EAAE,GAAG,CAAC;CACX,KAAK,OAAO,CAAC;IACZ,MAAM,EAAE,GAAG,CAAC;IACZ,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC3C,CAAC,CAAC"}

package/dist/evals/type.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/evals/verification-agent.evals.d.ts

@@ -0,0 +1,4 @@
+import { EvaluateFn } from "./type";
+export declare const verifierAgentEvaluate: EvaluateFn;
+export default verifierAgentEvaluate;
+//# sourceMappingURL=verification-agent.evals.d.ts.map

package/dist/evals/verification-agent.evals.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"verification-agent.evals.d.ts","sourceRoot":"","sources":["../../src/evals/verification-agent.evals.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,eAAO,MAAM,qBAAqB,EAAE,UAgBnC,CAAC;AAEF,eAAe,qBAAqB,CAAC"}

package/dist/evals/verification-agent.evals.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.verifierAgentEvaluate = void 0;
+const verification_1 = require("../agent/verification");
+const verifierAgentEvaluate = async ({ item, trace }) => {
+    const { conversation = [], task = "" } = item.input;
+    const output = await (0, verification_1.verificationAgent)({
+        conversation,
+        trace,
+        task,
+    });
+    return {
+        scores: [
+            {
+                name: "equality",
+                value: item.expectedOutput.isDone === output.isDone ? 1 : 0,
+            },
+        ],
+        output,
+    };
+};
+exports.verifierAgentEvaluate = verifierAgentEvaluate;
+exports.default = exports.verifierAgentEvaluate;

package/dist/session/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session/index.ts"],"names":[],"mappings":"AAeA,iBAAS,iBAAiB;;;;EAMzB;AAED,wBAAgB,iBAAiB,CAAC,EAChC,SAAS,EACT,YAAY,GACb,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB,QAGA;AAED,wBAAsB,iBAAiB,qBAGtC;AAED,wBAAsB,eAAe,0DAkBpC;AAED,wBAAsB,UAAU,kBAkB/B;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session/index.ts"],"names":[],"mappings":"AAgBA,iBAAS,iBAAiB;;;;EAMzB;AAED,wBAAgB,iBAAiB,CAAC,EAChC,SAAS,EACT,YAAY,GACb,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB,QAGA;AAED,wBAAsB,iBAAiB,qBAGtC;AAED,wBAAsB,eAAe,0DAqBpC;AAED,wBAAsB,UAAU,kBAqB/B;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC"}

package/dist/session/index.js

@@ -10,7 +10,8 @@ const sessionDetails = {
     version: package_json_1.default.version,
     generationId: undefined,
 };
-const DASHBOARD_DOMAIN = process.env.DASHBOARD_DOMAIN || "https://dash.empirical.run";
+const DASHBOARD_DOMAIN = process.env.DASHBOARD_DOMAIN ||
+    (process.env.CI === "true" ? "https://dash.empirical.run" : "");
 function getSessionDetails() {
     return {
         generationId: sessionDetails.generationId,
@@ -30,6 +31,9 @@ async function shouldStopSession() {
 }
 exports.shouldStopSession = shouldStopSession;
 async function getSessionState() {
+    if (!DASHBOARD_DOMAIN) {
+        return "started";
+    }
     const apiPath = `${DASHBOARD_DOMAIN}/api/sessions/${sessionDetails.sessionId}/generations/${sessionDetails.generationId}/state`;
     const response = await fetch(apiPath, {
         method: "GET",
@@ -44,6 +48,9 @@ async function getSessionState() {
 }
 exports.getSessionState = getSessionState;
 async function endSession() {
+    if (!DASHBOARD_DOMAIN) {
+        return;
+    }
     const apiPath = `${DASHBOARD_DOMAIN}/api/sessions/${sessionDetails.sessionId}/generations/${sessionDetails.generationId}/state`;
     try {
         await fetch(apiPath, {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.34.3",
+  "version": "0.34.5",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -44,7 +44,7 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.9.18",
+    "@empiricalrun/llm": "^0.9.20",
     "@empiricalrun/r2-uploader": "^0.3.6",
     "@empiricalrun/reporter": "^0.21.2"
   },