@empiricalrun/test-gen 0.53.9 → 0.53.11

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @empiricalrun/test-gen
 
+## 0.53.11
+
+### Patch Changes
+
+- fb78386: feat: enabled LLM tracing for chatAgent dashboard
+- Updated dependencies [a40b338]
+  - @empiricalrun/llm@0.14.6
+
+## 0.53.10
+
+### Patch Changes
+
+- 088545c: feat: upload test reports from run-test tool calls
+- d003ea8: feat: add description to PRs created by chat agent
+- Updated dependencies [088545c]
+  - @empiricalrun/test-run@0.7.7
+
 ## 0.53.9
 
 ### Patch Changes

package/dist/agent/chat/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/index.ts"],"names":[],"mappings":"AAYA,OAAO,EAAoB,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAiBhE,wBAAsB,kBAAkB,CAAC,EACvC,mBAAmB,EACnB,aAAa,EACb,oBAAoB,GACrB,EAAE;IACD,aAAa,EAAE,mBAAmB,CAAC;IACnC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,oBAAoB,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1C,iBAoFA;AAqBD,wBAAsB,wBAAwB,CAAC,EAC7C,aAAa,EACb,aAAa,GACd,EAAE;IACD,aAAa,EAAE,mBAAmB,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;CACvB,iBA+BA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/chat/index.ts"],"names":[],"mappings":"AAYA,OAAO,EAAoB,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAiBhE,wBAAsB,kBAAkB,CAAC,EACvC,mBAAmB,EACnB,aAAa,EACb,oBAAoB,GACrB,EAAE;IACD,aAAa,EAAE,mBAAmB,CAAC;IACnC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,oBAAoB,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1C,iBAoFA;AAuBD,wBAAsB,wBAAwB,CAAC,EAC7C,aAAa,EACb,aAAa,GACd,EAAE;IACD,aAAa,EAAE,mBAAmB,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;CACvB,iBAyCA"}

package/dist/agent/chat/index.js

@@ -116,6 +116,15 @@ async function runChatAgentForDashboard({ chatSessionId, selectedModel, }) {
     const chatSession = await getChatSessionFromDashboard(chatSessionId);
     const chatState = chatSession.chat_state;
     const branchName = chatSession.branch_name;
+    const trace = llm_1.langfuseInstance?.trace({
+        id: chatSession.langfuse_trace_id,
+        name: "chat_agent",
+        input: chatState,
+        tags: [selectedModel, "chat_agent"],
+        metadata: {
+            chatSessionId,
+        },
+    });
     await (0, git_1.checkoutBranch)(branchName);
     let chatModel = (0, model_1.createChatModel)(chatState.messages, selectedModel);
     let reporterFunc = async (chatState, latest) => {
@@ -137,6 +146,7 @@ async function runChatAgentForDashboard({ chatSessionId, selectedModel, }) {
         chatModel,
         selectedModel,
         reporter: reporterFunc,
+        trace,
     });
     await (0, git_1.commitLocalAndPushBranchToRemote)(branchName);
 }

package/dist/tools/commit-and-create-pr.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"commit-and-create-pr.d.ts","sourceRoot":"","sources":["../../src/tools/commit-and-create-pr.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAQnD,UAAU,cAAc;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,cAAc,CAiB1D;AAiBD,eAAO,MAAM,wBAAwB,EAAE,IAkFtC,CAAC"}
+{"version":3,"file":"commit-and-create-pr.d.ts","sourceRoot":"","sources":["../../src/tools/commit-and-create-pr.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAQnD,UAAU,cAAc;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,cAAc,CAiB1D;AA2CD,eAAO,MAAM,wBAAwB,EAAE,IA8FtC,CAAC"}

package/dist/tools/commit-and-create-pr.js

@@ -28,7 +28,24 @@ const CommitAndPushChangesSchema = zod_1.z.object({
     commitMessage: zod_1.z
         .string()
         .describe("A short message to use for the commit. Should not be more than 8 words. Should follow conventional commit format."),
+    description: zod_1.z.string().describe(`A longer description of the changes you made. This will be used as the description of a pull request on GitHub, and so you should follow markdown formatting.
+Your code will be reviewed by a human, and you should include everything that will provide context and improve the reviewer's confidence in the changes.
+
+For example, if you used the test run tool, you should include the results (and the report URL if available). Report URL is especially important, because it contains
+videos and other artifacts that help the reviewer gain more context and confidence in the changes. If tests pass, reviewer will see the video and merge the PR.
+If tests fail, reviewer will see the video and the test artifacts, and will be able to help you debug the issue.`),
 });
+function formatDescriptionWithTimestamp(description, existingBody, type = "create") {
+    const timestamp = new Date()
+        .toISOString()
+        .replace("T", " ")
+        .replace("Z", " UTC");
+    const timestampText = `<sup>${type === "create" ? "Created" : "Updated"} at ${timestamp}</sup>`;
+    if (existingBody) {
+        return `${existingBody}\n\n---\n${description}\n\n${timestampText}`;
+    }
+    return `${description}\n\n${timestampText}`;
+}
 exports.commitAndPushChangesTool = {
     schema: {
         name: "commitAndPushChanges",
@@ -41,7 +58,7 @@ Returns the URL of the created or updated pull request.`,
     },
     execute: async (input) => {
         try {
-            const { commitMessage } = input;
+            const { commitMessage, description } = input;
             const currentBranch = (0, child_process_1.execSync)("git rev-parse --abbrev-ref HEAD")
                 .toString()
                 .trim();
@@ -82,11 +99,21 @@ Returns the URL of the created or updated pull request.`,
             (0, child_process_1.execSync)(`git push origin ${branchName} --set-upstream`);
             const existingPR = existingPRs?.find((pr) => pr.head.ref === branchName);
             if (existingPR) {
+                // Append the new description to the existing PR description
+                const updatedDescription = formatDescriptionWithTimestamp(description, existingPR.body, "update");
+                await (0, utils_1.callGitHubProxy)({
+                    method: "PATCH",
+                    url: `https://api.github.com/repos/${owner}/${repo}/pulls/${existingPR.number}`,
+                    body: {
+                        body: updatedDescription,
+                    },
+                });
                 return {
                     isError: false,
                     result: `Committed and pushed changes to existing PR: ${existingPR.html_url}`,
                 };
             }
+            const initialDescription = formatDescriptionWithTimestamp(description);
             const pr = (await (0, utils_1.callGitHubProxy)({
                 method: "POST",
                 url: `https://api.github.com/repos/${owner}/${repo}/pulls`,
@@ -94,7 +121,7 @@ Returns the URL of the created or updated pull request.`,
                     title: commitMessage,
                     head: branchName,
                     base: "main",
-                    body: "Created via CommitAndPushChanges tool",
+                    body: initialDescription,
                 },
             }));
             return {

package/dist/tools/test-run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"test-run.d.ts","sourceRoot":"","sources":["../../src/tools/test-run.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAuBnD,eAAO,MAAM,WAAW,EAAE,IAgCzB,CAAC"}
+{"version":3,"file":"test-run.d.ts","sourceRoot":"","sources":["../../src/tools/test-run.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAgDnD,eAAO,MAAM,WAAW,EAAE,IAuDzB,CAAC"}

package/dist/tools/test-run.js

@@ -12,11 +12,25 @@ const RunTestSchema = zod_1.z.object({
         .string()
         .describe("The name of the file where the test is located. File name must end with .spec.ts"),
     project: zod_1.z.string().describe("The project to run the test on"),
-    headed: zod_1.z
-        .boolean()
-        .describe("Whether to run the test in headed mode (default is false, which is headless)")
-        .optional(),
 });
+function hasCloudflareCredentials() {
+    return (process.env.R2_ACCOUNT_ID &&
+        process.env.R2_ACCESS_KEY_ID &&
+        process.env.R2_SECRET_ACCESS_KEY);
+}
+function buildReportUrl(projectName, testRunId) {
+    return `https://reports.empirical.run/${projectName}/${testRunId}/index.html`;
+}
+function buildResult({ hasTestPassed, summaryJson, reportUrl, }) {
+    return `
+Test run is complete. Result: ${hasTestPassed ? "Passed" : "Failed"}
+
+${reportUrl ? `Report URL: ${reportUrl}` : ""}
+
+# Raw result (in Playwright JSON format)
+${JSON.stringify(summaryJson)}
+`;
+}
 exports.runTestTool = {
     schema: {
         name: "runTest",
@@ -24,7 +38,18 @@ exports.runTestTool = {
         parameters: RunTestSchema,
     },
     execute: async (input) => {
-        const { testName, suites, fileName, project, headed } = input;
+        let reportUrl = undefined;
+        let projectName = undefined;
+        let testRunId = undefined;
+        if (hasCloudflareCredentials()) {
+            projectName = "test-gen-chat-agent";
+            testRunId = Date.now().toString();
+            reportUrl = buildReportUrl(projectName, testRunId);
+        }
+        else {
+            console.warn("R2 credentials not found: report artifacts will not be uploaded");
+        }
+        const { testName, suites, fileName, project } = input;
         try {
             // {"project":"chromium","suites":[],"fileName":"tests/quizizz-for-work/group.spec.ts","testName":"Create a group"}
             // This runs all tests - TODO: Debug this, should only run the testName
@@ -33,10 +58,20 @@ exports.runTestTool = {
                 suites,
                 fileName,
                 projects: [project],
-                headed,
+                // Adding these to enforce report artifacts are uploaded
+                envOverrides: projectName && testRunId
+                    ? {
+                        PROJECT_NAME: projectName,
+                        TEST_RUN_GITHUB_ACTION_ID: testRunId,
+                    }
+                    : undefined,
             });
             return {
-                result: JSON.stringify(result),
+                result: buildResult({
+                    hasTestPassed: result.hasTestPassed,
+                    summaryJson: result.summaryJson,
+                    reportUrl: reportUrl,
+                }),
                 isError: false,
             };
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.53.9",
+  "version": "0.53.11",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -56,9 +56,9 @@
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
     "zod": "^3.23.8",
-    "@empiricalrun/llm": "^0.14.5",
+    "@empiricalrun/llm": "^0.14.6",
     "@empiricalrun/r2-uploader": "^0.3.8",
-    "@empiricalrun/test-run": "^0.7.6"
+    "@empiricalrun/test-run": "^0.7.7"
   },
   "devDependencies": {
     "@playwright/test": "1.47.1",