@empiricalrun/test-gen 0.38.44 → 0.38.46

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/dist/actions/fill.d.ts.map +1 -1
  3. package/dist/actions/fill.js +9 -5
  4. package/dist/agent/codegen/repo-edit.d.ts.map +1 -1
  5. package/dist/agent/codegen/repo-edit.js +20 -3
  6. package/dist/agent/codegen/test-update-feedback.d.ts +17 -0
  7. package/dist/agent/codegen/test-update-feedback.d.ts.map +1 -0
  8. package/dist/agent/codegen/test-update-feedback.js +40 -0
  9. package/dist/agent/codegen/update-flow.d.ts +10 -5
  10. package/dist/agent/codegen/update-flow.d.ts.map +1 -1
  11. package/dist/agent/codegen/update-flow.js +33 -6
  12. package/dist/agent/codegen/utils.d.ts +5 -1
  13. package/dist/agent/codegen/utils.d.ts.map +1 -1
  14. package/dist/agent/codegen/utils.js +111 -88
  15. package/dist/agent/master/run.js +1 -1
  16. package/dist/agent/master/with-hints.d.ts +3 -3
  17. package/dist/agent/master/with-hints.d.ts.map +1 -1
  18. package/dist/agent/master/with-hints.js +1 -3
  19. package/dist/evals/update-scenario-agent.evals.js +1 -1
  20. package/dist/types/index.d.ts +2 -0
  21. package/dist/types/index.d.ts.map +1 -1
  22. package/package.json +3 -3
package/CHANGELOG.md CHANGED
@@ -1,5 +1,17 @@
1
1
  # @empiricalrun/test-gen
2
2
 
3
+ ## 0.38.46
4
+
5
+ ### Patch Changes
6
+
7
+ - 4ca5faa: fix: feedback for wrong oldCode in coding agent output
8
+
9
+ ## 0.38.45
10
+
11
+ ### Patch Changes
12
+
13
+ - efb88aa: fix: update fill action to use state variables properly
14
+
3
15
  ## 0.38.44
4
16
 
5
17
  ### Patch Changes
package/dist/actions/fill.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"fill.d.ts","sourceRoot":"","sources":["../../src/actions/fill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,2BAA2B,uBAAuB,CAAC;AAGhE,eAAO,MAAM,mBAAmB,EAAE,yBAkFjC,CAAC"}
1
+ {"version":3,"file":"fill.d.ts","sourceRoot":"","sources":["../../src/actions/fill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAIrD,eAAO,MAAM,2BAA2B,uBAAuB,CAAC;AAGhE,eAAO,MAAM,mBAAmB,EAAE,yBA6FjC,CAAC"}
package/dist/actions/fill.js CHANGED
@@ -12,17 +12,21 @@ const fillActionGenerator = (page, options) => {
12
12
  execute: async ({ args }) => {
13
13
  const css = args.css_selector;
14
14
  const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(css, args.xpath, page.pwPageInstance, args?.elementAnnotation);
15
- const textToFill = options?.stateVariables[args.variable_name] || args.text;
16
- const exec = new Function("page", `return page.${locator}.fill("${textToFill}", { timeout: 3000 })`);
17
- await exec(page.pwPageInstance);
15
+ const stateVariableKeys = Object.keys(options?.stateVariables);
16
+ const testCaseLine = `${locator}.fill(${options?.stateVariables[args.variable_name]
17
+ ? args.variable_name
18
+ : `"${args.text}"`})`;
19
+ const exec = new Function("page", ...stateVariableKeys, `return page.${testCaseLine}`);
20
+ await exec(page.pwPageInstance, ...stateVariableKeys.map((key) => options?.stateVariables[key]));
18
21
  return {
19
22
  locator,
23
+ testCaseLine,
20
24
  };
21
25
  },
22
26
  // TODO: args transformer to be kept at a single place
23
- template: (args, options) => {
27
+ template: (_, options) => {
24
28
  return {
25
- code: `await ${page.name}.${options.locator}.fill("${args.text}");`,
29
+ code: `await ${page.name}.${options.testCaseLine}`,
26
30
  };
27
31
  },
28
32
  name: exports.PLAYWRIGHT_FILL_ACTION_NAME,
package/dist/agent/codegen/repo-edit.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"repo-edit.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/repo-edit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAarE,wBAAsB,aAAa,CAAC,EAClC,KAAK,EACL,IAAI,GACL,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,iBA+IA"}
1
+ {"version":3,"file":"repo-edit.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/repo-edit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAcrD,wBAAsB,aAAa,CAAC,EAClC,KAAK,EACL,IAAI,GACL,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,iBAgKA"}
package/dist/agent/codegen/repo-edit.js CHANGED
@@ -35,6 +35,7 @@ const context_1 = require("../../bin/utils/context");
35
35
  const web_1 = require("../../bin/utils/platform/web");
36
36
  const constants_1 = require("../../constants");
37
37
  const reporter_1 = require("../../reporter");
38
+ const test_update_feedback_1 = require("./test-update-feedback");
38
39
  const utils_1 = require("./utils");
39
40
  async function repoEditAgent({ trace, task, }) {
40
41
  const testgenUpdatesReporter = new reporter_1.TestGenUpdatesReporter();
@@ -141,14 +142,30 @@ Task: ${task}
141
142
  task,
142
143
  },
143
144
  });
144
- // TODO: remove this
145
- await (0, llm_1.flushAllTraces)();
146
- await (0, utils_1.applyFileChanges)({
145
+ const fileUpdateResponses = await (0, utils_1.applyFileChanges)({
147
146
  validateTypes: false,
148
147
  trace,
149
148
  testCase: {},
150
149
  fileChanges: updates.filter((f) => f.filePath && fsSync.existsSync(f.filePath)),
151
150
  });
151
+ const errorResponses = fileUpdateResponses.filter((f) => f?.error);
152
+ if (errorResponses.length > 0) {
153
+ const updatedFileChanges = await (0, test_update_feedback_1.applyTestUpdateFeedbacks)({
154
+ trace,
155
+ oldPrompt: prompt,
156
+ agentResponse: updatedUsageExampleMessage?.content,
157
+ feedbacks: errorResponses.map((e) => ({
158
+ filePath: e?.filePath,
159
+ errorMessage: e?.errorMessage,
160
+ })),
161
+ });
162
+ await (0, utils_1.applyFileChanges)({
163
+ validateTypes: false,
164
+ trace,
165
+ testCase: {},
166
+ fileChanges: updatedFileChanges,
167
+ });
168
+ }
152
169
  // TODO: identify new file updates
153
170
  const newFileUpdates = updates.filter((f) => !fsSync.existsSync(f.filePath));
154
171
  await Promise.all(newFileUpdates.map((f) => {
package/dist/agent/codegen/test-update-feedback.d.ts ADDED
@@ -0,0 +1,17 @@
1
+ import { TraceClient } from "@empiricalrun/llm";
2
+ import { ChatCompletionMessageParam } from "openai/resources/index.mjs";
3
+ export declare function applyTestUpdateFeedbacks({ trace, oldPrompt, feedbacks, agentResponse, }: {
4
+ oldPrompt: ChatCompletionMessageParam[];
5
+ agentResponse: string;
6
+ feedbacks: {
7
+ filePath: string | undefined;
8
+ errorMessage: string;
9
+ }[];
10
+ trace?: TraceClient;
11
+ }): Promise<{
12
+ filePath: string | undefined;
13
+ oldCode: string | undefined;
14
+ newCode: string | undefined;
15
+ reason: string | undefined;
16
+ }[]>;
17
+ //# sourceMappingURL=test-update-feedback.d.ts.map
package/dist/agent/codegen/test-update-feedback.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"test-update-feedback.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/test-update-feedback.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAKxE,wBAAsB,wBAAwB,CAAC,EAC7C,KAAK,EACL,SAAS,EACT,SAAS,EACT,aAAa,GACd,EAAE;IACD,SAAS,EAAE,0BAA0B,EAAE,CAAC;IACxC,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,YAAY,EAAE,MAAM,CAAC;KACtB,EAAE,CAAC;IACJ,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;;;;;KAkCA"}
package/dist/agent/codegen/test-update-feedback.js ADDED
@@ -0,0 +1,40 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.applyTestUpdateFeedbacks = void 0;
4
+ const llm_1 = require("@empiricalrun/llm");
5
+ const constants_1 = require("../../constants");
6
+ const utils_1 = require("./utils");
7
+ async function applyTestUpdateFeedbacks({ trace, oldPrompt, feedbacks, agentResponse, }) {
8
+ const applyTestUpdateFeedbacksSpan = trace?.span({
9
+ name: "apply-test-update-feedbacks",
10
+ });
11
+ const llm = new llm_1.LLM({
12
+ trace: applyTestUpdateFeedbacksSpan,
13
+ provider: "openai",
14
+ defaultModel: "gpt-4o",
15
+ providerApiKey: constants_1.MODEL_API_KEYS["openai"],
16
+ });
17
+ const response = await llm.createChatCompletion({
18
+ messages: [
19
+ ...oldPrompt,
20
+ {
21
+ role: "assistant",
22
+ content: agentResponse,
23
+ },
24
+ {
25
+ role: "user",
26
+ content: `
27
+ Following errors were found in the previous response
28
+ ${feedbacks.map((f) => `- ${f.errorMessage}`).join("\n")}
29
+
30
+ Fix the above failures and respond with same format as the previous response`,
31
+ },
32
+ ],
33
+ });
34
+ const updatedResponse = response?.content || "";
35
+ const fileChanges = (0, utils_1.extractTestUpdates)(updatedResponse);
36
+ const filteredFileChanges = fileChanges.filter((f) => !feedbacks.some((e) => e?.filePath === f.filePath));
37
+ applyTestUpdateFeedbacksSpan?.end({ output: { updatedResponse } });
38
+ return filteredFileChanges;
39
+ }
40
+ exports.applyTestUpdateFeedbacks = applyTestUpdateFeedbacks;
package/dist/agent/codegen/update-flow.d.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { TraceClient } from "@empiricalrun/llm";
2
+ import { ChatCompletionMessageParam } from "openai/resources/index.mjs";
2
3
  import { TestCase, TestGenConfigOptions } from "../../types";
3
4
  import { UpdatedTestCase } from "./utils";
4
5
  export declare function getUpdateTestCodeCompletion({ testCase, testFileContent, testFiles, pageFiles, testFilePath, trace, options, }: {
@@ -10,11 +11,15 @@ export declare function getUpdateTestCodeCompletion({ testCase, testFileContent,
10
11
  trace?: TraceClient;
11
12
  options?: TestGenConfigOptions;
12
13
  }): Promise<{
13
- filePath: string | undefined;
14
- oldCode: string | undefined;
15
- newCode: string | undefined;
16
- reason: string | undefined;
17
- }[]>;
14
+ prompt: ChatCompletionMessageParam[];
15
+ agentResponse: string;
16
+ fileChanges: {
17
+ filePath: string | undefined;
18
+ oldCode: string | undefined;
19
+ newCode: string | undefined;
20
+ reason: string | undefined;
21
+ }[];
22
+ }>;
18
23
  export declare function updateTest(testCase: TestCase, file: string, options: TestGenConfigOptions | undefined, logging?: boolean, validate?: boolean, trace?: TraceClient): Promise<UpdatedTestCase[]>;
19
24
  export declare function getAppendCreateTestBlockCompletion({ testFiles, pageFiles, testCase, testFilePath, options, trace, }: {
20
25
  trace?: TraceClient;
package/dist/agent/codegen/update-flow.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAc3B,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAC7D,OAAO,EAIL,eAAe,EAChB,MAAM,SAAS,CAAC;AAEjB,wBAAsB,2BAA2B,CAAC,EAChD,QAAQ,EACR,eAAe,EACf,SAAS,EACT,SAAS,EACT,YAAY,EACZ,KAAK,EACL,OAAO,GACR,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,GAAG,OAAO,CACT;IACE,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CACJ,CA+CA;AAED,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAAG,SAAS,EACzC,OAAO,GAAE,OAAc,EACvB,QAAQ,GAAE,OAAc,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,EAAE,CAAC,CA6D5B;AAED,wBAAsB,kCAAkC,CAAC,EACvD,SAAS,EACT,SAAS,EACT,QAAQ,EACR,YAAY,EACZ,OAAO,EACP,KAAK,GACN,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB,mBAuGA;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,EACL,aAAoB,GACrB,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA4C7B"}
1
+ {"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAYxE,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,OAAO,EAIL,eAAe,EAChB,MAAM,SAAS,CAAC;AAEjB,wBAAsB,2BAA2B,CAAC,EAChD,QAAQ,EACR,eAAe,EACf,SAAS,EACT,SAAS,EACT,YAAY,EACZ,KAAK,EACL,OAAO,GACR,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;CACL,CAAC,CAmDD;AAED,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAAG,SAAS,EACzC,OAAO,GAAE,OAAc,EACvB,QAAQ,GAAE,OAAc,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,EAAE,CAAC,CAqF5B;AAED,wBAAsB,kCAAkC,CAAC,EACvD,SAAS,EACT,SAAS,EACT,QAAQ,EACR,YAAY,EACZ,OAAO,EACP,KAAK,GACN,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB,mBAuGA;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,EACL,aAAoB,GACrB,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA4C7B"}
package/dist/agent/codegen/update-flow.js CHANGED
@@ -13,6 +13,7 @@ const fs_1 = require("../../bin/utils/fs");
13
13
  const web_1 = require("../../bin/utils/platform/web");
14
14
  const constants_1 = require("../../constants");
15
15
  const session_1 = require("../../session");
16
+ const test_update_feedback_1 = require("./test-update-feedback");
16
17
  const utils_1 = require("./utils");
17
18
  async function getUpdateTestCodeCompletion({ testCase, testFileContent, testFiles, pageFiles, testFilePath, trace, options, }) {
18
19
  const promptSpan = trace?.span({
@@ -31,7 +32,7 @@ async function getUpdateTestCodeCompletion({ testCase, testFileContent, testFile
31
32
  content: testFileContent,
32
33
  suites: testCase?.suites || [],
33
34
  });
34
- const instruction = await (0, llm_1.getPrompt)(promptName, {
35
+ const prompt = await (0, llm_1.getPrompt)(promptName, {
35
36
  testFiles: testFiles,
36
37
  pageFiles: pageFiles,
37
38
  scenarioName,
@@ -46,7 +47,7 @@ async function getUpdateTestCodeCompletion({ testCase, testFileContent, testFile
46
47
  providerApiKey: constants_1.MODEL_API_KEYS["anthropic"],
47
48
  });
48
49
  const firstShotMessage = await llm.createChatCompletion({
49
- messages: instruction,
50
+ messages: prompt,
50
51
  modelParameters: {
51
52
  ...constants_1.DEFAULT_MODEL_PARAMETERS,
52
53
  ...options?.modelParameters,
@@ -56,7 +57,11 @@ async function getUpdateTestCodeCompletion({ testCase, testFileContent, testFile
56
57
  let response = firstShotMessage?.content || "";
57
58
  const fileChanges = (0, utils_1.extractTestUpdates)(response);
58
59
  promptSpan?.end({ output: fileChanges });
59
- return fileChanges;
60
+ return {
61
+ fileChanges,
62
+ prompt: prompt,
63
+ agentResponse: response,
64
+ };
60
65
  }
61
66
  exports.getUpdateTestCodeCompletion = getUpdateTestCodeCompletion;
62
67
  async function updateTest(testCase, file, options, logging = true, validate = true, trace) {
@@ -89,13 +94,12 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
89
94
  name: "update-test",
90
95
  input: request,
91
96
  });
92
- const fileChanges = await getUpdateTestCodeCompletion({
97
+ const { fileChanges, prompt, agentResponse } = await getUpdateTestCodeCompletion({
93
98
  ...request,
94
99
  trace: updateTestSpan,
95
100
  });
96
101
  console.log(`fileChanges ${testCase.steps.join("")}`, fileChanges);
97
- logger.success("Test generated successfully!");
98
- await (0, utils_1.applyFileChanges)({
102
+ const fileUpdateResponses = await (0, utils_1.applyFileChanges)({
99
103
  validateTypes: validate,
100
104
  trace: updateTestSpan,
101
105
  testCase,
@@ -105,6 +109,29 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
105
109
  pomPrompt: pomPrompt,
106
110
  nonSpecFilePrompt: nonSpecFilePrompt,
107
111
  });
112
+ const errorResponses = fileUpdateResponses.filter((f) => f?.error);
113
+ if (errorResponses.length > 0) {
114
+ const filteredFileChanges = await (0, test_update_feedback_1.applyTestUpdateFeedbacks)({
115
+ trace: updateTestSpan,
116
+ oldPrompt: prompt,
117
+ agentResponse,
118
+ feedbacks: errorResponses.map((e) => ({
119
+ filePath: e?.filePath,
120
+ errorMessage: e?.errorMessage,
121
+ })),
122
+ });
123
+ await (0, utils_1.applyFileChanges)({
124
+ validateTypes: validate,
125
+ trace: updateTestSpan,
126
+ testCase,
127
+ fileChanges: filteredFileChanges,
128
+ logger,
129
+ testGenOptions: options,
130
+ pomPrompt: pomPrompt,
131
+ nonSpecFilePrompt: nonSpecFilePrompt,
132
+ });
133
+ }
134
+ logger.success("Test generated successfully!");
108
135
  if (trace) {
109
136
  logger.log(`View [trace](${trace.getTraceUrl()})`);
110
137
  }
package/dist/agent/codegen/utils.d.ts CHANGED
@@ -65,5 +65,9 @@ export declare function applyFileChanges({ validateTypes, trace, testCase, fileC
65
65
  testGenOptions?: TestGenConfigOptions;
66
66
  pomPrompt?: string;
67
67
  nonSpecFilePrompt?: string;
68
- }): Promise<void>;
68
+ }): Promise<({
69
+ error: boolean;
70
+ errorMessage: string;
71
+ filePath: string;
72
+ } | undefined)[]>;
69
73
  //# sourceMappingURL=utils.d.ts.map
package/dist/agent/codegen/utils.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMhD,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG;IACjD,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CAiBF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,GAAG;IACvD,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CA8BF;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,GAAG;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB,EAAE,CAgBF;AAED,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG;IACvC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAEF,wBAAsB,gBAAgB,CAAC,EACrC,aAAoB,EACpB,KAAK,EACL,QAAQ,EACR,WAAW,EACX,MAAM,EACN,cAAc,EACd,SAAS,EACT,iBAAiB,GAClB,EAAE;IACD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;IACJ,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,iBA4HA"}
1
+ {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMhD,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG;IACjD,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CAiBF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,GAAG;IACvD,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,EAAE,CA8BF;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,GAAG;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB,EAAE,CAgBF;AAED,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG;IACvC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAEF,wBAAsB,gBAAgB,CAAC,EACrC,aAAoB,EACpB,KAAK,EACL,QAAQ,EACR,WAAW,EACX,MAAM,EACN,cAAc,EACd,SAAS,EACT,iBAAiB,GAClB,EAAE;IACD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;QAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;QAC5B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;KAC5B,EAAE,CAAC;IACJ,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,GAAG,OAAO,CACT,CAAC;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAAC,EAAE,CAC3E,CAmJA"}
package/dist/agent/codegen/utils.js CHANGED
@@ -89,107 +89,130 @@ function extractTestStepsSuggestions(input) {
89
89
  }
90
90
  exports.extractTestStepsSuggestions = extractTestStepsSuggestions;
91
91
  async function applyFileChanges({ validateTypes = true, trace, testCase, fileChanges, logger, testGenOptions, pomPrompt, nonSpecFilePrompt, }) {
92
- await Promise.allSettled(fileChanges.map(async (fileChange) => {
92
+ return await Promise.all(fileChanges.map(async (fileChange) => {
93
93
  if (!fileChange.filePath) {
94
94
  return;
95
95
  }
96
- const hasTestCaseAsUpdateContext = !!testCase?.name;
97
- let testBlockUpdate = undefined;
98
- if (hasTestCaseAsUpdateContext) {
99
- const applyFileChangesSpan = trace?.span({
100
- name: "apply-file-changes",
101
- });
102
- const block = (0, web_1.getTypescriptTestBlock)({
103
- scenarioName: testCase?.name,
104
- content: fileChange.newCode || "",
105
- suites: [], // suites should be empty here since we ask LLM to send immediate parent AST node for the code update. so there won't be any nesting here, just the test block
106
- });
107
- testBlockUpdate = block.testBlock;
108
- applyFileChangesSpan?.end({ output: { testBlock: testBlockUpdate } });
109
- }
110
- if (testBlockUpdate) {
111
- // assuming the test case getting updated
112
- // maintaining the previous accuracy of the test case update
113
- const readWriteFileSpan = trace?.span({ name: "write-to-file" });
114
- let contents = await fs_extra_1.default.readFile(fileChange.filePath, "utf-8");
115
- const [prependContent, strippedContent] = await (0, web_1.stripAndPrependImports)(fileChange.newCode, testCase?.name);
116
- let updatedContent = prependContent + contents + `\n\n${strippedContent}`;
117
- const { testBlock } = (0, web_1.getTypescriptTestBlock)({
118
- scenarioName: testCase?.name,
119
- content: contents,
120
- suites: testCase?.suites,
121
- });
122
- contents = contents.replace(testBlock, `\n\n${strippedContent}`);
123
- updatedContent = prependContent + contents;
124
- await fs_extra_1.default.writeFile(fileChange.filePath, updatedContent, "utf-8");
125
- readWriteFileSpan?.end({ output: { updatedContent } });
126
- }
127
- else {
128
- const readWriteFileSpan = trace?.span({ name: "write-to-file" });
129
- let contents = await fs_extra_1.default.readFile(fileChange.filePath, "utf-8");
130
- const project = new ts_morph_1.Project();
131
- const sourceFile = project.createSourceFile("updated-code.ts", fileChange.newCode);
132
- const functions = sourceFile.getFunctions();
133
- const checkForMethodSrc = project.createSourceFile("check-method.ts", `class A {
134
- ${fileChange.newCode}
135
- }`);
136
- const methods = checkForMethodSrc.getDescendantsOfKind(ts_morph_1.SyntaxKind.MethodDeclaration);
137
- const originalSource = project.createSourceFile("current-code.ts", contents);
138
- // if there is a single function update in the file
139
- if (functions.length === 1 &&
140
- functions[0]?.getText() === fileChange.newCode) {
141
- const updatedCodeFuncNames = functions.map((f) => f.getName());
142
- const funcName = updatedCodeFuncNames[0];
143
- const matchingNodes = originalSource
144
- .getDescendantsOfKind(ts_morph_1.SyntaxKind.FunctionDeclaration)
145
- .filter((node) => node.getName() === funcName);
146
- matchingNodes[0]?.replaceWithText(functions[0]?.getText());
147
- contents = originalSource.getFullText();
96
+ try {
97
+ const hasTestCaseAsUpdateContext = !!testCase?.name;
98
+ let testBlockUpdate = undefined;
99
+ if (hasTestCaseAsUpdateContext) {
100
+ const applyFileChangesSpan = trace?.span({
101
+ name: "apply-file-changes",
102
+ });
103
+ const block = (0, web_1.getTypescriptTestBlock)({
104
+ scenarioName: testCase?.name,
105
+ content: fileChange.newCode || "",
106
+ suites: [], // suites should be empty here since we ask LLM to send immediate parent AST node for the code update. so there won't be any nesting here, just the test block
107
+ });
108
+ testBlockUpdate = block.testBlock;
109
+ applyFileChangesSpan?.end({ output: { testBlock: testBlockUpdate } });
148
110
  }
149
- else if (
150
- // if there is a update in method of a class in the file
151
- methods.length === 1 &&
152
- methods[0]?.getText() === fileChange.newCode) {
153
- const method = methods[0];
154
- const funcName = method?.getName();
155
- const matchingNodes = originalSource
156
- .getDescendantsOfKind(ts_morph_1.SyntaxKind.MethodDeclaration)
157
- .filter((node) => node.getName() === funcName);
158
- matchingNodes[0]?.replaceWithText(method?.getText());
159
- contents = originalSource.getFullText();
111
+ if (testBlockUpdate) {
112
+ // assuming the test case getting updated
113
+ // maintaining the previous accuracy of the test case update
114
+ const readWriteFileSpan = trace?.span({ name: "write-to-file" });
115
+ let contents = await fs_extra_1.default.readFile(fileChange.filePath, "utf-8");
116
+ const [prependContent, strippedContent] = await (0, web_1.stripAndPrependImports)(fileChange.newCode, testCase?.name);
117
+ let updatedContent = prependContent + contents + `\n\n${strippedContent}`;
118
+ const { testBlock } = (0, web_1.getTypescriptTestBlock)({
119
+ scenarioName: testCase?.name,
120
+ content: contents,
121
+ suites: testCase?.suites,
122
+ });
123
+ contents = contents.replace(testBlock, `\n\n${strippedContent}`);
124
+ updatedContent = prependContent + contents;
125
+ await fs_extra_1.default.writeFile(fileChange.filePath, updatedContent, "utf-8");
126
+ readWriteFileSpan?.end({ output: { updatedContent } });
160
127
  }
161
128
  else {
162
- // since we dont know what is getting updated,
163
- // we believe that the patch is correct and contains few before and after lines
164
- // to make the change unique for search & replace
165
- // since we dont know what is getting updated,
166
- // we believe that the patch is correct and contains few before and after lines
167
- // to make the change unique for search & replace
168
- if (contents.includes(fileChange.oldCode)) {
169
- contents = contents.replace(fileChange.oldCode, `\n\n${fileChange.newCode}`);
129
+ const readWriteFileSpan = trace?.span({ name: "write-to-file" });
130
+ let contents = await fs_extra_1.default.readFile(fileChange.filePath, "utf-8");
131
+ const project = new ts_morph_1.Project();
132
+ const sourceFile = project.createSourceFile("updated-code.ts", fileChange.newCode);
133
+ const functions = sourceFile.getFunctions();
134
+ const checkForMethodSrc = project.createSourceFile("check-method.ts", `class A {
135
+ ${fileChange.newCode}
136
+ }`);
137
+ const methods = checkForMethodSrc.getDescendantsOfKind(ts_morph_1.SyntaxKind.MethodDeclaration);
138
+ const originalSource = project.createSourceFile("current-code.ts", contents);
139
+ // if there is a single function update in the file
140
+ if (functions.length === 1 &&
141
+ functions[0]?.getText() === fileChange.newCode) {
142
+ const updatedCodeFuncNames = functions.map((f) => f.getName());
143
+ const funcName = updatedCodeFuncNames[0];
144
+ const matchingNodes = originalSource
145
+ .getDescendantsOfKind(ts_morph_1.SyntaxKind.FunctionDeclaration)
146
+ .filter((node) => node.getName() === funcName);
147
+ matchingNodes[0]?.replaceWithText(functions[0]?.getText());
148
+ contents = originalSource.getFullText();
149
+ }
150
+ else if (
151
+ // if there is a update in method of a class in the file
152
+ methods.length === 1 &&
153
+ methods[0]?.getText() === fileChange.newCode) {
154
+ const method = methods[0];
155
+ const funcName = method?.getName();
156
+ const matchingNodes = originalSource
157
+ .getDescendantsOfKind(ts_morph_1.SyntaxKind.MethodDeclaration)
158
+ .filter((node) => node.getName() === funcName);
159
+ matchingNodes[0]?.replaceWithText(method?.getText());
160
+ contents = originalSource.getFullText();
170
161
  }
171
162
  else {
172
- logger?.error(`Unable to find the code to update in ${fileChange.filePath}`);
163
+ // since we dont know what is getting updated,
164
+ // we believe that the patch is correct and contains few before and after lines
165
+ // to make the change unique for search & replace
166
+ // since we dont know what is getting updated,
167
+ // we believe that the patch is correct and contains few before and after lines
168
+ // to make the change unique for search & replace
169
+ if (contents.includes(fileChange.oldCode)) {
170
+ contents = contents.replace(fileChange.oldCode, `\n\n${fileChange.newCode}`);
171
+ }
172
+ else {
173
+ logger?.error(`Unable to find the code to update in ${fileChange.filePath}`);
174
+ return {
175
+ filePath: fileChange.filePath,
176
+ error: true,
177
+ errorMessage: `The content of "old_code_block" corresponding to file path "${fileChange.filePath}" did not match the current content of the file "${fileChange.filePath}"`,
178
+ };
179
+ }
173
180
  }
181
+ await fs_extra_1.default.writeFile(fileChange.filePath, contents, "utf-8");
182
+ readWriteFileSpan?.end({ output: { contents } });
183
+ }
184
+ // format and validate file change
185
+ if (validateTypes) {
186
+ await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
187
+ trace,
188
+ logger,
189
+ file: fileChange.filePath,
190
+ pomCode: pomPrompt,
191
+ nonSpecFileCode: nonSpecFilePrompt,
192
+ testCase: testCase,
193
+ options: testGenOptions,
194
+ });
174
195
  }
175
- await fs_extra_1.default.writeFile(fileChange.filePath, contents, "utf-8");
176
- readWriteFileSpan?.end({ output: { contents } });
196
+ trace?.event({ name: "format-file" });
197
+ await (0, web_1.formatCode)(fileChange.filePath);
198
+ logger?.success(`${fileChange.filePath} file formatted successfully!`);
199
+ return {
200
+ filePath: fileChange.filePath,
201
+ error: false,
202
+ errorMessage: "",
203
+ };
177
204
  }
178
- // format and validate file change
179
- if (validateTypes) {
180
- await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
181
- trace,
182
- logger,
183
- file: fileChange.filePath,
184
- pomCode: pomPrompt,
185
- nonSpecFileCode: nonSpecFilePrompt,
186
- testCase: testCase,
187
- options: testGenOptions,
205
+ catch (e) {
206
+ trace?.event({
207
+ name: "apply-file-changes-error",
208
+ output: {
209
+ filePath: fileChange.filePath,
210
+ error: e,
211
+ },
188
212
  });
213
+ console.error(`Error while applying changes to file ${fileChange.filePath}`, e);
214
+ return;
189
215
  }
190
- trace?.event({ name: "format-file" });
191
- await (0, web_1.formatCode)(fileChange.filePath);
192
- logger?.success(`${fileChange.filePath} file formatted successfully!`);
193
216
  }));
194
217
  }
195
218
  exports.applyFileChanges = applyFileChanges;
package/dist/agent/master/run.js CHANGED
@@ -231,7 +231,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
231
231
  const result = await (0, with_hints_1.triggerHintsFlow)({
232
232
  outputFromGetNextAction: output,
233
233
  generatedAnnotations: annotationKeys,
234
- page: testGenPage,
234
+ actions,
235
235
  llm,
236
236
  trace: triggerHintsFlowSpan,
237
237
  });
package/dist/agent/master/with-hints.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { LLM, TraceClient } from "@empiricalrun/llm";
2
2
  import OpenAI from "openai";
3
- import { TestGenPage } from "../../page";
3
+ import { PlaywrightActions } from "../../actions";
4
4
  import { BrowsingAgentOptions } from "../browsing";
5
5
  export declare const getUserMessageWithForHints: ({ userMessage, options, pageScreenshot, annotatedPageScreenshot, }: {
6
6
  userMessage: OpenAI.ChatCompletionUserMessageParam;
@@ -8,13 +8,13 @@ export declare const getUserMessageWithForHints: ({ userMessage, options, pageSc
8
8
  pageScreenshot: string;
9
9
  annotatedPageScreenshot: string;
10
10
  }) => string | OpenAI.ChatCompletionContentPart[];
11
- export declare const triggerHintsFlow: ({ outputFromGetNextAction, generatedAnnotations, page, llm, trace, }: {
11
+ export declare const triggerHintsFlow: ({ outputFromGetNextAction, generatedAnnotations, actions, llm, trace, }: {
12
12
  outputFromGetNextAction: {
13
13
  action: string;
14
14
  elementAnnotation?: string;
15
15
  };
16
16
  generatedAnnotations: Record<string, any>;
17
- page: TestGenPage;
17
+ actions: PlaywrightActions;
18
18
  llm: LLM;
19
19
  trace?: TraceClient | undefined;
20
20
  }) => Promise<{
package/dist/agent/master/with-hints.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAU5B,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,0BAA0B;iBAMxB,OAAO,8BAA8B;;oBAElC,MAAM;6BACG,MAAM;MAC7B,MAAM,GAAG,OAAO,yBAAyB,EAiC5C,CAAC;AAEF,eAAO,MAAM,gBAAgB;6BAOF;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B;0BACqB,OAAO,MAAM,EAAE,GAAG,CAAC;UACnC,WAAW;SACZ,GAAG;;MAEN,QAAQ;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,OAAO,qBAAqB,GAAG,SAAS,CAAC;CACpE,CA0GA,CAAC"}
1
+ {"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAQlD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,0BAA0B;iBAMxB,OAAO,8BAA8B;;oBAElC,MAAM;6BACG,MAAM;MAC7B,MAAM,GAAG,OAAO,yBAAyB,EAiC5C,CAAC;AAEF,eAAO,MAAM,gBAAgB;6BAOF;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B;0BACqB,OAAO,MAAM,EAAE,GAAG,CAAC;aAChC,iBAAiB;SACrB,GAAG;;MAEN,QAAQ;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,OAAO,qBAAqB,GAAG,SAAS,CAAC;CACpE,CAuGA,CAAC"}
package/dist/agent/master/with-hints.js CHANGED
@@ -2,7 +2,6 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.triggerHintsFlow = exports.getUserMessageWithForHints = void 0;
4
4
  const vision_1 = require("@empiricalrun/llm/vision");
5
- const actions_1 = require("../../actions");
6
5
  const click_1 = require("../../actions/click");
7
6
  const fill_1 = require("../../actions/fill");
8
7
  const hover_1 = require("../../actions/hover");
@@ -36,7 +35,7 @@ const getUserMessageWithForHints = ({ userMessage, options, pageScreenshot, anno
36
35
  ];
37
36
  };
38
37
  exports.getUserMessageWithForHints = getUserMessageWithForHints;
39
- const triggerHintsFlow = async ({ outputFromGetNextAction, generatedAnnotations, page, llm, trace, }) => {
38
+ const triggerHintsFlow = async ({ outputFromGetNextAction, generatedAnnotations, actions, llm, trace, }) => {
40
39
  try {
41
40
  const hasElementAnnotation = outputFromGetNextAction?.elementAnnotation?.length &&
42
41
  outputFromGetNextAction?.elementAnnotation?.trim()?.length &&
@@ -56,7 +55,6 @@ const triggerHintsFlow = async ({ outputFromGetNextAction, generatedAnnotations,
56
55
  hintsExecutionCompletion: undefined,
57
56
  };
58
57
  }
59
- const actions = new actions_1.PlaywrightActions(page);
60
58
  const completion = await llm.createChatCompletion({
61
59
  messages: [
62
60
  {
package/dist/evals/update-scenario-agent.evals.js CHANGED
@@ -7,7 +7,7 @@ const js_levenshtein_1 = __importDefault(require("js-levenshtein"));
7
7
  const update_flow_1 = require("../agent/codegen/update-flow");
8
8
  const updateScenarioCodeAgentEvaluate = async ({ item, trace }) => {
9
9
  const { testCase, testFiles, pageFiles, testFilePath, testFileContent } = item.input;
10
- const fileChanges = await (0, update_flow_1.getUpdateTestCodeCompletion)({
10
+ const { fileChanges } = await (0, update_flow_1.getUpdateTestCodeCompletion)({
11
11
  testCase,
12
12
  testFiles,
13
13
  pageFiles,
package/dist/types/index.d.ts CHANGED
@@ -57,9 +57,11 @@ export type Action = {
57
57
  trace?: TraceClient;
58
58
  }) => Promise<{
59
59
  locator: string;
60
+ testCaseLine?: string;
60
61
  } | void>;
61
62
  template: (args: Record<string, any>, options: {
62
63
  locator: string;
64
+ testCaseLine?: string;
63
65
  }) => {
64
66
  code: string;
65
67
  importPath?: string;
package/dist/types/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,WAAW,EACX,eAAe,EACf,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEtC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,EAAE,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAC;AAExD,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CACtC,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE;IACP,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACpC,iBAAiB,EAAE,CAAC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,CAAC;CAClE,KACE,MAAM,CAAC;AAEZ,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,OAAO,EAAE;QACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,WAAW,CAAC;KACrB,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC1C,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KACzB;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5C,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,WAAW,EACX,eAAe,EACf,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEtC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,EAAE,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAC;AAExD,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CACtC,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE;IACP,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACpC,iBAAiB,EAAE,CAAC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,CAAC;CAClE,KACE,MAAM,CAAC;AAEZ,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,OAAO,EAAE;QACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,WAAW,CAAC;KACrB,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IACjE,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,KAChD;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5C,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@empiricalrun/test-gen",
3
- "version": "0.38.44",
3
+ "version": "0.38.46",
4
4
  "publishConfig": {
5
5
  "registry": "https://registry.npmjs.org/",
6
6
  "access": "public"
@@ -71,9 +71,9 @@
71
71
  "ts-morph": "^24.0.0",
72
72
  "tsx": "^4.16.2",
73
73
  "typescript": "^5.3.3",
74
- "@empiricalrun/r2-uploader": "^0.3.7",
74
+ "@empiricalrun/llm": "^0.9.28",
75
75
  "@empiricalrun/reporter": "^0.21.6",
76
- "@empiricalrun/llm": "^0.9.28"
76
+ "@empiricalrun/r2-uploader": "^0.3.7"
77
77
  },
78
78
  "devDependencies": {
79
79
  "@types/detect-port": "^1.3.5",