@empiricalrun/test-gen 0.33.13 → 0.33.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,17 @@
1
1
  # @empiricalrun/test-gen
2
2
 
3
+ ## 0.33.15
4
+
5
+ ### Patch Changes
6
+
7
+ - 4091222: feat: infer master / code agent from the task provided
8
+
9
+ ## 0.33.14
10
+
11
+ ### Patch Changes
12
+
13
+ - ec95c2a: fix: update flow prompt
14
+
3
15
  ## 0.33.13
4
16
 
5
17
  ### Patch Changes
@@ -1,3 +1,4 @@
1
+ import { TraceClient } from "@empiricalrun/llm";
1
2
  import { Page } from "playwright";
2
3
  import { PlaywrightTestConfig } from "playwright/test";
3
4
  import { TestGenConfig } from "../../types";
@@ -8,7 +9,7 @@ export declare function prepareBrowsingAgentTask(steps: string[]): string;
8
9
  * @param {TestGenConfig} genConfig
9
10
  * @return {*} {Promise<string>}
10
11
  */
11
- export declare function prepareFileForMasterAgent(genConfig: TestGenConfig): Promise<string>;
12
+ export declare function prepareFileForMasterAgent(genConfig: TestGenConfig, trace?: TraceClient): Promise<string>;
12
13
  export declare function injectPwLocatorGenerator(page: Page): Promise<void>;
13
14
  /**
14
15
  * Function to validate if the test file path are valid.
@@ -1 +1 @@
1
- {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAiBvD,OAAO,EAAe,aAAa,EAAE,MAAM,aAAa,CAAC;AAKzD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA4ED;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,GACvB,OAAO,CAAC,MAAM,CAAC,CA0DjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAiBxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QA+BjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}
1
+ {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAI3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAgBvD,OAAO,EAAe,aAAa,EAAE,MAAM,aAAa,CAAC;AAKzD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA4ED;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CAuDjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAiBxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QA+BjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}
@@ -14,7 +14,6 @@ const logger_1 = require("../../bin/logger");
14
14
  const context_1 = require("../../bin/utils/context");
15
15
  const fs_1 = require("../../bin/utils/fs");
16
16
  const web_1 = require("../../bin/utils/platform/web");
17
- const reporter_1 = require("../../reporter");
18
17
  const create_test_block_1 = require("../codegen/create-test-block");
19
18
  const fix_ts_errors_1 = require("../codegen/fix-ts-errors");
20
19
  const update_flow_1 = require("../codegen/update-flow");
@@ -90,27 +89,24 @@ async function prepareFileForUpdateScenario(genConfig, trace) {
90
89
  * @param {TestGenConfig} genConfig
91
90
  * @return {*} {Promise<string>}
92
91
  */
93
- async function prepareFileForMasterAgent(genConfig) {
94
- const trace = llm_1.langfuseInstance.trace({
92
+ async function prepareFileForMasterAgent(genConfig, trace) {
93
+ const prepareFileSpan = trace?.span({
95
94
  name: "prepare-file-for-master-agent",
96
95
  });
97
- new logger_1.CustomLogger({ useReporter: true }).log(`Preparing file for master agent. [view trace](${trace.getTraceUrl()})`);
98
- if (trace.getTraceUrl()) {
99
- await new reporter_1.TestGenUpdatesReporter().sendAgentTraceUrl(trace.getTraceUrl());
100
- }
96
+ new logger_1.CustomLogger({ useReporter: true }).log(`Preparing file for master agent. [view trace](${trace?.getTraceUrl()})`);
101
97
  const { specPath, testCase } = genConfig;
102
98
  const { name, suites } = testCase;
103
99
  // check if the spec file exists
104
100
  // if no then create a new file with test and expect imports
105
101
  if (!fs_extra_1.default.existsSync(specPath)) {
106
- const fileCreateSpan = trace.span({
102
+ const fileCreateSpan = prepareFileSpan?.span({
107
103
  name: "create-file",
108
104
  input: { specPath },
109
105
  });
110
106
  await fs_extra_1.default.createFile(specPath);
111
107
  const fileContentWithImports = (0, web_1.addNewImport)("", ["test", "expect"], (0, web_1.getFixtureImportPath)(specPath));
112
108
  await fs_extra_1.default.writeFile(specPath, fileContentWithImports, "utf-8");
113
- fileCreateSpan.end({ output: { specPath, fileContentWithImports } });
109
+ fileCreateSpan?.end({ output: { specPath, fileContentWithImports } });
114
110
  }
115
111
  const existingContents = await fs_extra_1.default.readFile(specPath, "utf-8");
116
112
  const { testBlock } = (0, web_1.getTypescriptTestBlock)({
@@ -120,7 +116,7 @@ async function prepareFileForMasterAgent(genConfig) {
120
116
  });
121
117
  if (!testBlock) {
122
118
  const newTestBlock = await (0, create_test_block_1.createEmptyTestCaseBlock)({
123
- trace,
119
+ trace: prepareFileSpan,
124
120
  testCase: genConfig.testCase,
125
121
  file: specPath,
126
122
  options: genConfig.options,
@@ -133,7 +129,7 @@ async function prepareFileForMasterAgent(genConfig) {
133
129
  });
134
130
  await fs_extra_1.default.writeFile(specPath, updatedContent, "utf-8");
135
131
  }
136
- await prepareFileForUpdateScenario(genConfig, trace);
132
+ await prepareFileForUpdateScenario(genConfig, prepareFileSpan);
137
133
  return specPath;
138
134
  }
139
135
  exports.prepareFileForMasterAgent = prepareFileForMasterAgent;
@@ -1,3 +1,4 @@
1
+ import { TraceClient } from "@empiricalrun/llm";
1
2
  import { TestCase, TestGenConfigOptions } from "../../types";
2
- export declare function generateTest(testCase: TestCase, file: string, options: TestGenConfigOptions): Promise<TestCase[]>;
3
+ export declare function generateTest(testCase: TestCase, file: string, options: TestGenConfigOptions, trace: TraceClient): Promise<TestCase[]>;
3
4
  //# sourceMappingURL=run.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AA0BA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAgHrB"}
1
+ {"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkC,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAkBhF,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,EAC7B,KAAK,EAAE,WAAW,GACjB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAwGrB"}
@@ -5,17 +5,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
6
  exports.generateTest = void 0;
7
7
  const llm_1 = require("@empiricalrun/llm");
8
- const crypto_1 = __importDefault(require("crypto"));
9
8
  const fs_extra_1 = __importDefault(require("fs-extra"));
10
9
  const logger_1 = require("../../bin/logger");
11
10
  const context_1 = require("../../bin/utils/context");
12
11
  const web_1 = require("../../bin/utils/platform/web");
13
12
  const constants_1 = require("../../constants");
14
- const reporter_1 = require("../../reporter");
15
- const session_1 = require("../../session");
16
13
  const fix_ts_errors_1 = require("./fix-ts-errors");
17
14
  const update_flow_1 = require("./update-flow");
18
- async function generateTest(testCase, file, options) {
15
+ async function generateTest(testCase, file, options, trace) {
19
16
  const logger = new logger_1.CustomLogger();
20
17
  if (!fs_extra_1.default.existsSync(file)) {
21
18
  logger.log(`Creating a new spec file: ${file}`);
@@ -30,24 +27,19 @@ async function generateTest(testCase, file, options) {
30
27
  });
31
28
  const isUpdate = !!testBlock;
32
29
  if (isUpdate) {
33
- return await (0, update_flow_1.updateTest)(testCase, file, options);
30
+ return await (0, update_flow_1.updateTest)(testCase, file, options, true, true, trace);
34
31
  }
35
32
  const generatedTestCases = [];
36
33
  logger.logEmptyLine();
37
- const session = (0, session_1.getSessionDetails)();
38
- const trace = llm_1.langfuseInstance.trace({
39
- name: "generate-test",
40
- id: crypto_1.default.randomUUID(),
41
- release: session.version,
42
- tags: [options.metadata.projectName, options.metadata.environment].filter((s) => !!s),
34
+ const createTestSpan = trace.span({
35
+ name: "create-test",
36
+ input: {
37
+ testCase,
38
+ file,
39
+ options,
40
+ },
43
41
  });
44
- try {
45
- await new reporter_1.TestGenUpdatesReporter().sendAgentTraceUrl(trace.getTraceUrl());
46
- }
47
- catch (e) {
48
- console.warn("Failed to send trace url", e);
49
- }
50
- trace.event({
42
+ createTestSpan.event({
51
43
  name: "collate-files-as-text",
52
44
  output: {
53
45
  codePrompt,
@@ -55,8 +47,7 @@ async function generateTest(testCase, file, options) {
55
47
  testFileContent,
56
48
  },
57
49
  });
58
- trace.update({ input: { testCase } });
59
- const promptSpan = trace.span({
50
+ const promptSpan = createTestSpan.span({
60
51
  name: "add-scenario-prompt",
61
52
  });
62
53
  const instruction = await (0, llm_1.getPrompt)("add-scenario", {
@@ -94,7 +85,7 @@ async function generateTest(testCase, file, options) {
94
85
  await fs_extra_1.default.writeFile(file, updatedContent, "utf-8");
95
86
  readWriteFileSpan.end({ output: { updatedContent } });
96
87
  logger.log("Linting generated code...");
97
- trace.event({ name: "lint-file" });
88
+ createTestSpan.event({ name: "lint-file" });
98
89
  await (0, web_1.lintErrors)(file);
99
90
  await (0, fix_ts_errors_1.validateAndFixTypescriptErrors)({
100
91
  trace,
@@ -105,12 +96,12 @@ async function generateTest(testCase, file, options) {
105
96
  testCase: testCase,
106
97
  options,
107
98
  });
108
- trace.event({ name: "format-file" });
99
+ createTestSpan.event({ name: "format-file" });
109
100
  await (0, web_1.formatCode)(file);
110
101
  logger.success("File formatted successfully!");
111
102
  logger.log(`Successfully generated code for the given task. \n View [trace](${trace.getTraceUrl()})`);
112
103
  generatedTestCases.push(testCase);
113
- trace.update({ input: { testCase }, output: { response } });
104
+ createTestSpan.end({ output: { response } });
114
105
  await (0, llm_1.flushAllTraces)();
115
106
  return generatedTestCases;
116
107
  }
@@ -3,7 +3,7 @@ import { TestCase, TestGenConfigOptions } from "../../types";
3
3
  type UpdatedTestCase = TestCase & {
4
4
  updatedFiles: string[];
5
5
  };
6
- export declare function updateTest(testCase: TestCase, file: string, options: TestGenConfigOptions | undefined, logging?: boolean, validate?: boolean): Promise<UpdatedTestCase[]>;
6
+ export declare function updateTest(testCase: TestCase, file: string, options: TestGenConfigOptions | undefined, logging?: boolean, validate?: boolean, trace?: TraceClient): Promise<UpdatedTestCase[]>;
7
7
  export declare function appendCreateTestBlock({ testCase, file, options, trace, validateTypes, }: {
8
8
  testCase: TestCase;
9
9
  file: string;
@@ -1 +1 @@
1
- {"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAsB3B,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,KAAK,eAAe,GAAG,QAAQ,GAAG;IAChC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAqIF,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAAG,SAAS,EACzC,OAAO,GAAE,OAAc,EACvB,QAAQ,GAAE,OAAc,GACvB,OAAO,CAAC,eAAe,EAAE,CAAC,CAkG5B;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,EACL,aAAoB,GACrB,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA6E7B"}
1
+ {"version":3,"file":"update-flow.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/update-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAsB3B,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAI7D,KAAK,eAAe,GAAG,QAAQ,GAAG;IAChC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,CAAC;AAqIF,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAAG,SAAS,EACzC,OAAO,GAAE,OAAc,EACvB,QAAQ,GAAE,OAAc,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,EAAE,CAAC,CAoG5B;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,EACL,aAAoB,GACrB,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA6E7B"}
@@ -13,7 +13,6 @@ const context_1 = require("../../bin/utils/context");
13
13
  const fs_1 = require("../../bin/utils/fs");
14
14
  const web_1 = require("../../bin/utils/platform/web");
15
15
  const constants_1 = require("../../constants");
16
- const reporter_1 = require("../../reporter");
17
16
  const session_1 = require("../../session");
18
17
  const fix_ts_errors_1 = require("./fix-ts-errors");
19
18
  const utils_1 = require("./utils");
@@ -104,29 +103,33 @@ async function applyFileChanges({ validateTypes = true, trace, testCase, fileCha
104
103
  logger.success(`${fileChange.filePath} file formatted successfully!`);
105
104
  }));
106
105
  }
107
- async function updateTest(testCase, file, options, logging = true, validate = true) {
106
+ async function updateTest(testCase, file, options, logging = true, validate = true, trace) {
108
107
  const logger = new logger_1.CustomLogger({ useReporter: logging });
109
108
  const context = await (0, context_1.contextForGeneration)(file);
110
109
  const { codePrompt, pomPrompt, testFileContent } = context;
111
110
  const generatedTestCases = [];
112
111
  logger.logEmptyLine();
113
112
  const session = (0, session_1.getSessionDetails)();
114
- const trace = llm_1.langfuseInstance.trace({
113
+ trace =
114
+ trace ||
115
+ llm_1.langfuseInstance.trace({
116
+ name: "update-test",
117
+ id: crypto_1.default.randomUUID(),
118
+ release: session.version,
119
+ tags: [
120
+ options?.metadata.projectName || "",
121
+ options?.metadata.environment || "",
122
+ ].filter((s) => !!s),
123
+ });
124
+ const updateTestSpan = trace.span({
115
125
  name: "update-test",
116
- id: crypto_1.default.randomUUID(),
117
- release: session.version,
118
- tags: [
119
- options?.metadata.projectName || "",
120
- options?.metadata.environment || "",
121
- ].filter((s) => !!s),
126
+ input: {
127
+ testCase,
128
+ file,
129
+ options,
130
+ },
122
131
  });
123
- try {
124
- await new reporter_1.TestGenUpdatesReporter().sendAgentTraceUrl(trace.getTraceUrl());
125
- }
126
- catch (e) {
127
- console.warn("Failed to send trace url as test gen update", e);
128
- }
129
- trace.event({
132
+ updateTestSpan.event({
130
133
  name: "collate-files-as-text",
131
134
  output: {
132
135
  codePrompt,
@@ -134,20 +137,17 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
134
137
  testFileContent,
135
138
  },
136
139
  });
137
- trace.update({ input: { testCase } });
138
- const promptSpan = trace.span({
140
+ const promptSpan = updateTestSpan.span({
139
141
  name: "update-scenario-prompt",
140
142
  });
141
143
  const promptName = "update-scenario";
142
144
  // if describe blocks are present, we need to add them to the scenario name
143
145
  // e.g. describe block: login ---> login with email
144
146
  // this is help LLM navigate to the right test block
145
- const scenarioName = testCase.suites?.length
146
- ? testCase.suites.reduce((text, suite) => {
147
- text += `Describe block: ${suite} ----> `;
148
- return text;
149
- }, "") + testCase.name
150
- : testCase.name;
147
+ const scenarioName = (0, web_1.buildTestNamePrompt)({
148
+ testName: testCase.name,
149
+ suites: testCase.suites,
150
+ });
151
151
  const { testBlock: currentScenarioCodeBlock } = (0, web_1.getTypescriptTestBlock)({
152
152
  scenarioName: testCase?.name,
153
153
  content: testFileContent,
@@ -163,7 +163,7 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
163
163
  });
164
164
  promptSpan.end({ output: { instruction } });
165
165
  const llm = new llm_1.LLM({
166
- trace,
166
+ trace: updateTestSpan,
167
167
  provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
168
168
  defaultModel: options?.model || constants_1.DEFAULT_MODEL,
169
169
  providerApiKey: constants_1.MODEL_API_KEYS[options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
@@ -180,7 +180,7 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
180
180
  const fileChanges = (0, utils_1.extractTestUpdates)(response);
181
181
  await applyFileChanges({
182
182
  validateTypes: validate,
183
- trace,
183
+ trace: updateTestSpan,
184
184
  testCase,
185
185
  fileChanges,
186
186
  logger,
@@ -193,7 +193,7 @@ async function updateTest(testCase, file, options, logging = true, validate = tr
193
193
  ...testCase,
194
194
  updatedFiles: fileChanges.map((f) => f.filePath),
195
195
  });
196
- trace.update({ input: { testCase }, output: { response } });
196
+ updateTestSpan.end({ output: { response } });
197
197
  await (0, llm_1.flushAllTraces)();
198
198
  return generatedTestCases;
199
199
  }
@@ -0,0 +1,13 @@
1
+ import { TraceClient } from "@empiricalrun/llm";
2
+ import { Agent, TestGenConfigOptions } from "../../types";
3
+ export declare function inferAgentBasedTask({ task, testCode, options, trace, testName, }: {
4
+ task: string;
5
+ testCode?: string;
6
+ options?: TestGenConfigOptions;
7
+ trace?: TraceClient;
8
+ testName: string;
9
+ }): Promise<{
10
+ response: Agent;
11
+ trace: TraceClient;
12
+ }>;
13
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/infer-agent/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAS3B,OAAO,EAAE,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG1D,wBAAsB,mBAAmB,CAAC,EACxC,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,KAAK,EACL,QAAQ,GACT,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,WAAW,CAAA;CAAE,CAAC,CAyFnD"}
@@ -0,0 +1,91 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.inferAgentBasedTask = void 0;
4
+ const llm_1 = require("@empiricalrun/llm");
5
+ const constants_1 = require("../../constants");
6
+ const session_1 = require("../../session");
7
+ const session = (0, session_1.getSessionDetails)();
8
+ async function inferAgentBasedTask({ task, testCode, options, trace, testName, }) {
9
+ trace =
10
+ trace ||
11
+ llm_1.langfuseInstance.trace({
12
+ name: "infer-agent-task",
13
+ id: crypto.randomUUID(),
14
+ release: session.version,
15
+ });
16
+ const inferAgentSpan = trace.span({
17
+ name: "infer-agent",
18
+ input: {
19
+ task,
20
+ testCode,
21
+ testName,
22
+ options,
23
+ },
24
+ });
25
+ const promptSpan = inferAgentSpan.span({
26
+ name: "infer-agent-prompt",
27
+ input: {
28
+ task,
29
+ testCode,
30
+ },
31
+ });
32
+ const testNameAndCode = testCode
33
+ ? `
34
+ Test name: ${testName}
35
+
36
+ Test code:
37
+ ${testCode}
38
+ `
39
+ : "";
40
+ const prompt = await (0, llm_1.getPrompt)("infer-agent", {
41
+ task,
42
+ testNameAndCode,
43
+ });
44
+ promptSpan?.end({ output: { prompt } });
45
+ const llm = new llm_1.LLM({
46
+ trace: inferAgentSpan,
47
+ provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
48
+ defaultModel: options?.model || constants_1.DEFAULT_MODEL,
49
+ providerApiKey: constants_1.MODEL_API_KEYS[options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
50
+ });
51
+ const firstShotMessage = await llm.createChatCompletion({
52
+ messages: prompt,
53
+ modelParameters: {
54
+ ...constants_1.DEFAULT_MODEL_PARAMETERS,
55
+ ...options?.modelParameters,
56
+ temperature: 1,
57
+ },
58
+ responseFormat: {
59
+ type: "json_schema",
60
+ json_schema: {
61
+ name: "infer-master-code",
62
+ strict: true,
63
+ schema: {
64
+ type: "object",
65
+ properties: {
66
+ reason: {
67
+ type: "string",
68
+ },
69
+ response: {
70
+ type: "string",
71
+ },
72
+ },
73
+ additionalProperties: false,
74
+ required: ["reason", "response"],
75
+ },
76
+ },
77
+ },
78
+ });
79
+ const output = JSON.parse(firstShotMessage?.content || "{}");
80
+ inferAgentSpan.end({
81
+ output: {
82
+ response: output.response,
83
+ reason: output.reason,
84
+ },
85
+ });
86
+ return {
87
+ response: output.response,
88
+ trace: inferAgentSpan,
89
+ };
90
+ }
91
+ exports.inferAgentBasedTask = inferAgentBasedTask;
package/dist/bin/index.js CHANGED
@@ -9,12 +9,15 @@ const dotenv_1 = __importDefault(require("dotenv"));
9
9
  const run_1 = require("../agent/browsing/run");
10
10
  const utils_1 = require("../agent/browsing/utils");
11
11
  const run_2 = require("../agent/codegen/run");
12
+ const infer_agent_1 = require("../agent/infer-agent");
12
13
  const reporter_1 = require("../reporter");
13
14
  const ci_1 = require("../reporter/ci");
14
15
  const session_1 = require("../session");
15
16
  const test_build_1 = require("../test-build");
16
17
  const logger_1 = require("./logger");
17
18
  const utils_2 = require("./utils");
19
+ const context_1 = require("./utils/context");
20
+ const web_1 = require("./utils/platform/web");
18
21
  dotenv_1.default.config({
19
22
  path: [".env.local", ".env"],
20
23
  });
@@ -22,6 +25,25 @@ process.on("beforeExit", async () => await (0, llm_1.flushAllTraces)());
22
25
  process.on("exit", async () => await (0, llm_1.flushAllTraces)());
23
26
  process.on("SIGINT", async () => await (0, llm_1.flushAllTraces)());
24
27
  process.on("SIGTERM", async () => await (0, llm_1.flushAllTraces)());
28
+ async function resolveAgentUsingTask({ specPath, testCase, trace, }) {
29
+ const { pomPrompt, testFileContent } = await (0, context_1.contextForGeneration)(specPath);
30
+ const isUpdateFlow = (0, web_1.isTestPresent)({
31
+ specPath,
32
+ testCase,
33
+ });
34
+ const { response } = await (0, infer_agent_1.inferAgentBasedTask)({
35
+ task: testCase.steps.join("\n"),
36
+ testCode: isUpdateFlow ? `${pomPrompt}\n\n${testFileContent}` : "",
37
+ testName: isUpdateFlow
38
+ ? (0, web_1.buildTestNamePrompt)({
39
+ testName: testCase.name,
40
+ suites: testCase.suites,
41
+ })
42
+ : "",
43
+ trace,
44
+ });
45
+ return response;
46
+ }
25
47
  async function runAgent(testGenConfig) {
26
48
  const logger = new logger_1.CustomLogger();
27
49
  const { specPath, testCase } = testGenConfig;
@@ -36,10 +58,35 @@ async function runAgent(testGenConfig) {
36
58
  if (await (0, session_1.shouldStopSession)()) {
37
59
  return;
38
60
  }
39
- if (testGenConfig.options?.agent !== "code") {
61
+ let agent = testGenConfig.options?.agent;
62
+ const session = (0, session_1.getSessionDetails)();
63
+ const trace = llm_1.langfuseInstance.trace({
64
+ name: "generate-test",
65
+ id: crypto.randomUUID(),
66
+ release: session.version,
67
+ tags: [
68
+ testGenConfig.options?.metadata.projectName || "",
69
+ testGenConfig.options?.metadata.environment || "",
70
+ ].filter((s) => !!s),
71
+ });
72
+ try {
73
+ await new reporter_1.TestGenUpdatesReporter().sendAgentTraceUrl(trace.getTraceUrl());
74
+ }
75
+ catch (e) {
76
+ console.warn("Failed to send trace url as test gen update", e);
77
+ }
78
+ if (!agent || agent === "auto") {
79
+ agent = await resolveAgentUsingTask({
80
+ specPath,
81
+ testCase,
82
+ trace,
83
+ });
84
+ testGenConfig.options.agent = agent;
85
+ }
86
+ if (agent !== "code") {
40
87
  // this assumes we have only one scenario in test config
41
88
  logger.success(`Generating test using ${testGenConfig.options?.agent} agent. ${process.env.LOG_URL ? `[view log](${process.env.LOG_URL})` : ""}`);
42
- const filePathToUpdate = await (0, utils_1.prepareFileForMasterAgent)(testGenConfig);
89
+ const filePathToUpdate = await (0, utils_1.prepareFileForMasterAgent)(testGenConfig, trace);
43
90
  await (0, run_1.generateTestsUsingMasterAgent)({
44
91
  testFilePath: specPath,
45
92
  filePathToUpdate,
@@ -48,7 +95,7 @@ async function runAgent(testGenConfig) {
48
95
  }
49
96
  else {
50
97
  logger.success(`Generating test using coding agent. ${process.env.LOG_URL ? `[view log](${process.env.LOG_URL})` : ""}`);
51
- await (0, run_2.generateTest)(testCase, specPath, testGenConfig.options);
98
+ await (0, run_2.generateTest)(testCase, specPath, testGenConfig.options, trace);
52
99
  }
53
100
  }
54
101
  (async function main() {
@@ -1,4 +1,5 @@
1
1
  import { Node } from "ts-morph";
2
+ import { TestCase } from "../../../../types";
2
3
  /**
3
4
  * function to get the test block and test node for the scenario
4
5
  * @export
@@ -60,4 +61,12 @@ export declare function addUserContextFixture({ scenarioName, filePath, suites,
60
61
  filePath: string;
61
62
  suites: string[];
62
63
  }): Promise<void>;
64
+ export declare function isTestPresent({ specPath, testCase, }: {
65
+ specPath: string;
66
+ testCase: TestCase;
67
+ }): boolean;
68
+ export declare function buildTestNamePrompt({ testName, suites, }: {
69
+ testName: string;
70
+ suites: string[];
71
+ }): string;
63
72
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAGL,IAAI,EAIL,MAAM,UAAU,CAAC;AAGlB;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,EACrC,YAAY,EACZ,MAAM,EACN,OAAO,GACR,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB,GAAG;IACF,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,EAAE,IAAI,GAAG,SAAS,CAAC;CAC5B,CAuCA;AAwBD,wBAAsB,0CAA0C,CAC9D,QAAQ,EAAE,MAAM,oBA+BjB;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,IAAI,GAAG,SAAS,GACrB,IAAI,GAAG,SAAS,CA4BlB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CA8C7D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,mCAWjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAMpD;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,UAcpD;AAED,wBAAsB,iCAAiC,CAAC,QAAQ,EAAE,MAAM,+BAoBvE;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,MAAM,UA0CtB;AAED,eAAO,MAAM,6BAA6B;qBAKvB,MAAM;iBACV,MAAM;YACX,MAAM,EAAE;YA2DjB,CAAC;AAEF,eAAO,MAAM,iCAAiC,cACjC,MAAM,EAAE,gBACL,MAAM,sBAyBrB,CAAC;AAEF,wBAAsB,qBAAqB,CAAC,EAC1C,YAAY,EACZ,QAAQ,EACR,MAAM,GACP,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,iBAsDA"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAGL,IAAI,EAIL,MAAM,UAAU,CAAC;AAGlB,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAE7C;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,EACrC,YAAY,EACZ,MAAM,EACN,OAAO,GACR,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB,GAAG;IACF,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,EAAE,IAAI,GAAG,SAAS,CAAC;CAC5B,CAuCA;AAwBD,wBAAsB,0CAA0C,CAC9D,QAAQ,EAAE,MAAM,oBA+BjB;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,IAAI,GAAG,SAAS,GACrB,IAAI,GAAG,SAAS,CA4BlB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CA8C7D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,mCAWjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAMpD;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,UAcpD;AAED,wBAAsB,iCAAiC,CAAC,QAAQ,EAAE,MAAM,+BAoBvE;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,MAAM,UA0CtB;AAED,eAAO,MAAM,6BAA6B;qBAKvB,MAAM;iBACV,MAAM;YACX,MAAM,EAAE;YA2DjB,CAAC;AAEF,eAAO,MAAM,iCAAiC,cACjC,MAAM,EAAE,gBACL,MAAM,sBAyBrB,CAAC;AAEF,wBAAsB,qBAAqB,CAAC,EAC1C,YAAY,EACZ,QAAQ,EACR,MAAM,GACP,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,iBAsDA;AAED,wBAAgB,aAAa,CAAC,EAC5B,QAAQ,EACR,QAAQ,GACT,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;CACpB,WAYA;AAED,wBAAgB,mBAAmB,CAAC,EAClC,QAAQ,EACR,MAAM,GACP,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,UAOA"}
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.addUserContextFixture = exports.importAllExportsStmtFromFilePaths = exports.injectCodeSnippetBySuiteChain = exports.replaceCreateTestWithNewCode = exports.getPageVariableNameFromCreateTest = exports.getFixtureImportPath = exports.removeTestOnly = exports.addNewImport = exports.formatCode = exports.lintErrors = exports.stripAndPrependImports = exports.validateTypescript = exports.appendToTestBlock = exports.findFirstSerialDescribeBlock = exports.hasTopLevelDescribeConfigureWithSerialMode = exports.getTypescriptTestBlock = void 0;
6
+ exports.buildTestNamePrompt = exports.isTestPresent = exports.addUserContextFixture = exports.importAllExportsStmtFromFilePaths = exports.injectCodeSnippetBySuiteChain = exports.replaceCreateTestWithNewCode = exports.getPageVariableNameFromCreateTest = exports.getFixtureImportPath = exports.removeTestOnly = exports.addNewImport = exports.formatCode = exports.lintErrors = exports.stripAndPrependImports = exports.validateTypescript = exports.appendToTestBlock = exports.findFirstSerialDescribeBlock = exports.hasTopLevelDescribeConfigureWithSerialMode = exports.getTypescriptTestBlock = void 0;
7
7
  const eslint_1 = require("eslint");
8
8
  const fs_extra_1 = __importDefault(require("fs-extra"));
9
9
  const lodash_isequal_1 = __importDefault(require("lodash.isequal"));
@@ -413,3 +413,26 @@ async function addUserContextFixture({ scenarioName, filePath, suites, }) {
413
413
  }
414
414
  }
415
415
  exports.addUserContextFixture = addUserContextFixture;
416
+ function isTestPresent({ specPath, testCase, }) {
417
+ let isTestPresent = true;
418
+ isTestPresent = fs_extra_1.default.existsSync(specPath);
419
+ if (isTestPresent) {
420
+ const { testBlock } = getTypescriptTestBlock({
421
+ suites: testCase.suites,
422
+ scenarioName: testCase.name,
423
+ content: fs_extra_1.default.readFileSync(specPath, "utf-8"),
424
+ });
425
+ isTestPresent = Boolean(testBlock);
426
+ }
427
+ return isTestPresent;
428
+ }
429
+ exports.isTestPresent = isTestPresent;
430
+ function buildTestNamePrompt({ testName, suites, }) {
431
+ return suites?.length
432
+ ? suites.reduce((text, suite) => {
433
+ text += `Describe block: ${suite} ----> `;
434
+ return text;
435
+ }, "") + testName
436
+ : testName;
437
+ }
438
+ exports.buildTestNamePrompt = buildTestNamePrompt;
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/constants/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAE3E,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAItD,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,WAAsB,CAAC;AAE5D,eAAO,MAAM,aAAa,EAAE,QAA8B,CAAC;AAE3D,eAAO,MAAM,wBAAwB,EAAE,eAGtC,CAAC;AAEF,eAAO,MAAM,2BAA2B,EAAE,eAGzC,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/constants/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAE3E,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAItD,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,WAAsB,CAAC;AAE5D,eAAO,MAAM,aAAa,EAAE,QAAmB,CAAC;AAEhD,eAAO,MAAM,wBAAwB,EAAE,eAGtC,CAAC;AAEF,eAAO,MAAM,2BAA2B,EAAE,eAGzC,CAAC"}
@@ -7,7 +7,7 @@ exports.MODEL_API_KEYS = {
7
7
  openai: process.env.OPENAI_API_KEY || "",
8
8
  };
9
9
  exports.DEFAULT_MODEL_PROVIDER = "openai";
10
- exports.DEFAULT_MODEL = "gpt-4o-2024-08-06";
10
+ exports.DEFAULT_MODEL = "gpt-4o";
11
11
  exports.DEFAULT_MODEL_PARAMETERS = {
12
12
  temperature: 0.5,
13
13
  max_completion_tokens: 4000,
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=infer-master-code.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"infer-master-code.d.ts","sourceRoot":"","sources":["../../src/evals/infer-master-code.ts"],"names":[],"mappings":""}
@@ -0,0 +1,18 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const llm_1 = require("@empiricalrun/llm");
4
+ const infer_agent_1 = require("../agent/infer-agent");
5
+ const datasetName = "infer-master-or-code-agent";
6
+ (async function main() {
7
+ const dataset = await llm_1.langfuseInstance.getDataset(datasetName);
8
+ const runName = `${datasetName}-${Date.now()}`;
9
+ for (const item of dataset.items) {
10
+ const { response, trace } = await (0, infer_agent_1.inferAgentBasedTask)(item.input);
11
+ await item.link(trace, runName, {});
12
+ trace?.score({
13
+ name: "equality",
14
+ value: item.expectedOutput === response ? 1 : 0, // score value
15
+ });
16
+ }
17
+ await llm_1.langfuseInstance.flushAsync();
18
+ })();
@@ -10,8 +10,9 @@ export type Environment = {
10
10
  slug: string;
11
11
  playwrightProjects: string[];
12
12
  };
13
+ export type Agent = "code" | "master" | "auto";
13
14
  export type TestGenConfigOptions = {
14
- agent: "code" | "master";
15
+ agent: Agent;
15
16
  model: LLMModel;
16
17
  modelProvider: LLMProvider;
17
18
  modelParameters?: ModelParameters;
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,WAAW,EACX,eAAe,EACf,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,EAAE,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,GAAG,QAAQ,CAAC;IACzB,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CACtC,IAAI,EAAE,IAAI,EACV,OAAO,EAAE;IACP,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACrC,KACE,MAAM,CAAC;AAEZ,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,OAAO,EAAE;QACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,WAAW,CAAC;KACrB,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC1C,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KACzB;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5C,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,WAAW,EACX,eAAe,EACf,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,EAAE,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;AAE/C,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;IAC3B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG,CACtC,IAAI,EAAE,IAAI,EACV,OAAO,EAAE;IACP,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACrC,KACE,MAAM,CAAC;AAEZ,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;AAEtE,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,CAAC,OAAO,EAAE;QACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,WAAW,CAAC;KACrB,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC1C,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KACzB;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5C,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@empiricalrun/test-gen",
3
- "version": "0.33.13",
3
+ "version": "0.33.15",
4
4
  "publishConfig": {
5
5
  "registry": "https://registry.npmjs.org/",
6
6
  "access": "public"