@empiricalrun/test-gen 0.21.9 → 0.22.2

package/CHANGELOG.md

@@ -1,5 +1,44 @@
 # @empiricalrun/test-gen
 
+## 0.22.2
+
+### Patch Changes
+
+- d35239e: fix: update imports for test and expect to fixtures
+- Updated dependencies [2de783d]
+  - @empiricalrun/llm@0.7.0
+
+## 0.22.1
+
+### Patch Changes
+
+- b96491f: fix: typescript errors for valid CommonJS module imports
+
+## 0.22.0
+
+### Minor Changes
+
+- 4676ab7: feat: use gpt-4o-2024-08-06 as default
+
+### Patch Changes
+
+- Updated dependencies [1770926]
+- Updated dependencies [4676ab7]
+  - @empiricalrun/llm@0.6.0
+
+## 0.21.11
+
+### Patch Changes
+
+- e2b7a29: fix: codegen adding extra changes than the requested task
+- 29cad4a: feat: use vercel kv
+
+## 0.21.10
+
+### Patch Changes
+
+- e077e2a: fix: verification agent should use model configured from test token
+
 ## 0.21.9
 
 ### Patch Changes

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AASvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAED,wBAAsB,2BAA2B,CAAC,SAAS,EAAE,aAAa,iBAqCzE;AAiBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBASxD;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,QA4BnD;AAED,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,CAAC,CAkDjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAUvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAED,wBAAsB,2BAA2B,CAAC,SAAS,EAAE,aAAa,iBAuCzE;AAiBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBASxD;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,QA4BnD;AAED,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,CAAC,CAkDjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA"}

package/dist/agent/browsing/utils.js

@@ -28,7 +28,8 @@ async function prepareFileForBrowsingAgent(genConfig) {
     const logger = new logger_1.CustomLogger();
     if (!fs_extra_1.default.existsSync(specPath)) {
         await fs_extra_1.default.createFile(specPath);
-        await fs_extra_1.default.writeFile(specPath, (0, web_1.addNewImport)("", ["test", "expect"], "@playwright/test"));
+        const fileContentWithImports = (0, web_1.addNewImport)("", ["test", "expect"], (0, web_1.getFixtureImportPath)(specPath));
+        await fs_extra_1.default.writeFile(specPath, fileContentWithImports, "utf-8");
     }
     if (name && steps && steps.length) {
         const existingContents = await fs_extra_1.default.readFile(specPath, "utf-8");

package/dist/agent/codegen/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAgJrB"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/run.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAkJrB"}

package/dist/agent/codegen/run.js

@@ -67,7 +67,7 @@ async function generateTest(testCase, file, options) {
     logger.success("Test generated successfully!");
     const readWriteFileSpan = trace.span({ name: "write-to-file" });
     let contents = fs_extra_1.default.readFileSync(file, "utf-8");
-    const [prependContent, strippedContent] = await (0, web_1.stripAndPrependImports)(response);
+    const [prependContent, strippedContent] = await (0, web_1.stripAndPrependImports)(response, testCase?.name);
     let updatedContent = prependContent + contents + `\n\n${strippedContent}`;
     if (isUpdate) {
         const testBlock = (0, web_1.getTypescriptTestBlock)(testCase?.name, contents);

package/dist/agent/verification/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/verification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,GAAG,EACH,KAAK,EACL,IAAI,EACJ,YAAY,GACb,EAAE;IACD,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd;;;GA8DA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/verification/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,GAAG,EACH,KAAK,EACL,IAAI,EACJ,YAAY,GACb,EAAE;IACD,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd;;;GA6DA"}

package/dist/agent/verification/index.js

@@ -48,7 +48,6 @@ ${conversation.join("\n")}
                 },
             },
         ],
-        model: "gpt-4o",
         modelParameters: {
             tool_choice: "required",
         },

package/dist/bin/utils/platform/web/index.d.ts

@@ -1,9 +1,10 @@
 export declare function getTypescriptTestBlock(scenarioName: string, content: string): string | undefined;
 export declare function appendToTestBlock(testBlock: string, content: string): string;
 export declare function validateTypescript(filePath: string): string[];
-export declare function stripAndPrependImports(content: string): Promise<string[]>;
+export declare function stripAndPrependImports(content: string, testName: string): Promise<(string | undefined)[]>;
 export declare function lintErrors(filePath: string): Promise<void>;
 export declare function formatCode(filePath: string): Promise<void>;
 export declare function addNewImport(contents: string, modules: string[], pkg: string): string;
 export declare function removeTestOnly(filePath: string): Promise<void>;
+export declare function getFixtureImportPath(filePath: string): string;
 //# sourceMappingURL=index.d.ts.map

package/dist/bin/utils/platform/web/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAMA,wBAAgB,sBAAsB,CACpC,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,GACd,MAAM,GAAG,SAAS,CAYpB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAmD7D;AAED,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,MAAM,qBAM3D;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAIpD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAMA,wBAAgB,sBAAsB,CACpC,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,GACd,MAAM,GAAG,SAAS,CAYpB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAwD7D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,mCAOjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAIpD;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,UAcpD"}

package/dist/bin/utils/platform/web/index.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.removeTestOnly = exports.addNewImport = exports.formatCode = exports.lintErrors = exports.stripAndPrependImports = exports.validateTypescript = exports.appendToTestBlock = exports.getTypescriptTestBlock = void 0;
+exports.getFixtureImportPath = exports.removeTestOnly = exports.addNewImport = exports.formatCode = exports.lintErrors = exports.stripAndPrependImports = exports.validateTypescript = exports.appendToTestBlock = exports.getTypescriptTestBlock = void 0;
 const eslint_1 = require("eslint");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const prettier_1 = __importDefault(require("prettier"));
@@ -25,12 +25,13 @@ function appendToTestBlock(testBlock, content) {
 exports.appendToTestBlock = appendToTestBlock;
 function validateTypescript(filePath) {
     // Create a compiler host to read files
-    const compilerHost = typescript_1.default.createCompilerHost({
-        esModuleInterop: true,
-    });
+    const compilerHost = typescript_1.default.createCompilerHost({});
     compilerHost.readFile = (file) => fs_extra_1.default.readFileSync(file, "utf8");
     // Create a program with a single source file
-    const program = typescript_1.default.createProgram([filePath], {}, compilerHost);
+    const program = typescript_1.default.createProgram([filePath], {
+        esModuleInterop: true,
+        allowSyntheticDefaultImports: true,
+    }, compilerHost);
     // Get the source file
     const sourceFile = program.getSourceFile(filePath);
     // Get and report any syntactic errors
@@ -73,10 +74,10 @@ function validateTypescript(filePath) {
     return errors;
 }
 exports.validateTypescript = validateTypescript;
-async function stripAndPrependImports(content) {
+async function stripAndPrependImports(content, testName) {
     const importRegexp = /import\s+\{[^}]*\}\s+from\s+["'][^"']+["'];?/g;
     const imports = content.match(importRegexp);
-    const strippedContent = content.replace(importRegexp, "");
+    const strippedContent = getTypescriptTestBlock(testName, content);
     const prependContent = (imports?.join("\n") || "") + "\n\n";
     return [prependContent, strippedContent];
 }
@@ -112,3 +113,19 @@ async function removeTestOnly(filePath) {
     return fs_extra_1.default.writeFile(filePath, updatedContent);
 }
 exports.removeTestOnly = removeTestOnly;
+function getFixtureImportPath(filePath) {
+    // get the depth from the tests folder
+    // assumption fixtures are always int the folder tests/fixtures
+    const subFolder = filePath.split("/tests/")[1];
+    const subFolderDepth = subFolder?.split("/").length
+        ? subFolder?.split("/").length - 1
+        : 0;
+    // create an array of path segments to the fixtures file
+    const fixturesPath = [
+        "./",
+        subFolderDepth ? new Array(subFolderDepth).fill("../").join("") : "",
+        "fixtures",
+    ].join("");
+    return fixturesPath;
+}
+exports.getFixtureImportPath = getFixtureImportPath;

package/dist/bin/utils/platform/web/test-files/ts-path-import-validate.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=ts-path-import-validate.d.ts.map

package/dist/bin/utils/platform/web/test-files/ts-path-import-validate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ts-path-import-validate.d.ts","sourceRoot":"","sources":["../../../../../../src/bin/utils/platform/web/test-files/ts-path-import-validate.ts"],"names":[],"mappings":""}

package/dist/bin/utils/platform/web/test-files/ts-path-import-validate.js

@@ -0,0 +1,7 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const path_1 = __importDefault(require("path"));
+console.log(path_1.default);

package/dist/constants/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/constants/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAE3E,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAItD,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,WAAsB,CAAC;AAE5D,eAAO,MAAM,aAAa,EAAE,QAAmB,CAAC;AAEhD,eAAO,MAAM,wBAAwB,EAAE,eAEtC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/constants/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAE3E,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAItD,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,WAAsB,CAAC;AAE5D,eAAO,MAAM,aAAa,EAAE,QAA8B,CAAC;AAE3D,eAAO,MAAM,wBAAwB,EAAE,eAEtC,CAAC"}

package/dist/constants/index.js

@@ -7,7 +7,7 @@ exports.MODEL_API_KEYS = {
     openai: process.env.OPENAI_API_KEY || "",
 };
 exports.DEFAULT_MODEL_PROVIDER = "openai";
-exports.DEFAULT_MODEL = "gpt-4o";
+exports.DEFAULT_MODEL = "gpt-4o-2024-08-06";
 exports.DEFAULT_MODEL_PARAMETERS = {
     temperature: 0.5,
 };

package/dist/file/server.js

@@ -27,7 +27,7 @@ class FileService {
                 if (testFilePath) {
                     const testFile = fs_1.default.readFileSync(testFilePath, "utf-8");
                     const updatedTestFile = testFile.replace(/await createTest\([\s\S]*?\);\n/, "\n" + generatedCode);
-                    const importStatement = `import { test, expect } from "@playwright/test";`;
+                    const importStatement = `import { test, expect } from "${(0, web_1.getFixtureImportPath)(testFilePath)}";`;
                     fs_1.default.writeFileSync(testFilePath, importStatement + "\n" + updatedTestFile, "utf-8");
                     await (0, web_1.lintErrors)(testFilePath);
                     return res.send({ success: true });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.21.9",
+  "version": "0.22.2",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -41,7 +41,7 @@
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
     "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.5.0",
+    "@empiricalrun/llm": "^0.7.0",
     "@empiricalrun/reporter": "^0.16.1"
   },
   "devDependencies": {