@empiricalrun/test-gen 0.1.1

package/CHANGELOG.md

@@ -0,0 +1,13 @@
+# @empiricalrun/test-gen
+
+## 0.1.1
+
+### Patch Changes
+
+- c75a102: chore: add publish package support
+
+## 0.1.0
+
+### Minor Changes
+
+- 20c6004: feat: add support for package publish

package/README.md

@@ -0,0 +1,24 @@
+# test-gen
+
+Package to generate e2e Playwright tests.
+
+## Usages
+
+```
+npx @empiricalrun/test-gen ./scenarios/create-flow.yaml
+```
+
+## Sample test scenarios
+
+```yaml ./tests/scenarios.yaml
+dir: ./tests/ # [optional] directory where you want to create tests. Default - ./tests
+scenarios: # list of scenarios
+    - name: user should be able to login successfully # name of the test case to be generated
+      steps: # steps for the test case
+        - cLick on "Login" button
+        - click on input "username"
+        - fill username as "foo@bar.com"
+        - fill password as "bar@123"
+        - click on "Login Button"
+      assert: "Login successful" text should be visible # assertion
+```

package/dist/bin/index.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/bin/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/bin/index.ts"],"names":[],"mappings":""}

package/dist/bin/index.js

@@ -0,0 +1,264 @@
+#!/usr/bin/env node
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const openai_1 = __importDefault(require("openai"));
+const path_1 = __importDefault(require("path"));
+// const { scenarios } = require("./scenarios");
+const typescript_1 = __importDefault(require("typescript"));
+const prettier_1 = __importDefault(require("prettier"));
+const eslint_1 = require("eslint");
+const dotenv_1 = __importDefault(require("dotenv"));
+const yaml_1 = require("yaml");
+dotenv_1.default.config({
+    path: [".env.local", ".env"],
+});
+async function readFilesInDirectory(dir = "") {
+    let files = [];
+    const items = await fs_extra_1.default.readdir(dir);
+    for (const item of items) {
+        const fullPath = path_1.default.join(dir, item);
+        const stat = await fs_extra_1.default.stat(fullPath);
+        if (stat.isDirectory()) {
+            files = files.concat(await readFilesInDirectory(fullPath));
+        }
+        else if (stat.isFile()) {
+            const content = await fs_extra_1.default.readFile(fullPath, "utf-8");
+            files.push({ filePath: fullPath, content });
+        }
+    }
+    return files;
+}
+function mergeFilesToPrompt(files = []) {
+    let prompt = "";
+    files.forEach((file) => {
+        prompt += `File Path: ${file.filePath}\n`;
+        prompt += `File:\n`;
+        prompt += `${file.content}\n\n ------ \n\n`;
+    });
+    return prompt;
+}
+async function generatePromptFromDirectory(dir = "") {
+    try {
+        const files = await readFilesInDirectory(dir);
+        const prompt = mergeFilesToPrompt(files);
+        return prompt;
+    }
+    catch (error) {
+        console.error("Error reading directory:", error);
+    }
+}
+async function getLLMResult(instruction) {
+    const openai = new openai_1.default();
+    const completion = await openai.chat.completions.create({
+        messages: [
+            {
+                role: "user",
+                content: [
+                    {
+                        type: "text",
+                        text: instruction,
+                    },
+                ],
+            },
+        ],
+        model: "gpt-4o",
+    });
+    console.log("generated completion");
+    const response = completion.choices[0]?.message.content || "";
+    return response;
+}
+function validateTypescript(filePath) {
+    // Create a compiler host to read files
+    const compilerHost = typescript_1.default.createCompilerHost({});
+    compilerHost.readFile = (file) => fs_extra_1.default.readFileSync(file, "utf8");
+    // Create a program with a single source file
+    const program = typescript_1.default.createProgram([filePath], {}, compilerHost);
+    // Get the source file
+    const sourceFile = program.getSourceFile(filePath);
+    // Get and report any syntactic errors
+    const errors = [];
+    const syntacticDiagnostics = program.getSyntacticDiagnostics(sourceFile);
+    if (syntacticDiagnostics.length > 0) {
+        console.log("Syntactic errors:");
+        syntacticDiagnostics.forEach((diagnostic) => {
+            const { line, character } = diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start);
+            const message = typescript_1.default.flattenDiagnosticMessageText(diagnostic.messageText, "\n");
+            console.log(`${diagnostic.file.fileName} (${line + 1},${character + 1}): ${message}`);
+            if (typeof diagnostic.messageText === "string") {
+                errors.push(diagnostic.messageText);
+            }
+        });
+    }
+    else {
+        console.log("No syntactic errors.");
+    }
+    // Get and report any semantic errors
+    const semanticDiagnostics = program.getSemanticDiagnostics(sourceFile);
+    if (semanticDiagnostics.length > 0) {
+        console.log("Semantic errors:");
+        semanticDiagnostics.forEach((diagnostic) => {
+            console.log(diagnostic.messageText);
+            if (typeof diagnostic.messageText == "string") {
+                errors.push(diagnostic.messageText);
+            }
+        });
+    }
+    else {
+        console.log("No semantic errors.");
+    }
+    return errors;
+}
+async function formatCode(filePath) {
+    const fileContent = fs_extra_1.default.readFileSync(filePath, "utf8");
+    const prettierConfig = {};
+    const formattedContent = await prettier_1.default.format(fileContent, {
+        ...prettierConfig,
+        filepath: filePath,
+    });
+    fs_extra_1.default.writeFileSync(filePath, formattedContent);
+    console.log("File formatted successfully.");
+}
+async function stripAndPrependImports(content) {
+    const imports = content.match(/import.* from.*;/g);
+    const strippedContent = content.replace(/import.* from.*;/g, "");
+    const prependContent = (imports?.join("\n") || "") + "\n\n";
+    return [prependContent, strippedContent];
+}
+async function lintErrors(filePath) {
+    const eslint = new eslint_1.ESLint({
+        fix: true,
+        useEslintrc: true,
+    });
+    const [result] = await eslint.lintFiles(filePath);
+    fs_extra_1.default.writeFileSync(filePath, result?.output || "");
+}
+async function generateTest(scenarios, file) {
+    const codePrompt = await generatePromptFromDirectory("./tests");
+    const pomPrompt = await generatePromptFromDirectory("./pages");
+    const testFileContent = fs_extra_1.default.readFileSync(file, "utf-8");
+    for (const i in scenarios) {
+        const scenario = scenarios[i];
+        console.log("Generating test for scenario: ", scenario?.name);
+        //TODO: improve this logic. its buggy
+        if (testFileContent.includes(`test("${scenario?.name}"`)) {
+            console.log("test already exists for this scenario", scenario?.name);
+            console.log("skipping test generation");
+            continue;
+        }
+        const instruction = `
+        You are a software test engineer who is given a task to create test basis a scenario provided to you.
+        You will be provided with current tests, fixtures and page object models for you to use and write test.
+        You need to respond with only with the test case in typescript language using playwright.
+
+        Here is the list of current tests and fixtures:
+
+        ${codePrompt}
+
+
+        Here is the list of current page object models:
+
+        ${pomPrompt}
+
+
+        Following is the test scenario for which you need to write test:
+          name: ${scenario?.name}
+          Steps:
+          ${scenario?.steps?.join("\n")}
+
+          Assert:
+          ${scenario?.assert}
+
+
+        The code needs to be written inside ${file} file. Write the tests accordingly.
+
+        Follow these guidelines before responding with output
+        - Ensure there are no type issues in the code generated
+        - For the given spec file respond with only the test case for the given scenario
+        - Donot respond with markdown syntax or backticks
+        - Respond only with the code
+        - Donot repeat steps which are already mentioned in the "test.beforeEach" block
+        - Add import statements at the beginning of the output.
+        `;
+        console.log("constructed instruction for llm");
+        let response = await getLLMResult(instruction);
+        const contents = fs_extra_1.default.readFileSync(file, "utf-8");
+        const [prependContent, strippedContent] = await stripAndPrependImports(response);
+        await fs_extra_1.default.writeFile(file, prependContent + contents + `\n\n${strippedContent}`, "utf-8");
+        console.log("Linting generated output");
+        await lintErrors(file);
+        console.log("Validating types");
+        let errors = validateTypescript(file);
+        const maxIteration = 2;
+        let counter = 0;
+        while (errors.length > 0) {
+            console.log(errors);
+            const fileContent = fs_extra_1.default.readFileSync(file, "utf-8");
+            counter += 1;
+            if (counter > maxIteration) {
+                console.log(`Max iteration limit reached. Please review the file ${file} and manually fix the typescript errors.`);
+                break;
+            }
+            const instruction = `
+        You are a software engineer who is given a task to create test basis a scenario provided to you.
+        You will be provided with current tests, fixtures and page object models for you to use and write test.
+        There are following typescript errors in ${file} which you need to fix.
+
+        Here is the list of current tests and fixtures:
+
+        ${codePrompt}
+
+        Here is the list of current page object models:
+
+        ${pomPrompt}
+
+        Here are the typescript errors:
+        ${errors.join("\n")}
+
+        Here is the content of the file ${file}:
+
+        ${fileContent}
+
+        Follow following guidelines before responding with output
+        - Ensure there are no type issues in the file ${file}
+        - For the given file respond with only the code
+        - Donot respond with markdown syntax or backticks
+        - Respond only with the code
+        - Donot modify anything else apart from the code required to fix typescript error
+        `;
+            response = await getLLMResult(instruction);
+            console.log("fixed");
+            console.log(response);
+            errors = validateTypescript(file);
+            await fs_extra_1.default.writeFile(file, response, "utf-8");
+        }
+        console.log("formatting spec file");
+        await formatCode(file);
+    }
+}
+(async function main() {
+    if (process.argv.length != 3) {
+        console.log("Provide a path to scenarios path to generate test");
+        process.exit(1);
+    }
+    else {
+        const scenariosPath = process.argv[2];
+        const file = fs_extra_1.default.readFileSync(path_1.default.resolve(process.cwd(), scenariosPath), 'utf8');
+        const fileName = scenariosPath.split('/').pop()?.split(".")[0];
+        const config = (0, yaml_1.parse)(file);
+        const scenarios = config.scenarios;
+        const fileDir = config.dir || "./tests";
+        const filePath = `${fileDir}/${fileName}.spec.ts`;
+        if (fs_extra_1.default.existsSync(filePath)) {
+            console.log("Spec file already exists, skipping generation");
+        }
+        else {
+            console.log("Spec file does not exist, creating new spec file");
+            fs_extra_1.default.createFileSync(filePath);
+        }
+        await generateTest(scenarios, filePath);
+    }
+})();

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@empiricalrun/test-gen",
+  "version": "0.1.1",
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org/",
+    "access": "public"
+  },
+  "bin": {
+    "@empiricalrun/test-gen": "dist/bin/index.js"
+  },
+  "main": "dist/index.js",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/empirical-run/empirical.git"
+  },
+  "author": "Empirical Team <hey@empirical.run>",
+  "dependencies": {
+    "dotenv": "^16.4.5",
+    "fs-extra": "^11.2.0",
+    "openai": "^4.47.2",
+    "yaml": "^2.4.2"
+  },
+  "devDependencies": {
+    "@types/fs-extra": "^11.0.4"
+  },
+  "scripts": {
+    "dev": "tsc --build --watch",
+    "build": "tsc --build",
+    "clean": "tsc --build --clean",
+    "lint": "eslint .",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  }
+}