@empiricalrun/test-gen 0.4.2 → 0.6.0

package/CHANGELOG.md

@@ -1,5 +1,28 @@
 # @empiricalrun/test-gen
 
+## 0.6.0
+
+### Minor Changes
+
+- 3e0e6d1: feat: add support for json string support for adding / updating tests using test-gen
+
+## 0.5.1
+
+### Patch Changes
+
+- fb09f23: fix: build action in gh
+
+## 0.5.0
+
+### Minor Changes
+
+- c09db7e: feat: add observability
+
+### Patch Changes
+
+- 77dec1d: feat: add support for editing a scenario
+- 453c8f6: fix: ts feedbacks getting applied to other scenarios
+
 ## 0.4.2
 
 ### Patch Changes

package/dist/bin/ai/index.d.ts

@@ -0,0 +1,4 @@
+import OpenAI from "openai";
+import LLMTracing from "./trace";
+export declare function getLLMResult(messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[], trace?: LLMTracing): Promise<string>;
+//# sourceMappingURL=index.d.ts.map

package/dist/bin/ai/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/ai/index.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,UAAU,MAAM,SAAS,CAAC;AAEjC,wBAAsB,YAAY,CAChC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,EAC9D,KAAK,CAAC,EAAE,UAAU,GACjB,OAAO,CAAC,MAAM,CAAC,CAoBjB"}

package/dist/bin/ai/index.js

@@ -0,0 +1,29 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getLLMResult = void 0;
+const openai_1 = __importDefault(require("openai"));
+async function getLLMResult(messages, trace) {
+    const openai = new openai_1.default();
+    const model = "gpt-4o";
+    const parameters = {
+        temperature: 0.5,
+    };
+    const generation = trace?.startGeneration({
+        name: "get-llm-result",
+        model,
+        parameters,
+        input: messages,
+    });
+    const completion = await openai.chat.completions.create({
+        messages,
+        model,
+        ...parameters,
+    });
+    const output = completion.choices[0]?.message.content || "";
+    generation?.end({ output });
+    return output;
+}
+exports.getLLMResult = getLLMResult;

package/dist/bin/ai/prompts/provider/index.d.ts

@@ -0,0 +1,3 @@
+import OpenAI from "openai";
+export declare function getPrompt(name: string, vars: any): Promise<OpenAI.Chat.Completions.ChatCompletionMessageParam[]>;
+//# sourceMappingURL=index.d.ts.map

package/dist/bin/ai/prompts/provider/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/ai/prompts/provider/index.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAS5B,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,GAAG,GACR,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,CAAC,CAI/D"}

package/dist/bin/ai/prompts/provider/index.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getPrompt = void 0;
+const langfuse_1 = require("langfuse");
+const langfuse = new langfuse_1.Langfuse({
+    secretKey: "sk-lf-30eee545-9c2c-4646-bfd9-8f38acf2fa8f",
+    publicKey: "pk-lf-c5c7cf23-de76-4098-ad81-594bce7cf330",
+    baseUrl: "https://us.cloud.langfuse.com",
+    flushAt: 1,
+});
+async function getPrompt(name, vars) {
+    const prompt = await langfuse.getPrompt(name, undefined, { type: "chat" });
+    const compiledPrompt = prompt.compile(vars);
+    return compiledPrompt;
+}
+exports.getPrompt = getPrompt;

package/dist/bin/ai/trace/index.d.ts

@@ -0,0 +1,36 @@
+import OpenAI from "openai";
+export default class LLMTracing {
+    private trace;
+    constructor({ name }?: {
+        name?: string;
+    });
+    get id(): string;
+    event({ name, input, metadata, output, }: {
+        name: string;
+        input?: any;
+        metadata?: any;
+        output?: any;
+    }): void;
+    update({ input, output, metadata, }: {
+        input?: any;
+        output?: any;
+        metadata?: any;
+    }): void;
+    startSpan(name: string, input?: Record<string, any>, metadata?: Record<string, any>): {
+        end: ({ output }: {
+            output: Record<string, any>;
+        }) => void;
+    };
+    startGeneration({ name, model, parameters, input, }: {
+        name: string;
+        model: string;
+        parameters: Record<string, any>;
+        input: any;
+    }): {
+        end: ({ output, usage, }: {
+            output: string;
+            usage?: OpenAI.Completions.CompletionUsage | undefined;
+        }) => void;
+    };
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/bin/ai/trace/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/bin/ai/trace/index.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAe5B,MAAM,CAAC,OAAO,OAAO,UAAU;IAC7B,OAAO,CAAC,KAAK,CAAsB;gBACvB,EAAE,IAAuB,EAAE,GAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAO;IAS/D,IAAI,EAAE,WAEL;IAED,KAAK,CAAC,EACJ,IAAI,EACJ,KAAK,EACL,QAAQ,EACR,MAAM,GACP,EAAE;QACD,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,CAAC,EAAE,GAAG,CAAC;QACZ,QAAQ,CAAC,EAAE,GAAG,CAAC;QACf,MAAM,CAAC,EAAE,GAAG,CAAC;KACd;IAID,MAAM,CAAC,EACL,KAAU,EACV,MAAW,EACX,QAAa,GACd,EAAE;QACD,KAAK,CAAC,EAAE,GAAG,CAAC;QACZ,MAAM,CAAC,EAAE,GAAG,CAAC;QACb,QAAQ,CAAC,EAAE,GAAG,CAAC;KAChB;IAID,SAAS,CACP,IAAI,EAAE,MAAM,EACZ,KAAK,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,EAC/B,QAAQ,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM;;oBAIJ,OAAO,MAAM,EAAE,GAAG,CAAC;;;IAMnD,eAAe,CAAC,EACd,IAAS,EACT,KAAU,EACV,UAAe,EACf,KAAU,GACX,EAAE;QACD,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAChC,KAAK,EAAE,GAAG,CAAC;KACZ;;oBAYa,MAAM;;cAEZ,IAAI;;CAcb"}

package/dist/bin/ai/trace/index.js

@@ -0,0 +1,64 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const langfuse_1 = require("langfuse");
+const session_1 = require("../../session");
+const langfuse = new langfuse_1.Langfuse({
+    secretKey: "sk-lf-30eee545-9c2c-4646-bfd9-8f38acf2fa8f",
+    publicKey: "pk-lf-c5c7cf23-de76-4098-ad81-594bce7cf330",
+    baseUrl: "https://us.cloud.langfuse.com",
+    flushAt: 1,
+});
+process.on("exit", () => langfuse.flush());
+process.on("SIGINT", () => langfuse.flush());
+process.on("SIGTERM", async () => langfuse.flush());
+class LLMTracing {
+    trace;
+    constructor({ name = "test-generator" } = {}) {
+        const sessionDetails = (0, session_1.getSessionDetails)();
+        this.trace = langfuse.trace({
+            name,
+            sessionId: sessionDetails.id,
+            release: sessionDetails.version,
+        });
+    }
+    get id() {
+        return this.trace.traceId;
+    }
+    event({ name, input, metadata, output, }) {
+        this.trace.event({ name, input, metadata, output });
+    }
+    update({ input = {}, output = {}, metadata = {}, }) {
+        this.trace.update({ input, output, metadata });
+    }
+    startSpan(name, input = {}, metadata = {}) {
+        const span = this.trace.span({ name, metadata, input });
+        return {
+            end: ({ output }) => {
+                span.end({ output });
+            },
+        };
+    }
+    startGeneration({ name = "", model = "", parameters = {}, input = {}, }) {
+        const generation = this.trace.generation({
+            name,
+            model,
+            modelParameters: parameters,
+            input,
+        });
+        return {
+            end: ({ output, usage, }) => {
+                generation.end({
+                    output,
+                    usage: usage
+                        ? {
+                            promptTokens: usage.prompt_tokens,
+                            completionTokens: usage.completion_tokens,
+                            totalTokens: usage.total_tokens,
+                        }
+                        : undefined,
+                });
+            },
+        };
+    }
+}
+exports.default = LLMTracing;

package/dist/bin/index.js

@@ -4,252 +4,152 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const fs_extra_1 = __importDefault(require("fs-extra"));
-const openai_1 = __importDefault(require("openai"));
-const path_1 = __importDefault(require("path"));
-const typescript_1 = __importDefault(require("typescript"));
-const prettier_1 = __importDefault(require("prettier"));
-const eslint_1 = require("eslint");
 const dotenv_1 = __importDefault(require("dotenv"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const ai_1 = require("./ai");
+const provider_1 = require("./ai/prompts/provider");
+const trace_1 = __importDefault(require("./ai/trace"));
 const logger_1 = require("./logger");
-const scenarios_1 = require("./scenarios");
 const ci_1 = require("./reporter/ci");
+const scenarios_1 = require("./scenarios");
+const fs_1 = require("./utils/fs");
+const web_1 = require("./utils/platform/web");
 dotenv_1.default.config({
     path: [".env.local", ".env"],
 });
 const logger = new logger_1.CustomLogger();
-async function readFilesInDirectory(dir = "") {
-    let files = [];
-    const items = await fs_extra_1.default.readdir(dir);
-    for (const item of items) {
-        const fullPath = path_1.default.join(dir, item);
-        const stat = await fs_extra_1.default.stat(fullPath);
-        if (stat.isDirectory()) {
-            files = files.concat(await readFilesInDirectory(fullPath));
-        }
-        else if (stat.isFile()) {
-            const content = await fs_extra_1.default.readFile(fullPath, "utf-8");
-            files.push({ filePath: fullPath, content });
-        }
-    }
-    return files;
-}
-function mergeFilesToPrompt(files = []) {
-    let prompt = "";
-    files.forEach((file) => {
-        prompt += `File Path: ${file.filePath}\n`;
-        prompt += `File:\n`;
-        prompt += `${file.content}\n\n ------ \n\n`;
-    });
-    return prompt;
-}
-async function generatePromptFromDirectory(dir = "") {
-    try {
-        const files = await readFilesInDirectory(dir);
-        const prompt = mergeFilesToPrompt(files);
-        return prompt;
-    }
-    catch (error) {
-        console.error("Error reading directory:", error);
-    }
-}
-async function getLLMResult(instruction) {
-    const openai = new openai_1.default();
-    const completion = await openai.chat.completions.create({
-        messages: [
-            {
-                role: "user",
-                content: [
-                    {
-                        type: "text",
-                        text: instruction,
-                    },
-                ],
-            },
-        ],
-        model: "gpt-4o",
-    });
-    const response = completion.choices[0]?.message.content || "";
-    return response;
-}
-function validateTypescript(filePath) {
-    // Create a compiler host to read files
-    const compilerHost = typescript_1.default.createCompilerHost({});
-    compilerHost.readFile = (file) => fs_extra_1.default.readFileSync(file, "utf8");
-    // Create a program with a single source file
-    const program = typescript_1.default.createProgram([filePath], {}, compilerHost);
-    // Get the source file
-    const sourceFile = program.getSourceFile(filePath);
-    // Get and report any syntactic errors
-    const errors = [];
-    const syntacticDiagnostics = program.getSyntacticDiagnostics(sourceFile);
-    if (syntacticDiagnostics.length > 0) {
-        syntacticDiagnostics.forEach((diagnostic) => {
-            const { line, character } = diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start);
-            const message = typescript_1.default.flattenDiagnosticMessageText(diagnostic.messageText, "\n");
-            logger.log(`${diagnostic.file.fileName} (${line + 1},${character + 1}): ${message}`);
-            if (typeof diagnostic.messageText === "string") {
-                errors.push(diagnostic.messageText);
-            }
-        });
-    }
-    // Get and report any semantic errors
-    const semanticDiagnostics = program.getSemanticDiagnostics(sourceFile);
-    if (semanticDiagnostics.length > 0) {
-        semanticDiagnostics.forEach((diagnostic) => {
-            errors.push(diagnostic.messageText.toString());
-        });
-    }
-    if (!errors.length) {
-        logger.success("Found no type issues!");
-    }
-    return errors;
-}
-async function formatCode(filePath) {
-    const fileContent = fs_extra_1.default.readFileSync(filePath, "utf8");
-    const prettierConfig = {};
-    const formattedContent = await prettier_1.default.format(fileContent, {
-        ...prettierConfig,
-        filepath: filePath,
-    });
-    fs_extra_1.default.writeFileSync(filePath, formattedContent);
-    logger.success("File formatted successfully!");
-}
-async function stripAndPrependImports(content) {
-    const imports = content.match(/import.* from.*;/g);
-    const strippedContent = content.replace(/import.* from.*;/g, "");
-    const prependContent = (imports?.join("\n") || "") + "\n\n";
-    return [prependContent, strippedContent];
-}
-async function lintErrors(filePath) {
-    const eslint = new eslint_1.ESLint({
-        fix: true,
-        useEslintrc: true,
-    });
-    const [result] = await eslint.lintFiles(filePath);
-    fs_extra_1.default.writeFileSync(filePath, result?.output || "");
-}
-async function generateTest(scenarios, file) {
-    const codePrompt = await generatePromptFromDirectory("./tests");
-    const pomPrompt = await generatePromptFromDirectory("./pages");
+async function generateTest(scenarios, file, isUpdate) {
+    const codePrompt = await (0, fs_1.generatePromptFromDirectory)("./tests");
+    const pomPrompt = await (0, fs_1.generatePromptFromDirectory)("./pages");
     const testFileContent = fs_extra_1.default.readFileSync(file, "utf-8");
     const generatedScenarios = [];
     for (const i in scenarios) {
-        console.log("\n\n");
+        logger.logEmptyLine();
+        const trace = new trace_1.default({ name: "generate-test" });
+        trace.event({
+            name: "collate-files-as-text",
+            output: {
+                codePrompt,
+                pomPrompt,
+                testFileContent,
+            },
+        });
         const scenario = scenarios[i];
+        trace.update({ input: { scenario } });
         logger.log("Generating test for scenario:", scenario?.name);
-        //TODO: improve this logic. its buggy
-        if (testFileContent.includes(`test("${scenario?.name}"`)) {
+        if (!isUpdate && testFileContent.includes(`test("${scenario?.name}"`)) {
             logger.success("Test already exists for this scenario");
+            trace.event({ name: "test-already-exists" });
             continue;
         }
-        const instruction = `
-        You are a software test engineer who is given a task to create test basis a scenario provided to you.
-        You will be provided with current tests, fixtures and page object models for you to use and write test.
-        You need to respond with only with the test case in typescript language using playwright.
-
-        Here is the list of current tests and fixtures:
-
-        ${codePrompt}
-
-
-        Here is the list of current page object models:
-
-        ${pomPrompt}
-
-
-        Following is the test scenario for which you need to write test:
-          name: ${scenario?.name}
-          Steps:
-          ${scenario?.steps?.join("\n")}
-
-          Assert:
-          ${scenario?.assert}
-
-
-        The code needs to be written inside ${file} file. Write the tests accordingly.
-
-        Follow these guidelines before responding with output
-        - Ensure there are no type issues in the code generated
-        - For the given spec file respond with only the test case for the given scenario
-        - Donot respond with markdown syntax or backticks
-        - Respond only with the code
-        - Donot repeat steps which are already mentioned in the "test.beforeEach" block
-        - Add import statements at the beginning of the output.
-        `;
-        let response = await getLLMResult(instruction);
+        const promptSpan = trace.startSpan(isUpdate ? "update-scenario-prompt" : "add-scenario-prompt");
+        const promptName = isUpdate ? "update-scenario" : "add-scenario";
+        const instruction = await (0, provider_1.getPrompt)(promptName, {
+            testFiles: codePrompt,
+            pageFiles: pomPrompt,
+            scenarioName: scenario.name,
+            scenarioSteps: scenario.steps.join("\n"),
+            assert: scenario.assert,
+            scenarioFile: file,
+        });
+        promptSpan.end({ output: { instruction } });
+        let response = await (0, ai_1.getLLMResult)(instruction, trace);
         logger.success("Test generated successfully!");
-        const contents = fs_extra_1.default.readFileSync(file, "utf-8");
-        const [prependContent, strippedContent] = await stripAndPrependImports(response);
-        await fs_extra_1.default.writeFile(file, prependContent + contents + `\n\n${strippedContent}`, "utf-8");
+        const readWriteFileSpan = trace.startSpan("write-to-file");
+        let contents = fs_extra_1.default.readFileSync(file, "utf-8");
+        const [prependContent, strippedContent] = await (0, web_1.stripAndPrependImports)(response);
+        let updatedContent = prependContent + contents + `\n\n${strippedContent}`;
+        if (isUpdate) {
+            const testBlock = (0, web_1.getTypescriptTestBlock)(scenario?.name, contents);
+            contents = contents.replace(testBlock[0], `\n\n${strippedContent}`);
+            updatedContent = prependContent + contents;
+        }
+        await fs_extra_1.default.writeFile(file, updatedContent, "utf-8");
+        readWriteFileSpan.end({ output: { updatedContent } });
         logger.log("Linting generated code...");
-        await lintErrors(file);
+        trace.event({ name: "lint-file" });
+        await (0, web_1.lintErrors)(file);
+        const validateTypesSpan = trace.startSpan("detect-type-errors-in-file");
         logger.log("Validating types...");
-        let errors = validateTypescript(file);
+        let errors = (0, web_1.validateTypescript)(file);
+        validateTypesSpan.end({ output: { errors } });
+        if (!errors.length) {
+            logger.success("Found no type issues!");
+        }
         const maxIteration = 2;
         let counter = 0;
         while (errors.length > 0) {
             const fileContent = fs_extra_1.default.readFileSync(file, "utf-8");
             counter += 1;
             if (counter > maxIteration) {
-                logger.error(`Unable to fix typescript errors. Please review ${file} manually and fix the typescript errors. Run the test-gen command again, once errors are fixed`);
+                trace.event({ name: "code-fix-iteration-max-out" });
+                logger.error(`Unable to fix typescript errors. Please review ${file} manually and fix the typescript errors. Run the test-gen command again, once errors are fixed. [Trace: ${trace.id}]`);
                 break;
             }
+            trace.event({ name: "Found errors fixing" });
             logger.warn("Found few errors while validating types:");
-            errors.forEach(e => logger.warn(e));
+            errors.forEach((e) => logger.warn(e));
             logger.log("Trying to fix above errors...");
-            const instruction = `
-        You are a software engineer who is given a task to create test basis a scenario provided to you.
-        You will be provided with current tests, fixtures and page object models for you to use and write test.
-        There are following typescript errors in ${file} which you need to fix.
-
-        Here is the list of current tests and fixtures:
-
-        ${codePrompt}
-
-        Here is the list of current page object models:
-
-        ${pomPrompt}
-
-        Here are the typescript errors:
-        ${errors.join("\n")}
-
-        Here is the content of the file ${file}:
-
-        ${fileContent}
-
-        Follow following guidelines before responding with output
-        - Ensure there are no type issues in the file ${file}
-        - For the given file respond with only the code
-        - Donot respond with markdown syntax or backticks
-        - Respond only with the code
-        - Donot modify anything else apart from the code required to fix typescript error
-        `;
-            response = await getLLMResult(instruction);
+            const promptSpan = trace.startSpan("fix-type-errors-prompt");
+            const instruction = await (0, provider_1.getPrompt)("fix-file-errors-ts", {
+                testFiles: codePrompt || "",
+                pageFiles: pomPrompt || "",
+                scenarioFile: file,
+                errors: errors,
+                fileContent: fileContent,
+                scenaioName: scenario.name,
+            });
+            promptSpan.end({ output: { instruction } });
+            response = await (0, ai_1.getLLMResult)(instruction, trace);
+            const readWriteFileSpan = trace.startSpan("write-to-file");
             await fs_extra_1.default.writeFile(file, response, "utf-8");
-            await lintErrors(file);
-            errors = validateTypescript(file);
+            readWriteFileSpan.end({ output: { response } });
+            trace.event({ name: "lint-file" });
+            await (0, web_1.lintErrors)(file);
+            const validateTypesSpan = trace.startSpan("detect-type-errors-in-file");
+            errors = (0, web_1.validateTypescript)(file);
+            validateTypesSpan.end({ output: { errors } });
+            if (!errors.length) {
+                logger.success("Found no type issues!");
+            }
         }
-        await formatCode(file);
+        trace.event({ name: "format-file" });
+        await (0, web_1.formatCode)(file);
+        logger.success("File formatted successfully!", `[Trace: ${trace.id}]`);
         generatedScenarios.push(scenario);
+        trace.update({ input: { scenario }, output: { response } });
     }
     return generatedScenarios;
 }
 (async function main() {
-    if (process.argv.length != 3) {
-        logger.error("Please provide path to scenarios using command:", "npx @empiricalrun/test-gen <SCENARIOS_FILE_PATH>");
+    if (process.argv.length < 3) {
+        logger.error("Please provide path to scenarios using command:", "npx @empiricalrun/test-gen <SCENARIOS_FILE_PATH> <SCENARIO_NAME> -u");
         process.exit(1);
     }
     const scenariosPath = process.argv[2];
-    const testGenConfigs = await (0, scenarios_1.generateScenarios)(scenariosPath);
+    const scenario = process.argv[3];
+    const isUpdate = process.argv.includes("-u");
+    let testGenConfigs = await (0, scenarios_1.generateScenarios)(scenariosPath);
+    if (scenario) {
+        // filter config
+        testGenConfigs = testGenConfigs.filter((t) => t.scenarios.some((s) => s.name === scenario));
+        // if no config available then throw error
+        if (testGenConfigs.length == 0) {
+            logger.error("No scenarios found for the given scenario name", scenario);
+            process.exit(1);
+        }
+        const [testGenConfig] = testGenConfigs;
+        // filter scenarios
+        testGenConfig.scenarios = testGenConfig.scenarios.filter((s) => s.name === scenario);
+    }
     const generatedTestScenarios = [];
     for (const testGenConfig of testGenConfigs) {
-        const specPath = testGenConfig.specPath;
+        const { specPath, scenarios } = testGenConfig;
         if (!fs_extra_1.default.existsSync(specPath)) {
             logger.log(`Creating a new spec file: ${specPath}`);
             fs_extra_1.default.createFileSync(specPath);
         }
-        const gen = await generateTest(testGenConfig.scenarios, specPath);
+        const gen = await generateTest(scenarios, specPath, isUpdate);
         generatedTestScenarios.push(...gen);
     }
     await (0, ci_1.reportOnCI)(generatedTestScenarios);

package/dist/bin/logger/index.d.ts

@@ -1,13 +1,8 @@
-export interface Logger {
-    log: (message?: string, ...optionalParams: any[]) => void;
-    warn: (message?: string, ...optionalParams: any[]) => void;
-    error: (message?: string, ...optionalParams: any[]) => void;
-    success: (message?: string, ...optionalParams: any[]) => void;
-}
-export declare class CustomLogger implements Logger {
+export declare class CustomLogger {
     log(message?: string, ...optionalParams: any[]): void;
     warn(message?: string, ...optionalParams: any[]): void;
     success(message?: string, ...optionalParams: any[]): void;
     error(message?: string, ...optionalParams: any[]): void;
+    logEmptyLine(): void;
 }
 //# sourceMappingURL=index.d.ts.map

package/dist/bin/logger/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/logger/index.ts"],"names":[],"mappings":"AACA,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE,KAAK,IAAI,CAAC;IAC1D,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE,KAAK,IAAI,CAAC;IAC3D,KAAK,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE,KAAK,IAAI,CAAC;IAC5D,OAAO,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE,KAAK,IAAI,CAAC;CAC/D;AAED,qBAAa,YAAa,YAAW,MAAM;IACzC,GAAG,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE;IAI9C,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE;IAI/C,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE;IAIlD,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE;CAGjD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/logger/index.ts"],"names":[],"mappings":"AAEA,qBAAa,YAAY;IACvB,GAAG,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE;IAI9C,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE;IAI/C,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE;IAIlD,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,GAAG,EAAE;IAIhD,YAAY;CAGb"}

package/dist/bin/logger/index.js

@@ -15,5 +15,8 @@ class CustomLogger {
     error(message, ...optionalParams) {
         console.log("🚨", (0, picocolors_1.red)(message), ...optionalParams);
     }
+    logEmptyLine() {
+        console.log("\n\n");
+    }
 }
 exports.CustomLogger = CustomLogger;

package/dist/bin/reporter/ci.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../../src/bin/reporter/ci.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAGpC,wBAAsB,UAAU,CAAE,SAAS,EAAE,QAAQ,EAAE,uBAatD"}
+{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../../src/bin/reporter/ci.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEpC,wBAAsB,UAAU,CAAC,SAAS,EAAE,QAAQ,EAAE,uBAerD"}

package/dist/bin/reporter/ci.js

@@ -8,13 +8,15 @@ const fs_extra_1 = __importDefault(require("fs-extra"));
 async function reportOnCI(scenarios) {
     if (process.env.CI && process.env.GITHUB_OUTPUT) {
         const testNames = scenarios.map((s) => s.name).join(" ");
-        const scenariosOutput = scenarios.map((s) => {
+        const scenariosOutput = scenarios
+            .map((s) => {
             return `**Scenario:** ${s.name} \n\n**Steps:**\n  - ${s.steps.join("\n  - ")} \n\n**Assert**:\n  - ${s.assert}\n\n ----`;
-        }).join("\n ----- \n");
+        })
+            .join("\n ----- \n");
         const envVars = [
             `summary<<EOF\n${scenariosOutput}\nEOF`,
             `test_names=${testNames}`,
-        ].join('\n');
+        ].join("\n");
         await fs_extra_1.default.appendFile(process.env.GITHUB_OUTPUT, envVars);
     }
     return scenarios;

package/dist/bin/scenarios/index.d.ts

@@ -1,4 +1,4 @@
-import { Scenario } from '../types';
+import { Scenario } from "../types";
 declare function generateScenarios(scenariosPath: string): Promise<{
     specPath: string;
     scenarios: Scenario[];

package/dist/bin/scenarios/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/scenarios/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAsEpC,iBAAe,iBAAiB,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,QAAQ,EAAE,CAAA;CAAC,EAAE,CAAC,CAM7G;AAID,OAAO,EACH,iBAAiB,EACpB,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/scenarios/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAiGpC,iBAAe,iBAAiB,CAC9B,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,QAAQ,EAAE,CAAA;CAAE,EAAE,CAAC,CAqBxD;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC"}

package/dist/bin/scenarios/index.js

@@ -4,11 +4,20 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.generateScenarios = void 0;
-const google_auth_library_1 = require("google-auth-library");
-const slugify_1 = __importDefault(require("slugify"));
 const fs_extra_1 = __importDefault(require("fs-extra"));
+const google_auth_library_1 = require("google-auth-library");
 const path_1 = __importDefault(require("path"));
+const slugify_1 = __importDefault(require("slugify"));
 const yaml_1 = require("yaml");
+function isValidJSON(str) {
+    try {
+        JSON.parse(str);
+        return true;
+    }
+    catch (e) {
+        return false;
+    }
+}
 /**
  * Method to update / add scenarios to the repo.
  * @param path
@@ -24,24 +33,30 @@ async function generateScenariosUsingGsheet(path) {
     const serviceAccountAuth = new google_auth_library_1.JWT({
         email: process.env.GOOGLE_SERVICE_EMAIL,
         key: Buffer.from(process.env.GOOGLE_SERVICE_EMAIL_PRIVATE_KEY, "base64").toString(),
-        scopes: ['https://www.googleapis.com/auth/spreadsheets'],
+        scopes: ["https://www.googleapis.com/auth/spreadsheets"],
     });
     const doc = new GoogleSpreadsheet(docId, serviceAccountAuth);
     await doc.loadInfo();
     const sheet = doc.sheetsById[sheetId];
     const rows = await sheet.getRows();
     const map = new Map();
-    rows.forEach(r => {
+    rows.forEach((r) => {
         // TODO: fix for case insensitive
         const category = r.get("Category");
         const name = r.get("Scenario");
-        const steps = (r.get("Steps").split("\n")).map((s) => s.trim()).filter((s) => !!s.length);
+        const steps = r
+            .get("Steps")
+            .split("\n")
+            .map((s) => s.trim())
+            .filter((s) => !!s.length);
         const assert = r.get("Assert");
-        const specPath = category ? `./tests/${category}.spec.ts` : `./tests/${(0, slugify_1.default)(name)}.spec.ts`;
+        const specPath = category
+            ? `./tests/${category}.spec.ts`
+            : `./tests/${(0, slugify_1.default)(name)}.spec.ts`;
         const scenario = {
             steps,
             name,
-            assert
+            assert,
         };
         if (!map.get(specPath)) {
             map.set(specPath, [scenario]);
@@ -56,14 +71,14 @@ async function generateScenariosUsingGsheet(path) {
     for (const [specPath, scenarios] of map.entries()) {
         results.push({
             specPath,
-            scenarios
+            scenarios,
         });
     }
     return results;
 }
 async function generateScenariosUsingYAML(scenariosPath) {
-    const file = await fs_extra_1.default.readFile(path_1.default.resolve(process.cwd(), scenariosPath), 'utf8');
-    const fileName = scenariosPath.split('/').pop()?.split(".")[0];
+    const file = await fs_extra_1.default.readFile(path_1.default.resolve(process.cwd(), scenariosPath), "utf8");
+    const fileName = scenariosPath.split("/").pop()?.split(".")[0];
     const config = (0, yaml_1.parse)(file);
     const fileDir = config.dir || "./tests";
     config.specPath = `${fileDir}/${fileName}.spec.ts`;
@@ -73,6 +88,22 @@ async function generateScenarios(scenariosPath) {
     if (scenariosPath.startsWith("https://docs.google.com/spreadsheets")) {
         return await generateScenariosUsingGsheet(scenariosPath);
     }
+    else if (isValidJSON(scenariosPath)) {
+        const config = JSON.parse(scenariosPath);
+        const specPath = `./tests/${config.group || "index"}.spec.ts`;
+        return [
+            {
+                specPath,
+                scenarios: [
+                    {
+                        name: config.name,
+                        steps: config.steps,
+                        assert: config.assert,
+                    },
+                ],
+            },
+        ];
+    }
     else {
         return await generateScenariosUsingYAML(scenariosPath);
     }

package/dist/bin/session/index.d.ts

@@ -0,0 +1,6 @@
+declare function getSessionDetails(): {
+    id: `${string}-${string}-${string}-${string}-${string}`;
+    version: string;
+};
+export { getSessionDetails };
+//# sourceMappingURL=index.d.ts.map

package/dist/bin/session/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/session/index.ts"],"names":[],"mappings":"AAKA,iBAAS,iBAAiB;;;EAKzB;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC"}

package/dist/bin/session/index.js

@@ -0,0 +1,16 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getSessionDetails = void 0;
+const crypto_1 = __importDefault(require("crypto"));
+const package_json_1 = __importDefault(require("./../../../package.json"));
+const sessionId = crypto_1.default.randomUUID();
+function getSessionDetails() {
+    return {
+        id: sessionId,
+        version: package_json_1.default.version,
+    };
+}
+exports.getSessionDetails = getSessionDetails;

package/dist/bin/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/types/index.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,WAAW,GAAG;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAA;AAEH,MAAM,MAAM,aAAa,GAAG;IACxB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,QAAQ,EAAE,CAAC;CACzB,CAAA;AAED,MAAM,MAAM,QAAQ,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAClB,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/types/index.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,QAAQ,EAAE,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC"}

package/dist/bin/utils/fs/index.d.ts

@@ -0,0 +1,5 @@
+import { FileContent } from "../../types";
+export declare function readFilesInDirectory(dir?: string): Promise<FileContent[]>;
+export declare function convertFileContentsToString(files?: FileContent[]): string;
+export declare function generatePromptFromDirectory(dir?: string): Promise<string | undefined>;
+//# sourceMappingURL=index.d.ts.map

package/dist/bin/utils/fs/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/bin/utils/fs/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,wBAAsB,oBAAoB,CAAC,GAAG,GAAE,MAAW,0BAgB1D;AAED,wBAAgB,2BAA2B,CAAC,KAAK,GAAE,WAAW,EAAO,UAQpE;AAED,wBAAsB,2BAA2B,CAAC,GAAG,SAAK,+BAQzD"}

package/dist/bin/utils/fs/index.js

@@ -0,0 +1,46 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generatePromptFromDirectory = exports.convertFileContentsToString = exports.readFilesInDirectory = void 0;
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const path_1 = __importDefault(require("path"));
+async function readFilesInDirectory(dir = "") {
+    let files = [];
+    const items = await fs_extra_1.default.readdir(dir);
+    for (const item of items) {
+        const fullPath = path_1.default.join(dir, item);
+        const stat = await fs_extra_1.default.stat(fullPath);
+        if (stat.isDirectory()) {
+            files = files.concat(await readFilesInDirectory(fullPath));
+        }
+        else if (stat.isFile()) {
+            const content = await fs_extra_1.default.readFile(fullPath, "utf-8");
+            files.push({ filePath: fullPath, content });
+        }
+    }
+    return files;
+}
+exports.readFilesInDirectory = readFilesInDirectory;
+function convertFileContentsToString(files = []) {
+    let prompt = "";
+    files.forEach((file) => {
+        prompt += `File Path: ${file.filePath}\n`;
+        prompt += `File:\n`;
+        prompt += `${file.content}\n\n ------ \n\n`;
+    });
+    return prompt;
+}
+exports.convertFileContentsToString = convertFileContentsToString;
+async function generatePromptFromDirectory(dir = "") {
+    try {
+        const files = await readFilesInDirectory(dir);
+        const prompt = convertFileContentsToString(files);
+        return prompt;
+    }
+    catch (error) {
+        console.error("Error reading directory:", error);
+    }
+}
+exports.generatePromptFromDirectory = generatePromptFromDirectory;

package/dist/bin/utils/platform/web/index.d.ts

@@ -0,0 +1,6 @@
+export declare function getTypescriptTestBlock(scenarioName: string, content: string): RegExpExecArray | null;
+export declare function validateTypescript(filePath: string): string[];
+export declare function stripAndPrependImports(content: string): Promise<string[]>;
+export declare function lintErrors(filePath: string): Promise<void>;
+export declare function formatCode(filePath: string): Promise<void>;
+//# sourceMappingURL=index.d.ts.map

package/dist/bin/utils/platform/web/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAKA,wBAAgB,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,0BAG3E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAwC7D;AAED,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,MAAM,qBAK3D;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAOhD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD"}

package/dist/bin/utils/platform/web/index.js

@@ -0,0 +1,79 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatCode = exports.lintErrors = exports.stripAndPrependImports = exports.validateTypescript = exports.getTypescriptTestBlock = void 0;
+const eslint_1 = require("eslint");
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const prettier_1 = __importDefault(require("prettier"));
+const typescript_1 = __importDefault(require("typescript"));
+function getTypescriptTestBlock(scenarioName, content) {
+    const regex = new RegExp(`test\\("${scenarioName}"[^]*?\\}\\);`, "g");
+    return regex.exec(content);
+}
+exports.getTypescriptTestBlock = getTypescriptTestBlock;
+function validateTypescript(filePath) {
+    // Create a compiler host to read files
+    const compilerHost = typescript_1.default.createCompilerHost({});
+    compilerHost.readFile = (file) => fs_extra_1.default.readFileSync(file, "utf8");
+    // Create a program with a single source file
+    const program = typescript_1.default.createProgram([filePath], {}, compilerHost);
+    // Get the source file
+    const sourceFile = program.getSourceFile(filePath);
+    // Get and report any syntactic errors
+    const errors = [];
+    const syntacticDiagnostics = program.getSyntacticDiagnostics(sourceFile);
+    if (syntacticDiagnostics.length > 0) {
+        syntacticDiagnostics.forEach((diagnostic) => {
+            // const { line, character } = diagnostic.file.getLineAndCharacterOfPosition(
+            //   diagnostic.start,
+            // );
+            // const message = ts.flattenDiagnosticMessageText(
+            //   diagnostic.messageText,
+            //   "\n",
+            // );
+            // logger.log(
+            //   `${diagnostic.file.fileName} (${line + 1},${character + 1}): ${message}`,
+            // );
+            if (typeof diagnostic.messageText === "string") {
+                errors.push(diagnostic.messageText);
+            }
+        });
+    }
+    // Get and report any semantic errors
+    const semanticDiagnostics = program.getSemanticDiagnostics(sourceFile);
+    if (semanticDiagnostics.length > 0) {
+        semanticDiagnostics.forEach((diagnostic) => {
+            errors.push(diagnostic.messageText.toString());
+        });
+    }
+    return errors;
+}
+exports.validateTypescript = validateTypescript;
+async function stripAndPrependImports(content) {
+    const imports = content.match(/import.* from.*;/g);
+    const strippedContent = content.replace(/import.* from.*;/g, "");
+    const prependContent = (imports?.join("\n") || "") + "\n\n";
+    return [prependContent, strippedContent];
+}
+exports.stripAndPrependImports = stripAndPrependImports;
+async function lintErrors(filePath) {
+    const eslint = new eslint_1.ESLint({
+        fix: true,
+        useEslintrc: true,
+    });
+    const [result] = await eslint.lintFiles(filePath);
+    fs_extra_1.default.writeFileSync(filePath, result?.output || "");
+}
+exports.lintErrors = lintErrors;
+async function formatCode(filePath) {
+    const fileContent = fs_extra_1.default.readFileSync(filePath, "utf8");
+    const prettierConfig = {};
+    const formattedContent = await prettier_1.default.format(fileContent, {
+        ...prettierConfig,
+        filepath: filePath,
+    });
+    fs_extra_1.default.writeFileSync(filePath, formattedContent);
+}
+exports.formatCode = formatCode;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.4.2",
+  "version": "0.6.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -15,11 +15,14 @@
   },
   "author": "Empirical Team <hey@empirical.run>",
   "dependencies": {
+    "commander": "^12.1.0",
     "dotenv": "^16.4.5",
     "eslint": "^8.57.0",
     "fs-extra": "^11.2.0",
     "google-auth-library": "^9.10.0",
     "google-spreadsheet": "^4.1.2",
+    "langfuse": "^3.11.2",
+    "langfuse-core": "^3.11.2",
     "openai": "^4.47.2",
     "picocolors": "^1.0.1",
     "prettier": "^3.2.5",
@@ -32,7 +35,7 @@
   },
   "scripts": {
     "dev": "tsc --build --watch",
-    "build": "tsc --build",
+    "build": "tsc --build && node ./../../tools/static-env-vars.js dist",
     "clean": "tsc --build --clean",
     "lint": "eslint .",
     "test": "vitest run",