@empiricalrun/test-gen 0.22.7 → 0.22.9-beta.2

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @empiricalrun/test-gen
 
+## 0.22.9
+
+### Patch Changes
+
+- Updated dependencies [8a755ba]
+  - @empiricalrun/reporter@0.17.5
+
+## 0.22.8
+
+### Patch Changes
+
+- Updated dependencies [9fa7d90]
+  - @empiricalrun/reporter@0.17.4
+
 ## 0.22.7
 
 ### Patch Changes

package/dist/actions/assertTextVisibility.d.ts

@@ -0,0 +1,4 @@
+import { PlaywrightActionGenerator } from "../types";
+export declare const PLAYWRIGHT_ASSERT_TEXT_VISIBILITY_ACTION_NAME = "assert_text_visibility";
+export declare const assertTextVisibilityActionGenerator: PlaywrightActionGenerator;
+//# sourceMappingURL=assertTextVisibility.d.ts.map

package/dist/actions/assertTextVisibility.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertTextVisibility.d.ts","sourceRoot":"","sources":["../../src/actions/assertTextVisibility.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,UAAU,CAAC;AAGrD,eAAO,MAAM,6CAA6C,2BAChC,CAAC;AAE3B,eAAO,MAAM,mCAAmC,EAAE,yBAsDjD,CAAC"}

package/dist/actions/assertTextVisibility.js

@@ -0,0 +1,52 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.assertTextVisibilityActionGenerator = exports.PLAYWRIGHT_ASSERT_TEXT_VISIBILITY_ACTION_NAME = void 0;
+const utils_1 = require("./utils");
+exports.PLAYWRIGHT_ASSERT_TEXT_VISIBILITY_ACTION_NAME = "assert_text_visibility";
+const assertTextVisibilityActionGenerator = (page) => {
+    return {
+        execute: async (args) => {
+            const locator = await (0, utils_1.getPlaywrightLocatorUsingCssSelector)(args.css_selector, page);
+            const exec = new Function("page", `return page.${locator}.isVisible({ timeout: 3000 })`);
+            await exec(page);
+            return {
+                locator,
+            };
+        },
+        // TODO: args transformer to be kept at a single place
+        template: (args, options) => {
+            const css = args.css_selector;
+            const assertStr = `await expect(page.locator("${css}")).toBeVisible();`;
+            if (options?.locator) {
+                return `
+        // ${assertStr}
+        await expect(page.${options.locator}).toBeVisible();
+        `;
+            }
+            return assertStr;
+        },
+        name: exports.PLAYWRIGHT_ASSERT_TEXT_VISIBILITY_ACTION_NAME,
+        schema: {
+            type: "function",
+            function: {
+                name: exports.PLAYWRIGHT_ASSERT_TEXT_VISIBILITY_ACTION_NAME,
+                description: "assert whether the given element on the page is visible",
+                parameters: {
+                    type: "object",
+                    properties: {
+                        css_selector: {
+                            type: "string",
+                            description: "CSS selector to identify the element uniquely and click it. When creating CSS selectors, ensure they are unique and specific enough to select only one element, even if there are multiple elements of the same type (like multiple h1 elements)",
+                        },
+                        reason: {
+                            type: "string",
+                            description: "reason for calling this function",
+                        },
+                    },
+                    required: ["css_selector", "reason"],
+                },
+            },
+        },
+    };
+};
+exports.assertTextVisibilityActionGenerator = assertTextVisibilityActionGenerator;

package/dist/agent/browsing/run.js

@@ -23,7 +23,7 @@ async function generateTestsUsingBrowsingAgent(testFilePath) {
     logger.log(`Detected playwright project name: ${project}`);
     //TODO: change this to per test
     let command = `npx playwright test ${testFilePath} --retries 0 --project ${project}`;
-    if (!"true") {
+    if (!process.env.CI) {
         command = command.concat(` --headed`);
     }
     try {

package/dist/agent/browsing/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAUvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAED,wBAAsB,2BAA2B,CAAC,SAAS,EAAE,aAAa,iBAuCzE;AAiBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBASxD;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,QA4BnD;AAED,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,CAAC,CAkDjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAUvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AAED,wBAAsB,2BAA2B,CAAC,SAAS,EAAE,aAAa,iBA+CzE;AAwCD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBASxD;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,QA4BnD;AAED,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,GACrC,OAAO,CAAC,MAAM,CAAC,CAkDjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA"}

package/dist/agent/browsing/utils.js

@@ -33,14 +33,14 @@ async function prepareFileForBrowsingAgent(genConfig) {
     }
     if (name && steps && steps.length) {
         const existingContents = await fs_extra_1.default.readFile(specPath, "utf-8");
-        const testBlock = (0, web_1.getTypescriptTestBlock)(name, existingContents);
+        const { testBlock, parentDescribe } = (0, web_1.getTypescriptTestBlock)(name, existingContents);
         const mergedSteps = prepareBrowsingAgentTask(steps);
         let newContents = existingContents;
         if (testBlock) {
             logger.log("appending to existing test block");
             // test scenario is already present
-            const updateTestBlock = (0, web_1.appendToTestBlock)(testBlock, createTestGenBlock(mergedSteps)).replace("test(", "test.only("); // mark the test as test.only
-            newContents = existingContents.replace(testBlock, updateTestBlock);
+            const updatedTestBlock = (0, web_1.appendToTestBlock)(testBlock, createTestGenBlock(mergedSteps));
+            newContents = newContentsWithTestOnly(existingContents, testBlock, updatedTestBlock, parentDescribe);
         }
         else {
             logger.log("creating new test block");
@@ -52,6 +52,18 @@ async function prepareFileForBrowsingAgent(genConfig) {
     }
 }
 exports.prepareFileForBrowsingAgent = prepareFileForBrowsingAgent;
+function newContentsWithTestOnly(existingContents, originalTestBlock, updatedTestBlock, parentDescribeBlock) {
+    if (!parentDescribeBlock) {
+        const testMarkedAsOnly = updatedTestBlock.replace("test(", "test.only(");
+        return existingContents.replace(originalTestBlock, testMarkedAsOnly);
+    }
+    else {
+        const updatedDescribeBlock = parentDescribeBlock.replace(originalTestBlock, updatedTestBlock);
+        // TODO: this should only happen when the describe block has "serial" execution
+        const describeMarkedAsOnly = updatedDescribeBlock.replace("test.describe(", "test.describe.only(");
+        return existingContents.replace(parentDescribeBlock, describeMarkedAsOnly);
+    }
+}
 function createTestBlockForCreateTest(name, steps) {
     return `
 test.only("${name}", async ({page}) => {

package/dist/agent/codegen/run.js

@@ -70,7 +70,7 @@ async function generateTest(testCase, file, options) {
     const [prependContent, strippedContent] = await (0, web_1.stripAndPrependImports)(response, testCase?.name);
     let updatedContent = prependContent + contents + `\n\n${strippedContent}`;
     if (isUpdate) {
-        const testBlock = (0, web_1.getTypescriptTestBlock)(testCase?.name, contents);
+        const { testBlock } = (0, web_1.getTypescriptTestBlock)(testCase?.name, contents);
         contents = contents.replace(testBlock, `\n\n${strippedContent}`);
         updatedContent = prependContent + contents;
     }

package/dist/bin/ai/index.d.ts

@@ -0,0 +1,9 @@
+import OpenAI from "openai";
+import LLMTracing from "./trace";
+export declare function getLLMResult({ messages, trace, tools, tool_choice, }: {
+    messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[];
+    trace?: LLMTracing;
+    tools?: OpenAI.Chat.Completions.ChatCompletionTool[];
+    tool_choice?: OpenAI.Chat.Completions.ChatCompletionToolChoiceOption;
+}): Promise<OpenAI.Chat.Completions.ChatCompletionMessage | undefined>;
+//# sourceMappingURL=index.d.ts.map

package/dist/bin/ai/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/ai/index.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,UAAU,MAAM,SAAS,CAAC;AAEjC,wBAAsB,YAAY,CAAC,EACjC,QAAQ,EACR,KAAK,EACL,KAAK,EACL,WAAW,GACZ,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,CAAC;IAC/D,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,EAAE,CAAC;IACrD,WAAW,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,8BAA8B,CAAC;CACtE,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,GAAG,SAAS,CAAC,CAsBrE"}

package/dist/bin/ai/index.js

@@ -0,0 +1,31 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getLLMResult = void 0;
+const openai_1 = __importDefault(require("openai"));
+async function getLLMResult({ messages, trace, tools, tool_choice, }) {
+    const openai = new openai_1.default();
+    const model = "gpt-4o";
+    const parameters = {
+        temperature: 0.5,
+    };
+    const generation = trace?.startGeneration({
+        name: "get-llm-result",
+        model,
+        parameters,
+        input: messages,
+    });
+    const completion = await openai.chat.completions.create({
+        messages,
+        model,
+        tools,
+        ...parameters,
+        tool_choice,
+    });
+    const output = completion.choices[0]?.message;
+    generation?.end({ output });
+    return output;
+}
+exports.getLLMResult = getLLMResult;

package/dist/bin/ai/prompts/provider/index.d.ts

@@ -0,0 +1,9 @@
+import OpenAI from "openai";
+export declare function getPrompt(name: string, vars: any): Promise<OpenAI.Chat.Completions.ChatCompletionMessageParam[]>;
+export declare function getPromptForNextAction({ pageSnapshot, task, previousActions, lastActionErrors, }: {
+    pageSnapshot: string;
+    task: string;
+    previousActions: string[];
+    lastActionErrors: string[];
+}): Promise<OpenAI.Chat.Completions.ChatCompletionMessageParam[]>;
+//# sourceMappingURL=index.d.ts.map

package/dist/bin/ai/prompts/provider/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/ai/prompts/provider/index.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAS5B,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,GAAG,GACR,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,CAAC,CAI/D;AAGD,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,GACtB,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B,iEASA"}

package/dist/bin/ai/prompts/provider/index.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getPromptForNextAction = exports.getPrompt = void 0;
+const langfuse_1 = require("langfuse");
+const langfuse = new langfuse_1.Langfuse({
+    secretKey: "sk-lf-30eee545-9c2c-4646-bfd9-8f38acf2fa8f",
+    publicKey: "pk-lf-c5c7cf23-de76-4098-ad81-594bce7cf330",
+    baseUrl: "https://us.cloud.langfuse.com",
+    flushAt: 1,
+});
+async function getPrompt(name, vars) {
+    const prompt = await langfuse.getPrompt(name, undefined, { type: "chat" });
+    const compiledPrompt = prompt.compile(vars);
+    return compiledPrompt;
+}
+exports.getPrompt = getPrompt;
+// TODO: move this prompt to langfuse
+async function getPromptForNextAction({ pageSnapshot = "", task = "", previousActions = [], lastActionErrors = [], }) {
+    const previousActionsStr = previousActions.join("\n\n ---- \n\n");
+    const prompt = await getPrompt("browsing-agent-next-action", {
+        pageSnapshot,
+        previousActionsStr,
+        task,
+        lastActionErrors,
+    });
+    return prompt;
+}
+exports.getPromptForNextAction = getPromptForNextAction;

package/dist/bin/ai/trace/index.d.ts

@@ -0,0 +1,37 @@
+import OpenAI from "openai";
+export default class LLMTracing {
+    private trace;
+    constructor({ name }: {
+        name: string;
+    });
+    get id(): string;
+    get url(): string;
+    event({ name, input, metadata, output, }: {
+        name: string;
+        input?: any;
+        metadata?: any;
+        output?: any;
+    }): void;
+    update({ input, output, metadata, }: {
+        input?: any;
+        output?: any;
+        metadata?: any;
+    }): void;
+    startSpan(name: string, input?: Record<string, any>, metadata?: Record<string, any>): {
+        end: ({ output }: {
+            output: Record<string, any>;
+        }) => void;
+    };
+    startGeneration({ name, model, parameters, input, }: {
+        name: string;
+        model: string;
+        parameters: Record<string, any>;
+        input: any;
+    }): {
+        end: ({ output, usage, }: {
+            output: any;
+            usage?: OpenAI.Completions.CompletionUsage | undefined;
+        }) => void;
+    };
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/bin/ai/trace/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/bin/ai/trace/index.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAe5B,MAAM,CAAC,OAAO,OAAO,UAAU;IAC7B,OAAO,CAAC,KAAK,CAAsB;gBACvB,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE;IAStC,IAAI,EAAE,WAEL;IAED,IAAI,GAAG,WAEN;IAED,KAAK,CAAC,EACJ,IAAI,EACJ,KAAK,EACL,QAAQ,EACR,MAAM,GACP,EAAE;QACD,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,CAAC,EAAE,GAAG,CAAC;QACZ,QAAQ,CAAC,EAAE,GAAG,CAAC;QACf,MAAM,CAAC,EAAE,GAAG,CAAC;KACd;IAID,MAAM,CAAC,EACL,KAAU,EACV,MAAW,EACX,QAAa,GACd,EAAE;QACD,KAAK,CAAC,EAAE,GAAG,CAAC;QACZ,MAAM,CAAC,EAAE,GAAG,CAAC;QACb,QAAQ,CAAC,EAAE,GAAG,CAAC;KAChB;IAID,SAAS,CACP,IAAI,EAAE,MAAM,EACZ,KAAK,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,EAC/B,QAAQ,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM;;oBAIJ,OAAO,MAAM,EAAE,GAAG,CAAC;;;IAMnD,eAAe,CAAC,EACd,IAAS,EACT,KAAU,EACV,UAAe,EACf,KAAU,GACX,EAAE;QACD,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAChC,KAAK,EAAE,GAAG,CAAC;KACZ;;oBAYa,GAAG;;cAET,IAAI;;CAcb"}

package/dist/bin/ai/trace/index.js

@@ -0,0 +1,67 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const langfuse_1 = require("langfuse");
+const session_1 = require("../../session");
+const langfuse = new langfuse_1.Langfuse({
+    secretKey: "sk-lf-30eee545-9c2c-4646-bfd9-8f38acf2fa8f",
+    publicKey: "pk-lf-c5c7cf23-de76-4098-ad81-594bce7cf330",
+    baseUrl: "https://us.cloud.langfuse.com",
+    flushAt: 1,
+});
+process.on("exit", async () => await langfuse.flushAsync());
+process.on("SIGINT", async () => await langfuse.flushAsync());
+process.on("SIGTERM", async () => await langfuse.flushAsync());
+class LLMTracing {
+    trace;
+    constructor({ name }) {
+        const sessionDetails = (0, session_1.getSessionDetails)();
+        this.trace = langfuse.trace({
+            name,
+            sessionId: sessionDetails.id,
+            release: sessionDetails.version,
+        });
+    }
+    get id() {
+        return this.trace.traceId;
+    }
+    get url() {
+        return this.trace.getTraceUrl();
+    }
+    event({ name, input, metadata, output, }) {
+        this.trace.event({ name, input, metadata, output });
+    }
+    update({ input = {}, output = {}, metadata = {}, }) {
+        this.trace.update({ input, output, metadata });
+    }
+    startSpan(name, input = {}, metadata = {}) {
+        const span = this.trace.span({ name, metadata, input });
+        return {
+            end: ({ output }) => {
+                span.end({ output });
+            },
+        };
+    }
+    startGeneration({ name = "", model = "", parameters = {}, input = {}, }) {
+        const generation = this.trace.generation({
+            name,
+            model,
+            modelParameters: parameters,
+            input,
+        });
+        return {
+            end: ({ output, usage, }) => {
+                generation.end({
+                    output,
+                    usage: usage
+                        ? {
+                            promptTokens: usage.prompt_tokens,
+                            completionTokens: usage.completion_tokens,
+                            totalTokens: usage.total_tokens,
+                        }
+                        : undefined,
+                });
+            },
+        };
+    }
+}
+exports.default = LLMTracing;

package/dist/bin/reporter/ci.d.ts

@@ -0,0 +1,3 @@
+import { Scenario } from "../../types";
+export declare function reportOnCI(scenarios: Scenario[]): Promise<Scenario[]>;
+//# sourceMappingURL=ci.d.ts.map

package/dist/bin/reporter/ci.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../../src/bin/reporter/ci.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,wBAAsB,UAAU,CAAC,SAAS,EAAE,QAAQ,EAAE,uBAerD"}

package/dist/bin/reporter/ci.js

@@ -0,0 +1,24 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.reportOnCI = void 0;
+const fs_extra_1 = __importDefault(require("fs-extra"));
+async function reportOnCI(scenarios) {
+    if (process.env.CI && process.env.GITHUB_OUTPUT) {
+        const testNames = scenarios.map((s) => s.name).join(" ");
+        const scenariosOutput = scenarios
+            .map((s) => {
+            return `**Scenario:** ${s.name} \n\n**Steps:**\n  - ${s.steps.join("\n  - ")} \n\n**Assert**:\n  - ${s.assert}\n\n ----`;
+        })
+            .join("\n ----- \n");
+        const envVars = [
+            `summary<<EOF\n${scenariosOutput}\nEOF`,
+            `test_names=${testNames}`,
+        ].join("\n");
+        await fs_extra_1.default.appendFile(process.env.GITHUB_OUTPUT, envVars);
+    }
+    return scenarios;
+}
+exports.reportOnCI = reportOnCI;

package/dist/bin/scenarios/index.d.ts

@@ -0,0 +1,4 @@
+import { TestGenConfig } from "../../types";
+declare function loadTestConfigs(scenariosPath: string): Promise<TestGenConfig[]>;
+export { loadTestConfigs };
+//# sourceMappingURL=index.d.ts.map

package/dist/bin/scenarios/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/scenarios/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAY,aAAa,EAAwB,MAAM,aAAa,CAAC;AAkF5E,iBAAe,eAAe,CAC5B,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC,aAAa,EAAE,CAAC,CAiC1B;AAED,OAAO,EAAE,eAAe,EAAE,CAAC"}

package/dist/bin/scenarios/index.js

@@ -0,0 +1,112 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadTestConfigs = void 0;
+const google_auth_library_1 = require("google-auth-library");
+const slugify_1 = __importDefault(require("slugify"));
+function isValidJSON(str) {
+    try {
+        JSON.parse(str);
+        return true;
+    }
+    catch (e) {
+        return false;
+    }
+}
+/**
+ * Method to update / add scenarios to the repo.
+ * @param path
+ * @returns updated paths of scenarios
+ */
+async function loadScenariosFromGsheet(path) {
+    const { GoogleSpreadsheet } = await import("google-spreadsheet");
+    const url = new URL(path);
+    const docId = url.pathname.split("/")[3];
+    const searchParams = new URLSearchParams(url.hash.split("#")[1]);
+    const sheetId = Number(searchParams.get("gid")) || 0;
+    // TODO: use oauth 2
+    const serviceAccountAuth = new google_auth_library_1.JWT({
+        email: process.env.GOOGLE_SERVICE_EMAIL,
+        key: Buffer.from(process.env.GOOGLE_SERVICE_EMAIL_PRIVATE_KEY, "base64").toString(),
+        scopes: ["https://www.googleapis.com/auth/spreadsheets"],
+    });
+    const doc = new GoogleSpreadsheet(docId, serviceAccountAuth);
+    await doc.loadInfo();
+    const sheet = doc.sheetsById[sheetId];
+    const rows = await sheet.getRows();
+    const map = new Map();
+    rows.forEach((r) => {
+        // TODO: fix for case insensitive
+        const category = r.get("Category");
+        const name = r.get("Scenario");
+        const steps = r
+            .get("Steps")
+            .split("\n")
+            .map((s) => s.trim())
+            .filter((s) => !!s.length);
+        const assert = r.get("Assert");
+        const specPath = category
+            ? `./tests/${category}.spec.ts`
+            : `./tests/${(0, slugify_1.default)(name)}.spec.ts`;
+        const scenario = {
+            steps,
+            name,
+            assert,
+        };
+        if (!map.get(specPath)) {
+            map.set(specPath, [scenario]);
+        }
+        else {
+            const scenarios = map.get(specPath);
+            scenarios.push(scenario);
+            map.set(specPath, scenarios);
+        }
+    });
+    const results = [];
+    for (const [specPath, scenarios] of map.entries()) {
+        results.push({
+            specPath,
+            scenarios,
+        });
+    }
+    return results;
+}
+async function loadTestConfigs(scenariosPath) {
+    // google sheets generation flow
+    // TODO: remove this whenever we are comfortable demoing using dashboard
+    if (scenariosPath.startsWith("https://docs.google.com/spreadsheets")) {
+        return await loadScenariosFromGsheet(scenariosPath);
+        // dev testing flow
+    }
+    else if (scenariosPath.endsWith(".ts")) {
+        return [
+            {
+                specPath: scenariosPath,
+                scenarios: [],
+            },
+        ];
+        // api flow
+    }
+    else if (isValidJSON(atob(scenariosPath))) {
+        const str = atob(scenariosPath);
+        const config = JSON.parse(str);
+        const specPath = `./tests/${config.group || "index"}.spec.ts`;
+        return [
+            {
+                specPath,
+                scenarios: [
+                    {
+                        name: config.name,
+                        steps: config.steps.filter((s) => !!s),
+                        assert: config.assert,
+                    },
+                ],
+                options: config.options,
+            },
+        ];
+    }
+    throw Error("Invalid path for test scenarios");
+}
+exports.loadTestConfigs = loadTestConfigs;

package/dist/bin/session/index.d.ts

@@ -0,0 +1,6 @@
+declare function getSessionDetails(): {
+    id: `${string}-${string}-${string}-${string}-${string}`;
+    version: string;
+};
+export { getSessionDetails };
+//# sourceMappingURL=index.d.ts.map

package/dist/bin/session/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/bin/session/index.ts"],"names":[],"mappings":"AAKA,iBAAS,iBAAiB;;;EAKzB;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC"}

package/dist/bin/session/index.js

@@ -0,0 +1,16 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getSessionDetails = void 0;
+const crypto_1 = __importDefault(require("crypto"));
+const package_json_1 = __importDefault(require("./../../../package.json"));
+const sessionId = crypto_1.default.randomUUID();
+function getSessionDetails() {
+    return {
+        id: sessionId,
+        version: package_json_1.default.version,
+    };
+}
+exports.getSessionDetails = getSessionDetails;

package/dist/bin/utils/platform/web/index.d.ts

@@ -1,4 +1,7 @@
-export declare function getTypescriptTestBlock(scenarioName: string, content: string): string | undefined;
+export declare function getTypescriptTestBlock(scenarioName: string, content: string): {
+    testBlock: string | undefined;
+    parentDescribe: string | undefined;
+};
 export declare function appendToTestBlock(testBlock: string, content: string): string;
 export declare function validateTypescript(filePath: string): string[];
 export declare function stripAndPrependImports(content: string, testName: string): Promise<(string | undefined)[]>;

package/dist/bin/utils/platform/web/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAMA,wBAAgB,sBAAsB,CACpC,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,GACd,MAAM,GAAG,SAAS,CAYpB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAwD7D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,mCAOjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAIpD;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,UAcpD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/bin/utils/platform/web/index.ts"],"names":[],"mappings":"AAMA,wBAAgB,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;;;EAwB3E;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAwD7D;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,mCAUjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAShD;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,iBAQhD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,UAE5E;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,iBAMpD;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,UAcpD"}

package/dist/bin/utils/platform/web/index.js

@@ -12,10 +12,17 @@ const typescript_1 = __importDefault(require("typescript"));
 function getTypescriptTestBlock(scenarioName, content) {
     const project = new ts_morph_1.Project();
     const sourceFile = project.createSourceFile("test.ts", content);
-    const testFunction = sourceFile.getFirstDescendant((node) => !!(node.isKind(ts_morph_1.SyntaxKind.CallExpression) &&
+    const testFunctionNode = sourceFile.getFirstDescendant((node) => !!(node.isKind(ts_morph_1.SyntaxKind.CallExpression) &&
         node.getExpression().getText() === "test" &&
         node.getArguments()[0]?.getText().includes(scenarioName)));
-    return testFunction?.getText();
+    // TODO: support files that have 2 describe blocks with the same test name in them
+    const describeBlockNode = sourceFile.getFirstDescendant((node) => !!(node.isKind(ts_morph_1.SyntaxKind.CallExpression) &&
+        node.getExpression().getText() === "test.describe" &&
+        node.getText().includes(scenarioName)));
+    return {
+        testBlock: testFunctionNode?.getText(),
+        parentDescribe: describeBlockNode?.getText(),
+    };
 }
 exports.getTypescriptTestBlock = getTypescriptTestBlock;
 function appendToTestBlock(testBlock, content) {
@@ -77,7 +84,7 @@ exports.validateTypescript = validateTypescript;
 async function stripAndPrependImports(content, testName) {
     const importRegexp = /import\s+\{[^}]*\}\s+from\s+["'][^"']+["'];?/g;
     const imports = content.match(importRegexp);
-    const strippedContent = getTypescriptTestBlock(testName, content);
+    const { testBlock: strippedContent } = getTypescriptTestBlock(testName, content);
     const prependContent = (imports?.join("\n") || "") + "\n\n";
     return [prependContent, strippedContent];
 }
@@ -109,7 +116,9 @@ function addNewImport(contents, modules, pkg) {
 exports.addNewImport = addNewImport;
 async function removeTestOnly(filePath) {
     const contents = await fs_extra_1.default.readFile(filePath, "utf8");
-    const updatedContent = contents.replace(/test\.only/, "test");
+    const updatedContent = contents
+        .replace("test.only(", "test(")
+        .replace("test.describe.only(", "test.describe(");
     return fs_extra_1.default.writeFile(filePath, updatedContent);
 }
 exports.removeTestOnly = removeTestOnly;

package/dist/reporter/ci.js

@@ -26,7 +26,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.reportOnCI = void 0;
 const core = __importStar(require("@actions/core"));
 async function reportOnCI(testCase) {
-    if ("true") {
+    if (process.env.CI) {
         const scenariosOutput = `**Scenario:** ${testCase.name} \n\n**Steps:**\n  - ${testCase.steps.join("\n  - ")}`;
         core.setOutput("summary", scenariosOutput);
         core.setOutput("test_names", testCase.name);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/test-gen",
-  "version": "0.22.7",
+  "version": "0.22.9-beta.2",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -14,10 +14,20 @@
     "url": "https://github.com/empirical-run/empirical.git"
   },
   "author": "Empirical Team <hey@empirical.run>",
+  "scripts": {
+    "dev": "tsc --build --watch",
+    "build": "tsc --build && node ./../../tools/static-env-vars.js dist",
+    "clean": "tsc --build --clean",
+    "lint": "eslint .",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  },
   "dependencies": {
     "@actions/core": "^1.10.1",
     "@aws-sdk/client-s3": "^3.614.0",
     "@aws-sdk/s3-request-presigner": "^3.614.0",
+    "@empiricalrun/llm": "workspace:^",
+    "@empiricalrun/reporter": "workspace:^",
     "@playwright/test": "^1.44.1",
     "@types/sanitize-html": "^2.11.0",
     "commander": "^12.1.0",
@@ -40,22 +50,12 @@
     "slugify": "^1.6.6",
     "ts-morph": "^23.0.0",
     "tsx": "^4.16.2",
-    "typescript": "^5.3.3",
-    "@empiricalrun/llm": "^0.7.2",
-    "@empiricalrun/reporter": "^0.17.3"
+    "typescript": "^5.3.3"
   },
   "devDependencies": {
     "@types/detect-port": "^1.3.5",
     "@types/express": "^4.17.21",
     "@types/fs-extra": "^11.0.4",
     "@types/md5": "^2.3.5"
-  },
-  "scripts": {
-    "dev": "tsc --build --watch",
-    "build": "tsc --build && node ./../../tools/static-env-vars.js dist",
-    "clean": "tsc --build --clean",
-    "lint": "eslint .",
-    "test": "vitest run",
-    "test:watch": "vitest"
   }
 }