@empiricalrun/test-gen 0.79.2 → 0.79.3

package/dist/generate-summary/generate-failed-step-screenshot-diff-summary.js

@@ -0,0 +1,460 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateFailedStepScreenshotDiffSummary = exports.getScreenshotsInFailedStepWindow = exports.getFailedStep = exports.findSimilarActionFromList = void 0;
+const llm_1 = require("@empiricalrun/llm");
+const r2_uploader_1 = require("@empiricalrun/r2-uploader");
+const trace_utils_1 = require("@empiricalrun/test-gen/trace-utils");
+// import { waitUntil } from "@vercel/functions";
+const buffer_1 = require("buffer");
+// import fs from "fs";
+const lodash_isequal_1 = __importDefault(require("lodash.isequal"));
+// import prompt from "../../../../prompts/video-diff-summary-prompt.handlebars";
+const trace_utils_2 = require("../trace-utils");
+const frame_sampling_1 = require("./frame-sampling");
+const responseFormat = {
+    type: "json_schema",
+    json_schema: {
+        name: "screenshot-difference-with-failure-reason",
+        strict: true,
+        schema: {
+            type: "object",
+            required: ["difference_in_screenshots", "reason_for_test_failure"],
+            properties: {
+                difference_in_screenshots: {
+                    type: "string",
+                    description: "what has changed between the successful run and failed run screenshots",
+                },
+                reason_for_test_failure: {
+                    type: "string",
+                    description: "reason for test failure",
+                },
+            },
+            additionalProperties: false,
+        },
+    },
+};
+const convertBufferToBase64 = (imageBuffer, mimeType) => {
+    const base64Image = imageBuffer.toString("base64");
+    const dataUrl = `data:${mimeType};base64,${base64Image}`;
+    return dataUrl;
+};
+const uploadScreenshotsToR2 = async ({ successScreenshots, failureScreenshots, testRunId, project, test, }) => {
+    try {
+        const file = {
+            buffer: buffer_1.Buffer.from(JSON.stringify({
+                success: successScreenshots.map((screenshot) => {
+                    return screenshot.base64;
+                }),
+                failure: failureScreenshots.map((screenshot) => {
+                    return screenshot.base64;
+                }),
+            }), "utf-8"),
+            fileName: `test-case-${test.testCaseId}-${test.slug}.json`,
+            mimeType: "application/json",
+        };
+        const folderName = process.env.NODE_ENV === "production"
+            ? "visual-comparison-screenshots"
+            : "visual-comparison-screenshots-development";
+        const fileUrls = await (0, r2_uploader_1.uploadInMemoryFiles)({
+            files: [file],
+            destinationDir: `${project.repo_name.replace("-tests", "")}/${testRunId}/${folderName}`,
+            uploadBucket: "test-report",
+            accountId: process.env.R2_ACCOUNT_ID,
+            accessKeyId: process.env.R2_ACCESS_KEY_ID,
+            secretAccessKey: process.env.R2_SECRET_ACCESS_KEY,
+        });
+        return fileUrls;
+    }
+    catch (e) {
+        console.error(`[testRunId - ${testRunId}] uploadScreenshotsToR2 failed`, e);
+        return {};
+    }
+};
+const findSimilarActionFromList = ({ actionList, actionToFind, }) => {
+    const actionStack = actionToFind?.stack?.[0];
+    let matchingAction = actionList.find((currentAction) => {
+        const currentStack = currentAction?.stack?.[0];
+        return (currentStack?.file === actionStack?.file &&
+            currentStack?.line === actionStack?.line &&
+            currentStack?.column === actionStack?.column &&
+            currentAction?.apiName === actionToFind?.apiName &&
+            (0, lodash_isequal_1.default)(currentAction?.params, actionToFind?.params));
+    });
+    if (matchingAction) {
+        return matchingAction;
+    }
+    matchingAction = actionList.find((currentAction) => {
+        const currentStack = currentAction?.stack?.[0];
+        return (currentStack?.file === actionStack?.file &&
+            currentAction?.apiName === actionToFind?.apiName &&
+            (0, lodash_isequal_1.default)(currentAction?.params, actionToFind?.params));
+    });
+    if (matchingAction) {
+        return matchingAction;
+    }
+    matchingAction = actionList.find((currentAction) => {
+        const currentStack = currentAction?.stack?.[0];
+        return (currentStack?.file === actionStack?.file &&
+            currentStack?.line === actionStack?.line &&
+            currentStack?.column === actionStack?.column &&
+            currentAction?.apiName === actionToFind?.apiName);
+    });
+    return matchingAction;
+};
+exports.findSimilarActionFromList = findSimilarActionFromList;
+const getFailedStep = async ({ zipUrl, sendParentForBeforeAction = true, }) => {
+    const allTestActions = [];
+    const testTraceFile = "test.trace";
+    await (0, trace_utils_2.extractFileFromZipFromUrl)({
+        zipUrl,
+        fileNames: [testTraceFile],
+        chunkProcessor: (_, fileName) => {
+            let testTraceTempBuffer = "";
+            return (chunk) => {
+                switch (fileName) {
+                    case testTraceFile: {
+                        testTraceTempBuffer += chunk;
+                        let lines = testTraceTempBuffer.split("\n");
+                        lines.slice(0, -1).forEach((line) => {
+                            const parsedEvent = JSON.parse(line);
+                            allTestActions.push(parsedEvent);
+                        });
+                        testTraceTempBuffer = lines[lines.length - 1];
+                        break;
+                    }
+                    default:
+                        break;
+                }
+            };
+        },
+    });
+    const totalActions = allTestActions.length;
+    let lastBeforeAction;
+    let lastBeforeActionParent;
+    for (let actionIndex = 0; actionIndex < totalActions; actionIndex++) {
+        const currentAction = allTestActions[actionIndex];
+        if (currentAction.type === "before") {
+            lastBeforeAction = currentAction;
+        }
+        if (currentAction.type === "error") {
+            break;
+        }
+    }
+    // This handles cases where the failed step is inside a locator
+    // patch method of playwright-utils, and our intent is to find
+    // the parent action -- which is the caller in the actual test file
+    const isPlaywrightUtilsPatch = lastBeforeAction?.stack?.some((stackFrame) => stackFrame.file.includes("@empiricalrun/playwright-utils"));
+    if (isPlaywrightUtilsPatch && lastBeforeAction?.parentId) {
+        lastBeforeActionParent = allTestActions.find((action) => action.type === "before" &&
+            action.callId === lastBeforeAction?.parentId);
+    }
+    if (!lastBeforeAction) {
+        throw new Error("No before action found before error");
+    }
+    // Return parent action if requested and available, otherwise return the last before action
+    return sendParentForBeforeAction && lastBeforeActionParent
+        ? lastBeforeActionParent
+        : lastBeforeAction;
+};
+exports.getFailedStep = getFailedStep;
+const getScreenshotsInFailedStepWindow = async ({ failedStep, testRunId, zipUrl, isSuccessRun = false, }) => {
+    try {
+        const allTestActions = [];
+        const allScreenshots = [];
+        const stepIdToCallIdMap = {};
+        const detailedBeforeTraceActions = [];
+        const wallTimeToCallIdMap = {};
+        const testTraceFile = "test.trace";
+        const fileNames = await (0, trace_utils_2.getFilenamesInZip)(zipUrl, {
+            recursive: false,
+        });
+        const detailedTraceFileList = fileNames.filter((fileName) => fileName.includes("trace.trace"));
+        await (0, trace_utils_2.extractFileFromZipFromUrl)({
+            zipUrl,
+            fileNames: [testTraceFile, ...detailedTraceFileList],
+            chunkProcessor: (_, fileName) => {
+                let testTraceTempBuffer = "";
+                let detailedTraceTempBuffer = "";
+                return (chunk) => {
+                    if (fileName === testTraceFile) {
+                        testTraceTempBuffer += chunk;
+                        let lines = testTraceTempBuffer.split("\n");
+                        lines.slice(0, -1).forEach((line) => {
+                            const parsedEvent = JSON.parse(line);
+                            if (parsedEvent.type === "before" ||
+                                parsedEvent.type === "after") {
+                                allTestActions.push(parsedEvent);
+                            }
+                        });
+                        testTraceTempBuffer = lines[lines.length - 1];
+                    }
+                    else if (fileName.includes("trace.trace")) {
+                        detailedTraceTempBuffer += chunk;
+                        let lines = detailedTraceTempBuffer.split("\n");
+                        lines.slice(0, -1).forEach((line) => {
+                            const parsedEvent = JSON.parse(line);
+                            if (parsedEvent.type === "screencast-frame") {
+                                allScreenshots.push({
+                                    ...parsedEvent,
+                                    sha1: `resources/${parsedEvent.sha1}`,
+                                });
+                            }
+                            else if (parsedEvent.type === "before") {
+                                detailedBeforeTraceActions.push(parsedEvent);
+                                if (parsedEvent.stepId) {
+                                    stepIdToCallIdMap[parsedEvent.stepId] =
+                                        parsedEvent;
+                                }
+                                if (parsedEvent.wallTime) {
+                                    wallTimeToCallIdMap[parsedEvent.wallTime] =
+                                        parsedEvent;
+                                }
+                            }
+                        });
+                        detailedTraceTempBuffer = lines[lines.length - 1];
+                    }
+                };
+            },
+        });
+        const beforeActions = allTestActions.filter((action) => {
+            const isBefore = action.type === "before";
+            return isBefore;
+        });
+        let currentFailedStep = (0, exports.findSimilarActionFromList)({
+            actionList: beforeActions,
+            actionToFind: failedStep,
+        });
+        // if the callId is a step then that mapping doesnt exist inside the detail trace.
+        // we need to find the right step inside it to map it
+        // currently assuming test.step can only be in case playwright utils patched apis
+        if (currentFailedStep?.callId.includes("test.step")) {
+            currentFailedStep =
+                beforeActions.find((a) => a.parentId === currentFailedStep.callId) ||
+                    currentFailedStep;
+        }
+        // console.log("allscreenshots", allScreenshots.slice(0, 3));
+        const failedStepIndex = beforeActions.findIndex((a) => {
+            return a.callId === currentFailedStep.callId;
+        });
+        console.log("failedStepIndex", failedStepIndex);
+        console.log("beforeActions", beforeActions[failedStepIndex]);
+        let stepEndTime = allTestActions.find((action) => {
+            return (action.type === "after" && action.callId === currentFailedStep.callId);
+        }).endTime;
+        let lastActionCapturedInTestTrace = currentFailedStep;
+        let lastActionCapturedInDetailTrace = (stepIdToCallIdMap[currentFailedStep.callId] ||
+            wallTimeToCallIdMap[currentFailedStep.wallTime]);
+        if (!lastActionCapturedInDetailTrace) {
+            const actionsTakenTillLastFailedStepInReverseOrder = beforeActions
+                .slice(0, failedStepIndex)
+                .reverse();
+            const totalReversedActions = actionsTakenTillLastFailedStepInReverseOrder.length;
+            for (let actionIndex = 0; actionIndex < totalReversedActions; actionIndex++) {
+                const action = actionsTakenTillLastFailedStepInReverseOrder[actionIndex];
+                const correspondingDetailTraceAction = stepIdToCallIdMap[action.callId] ||
+                    wallTimeToCallIdMap[action.wallTime];
+                if (correspondingDetailTraceAction) {
+                    lastActionCapturedInTestTrace = action;
+                    lastActionCapturedInDetailTrace = correspondingDetailTraceAction;
+                }
+                if (lastActionCapturedInDetailTrace) {
+                    break;
+                }
+            }
+        }
+        const defaultWindowSize = 2000;
+        const pageIdForFailedStep = lastActionCapturedInDetailTrace.pageId;
+        const stepStartTime = isSuccessRun
+            ? lastActionCapturedInTestTrace.startTime - defaultWindowSize / 2
+            : lastActionCapturedInTestTrace.startTime;
+        const endTime = isSuccessRun
+            ? stepEndTime + defaultWindowSize
+            : stepEndTime;
+        // console.log("stepStartTime", stepStartTime);
+        // console.log("endTime", endTime);
+        const screenshotsForPage = allScreenshots.filter((screenshot) => {
+            return screenshot.pageId === pageIdForFailedStep;
+        });
+        allScreenshots.sort((a, b) => a.timestamp - b.timestamp);
+        const availableScreenshotTimestamps = screenshotsForPage.map((s) => s.timestamp);
+        let screenshotsToConsider = allScreenshots.filter((screenshot) => {
+            return (screenshot.pageId === pageIdForFailedStep &&
+                screenshot.timestamp <= endTime &&
+                screenshot.timestamp >= stepStartTime);
+        });
+        if (screenshotsToConsider.length === 0) {
+            const lastScreenshotTimestamp = availableScreenshotTimestamps[availableScreenshotTimestamps.length - 1];
+            // if the available screenshot timestamps are a lot less than the step start time
+            // take a window of last 2 seconds from the end of the screenshot timeline
+            if (lastScreenshotTimestamp < stepStartTime) {
+                screenshotsToConsider = allScreenshots.filter((screenshot) => {
+                    return (screenshot.pageId === pageIdForFailedStep &&
+                        screenshot.timestamp <= lastScreenshotTimestamp &&
+                        screenshot.timestamp >= lastScreenshotTimestamp - defaultWindowSize);
+                });
+            }
+            else {
+                // if the failed step is in the middle of screenshot timestamp but no corresponding image is found
+                // take a window of 4 seconds before and after the failed step
+                screenshotsToConsider = allScreenshots.filter((screenshot) => {
+                    return (screenshot.pageId === pageIdForFailedStep &&
+                        screenshot.timestamp <= endTime + defaultWindowSize / 2 &&
+                        screenshot.timestamp >= stepStartTime - defaultWindowSize / 2);
+                });
+            }
+        }
+        const imageBufferMap = {};
+        await (0, trace_utils_2.extractFileFromZipFromUrl)({
+            zipUrl,
+            fileNames: screenshotsToConsider.map((s) => s.sha1),
+            chunkProcessor: (_, fileName) => {
+                imageBufferMap[fileName] = imageBufferMap[fileName] || [];
+                return (chunk) => {
+                    if (chunk) {
+                        imageBufferMap[fileName].push(chunk);
+                    }
+                };
+            },
+        });
+        const screenshotsWithBase64 = screenshotsToConsider.map((screenshot) => {
+            const imageBuffer = buffer_1.Buffer.concat(imageBufferMap[screenshot.sha1]);
+            const base64 = convertBufferToBase64(imageBuffer, "image/jpeg");
+            return {
+                metadata: { ...screenshot, base64 },
+                image: imageBuffer.toString("base64"),
+            };
+        });
+        // console.log("SCREENSHOT", screenshotsWithBase64.length);
+        const images = await (0, trace_utils_1.deduplicateImages)({
+            base64Images: screenshotsWithBase64,
+            threshold: 0.001,
+            logPrefix: `TestRun: ${testRunId}`,
+        });
+        // console.log("DEDUPED IMAGES", images.length);
+        return images.map((i) => i.metadata);
+    }
+    catch (e) {
+        console.warn(`Error extracting screenshots for testRunId - "${testRunId}", zipUrl - "${zipUrl}"`, e?.message);
+        return [];
+    }
+};
+exports.getScreenshotsInFailedStepWindow = getScreenshotsInFailedStepWindow;
+const generateFailedStepScreenshotDiffSummary = async ({ testRunId, test, modelConfig, trace, project, }) => {
+    if (!test.success?.trace && !test.failure?.trace) {
+        console.warn(`[testRunId - ${testRunId}][test - ${test.title}] No success and failure trace found for test, exiting early ...`);
+        return "";
+    }
+    if (!test.failure?.trace) {
+        console.warn(`[testRunId - ${testRunId}][test - ${test.title}] No failure trace found for test, exiting early ...`);
+        return "";
+    }
+    try {
+        const failedStep = await (0, exports.getFailedStep)({
+            testRunId,
+            zipUrl: test.failure.trace,
+        });
+        let failureScreenshots = await (0, exports.getScreenshotsInFailedStepWindow)({
+            failedStep,
+            testRunId,
+            zipUrl: test.failure.trace,
+        });
+        const totalFailureScreenshots = failureScreenshots?.length ?? 0;
+        console.log(`[testRunId - ${testRunId}][test - ${test.title}] has - "${totalFailureScreenshots}" failure screenshots`);
+        trace?.event({
+            name: "collected-failure-actions",
+            output: {
+                success: true,
+                data: { count: totalFailureScreenshots },
+            },
+        });
+        if (!totalFailureScreenshots) {
+            return "";
+        }
+        let successScreenshots = test.success.trace
+            ? await (0, exports.getScreenshotsInFailedStepWindow)({
+                failedStep,
+                testRunId,
+                zipUrl: test.success.trace,
+                isSuccessRun: true,
+            })
+            : [];
+        const totalSuccessScreenshots = successScreenshots?.length ?? 0;
+        console.log(`[testRunId - ${testRunId}][test - ${test.title}] has - "${totalSuccessScreenshots}" success screenshots`);
+        trace?.event({
+            name: "collected-success-actions",
+            output: {
+                success: true,
+                data: { count: totalSuccessScreenshots },
+            },
+        });
+        const totalScreenshots = totalSuccessScreenshots + totalFailureScreenshots;
+        const maxScreenshotsAllowed = 250;
+        if (totalScreenshots > maxScreenshotsAllowed) {
+            console.log(`[testRunId - ${testRunId}][testCaseId - ${test.testCaseId}] has - "${totalScreenshots}" screenshots, sampling to - "${maxScreenshotsAllowed}"`);
+            const sampledList = (0, frame_sampling_1.sampleCombinedList)(failureScreenshots, successScreenshots, maxScreenshotsAllowed);
+            failureScreenshots = sampledList.list1;
+            successScreenshots = sampledList.list2;
+        }
+        // waitUntil(
+        //   uploadScreenshotsToR2({
+        //     successScreenshots,
+        //     failureScreenshots,
+        //     testRunId,
+        //     project,
+        //     test,
+        //   }),
+        // );
+        const modelProvider = modelConfig?.provider ?? "openai";
+        const model = modelConfig?.model ?? "gpt-4o";
+        const temperature = modelConfig?.temperature ?? 0.5;
+        const llm = new llm_1.LLM({
+            provider: modelProvider,
+            defaultModel: model,
+        });
+        const generationTrace = trace?.generation({
+            name: "generate-trace-screenshot-diff-summary",
+            model,
+            modelParameters: {
+                temperature,
+            },
+        });
+        const totalImageCount = successScreenshots.length + failureScreenshots.length;
+        // If there are too many images LLM failes to respond within alloted timeout
+        const imageDetail = totalImageCount > 40 ? "low" : "high";
+        const messages = (0, llm_1.compilePrompt)("prompt", {
+            successScreenshots: successScreenshots.map((s) => s.base64),
+            failureScreenshots: failureScreenshots.map((s) => s.base64),
+            errorStack: test.failure.stack,
+        }, {
+            imageDetail,
+        });
+        const llmResponse = await llm.createChatCompletion({
+            messages,
+            trace: generationTrace,
+            modelParameters: {
+                temperature,
+            },
+            responseFormat,
+        });
+        const parsedResponse = JSON.parse(llmResponse?.content);
+        generationTrace?.end({
+            output: {
+                videoDiffSummary: JSON.stringify(parsedResponse),
+            },
+            usage: {
+                input: llm.promptTokens,
+                output: llm.completionTokens,
+                unit: "TOKENS",
+            },
+        });
+        return parsedResponse?.reason_for_test_failure ?? "";
+    }
+    catch (e) {
+        console.error(`[testRunId - ${testRunId}][testCaseId - ${test.testCaseId}] Error generating trace screenshot diff summary`, e);
+        return "";
+    }
+};
+exports.generateFailedStepScreenshotDiffSummary = generateFailedStepScreenshotDiffSummary;

package/dist/generate-summary/generate-grouped-summary.d.ts

@@ -0,0 +1,18 @@
+import { TraceClient } from "@empiricalrun/llm";
+import { z } from "zod";
+declare const groupedSummarySchema: z.ZodArray<z.ZodObject<{
+    testIds: z.ZodArray<z.ZodString>;
+    groupSummary: z.ZodString;
+}, z.core.$strip>>;
+export type GroupedSummarySchemaType = z.infer<typeof groupedSummarySchema>;
+type ArgsT = {
+    testRunId: number;
+    tests: {
+        testId: string;
+        errorSummary: string;
+    }[];
+    trace?: TraceClient;
+};
+export declare const generateGroupedSummary: ({ testRunId, tests, trace, }: ArgsT) => Promise<GroupedSummarySchemaType | null>;
+export {};
+//# sourceMappingURL=generate-grouped-summary.d.ts.map

package/dist/generate-summary/generate-grouped-summary.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate-grouped-summary.d.ts","sourceRoot":"","sources":["../../src/generate-summary/generate-grouped-summary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,QAAA,MAAM,oBAAoB;;;kBAKzB,CAAC;AACF,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AA0C5E,KAAK,KAAK,GAAG;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAClD,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,CAAC;AACF,eAAO,MAAM,sBAAsB,GAAU,8BAI1C,KAAK,KAAG,OAAO,CAAC,wBAAwB,GAAG,IAAI,CAgDjD,CAAC"}

package/dist/generate-summary/generate-grouped-summary.js

@@ -0,0 +1,91 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateGroupedSummary = void 0;
+const llm_1 = require("@empiricalrun/llm");
+const zod_1 = require("zod");
+const group_summary_handlebars_1 = __importDefault(require("@/prompts/group-summary.handlebars"));
+const groupedSummarySchema = zod_1.z.array(zod_1.z.object({
+    testIds: zod_1.z.array(zod_1.z.string()),
+    groupSummary: zod_1.z.string(),
+}));
+const getResponseFormat = (availableTestIds) => ({
+    type: "json_schema",
+    json_schema: {
+        name: "test-case-failure-grouped-summary",
+        strict: true,
+        schema: {
+            type: "object",
+            properties: {
+                groups: {
+                    type: "array",
+                    description: "array containing tests grouped by similar error summaries",
+                    items: {
+                        type: "object",
+                        properties: {
+                            testIds: {
+                                type: "array",
+                                description: "array of test ids which are grouped together",
+                                items: {
+                                    type: "string",
+                                    description: "Unique identifier for the tests, provided by the user",
+                                    enum: availableTestIds,
+                                },
+                            },
+                            groupSummary: {
+                                type: "string",
+                                description: "Summary of the grouped tests",
+                            },
+                        },
+                        required: ["testIds", "groupSummary"],
+                        additionalProperties: false,
+                    },
+                },
+            },
+            required: ["groups"],
+            additionalProperties: false,
+        },
+    },
+});
+const generateGroupedSummary = async ({ testRunId, tests, trace, }) => {
+    let output = null;
+    const availableTestIds = tests.map((t) => t.testId);
+    try {
+        const messages = (0, llm_1.compilePrompt)(promptTemplate_0, {
+            testList: JSON.stringify(tests),
+        });
+        trace?.event({
+            input: {
+                testList: JSON.stringify(tests),
+            },
+            output: { messages },
+        });
+        const llm = new llm_1.LLM({
+            trace,
+            provider: "openai",
+            defaultModel: "gpt-4o",
+        });
+        const llmResponse = await llm.createChatCompletion({
+            messages,
+            trace,
+            modelParameters: {
+                temperature: 0.2,
+            },
+            responseFormat: getResponseFormat(availableTestIds),
+        });
+        output = groupedSummarySchema.parse(JSON.parse(llmResponse?.content).groups);
+    }
+    catch (e) {
+        console.error(`Error grouping tests, returning all tests without groups for testRunId - "${testRunId}"`, e?.message);
+        output = tests.map((test) => {
+            return {
+                testIds: [test.testId],
+                groupSummary: test.errorSummary,
+            };
+        });
+    }
+    return output;
+};
+exports.generateGroupedSummary = generateGroupedSummary;

package/dist/generate-summary/merge-summary.d.ts

@@ -0,0 +1,16 @@
+import { TraceClient } from "@empiricalrun/llm";
+import { TestGroup } from "@empiricalrun/shared-types/failure-workflow";
+type ArgsT = {
+    testRunId: number;
+    test: TestGroup;
+    errorSummary: string;
+    videoDiffSummary: string;
+    networkFailures: {
+        endpoint: string;
+        status: number;
+    }[];
+    trace?: TraceClient;
+};
+export declare const mergeSummary: ({ testRunId, test, errorSummary: errorStackSummary, networkFailures, videoDiffSummary, trace, }: ArgsT) => Promise<string>;
+export {};
+//# sourceMappingURL=merge-summary.d.ts.map

package/dist/generate-summary/merge-summary.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"merge-summary.d.ts","sourceRoot":"","sources":["../../src/generate-summary/merge-summary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,SAAS,EAAE,MAAM,6CAA6C,CAAC;AAGxE,KAAK,KAAK,GAAG;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,SAAS,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACxD,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,CAAC;AACF,eAAO,MAAM,YAAY,GAAU,iGAOhC,KAAK,KAAG,OAAO,CAAC,MAAM,CA2CxB,CAAC"}

package/dist/generate-summary/merge-summary.js

@@ -0,0 +1,46 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mergeSummary = void 0;
+const llm_1 = require("@empiricalrun/llm");
+const merge_summary_handlebars_1 = __importDefault(require("@/prompts/merge-summary.handlebars"));
+const mergeSummary = async ({ testRunId, test, errorSummary: errorStackSummary, networkFailures, videoDiffSummary, trace, }) => {
+    let output = "";
+    const payload = {
+        test: JSON.stringify({
+            errorSummary: videoDiffSummary || errorStackSummary,
+            networkFailures,
+        }),
+    };
+    try {
+        const messages = (0, llm_1.compilePrompt)(promptTemplate_0, payload);
+        trace?.event({
+            input: {
+                payload,
+            },
+            output: { messages },
+        });
+        const llm = new llm_1.LLM({
+            trace,
+            // defaultModel: "gpt-4o-2024-08-06",
+            providerApiKey: process.env.ANTHROPIC_API_KEY,
+            provider: "anthropic",
+            defaultModel: "claude-3-5-sonnet-latest",
+        });
+        const llmResponse = await llm.createChatCompletion({
+            messages: messages,
+            trace,
+            modelParameters: {
+                temperature: 0.2,
+            },
+        });
+        output = llmResponse?.content ?? "";
+    }
+    catch (e) {
+        throw new Error(`Failed to merge network+video+error stack summary  for test - "${test.title}", testRunId - "${testRunId}"`, e.message);
+    }
+    return output;
+};
+exports.mergeSummary = mergeSummary;

package/dist/generate-summary/pick-videos-for-comparison.d.ts

@@ -0,0 +1,9 @@
+import { TestGroup } from "@empiricalrun/shared-types/failure-workflow";
+export declare const pickVideosForComparison: ({ testRunId, test, }: {
+    testRunId: number;
+    test: TestGroup;
+}) => Promise<{
+    failure: string;
+    success: string;
+}>;
+//# sourceMappingURL=pick-videos-for-comparison.d.ts.map

package/dist/generate-summary/pick-videos-for-comparison.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pick-videos-for-comparison.d.ts","sourceRoot":"","sources":["../../src/generate-summary/pick-videos-for-comparison.ts"],"names":[],"mappings":"AAqBA,OAAO,EAAE,SAAS,EAAE,MAAM,6CAA6C,CAAC;AAExE,eAAO,MAAM,uBAAuB,GAAU,sBAG3C;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,SAAS,CAAC;CACjB,KAAG,OAAO,CAAC;IACV,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB,CAiEA,CAAC"}