@empiricalrun/test-gen 0.81.0 → 0.81.2

package/dist/generate-summary/generate-failed-step-screenshot-diff-summary.d.ts

@@ -1,58 +0,0 @@
-import { LLMModel, LLMProvider, TraceClient } from "@empiricalrun/llm";
-import { Project } from "@empiricalrun/shared-types/api/projects";
-import { TestGroup } from "@empiricalrun/shared-types/failure-workflow";
-type ArgsT = {
-    testRunId: number;
-    test: TestGroup;
-    trace?: TraceClient;
-    project: Project;
-    modelConfig?: {
-        provider?: LLMProvider;
-        providerApiKey?: string;
-        model?: LLMModel;
-        temperature?: number;
-        prompt?: string;
-        promptVersion?: number;
-    };
-};
-type BeforeAction = {
-    apiName: string;
-    params: Record<string, any>;
-    callId: string;
-    type: "before";
-    startTime: number;
-    wallTime?: number;
-    parentId?: string;
-    stack?: {
-        file: string;
-        line: number;
-        column: number;
-    }[];
-};
-type ScreenshotsWithBase64 = {
-    type: string;
-    pageId: string;
-    sha1: string;
-    width: number;
-    height: number;
-    timestamp: number;
-    base64?: string;
-};
-export declare const findSimilarActionFromList: ({ actionList, actionToFind, }: {
-    actionList: BeforeAction[];
-    actionToFind: BeforeAction;
-}) => BeforeAction;
-export declare const getFailedStep: ({ zipUrl, sendParentForBeforeAction, }: {
-    testRunId?: number;
-    zipUrl: string;
-    sendParentForBeforeAction?: boolean;
-}) => Promise<BeforeAction>;
-export declare const getScreenshotsInFailedStepWindow: ({ failedStep, testRunId, zipUrl, isSuccessRun, }: {
-    failedStep: BeforeAction;
-    testRunId: number;
-    zipUrl: string;
-    isSuccessRun?: boolean;
-}) => Promise<ScreenshotsWithBase64[]>;
-export declare const generateFailedStepScreenshotDiffSummary: ({ testRunId, test, modelConfig, trace, project, }: ArgsT) => Promise<string>;
-export {};
-//# sourceMappingURL=generate-failed-step-screenshot-diff-summary.d.ts.map

package/dist/generate-summary/generate-failed-step-screenshot-diff-summary.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"generate-failed-step-screenshot-diff-summary.d.ts","sourceRoot":"","sources":["../../src/generate-summary/generate-failed-step-screenshot-diff-summary.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,QAAQ,EACR,WAAW,EACX,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,6CAA6C,CAAC;AAaxE,KAAK,KAAK,GAAG;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,SAAS,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE;QACZ,QAAQ,CAAC,EAAE,WAAW,CAAC;QACvB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,KAAK,CAAC,EAAE,QAAQ,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,aAAa,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;CACH,CAAC;AAwCF,KAAK,YAAY,GAAG;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,QAAQ,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC1D,CAAC;AAgBF,KAAK,qBAAqB,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAqDF,eAAO,MAAM,yBAAyB,GAAI,+BAGvC;IACD,UAAU,EAAE,YAAY,EAAE,CAAC;IAC3B,YAAY,EAAE,YAAY,CAAC;CAC5B,KAAG,YA6CH,CAAC;AAEF,eAAO,MAAM,aAAa,GAAU,wCAGjC;IACD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,yBAAyB,CAAC,EAAE,OAAO,CAAC;CACrC,KAAG,OAAO,CAAC,YAAY,CA8DvB,CAAC;AAEF,eAAO,MAAM,gCAAgC,GAAU,kDAKpD;IACD,UAAU,EAAE,YAAY,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,KAAG,OAAO,CAAC,qBAAqB,EAAE,CA+OlC,CAAC;AAEF,eAAO,MAAM,uCAAuC,GAAU,mDAM3D,KAAK,KAAG,OAAO,CAAC,MAAM,CA2JxB,CAAC"}

package/dist/generate-summary/generate-failed-step-screenshot-diff-summary.js

@@ -1,460 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateFailedStepScreenshotDiffSummary = exports.getScreenshotsInFailedStepWindow = exports.getFailedStep = exports.findSimilarActionFromList = void 0;
-const llm_1 = require("@empiricalrun/llm");
-const r2_uploader_1 = require("@empiricalrun/r2-uploader");
-// import prompt from "../../../../prompts/video-diff-summary-prompt.handlebars";
-const trace_utils_1 = require("@empiricalrun/trace-utils");
-// import { waitUntil } from "@vercel/functions";
-const buffer_1 = require("buffer");
-// import fs from "fs";
-const lodash_isequal_1 = __importDefault(require("lodash.isequal"));
-const dedup_image_1 = require("../utils/dedup/dedup-image");
-const frame_sampling_1 = require("./frame-sampling");
-const responseFormat = {
-    type: "json_schema",
-    json_schema: {
-        name: "screenshot-difference-with-failure-reason",
-        strict: true,
-        schema: {
-            type: "object",
-            required: ["difference_in_screenshots", "reason_for_test_failure"],
-            properties: {
-                difference_in_screenshots: {
-                    type: "string",
-                    description: "what has changed between the successful run and failed run screenshots",
-                },
-                reason_for_test_failure: {
-                    type: "string",
-                    description: "reason for test failure",
-                },
-            },
-            additionalProperties: false,
-        },
-    },
-};
-const convertBufferToBase64 = (imageBuffer, mimeType) => {
-    const base64Image = imageBuffer.toString("base64");
-    const dataUrl = `data:${mimeType};base64,${base64Image}`;
-    return dataUrl;
-};
-const uploadScreenshotsToR2 = async ({ successScreenshots, failureScreenshots, testRunId, project, test, }) => {
-    try {
-        const file = {
-            buffer: buffer_1.Buffer.from(JSON.stringify({
-                success: successScreenshots.map((screenshot) => {
-                    return screenshot.base64;
-                }),
-                failure: failureScreenshots.map((screenshot) => {
-                    return screenshot.base64;
-                }),
-            }), "utf-8"),
-            fileName: `test-case-${test.testCaseId}-${test.slug}.json`,
-            mimeType: "application/json",
-        };
-        const folderName = process.env.NODE_ENV === "production"
-            ? "visual-comparison-screenshots"
-            : "visual-comparison-screenshots-development";
-        const fileUrls = await (0, r2_uploader_1.uploadInMemoryFiles)({
-            files: [file],
-            destinationDir: `${project.repo_name.replace("-tests", "")}/${testRunId}/${folderName}`,
-            uploadBucket: "test-report",
-            accountId: process.env.R2_ACCOUNT_ID,
-            accessKeyId: process.env.R2_ACCESS_KEY_ID,
-            secretAccessKey: process.env.R2_SECRET_ACCESS_KEY,
-        });
-        return fileUrls;
-    }
-    catch (e) {
-        console.error(`[testRunId - ${testRunId}] uploadScreenshotsToR2 failed`, e);
-        return {};
-    }
-};
-const findSimilarActionFromList = ({ actionList, actionToFind, }) => {
-    const actionStack = actionToFind?.stack?.[0];
-    let matchingAction = actionList.find((currentAction) => {
-        const currentStack = currentAction?.stack?.[0];
-        return (currentStack?.file === actionStack?.file &&
-            currentStack?.line === actionStack?.line &&
-            currentStack?.column === actionStack?.column &&
-            currentAction?.apiName === actionToFind?.apiName &&
-            (0, lodash_isequal_1.default)(currentAction?.params, actionToFind?.params));
-    });
-    if (matchingAction) {
-        return matchingAction;
-    }
-    matchingAction = actionList.find((currentAction) => {
-        const currentStack = currentAction?.stack?.[0];
-        return (currentStack?.file === actionStack?.file &&
-            currentAction?.apiName === actionToFind?.apiName &&
-            (0, lodash_isequal_1.default)(currentAction?.params, actionToFind?.params));
-    });
-    if (matchingAction) {
-        return matchingAction;
-    }
-    matchingAction = actionList.find((currentAction) => {
-        const currentStack = currentAction?.stack?.[0];
-        return (currentStack?.file === actionStack?.file &&
-            currentStack?.line === actionStack?.line &&
-            currentStack?.column === actionStack?.column &&
-            currentAction?.apiName === actionToFind?.apiName);
-    });
-    return matchingAction;
-};
-exports.findSimilarActionFromList = findSimilarActionFromList;
-const getFailedStep = async ({ zipUrl, sendParentForBeforeAction = true, }) => {
-    const allTestActions = [];
-    const testTraceFile = "test.trace";
-    await (0, trace_utils_1.extractFileFromZip)({
-        zipUrl,
-        fileNames: [testTraceFile],
-        chunkProcessor: (_, fileName) => {
-            let testTraceTempBuffer = "";
-            return (chunk) => {
-                switch (fileName) {
-                    case testTraceFile: {
-                        testTraceTempBuffer += chunk;
-                        let lines = testTraceTempBuffer.split("\n");
-                        lines.slice(0, -1).forEach((line) => {
-                            const parsedEvent = JSON.parse(line);
-                            allTestActions.push(parsedEvent);
-                        });
-                        testTraceTempBuffer = lines[lines.length - 1];
-                        break;
-                    }
-                    default:
-                        break;
-                }
-            };
-        },
-    });
-    const totalActions = allTestActions.length;
-    let lastBeforeAction;
-    let lastBeforeActionParent;
-    for (let actionIndex = 0; actionIndex < totalActions; actionIndex++) {
-        const currentAction = allTestActions[actionIndex];
-        if (currentAction.type === "before") {
-            lastBeforeAction = currentAction;
-        }
-        if (currentAction.type === "error") {
-            break;
-        }
-    }
-    // This handles cases where the failed step is inside a locator
-    // patch method of playwright-utils, and our intent is to find
-    // the parent action -- which is the caller in the actual test file
-    const isPlaywrightUtilsPatch = lastBeforeAction?.stack?.some((stackFrame) => stackFrame.file.includes("@empiricalrun/playwright-utils"));
-    if (isPlaywrightUtilsPatch && lastBeforeAction?.parentId) {
-        lastBeforeActionParent = allTestActions.find((action) => action.type === "before" &&
-            action.callId === lastBeforeAction?.parentId);
-    }
-    if (!lastBeforeAction) {
-        throw new Error("No before action found before error");
-    }
-    // Return parent action if requested and available, otherwise return the last before action
-    return sendParentForBeforeAction && lastBeforeActionParent
-        ? lastBeforeActionParent
-        : lastBeforeAction;
-};
-exports.getFailedStep = getFailedStep;
-const getScreenshotsInFailedStepWindow = async ({ failedStep, testRunId, zipUrl, isSuccessRun = false, }) => {
-    try {
-        const allTestActions = [];
-        const allScreenshots = [];
-        const stepIdToCallIdMap = {};
-        const detailedBeforeTraceActions = [];
-        const wallTimeToCallIdMap = {};
-        const testTraceFile = "test.trace";
-        const fileNames = await (0, trace_utils_1.getFilenamesInZip)(zipUrl, {
-            recursive: false,
-        });
-        const detailedTraceFileList = fileNames.filter((fileName) => fileName.includes("trace.trace"));
-        await (0, trace_utils_1.extractFileFromZip)({
-            zipUrl,
-            fileNames: [testTraceFile, ...detailedTraceFileList],
-            chunkProcessor: (_, fileName) => {
-                let testTraceTempBuffer = "";
-                let detailedTraceTempBuffer = "";
-                return (chunk) => {
-                    if (fileName === testTraceFile) {
-                        testTraceTempBuffer += chunk;
-                        let lines = testTraceTempBuffer.split("\n");
-                        lines.slice(0, -1).forEach((line) => {
-                            const parsedEvent = JSON.parse(line);
-                            if (parsedEvent.type === "before" ||
-                                parsedEvent.type === "after") {
-                                allTestActions.push(parsedEvent);
-                            }
-                        });
-                        testTraceTempBuffer = lines[lines.length - 1];
-                    }
-                    else if (fileName.includes("trace.trace")) {
-                        detailedTraceTempBuffer += chunk;
-                        let lines = detailedTraceTempBuffer.split("\n");
-                        lines.slice(0, -1).forEach((line) => {
-                            const parsedEvent = JSON.parse(line);
-                            if (parsedEvent.type === "screencast-frame") {
-                                allScreenshots.push({
-                                    ...parsedEvent,
-                                    sha1: `resources/${parsedEvent.sha1}`,
-                                });
-                            }
-                            else if (parsedEvent.type === "before") {
-                                detailedBeforeTraceActions.push(parsedEvent);
-                                if (parsedEvent.stepId) {
-                                    stepIdToCallIdMap[parsedEvent.stepId] =
-                                        parsedEvent;
-                                }
-                                if (parsedEvent.wallTime) {
-                                    wallTimeToCallIdMap[parsedEvent.wallTime] =
-                                        parsedEvent;
-                                }
-                            }
-                        });
-                        detailedTraceTempBuffer = lines[lines.length - 1];
-                    }
-                };
-            },
-        });
-        const beforeActions = allTestActions.filter((action) => {
-            const isBefore = action.type === "before";
-            return isBefore;
-        });
-        let currentFailedStep = (0, exports.findSimilarActionFromList)({
-            actionList: beforeActions,
-            actionToFind: failedStep,
-        });
-        // if the callId is a step then that mapping doesnt exist inside the detail trace.
-        // we need to find the right step inside it to map it
-        // currently assuming test.step can only be in case playwright utils patched apis
-        if (currentFailedStep?.callId.includes("test.step")) {
-            currentFailedStep =
-                beforeActions.find((a) => a.parentId === currentFailedStep.callId) ||
-                    currentFailedStep;
-        }
-        // console.log("allscreenshots", allScreenshots.slice(0, 3));
-        const failedStepIndex = beforeActions.findIndex((a) => {
-            return a.callId === currentFailedStep.callId;
-        });
-        console.log("failedStepIndex", failedStepIndex);
-        console.log("beforeActions", beforeActions[failedStepIndex]);
-        let stepEndTime = allTestActions.find((action) => {
-            return (action.type === "after" && action.callId === currentFailedStep.callId);
-        }).endTime;
-        let lastActionCapturedInTestTrace = currentFailedStep;
-        let lastActionCapturedInDetailTrace = (stepIdToCallIdMap[currentFailedStep.callId] ||
-            wallTimeToCallIdMap[currentFailedStep.wallTime]);
-        if (!lastActionCapturedInDetailTrace) {
-            const actionsTakenTillLastFailedStepInReverseOrder = beforeActions
-                .slice(0, failedStepIndex)
-                .reverse();
-            const totalReversedActions = actionsTakenTillLastFailedStepInReverseOrder.length;
-            for (let actionIndex = 0; actionIndex < totalReversedActions; actionIndex++) {
-                const action = actionsTakenTillLastFailedStepInReverseOrder[actionIndex];
-                const correspondingDetailTraceAction = stepIdToCallIdMap[action.callId] ||
-                    wallTimeToCallIdMap[action.wallTime];
-                if (correspondingDetailTraceAction) {
-                    lastActionCapturedInTestTrace = action;
-                    lastActionCapturedInDetailTrace = correspondingDetailTraceAction;
-                }
-                if (lastActionCapturedInDetailTrace) {
-                    break;
-                }
-            }
-        }
-        const defaultWindowSize = 2000;
-        const pageIdForFailedStep = lastActionCapturedInDetailTrace.pageId;
-        const stepStartTime = isSuccessRun
-            ? lastActionCapturedInTestTrace.startTime - defaultWindowSize / 2
-            : lastActionCapturedInTestTrace.startTime;
-        const endTime = isSuccessRun
-            ? stepEndTime + defaultWindowSize
-            : stepEndTime;
-        // console.log("stepStartTime", stepStartTime);
-        // console.log("endTime", endTime);
-        const screenshotsForPage = allScreenshots.filter((screenshot) => {
-            return screenshot.pageId === pageIdForFailedStep;
-        });
-        allScreenshots.sort((a, b) => a.timestamp - b.timestamp);
-        const availableScreenshotTimestamps = screenshotsForPage.map((s) => s.timestamp);
-        let screenshotsToConsider = allScreenshots.filter((screenshot) => {
-            return (screenshot.pageId === pageIdForFailedStep &&
-                screenshot.timestamp <= endTime &&
-                screenshot.timestamp >= stepStartTime);
-        });
-        if (screenshotsToConsider.length === 0) {
-            const lastScreenshotTimestamp = availableScreenshotTimestamps[availableScreenshotTimestamps.length - 1];
-            // if the available screenshot timestamps are a lot less than the step start time
-            // take a window of last 2 seconds from the end of the screenshot timeline
-            if (lastScreenshotTimestamp < stepStartTime) {
-                screenshotsToConsider = allScreenshots.filter((screenshot) => {
-                    return (screenshot.pageId === pageIdForFailedStep &&
-                        screenshot.timestamp <= lastScreenshotTimestamp &&
-                        screenshot.timestamp >= lastScreenshotTimestamp - defaultWindowSize);
-                });
-            }
-            else {
-                // if the failed step is in the middle of screenshot timestamp but no corresponding image is found
-                // take a window of 4 seconds before and after the failed step
-                screenshotsToConsider = allScreenshots.filter((screenshot) => {
-                    return (screenshot.pageId === pageIdForFailedStep &&
-                        screenshot.timestamp <= endTime + defaultWindowSize / 2 &&
-                        screenshot.timestamp >= stepStartTime - defaultWindowSize / 2);
-                });
-            }
-        }
-        const imageBufferMap = {};
-        await (0, trace_utils_1.extractFileFromZip)({
-            zipUrl,
-            fileNames: screenshotsToConsider.map((s) => s.sha1),
-            chunkProcessor: (_, fileName) => {
-                imageBufferMap[fileName] = imageBufferMap[fileName] || [];
-                return (chunk) => {
-                    if (chunk) {
-                        imageBufferMap[fileName].push(chunk);
-                    }
-                };
-            },
-        });
-        const screenshotsWithBase64 = screenshotsToConsider.map((screenshot) => {
-            const imageBuffer = buffer_1.Buffer.concat(imageBufferMap[screenshot.sha1]);
-            const base64 = convertBufferToBase64(imageBuffer, "image/jpeg");
-            return {
-                metadata: { ...screenshot, base64 },
-                image: imageBuffer.toString("base64"),
-            };
-        });
-        // console.log("SCREENSHOT", screenshotsWithBase64.length);
-        const images = await (0, dedup_image_1.deduplicateImages)({
-            base64Images: screenshotsWithBase64,
-            threshold: 0.001,
-            logPrefix: `TestRun: ${testRunId}`,
-        });
-        // console.log("DEDUPED IMAGES", images.length);
-        return images.map((i) => i.metadata);
-    }
-    catch (e) {
-        console.warn(`Error extracting screenshots for testRunId - "${testRunId}", zipUrl - "${zipUrl}"`, e?.message);
-        return [];
-    }
-};
-exports.getScreenshotsInFailedStepWindow = getScreenshotsInFailedStepWindow;
-const generateFailedStepScreenshotDiffSummary = async ({ testRunId, test, modelConfig, trace, project, }) => {
-    if (!test.success?.trace && !test.failure?.trace) {
-        console.warn(`[testRunId - ${testRunId}][test - ${test.title}] No success and failure trace found for test, exiting early ...`);
-        return "";
-    }
-    if (!test.failure?.trace) {
-        console.warn(`[testRunId - ${testRunId}][test - ${test.title}] No failure trace found for test, exiting early ...`);
-        return "";
-    }
-    try {
-        const failedStep = await (0, exports.getFailedStep)({
-            testRunId,
-            zipUrl: test.failure.trace,
-        });
-        let failureScreenshots = await (0, exports.getScreenshotsInFailedStepWindow)({
-            failedStep,
-            testRunId,
-            zipUrl: test.failure.trace,
-        });
-        const totalFailureScreenshots = failureScreenshots?.length ?? 0;
-        console.log(`[testRunId - ${testRunId}][test - ${test.title}] has - "${totalFailureScreenshots}" failure screenshots`);
-        trace?.event({
-            name: "collected-failure-actions",
-            output: {
-                success: true,
-                data: { count: totalFailureScreenshots },
-            },
-        });
-        if (!totalFailureScreenshots) {
-            return "";
-        }
-        let successScreenshots = test.success.trace
-            ? await (0, exports.getScreenshotsInFailedStepWindow)({
-                failedStep,
-                testRunId,
-                zipUrl: test.success.trace,
-                isSuccessRun: true,
-            })
-            : [];
-        const totalSuccessScreenshots = successScreenshots?.length ?? 0;
-        console.log(`[testRunId - ${testRunId}][test - ${test.title}] has - "${totalSuccessScreenshots}" success screenshots`);
-        trace?.event({
-            name: "collected-success-actions",
-            output: {
-                success: true,
-                data: { count: totalSuccessScreenshots },
-            },
-        });
-        const totalScreenshots = totalSuccessScreenshots + totalFailureScreenshots;
-        const maxScreenshotsAllowed = 250;
-        if (totalScreenshots > maxScreenshotsAllowed) {
-            console.log(`[testRunId - ${testRunId}][testCaseId - ${test.testCaseId}] has - "${totalScreenshots}" screenshots, sampling to - "${maxScreenshotsAllowed}"`);
-            const sampledList = (0, frame_sampling_1.sampleCombinedList)(failureScreenshots, successScreenshots, maxScreenshotsAllowed);
-            failureScreenshots = sampledList.list1;
-            successScreenshots = sampledList.list2;
-        }
-        // waitUntil(
-        //   uploadScreenshotsToR2({
-        //     successScreenshots,
-        //     failureScreenshots,
-        //     testRunId,
-        //     project,
-        //     test,
-        //   }),
-        // );
-        const modelProvider = modelConfig?.provider ?? "openai";
-        const model = modelConfig?.model ?? "gpt-4o";
-        const temperature = modelConfig?.temperature ?? 0.5;
-        const llm = new llm_1.LLM({
-            provider: modelProvider,
-            defaultModel: model,
-        });
-        const generationTrace = trace?.generation({
-            name: "generate-trace-screenshot-diff-summary",
-            model,
-            modelParameters: {
-                temperature,
-            },
-        });
-        const totalImageCount = successScreenshots.length + failureScreenshots.length;
-        // If there are too many images LLM failes to respond within alloted timeout
-        const imageDetail = totalImageCount > 40 ? "low" : "high";
-        const messages = (0, llm_1.compilePrompt)("prompt", {
-            successScreenshots: successScreenshots.map((s) => s.base64),
-            failureScreenshots: failureScreenshots.map((s) => s.base64),
-            errorStack: test.failure.stack,
-        }, {
-            imageDetail,
-        });
-        const llmResponse = await llm.createChatCompletion({
-            messages,
-            trace: generationTrace,
-            modelParameters: {
-                temperature,
-            },
-            responseFormat,
-        });
-        const parsedResponse = JSON.parse(llmResponse?.content);
-        generationTrace?.end({
-            output: {
-                videoDiffSummary: JSON.stringify(parsedResponse),
-            },
-            usage: {
-                input: llm.promptTokens,
-                output: llm.completionTokens,
-                unit: "TOKENS",
-            },
-        });
-        return parsedResponse?.reason_for_test_failure ?? "";
-    }
-    catch (e) {
-        console.error(`[testRunId - ${testRunId}][testCaseId - ${test.testCaseId}] Error generating trace screenshot diff summary`, e);
-        return "";
-    }
-};
-exports.generateFailedStepScreenshotDiffSummary = generateFailedStepScreenshotDiffSummary;

package/dist/generate-summary/generate-grouped-summary.d.ts

@@ -1,18 +0,0 @@
-import { TraceClient } from "@empiricalrun/llm";
-import { z } from "zod";
-declare const groupedSummarySchema: z.ZodArray<z.ZodObject<{
-    testIds: z.ZodArray<z.ZodString>;
-    groupSummary: z.ZodString;
-}, z.core.$strip>>;
-export type GroupedSummarySchemaType = z.infer<typeof groupedSummarySchema>;
-type ArgsT = {
-    testRunId: number;
-    tests: {
-        testId: string;
-        errorSummary: string;
-    }[];
-    trace?: TraceClient;
-};
-export declare const generateGroupedSummary: ({ testRunId, tests, trace, }: ArgsT) => Promise<GroupedSummarySchemaType | null>;
-export {};
-//# sourceMappingURL=generate-grouped-summary.d.ts.map

package/dist/generate-summary/generate-grouped-summary.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"generate-grouped-summary.d.ts","sourceRoot":"","sources":["../../src/generate-summary/generate-grouped-summary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,QAAA,MAAM,oBAAoB;;;kBAKzB,CAAC;AACF,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AA0C5E,KAAK,KAAK,GAAG;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAClD,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,CAAC;AACF,eAAO,MAAM,sBAAsB,GAAU,8BAI1C,KAAK,KAAG,OAAO,CAAC,wBAAwB,GAAG,IAAI,CAgDjD,CAAC"}

package/dist/generate-summary/generate-grouped-summary.js

@@ -1,88 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateGroupedSummary = void 0;
-const llm_1 = require("@empiricalrun/llm");
-const zod_1 = require("zod");
-const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software engineer, who is given a task is to summarize test failures\n\nThe test failures will be provided to you in this format\n```\n[\n  {\n    testId: string, // this is a unique id of the test\n    errorSummary: string, // this is the error summary of the test failure\n  }\n]\n```\n\nThe errorSummary for each test is in this format:\n```\n<reason for failure of the test>\n\n<steps taken in the tests before it failed>\n```\nYou need to group the tests based on the test failure reason. steps taken can be ignored\n\nYour output should be of the following format:\n```\n{\n  \"groups\": [\n    // Group 1\n    {\n      testIds: string[] // testIds grouped with similar errors\n      groupSummary: string // a single error summary of all the grouped tests\n    },\n    // Group 2\n    {\n      testIds: string[]\n      groupSummary: string\n    },\n      ... and so on\n  ]\n}\n```\n\nThe group summary string should follow this format:\n```\n<common reason for failure of the tests>\n\n<common steps or pattern of steps taken in the tests before it failed>\n```\n\nFollow these instructions before responding with output:\n- Do not respond with backticks\n- The output should be a valid JSON object\n- Do not group tests which have different error summary\n- Do not drop useful information about unique elements, locators, status code, error messages in the summary\n- Avoid repetitions in the group summary content\n- Timeout difference in errorSummary can be ignored during grouping\n- If a test is grouped alone, just send its errorSummary as groupSummary. Do not make changes in this case\n{{/section}}\n\n{{#section \"user\"}}\nFollowing are the test failures which needs to be summarized:\n{{testList}}\n{{/section}}\n";
-const groupedSummarySchema = zod_1.z.array(zod_1.z.object({
-    testIds: zod_1.z.array(zod_1.z.string()),
-    groupSummary: zod_1.z.string(),
-}));
-const getResponseFormat = (availableTestIds) => ({
-    type: "json_schema",
-    json_schema: {
-        name: "test-case-failure-grouped-summary",
-        strict: true,
-        schema: {
-            type: "object",
-            properties: {
-                groups: {
-                    type: "array",
-                    description: "array containing tests grouped by similar error summaries",
-                    items: {
-                        type: "object",
-                        properties: {
-                            testIds: {
-                                type: "array",
-                                description: "array of test ids which are grouped together",
-                                items: {
-                                    type: "string",
-                                    description: "Unique identifier for the tests, provided by the user",
-                                    enum: availableTestIds,
-                                },
-                            },
-                            groupSummary: {
-                                type: "string",
-                                description: "Summary of the grouped tests",
-                            },
-                        },
-                        required: ["testIds", "groupSummary"],
-                        additionalProperties: false,
-                    },
-                },
-            },
-            required: ["groups"],
-            additionalProperties: false,
-        },
-    },
-});
-const generateGroupedSummary = async ({ testRunId, tests, trace, }) => {
-    let output = null;
-    const availableTestIds = tests.map((t) => t.testId);
-    try {
-        const messages = (0, llm_1.compilePrompt)(promptTemplate_0, {
-            testList: JSON.stringify(tests),
-        });
-        trace?.event({
-            input: {
-                testList: JSON.stringify(tests),
-            },
-            output: { messages },
-        });
-        const llm = new llm_1.LLM({
-            trace,
-            provider: "openai",
-            defaultModel: "gpt-4o",
-        });
-        const llmResponse = await llm.createChatCompletion({
-            messages,
-            trace,
-            modelParameters: {
-                temperature: 0.2,
-            },
-            responseFormat: getResponseFormat(availableTestIds),
-        });
-        output = groupedSummarySchema.parse(JSON.parse(llmResponse?.content).groups);
-    }
-    catch (e) {
-        console.error(`Error grouping tests, returning all tests without groups for testRunId - "${testRunId}"`, e?.message);
-        output = tests.map((test) => {
-            return {
-                testIds: [test.testId],
-                groupSummary: test.errorSummary,
-            };
-        });
-    }
-    return output;
-};
-exports.generateGroupedSummary = generateGroupedSummary;

package/dist/generate-summary/merge-summary.d.ts

@@ -1,16 +0,0 @@
-import { TraceClient } from "@empiricalrun/llm";
-import { TestGroup } from "@empiricalrun/shared-types/failure-workflow";
-type ArgsT = {
-    testRunId: number;
-    test: TestGroup;
-    errorSummary: string;
-    videoDiffSummary: string;
-    networkFailures: {
-        endpoint: string;
-        status: number;
-    }[];
-    trace?: TraceClient;
-};
-export declare const mergeSummary: ({ testRunId, test, errorSummary: errorStackSummary, networkFailures, videoDiffSummary, trace, }: ArgsT) => Promise<string>;
-export {};
-//# sourceMappingURL=merge-summary.d.ts.map

package/dist/generate-summary/merge-summary.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"merge-summary.d.ts","sourceRoot":"","sources":["../../src/generate-summary/merge-summary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,SAAS,EAAE,MAAM,6CAA6C,CAAC;AAGxE,KAAK,KAAK,GAAG;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,SAAS,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACxD,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,CAAC;AACF,eAAO,MAAM,YAAY,GAAU,iGAOhC,KAAK,KAAG,OAAO,CAAC,MAAM,CA2CxB,CAAC"}