@empiricalrun/test-gen 0.75.0 → 0.76.0

package/dist/tools/definitions/fetch-video-analysis.d.ts

@@ -9,29 +9,29 @@ export declare const videoAnalysisSchema: z.ZodObject<{
         featureFlag: z.ZodOptional<z.ZodDefault<z.ZodEnum<["none", "send-all-frames", "separate-threshold", "ai-based-frames"]>>>;
     }, "strip", z.ZodTypeAny, {
         model?: string | undefined;
-        fps?: number | undefined;
         threshold?: number | undefined;
+        fps?: number | undefined;
         featureFlag?: "none" | "send-all-frames" | "separate-threshold" | "ai-based-frames" | undefined;
     }, {
         model?: string | undefined;
-        fps?: number | undefined;
         threshold?: number | undefined;
+        fps?: number | undefined;
         featureFlag?: "none" | "send-all-frames" | "separate-threshold" | "ai-based-frames" | undefined;
     }>>;
 }, "strip", z.ZodTypeAny, {
     videoUrl: string;
     params?: {
         model?: string | undefined;
-        fps?: number | undefined;
         threshold?: number | undefined;
+        fps?: number | undefined;
         featureFlag?: "none" | "send-all-frames" | "separate-threshold" | "ai-based-frames" | undefined;
     } | undefined;
 }, {
     videoUrl: string;
     params?: {
         model?: string | undefined;
-        fps?: number | undefined;
         threshold?: number | undefined;
+        fps?: number | undefined;
         featureFlag?: "none" | "send-all-frames" | "separate-threshold" | "ai-based-frames" | undefined;
     } | undefined;
 }>;

package/dist/tools/executor/index.d.ts

@@ -1,6 +1,6 @@
 import { TraceClient } from "@empiricalrun/llm";
 import { PendingToolCall, ToolResult } from "@empiricalrun/llm/chat";
-import { ChatSessionInfo, IDashboardAPIClient, ToolExecute } from "@empiricalrun/shared-types";
+import type { ChatSessionInfo, IDashboardAPIClient, ToolExecute } from "@empiricalrun/shared-types";
 export declare class ToolExecutor {
     toolExecutors: Record<string, ToolExecute>;
     chatSession: ChatSessionInfo | null;

package/dist/tools/executor/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/executor/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACrE,OAAO,EAEL,eAAe,EACf,mBAAmB,EACnB,WAAW,EACZ,MAAM,4BAA4B,CAAC;AA0BpC,qBAAa,YAAY;IACvB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAM;IAChD,WAAW,EAAE,eAAe,GAAG,IAAI,CAAC;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,mBAAmB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAEjC,EACV,WAAW,EACX,UAAU,EACV,QAAQ,EACR,SAAS,EACT,KAAK,EACL,YAAY,EACZ,oBAAyB,GAC1B,EAAE;QACD,WAAW,EAAE,eAAe,GAAG,IAAI,CAAC;QACpC,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,mBAAmB,CAAC;QAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;QACpB,YAAY,EAAE,MAAM,EAAE,CAAC;QACvB,oBAAoB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC/C;IA8CK,OAAO,CAAC,SAAS,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;CAoEnE"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/executor/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACrE,OAAO,KAAK,EAEV,eAAe,EACf,mBAAmB,EACnB,WAAW,EACZ,MAAM,4BAA4B,CAAC;AA8BpC,qBAAa,YAAY;IACvB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAM;IAChD,WAAW,EAAE,eAAe,GAAG,IAAI,CAAC;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,mBAAmB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAEjC,EACV,WAAW,EACX,UAAU,EACV,QAAQ,EACR,SAAS,EACT,KAAK,EACL,YAAY,EACZ,oBAAyB,GAC1B,EAAE;QACD,WAAW,EAAE,eAAe,GAAG,IAAI,CAAC;QACpC,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,mBAAmB,CAAC;QAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;QACpB,YAAY,EAAE,MAAM,EAAE,CAAC;QACvB,oBAAoB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC/C;IAiDK,OAAO,CAAC,SAAS,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;CA8EnE"}

package/dist/tools/executor/index.js

@@ -2,12 +2,15 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ToolExecutor = void 0;
 const artifacts_1 = require("../../artifacts");
-const commit_and_create_pr_1 = require("../commit-and-create-pr");
+const file_1 = require("../../utils/file");
+const create_pull_request_1 = require("../create-pull-request");
 const delete_file_1 = require("../delete-file");
 const diagnosis_fetcher_1 = require("../diagnosis-fetcher");
 const download_build_1 = require("../download-build");
+const extract_frames_from_video_1 = require("../extract-frames-from-video");
 const fetch_file_1 = require("../fetch-file");
 const fetch_last_successful_test_run_1 = require("../fetch-last-successful-test-run");
+const fetch_session_diff_1 = require("../fetch-session-diff");
 const fetch_video_analysis_1 = require("../fetch-video-analysis");
 const file_operations_1 = require("../file-operations");
 const grep_1 = require("../grep");
@@ -16,6 +19,7 @@ const list_environments_1 = require("../list-environments");
 const list_tests_and_projects_1 = require("../list-tests-and-projects");
 const merge_conflicts_1 = require("../merge-conflicts");
 const rename_file_1 = require("../rename-file");
+const review_pull_request_1 = require("../review-pull-request");
 const run_test_1 = require("../run-test");
 const test_gen_browser_1 = require("../test-gen-browser");
 const test_run_fetcher_1 = require("../test-run-fetcher");
@@ -48,7 +52,7 @@ class ToolExecutor {
             test_run_fetcher_1.fetchTestRunReportTool,
             diagnosis_fetcher_1.fetchDiagnosisReportTool,
             test_gen_browser_1.generateTestWithBrowserAgent,
-            commit_and_create_pr_1.createPullRequestTool,
+            create_pull_request_1.createPullRequestTool,
             merge_conflicts_1.mergeConflictsTool,
             list_environments_1.listEnvironmentsTool,
             issues_1.listIssuesTool,
@@ -65,6 +69,9 @@ class ToolExecutor {
             fetch_last_successful_test_run_1.fetchLastSuccessfulTestRunTool,
             triage_summary_1.sendTriageSummaryTool,
             list_tests_and_projects_1.listProjectsTool,
+            extract_frames_from_video_1.extractFramesFromVideo,
+            fetch_session_diff_1.fetchSessionDiffTool,
+            review_pull_request_1.reviewPullRequestTool,
             ...file_operations_1.textEditorTools,
         ];
         tools.forEach((tool) => {
@@ -89,8 +96,15 @@ class ToolExecutor {
                 input: toolCall.input,
             });
             const uploadArtifactsQueue = new artifacts_1.UploadArtifactsQueue(this.repoPath, toolCall.id);
-            const collectArtifactsFn = (artifactsInput) => {
-                uploadArtifactsQueue.addTask(artifactsInput).catch((error) => {
+            const collectArtifactsFn = (artifactsInput, cleanUpDir) => {
+                uploadArtifactsQueue
+                    .addTask(artifactsInput)
+                    .then(() => {
+                    if (cleanUpDir) {
+                        void (0, file_1.safeCleanupDirectory)(cleanUpDir);
+                    }
+                })
+                    .catch((error) => {
                     console.error("Error collecting artifacts:", error);
                 });
             };

package/dist/tools/extract-frames-from-video/index.d.ts

@@ -0,0 +1,7 @@
+import type { Tool } from "@empiricalrun/shared-types";
+import type { z } from "zod";
+import { type extractFramesFromVideoSchema } from "../definitions/extract-frames-from-video";
+type ExtractFramesInput = z.infer<typeof extractFramesFromVideoSchema>;
+export declare const extractFramesFromVideo: Tool<ExtractFramesInput>;
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/tools/extract-frames-from-video/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/extract-frames-from-video/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAEV,IAAI,EAEL,MAAM,4BAA4B,CAAC;AAEpC,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAI7B,OAAO,EAEL,KAAK,4BAA4B,EAClC,MAAM,0CAA0C,CAAC;AAElD,KAAK,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AA0EvE,eAAO,MAAM,sBAAsB,EAAE,IAAI,CAAC,kBAAkB,CAkH3D,CAAC"}

package/dist/tools/extract-frames-from-video/index.js

@@ -0,0 +1,145 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractFramesFromVideo = void 0;
+const node_crypto_1 = __importDefault(require("node:crypto"));
+const fs_1 = require("fs");
+const file_1 = require("../../utils/file");
+const local_ffmpeg_client_1 = require("../../utils/local-ffmpeg-client");
+const extract_frames_from_video_1 = require("../definitions/extract-frames-from-video");
+function parseTimeToSeconds(timeString) {
+    const timeRegex = /^(\d{1,2}):(\d{2})$/;
+    const match = timeString.match(timeRegex);
+    if (!match || match.length < 3) {
+        throw new Error(`Invalid time format: ${timeString}. Expected MM:SS format (e.g., "01:32")`);
+    }
+    const minutesStr = match[1];
+    const secondsStr = match[2];
+    if (!minutesStr || !secondsStr) {
+        throw new Error(`Invalid time format: ${timeString}. Expected MM:SS format (e.g., "01:32")`);
+    }
+    const minutes = parseInt(minutesStr, 10);
+    const seconds = parseInt(secondsStr, 10);
+    if (seconds >= 60) {
+        throw new Error(`Invalid seconds: ${seconds}. Seconds must be 0-59`);
+    }
+    return minutes * 60 + seconds;
+}
+function getExtractFramesParams(params) {
+    return {
+        fps: params?.fps ?? 30,
+        threshold: params?.threshold ?? 0.001,
+        startTime: params?.startTime
+            ? parseTimeToSeconds(params.startTime)
+            : undefined,
+        duration: params?.duration
+            ? parseTimeToSeconds(params.duration)
+            : undefined,
+    };
+}
+function hashExtractParams(videoUrl, params) {
+    const sortedParams = Object.keys(params)
+        .sort()
+        .reduce((acc, key) => {
+        acc[key] = params[key];
+        return acc;
+    }, {});
+    const json = JSON.stringify({ videoUrl, ...sortedParams });
+    return node_crypto_1.default
+        .createHash("sha256")
+        .update(json)
+        .digest("hex")
+        .substring(0, 16);
+}
+exports.extractFramesFromVideo = {
+    ...extract_frames_from_video_1.extractFramesFromVideo,
+    execute: async ({ input, trace, }) => {
+        const { videoUrl } = input;
+        const params = getExtractFramesParams(input.params);
+        const videoUrlHash = hashExtractParams(videoUrl, params);
+        const WORKING_DIR = `./extract-frames-artifacts/${videoUrlHash}`;
+        // const R2_BASE_URL = `https://video-analysis.empirical.run/${videoUrlHash}`;
+        const extractionSpan = trace?.span({
+            name: "extract-frames-from-video",
+            input: { videoUrl, params },
+        });
+        try {
+            const response = await fetch(videoUrl, { method: "HEAD" });
+            if (!response.ok) {
+                return {
+                    result: `Failed to access video: ${response.statusText}`,
+                    isError: true,
+                };
+            }
+            const ffmpegClient = new local_ffmpeg_client_1.LocalFFmpegClient();
+            const extractionResult = await ffmpegClient.extractVideoFrames(videoUrl, WORKING_DIR, {
+                fps: params.fps,
+                threshold: params.threshold,
+                startTime: params.startTime,
+                duration: params.duration,
+            });
+            const { totalFramesCount, uniqueFrames, videoDurationSeconds } = extractionResult;
+            // Create response metadata
+            const extractFramesResponse = {
+                videoUrl,
+                videoDurationSeconds,
+                totalFramesCount,
+                extractedFramesCount: uniqueFrames.length,
+                timeRange: params.startTime !== undefined || params.duration !== undefined
+                    ? {
+                        startTime: params.startTime || 0,
+                        duration: params.duration || 0,
+                        endTime: (params.startTime || 0) + (params.duration || 0),
+                    }
+                    : undefined,
+                frames: uniqueFrames.map((frame) => ({
+                    index: frame.metadata.index,
+                    path: frame.metadata.path,
+                    frameId: `frame_${frame.metadata.index}`,
+                })),
+            };
+            // Convert frames to base64 images for multimodal response
+            const frameImageParts = await Promise.all(uniqueFrames.map(async (frame) => {
+                try {
+                    const imageBuffer = await fs_1.promises.readFile(frame.metadata.path);
+                    const base64Data = imageBuffer.toString("base64");
+                    return {
+                        type: "image/png",
+                        base64Data: base64Data,
+                    };
+                }
+                catch (error) {
+                    console.warn(`Failed to read frame ${frame.metadata.path}:`, error);
+                    return null;
+                }
+            }));
+            await (0, file_1.safeCleanupDirectory)(WORKING_DIR, "extract-frames-tool");
+            // Filter out any failed frame reads
+            const validFrameImageParts = frameImageParts.filter((part) => part !== null);
+            return {
+                result: [
+                    {
+                        type: "text",
+                        text: JSON.stringify(extractFramesResponse, null, 2),
+                    },
+                    ...validFrameImageParts,
+                ],
+                isError: false,
+            };
+        }
+        catch (error) {
+            console.error("Error in extractFramesFromVideo", error);
+            extractionSpan?.end();
+            const message = error instanceof Error ? error.message : String(error);
+            return {
+                result: `Error extracting frames from video: ${message}`,
+                isError: true,
+            };
+        }
+        finally {
+            extractionSpan?.end();
+        }
+    },
+};

package/dist/tools/fetch-session-diff/index.d.ts

@@ -0,0 +1,3 @@
+import { Tool } from "@empiricalrun/shared-types";
+export declare const fetchSessionDiffTool: Tool;
+//# sourceMappingURL=index.d.ts.map

package/dist/tools/fetch-session-diff/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/fetch-session-diff/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAc,MAAM,4BAA4B,CAAC;AAO9D,eAAO,MAAM,oBAAoB,EAAE,IA0ClC,CAAC"}

package/dist/tools/fetch-session-diff/index.js

@@ -0,0 +1,46 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchSessionDiffTool = void 0;
+const zod_1 = require("zod");
+const SessionDiffInputSchema = zod_1.z.object({
+    url: zod_1.z.string().describe(`The URL of the chat session.`),
+});
+exports.fetchSessionDiffTool = {
+    schema: {
+        name: "fetchSessionDiff",
+        description: `Extracts the session url and returns the code diff for the session.`,
+        parameters: SessionDiffInputSchema,
+    },
+    needsBrowser: false,
+    isInlineTool: true,
+    execute: async ({ input, apiClient }) => {
+        try {
+            const sessionUrl = input.url;
+            if (!sessionUrl) {
+                return {
+                    isError: true,
+                    result: "No session URL provided",
+                };
+            }
+            if (!apiClient) {
+                throw new Error("Dashboard API client is not available.");
+            }
+            const sessionId = sessionUrl.split("/").pop();
+            const sessionDiffApiUrl = `/api/chat-sessions/${sessionId}/diff`;
+            const sessionDiff = await apiClient.request(sessionDiffApiUrl, {
+                method: "GET",
+            });
+            return {
+                isError: false,
+                result: JSON.stringify(sessionDiff, null, 2),
+            };
+        }
+        catch (error) {
+            console.error("Error fetching session diff", error);
+            return {
+                isError: true,
+                result: error instanceof Error ? error.message : String(error),
+            };
+        }
+    },
+};

package/dist/tools/fetch-video-analysis/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/fetch-video-analysis/index.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAEV,IAAI,EAIL,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,qCAAqC,CAAC;AAwC7C,eAAO,MAAM,kBAAkB,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAoJxE,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/fetch-video-analysis/index.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAEV,IAAI,EAKL,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAI7B,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,qCAAqC,CAAC;AAuC7C,eAAO,MAAM,kBAAkB,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAsKxE,CAAC"}

package/dist/tools/fetch-video-analysis/index.js

@@ -6,8 +6,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.fetchVideoAnalysis = void 0;
 const node_crypto_1 = __importDefault(require("node:crypto"));
 const node_path_1 = __importDefault(require("node:path"));
+const file_1 = require("../../utils/file");
+const local_ffmpeg_client_1 = require("../../utils/local-ffmpeg-client");
 const fetch_video_analysis_1 = require("../definitions/fetch-video-analysis");
-const local_ffmpeg_client_1 = require("./local-ffmpeg-client");
 const utils_1 = require("./utils");
 const video_analysis_1 = require("./video-analysis");
 function getVideoAnalysisParams(params) {
@@ -34,7 +35,7 @@ function hashObject(obj) {
 }
 exports.fetchVideoAnalysis = {
     ...fetch_video_analysis_1.fetchVideoAnalysis,
-    execute: async ({ input, trace, }) => {
+    execute: async ({ input, trace, featureFlags, }) => {
         const { videoUrl } = input;
         const params = getVideoAnalysisParams(input.params);
         const videoUrlHash = hashObject({
@@ -82,8 +83,15 @@ exports.fetchVideoAnalysis = {
                 const zipUploadPromise = (0, utils_1.zipAndUploadFramesToR2)(uniqueFrames, outputZipPath, videoUrlHash).catch((error) => {
                     throw error; // Re-throw to maintain error in Promise.all
                 });
-                const { analysis: llmAnalysis, usage } = await (0, video_analysis_1.analyzeFramesWithLLM)(uniqueFrames.map((frame) => frame.image), videoAnalysisSpan, selectedModel);
+                const { analysis: llmAnalysis, usage } = await (0, video_analysis_1.analyzeFramesWithLLM)(uniqueFrames, videoAnalysisSpan, selectedModel);
                 console.log(`[video-analysis] Finished Analyzing ${uniqueFrames.length} frames with LLM`);
+                let finalAnalysis = llmAnalysis;
+                let finalKeyFramesData = [];
+                if (featureFlags.includes("enableFetchVideoAnalysisToolMultiModal")) {
+                    const { updatedAnalysis, keyFramesData } = await (0, utils_1.getUpdatedAnalysisAndToolResultImagePart)(llmAnalysis, uniqueFrames);
+                    finalAnalysis = updatedAnalysis;
+                    finalKeyFramesData = [...keyFramesData];
+                }
                 const videoInfo = {
                     total_frames_count: totalFramesCount,
                     unique_frames_count: uniqueFrames.length,
@@ -99,20 +107,23 @@ exports.fetchVideoAnalysis = {
                     usage,
                     langfuse_trace_id: trace?.id || undefined,
                     frames_zip_url: `${R2_BASE_URL}frames.zip`,
-                    analysis: llmAnalysis,
+                    analysis: finalAnalysis,
                 };
                 await Promise.all([
                     zipUploadPromise,
                     (0, utils_1.uploadSummaryToR2)(videoInfo, R2_BASE_URL),
                 ]);
-                await (0, utils_1.safeCleanupDirectory)(WORKING_DIR, "video-analysis-cleanup");
+                await (0, file_1.safeCleanupDirectory)(WORKING_DIR, "video-analysis-cleanup");
                 const toolResult = {
                     video_url: videoUrl,
-                    analysis: llmAnalysis,
                     analysis_id: videoUrlHash,
+                    analysis: finalAnalysis,
                 };
                 return {
-                    result: JSON.stringify(toolResult, null, 2),
+                    result: [
+                        { type: "text", text: JSON.stringify(toolResult, null, 2) },
+                        ...finalKeyFramesData,
+                    ],
                     isError: false,
                     usage,
                 };

package/dist/tools/fetch-video-analysis/utils.d.ts

@@ -1,6 +1,5 @@
 import { type FileMap } from "@empiricalrun/r2-uploader";
-import { UniqueFrameInfo, UniqueFrameWithMetadata, VideoAnalysisSummary } from "@empiricalrun/shared-types";
-export declare const safeCleanupDirectory: (dirPath: string, label?: string) => Promise<void>;
+import { ToolResultImagePart, UniqueFrameInfo, UniqueFrameWithMetadata, VideoAnalysisSummary } from "@empiricalrun/shared-types";
 export declare const uploadFramesToR2: (videoInfo: Omit<VideoAnalysisSummary, "analysis">, frames: {
     metadata: {
         index: number;
@@ -10,4 +9,8 @@ export declare const uploadFramesToR2: (videoInfo: Omit<VideoAnalysisSummary, "a
 }[], r2BaseUrl: string) => Promise<UniqueFrameInfo[]>;
 export declare const uploadSummaryToR2: (videoInfo: VideoAnalysisSummary, r2BaseUrl: string) => Promise<string>;
 export declare const zipAndUploadFramesToR2: (uniqueFrames: UniqueFrameWithMetadata[], outputZipPath: string, videoUrlHash: string) => Promise<FileMap>;
+export declare const getUpdatedAnalysisAndToolResultImagePart: (llmAnalysis: string, uniqueFrames: UniqueFrameWithMetadata[]) => Promise<{
+    updatedAnalysis: string;
+    keyFramesData: ToolResultImagePart[];
+}>;
 //# sourceMappingURL=utils.d.ts.map

package/dist/tools/fetch-video-analysis/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/tools/fetch-video-analysis/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,OAAO,EAGb,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,eAAe,EACf,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAKpC,eAAO,MAAM,oBAAoB,GAC/B,SAAS,MAAM,EACf,cAAiB,KAChB,OAAO,CAAC,IAAI,CAUd,CAAC;AAoCF,eAAO,MAAM,gBAAgB,GAC3B,WAAW,IAAI,CAAC,oBAAoB,EAAE,UAAU,CAAC,EACjD,QAAQ;IACN,QAAQ,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1C,KAAK,EAAE,MAAM,CAAC;CACf,EAAE,EACH,WAAW,MAAM,KAChB,OAAO,CAAC,eAAe,EAAE,CAqB3B,CAAC;AAEF,eAAO,MAAM,iBAAiB,GAC5B,WAAW,oBAAoB,EAC/B,WAAW,MAAM,KAChB,OAAO,CAAC,MAAM,CAsBhB,CAAC;AAEF,eAAO,MAAM,sBAAsB,GACjC,cAAc,uBAAuB,EAAE,EACvC,eAAe,MAAM,EACrB,cAAc,MAAM,KACnB,OAAO,CAAC,OAAO,CAsBjB,CAAC"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/tools/fetch-video-analysis/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,OAAO,EAGb,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAuCpC,eAAO,MAAM,gBAAgB,GAC3B,WAAW,IAAI,CAAC,oBAAoB,EAAE,UAAU,CAAC,EACjD,QAAQ;IACN,QAAQ,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1C,KAAK,EAAE,MAAM,CAAC;CACf,EAAE,EACH,WAAW,MAAM,KAChB,OAAO,CAAC,eAAe,EAAE,CAqB3B,CAAC;AAEF,eAAO,MAAM,iBAAiB,GAC5B,WAAW,oBAAoB,EAC/B,WAAW,MAAM,KAChB,OAAO,CAAC,MAAM,CAsBhB,CAAC;AAEF,eAAO,MAAM,sBAAsB,GACjC,cAAc,uBAAuB,EAAE,EACvC,eAAe,MAAM,EACrB,cAAc,MAAM,KACnB,OAAO,CAAC,OAAO,CAsBjB,CAAC;AAEF,eAAO,MAAM,wCAAwC,GACnD,aAAa,MAAM,EACnB,cAAc,uBAAuB,EAAE,KACtC,OAAO,CAAC;IACT,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,mBAAmB,EAAE,CAAC;CACtC,CA2CA,CAAC"}

package/dist/tools/fetch-video-analysis/utils.js

@@ -3,21 +3,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.zipAndUploadFramesToR2 = exports.uploadSummaryToR2 = exports.uploadFramesToR2 = exports.safeCleanupDirectory = void 0;
+exports.getUpdatedAnalysisAndToolResultImagePart = exports.zipAndUploadFramesToR2 = exports.uploadSummaryToR2 = exports.uploadFramesToR2 = void 0;
 const r2_uploader_1 = require("@empiricalrun/r2-uploader");
 const child_process_1 = require("child_process");
 const fs_1 = require("fs");
 const path_1 = __importDefault(require("path"));
-const safeCleanupDirectory = async (dirPath, label = "cleanup") => {
-    try {
-        console.log(`[${label}] Cleaning up directory: ${dirPath}`);
-        await fs_1.promises.rm(dirPath, { recursive: true });
-    }
-    catch (error) {
-        console.warn(`[${label}] Failed to cleanup directory ${dirPath}:`, error instanceof Error ? error.message : String(error));
-    }
-};
-exports.safeCleanupDirectory = safeCleanupDirectory;
 const uploadFilesToR2 = async (files, videoUrlHash) => {
     const label = `video-analysis-upload:${videoUrlHash}`;
     const start = Date.now();
@@ -96,3 +86,36 @@ const zipAndUploadFramesToR2 = async (uniqueFrames, outputZipPath, videoUrlHash)
     });
 };
 exports.zipAndUploadFramesToR2 = zipAndUploadFramesToR2;
+const getUpdatedAnalysisAndToolResultImagePart = async (llmAnalysis, uniqueFrames) => {
+    const keyFramesMatch = llmAnalysis.match(/KEY FRAMES:\s*(.+)$/);
+    const updatedAnalysis = keyFramesMatch
+        ? llmAnalysis.replace(/\n\nKEY FRAMES:\s*(.+)$/, "")
+        : llmAnalysis;
+    const keyFrameIndices = keyFramesMatch?.[1]
+        ?.match(/<frame_(\d+)>/g)
+        ?.map((match) => parseInt(match.match(/<frame_(\d+)>/)?.[1] || "0", 10)) || [];
+    const frameDataMap = new Map();
+    await Promise.all(uniqueFrames.map(async (frame) => {
+        try {
+            const fileBuffer = await fs_1.promises.readFile(frame.metadata.path);
+            const base64Data = fileBuffer.toString("base64");
+            frameDataMap.set(frame.metadata.index, base64Data);
+        }
+        catch (error) {
+            console.error(`Failed to read frame file ${frame.metadata.path}:`, error);
+        }
+    }));
+    const keyFramesData = keyFrameIndices
+        .map((frameIndex) => {
+        const base64Data = frameDataMap.get(frameIndex);
+        return base64Data
+            ? {
+                type: "image/png",
+                base64Data: base64Data,
+            }
+            : null;
+    })
+        .filter(Boolean);
+    return { updatedAnalysis, keyFramesData };
+};
+exports.getUpdatedAnalysisAndToolResultImagePart = getUpdatedAnalysisAndToolResultImagePart;

package/dist/tools/fetch-video-analysis/video-analysis.d.ts

@@ -1,6 +1,6 @@
 import { TraceClient } from "@empiricalrun/llm";
-import { SupportedChatModels, Usage } from "@empiricalrun/shared-types";
-export declare function analyzeFramesWithLLM(frameBase64Data: string[], trace?: TraceClient, selectedModel?: SupportedChatModels, apiKey?: string): Promise<{
+import { SupportedChatModels, UniqueFrameWithMetadata, Usage } from "@empiricalrun/shared-types";
+export declare function analyzeFramesWithLLM(uniqueFrames: UniqueFrameWithMetadata[], trace?: TraceClient, selectedModel?: SupportedChatModels, apiKey?: string): Promise<{
     analysis: string;
     usage: Usage;
 }>;

package/dist/tools/fetch-video-analysis/video-analysis.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"video-analysis.d.ts","sourceRoot":"","sources":["../../../src/tools/fetch-video-analysis/video-analysis.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAEL,mBAAmB,EACnB,KAAK,EACN,MAAM,4BAA4B,CAAC;AAIpC,wBAAsB,oBAAoB,CACxC,eAAe,EAAE,MAAM,EAAE,EACzB,KAAK,CAAC,EAAE,WAAW,EACnB,aAAa,CAAC,EAAE,mBAAmB,EACnC,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,KAAK,CAAA;CAAE,CAAC,CAgE7C"}
+{"version":3,"file":"video-analysis.d.ts","sourceRoot":"","sources":["../../../src/tools/fetch-video-analysis/video-analysis.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAEL,mBAAmB,EACnB,uBAAuB,EACvB,KAAK,EACN,MAAM,4BAA4B,CAAC;AAIpC,wBAAsB,oBAAoB,CACxC,YAAY,EAAE,uBAAuB,EAAE,EACvC,KAAK,CAAC,EAAE,WAAW,EACnB,aAAa,CAAC,EAAE,mBAAmB,EACnC,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,KAAK,CAAA;CAAE,CAAC,CA8E7C"}

package/dist/tools/fetch-video-analysis/video-analysis.js

@@ -3,32 +3,48 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.analyzeFramesWithLLM = analyzeFramesWithLLM;
 const chat_1 = require("@empiricalrun/llm/chat");
 const models_1 = require("../../agent/chat/models");
-async function analyzeFramesWithLLM(frameBase64Data, trace, selectedModel, apiKey) {
+async function analyzeFramesWithLLM(uniqueFrames, trace, selectedModel, apiKey) {
     const llmSpan = trace?.span({
         name: "llm-frame-analysis",
-        input: { frameCount: frameBase64Data.length },
+        input: { frameCount: uniqueFrames.length },
     });
     const selectedChatModel = selectedModel || (0, models_1.getDefaultChatModelId)();
     try {
         // TODO: Move to canonical chat model
         let chatModel = new chat_1.GeminiChatModel(selectedChatModel, [], apiKey || process.env.GOOGLE_API_KEY);
         chatModel.validateEnvVarsForAuth();
-        const frameAttachments = frameBase64Data.map((frameBase64, index) => ({
-            url: frameBase64,
+        const frameAttachments = uniqueFrames.map((data) => ({
+            url: data.image,
             contentType: "image/png",
-            name: `frame-${index}.png`,
+            name: `frame_${data.metadata.index}.png`,
         }));
         const userMessage = `
-Analyze the ${frameBase64Data.length} frames that are extracted from a screen recording.`;
+Analyze the ${uniqueFrames.length} frames that are extracted from a screen recording.`;
         chatModel.pushUserMessage(userMessage, frameAttachments);
         const systemPrompt = `
-You are given a set of unique frames from a screen recording. Your job is to return a verbose bullet list of what is going on in the video. Don't miss out on anything.
+You are given a set of unique frames from a screen recording. Your job is to provide a comprehensive analysis of the video content.
+
+REQUIREMENTS:
+1. Provide a detailed verbose bullet list of everything happening in the video
+2. Analyze each frame thoroughly - do not skip any important details
+3. Include UI elements, text content, user interactions, state changes, and visual elements
+4. Be extremely thorough and capture every significant action or change
+
+OUTPUT FORMAT:
+- Start with detailed bullet points of your analysis
+- End your response with a dedicated "KEY FRAMES:" section
+- List the most important frame IDs in this exact format: KEY FRAMES: <frame_0>, <frame_5>, <frame_12>
+- The key frames section MUST be the last part of your response
+
+STRICT FORMATTING:
+- Use bullet points for the analysis
+- Key frames must be listed at the very end using the exact format specified above
+- Do not include any text after the key frames list
 `;
         // Get LLM response
         const response = await chatModel.getLLMResponse({
             systemPrompt,
             tools: [], // No tools needed for this analysis
-            selectedModel: selectedChatModel,
             trace: llmSpan,
             hasThinkingEnabled: false,
             hasInterleavedThinkingEnabled: false,

package/dist/tools/index.d.ts

@@ -1,9 +1,35 @@
 import { PendingToolCall, ServicePayload, SupportedChatModels, Tool, ToolDefinition } from "@empiricalrun/shared-types";
+import { createPullRequestTool } from "./create-pull-request";
+import { deleteFileTool } from "./definitions/delete-file";
+import { downloadBuildTool } from "./definitions/download-build";
+import { extractFramesFromVideo } from "./definitions/extract-frames-from-video";
+import { fetchVideoAnalysis } from "./definitions/fetch-video-analysis";
+import { grepTool } from "./definitions/grep";
+import { listProjectsTool, listTestsForProjectTool } from "./definitions/list-tests-and-projects";
+import { mergeConflictsTool } from "./definitions/merge-conflicts";
+import { renameFileTool } from "./definitions/rename-file";
+import { runTestTool } from "./definitions/run-test";
+import { generateTestWithBrowserAgent } from "./definitions/test-gen-browser";
+import { upgradePackagesTool } from "./definitions/upgrade-packages";
+import { fetchDiagnosisReportTool } from "./diagnosis-fetcher";
+import { fetchFileTool } from "./fetch-file";
+import { fetchLastSuccessfulTestRunTool } from "./fetch-last-successful-test-run";
+import { fetchSessionDiffTool } from "./fetch-session-diff";
+import { createIssueTool, listIssuesTool, updateIssueTool } from "./issues";
+import { listEnvironmentsTool } from "./list-environments";
+import { reviewPullRequestTool } from "./review-pull-request";
+import { fetchTestRunReportTool } from "./test-run-fetcher";
+import { traceDotZipTool } from "./trace-dot-zip";
+import { sendTriageSummaryTool } from "./triage-summary";
+import { viewFailedTestRunReportTool } from "./view-failed-test-run-report";
 type ToolOrToolDefinition = Tool | ToolDefinition;
+export declare const commonTools: ToolOrToolDefinition[];
+export declare const testGenerationTools: ToolOrToolDefinition[];
+export declare const triageOnlyTools: ToolOrToolDefinition[];
+export { createIssueTool, createPullRequestTool, deleteFileTool, downloadBuildTool, extractFramesFromVideo, fetchDiagnosisReportTool, fetchFileTool, fetchLastSuccessfulTestRunTool, fetchSessionDiffTool, fetchTestRunReportTool, fetchVideoAnalysis, generateTestWithBrowserAgent, grepTool, listEnvironmentsTool, listIssuesTool, listProjectsTool, listTestsForProjectTool, mergeConflictsTool, renameFileTool, reviewPullRequestTool, runTestTool, sendTriageSummaryTool, traceDotZipTool, updateIssueTool, upgradePackagesTool, viewFailedTestRunReportTool, };
 export declare const allToolsDefinitions: ToolOrToolDefinition[];
-export declare function availableTools(model?: SupportedChatModels, isTriage?: boolean, featureFlags?: string[]): Array<ToolDefinition | Tool>;
+export declare function textEditorToolsForModel(model: SupportedChatModels): ToolOrToolDefinition[];
 export declare function toolsNeedBrowser(toolCalls: PendingToolCall[]): boolean;
 export declare function nameForTelemetry(toolCall: PendingToolCall): string;
 export declare function sendToolRequestToRemoteQueue(payload: ServicePayload): Promise<void>;
-export {};
 //# sourceMappingURL=index.d.ts.map

package/dist/tools/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AACA,OAAO,EACL,eAAe,EACf,cAAc,EACd,mBAAmB,EACnB,IAAI,EACJ,cAAc,EACf,MAAM,4BAA4B,CAAC;AA6BpC,KAAK,oBAAoB,GAAG,IAAI,GAAG,cAAc,CAAC;AAkClD,eAAO,MAAM,mBAAmB,EAAE,oBAAoB,EAMrD,CAAC;AAEF,wBAAgB,cAAc,CAC5B,KAAK,CAAC,EAAE,mBAAmB,EAC3B,QAAQ,CAAC,EAAE,OAAO,EAClB,YAAY,CAAC,EAAE,MAAM,EAAE,GACtB,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC,CAmB9B;AAED,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,eAAe,EAAE,WAM5D;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,CAQlE;AAED,wBAAsB,4BAA4B,CAAC,OAAO,EAAE,cAAc,iBAoBzE"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AACA,OAAO,EACL,eAAe,EACf,cAAc,EACd,mBAAmB,EACnB,IAAI,EACJ,cAAc,EACf,MAAM,4BAA4B,CAAC;AAGpC,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AACxE,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EACL,gBAAgB,EAChB,uBAAuB,EACxB,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAErD,OAAO,EAAE,4BAA4B,EAAE,MAAM,gCAAgC,CAAC;AAC9E,OAAO,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,8BAA8B,EAAE,MAAM,kCAAkC,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC5E,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,2BAA2B,EAAE,MAAM,+BAA+B,CAAC;AAE5E,KAAK,oBAAoB,GAAG,IAAI,GAAG,cAAc,CAAC;AAIlD,eAAO,MAAM,WAAW,EAAE,oBAAoB,EAQ7C,CAAC;AAEF,eAAO,MAAM,mBAAmB,EAAE,oBAAoB,EAQrD,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,oBAAoB,EASjD,CAAC;AAGF,OAAO,EACL,eAAe,EACf,qBAAqB,EACrB,cAAc,EACd,iBAAiB,EAEjB,sBAAsB,EACtB,wBAAwB,EACxB,aAAa,EACb,8BAA8B,EAE9B,oBAAoB,EACpB,sBAAsB,EACtB,kBAAkB,EAElB,4BAA4B,EAC5B,QAAQ,EACR,oBAAoB,EAEpB,cAAc,EAEd,gBAAgB,EAChB,uBAAuB,EACvB,kBAAkB,EAClB,cAAc,EACd,qBAAqB,EAErB,WAAW,EACX,qBAAqB,EACrB,eAAe,EACf,eAAe,EACf,mBAAmB,EACnB,2BAA2B,GAC5B,CAAC;AAEF,eAAO,MAAM,mBAAmB,EAAE,oBAAoB,EASrD,CAAC;AAEF,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,mBAAmB,GACzB,oBAAoB,EAAE,CAKxB;AAED,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,eAAe,EAAE,WAM5D;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,CAQlE;AAED,wBAAsB,4BAA4B,CAAC,OAAO,EAAE,cAAc,iBAoBzE"}