@empiricalrun/test-gen 0.74.2 → 0.76.0

package/dist/tools/extract-frames-from-video/index.js

@@ -0,0 +1,145 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractFramesFromVideo = void 0;
+const node_crypto_1 = __importDefault(require("node:crypto"));
+const fs_1 = require("fs");
+const file_1 = require("../../utils/file");
+const local_ffmpeg_client_1 = require("../../utils/local-ffmpeg-client");
+const extract_frames_from_video_1 = require("../definitions/extract-frames-from-video");
+function parseTimeToSeconds(timeString) {
+    const timeRegex = /^(\d{1,2}):(\d{2})$/;
+    const match = timeString.match(timeRegex);
+    if (!match || match.length < 3) {
+        throw new Error(`Invalid time format: ${timeString}. Expected MM:SS format (e.g., "01:32")`);
+    }
+    const minutesStr = match[1];
+    const secondsStr = match[2];
+    if (!minutesStr || !secondsStr) {
+        throw new Error(`Invalid time format: ${timeString}. Expected MM:SS format (e.g., "01:32")`);
+    }
+    const minutes = parseInt(minutesStr, 10);
+    const seconds = parseInt(secondsStr, 10);
+    if (seconds >= 60) {
+        throw new Error(`Invalid seconds: ${seconds}. Seconds must be 0-59`);
+    }
+    return minutes * 60 + seconds;
+}
+function getExtractFramesParams(params) {
+    return {
+        fps: params?.fps ?? 30,
+        threshold: params?.threshold ?? 0.001,
+        startTime: params?.startTime
+            ? parseTimeToSeconds(params.startTime)
+            : undefined,
+        duration: params?.duration
+            ? parseTimeToSeconds(params.duration)
+            : undefined,
+    };
+}
+function hashExtractParams(videoUrl, params) {
+    const sortedParams = Object.keys(params)
+        .sort()
+        .reduce((acc, key) => {
+        acc[key] = params[key];
+        return acc;
+    }, {});
+    const json = JSON.stringify({ videoUrl, ...sortedParams });
+    return node_crypto_1.default
+        .createHash("sha256")
+        .update(json)
+        .digest("hex")
+        .substring(0, 16);
+}
+exports.extractFramesFromVideo = {
+    ...extract_frames_from_video_1.extractFramesFromVideo,
+    execute: async ({ input, trace, }) => {
+        const { videoUrl } = input;
+        const params = getExtractFramesParams(input.params);
+        const videoUrlHash = hashExtractParams(videoUrl, params);
+        const WORKING_DIR = `./extract-frames-artifacts/${videoUrlHash}`;
+        // const R2_BASE_URL = `https://video-analysis.empirical.run/${videoUrlHash}`;
+        const extractionSpan = trace?.span({
+            name: "extract-frames-from-video",
+            input: { videoUrl, params },
+        });
+        try {
+            const response = await fetch(videoUrl, { method: "HEAD" });
+            if (!response.ok) {
+                return {
+                    result: `Failed to access video: ${response.statusText}`,
+                    isError: true,
+                };
+            }
+            const ffmpegClient = new local_ffmpeg_client_1.LocalFFmpegClient();
+            const extractionResult = await ffmpegClient.extractVideoFrames(videoUrl, WORKING_DIR, {
+                fps: params.fps,
+                threshold: params.threshold,
+                startTime: params.startTime,
+                duration: params.duration,
+            });
+            const { totalFramesCount, uniqueFrames, videoDurationSeconds } = extractionResult;
+            // Create response metadata
+            const extractFramesResponse = {
+                videoUrl,
+                videoDurationSeconds,
+                totalFramesCount,
+                extractedFramesCount: uniqueFrames.length,
+                timeRange: params.startTime !== undefined || params.duration !== undefined
+                    ? {
+                        startTime: params.startTime || 0,
+                        duration: params.duration || 0,
+                        endTime: (params.startTime || 0) + (params.duration || 0),
+                    }
+                    : undefined,
+                frames: uniqueFrames.map((frame) => ({
+                    index: frame.metadata.index,
+                    path: frame.metadata.path,
+                    frameId: `frame_${frame.metadata.index}`,
+                })),
+            };
+            // Convert frames to base64 images for multimodal response
+            const frameImageParts = await Promise.all(uniqueFrames.map(async (frame) => {
+                try {
+                    const imageBuffer = await fs_1.promises.readFile(frame.metadata.path);
+                    const base64Data = imageBuffer.toString("base64");
+                    return {
+                        type: "image/png",
+                        base64Data: base64Data,
+                    };
+                }
+                catch (error) {
+                    console.warn(`Failed to read frame ${frame.metadata.path}:`, error);
+                    return null;
+                }
+            }));
+            await (0, file_1.safeCleanupDirectory)(WORKING_DIR, "extract-frames-tool");
+            // Filter out any failed frame reads
+            const validFrameImageParts = frameImageParts.filter((part) => part !== null);
+            return {
+                result: [
+                    {
+                        type: "text",
+                        text: JSON.stringify(extractFramesResponse, null, 2),
+                    },
+                    ...validFrameImageParts,
+                ],
+                isError: false,
+            };
+        }
+        catch (error) {
+            console.error("Error in extractFramesFromVideo", error);
+            extractionSpan?.end();
+            const message = error instanceof Error ? error.message : String(error);
+            return {
+                result: `Error extracting frames from video: ${message}`,
+                isError: true,
+            };
+        }
+        finally {
+            extractionSpan?.end();
+        }
+    },
+};

package/dist/tools/{fetch-image → fetch-file}/index.d.ts

@@ -1,12 +1,12 @@
 import { Tool } from "@empiricalrun/shared-types";
 import { z } from "zod";
-declare const fetchImageSchema: z.ZodObject<{
+declare const fetchFileSchema: z.ZodObject<{
     url: z.ZodString;
 }, "strip", z.ZodTypeAny, {
     url: string;
 }, {
     url: string;
 }>;
-export declare const fetchImageTool: Tool<z.infer<typeof fetchImageSchema>>;
+export declare const fetchFileTool: Tool<z.infer<typeof fetchFileSchema>>;
 export {};
 //# sourceMappingURL=index.d.ts.map

package/dist/tools/fetch-file/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/fetch-file/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,IAAI,EAGL,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,QAAA,MAAM,eAAe;;;;;;EAEnB,CAAC;AAEH,eAAO,MAAM,aAAa,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAyF/D,CAAC"}

package/dist/tools/fetch-file/index.js

@@ -0,0 +1,97 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchFileTool = void 0;
+const zod_1 = require("zod");
+const SUPPORTED_IMAGE_TYPES = [
+    "image/png",
+    "image/jpeg",
+    "image/gif",
+    "image/webp",
+];
+const SUPPORTED_TEXT_TYPES = ["text/markdown", "text/plain"];
+const fetchFileSchema = zod_1.z.object({
+    url: zod_1.z.string(),
+});
+exports.fetchFileTool = {
+    schema: {
+        name: "fetchFile",
+        description: `Use this tool to fetch file data from any valid URL that responds with an image (PNG, JPEG, GIF, WebP) or markdown file.
+For images, it returns the image in base64 format for you to view or analyze. For markdown files, it returns the text content for you to read and process.
+
+## Caveats
+1. This will not work to fetch markdown files from a repo due to access control issues. Use file view tools for that.
+
+## Scenarios to use fetchFile
+1. Understand a test report (for the runTest tool) by fetching screenshots and error context files (.md) which contain the accessibility tree of the page at the
+time of test failures. Both of these are available in the attachments section of the test report from the runTest tool call.
+
+2. While adding new tests, if the user message contains image URLs, use this tool to fetch the images, and understand the steps that the user is looking to cover..`,
+        parameters: fetchFileSchema,
+    },
+    needsBrowser: false,
+    isInlineTool: true,
+    execute: async ({ input }) => {
+        const { url } = input;
+        try {
+            const response = await fetch(url);
+            if (!response.ok) {
+                console.error(`Failed to fetch file from ${url}: ${response.statusText}`);
+                return {
+                    result: `Failed to fetch file from ${url}: ${response.statusText}`,
+                    isError: true,
+                };
+            }
+            const contentType = response.headers.get("content-type");
+            if (!contentType) {
+                const errorMessage = `No content type header found. URL must return an image or markdown file.`;
+                return {
+                    result: errorMessage,
+                    isError: true,
+                };
+            }
+            const isImage = SUPPORTED_IMAGE_TYPES.some((type) => contentType.startsWith(type));
+            const isText = SUPPORTED_TEXT_TYPES.some((type) => contentType.startsWith(type));
+            if (!isImage && !isText) {
+                const errorMessage = `Invalid content type: ${contentType}. URL must return an image (PNG, JPEG, GIF, WebP) or markdown file.`;
+                console.error(errorMessage);
+                return {
+                    result: errorMessage,
+                    isError: true,
+                };
+            }
+            if (isImage) {
+                const buffer = await response.arrayBuffer();
+                const base64 = Buffer.from(buffer).toString("base64");
+                return {
+                    result: [
+                        {
+                            type: contentType,
+                            base64Data: base64,
+                        },
+                    ],
+                    isError: false,
+                };
+            }
+            else {
+                // Handle text/markdown files
+                const text = await response.text();
+                return {
+                    result: [
+                        {
+                            type: "text",
+                            text: text,
+                        },
+                    ],
+                    isError: false,
+                };
+            }
+        }
+        catch (error) {
+            console.error("Error fetching file", error);
+            return {
+                result: `Error fetching file: ${error}`,
+                isError: true,
+            };
+        }
+    },
+};

package/dist/tools/fetch-session-diff/index.d.ts

@@ -0,0 +1,3 @@
+import { Tool } from "@empiricalrun/shared-types";
+export declare const fetchSessionDiffTool: Tool;
+//# sourceMappingURL=index.d.ts.map

package/dist/tools/fetch-session-diff/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/fetch-session-diff/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAc,MAAM,4BAA4B,CAAC;AAO9D,eAAO,MAAM,oBAAoB,EAAE,IA0ClC,CAAC"}

package/dist/tools/fetch-session-diff/index.js

@@ -0,0 +1,46 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchSessionDiffTool = void 0;
+const zod_1 = require("zod");
+const SessionDiffInputSchema = zod_1.z.object({
+    url: zod_1.z.string().describe(`The URL of the chat session.`),
+});
+exports.fetchSessionDiffTool = {
+    schema: {
+        name: "fetchSessionDiff",
+        description: `Extracts the session url and returns the code diff for the session.`,
+        parameters: SessionDiffInputSchema,
+    },
+    needsBrowser: false,
+    isInlineTool: true,
+    execute: async ({ input, apiClient }) => {
+        try {
+            const sessionUrl = input.url;
+            if (!sessionUrl) {
+                return {
+                    isError: true,
+                    result: "No session URL provided",
+                };
+            }
+            if (!apiClient) {
+                throw new Error("Dashboard API client is not available.");
+            }
+            const sessionId = sessionUrl.split("/").pop();
+            const sessionDiffApiUrl = `/api/chat-sessions/${sessionId}/diff`;
+            const sessionDiff = await apiClient.request(sessionDiffApiUrl, {
+                method: "GET",
+            });
+            return {
+                isError: false,
+                result: JSON.stringify(sessionDiff, null, 2),
+            };
+        }
+        catch (error) {
+            console.error("Error fetching session diff", error);
+            return {
+                isError: true,
+                result: error instanceof Error ? error.message : String(error),
+            };
+        }
+    },
+};

package/dist/tools/fetch-video-analysis/index.d.ts

@@ -1,5 +1,5 @@
-import { Tool } from "@empiricalrun/shared-types";
-import { z } from "zod";
-import { videoAnalysisSchema } from "../definitions/fetch-video-analysis";
+import type { Tool } from "@empiricalrun/shared-types";
+import type { z } from "zod";
+import { type videoAnalysisSchema } from "../definitions/fetch-video-analysis";
 export declare const fetchVideoAnalysis: Tool<z.infer<typeof videoAnalysisSchema>>;
 //# sourceMappingURL=index.d.ts.map

package/dist/tools/fetch-video-analysis/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/fetch-video-analysis/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAqB,MAAM,4BAA4B,CAAC;AAErE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAEL,mBAAmB,EACpB,MAAM,qCAAqC,CAAC;AAK7C,eAAO,MAAM,kBAAkB,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAiHxE,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/fetch-video-analysis/index.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAEV,IAAI,EAKL,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAI7B,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,qCAAqC,CAAC;AAuC7C,eAAO,MAAM,kBAAkB,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAsKxE,CAAC"}

package/dist/tools/fetch-video-analysis/index.js

@@ -4,20 +4,46 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.fetchVideoAnalysis = void 0;
-const crypto_1 = __importDefault(require("crypto"));
+const node_crypto_1 = __importDefault(require("node:crypto"));
+const node_path_1 = __importDefault(require("node:path"));
+const file_1 = require("../../utils/file");
+const local_ffmpeg_client_1 = require("../../utils/local-ffmpeg-client");
 const fetch_video_analysis_1 = require("../definitions/fetch-video-analysis");
-const local_ffmpeg_client_1 = require("./local-ffmpeg-client");
 const utils_1 = require("./utils");
 const video_analysis_1 = require("./video-analysis");
+function getVideoAnalysisParams(params) {
+    return {
+        model: params?.model || "gemini-2.5-pro",
+        fps: params?.fps ?? 30,
+        threshold: params?.threshold ?? 0.001,
+        featureFlag: params?.featureFlag ?? "send-all-frames",
+    };
+}
+function hashObject(obj) {
+    const sortedObj = Object.keys(obj)
+        .sort()
+        .reduce((acc, key) => {
+        acc[key] = obj[key];
+        return acc;
+    }, {});
+    const json = JSON.stringify(sortedObj);
+    return node_crypto_1.default
+        .createHash("sha256")
+        .update(json)
+        .digest("hex")
+        .substring(0, 16);
+}
 exports.fetchVideoAnalysis = {
     ...fetch_video_analysis_1.fetchVideoAnalysis,
-    execute: async ({ input, trace, }) => {
+    execute: async ({ input, trace, featureFlags, }) => {
         const { videoUrl } = input;
-        const videoUrlHash = crypto_1.default
-            .createHash("sha256")
-            .update(videoUrl)
-            .digest("hex")
-            .substring(0, 16);
+        const params = getVideoAnalysisParams(input.params);
+        const videoUrlHash = hashObject({
+            videoUrl,
+            ...params,
+        });
+        const { model: selectedModel, fps: effectiveFps, threshold: effectiveThreshold, featureFlag: effectiveFeatureFlag, } = params;
+        const WORKING_DIR = `./video-analysis-artifacts/${videoUrlHash}`;
         const R2_BASE_URL = `https://video-analysis.empirical.run/${videoUrlHash}/`;
         const videoAnalysisSpan = trace?.span({
             name: "video-analysis",
@@ -34,36 +60,70 @@ exports.fetchVideoAnalysis = {
             }
             const processingSpan = videoAnalysisSpan?.span({
                 name: "ffmpeg-processing",
-                input: { videoUrl },
+                input: {
+                    videoUrl,
+                    fps: effectiveFps,
+                    threshold: effectiveThreshold,
+                },
             });
             try {
-                const { totalFramesCount, uniqueFramesCount, uniqueFrames } = await ffmpegClient.extractVideoFrames(videoUrl, "./temp");
+                const extractionResult = await ffmpegClient.extractVideoFrames(videoUrl, WORKING_DIR, {
+                    fps: effectiveFps,
+                    threshold: effectiveThreshold,
+                });
+                const { totalFramesCount, uniqueFrames } = extractionResult;
                 processingSpan?.end({
                     output: {
                         totalFramesCount,
-                        uniqueFramesCount,
+                        uniqueFramesCount: uniqueFrames.length,
                     },
                 });
                 console.log(`[video-analysis] Analyzing ${uniqueFrames.length} frames with LLM`);
-                const videoInfoWithoutAnalysis = {
+                const outputZipPath = node_path_1.default.join(WORKING_DIR, "frames.zip");
+                const zipUploadPromise = (0, utils_1.zipAndUploadFramesToR2)(uniqueFrames, outputZipPath, videoUrlHash).catch((error) => {
+                    throw error; // Re-throw to maintain error in Promise.all
+                });
+                const { analysis: llmAnalysis, usage } = await (0, video_analysis_1.analyzeFramesWithLLM)(uniqueFrames, videoAnalysisSpan, selectedModel);
+                console.log(`[video-analysis] Finished Analyzing ${uniqueFrames.length} frames with LLM`);
+                let finalAnalysis = llmAnalysis;
+                let finalKeyFramesData = [];
+                if (featureFlags.includes("enableFetchVideoAnalysisToolMultiModal")) {
+                    const { updatedAnalysis, keyFramesData } = await (0, utils_1.getUpdatedAnalysisAndToolResultImagePart)(llmAnalysis, uniqueFrames);
+                    finalAnalysis = updatedAnalysis;
+                    finalKeyFramesData = [...keyFramesData];
+                }
+                const videoInfo = {
                     total_frames_count: totalFramesCount,
-                    unique_frames_count: uniqueFramesCount,
+                    unique_frames_count: uniqueFrames.length,
                     video_url: videoUrl,
-                    video_url_hash: videoUrlHash,
+                    analysis_id: videoUrlHash,
                     created_at: new Date().toISOString(),
+                    params: {
+                        fps: effectiveFps,
+                        threshold: effectiveThreshold,
+                        model: selectedModel,
+                        featureFlag: effectiveFeatureFlag,
+                    },
+                    usage,
+                    langfuse_trace_id: trace?.id || undefined,
+                    frames_zip_url: `${R2_BASE_URL}frames.zip`,
+                    analysis: finalAnalysis,
                 };
-                const framesUploadPromise = (0, utils_1.uploadFramesToR2)(videoInfoWithoutAnalysis, uniqueFrames, R2_BASE_URL);
-                const { analysis: llmAnalysis, usage } = await (0, video_analysis_1.analyzeFramesWithLLM)(uniqueFrames.map((frame) => frame.image), videoAnalysisSpan, "gemini-2.5-pro");
-                console.log(`[video-analysis] Finished Analyzing ${uniqueFrames.length} frames with LLM`);
-                const videoInfo = {
-                    ...videoInfoWithoutAnalysis,
-                    llm_analysis: llmAnalysis,
-                    video_url_hash: videoUrlHash,
+                await Promise.all([
+                    zipUploadPromise,
+                    (0, utils_1.uploadSummaryToR2)(videoInfo, R2_BASE_URL),
+                ]);
+                await (0, file_1.safeCleanupDirectory)(WORKING_DIR, "video-analysis-cleanup");
+                const toolResult = {
+                    video_url: videoUrl,
+                    analysis_id: videoUrlHash,
+                    analysis: finalAnalysis,
                 };
-                const uniqueFramesWithUrls = await framesUploadPromise;
-                await (0, utils_1.uploadAnalysisToR2)(videoInfo, uniqueFramesWithUrls, R2_BASE_URL);
                 return {
-                    result: JSON.stringify(videoInfo, null, 2),
+                    result: [
+                        { type: "text", text: JSON.stringify(toolResult, null, 2) },
+                        ...finalKeyFramesData,
+                    ],
                     isError: false,
                     usage,
                 };

package/dist/tools/fetch-video-analysis/open-ai.d.ts

@@ -0,0 +1,6 @@
+export declare function analyzeImages({ systemPrompt, imageBase64Array, userPrompt, }: {
+    systemPrompt: string;
+    imageBase64Array: string[];
+    userPrompt: string;
+}): Promise<string>;
+//# sourceMappingURL=open-ai.d.ts.map

package/dist/tools/fetch-video-analysis/open-ai.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"open-ai.d.ts","sourceRoot":"","sources":["../../../src/tools/fetch-video-analysis/open-ai.ts"],"names":[],"mappings":"AAOA,wBAAsB,aAAa,CAAC,EAClC,YAAY,EACZ,gBAAgB,EAChB,UAAU,GACX,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;CACpB,mBA+BA"}

package/dist/tools/fetch-video-analysis/open-ai.js

@@ -0,0 +1,37 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.analyzeImages = analyzeImages;
+const openai_1 = __importDefault(require("openai"));
+const openai = new openai_1.default({
+    apiKey: ``,
+});
+async function analyzeImages({ systemPrompt, imageBase64Array, userPrompt, }) {
+    const response = await openai.responses.create({
+        model: "gpt-4.1-2025-04-14",
+        input: [
+            {
+                role: "system",
+                content: systemPrompt,
+            },
+            {
+                role: "user",
+                content: [
+                    {
+                        type: "input_text",
+                        text: userPrompt,
+                    },
+                    ...imageBase64Array.map((base64) => ({
+                        detail: "auto",
+                        type: "input_image",
+                        image_url: `data:image/png;base64,${base64}`,
+                    })),
+                ],
+            },
+        ],
+    });
+    console.log(response.output_text);
+    return response.output_text;
+}

package/dist/tools/fetch-video-analysis/utils.d.ts

@@ -1,10 +1,16 @@
-import { UniqueFrameInfo, VideoAnalysisInfo } from "@empiricalrun/shared-types";
-export declare const uploadFramesToR2: (videoInfo: Omit<VideoAnalysisInfo, "llm_analysis">, frames: {
+import { type FileMap } from "@empiricalrun/r2-uploader";
+import { ToolResultImagePart, UniqueFrameInfo, UniqueFrameWithMetadata, VideoAnalysisSummary } from "@empiricalrun/shared-types";
+export declare const uploadFramesToR2: (videoInfo: Omit<VideoAnalysisSummary, "analysis">, frames: {
     metadata: {
         index: number;
         path: string;
     };
     image: string;
 }[], r2BaseUrl: string) => Promise<UniqueFrameInfo[]>;
-export declare const uploadAnalysisToR2: (videoInfo: VideoAnalysisInfo, uniqueFrames: UniqueFrameInfo[], r2BaseUrl: string) => Promise<string>;
+export declare const uploadSummaryToR2: (videoInfo: VideoAnalysisSummary, r2BaseUrl: string) => Promise<string>;
+export declare const zipAndUploadFramesToR2: (uniqueFrames: UniqueFrameWithMetadata[], outputZipPath: string, videoUrlHash: string) => Promise<FileMap>;
+export declare const getUpdatedAnalysisAndToolResultImagePart: (llmAnalysis: string, uniqueFrames: UniqueFrameWithMetadata[]) => Promise<{
+    updatedAnalysis: string;
+    keyFramesData: ToolResultImagePart[];
+}>;
 //# sourceMappingURL=utils.d.ts.map

package/dist/tools/fetch-video-analysis/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/tools/fetch-video-analysis/utils.ts"],"names":[],"mappings":"AACA,OAAO,EACL,eAAe,EACf,iBAAiB,EAElB,MAAM,4BAA4B,CAAC;AAmCpC,eAAO,MAAM,gBAAgB,GAC3B,WAAW,IAAI,CAAC,iBAAiB,EAAE,cAAc,CAAC,EAClD,QAAQ;IACN,QAAQ,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1C,KAAK,EAAE,MAAM,CAAC;CACf,EAAE,EACH,WAAW,MAAM,KAChB,OAAO,CAAC,eAAe,EAAE,CAqB3B,CAAC;AAEF,eAAO,MAAM,kBAAkB,GAC7B,WAAW,iBAAiB,EAC5B,cAAc,eAAe,EAAE,EAC/B,WAAW,MAAM,KAChB,OAAO,CAAC,MAAM,CAwChB,CAAC"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/tools/fetch-video-analysis/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,OAAO,EAGb,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,4BAA4B,CAAC;AAuCpC,eAAO,MAAM,gBAAgB,GAC3B,WAAW,IAAI,CAAC,oBAAoB,EAAE,UAAU,CAAC,EACjD,QAAQ;IACN,QAAQ,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1C,KAAK,EAAE,MAAM,CAAC;CACf,EAAE,EACH,WAAW,MAAM,KAChB,OAAO,CAAC,eAAe,EAAE,CAqB3B,CAAC;AAEF,eAAO,MAAM,iBAAiB,GAC5B,WAAW,oBAAoB,EAC/B,WAAW,MAAM,KAChB,OAAO,CAAC,MAAM,CAsBhB,CAAC;AAEF,eAAO,MAAM,sBAAsB,GACjC,cAAc,uBAAuB,EAAE,EACvC,eAAe,MAAM,EACrB,cAAc,MAAM,KACnB,OAAO,CAAC,OAAO,CAsBjB,CAAC;AAEF,eAAO,MAAM,wCAAwC,GACnD,aAAa,MAAM,EACnB,cAAc,uBAAuB,EAAE,KACtC,OAAO,CAAC;IACT,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,mBAAmB,EAAE,CAAC;CACtC,CA2CA,CAAC"}

package/dist/tools/fetch-video-analysis/utils.js

@@ -1,7 +1,13 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.uploadAnalysisToR2 = exports.uploadFramesToR2 = void 0;
+exports.getUpdatedAnalysisAndToolResultImagePart = exports.zipAndUploadFramesToR2 = exports.uploadSummaryToR2 = exports.uploadFramesToR2 = void 0;
 const r2_uploader_1 = require("@empiricalrun/r2-uploader");
+const child_process_1 = require("child_process");
+const fs_1 = require("fs");
+const path_1 = __importDefault(require("path"));
 const uploadFilesToR2 = async (files, videoUrlHash) => {
     const label = `video-analysis-upload:${videoUrlHash}`;
     const start = Date.now();
@@ -25,7 +31,7 @@ const uploadFilesToR2 = async (files, videoUrlHash) => {
     }
 };
 const uploadFramesToR2 = async (videoInfo, frames, r2BaseUrl) => {
-    const { video_url_hash: videoUrlHash } = videoInfo;
+    const { analysis_id: videoUrlHash } = videoInfo;
     const frameFiles = frames.map((f) => {
         const fileName = `frame_${f.metadata.index}_${videoUrlHash}.png`;
         const buffer = Buffer.from(f.image, "base64");
@@ -41,21 +47,12 @@ const uploadFramesToR2 = async (videoInfo, frames, r2BaseUrl) => {
     }));
 };
 exports.uploadFramesToR2 = uploadFramesToR2;
-const uploadAnalysisToR2 = async (videoInfo, uniqueFrames, r2BaseUrl) => {
+const uploadSummaryToR2 = async (videoInfo, r2BaseUrl) => {
     try {
-        const { video_url_hash: videoUrlHash, total_frames_count, unique_frames_count, video_url, llm_analysis, created_at, } = videoInfo;
-        const summary = {
-            total_frames_count,
-            unique_frames_count,
-            video_url,
-            llm_analysis,
-            video_url_hash: videoUrlHash,
-            uniqueFrames,
-            created_at,
-        };
+        const { analysis_id: videoUrlHash } = videoInfo;
         const filesToUpload = [
             {
-                buffer: Buffer.from(JSON.stringify(summary, null, 2)),
+                buffer: Buffer.from(JSON.stringify(videoInfo, null, 2)),
                 fileName: "summary.json",
                 mimeType: "application/json",
             },
@@ -69,4 +66,56 @@ const uploadAnalysisToR2 = async (videoInfo, uniqueFrames, r2BaseUrl) => {
         throw error;
     }
 };
-exports.uploadAnalysisToR2 = uploadAnalysisToR2;
+exports.uploadSummaryToR2 = uploadSummaryToR2;
+const zipAndUploadFramesToR2 = async (uniqueFrames, outputZipPath, videoUrlHash) => {
+    const filePaths = uniqueFrames.map((u) => u.metadata.path);
+    await new Promise((resolve, reject) => {
+        (0, child_process_1.execFile)("zip", ["-0", "-j", outputZipPath, ...filePaths], (err) => {
+            if (err)
+                return reject(err);
+            resolve();
+        });
+    });
+    const tempUploadDir = path_1.default.dirname(outputZipPath);
+    console.log(`[zip-upload] Uploading zip file: ${outputZipPath} to video-analysis/${videoUrlHash}/`);
+    return await (0, r2_uploader_1.uploadDirectory)({
+        sourceDir: tempUploadDir,
+        fileList: [outputZipPath],
+        destinationDir: videoUrlHash,
+        uploadBucket: "video-analysis",
+    });
+};
+exports.zipAndUploadFramesToR2 = zipAndUploadFramesToR2;
+const getUpdatedAnalysisAndToolResultImagePart = async (llmAnalysis, uniqueFrames) => {
+    const keyFramesMatch = llmAnalysis.match(/KEY FRAMES:\s*(.+)$/);
+    const updatedAnalysis = keyFramesMatch
+        ? llmAnalysis.replace(/\n\nKEY FRAMES:\s*(.+)$/, "")
+        : llmAnalysis;
+    const keyFrameIndices = keyFramesMatch?.[1]
+        ?.match(/<frame_(\d+)>/g)
+        ?.map((match) => parseInt(match.match(/<frame_(\d+)>/)?.[1] || "0", 10)) || [];
+    const frameDataMap = new Map();
+    await Promise.all(uniqueFrames.map(async (frame) => {
+        try {
+            const fileBuffer = await fs_1.promises.readFile(frame.metadata.path);
+            const base64Data = fileBuffer.toString("base64");
+            frameDataMap.set(frame.metadata.index, base64Data);
+        }
+        catch (error) {
+            console.error(`Failed to read frame file ${frame.metadata.path}:`, error);
+        }
+    }));
+    const keyFramesData = keyFrameIndices
+        .map((frameIndex) => {
+        const base64Data = frameDataMap.get(frameIndex);
+        return base64Data
+            ? {
+                type: "image/png",
+                base64Data: base64Data,
+            }
+            : null;
+    })
+        .filter(Boolean);
+    return { updatedAnalysis, keyFramesData };
+};
+exports.getUpdatedAnalysisAndToolResultImagePart = getUpdatedAnalysisAndToolResultImagePart;