@blackbox_ai/blackbox-cli-core 0.0.9 → 0.8.1

package/dist/src/tools/browser_use.js

@@ -4,9 +4,14 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 import { chromium } from 'playwright';
+import path from 'node:path';
+import { mkdir, rename, readFile, stat } from 'node:fs/promises';
+import { randomUUID } from 'node:crypto';
 import { ToolNames } from './tool-names.js';
 import { ToolErrorType } from './tool-error.js';
 import { BaseDeclarativeTool, BaseToolInvocation, Kind, } from './tools.js';
+import { SignJWT } from 'jose';
+import { compressImage, shouldCompressImage } from '../utils/imageCompression.js';
 const screenshotPrompt = `Here are the action result, console logs and screenshot after the action execution.
 Carefully review and decide the next steps to complete the task successfully.`;
 /**
@@ -14,9 +19,15 @@ Carefully review and decide the next steps to complete the task successfully.`;
  */
 export class ServerBrowserSession {
     browser;
+    context; // added
     page;
     currentMousePosition;
     static instance;
+    // recording config/state
+    recordingEnabled = false; // added
+    recordingDirAbs; // added (absolute directory)
+    recordingName; // added (desired filename, optional)
+    lastVideoPath; // added (resolved saved path)
     constructor() { }
     static getInstance() {
         if (!ServerBrowserSession.instance) {
@@ -24,7 +35,7 @@ export class ServerBrowserSession {
         }
         return ServerBrowserSession.instance;
     }
-    async launchBrowser() {
+    async launchBrowser(opts) {
         console.log('Launching browser...');
         if (this.browser) {
             await this.closeBrowser();
@@ -43,19 +54,39 @@ export class ServerBrowserSession {
                     '--disable-background-timer-throttling',
                     '--disable-backgrounding-occluded-windows',
                     '--disable-renderer-backgrounding',
-                    '--use-gl=swiftshader', // software GL
+                    '--use-gl=swiftshader',
                 ],
                 headless: true,
             });
-            const context = await this.browser.newContext({
+            // Configure recording
+            this.recordingEnabled = !!opts?.record;
+            this.recordingName = opts?.videoName?.trim() || undefined;
+            if (this.recordingName && !this.recordingName.endsWith('.webm')) {
+                this.recordingName = `${this.recordingName}.webm`;
+            }
+            if (this.recordingEnabled) {
+                const dir = opts?.videoDir?.trim();
+                const abs = dir && path.isAbsolute(dir) ? dir : path.resolve(process.cwd(), dir || 'videos');
+                await mkdir(abs, { recursive: true });
+                this.recordingDirAbs = abs;
+                console.log(`[Recording] Enabled. Output dir: ${this.recordingDirAbs}${this.recordingName ? `, name: ${this.recordingName}` : ''}`);
+            }
+            else {
+                this.recordingDirAbs = undefined;
+                this.recordingName = undefined;
+            }
+            // Create context with optional video
+            this.context = await this.browser.newContext({
                 viewport: { width: 900, height: 600 },
                 deviceScaleFactor: 1,
                 screen: { width: 900, height: 600 },
                 ignoreHTTPSErrors: true,
                 bypassCSP: true,
+                ...(this.recordingEnabled && this.recordingDirAbs
+                    ? { recordVideo: { dir: this.recordingDirAbs, size: { width: 900, height: 600 } } }
+                    : {}),
             });
-            this.page = await context.newPage();
-            // Set a default background to ensure screenshots work
+            this.page = await this.context.newPage();
             await this.page.addStyleTag({
                 content: `
           html, body {
@@ -64,11 +95,14 @@ export class ServerBrowserSession {
           }
         `,
             });
-            console.log('Browser launched successfully with 900x600 viewport');
+            const recordNote = this.recordingEnabled
+                ? ` with recording to ${this.recordingDirAbs}${this.recordingName ? `/${this.recordingName}` : ''}`
+                : '';
+            console.log('Browser launched successfully with 900x600 viewport' + recordNote);
             return {
                 execution_success: true,
-                logs: 'Browser session started successfully with 900x600 viewport',
-                execution_logs: 'Browser launched and ready for interaction at 900x600 resolution',
+                logs: 'Browser session started successfully with 900x600 viewport' + recordNote,
+                execution_logs: 'Browser launched and ready for interaction at 900x600 resolution' + recordNote,
             };
         }
         catch (_error) {
@@ -81,18 +115,176 @@ export class ServerBrowserSession {
             };
         }
     }
+    /* generate access token, jwt token with secret key */
+    async generateAccessToken() {
+        const secret = process.env['VIDEO_STORAGE_SECRET_KEY'];
+        if (!secret) {
+            throw new Error('VIDEO_STORAGE_SECRET_KEY is not set');
+        }
+        const expiresInSeconds = 3600; // 1 hour
+        const payload = {
+            sub: randomUUID(),
+        };
+        const secretKey = new TextEncoder().encode(secret);
+        return new SignJWT(payload)
+            .setProtectedHeader({ alg: 'HS256', typ: 'JWT' })
+            .setIssuedAt()
+            .setExpirationTime(Math.floor(Date.now() / 1000) + expiresInSeconds)
+            .sign(secretKey);
+    }
+    async uploadToStorage(videoPath, videoName) {
+        try {
+            const accessToken = await this.generateAccessToken();
+            const baseUrl = process.env['VIDEO_STORAGE_API_URL'];
+            if (!baseUrl) {
+                return { execution_success: false, error: 'VIDEO_STORAGE_API_URL is not set' };
+            }
+            // Read bytes once
+            const bytes = await readFile(videoPath);
+            // Attempt direct multipart /upload first
+            {
+                const form = new FormData();
+                const blob = new Blob([bytes], { type: 'video/webm' });
+                form.append('file', blob, videoName);
+                const uploadResponse = await fetch(`${baseUrl}/upload`, {
+                    method: 'POST',
+                    headers: { Authorization: `Bearer ${accessToken}` },
+                    body: form,
+                });
+                if (uploadResponse.ok) {
+                    const json = await uploadResponse.json();
+                    if (json.public_url) {
+                        return { execution_success: true, videoPath: json.public_url };
+                    }
+                    // fall through to signed-url if schema unexpected
+                }
+                else {
+                    const text = await uploadResponse.text().catch(() => '');
+                    const lower = text.toLowerCase();
+                    const isBodyParseError = uploadResponse.status === 400 && (lower.includes('parse') || lower.includes('parsing'));
+                    if (!isBodyParseError) {
+                        return {
+                            execution_success: false,
+                            error: `Upload failed (${uploadResponse.status}): ${text}`,
+                        };
+                    }
+                }
+            }
+            // Fallback: use signed URL flow to bypass multipart parser
+            {
+                const signedResp = await fetch(`${baseUrl}/signed-url`, {
+                    method: 'POST',
+                    headers: {
+                        Authorization: `Bearer ${accessToken}`,
+                        'Content-Type': 'application/json',
+                    },
+                    body: JSON.stringify({ filename: videoName, content_type: 'video/webm' }),
+                });
+                if (!signedResp.ok) {
+                    const text = await signedResp.text().catch(() => '');
+                    return {
+                        execution_success: false,
+                        error: `Signed URL request failed (${signedResp.status}): ${text}`,
+                    };
+                }
+                const signedJson = await signedResp.json();
+                if (!signedJson.signed_put_url || !signedJson.public_url) {
+                    return { execution_success: false, error: 'Invalid signed-url response' };
+                }
+                const putResp = await fetch(signedJson.signed_put_url, {
+                    method: 'PUT',
+                    headers: { 'Content-Type': 'video/webm' },
+                    body: bytes,
+                });
+                if (!putResp.ok) {
+                    const t = await putResp.text().catch(() => '');
+                    return { execution_success: false, error: `Signed PUT failed (${putResp.status}): ${t}` };
+                }
+                return { execution_success: true, videoPath: signedJson.public_url };
+            }
+        }
+        catch (error) {
+            console.error('Error uploading video to storage:', error);
+            return { execution_success: false, error: error instanceof Error ? error.message : String(error) };
+        }
+    }
     async closeBrowser() {
         if (this.browser || this.page) {
             console.log('Closing browser...');
             try {
+                let tmpVideoPath;
+                let finalVideoPath;
+                // Close page and get tmp video path
+                if (this.page) {
+                    const v = this.page.video?.();
+                    await this.page.close();
+                    if (this.recordingEnabled && v) {
+                        try {
+                            tmpVideoPath = await v.path();
+                        }
+                        catch (e) {
+                            console.warn('[Recording] Failed to obtain video path after page close:', e);
+                        }
+                    }
+                }
+                // Close context to flush video to disk
+                await this.context?.close().catch(() => { });
+                // Finalize and optionally upload
+                if (this.recordingEnabled && tmpVideoPath && this.recordingDirAbs) {
+                    try {
+                        if (this.recordingName) {
+                            const target = path.join(this.recordingDirAbs, this.recordingName);
+                            await rename(tmpVideoPath, target);
+                            finalVideoPath = target;
+                        }
+                        else {
+                            finalVideoPath = tmpVideoPath;
+                        }
+                        // Wait briefly until file is non-zero
+                        if (finalVideoPath) {
+                            for (let i = 0; i < 10; i++) {
+                                try {
+                                    const s = await stat(finalVideoPath);
+                                    if (s.size > 0)
+                                        break;
+                                }
+                                catch (_e) {
+                                    // no-op while file finalizes
+                                }
+                                await new Promise((r) => setTimeout(r, 100));
+                            }
+                        }
+                        this.lastVideoPath = finalVideoPath;
+                        if (process.env['VIDEO_STORAGE_API_URL'] && finalVideoPath) {
+                            const safeName = this.recordingName || path.basename(finalVideoPath);
+                            const uploadResult = await this.uploadToStorage(finalVideoPath, safeName);
+                            if (uploadResult.execution_success && uploadResult.videoPath) {
+                                this.lastVideoPath = uploadResult.videoPath;
+                            }
+                            console.log(`[Recording] Video saved: ${this.lastVideoPath}`);
+                        }
+                        else if (finalVideoPath) {
+                            console.log(`[Recording] Video saved: ${finalVideoPath}`);
+                        }
+                    }
+                    catch (e) {
+                        console.warn('[Recording] Failed to finalize/save/upload video:', e);
+                    }
+                }
+                // Close browser
                 await this.browser?.close();
                 this.browser = undefined;
+                this.context = undefined;
                 this.page = undefined;
                 this.currentMousePosition = undefined;
+                const msg = this.lastVideoPath
+                    ? `Browser session closed successfully. Video saved to: ${this.lastVideoPath}`
+                    : 'Browser session closed successfully';
                 return {
                     execution_success: true,
-                    logs: 'Browser session closed successfully',
-                    execution_logs: 'Browser closed successfully',
+                    logs: msg,
+                    execution_logs: msg,
+                    videoPath: this.lastVideoPath,
                 };
             }
             catch (_error) {
@@ -166,7 +358,7 @@ export class ServerBrowserSession {
                 lastLogTs = Date.now();
             }
             catch (_error) {
-                logs.push(`[Console Error] ${_error instanceof Error ? _error.message : String(_error)}`);
+                logs.push(`[Console Error] unknown console listener error`);
             }
         };
         this.page.on('console', consoleListener);
@@ -196,8 +388,24 @@ export class ServerBrowserSession {
                 omitBackground: false,
             });
             if (screenshotBytes && screenshotBytes.length > 0) {
-                const screenshotBase64 = screenshotBytes.toString('base64');
-                screenshot = `data:image/png;base64,${screenshotBase64}`;
+                // Compress screenshot if needed
+                let finalScreenshotBytes = screenshotBytes;
+                let finalMimeType = 'image/png';
+                if (shouldCompressImage(screenshotBytes, 'image/png')) {
+                    try {
+                        const compressionResult = await compressImage(screenshotBytes, 'image/png');
+                        finalScreenshotBytes = compressionResult.buffer;
+                        finalMimeType = compressionResult.mimeType;
+                        console.log(`[Screenshot Compression] ${(screenshotBytes.length / 1024 / 1024).toFixed(2)}MB → ` +
+                            `${(compressionResult.compressedSize / 1024 / 1024).toFixed(2)}MB ` +
+                            `(${compressionResult.compressionRatio.toFixed(1)}% reduction)`);
+                    }
+                    catch (error) {
+                        console.warn(`[Screenshot Compression] Failed, using original: ${error instanceof Error ? error.message : String(error)}`);
+                    }
+                }
+                const screenshotBase64 = finalScreenshotBytes.toString('base64');
+                screenshot = `data:${finalMimeType};base64,${screenshotBase64}`;
                 // Log screenshot success with dimensions
                 console.log(`Screenshot captured: 900x600px, ${screenshotBase64.length} chars, data URI length: ${screenshot.length}`);
                 executionLog += `\nScreenshot captured at 900x600 resolution (1:1 scale with viewport)`;
@@ -218,8 +426,21 @@ export class ServerBrowserSession {
                     fullPage: false,
                 });
                 if (fallbackBytes && fallbackBytes.length > 0) {
-                    const fallbackBase64 = fallbackBytes.toString('base64');
-                    screenshot = `data:image/png;base64,${fallbackBase64}`;
+                    // Compress fallback screenshot if needed
+                    let finalFallbackBytes = fallbackBytes;
+                    let finalFallbackMimeType = 'image/png';
+                    if (shouldCompressImage(fallbackBytes, 'image/png')) {
+                        try {
+                            const compressionResult = await compressImage(fallbackBytes, 'image/png');
+                            finalFallbackBytes = compressionResult.buffer;
+                            finalFallbackMimeType = compressionResult.mimeType;
+                        }
+                        catch (error) {
+                            console.warn(`[Fallback Screenshot Compression] Failed: ${error instanceof Error ? error.message : String(error)}`);
+                        }
+                    }
+                    const fallbackBase64 = finalFallbackBytes.toString('base64');
+                    screenshot = `data:${finalFallbackMimeType};base64,${fallbackBase64}`;
                     console.log(`Fallback screenshot captured: ${fallbackBase64.length} chars`);
                     executionLog += `\nFallback screenshot captured successfully`;
                 }
@@ -258,7 +479,11 @@ export class ServerBrowserSession {
     }
     async navigateToUrl(url) {
         if (!this.page || !this.browser) {
-            const launchResult = await this.launchBrowser();
+            const launchResult = await this.launchBrowser({
+                record: true,
+                videoDir: path.join(process.cwd(), 'videos'),
+                videoName: `browser_session_${Date.now()}.webm`,
+            });
             if (!launchResult.execution_success) {
                 return launchResult;
             }
@@ -366,18 +591,155 @@ export class ServerBrowserSession {
         return this.doAction(async () => {
             if (!this.page)
                 throw new Error('Page not available');
-            await this.page.evaluate("window.scrollBy({top: 400, behavior: 'auto'})");
+            // Try to find and scroll the most appropriate scrollable element
+            const scrollResult = await this.page.evaluate(() => {
+                // Helper function to check if element is scrollable
+                const isScrollable = (el) => {
+                    const style = window.getComputedStyle(el);
+                    const overflowY = style.overflowY;
+                    const hasScrollableContent = el.scrollHeight > el.clientHeight;
+                    const isScrollableStyle = overflowY === 'auto' || overflowY === 'scroll';
+                    return hasScrollableContent && (isScrollableStyle || el === document.documentElement);
+                };
+                // Helper function to check if element is visible
+                const isVisible = (el) => {
+                    const rect = el.getBoundingClientRect();
+                    const style = window.getComputedStyle(el);
+                    return (rect.width > 0 &&
+                        rect.height > 0 &&
+                        style.display !== 'none' &&
+                        style.visibility !== 'hidden' &&
+                        style.opacity !== '0');
+                };
+                // Find all scrollable elements
+                const allElements = Array.from(document.querySelectorAll('*'));
+                const scrollableElements = allElements.filter(el => isScrollable(el) && isVisible(el));
+                // Sort by size (prefer larger scrollable areas) and scroll position
+                scrollableElements.sort((a, b) => {
+                    const aArea = a.clientHeight * a.clientWidth;
+                    const bArea = b.clientHeight * b.clientWidth;
+                    return bArea - aArea;
+                });
+                let scrolled = false;
+                let scrolledElement = 'none';
+                const scrollAmount = 600; // Match viewport height
+                // Try to scroll the best candidate
+                if (scrollableElements.length > 0) {
+                    for (const element of scrollableElements) {
+                        const beforeScroll = element.scrollTop;
+                        const maxScroll = element.scrollHeight - element.clientHeight;
+                        if (beforeScroll < maxScroll) {
+                            element.scrollBy({ top: scrollAmount, behavior: 'auto' });
+                            const afterScroll = element.scrollTop;
+                            if (afterScroll > beforeScroll) {
+                                scrolled = true;
+                                scrolledElement = element.tagName + (element.id ? `#${element.id}` : '') +
+                                    (element.className ? `.${Array.from(element.classList).join('.')}` : '');
+                                break;
+                            }
+                        }
+                    }
+                }
+                // Fallback to window scrolling if no scrollable element found or scroll failed
+                if (!scrolled) {
+                    const beforeScroll = window.scrollY || document.documentElement.scrollTop;
+                    window.scrollBy({ top: scrollAmount, behavior: 'auto' });
+                    const afterScroll = window.scrollY || document.documentElement.scrollTop;
+                    if (afterScroll > beforeScroll) {
+                        scrolled = true;
+                        scrolledElement = 'window';
+                    }
+                }
+                return {
+                    success: scrolled,
+                    element: scrolledElement,
+                    foundScrollableElements: scrollableElements.length
+                };
+            });
             await new Promise((resolve) => setTimeout(resolve, 300));
-            return 'Scroll down action performed!';
+            if (scrollResult.success) {
+                return `Scroll down action performed! Scrolled element: ${scrollResult.element} (found ${scrollResult.foundScrollableElements} scrollable elements)`;
+            }
+            else {
+                return `Scroll down attempted but page may be at bottom or no scrollable content found (checked ${scrollResult.foundScrollableElements} elements)`;
+            }
         });
     }
     async scrollUp() {
         return this.doAction(async () => {
             if (!this.page)
                 throw new Error('Page not available');
-            await this.page.evaluate("window.scrollBy({top: -600, behavior: 'auto'})");
+            // Try to find and scroll the most appropriate scrollable element
+            const scrollResult = await this.page.evaluate(() => {
+                // Helper function to check if element is scrollable
+                const isScrollable = (el) => {
+                    const style = window.getComputedStyle(el);
+                    const overflowY = style.overflowY;
+                    const hasScrollableContent = el.scrollHeight > el.clientHeight;
+                    const isScrollableStyle = overflowY === 'auto' || overflowY === 'scroll';
+                    return hasScrollableContent && (isScrollableStyle || el === document.documentElement);
+                };
+                // Helper function to check if element is visible
+                const isVisible = (el) => {
+                    const rect = el.getBoundingClientRect();
+                    const style = window.getComputedStyle(el);
+                    return (rect.width > 0 &&
+                        rect.height > 0 &&
+                        style.display !== 'none' &&
+                        style.visibility !== 'hidden' &&
+                        style.opacity !== '0');
+                };
+                // Find all scrollable elements
+                const allElements = Array.from(document.querySelectorAll('*'));
+                const scrollableElements = allElements.filter(el => isScrollable(el) && isVisible(el));
+                // Sort by size (prefer larger scrollable areas)
+                scrollableElements.sort((a, b) => {
+                    const aArea = a.clientHeight * a.clientWidth;
+                    const bArea = b.clientHeight * b.clientWidth;
+                    return bArea - aArea;
+                });
+                let scrolled = false;
+                let scrolledElement = 'none';
+                const scrollAmount = 600; // Match viewport height
+                // Try to scroll the best candidate
+                if (scrollableElements.length > 0) {
+                    for (const element of scrollableElements) {
+                        const beforeScroll = element.scrollTop;
+                        if (beforeScroll > 0) {
+                            element.scrollBy({ top: -scrollAmount, behavior: 'auto' });
+                            const afterScroll = element.scrollTop;
+                            if (afterScroll < beforeScroll) {
+                                scrolled = true;
+                                scrolledElement = element.tagName + (element.id ? `#${element.id}` : '') +
+                                    (element.className ? `.${Array.from(element.classList).join('.')}` : '');
+                                break;
+                            }
+                        }
+                    }
+                }
+                // Fallback to window scrolling if no scrollable element found or scroll failed
+                if (!scrolled) {
+                    const beforeScroll = window.scrollY || document.documentElement.scrollTop;
+                    window.scrollBy({ top: -scrollAmount, behavior: 'auto' });
+                    const afterScroll = window.scrollY || document.documentElement.scrollTop;
+                    if (afterScroll < beforeScroll) {
+                        scrolled = true;
+                        scrolledElement = 'window';
+                    }
+                }
+                return {
+                    success: scrolled,
+                    element: scrolledElement,
+                    foundScrollableElements: scrollableElements.length
+                };
+            });
             await new Promise((resolve) => setTimeout(resolve, 300));
-            return 'Scroll up action performed!';
+            if (scrollResult.success) {
+                return `Scroll up action performed! Scrolled element: ${scrollResult.element} (found ${scrollResult.foundScrollableElements} scrollable elements)`;
+            }
+            else {
+                return `Scroll up attempted but page may be at top or no scrollable content found (checked ${scrollResult.foundScrollableElements} elements)`;
+            }
         });
     }
 }
@@ -390,7 +752,11 @@ class BrowserLaunchToolInvocation extends BaseToolInvocation {
     }
     async execute() {
         const session = ServerBrowserSession.getInstance();
-        const result = await session.launchBrowser();
+        const result = await session.launchBrowser({
+            record: this.params.record,
+            videoDir: this.params.videoDir,
+            videoName: this.params.videoName,
+        });
         console.log('[BrowserLaunchTool] Browser launch completed', {
             success: result.execution_success,
         });
@@ -415,9 +781,22 @@ export class BrowserLaunchTool extends BaseDeclarativeTool {
     static Name = ToolNames.BROWSER_LAUNCH;
     // @ts-expect-error - Required by base class pattern
     constructor(config) {
-        super(BrowserLaunchTool.Name, 'BrowserLaunch', 'Launches a Puppeteer-controlled browser instance with a 900x600 viewport. This must always be the first browser action before any other browser operations.', Kind.Execute, {
+        super(BrowserLaunchTool.Name, 'BrowserLaunch', 'Launches a Playwright-controlled browser instance with a 900x600 viewport. Optionally records the session to a .webm video.', Kind.Execute, {
             type: 'object',
-            properties: {},
+            properties: {
+                record: {
+                    type: 'boolean',
+                    description: 'Enable screen recording for the session (video saved on close)',
+                },
+                videoDir: {
+                    type: 'string',
+                    description: 'Directory to save the recording (absolute path preferred). Defaults to "<cwd>/videos".',
+                },
+                videoName: {
+                    type: 'string',
+                    description: 'Filename for the recording (".webm" appended if missing). If omitted, Playwright default name is used.',
+                },
+            },
             required: [],
         }, false);
         this.config = config;
@@ -844,6 +1223,7 @@ class BrowserCloseToolInvocation extends BaseToolInvocation {
         return {
             llmContent: result.execution_logs || 'Browser closed successfully',
             returnDisplay: result.logs || 'Browser closed successfully',
+            videoPath: result.videoPath, // Surface the saved video path
         };
     }
 }