@blackbox_ai/blackbox-cli-core 0.0.7 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -183
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/src/blackbox/blackboxOAuth2.js +17 -1
- package/dist/src/blackbox/blackboxOAuth2.js.map +1 -1
- package/dist/src/code_assist/oauth2.js +15 -3
- package/dist/src/code_assist/oauth2.js.map +1 -1
- package/dist/src/config/blackboxModels.d.ts +3 -2
- package/dist/src/config/blackboxModels.js +262 -33
- package/dist/src/config/blackboxModels.js.map +1 -1
- package/dist/src/config/config.d.ts +65 -0
- package/dist/src/config/config.js +282 -17
- package/dist/src/config/config.js.map +1 -1
- package/dist/src/config/models.d.ts +1 -1
- package/dist/src/config/models.js +1 -1
- package/dist/src/config/models.js.map +1 -1
- package/dist/src/config/multiAgentModels.d.ts +63 -0
- package/dist/src/config/multiAgentModels.js +194 -0
- package/dist/src/config/multiAgentModels.js.map +1 -0
- package/dist/src/core/client.js +11 -5
- package/dist/src/core/client.js.map +1 -1
- package/dist/src/core/contentGenerator.d.ts +1 -0
- package/dist/src/core/contentGenerator.js +57 -7
- package/dist/src/core/contentGenerator.js.map +1 -1
- package/dist/src/core/coreToolScheduler.js +2 -2
- package/dist/src/core/coreToolScheduler.js.map +1 -1
- package/dist/src/core/encryptedClientFactory.d.ts +17 -0
- package/dist/src/core/encryptedClientFactory.js +92 -0
- package/dist/src/core/encryptedClientFactory.js.map +1 -0
- package/dist/src/core/encryptedContentGenerator.d.ts +47 -0
- package/dist/src/core/encryptedContentGenerator.js +445 -0
- package/dist/src/core/encryptedContentGenerator.js.map +1 -0
- package/dist/src/core/encryptedGeminiClient.d.ts +59 -0
- package/dist/src/core/encryptedGeminiClient.js +177 -0
- package/dist/src/core/encryptedGeminiClient.js.map +1 -0
- package/dist/src/core/encryptedGeminiClientBridge.d.ts +107 -0
- package/dist/src/core/encryptedGeminiClientBridge.js +808 -0
- package/dist/src/core/encryptedGeminiClientBridge.js.map +1 -0
- package/dist/src/core/encryptedGeminiClientWrapper.d.ts +129 -0
- package/dist/src/core/encryptedGeminiClientWrapper.js +305 -0
- package/dist/src/core/encryptedGeminiClientWrapper.js.map +1 -0
- package/dist/src/core/encryptedTurn.d.ts +40 -0
- package/dist/src/core/encryptedTurn.js +114 -0
- package/dist/src/core/encryptedTurn.js.map +1 -0
- package/dist/src/core/logger.d.ts +21 -0
- package/dist/src/core/logger.js +110 -0
- package/dist/src/core/logger.js.map +1 -1
- package/dist/src/core/openaiContentGenerator/constants.d.ts +2 -0
- package/dist/src/core/openaiContentGenerator/constants.js +2 -0
- package/dist/src/core/openaiContentGenerator/constants.js.map +1 -1
- package/dist/src/core/openaiContentGenerator/converter.d.ts +16 -1
- package/dist/src/core/openaiContentGenerator/converter.js +135 -4
- package/dist/src/core/openaiContentGenerator/converter.js.map +1 -1
- package/dist/src/core/openaiContentGenerator/pipeline.js +22 -8
- package/dist/src/core/openaiContentGenerator/pipeline.js.map +1 -1
- package/dist/src/core/openaiContentGenerator/pipeline.test.js +51 -0
- package/dist/src/core/openaiContentGenerator/pipeline.test.js.map +1 -1
- package/dist/src/core/openaiContentGenerator/provider/default.js +10 -1
- package/dist/src/core/openaiContentGenerator/provider/default.js.map +1 -1
- package/dist/src/core/prompts.d.ts +18 -1
- package/dist/src/core/prompts.js +388 -459
- package/dist/src/core/prompts.js.map +1 -1
- package/dist/src/core/tokenLimits.d.ts +1 -0
- package/dist/src/core/tokenLimits.js +37 -2
- package/dist/src/core/tokenLimits.js.map +1 -1
- package/dist/src/core/tokenLimits.test.js +36 -1
- package/dist/src/core/tokenLimits.test.js.map +1 -1
- package/dist/src/encrypt/attestation.d.ts +5 -0
- package/dist/src/encrypt/attestation.js +100 -0
- package/dist/src/encrypt/attestation.js.map +1 -0
- package/dist/src/encrypt/client.d.ts +14 -0
- package/dist/src/encrypt/client.js +132 -0
- package/dist/src/encrypt/client.js.map +1 -0
- package/dist/src/encrypt/config.d.ts +22 -0
- package/dist/src/encrypt/config.js +43 -0
- package/dist/src/encrypt/config.js.map +1 -0
- package/dist/src/encrypt/crypto-utils.d.ts +57 -0
- package/dist/src/encrypt/crypto-utils.js +257 -0
- package/dist/src/encrypt/crypto-utils.js.map +1 -0
- package/dist/src/encrypt/history-manager.d.ts +43 -0
- package/dist/src/encrypt/history-manager.js +164 -0
- package/dist/src/encrypt/history-manager.js.map +1 -0
- package/dist/src/encrypt/minimax-template.d.ts +73 -0
- package/dist/src/encrypt/minimax-template.js +276 -0
- package/dist/src/encrypt/minimax-template.js.map +1 -0
- package/dist/src/encrypt/sessions.d.ts +17 -0
- package/dist/src/encrypt/sessions.js +221 -0
- package/dist/src/encrypt/sessions.js.map +1 -0
- package/dist/src/encrypt/streaming-client.d.ts +29 -0
- package/dist/src/encrypt/streaming-client.js +232 -0
- package/dist/src/encrypt/streaming-client.js.map +1 -0
- package/dist/src/encrypt/tool-formatter.d.ts +36 -0
- package/dist/src/encrypt/tool-formatter.js +353 -0
- package/dist/src/encrypt/tool-formatter.js.map +1 -0
- package/dist/src/encrypt/tool-parser.d.ts +93 -0
- package/dist/src/encrypt/tool-parser.js +567 -0
- package/dist/src/encrypt/tool-parser.js.map +1 -0
- package/dist/src/encrypt/types.d.ts +81 -0
- package/dist/src/encrypt/types.js +2 -0
- package/dist/src/encrypt/types.js.map +1 -0
- package/dist/src/generated/git-commit.d.ts +3 -3
- package/dist/src/generated/git-commit.js +3 -3
- package/dist/src/ide/ide-client.js +9 -19
- package/dist/src/ide/ide-client.js.map +1 -1
- package/dist/src/index.d.ts +15 -0
- package/dist/src/index.js +15 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/mcp/oauth-provider.js +2 -6
- package/dist/src/mcp/oauth-provider.js.map +1 -1
- package/dist/src/mcp/oauth-token-storage.d.ts +7 -0
- package/dist/src/mcp/oauth-token-storage.js +24 -0
- package/dist/src/mcp/oauth-token-storage.js.map +1 -1
- package/dist/src/services/EncryptedChatService.d.ts +80 -0
- package/dist/src/services/EncryptedChatService.js +202 -0
- package/dist/src/services/EncryptedChatService.js.map +1 -0
- package/dist/src/services/StatsHistoryService.d.ts +131 -0
- package/dist/src/services/StatsHistoryService.js +427 -0
- package/dist/src/services/StatsHistoryService.js.map +1 -0
- package/dist/src/services/checkpointApiService.d.ts +101 -0
- package/dist/src/services/checkpointApiService.js +215 -0
- package/dist/src/services/checkpointApiService.js.map +1 -0
- package/dist/src/services/environmentSanitization.d.ts +24 -0
- package/dist/src/services/environmentSanitization.js +152 -0
- package/dist/src/services/environmentSanitization.js.map +1 -0
- package/dist/src/telemetry/blackbox-logger/blackbox-logger.d.ts +2 -6
- package/dist/src/telemetry/blackbox-logger/blackbox-logger.js +29 -135
- package/dist/src/telemetry/blackbox-logger/blackbox-logger.js.map +1 -1
- package/dist/src/telemetry/blackbox-logger/blackbox-logger.test.js +1 -1
- package/dist/src/telemetry/blackbox-logger/blackbox-logger.test.js.map +1 -1
- package/dist/src/telemetry/uiTelemetry.d.ts +8 -0
- package/dist/src/telemetry/uiTelemetry.js +17 -0
- package/dist/src/telemetry/uiTelemetry.js.map +1 -1
- package/dist/src/tools/browser-interactive.d.ts +63 -0
- package/dist/src/tools/browser-interactive.js +394 -0
- package/dist/src/tools/browser-interactive.js.map +1 -0
- package/dist/src/tools/browser_use.d.ts +23 -2
- package/dist/src/tools/browser_use.js +424 -43
- package/dist/src/tools/browser_use.js.map +1 -1
- package/dist/src/tools/data-file-constants.d.ts +17 -0
- package/dist/src/tools/data-file-constants.js +30 -0
- package/dist/src/tools/data-file-constants.js.map +1 -0
- package/dist/src/tools/edit.js +44 -7
- package/dist/src/tools/edit.js.map +1 -1
- package/dist/src/tools/exitPlanMode.js +1 -1
- package/dist/src/tools/exitPlanMode.js.map +1 -1
- package/dist/src/tools/ls.js +40 -6
- package/dist/src/tools/ls.js.map +1 -1
- package/dist/src/tools/ls.test.js +4 -4
- package/dist/src/tools/ls.test.js.map +1 -1
- package/dist/src/tools/mcp-client-manager.d.ts +28 -2
- package/dist/src/tools/mcp-client-manager.js +62 -4
- package/dist/src/tools/mcp-client-manager.js.map +1 -1
- package/dist/src/tools/mcp-client.d.ts +5 -3
- package/dist/src/tools/mcp-client.js +39 -11
- package/dist/src/tools/mcp-client.js.map +1 -1
- package/dist/src/tools/mcp-tool.d.ts +3 -1
- package/dist/src/tools/mcp-tool.js +37 -9
- package/dist/src/tools/mcp-tool.js.map +1 -1
- package/dist/src/tools/memoryTool.d.ts +14 -4
- package/dist/src/tools/memoryTool.js +98 -39
- package/dist/src/tools/memoryTool.js.map +1 -1
- package/dist/src/tools/read-data-file.d.ts +31 -0
- package/dist/src/tools/read-data-file.js +469 -0
- package/dist/src/tools/read-data-file.js.map +1 -0
- package/dist/src/tools/read-file.js +64 -5
- package/dist/src/tools/read-file.js.map +1 -1
- package/dist/src/tools/read-file.test.js +40 -6
- package/dist/src/tools/read-file.test.js.map +1 -1
- package/dist/src/tools/shell.d.ts +3 -1
- package/dist/src/tools/shell.js +25 -4
- package/dist/src/tools/shell.js.map +1 -1
- package/dist/src/tools/skill.d.ts +34 -0
- package/dist/src/tools/skill.js +143 -0
- package/dist/src/tools/skill.js.map +1 -0
- package/dist/src/tools/sql_db.d.ts +101 -0
- package/dist/src/tools/sql_db.js +1033 -0
- package/dist/src/tools/sql_db.js.map +1 -0
- package/dist/src/tools/sql_db_configure.d.ts +18 -0
- package/dist/src/tools/sql_db_configure.js +96 -0
- package/dist/src/tools/sql_db_configure.js.map +1 -0
- package/dist/src/tools/taskCompletion.d.ts +29 -0
- package/dist/src/tools/taskCompletion.js +231 -0
- package/dist/src/tools/taskCompletion.js.map +1 -0
- package/dist/src/tools/todoWrite.js +0 -142
- package/dist/src/tools/todoWrite.js.map +1 -1
- package/dist/src/tools/tool-error.d.ts +3 -1
- package/dist/src/tools/tool-error.js +3 -0
- package/dist/src/tools/tool-error.js.map +1 -1
- package/dist/src/tools/tool-names.d.ts +8 -0
- package/dist/src/tools/tool-names.js +8 -0
- package/dist/src/tools/tool-names.js.map +1 -1
- package/dist/src/tools/tool-registry.d.ts +22 -0
- package/dist/src/tools/tool-registry.js +41 -1
- package/dist/src/tools/tool-registry.js.map +1 -1
- package/dist/src/tools/tools.d.ts +18 -2
- package/dist/src/tools/tools.js +3 -0
- package/dist/src/tools/tools.js.map +1 -1
- package/dist/src/tools/web-fetch.js +24 -4
- package/dist/src/tools/web-fetch.js.map +1 -1
- package/dist/src/tools/web-search.js +160 -2
- package/dist/src/tools/web-search.js.map +1 -1
- package/dist/src/tools/workspace-error-helper.d.ts +9 -0
- package/dist/src/tools/workspace-error-helper.js +43 -0
- package/dist/src/tools/workspace-error-helper.js.map +1 -0
- package/dist/src/tools/workspace-error-helper.test.js +85 -0
- package/dist/src/tools/workspace-error-helper.test.js.map +1 -0
- package/dist/src/tools/write-file.js +42 -7
- package/dist/src/tools/write-file.js.map +1 -1
- package/dist/src/utils/environmentContext.js +3 -1
- package/dist/src/utils/environmentContext.js.map +1 -1
- package/dist/src/utils/environmentContext.test.js +3 -2
- package/dist/src/utils/environmentContext.test.js.map +1 -1
- package/dist/src/utils/fetch.d.ts +3 -1
- package/dist/src/utils/fetch.js +35 -2
- package/dist/src/utils/fetch.js.map +1 -1
- package/dist/src/utils/fileUtils.js +30 -3
- package/dist/src/utils/fileUtils.js.map +1 -1
- package/dist/src/utils/filesearch/fileSearch.d.ts +2 -0
- package/dist/src/utils/filesearch/fileSearch.js +38 -7
- package/dist/src/utils/filesearch/fileSearch.js.map +1 -1
- package/dist/src/utils/git-worktree-utils.d.ts +56 -0
- package/dist/src/utils/git-worktree-utils.js +176 -0
- package/dist/src/utils/git-worktree-utils.js.map +1 -0
- package/dist/src/utils/imageCompression.d.ts +34 -0
- package/dist/src/utils/imageCompression.js +170 -0
- package/dist/src/utils/imageCompression.js.map +1 -0
- package/dist/src/utils/messageTruncator.d.ts +51 -0
- package/dist/src/utils/messageTruncator.js +346 -0
- package/dist/src/utils/messageTruncator.js.map +1 -0
- package/dist/src/utils/pathReader.js +26 -6
- package/dist/src/utils/pathReader.js.map +1 -1
- package/dist/src/utils/skill.d.ts +65 -0
- package/dist/src/utils/skill.js +241 -0
- package/dist/src/utils/skill.js.map +1 -0
- package/dist/src/utils/textCleaning.d.ts +51 -0
- package/dist/src/utils/textCleaning.js +327 -0
- package/dist/src/utils/textCleaning.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +19 -6
- package/dist/src/tools/mcp-client-manager.test.js +0 -39
- package/dist/src/tools/mcp-client-manager.test.js.map +0 -1
- package/dist/src/tools/mcp-client.test.d.ts +0 -6
- package/dist/src/tools/mcp-client.test.js +0 -454
- package/dist/src/tools/mcp-client.test.js.map +0 -1
- package/dist/src/tools/mcp-tool.test.d.ts +0 -6
- package/dist/src/tools/mcp-tool.test.js +0 -576
- package/dist/src/tools/mcp-tool.test.js.map +0 -1
- package/dist/src/tools/memoryTool.test.d.ts +0 -6
- package/dist/src/tools/memoryTool.test.js +0 -420
- package/dist/src/tools/memoryTool.test.js.map +0 -1
- package/dist/src/tools/tool-registry.test.d.ts +0 -6
- package/dist/src/tools/tool-registry.test.js +0 -332
- package/dist/src/tools/tool-registry.test.js.map +0 -1
- /package/dist/src/tools/{mcp-client-manager.test.d.ts → workspace-error-helper.test.d.ts} +0 -0
|
@@ -4,9 +4,14 @@
|
|
|
4
4
|
* SPDX-License-Identifier: Apache-2.0
|
|
5
5
|
*/
|
|
6
6
|
import { chromium } from 'playwright';
|
|
7
|
+
import path from 'node:path';
|
|
8
|
+
import { mkdir, rename, readFile, stat } from 'node:fs/promises';
|
|
9
|
+
import { randomUUID } from 'node:crypto';
|
|
7
10
|
import { ToolNames } from './tool-names.js';
|
|
8
11
|
import { ToolErrorType } from './tool-error.js';
|
|
9
12
|
import { BaseDeclarativeTool, BaseToolInvocation, Kind, } from './tools.js';
|
|
13
|
+
import { SignJWT } from 'jose';
|
|
14
|
+
import { compressImage, shouldCompressImage } from '../utils/imageCompression.js';
|
|
10
15
|
const screenshotPrompt = `Here are the action result, console logs and screenshot after the action execution.
|
|
11
16
|
Carefully review and decide the next steps to complete the task successfully.`;
|
|
12
17
|
/**
|
|
@@ -14,9 +19,15 @@ Carefully review and decide the next steps to complete the task successfully.`;
|
|
|
14
19
|
*/
|
|
15
20
|
export class ServerBrowserSession {
|
|
16
21
|
browser;
|
|
22
|
+
context; // added
|
|
17
23
|
page;
|
|
18
24
|
currentMousePosition;
|
|
19
25
|
static instance;
|
|
26
|
+
// recording config/state
|
|
27
|
+
recordingEnabled = false; // added
|
|
28
|
+
recordingDirAbs; // added (absolute directory)
|
|
29
|
+
recordingName; // added (desired filename, optional)
|
|
30
|
+
lastVideoPath; // added (resolved saved path)
|
|
20
31
|
constructor() { }
|
|
21
32
|
static getInstance() {
|
|
22
33
|
if (!ServerBrowserSession.instance) {
|
|
@@ -24,7 +35,7 @@ export class ServerBrowserSession {
|
|
|
24
35
|
}
|
|
25
36
|
return ServerBrowserSession.instance;
|
|
26
37
|
}
|
|
27
|
-
async launchBrowser() {
|
|
38
|
+
async launchBrowser(opts) {
|
|
28
39
|
console.log('Launching browser...');
|
|
29
40
|
if (this.browser) {
|
|
30
41
|
await this.closeBrowser();
|
|
@@ -43,18 +54,39 @@ export class ServerBrowserSession {
|
|
|
43
54
|
'--disable-background-timer-throttling',
|
|
44
55
|
'--disable-backgrounding-occluded-windows',
|
|
45
56
|
'--disable-renderer-backgrounding',
|
|
57
|
+
'--use-gl=swiftshader',
|
|
46
58
|
],
|
|
47
59
|
headless: true,
|
|
48
60
|
});
|
|
49
|
-
|
|
61
|
+
// Configure recording
|
|
62
|
+
this.recordingEnabled = !!opts?.record;
|
|
63
|
+
this.recordingName = opts?.videoName?.trim() || undefined;
|
|
64
|
+
if (this.recordingName && !this.recordingName.endsWith('.webm')) {
|
|
65
|
+
this.recordingName = `${this.recordingName}.webm`;
|
|
66
|
+
}
|
|
67
|
+
if (this.recordingEnabled) {
|
|
68
|
+
const dir = opts?.videoDir?.trim();
|
|
69
|
+
const abs = dir && path.isAbsolute(dir) ? dir : path.resolve(process.cwd(), dir || 'videos');
|
|
70
|
+
await mkdir(abs, { recursive: true });
|
|
71
|
+
this.recordingDirAbs = abs;
|
|
72
|
+
console.log(`[Recording] Enabled. Output dir: ${this.recordingDirAbs}${this.recordingName ? `, name: ${this.recordingName}` : ''}`);
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
this.recordingDirAbs = undefined;
|
|
76
|
+
this.recordingName = undefined;
|
|
77
|
+
}
|
|
78
|
+
// Create context with optional video
|
|
79
|
+
this.context = await this.browser.newContext({
|
|
50
80
|
viewport: { width: 900, height: 600 },
|
|
51
81
|
deviceScaleFactor: 1,
|
|
52
82
|
screen: { width: 900, height: 600 },
|
|
53
83
|
ignoreHTTPSErrors: true,
|
|
54
84
|
bypassCSP: true,
|
|
85
|
+
...(this.recordingEnabled && this.recordingDirAbs
|
|
86
|
+
? { recordVideo: { dir: this.recordingDirAbs, size: { width: 900, height: 600 } } }
|
|
87
|
+
: {}),
|
|
55
88
|
});
|
|
56
|
-
this.page = await context.newPage();
|
|
57
|
-
// Set a default background to ensure screenshots work
|
|
89
|
+
this.page = await this.context.newPage();
|
|
58
90
|
await this.page.addStyleTag({
|
|
59
91
|
content: `
|
|
60
92
|
html, body {
|
|
@@ -63,15 +95,18 @@ export class ServerBrowserSession {
|
|
|
63
95
|
}
|
|
64
96
|
`,
|
|
65
97
|
});
|
|
66
|
-
|
|
98
|
+
const recordNote = this.recordingEnabled
|
|
99
|
+
? ` with recording to ${this.recordingDirAbs}${this.recordingName ? `/${this.recordingName}` : ''}`
|
|
100
|
+
: '';
|
|
101
|
+
console.log('Browser launched successfully with 900x600 viewport' + recordNote);
|
|
67
102
|
return {
|
|
68
103
|
execution_success: true,
|
|
69
|
-
logs: 'Browser session started successfully with 900x600 viewport',
|
|
70
|
-
execution_logs: 'Browser launched and ready for interaction at 900x600 resolution',
|
|
104
|
+
logs: 'Browser session started successfully with 900x600 viewport' + recordNote,
|
|
105
|
+
execution_logs: 'Browser launched and ready for interaction at 900x600 resolution' + recordNote,
|
|
71
106
|
};
|
|
72
107
|
}
|
|
73
|
-
catch (
|
|
74
|
-
const errorMessage = `Failed to launch browser: ${
|
|
108
|
+
catch (_error) {
|
|
109
|
+
const errorMessage = `Failed to launch browser: ${_error instanceof Error ? _error.message : String(_error)}`;
|
|
75
110
|
console.error(`[Error] Exception during Starting browser - ${errorMessage}`);
|
|
76
111
|
return {
|
|
77
112
|
execution_success: false,
|
|
@@ -80,22 +115,180 @@ export class ServerBrowserSession {
|
|
|
80
115
|
};
|
|
81
116
|
}
|
|
82
117
|
}
|
|
118
|
+
/* generate access token, jwt token with secret key */
|
|
119
|
+
async generateAccessToken() {
|
|
120
|
+
const secret = process.env['VIDEO_STORAGE_SECRET_KEY'];
|
|
121
|
+
if (!secret) {
|
|
122
|
+
throw new Error('VIDEO_STORAGE_SECRET_KEY is not set');
|
|
123
|
+
}
|
|
124
|
+
const expiresInSeconds = 3600; // 1 hour
|
|
125
|
+
const payload = {
|
|
126
|
+
sub: randomUUID(),
|
|
127
|
+
};
|
|
128
|
+
const secretKey = new TextEncoder().encode(secret);
|
|
129
|
+
return new SignJWT(payload)
|
|
130
|
+
.setProtectedHeader({ alg: 'HS256', typ: 'JWT' })
|
|
131
|
+
.setIssuedAt()
|
|
132
|
+
.setExpirationTime(Math.floor(Date.now() / 1000) + expiresInSeconds)
|
|
133
|
+
.sign(secretKey);
|
|
134
|
+
}
|
|
135
|
+
async uploadToStorage(videoPath, videoName) {
|
|
136
|
+
try {
|
|
137
|
+
const accessToken = await this.generateAccessToken();
|
|
138
|
+
const baseUrl = process.env['VIDEO_STORAGE_API_URL'];
|
|
139
|
+
if (!baseUrl) {
|
|
140
|
+
return { execution_success: false, error: 'VIDEO_STORAGE_API_URL is not set' };
|
|
141
|
+
}
|
|
142
|
+
// Read bytes once
|
|
143
|
+
const bytes = await readFile(videoPath);
|
|
144
|
+
// Attempt direct multipart /upload first
|
|
145
|
+
{
|
|
146
|
+
const form = new FormData();
|
|
147
|
+
const blob = new Blob([bytes], { type: 'video/webm' });
|
|
148
|
+
form.append('file', blob, videoName);
|
|
149
|
+
const uploadResponse = await fetch(`${baseUrl}/upload`, {
|
|
150
|
+
method: 'POST',
|
|
151
|
+
headers: { Authorization: `Bearer ${accessToken}` },
|
|
152
|
+
body: form,
|
|
153
|
+
});
|
|
154
|
+
if (uploadResponse.ok) {
|
|
155
|
+
const json = await uploadResponse.json();
|
|
156
|
+
if (json.public_url) {
|
|
157
|
+
return { execution_success: true, videoPath: json.public_url };
|
|
158
|
+
}
|
|
159
|
+
// fall through to signed-url if schema unexpected
|
|
160
|
+
}
|
|
161
|
+
else {
|
|
162
|
+
const text = await uploadResponse.text().catch(() => '');
|
|
163
|
+
const lower = text.toLowerCase();
|
|
164
|
+
const isBodyParseError = uploadResponse.status === 400 && (lower.includes('parse') || lower.includes('parsing'));
|
|
165
|
+
if (!isBodyParseError) {
|
|
166
|
+
return {
|
|
167
|
+
execution_success: false,
|
|
168
|
+
error: `Upload failed (${uploadResponse.status}): ${text}`,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
// Fallback: use signed URL flow to bypass multipart parser
|
|
174
|
+
{
|
|
175
|
+
const signedResp = await fetch(`${baseUrl}/signed-url`, {
|
|
176
|
+
method: 'POST',
|
|
177
|
+
headers: {
|
|
178
|
+
Authorization: `Bearer ${accessToken}`,
|
|
179
|
+
'Content-Type': 'application/json',
|
|
180
|
+
},
|
|
181
|
+
body: JSON.stringify({ filename: videoName, content_type: 'video/webm' }),
|
|
182
|
+
});
|
|
183
|
+
if (!signedResp.ok) {
|
|
184
|
+
const text = await signedResp.text().catch(() => '');
|
|
185
|
+
return {
|
|
186
|
+
execution_success: false,
|
|
187
|
+
error: `Signed URL request failed (${signedResp.status}): ${text}`,
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
const signedJson = await signedResp.json();
|
|
191
|
+
if (!signedJson.signed_put_url || !signedJson.public_url) {
|
|
192
|
+
return { execution_success: false, error: 'Invalid signed-url response' };
|
|
193
|
+
}
|
|
194
|
+
const putResp = await fetch(signedJson.signed_put_url, {
|
|
195
|
+
method: 'PUT',
|
|
196
|
+
headers: { 'Content-Type': 'video/webm' },
|
|
197
|
+
body: bytes,
|
|
198
|
+
});
|
|
199
|
+
if (!putResp.ok) {
|
|
200
|
+
const t = await putResp.text().catch(() => '');
|
|
201
|
+
return { execution_success: false, error: `Signed PUT failed (${putResp.status}): ${t}` };
|
|
202
|
+
}
|
|
203
|
+
return { execution_success: true, videoPath: signedJson.public_url };
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
catch (error) {
|
|
207
|
+
console.error('Error uploading video to storage:', error);
|
|
208
|
+
return { execution_success: false, error: error instanceof Error ? error.message : String(error) };
|
|
209
|
+
}
|
|
210
|
+
}
|
|
83
211
|
async closeBrowser() {
|
|
84
212
|
if (this.browser || this.page) {
|
|
85
213
|
console.log('Closing browser...');
|
|
86
214
|
try {
|
|
215
|
+
let tmpVideoPath;
|
|
216
|
+
let finalVideoPath;
|
|
217
|
+
// Close page and get tmp video path
|
|
218
|
+
if (this.page) {
|
|
219
|
+
const v = this.page.video?.();
|
|
220
|
+
await this.page.close();
|
|
221
|
+
if (this.recordingEnabled && v) {
|
|
222
|
+
try {
|
|
223
|
+
tmpVideoPath = await v.path();
|
|
224
|
+
}
|
|
225
|
+
catch (e) {
|
|
226
|
+
console.warn('[Recording] Failed to obtain video path after page close:', e);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
// Close context to flush video to disk
|
|
231
|
+
await this.context?.close().catch(() => { });
|
|
232
|
+
// Finalize and optionally upload
|
|
233
|
+
if (this.recordingEnabled && tmpVideoPath && this.recordingDirAbs) {
|
|
234
|
+
try {
|
|
235
|
+
if (this.recordingName) {
|
|
236
|
+
const target = path.join(this.recordingDirAbs, this.recordingName);
|
|
237
|
+
await rename(tmpVideoPath, target);
|
|
238
|
+
finalVideoPath = target;
|
|
239
|
+
}
|
|
240
|
+
else {
|
|
241
|
+
finalVideoPath = tmpVideoPath;
|
|
242
|
+
}
|
|
243
|
+
// Wait briefly until file is non-zero
|
|
244
|
+
if (finalVideoPath) {
|
|
245
|
+
for (let i = 0; i < 10; i++) {
|
|
246
|
+
try {
|
|
247
|
+
const s = await stat(finalVideoPath);
|
|
248
|
+
if (s.size > 0)
|
|
249
|
+
break;
|
|
250
|
+
}
|
|
251
|
+
catch (_e) {
|
|
252
|
+
// no-op while file finalizes
|
|
253
|
+
}
|
|
254
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
this.lastVideoPath = finalVideoPath;
|
|
258
|
+
if (process.env['VIDEO_STORAGE_API_URL'] && finalVideoPath) {
|
|
259
|
+
const safeName = this.recordingName || path.basename(finalVideoPath);
|
|
260
|
+
const uploadResult = await this.uploadToStorage(finalVideoPath, safeName);
|
|
261
|
+
if (uploadResult.execution_success && uploadResult.videoPath) {
|
|
262
|
+
this.lastVideoPath = uploadResult.videoPath;
|
|
263
|
+
}
|
|
264
|
+
console.log(`[Recording] Video saved: ${this.lastVideoPath}`);
|
|
265
|
+
}
|
|
266
|
+
else if (finalVideoPath) {
|
|
267
|
+
console.log(`[Recording] Video saved: ${finalVideoPath}`);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
catch (e) {
|
|
271
|
+
console.warn('[Recording] Failed to finalize/save/upload video:', e);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
// Close browser
|
|
87
275
|
await this.browser?.close();
|
|
88
276
|
this.browser = undefined;
|
|
277
|
+
this.context = undefined;
|
|
89
278
|
this.page = undefined;
|
|
90
279
|
this.currentMousePosition = undefined;
|
|
280
|
+
const msg = this.lastVideoPath
|
|
281
|
+
? `Browser session closed successfully. Video saved to: ${this.lastVideoPath}`
|
|
282
|
+
: 'Browser session closed successfully';
|
|
91
283
|
return {
|
|
92
284
|
execution_success: true,
|
|
93
|
-
logs:
|
|
94
|
-
execution_logs:
|
|
285
|
+
logs: msg,
|
|
286
|
+
execution_logs: msg,
|
|
287
|
+
videoPath: this.lastVideoPath,
|
|
95
288
|
};
|
|
96
289
|
}
|
|
97
|
-
catch (
|
|
98
|
-
const errorMessage = `Error closing browser: ${
|
|
290
|
+
catch (_error) {
|
|
291
|
+
const errorMessage = `Error closing browser: ${_error instanceof Error ? _error.message : String(_error)}`;
|
|
99
292
|
console.warn(errorMessage);
|
|
100
293
|
return {
|
|
101
294
|
execution_success: false,
|
|
@@ -138,8 +331,8 @@ export class ServerBrowserSession {
|
|
|
138
331
|
await new Promise((resolve) => setTimeout(resolve, checkDurationMs));
|
|
139
332
|
checkCounts++;
|
|
140
333
|
}
|
|
141
|
-
catch (
|
|
142
|
-
console.warn('Error checking HTML stability:',
|
|
334
|
+
catch (_error) {
|
|
335
|
+
console.warn('Error checking HTML stability:', _error);
|
|
143
336
|
break;
|
|
144
337
|
}
|
|
145
338
|
}
|
|
@@ -164,8 +357,8 @@ export class ServerBrowserSession {
|
|
|
164
357
|
}
|
|
165
358
|
lastLogTs = Date.now();
|
|
166
359
|
}
|
|
167
|
-
catch (
|
|
168
|
-
logs.push(`[Console Error]
|
|
360
|
+
catch (_error) {
|
|
361
|
+
logs.push(`[Console Error] unknown console listener error`);
|
|
169
362
|
}
|
|
170
363
|
};
|
|
171
364
|
this.page.on('console', consoleListener);
|
|
@@ -181,7 +374,7 @@ export class ServerBrowserSession {
|
|
|
181
374
|
try {
|
|
182
375
|
await this.waitForConsoleInactivity(lastLogTs);
|
|
183
376
|
}
|
|
184
|
-
catch (
|
|
377
|
+
catch (_error) {
|
|
185
378
|
// Timeout is expected
|
|
186
379
|
}
|
|
187
380
|
try {
|
|
@@ -195,8 +388,24 @@ export class ServerBrowserSession {
|
|
|
195
388
|
omitBackground: false,
|
|
196
389
|
});
|
|
197
390
|
if (screenshotBytes && screenshotBytes.length > 0) {
|
|
198
|
-
|
|
199
|
-
|
|
391
|
+
// Compress screenshot if needed
|
|
392
|
+
let finalScreenshotBytes = screenshotBytes;
|
|
393
|
+
let finalMimeType = 'image/png';
|
|
394
|
+
if (shouldCompressImage(screenshotBytes, 'image/png')) {
|
|
395
|
+
try {
|
|
396
|
+
const compressionResult = await compressImage(screenshotBytes, 'image/png');
|
|
397
|
+
finalScreenshotBytes = compressionResult.buffer;
|
|
398
|
+
finalMimeType = compressionResult.mimeType;
|
|
399
|
+
console.log(`[Screenshot Compression] ${(screenshotBytes.length / 1024 / 1024).toFixed(2)}MB → ` +
|
|
400
|
+
`${(compressionResult.compressedSize / 1024 / 1024).toFixed(2)}MB ` +
|
|
401
|
+
`(${compressionResult.compressionRatio.toFixed(1)}% reduction)`);
|
|
402
|
+
}
|
|
403
|
+
catch (error) {
|
|
404
|
+
console.warn(`[Screenshot Compression] Failed, using original: ${error instanceof Error ? error.message : String(error)}`);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
const screenshotBase64 = finalScreenshotBytes.toString('base64');
|
|
408
|
+
screenshot = `data:${finalMimeType};base64,${screenshotBase64}`;
|
|
200
409
|
// Log screenshot success with dimensions
|
|
201
410
|
console.log(`Screenshot captured: 900x600px, ${screenshotBase64.length} chars, data URI length: ${screenshot.length}`);
|
|
202
411
|
executionLog += `\nScreenshot captured at 900x600 resolution (1:1 scale with viewport)`;
|
|
@@ -206,9 +415,9 @@ export class ServerBrowserSession {
|
|
|
206
415
|
executionLog += `\n[Error] Screenshot capture returned empty buffer`;
|
|
207
416
|
}
|
|
208
417
|
}
|
|
209
|
-
catch (
|
|
210
|
-
console.error('Screenshot capture failed:',
|
|
211
|
-
executionLog += `\n[Error] Error taking screenshot of the current state of page! ${
|
|
418
|
+
catch (_error) {
|
|
419
|
+
console.error('Screenshot capture failed:', _error);
|
|
420
|
+
executionLog += `\n[Error] Error taking screenshot of the current state of page! ${_error instanceof Error ? _error.message : String(_error)}`;
|
|
212
421
|
// Try alternative screenshot method as fallback
|
|
213
422
|
try {
|
|
214
423
|
console.log('Attempting fallback screenshot method...');
|
|
@@ -217,8 +426,21 @@ export class ServerBrowserSession {
|
|
|
217
426
|
fullPage: false,
|
|
218
427
|
});
|
|
219
428
|
if (fallbackBytes && fallbackBytes.length > 0) {
|
|
220
|
-
|
|
221
|
-
|
|
429
|
+
// Compress fallback screenshot if needed
|
|
430
|
+
let finalFallbackBytes = fallbackBytes;
|
|
431
|
+
let finalFallbackMimeType = 'image/png';
|
|
432
|
+
if (shouldCompressImage(fallbackBytes, 'image/png')) {
|
|
433
|
+
try {
|
|
434
|
+
const compressionResult = await compressImage(fallbackBytes, 'image/png');
|
|
435
|
+
finalFallbackBytes = compressionResult.buffer;
|
|
436
|
+
finalFallbackMimeType = compressionResult.mimeType;
|
|
437
|
+
}
|
|
438
|
+
catch (error) {
|
|
439
|
+
console.warn(`[Fallback Screenshot Compression] Failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
const fallbackBase64 = finalFallbackBytes.toString('base64');
|
|
443
|
+
screenshot = `data:${finalFallbackMimeType};base64,${fallbackBase64}`;
|
|
222
444
|
console.log(`Fallback screenshot captured: ${fallbackBase64.length} chars`);
|
|
223
445
|
executionLog += `\nFallback screenshot captured successfully`;
|
|
224
446
|
}
|
|
@@ -231,8 +453,8 @@ export class ServerBrowserSession {
|
|
|
231
453
|
try {
|
|
232
454
|
this.page.off('console', consoleListener);
|
|
233
455
|
}
|
|
234
|
-
catch (
|
|
235
|
-
console.log(`Error removing console listener: ${
|
|
456
|
+
catch (_error) {
|
|
457
|
+
console.log(`Error removing console listener: ${_error instanceof Error ? _error.message : String(_error)}`);
|
|
236
458
|
}
|
|
237
459
|
if (executionSuccess) {
|
|
238
460
|
executionLog += '\n Action executed Successfully!';
|
|
@@ -257,7 +479,11 @@ export class ServerBrowserSession {
|
|
|
257
479
|
}
|
|
258
480
|
async navigateToUrl(url) {
|
|
259
481
|
if (!this.page || !this.browser) {
|
|
260
|
-
const launchResult = await this.launchBrowser(
|
|
482
|
+
const launchResult = await this.launchBrowser({
|
|
483
|
+
record: true,
|
|
484
|
+
videoDir: path.join(process.cwd(), 'videos'),
|
|
485
|
+
videoName: `browser_session_${Date.now()}.webm`,
|
|
486
|
+
});
|
|
261
487
|
if (!launchResult.execution_success) {
|
|
262
488
|
return launchResult;
|
|
263
489
|
}
|
|
@@ -286,11 +512,11 @@ export class ServerBrowserSession {
|
|
|
286
512
|
await this.waitTillHTMLStable();
|
|
287
513
|
console.log(`Page navigation completed successfully for: ${url}`);
|
|
288
514
|
}
|
|
289
|
-
catch (
|
|
290
|
-
const errorMsg =
|
|
515
|
+
catch (_error) {
|
|
516
|
+
const errorMsg = _error instanceof Error ? _error.message : String(_error);
|
|
291
517
|
console.error(`Navigation error for ${url}:`, errorMsg);
|
|
292
518
|
executionLog += `\nNavigation error: ${errorMsg}`;
|
|
293
|
-
throw
|
|
519
|
+
throw _error;
|
|
294
520
|
}
|
|
295
521
|
return executionLog;
|
|
296
522
|
});
|
|
@@ -330,7 +556,7 @@ export class ServerBrowserSession {
|
|
|
330
556
|
await this.waitTillHTMLStable();
|
|
331
557
|
executionLog += '\nPage updated after click';
|
|
332
558
|
}
|
|
333
|
-
catch (
|
|
559
|
+
catch (_error) {
|
|
334
560
|
// Navigation timeout is common and not necessarily an error
|
|
335
561
|
console.log('Navigation wait timeout (expected for non-navigating clicks)');
|
|
336
562
|
executionLog += '\nClick completed (no page navigation)';
|
|
@@ -341,11 +567,11 @@ export class ServerBrowserSession {
|
|
|
341
567
|
}
|
|
342
568
|
console.log('Click action completed successfully');
|
|
343
569
|
}
|
|
344
|
-
catch (
|
|
345
|
-
const errorMsg =
|
|
570
|
+
catch (_error) {
|
|
571
|
+
const errorMsg = _error instanceof Error ? _error.message : String(_error);
|
|
346
572
|
console.error('Click action failed:', errorMsg);
|
|
347
573
|
executionLog += `\nClick error: ${errorMsg}`;
|
|
348
|
-
throw
|
|
574
|
+
throw _error;
|
|
349
575
|
}
|
|
350
576
|
finally {
|
|
351
577
|
this.page.off('request', requestListener);
|
|
@@ -365,18 +591,155 @@ export class ServerBrowserSession {
|
|
|
365
591
|
return this.doAction(async () => {
|
|
366
592
|
if (!this.page)
|
|
367
593
|
throw new Error('Page not available');
|
|
368
|
-
|
|
594
|
+
// Try to find and scroll the most appropriate scrollable element
|
|
595
|
+
const scrollResult = await this.page.evaluate(() => {
|
|
596
|
+
// Helper function to check if element is scrollable
|
|
597
|
+
const isScrollable = (el) => {
|
|
598
|
+
const style = window.getComputedStyle(el);
|
|
599
|
+
const overflowY = style.overflowY;
|
|
600
|
+
const hasScrollableContent = el.scrollHeight > el.clientHeight;
|
|
601
|
+
const isScrollableStyle = overflowY === 'auto' || overflowY === 'scroll';
|
|
602
|
+
return hasScrollableContent && (isScrollableStyle || el === document.documentElement);
|
|
603
|
+
};
|
|
604
|
+
// Helper function to check if element is visible
|
|
605
|
+
const isVisible = (el) => {
|
|
606
|
+
const rect = el.getBoundingClientRect();
|
|
607
|
+
const style = window.getComputedStyle(el);
|
|
608
|
+
return (rect.width > 0 &&
|
|
609
|
+
rect.height > 0 &&
|
|
610
|
+
style.display !== 'none' &&
|
|
611
|
+
style.visibility !== 'hidden' &&
|
|
612
|
+
style.opacity !== '0');
|
|
613
|
+
};
|
|
614
|
+
// Find all scrollable elements
|
|
615
|
+
const allElements = Array.from(document.querySelectorAll('*'));
|
|
616
|
+
const scrollableElements = allElements.filter(el => isScrollable(el) && isVisible(el));
|
|
617
|
+
// Sort by size (prefer larger scrollable areas) and scroll position
|
|
618
|
+
scrollableElements.sort((a, b) => {
|
|
619
|
+
const aArea = a.clientHeight * a.clientWidth;
|
|
620
|
+
const bArea = b.clientHeight * b.clientWidth;
|
|
621
|
+
return bArea - aArea;
|
|
622
|
+
});
|
|
623
|
+
let scrolled = false;
|
|
624
|
+
let scrolledElement = 'none';
|
|
625
|
+
const scrollAmount = 600; // Match viewport height
|
|
626
|
+
// Try to scroll the best candidate
|
|
627
|
+
if (scrollableElements.length > 0) {
|
|
628
|
+
for (const element of scrollableElements) {
|
|
629
|
+
const beforeScroll = element.scrollTop;
|
|
630
|
+
const maxScroll = element.scrollHeight - element.clientHeight;
|
|
631
|
+
if (beforeScroll < maxScroll) {
|
|
632
|
+
element.scrollBy({ top: scrollAmount, behavior: 'auto' });
|
|
633
|
+
const afterScroll = element.scrollTop;
|
|
634
|
+
if (afterScroll > beforeScroll) {
|
|
635
|
+
scrolled = true;
|
|
636
|
+
scrolledElement = element.tagName + (element.id ? `#${element.id}` : '') +
|
|
637
|
+
(element.className ? `.${Array.from(element.classList).join('.')}` : '');
|
|
638
|
+
break;
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
// Fallback to window scrolling if no scrollable element found or scroll failed
|
|
644
|
+
if (!scrolled) {
|
|
645
|
+
const beforeScroll = window.scrollY || document.documentElement.scrollTop;
|
|
646
|
+
window.scrollBy({ top: scrollAmount, behavior: 'auto' });
|
|
647
|
+
const afterScroll = window.scrollY || document.documentElement.scrollTop;
|
|
648
|
+
if (afterScroll > beforeScroll) {
|
|
649
|
+
scrolled = true;
|
|
650
|
+
scrolledElement = 'window';
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
return {
|
|
654
|
+
success: scrolled,
|
|
655
|
+
element: scrolledElement,
|
|
656
|
+
foundScrollableElements: scrollableElements.length
|
|
657
|
+
};
|
|
658
|
+
});
|
|
369
659
|
await new Promise((resolve) => setTimeout(resolve, 300));
|
|
370
|
-
|
|
660
|
+
if (scrollResult.success) {
|
|
661
|
+
return `Scroll down action performed! Scrolled element: ${scrollResult.element} (found ${scrollResult.foundScrollableElements} scrollable elements)`;
|
|
662
|
+
}
|
|
663
|
+
else {
|
|
664
|
+
return `Scroll down attempted but page may be at bottom or no scrollable content found (checked ${scrollResult.foundScrollableElements} elements)`;
|
|
665
|
+
}
|
|
371
666
|
});
|
|
372
667
|
}
|
|
373
668
|
async scrollUp() {
|
|
374
669
|
return this.doAction(async () => {
|
|
375
670
|
if (!this.page)
|
|
376
671
|
throw new Error('Page not available');
|
|
377
|
-
|
|
672
|
+
// Try to find and scroll the most appropriate scrollable element
|
|
673
|
+
const scrollResult = await this.page.evaluate(() => {
|
|
674
|
+
// Helper function to check if element is scrollable
|
|
675
|
+
const isScrollable = (el) => {
|
|
676
|
+
const style = window.getComputedStyle(el);
|
|
677
|
+
const overflowY = style.overflowY;
|
|
678
|
+
const hasScrollableContent = el.scrollHeight > el.clientHeight;
|
|
679
|
+
const isScrollableStyle = overflowY === 'auto' || overflowY === 'scroll';
|
|
680
|
+
return hasScrollableContent && (isScrollableStyle || el === document.documentElement);
|
|
681
|
+
};
|
|
682
|
+
// Helper function to check if element is visible
|
|
683
|
+
const isVisible = (el) => {
|
|
684
|
+
const rect = el.getBoundingClientRect();
|
|
685
|
+
const style = window.getComputedStyle(el);
|
|
686
|
+
return (rect.width > 0 &&
|
|
687
|
+
rect.height > 0 &&
|
|
688
|
+
style.display !== 'none' &&
|
|
689
|
+
style.visibility !== 'hidden' &&
|
|
690
|
+
style.opacity !== '0');
|
|
691
|
+
};
|
|
692
|
+
// Find all scrollable elements
|
|
693
|
+
const allElements = Array.from(document.querySelectorAll('*'));
|
|
694
|
+
const scrollableElements = allElements.filter(el => isScrollable(el) && isVisible(el));
|
|
695
|
+
// Sort by size (prefer larger scrollable areas)
|
|
696
|
+
scrollableElements.sort((a, b) => {
|
|
697
|
+
const aArea = a.clientHeight * a.clientWidth;
|
|
698
|
+
const bArea = b.clientHeight * b.clientWidth;
|
|
699
|
+
return bArea - aArea;
|
|
700
|
+
});
|
|
701
|
+
let scrolled = false;
|
|
702
|
+
let scrolledElement = 'none';
|
|
703
|
+
const scrollAmount = 600; // Match viewport height
|
|
704
|
+
// Try to scroll the best candidate
|
|
705
|
+
if (scrollableElements.length > 0) {
|
|
706
|
+
for (const element of scrollableElements) {
|
|
707
|
+
const beforeScroll = element.scrollTop;
|
|
708
|
+
if (beforeScroll > 0) {
|
|
709
|
+
element.scrollBy({ top: -scrollAmount, behavior: 'auto' });
|
|
710
|
+
const afterScroll = element.scrollTop;
|
|
711
|
+
if (afterScroll < beforeScroll) {
|
|
712
|
+
scrolled = true;
|
|
713
|
+
scrolledElement = element.tagName + (element.id ? `#${element.id}` : '') +
|
|
714
|
+
(element.className ? `.${Array.from(element.classList).join('.')}` : '');
|
|
715
|
+
break;
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
// Fallback to window scrolling if no scrollable element found or scroll failed
|
|
721
|
+
if (!scrolled) {
|
|
722
|
+
const beforeScroll = window.scrollY || document.documentElement.scrollTop;
|
|
723
|
+
window.scrollBy({ top: -scrollAmount, behavior: 'auto' });
|
|
724
|
+
const afterScroll = window.scrollY || document.documentElement.scrollTop;
|
|
725
|
+
if (afterScroll < beforeScroll) {
|
|
726
|
+
scrolled = true;
|
|
727
|
+
scrolledElement = 'window';
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
return {
|
|
731
|
+
success: scrolled,
|
|
732
|
+
element: scrolledElement,
|
|
733
|
+
foundScrollableElements: scrollableElements.length
|
|
734
|
+
};
|
|
735
|
+
});
|
|
378
736
|
await new Promise((resolve) => setTimeout(resolve, 300));
|
|
379
|
-
|
|
737
|
+
if (scrollResult.success) {
|
|
738
|
+
return `Scroll up action performed! Scrolled element: ${scrollResult.element} (found ${scrollResult.foundScrollableElements} scrollable elements)`;
|
|
739
|
+
}
|
|
740
|
+
else {
|
|
741
|
+
return `Scroll up attempted but page may be at top or no scrollable content found (checked ${scrollResult.foundScrollableElements} elements)`;
|
|
742
|
+
}
|
|
380
743
|
});
|
|
381
744
|
}
|
|
382
745
|
}
|
|
@@ -389,7 +752,11 @@ class BrowserLaunchToolInvocation extends BaseToolInvocation {
|
|
|
389
752
|
}
|
|
390
753
|
async execute() {
|
|
391
754
|
const session = ServerBrowserSession.getInstance();
|
|
392
|
-
const result = await session.launchBrowser(
|
|
755
|
+
const result = await session.launchBrowser({
|
|
756
|
+
record: this.params.record,
|
|
757
|
+
videoDir: this.params.videoDir,
|
|
758
|
+
videoName: this.params.videoName,
|
|
759
|
+
});
|
|
393
760
|
console.log('[BrowserLaunchTool] Browser launch completed', {
|
|
394
761
|
success: result.execution_success,
|
|
395
762
|
});
|
|
@@ -414,9 +781,22 @@ export class BrowserLaunchTool extends BaseDeclarativeTool {
|
|
|
414
781
|
static Name = ToolNames.BROWSER_LAUNCH;
|
|
415
782
|
// @ts-expect-error - Required by base class pattern
|
|
416
783
|
constructor(config) {
|
|
417
|
-
super(BrowserLaunchTool.Name, 'BrowserLaunch', 'Launches a
|
|
784
|
+
super(BrowserLaunchTool.Name, 'BrowserLaunch', 'Launches a Playwright-controlled browser instance with a 900x600 viewport. Optionally records the session to a .webm video.', Kind.Execute, {
|
|
418
785
|
type: 'object',
|
|
419
|
-
properties: {
|
|
786
|
+
properties: {
|
|
787
|
+
record: {
|
|
788
|
+
type: 'boolean',
|
|
789
|
+
description: 'Enable screen recording for the session (video saved on close)',
|
|
790
|
+
},
|
|
791
|
+
videoDir: {
|
|
792
|
+
type: 'string',
|
|
793
|
+
description: 'Directory to save the recording (absolute path preferred). Defaults to "<cwd>/videos".',
|
|
794
|
+
},
|
|
795
|
+
videoName: {
|
|
796
|
+
type: 'string',
|
|
797
|
+
description: 'Filename for the recording (".webm" appended if missing). If omitted, Playwright default name is used.',
|
|
798
|
+
},
|
|
799
|
+
},
|
|
420
800
|
required: [],
|
|
421
801
|
}, false);
|
|
422
802
|
this.config = config;
|
|
@@ -499,7 +879,7 @@ export class BrowserNavigateTool extends BaseDeclarativeTool {
|
|
|
499
879
|
try {
|
|
500
880
|
new URL(params.url);
|
|
501
881
|
}
|
|
502
|
-
catch (
|
|
882
|
+
catch (_error) {
|
|
503
883
|
// Check if it's a file path
|
|
504
884
|
if (!params.url.startsWith('file://') && !params.url.startsWith('http')) {
|
|
505
885
|
return `Invalid URL format: ${params.url}. Must be a valid URL (http://, https://, or file://)`;
|
|
@@ -843,6 +1223,7 @@ class BrowserCloseToolInvocation extends BaseToolInvocation {
|
|
|
843
1223
|
return {
|
|
844
1224
|
llmContent: result.execution_logs || 'Browser closed successfully',
|
|
845
1225
|
returnDisplay: result.logs || 'Browser closed successfully',
|
|
1226
|
+
videoPath: result.videoPath, // Surface the saved video path
|
|
846
1227
|
};
|
|
847
1228
|
}
|
|
848
1229
|
}
|