@blackbox_ai/blackbox-cli-core 0.0.7 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (255) hide show
  1. package/README.md +11 -183
  2. package/dist/index.d.ts +2 -1
  3. package/dist/index.js +2 -1
  4. package/dist/index.js.map +1 -1
  5. package/dist/src/blackbox/blackboxOAuth2.js +17 -1
  6. package/dist/src/blackbox/blackboxOAuth2.js.map +1 -1
  7. package/dist/src/code_assist/oauth2.js +15 -3
  8. package/dist/src/code_assist/oauth2.js.map +1 -1
  9. package/dist/src/config/blackboxModels.d.ts +3 -2
  10. package/dist/src/config/blackboxModels.js +262 -33
  11. package/dist/src/config/blackboxModels.js.map +1 -1
  12. package/dist/src/config/config.d.ts +65 -0
  13. package/dist/src/config/config.js +282 -17
  14. package/dist/src/config/config.js.map +1 -1
  15. package/dist/src/config/models.d.ts +1 -1
  16. package/dist/src/config/models.js +1 -1
  17. package/dist/src/config/models.js.map +1 -1
  18. package/dist/src/config/multiAgentModels.d.ts +63 -0
  19. package/dist/src/config/multiAgentModels.js +194 -0
  20. package/dist/src/config/multiAgentModels.js.map +1 -0
  21. package/dist/src/core/client.js +11 -5
  22. package/dist/src/core/client.js.map +1 -1
  23. package/dist/src/core/contentGenerator.d.ts +1 -0
  24. package/dist/src/core/contentGenerator.js +57 -7
  25. package/dist/src/core/contentGenerator.js.map +1 -1
  26. package/dist/src/core/coreToolScheduler.js +2 -2
  27. package/dist/src/core/coreToolScheduler.js.map +1 -1
  28. package/dist/src/core/encryptedClientFactory.d.ts +17 -0
  29. package/dist/src/core/encryptedClientFactory.js +92 -0
  30. package/dist/src/core/encryptedClientFactory.js.map +1 -0
  31. package/dist/src/core/encryptedContentGenerator.d.ts +47 -0
  32. package/dist/src/core/encryptedContentGenerator.js +445 -0
  33. package/dist/src/core/encryptedContentGenerator.js.map +1 -0
  34. package/dist/src/core/encryptedGeminiClient.d.ts +59 -0
  35. package/dist/src/core/encryptedGeminiClient.js +177 -0
  36. package/dist/src/core/encryptedGeminiClient.js.map +1 -0
  37. package/dist/src/core/encryptedGeminiClientBridge.d.ts +107 -0
  38. package/dist/src/core/encryptedGeminiClientBridge.js +808 -0
  39. package/dist/src/core/encryptedGeminiClientBridge.js.map +1 -0
  40. package/dist/src/core/encryptedGeminiClientWrapper.d.ts +129 -0
  41. package/dist/src/core/encryptedGeminiClientWrapper.js +305 -0
  42. package/dist/src/core/encryptedGeminiClientWrapper.js.map +1 -0
  43. package/dist/src/core/encryptedTurn.d.ts +40 -0
  44. package/dist/src/core/encryptedTurn.js +114 -0
  45. package/dist/src/core/encryptedTurn.js.map +1 -0
  46. package/dist/src/core/logger.d.ts +21 -0
  47. package/dist/src/core/logger.js +110 -0
  48. package/dist/src/core/logger.js.map +1 -1
  49. package/dist/src/core/openaiContentGenerator/constants.d.ts +2 -0
  50. package/dist/src/core/openaiContentGenerator/constants.js +2 -0
  51. package/dist/src/core/openaiContentGenerator/constants.js.map +1 -1
  52. package/dist/src/core/openaiContentGenerator/converter.d.ts +16 -1
  53. package/dist/src/core/openaiContentGenerator/converter.js +135 -4
  54. package/dist/src/core/openaiContentGenerator/converter.js.map +1 -1
  55. package/dist/src/core/openaiContentGenerator/pipeline.js +22 -8
  56. package/dist/src/core/openaiContentGenerator/pipeline.js.map +1 -1
  57. package/dist/src/core/openaiContentGenerator/pipeline.test.js +51 -0
  58. package/dist/src/core/openaiContentGenerator/pipeline.test.js.map +1 -1
  59. package/dist/src/core/openaiContentGenerator/provider/default.js +10 -1
  60. package/dist/src/core/openaiContentGenerator/provider/default.js.map +1 -1
  61. package/dist/src/core/prompts.d.ts +18 -1
  62. package/dist/src/core/prompts.js +388 -459
  63. package/dist/src/core/prompts.js.map +1 -1
  64. package/dist/src/core/tokenLimits.d.ts +1 -0
  65. package/dist/src/core/tokenLimits.js +37 -2
  66. package/dist/src/core/tokenLimits.js.map +1 -1
  67. package/dist/src/core/tokenLimits.test.js +36 -1
  68. package/dist/src/core/tokenLimits.test.js.map +1 -1
  69. package/dist/src/encrypt/attestation.d.ts +5 -0
  70. package/dist/src/encrypt/attestation.js +100 -0
  71. package/dist/src/encrypt/attestation.js.map +1 -0
  72. package/dist/src/encrypt/client.d.ts +14 -0
  73. package/dist/src/encrypt/client.js +132 -0
  74. package/dist/src/encrypt/client.js.map +1 -0
  75. package/dist/src/encrypt/config.d.ts +22 -0
  76. package/dist/src/encrypt/config.js +43 -0
  77. package/dist/src/encrypt/config.js.map +1 -0
  78. package/dist/src/encrypt/crypto-utils.d.ts +57 -0
  79. package/dist/src/encrypt/crypto-utils.js +257 -0
  80. package/dist/src/encrypt/crypto-utils.js.map +1 -0
  81. package/dist/src/encrypt/history-manager.d.ts +43 -0
  82. package/dist/src/encrypt/history-manager.js +164 -0
  83. package/dist/src/encrypt/history-manager.js.map +1 -0
  84. package/dist/src/encrypt/minimax-template.d.ts +73 -0
  85. package/dist/src/encrypt/minimax-template.js +276 -0
  86. package/dist/src/encrypt/minimax-template.js.map +1 -0
  87. package/dist/src/encrypt/sessions.d.ts +17 -0
  88. package/dist/src/encrypt/sessions.js +221 -0
  89. package/dist/src/encrypt/sessions.js.map +1 -0
  90. package/dist/src/encrypt/streaming-client.d.ts +29 -0
  91. package/dist/src/encrypt/streaming-client.js +232 -0
  92. package/dist/src/encrypt/streaming-client.js.map +1 -0
  93. package/dist/src/encrypt/tool-formatter.d.ts +36 -0
  94. package/dist/src/encrypt/tool-formatter.js +353 -0
  95. package/dist/src/encrypt/tool-formatter.js.map +1 -0
  96. package/dist/src/encrypt/tool-parser.d.ts +93 -0
  97. package/dist/src/encrypt/tool-parser.js +567 -0
  98. package/dist/src/encrypt/tool-parser.js.map +1 -0
  99. package/dist/src/encrypt/types.d.ts +81 -0
  100. package/dist/src/encrypt/types.js +2 -0
  101. package/dist/src/encrypt/types.js.map +1 -0
  102. package/dist/src/generated/git-commit.d.ts +3 -3
  103. package/dist/src/generated/git-commit.js +3 -3
  104. package/dist/src/ide/ide-client.js +9 -19
  105. package/dist/src/ide/ide-client.js.map +1 -1
  106. package/dist/src/index.d.ts +15 -0
  107. package/dist/src/index.js +15 -0
  108. package/dist/src/index.js.map +1 -1
  109. package/dist/src/mcp/oauth-provider.js +2 -6
  110. package/dist/src/mcp/oauth-provider.js.map +1 -1
  111. package/dist/src/mcp/oauth-token-storage.d.ts +7 -0
  112. package/dist/src/mcp/oauth-token-storage.js +24 -0
  113. package/dist/src/mcp/oauth-token-storage.js.map +1 -1
  114. package/dist/src/services/EncryptedChatService.d.ts +80 -0
  115. package/dist/src/services/EncryptedChatService.js +202 -0
  116. package/dist/src/services/EncryptedChatService.js.map +1 -0
  117. package/dist/src/services/StatsHistoryService.d.ts +131 -0
  118. package/dist/src/services/StatsHistoryService.js +427 -0
  119. package/dist/src/services/StatsHistoryService.js.map +1 -0
  120. package/dist/src/services/checkpointApiService.d.ts +101 -0
  121. package/dist/src/services/checkpointApiService.js +215 -0
  122. package/dist/src/services/checkpointApiService.js.map +1 -0
  123. package/dist/src/services/environmentSanitization.d.ts +24 -0
  124. package/dist/src/services/environmentSanitization.js +152 -0
  125. package/dist/src/services/environmentSanitization.js.map +1 -0
  126. package/dist/src/telemetry/blackbox-logger/blackbox-logger.d.ts +2 -6
  127. package/dist/src/telemetry/blackbox-logger/blackbox-logger.js +29 -135
  128. package/dist/src/telemetry/blackbox-logger/blackbox-logger.js.map +1 -1
  129. package/dist/src/telemetry/blackbox-logger/blackbox-logger.test.js +1 -1
  130. package/dist/src/telemetry/blackbox-logger/blackbox-logger.test.js.map +1 -1
  131. package/dist/src/telemetry/uiTelemetry.d.ts +8 -0
  132. package/dist/src/telemetry/uiTelemetry.js +17 -0
  133. package/dist/src/telemetry/uiTelemetry.js.map +1 -1
  134. package/dist/src/tools/browser-interactive.d.ts +63 -0
  135. package/dist/src/tools/browser-interactive.js +394 -0
  136. package/dist/src/tools/browser-interactive.js.map +1 -0
  137. package/dist/src/tools/browser_use.d.ts +23 -2
  138. package/dist/src/tools/browser_use.js +424 -43
  139. package/dist/src/tools/browser_use.js.map +1 -1
  140. package/dist/src/tools/data-file-constants.d.ts +17 -0
  141. package/dist/src/tools/data-file-constants.js +30 -0
  142. package/dist/src/tools/data-file-constants.js.map +1 -0
  143. package/dist/src/tools/edit.js +44 -7
  144. package/dist/src/tools/edit.js.map +1 -1
  145. package/dist/src/tools/exitPlanMode.js +1 -1
  146. package/dist/src/tools/exitPlanMode.js.map +1 -1
  147. package/dist/src/tools/ls.js +40 -6
  148. package/dist/src/tools/ls.js.map +1 -1
  149. package/dist/src/tools/ls.test.js +4 -4
  150. package/dist/src/tools/ls.test.js.map +1 -1
  151. package/dist/src/tools/mcp-client-manager.d.ts +28 -2
  152. package/dist/src/tools/mcp-client-manager.js +62 -4
  153. package/dist/src/tools/mcp-client-manager.js.map +1 -1
  154. package/dist/src/tools/mcp-client.d.ts +5 -3
  155. package/dist/src/tools/mcp-client.js +39 -11
  156. package/dist/src/tools/mcp-client.js.map +1 -1
  157. package/dist/src/tools/mcp-tool.d.ts +3 -1
  158. package/dist/src/tools/mcp-tool.js +37 -9
  159. package/dist/src/tools/mcp-tool.js.map +1 -1
  160. package/dist/src/tools/memoryTool.d.ts +14 -4
  161. package/dist/src/tools/memoryTool.js +98 -39
  162. package/dist/src/tools/memoryTool.js.map +1 -1
  163. package/dist/src/tools/read-data-file.d.ts +31 -0
  164. package/dist/src/tools/read-data-file.js +469 -0
  165. package/dist/src/tools/read-data-file.js.map +1 -0
  166. package/dist/src/tools/read-file.js +64 -5
  167. package/dist/src/tools/read-file.js.map +1 -1
  168. package/dist/src/tools/read-file.test.js +40 -6
  169. package/dist/src/tools/read-file.test.js.map +1 -1
  170. package/dist/src/tools/shell.d.ts +3 -1
  171. package/dist/src/tools/shell.js +25 -4
  172. package/dist/src/tools/shell.js.map +1 -1
  173. package/dist/src/tools/skill.d.ts +34 -0
  174. package/dist/src/tools/skill.js +143 -0
  175. package/dist/src/tools/skill.js.map +1 -0
  176. package/dist/src/tools/sql_db.d.ts +101 -0
  177. package/dist/src/tools/sql_db.js +1033 -0
  178. package/dist/src/tools/sql_db.js.map +1 -0
  179. package/dist/src/tools/sql_db_configure.d.ts +18 -0
  180. package/dist/src/tools/sql_db_configure.js +96 -0
  181. package/dist/src/tools/sql_db_configure.js.map +1 -0
  182. package/dist/src/tools/taskCompletion.d.ts +29 -0
  183. package/dist/src/tools/taskCompletion.js +231 -0
  184. package/dist/src/tools/taskCompletion.js.map +1 -0
  185. package/dist/src/tools/todoWrite.js +0 -142
  186. package/dist/src/tools/todoWrite.js.map +1 -1
  187. package/dist/src/tools/tool-error.d.ts +3 -1
  188. package/dist/src/tools/tool-error.js +3 -0
  189. package/dist/src/tools/tool-error.js.map +1 -1
  190. package/dist/src/tools/tool-names.d.ts +8 -0
  191. package/dist/src/tools/tool-names.js +8 -0
  192. package/dist/src/tools/tool-names.js.map +1 -1
  193. package/dist/src/tools/tool-registry.d.ts +22 -0
  194. package/dist/src/tools/tool-registry.js +41 -1
  195. package/dist/src/tools/tool-registry.js.map +1 -1
  196. package/dist/src/tools/tools.d.ts +18 -2
  197. package/dist/src/tools/tools.js +3 -0
  198. package/dist/src/tools/tools.js.map +1 -1
  199. package/dist/src/tools/web-fetch.js +24 -4
  200. package/dist/src/tools/web-fetch.js.map +1 -1
  201. package/dist/src/tools/web-search.js +160 -2
  202. package/dist/src/tools/web-search.js.map +1 -1
  203. package/dist/src/tools/workspace-error-helper.d.ts +9 -0
  204. package/dist/src/tools/workspace-error-helper.js +43 -0
  205. package/dist/src/tools/workspace-error-helper.js.map +1 -0
  206. package/dist/src/tools/workspace-error-helper.test.js +85 -0
  207. package/dist/src/tools/workspace-error-helper.test.js.map +1 -0
  208. package/dist/src/tools/write-file.js +42 -7
  209. package/dist/src/tools/write-file.js.map +1 -1
  210. package/dist/src/utils/environmentContext.js +3 -1
  211. package/dist/src/utils/environmentContext.js.map +1 -1
  212. package/dist/src/utils/environmentContext.test.js +3 -2
  213. package/dist/src/utils/environmentContext.test.js.map +1 -1
  214. package/dist/src/utils/fetch.d.ts +3 -1
  215. package/dist/src/utils/fetch.js +35 -2
  216. package/dist/src/utils/fetch.js.map +1 -1
  217. package/dist/src/utils/fileUtils.js +30 -3
  218. package/dist/src/utils/fileUtils.js.map +1 -1
  219. package/dist/src/utils/filesearch/fileSearch.d.ts +2 -0
  220. package/dist/src/utils/filesearch/fileSearch.js +38 -7
  221. package/dist/src/utils/filesearch/fileSearch.js.map +1 -1
  222. package/dist/src/utils/git-worktree-utils.d.ts +56 -0
  223. package/dist/src/utils/git-worktree-utils.js +176 -0
  224. package/dist/src/utils/git-worktree-utils.js.map +1 -0
  225. package/dist/src/utils/imageCompression.d.ts +34 -0
  226. package/dist/src/utils/imageCompression.js +170 -0
  227. package/dist/src/utils/imageCompression.js.map +1 -0
  228. package/dist/src/utils/messageTruncator.d.ts +51 -0
  229. package/dist/src/utils/messageTruncator.js +346 -0
  230. package/dist/src/utils/messageTruncator.js.map +1 -0
  231. package/dist/src/utils/pathReader.js +26 -6
  232. package/dist/src/utils/pathReader.js.map +1 -1
  233. package/dist/src/utils/skill.d.ts +65 -0
  234. package/dist/src/utils/skill.js +241 -0
  235. package/dist/src/utils/skill.js.map +1 -0
  236. package/dist/src/utils/textCleaning.d.ts +51 -0
  237. package/dist/src/utils/textCleaning.js +327 -0
  238. package/dist/src/utils/textCleaning.js.map +1 -0
  239. package/dist/tsconfig.tsbuildinfo +1 -1
  240. package/package.json +19 -6
  241. package/dist/src/tools/mcp-client-manager.test.js +0 -39
  242. package/dist/src/tools/mcp-client-manager.test.js.map +0 -1
  243. package/dist/src/tools/mcp-client.test.d.ts +0 -6
  244. package/dist/src/tools/mcp-client.test.js +0 -454
  245. package/dist/src/tools/mcp-client.test.js.map +0 -1
  246. package/dist/src/tools/mcp-tool.test.d.ts +0 -6
  247. package/dist/src/tools/mcp-tool.test.js +0 -576
  248. package/dist/src/tools/mcp-tool.test.js.map +0 -1
  249. package/dist/src/tools/memoryTool.test.d.ts +0 -6
  250. package/dist/src/tools/memoryTool.test.js +0 -420
  251. package/dist/src/tools/memoryTool.test.js.map +0 -1
  252. package/dist/src/tools/tool-registry.test.d.ts +0 -6
  253. package/dist/src/tools/tool-registry.test.js +0 -332
  254. package/dist/src/tools/tool-registry.test.js.map +0 -1
  255. /package/dist/src/tools/{mcp-client-manager.test.d.ts → workspace-error-helper.test.d.ts} +0 -0
@@ -4,9 +4,14 @@
4
4
  * SPDX-License-Identifier: Apache-2.0
5
5
  */
6
6
  import { chromium } from 'playwright';
7
+ import path from 'node:path';
8
+ import { mkdir, rename, readFile, stat } from 'node:fs/promises';
9
+ import { randomUUID } from 'node:crypto';
7
10
  import { ToolNames } from './tool-names.js';
8
11
  import { ToolErrorType } from './tool-error.js';
9
12
  import { BaseDeclarativeTool, BaseToolInvocation, Kind, } from './tools.js';
13
+ import { SignJWT } from 'jose';
14
+ import { compressImage, shouldCompressImage } from '../utils/imageCompression.js';
10
15
  const screenshotPrompt = `Here are the action result, console logs and screenshot after the action execution.
11
16
  Carefully review and decide the next steps to complete the task successfully.`;
12
17
  /**
@@ -14,9 +19,15 @@ Carefully review and decide the next steps to complete the task successfully.`;
14
19
  */
15
20
  export class ServerBrowserSession {
16
21
  browser;
22
+ context; // added
17
23
  page;
18
24
  currentMousePosition;
19
25
  static instance;
26
+ // recording config/state
27
+ recordingEnabled = false; // added
28
+ recordingDirAbs; // added (absolute directory)
29
+ recordingName; // added (desired filename, optional)
30
+ lastVideoPath; // added (resolved saved path)
20
31
  constructor() { }
21
32
  static getInstance() {
22
33
  if (!ServerBrowserSession.instance) {
@@ -24,7 +35,7 @@ export class ServerBrowserSession {
24
35
  }
25
36
  return ServerBrowserSession.instance;
26
37
  }
27
- async launchBrowser() {
38
+ async launchBrowser(opts) {
28
39
  console.log('Launching browser...');
29
40
  if (this.browser) {
30
41
  await this.closeBrowser();
@@ -43,18 +54,39 @@ export class ServerBrowserSession {
43
54
  '--disable-background-timer-throttling',
44
55
  '--disable-backgrounding-occluded-windows',
45
56
  '--disable-renderer-backgrounding',
57
+ '--use-gl=swiftshader',
46
58
  ],
47
59
  headless: true,
48
60
  });
49
- const context = await this.browser.newContext({
61
+ // Configure recording
62
+ this.recordingEnabled = !!opts?.record;
63
+ this.recordingName = opts?.videoName?.trim() || undefined;
64
+ if (this.recordingName && !this.recordingName.endsWith('.webm')) {
65
+ this.recordingName = `${this.recordingName}.webm`;
66
+ }
67
+ if (this.recordingEnabled) {
68
+ const dir = opts?.videoDir?.trim();
69
+ const abs = dir && path.isAbsolute(dir) ? dir : path.resolve(process.cwd(), dir || 'videos');
70
+ await mkdir(abs, { recursive: true });
71
+ this.recordingDirAbs = abs;
72
+ console.log(`[Recording] Enabled. Output dir: ${this.recordingDirAbs}${this.recordingName ? `, name: ${this.recordingName}` : ''}`);
73
+ }
74
+ else {
75
+ this.recordingDirAbs = undefined;
76
+ this.recordingName = undefined;
77
+ }
78
+ // Create context with optional video
79
+ this.context = await this.browser.newContext({
50
80
  viewport: { width: 900, height: 600 },
51
81
  deviceScaleFactor: 1,
52
82
  screen: { width: 900, height: 600 },
53
83
  ignoreHTTPSErrors: true,
54
84
  bypassCSP: true,
85
+ ...(this.recordingEnabled && this.recordingDirAbs
86
+ ? { recordVideo: { dir: this.recordingDirAbs, size: { width: 900, height: 600 } } }
87
+ : {}),
55
88
  });
56
- this.page = await context.newPage();
57
- // Set a default background to ensure screenshots work
89
+ this.page = await this.context.newPage();
58
90
  await this.page.addStyleTag({
59
91
  content: `
60
92
  html, body {
@@ -63,15 +95,18 @@ export class ServerBrowserSession {
63
95
  }
64
96
  `,
65
97
  });
66
- console.log('Browser launched successfully with 900x600 viewport');
98
+ const recordNote = this.recordingEnabled
99
+ ? ` with recording to ${this.recordingDirAbs}${this.recordingName ? `/${this.recordingName}` : ''}`
100
+ : '';
101
+ console.log('Browser launched successfully with 900x600 viewport' + recordNote);
67
102
  return {
68
103
  execution_success: true,
69
- logs: 'Browser session started successfully with 900x600 viewport',
70
- execution_logs: 'Browser launched and ready for interaction at 900x600 resolution',
104
+ logs: 'Browser session started successfully with 900x600 viewport' + recordNote,
105
+ execution_logs: 'Browser launched and ready for interaction at 900x600 resolution' + recordNote,
71
106
  };
72
107
  }
73
- catch (error) {
74
- const errorMessage = `Failed to launch browser: ${error instanceof Error ? error.message : String(error)}`;
108
+ catch (_error) {
109
+ const errorMessage = `Failed to launch browser: ${_error instanceof Error ? _error.message : String(_error)}`;
75
110
  console.error(`[Error] Exception during Starting browser - ${errorMessage}`);
76
111
  return {
77
112
  execution_success: false,
@@ -80,22 +115,180 @@ export class ServerBrowserSession {
80
115
  };
81
116
  }
82
117
  }
118
+ /* generate access token, jwt token with secret key */
119
+ async generateAccessToken() {
120
+ const secret = process.env['VIDEO_STORAGE_SECRET_KEY'];
121
+ if (!secret) {
122
+ throw new Error('VIDEO_STORAGE_SECRET_KEY is not set');
123
+ }
124
+ const expiresInSeconds = 3600; // 1 hour
125
+ const payload = {
126
+ sub: randomUUID(),
127
+ };
128
+ const secretKey = new TextEncoder().encode(secret);
129
+ return new SignJWT(payload)
130
+ .setProtectedHeader({ alg: 'HS256', typ: 'JWT' })
131
+ .setIssuedAt()
132
+ .setExpirationTime(Math.floor(Date.now() / 1000) + expiresInSeconds)
133
+ .sign(secretKey);
134
+ }
135
+ async uploadToStorage(videoPath, videoName) {
136
+ try {
137
+ const accessToken = await this.generateAccessToken();
138
+ const baseUrl = process.env['VIDEO_STORAGE_API_URL'];
139
+ if (!baseUrl) {
140
+ return { execution_success: false, error: 'VIDEO_STORAGE_API_URL is not set' };
141
+ }
142
+ // Read bytes once
143
+ const bytes = await readFile(videoPath);
144
+ // Attempt direct multipart /upload first
145
+ {
146
+ const form = new FormData();
147
+ const blob = new Blob([bytes], { type: 'video/webm' });
148
+ form.append('file', blob, videoName);
149
+ const uploadResponse = await fetch(`${baseUrl}/upload`, {
150
+ method: 'POST',
151
+ headers: { Authorization: `Bearer ${accessToken}` },
152
+ body: form,
153
+ });
154
+ if (uploadResponse.ok) {
155
+ const json = await uploadResponse.json();
156
+ if (json.public_url) {
157
+ return { execution_success: true, videoPath: json.public_url };
158
+ }
159
+ // fall through to signed-url if schema unexpected
160
+ }
161
+ else {
162
+ const text = await uploadResponse.text().catch(() => '');
163
+ const lower = text.toLowerCase();
164
+ const isBodyParseError = uploadResponse.status === 400 && (lower.includes('parse') || lower.includes('parsing'));
165
+ if (!isBodyParseError) {
166
+ return {
167
+ execution_success: false,
168
+ error: `Upload failed (${uploadResponse.status}): ${text}`,
169
+ };
170
+ }
171
+ }
172
+ }
173
+ // Fallback: use signed URL flow to bypass multipart parser
174
+ {
175
+ const signedResp = await fetch(`${baseUrl}/signed-url`, {
176
+ method: 'POST',
177
+ headers: {
178
+ Authorization: `Bearer ${accessToken}`,
179
+ 'Content-Type': 'application/json',
180
+ },
181
+ body: JSON.stringify({ filename: videoName, content_type: 'video/webm' }),
182
+ });
183
+ if (!signedResp.ok) {
184
+ const text = await signedResp.text().catch(() => '');
185
+ return {
186
+ execution_success: false,
187
+ error: `Signed URL request failed (${signedResp.status}): ${text}`,
188
+ };
189
+ }
190
+ const signedJson = await signedResp.json();
191
+ if (!signedJson.signed_put_url || !signedJson.public_url) {
192
+ return { execution_success: false, error: 'Invalid signed-url response' };
193
+ }
194
+ const putResp = await fetch(signedJson.signed_put_url, {
195
+ method: 'PUT',
196
+ headers: { 'Content-Type': 'video/webm' },
197
+ body: bytes,
198
+ });
199
+ if (!putResp.ok) {
200
+ const t = await putResp.text().catch(() => '');
201
+ return { execution_success: false, error: `Signed PUT failed (${putResp.status}): ${t}` };
202
+ }
203
+ return { execution_success: true, videoPath: signedJson.public_url };
204
+ }
205
+ }
206
+ catch (error) {
207
+ console.error('Error uploading video to storage:', error);
208
+ return { execution_success: false, error: error instanceof Error ? error.message : String(error) };
209
+ }
210
+ }
83
211
  async closeBrowser() {
84
212
  if (this.browser || this.page) {
85
213
  console.log('Closing browser...');
86
214
  try {
215
+ let tmpVideoPath;
216
+ let finalVideoPath;
217
+ // Close page and get tmp video path
218
+ if (this.page) {
219
+ const v = this.page.video?.();
220
+ await this.page.close();
221
+ if (this.recordingEnabled && v) {
222
+ try {
223
+ tmpVideoPath = await v.path();
224
+ }
225
+ catch (e) {
226
+ console.warn('[Recording] Failed to obtain video path after page close:', e);
227
+ }
228
+ }
229
+ }
230
+ // Close context to flush video to disk
231
+ await this.context?.close().catch(() => { });
232
+ // Finalize and optionally upload
233
+ if (this.recordingEnabled && tmpVideoPath && this.recordingDirAbs) {
234
+ try {
235
+ if (this.recordingName) {
236
+ const target = path.join(this.recordingDirAbs, this.recordingName);
237
+ await rename(tmpVideoPath, target);
238
+ finalVideoPath = target;
239
+ }
240
+ else {
241
+ finalVideoPath = tmpVideoPath;
242
+ }
243
+ // Wait briefly until file is non-zero
244
+ if (finalVideoPath) {
245
+ for (let i = 0; i < 10; i++) {
246
+ try {
247
+ const s = await stat(finalVideoPath);
248
+ if (s.size > 0)
249
+ break;
250
+ }
251
+ catch (_e) {
252
+ // no-op while file finalizes
253
+ }
254
+ await new Promise((r) => setTimeout(r, 100));
255
+ }
256
+ }
257
+ this.lastVideoPath = finalVideoPath;
258
+ if (process.env['VIDEO_STORAGE_API_URL'] && finalVideoPath) {
259
+ const safeName = this.recordingName || path.basename(finalVideoPath);
260
+ const uploadResult = await this.uploadToStorage(finalVideoPath, safeName);
261
+ if (uploadResult.execution_success && uploadResult.videoPath) {
262
+ this.lastVideoPath = uploadResult.videoPath;
263
+ }
264
+ console.log(`[Recording] Video saved: ${this.lastVideoPath}`);
265
+ }
266
+ else if (finalVideoPath) {
267
+ console.log(`[Recording] Video saved: ${finalVideoPath}`);
268
+ }
269
+ }
270
+ catch (e) {
271
+ console.warn('[Recording] Failed to finalize/save/upload video:', e);
272
+ }
273
+ }
274
+ // Close browser
87
275
  await this.browser?.close();
88
276
  this.browser = undefined;
277
+ this.context = undefined;
89
278
  this.page = undefined;
90
279
  this.currentMousePosition = undefined;
280
+ const msg = this.lastVideoPath
281
+ ? `Browser session closed successfully. Video saved to: ${this.lastVideoPath}`
282
+ : 'Browser session closed successfully';
91
283
  return {
92
284
  execution_success: true,
93
- logs: 'Browser session closed successfully',
94
- execution_logs: 'Browser closed successfully',
285
+ logs: msg,
286
+ execution_logs: msg,
287
+ videoPath: this.lastVideoPath,
95
288
  };
96
289
  }
97
- catch (error) {
98
- const errorMessage = `Error closing browser: ${error instanceof Error ? error.message : String(error)}`;
290
+ catch (_error) {
291
+ const errorMessage = `Error closing browser: ${_error instanceof Error ? _error.message : String(_error)}`;
99
292
  console.warn(errorMessage);
100
293
  return {
101
294
  execution_success: false,
@@ -138,8 +331,8 @@ export class ServerBrowserSession {
138
331
  await new Promise((resolve) => setTimeout(resolve, checkDurationMs));
139
332
  checkCounts++;
140
333
  }
141
- catch (error) {
142
- console.warn('Error checking HTML stability:', error);
334
+ catch (_error) {
335
+ console.warn('Error checking HTML stability:', _error);
143
336
  break;
144
337
  }
145
338
  }
@@ -164,8 +357,8 @@ export class ServerBrowserSession {
164
357
  }
165
358
  lastLogTs = Date.now();
166
359
  }
167
- catch (error) {
168
- logs.push(`[Console Error] ${error instanceof Error ? error.message : String(error)}`);
360
+ catch (_error) {
361
+ logs.push(`[Console Error] unknown console listener error`);
169
362
  }
170
363
  };
171
364
  this.page.on('console', consoleListener);
@@ -181,7 +374,7 @@ export class ServerBrowserSession {
181
374
  try {
182
375
  await this.waitForConsoleInactivity(lastLogTs);
183
376
  }
184
- catch (error) {
377
+ catch (_error) {
185
378
  // Timeout is expected
186
379
  }
187
380
  try {
@@ -195,8 +388,24 @@ export class ServerBrowserSession {
195
388
  omitBackground: false,
196
389
  });
197
390
  if (screenshotBytes && screenshotBytes.length > 0) {
198
- const screenshotBase64 = screenshotBytes.toString('base64');
199
- screenshot = `data:image/png;base64,${screenshotBase64}`;
391
+ // Compress screenshot if needed
392
+ let finalScreenshotBytes = screenshotBytes;
393
+ let finalMimeType = 'image/png';
394
+ if (shouldCompressImage(screenshotBytes, 'image/png')) {
395
+ try {
396
+ const compressionResult = await compressImage(screenshotBytes, 'image/png');
397
+ finalScreenshotBytes = compressionResult.buffer;
398
+ finalMimeType = compressionResult.mimeType;
399
+ console.log(`[Screenshot Compression] ${(screenshotBytes.length / 1024 / 1024).toFixed(2)}MB → ` +
400
+ `${(compressionResult.compressedSize / 1024 / 1024).toFixed(2)}MB ` +
401
+ `(${compressionResult.compressionRatio.toFixed(1)}% reduction)`);
402
+ }
403
+ catch (error) {
404
+ console.warn(`[Screenshot Compression] Failed, using original: ${error instanceof Error ? error.message : String(error)}`);
405
+ }
406
+ }
407
+ const screenshotBase64 = finalScreenshotBytes.toString('base64');
408
+ screenshot = `data:${finalMimeType};base64,${screenshotBase64}`;
200
409
  // Log screenshot success with dimensions
201
410
  console.log(`Screenshot captured: 900x600px, ${screenshotBase64.length} chars, data URI length: ${screenshot.length}`);
202
411
  executionLog += `\nScreenshot captured at 900x600 resolution (1:1 scale with viewport)`;
@@ -206,9 +415,9 @@ export class ServerBrowserSession {
206
415
  executionLog += `\n[Error] Screenshot capture returned empty buffer`;
207
416
  }
208
417
  }
209
- catch (error) {
210
- console.error('Screenshot capture failed:', error);
211
- executionLog += `\n[Error] Error taking screenshot of the current state of page! ${error instanceof Error ? error.message : String(error)}`;
418
+ catch (_error) {
419
+ console.error('Screenshot capture failed:', _error);
420
+ executionLog += `\n[Error] Error taking screenshot of the current state of page! ${_error instanceof Error ? _error.message : String(_error)}`;
212
421
  // Try alternative screenshot method as fallback
213
422
  try {
214
423
  console.log('Attempting fallback screenshot method...');
@@ -217,8 +426,21 @@ export class ServerBrowserSession {
217
426
  fullPage: false,
218
427
  });
219
428
  if (fallbackBytes && fallbackBytes.length > 0) {
220
- const fallbackBase64 = fallbackBytes.toString('base64');
221
- screenshot = `data:image/png;base64,${fallbackBase64}`;
429
+ // Compress fallback screenshot if needed
430
+ let finalFallbackBytes = fallbackBytes;
431
+ let finalFallbackMimeType = 'image/png';
432
+ if (shouldCompressImage(fallbackBytes, 'image/png')) {
433
+ try {
434
+ const compressionResult = await compressImage(fallbackBytes, 'image/png');
435
+ finalFallbackBytes = compressionResult.buffer;
436
+ finalFallbackMimeType = compressionResult.mimeType;
437
+ }
438
+ catch (error) {
439
+ console.warn(`[Fallback Screenshot Compression] Failed: ${error instanceof Error ? error.message : String(error)}`);
440
+ }
441
+ }
442
+ const fallbackBase64 = finalFallbackBytes.toString('base64');
443
+ screenshot = `data:${finalFallbackMimeType};base64,${fallbackBase64}`;
222
444
  console.log(`Fallback screenshot captured: ${fallbackBase64.length} chars`);
223
445
  executionLog += `\nFallback screenshot captured successfully`;
224
446
  }
@@ -231,8 +453,8 @@ export class ServerBrowserSession {
231
453
  try {
232
454
  this.page.off('console', consoleListener);
233
455
  }
234
- catch (error) {
235
- console.log(`Error removing console listener: ${error instanceof Error ? error.message : String(error)}`);
456
+ catch (_error) {
457
+ console.log(`Error removing console listener: ${_error instanceof Error ? _error.message : String(_error)}`);
236
458
  }
237
459
  if (executionSuccess) {
238
460
  executionLog += '\n Action executed Successfully!';
@@ -257,7 +479,11 @@ export class ServerBrowserSession {
257
479
  }
258
480
  async navigateToUrl(url) {
259
481
  if (!this.page || !this.browser) {
260
- const launchResult = await this.launchBrowser();
482
+ const launchResult = await this.launchBrowser({
483
+ record: true,
484
+ videoDir: path.join(process.cwd(), 'videos'),
485
+ videoName: `browser_session_${Date.now()}.webm`,
486
+ });
261
487
  if (!launchResult.execution_success) {
262
488
  return launchResult;
263
489
  }
@@ -286,11 +512,11 @@ export class ServerBrowserSession {
286
512
  await this.waitTillHTMLStable();
287
513
  console.log(`Page navigation completed successfully for: ${url}`);
288
514
  }
289
- catch (error) {
290
- const errorMsg = error instanceof Error ? error.message : String(error);
515
+ catch (_error) {
516
+ const errorMsg = _error instanceof Error ? _error.message : String(_error);
291
517
  console.error(`Navigation error for ${url}:`, errorMsg);
292
518
  executionLog += `\nNavigation error: ${errorMsg}`;
293
- throw error;
519
+ throw _error;
294
520
  }
295
521
  return executionLog;
296
522
  });
@@ -330,7 +556,7 @@ export class ServerBrowserSession {
330
556
  await this.waitTillHTMLStable();
331
557
  executionLog += '\nPage updated after click';
332
558
  }
333
- catch (error) {
559
+ catch (_error) {
334
560
  // Navigation timeout is common and not necessarily an error
335
561
  console.log('Navigation wait timeout (expected for non-navigating clicks)');
336
562
  executionLog += '\nClick completed (no page navigation)';
@@ -341,11 +567,11 @@ export class ServerBrowserSession {
341
567
  }
342
568
  console.log('Click action completed successfully');
343
569
  }
344
- catch (error) {
345
- const errorMsg = error instanceof Error ? error.message : String(error);
570
+ catch (_error) {
571
+ const errorMsg = _error instanceof Error ? _error.message : String(_error);
346
572
  console.error('Click action failed:', errorMsg);
347
573
  executionLog += `\nClick error: ${errorMsg}`;
348
- throw error;
574
+ throw _error;
349
575
  }
350
576
  finally {
351
577
  this.page.off('request', requestListener);
@@ -365,18 +591,155 @@ export class ServerBrowserSession {
365
591
  return this.doAction(async () => {
366
592
  if (!this.page)
367
593
  throw new Error('Page not available');
368
- await this.page.evaluate("window.scrollBy({top: 400, behavior: 'auto'})");
594
+ // Try to find and scroll the most appropriate scrollable element
595
+ const scrollResult = await this.page.evaluate(() => {
596
+ // Helper function to check if element is scrollable
597
+ const isScrollable = (el) => {
598
+ const style = window.getComputedStyle(el);
599
+ const overflowY = style.overflowY;
600
+ const hasScrollableContent = el.scrollHeight > el.clientHeight;
601
+ const isScrollableStyle = overflowY === 'auto' || overflowY === 'scroll';
602
+ return hasScrollableContent && (isScrollableStyle || el === document.documentElement);
603
+ };
604
+ // Helper function to check if element is visible
605
+ const isVisible = (el) => {
606
+ const rect = el.getBoundingClientRect();
607
+ const style = window.getComputedStyle(el);
608
+ return (rect.width > 0 &&
609
+ rect.height > 0 &&
610
+ style.display !== 'none' &&
611
+ style.visibility !== 'hidden' &&
612
+ style.opacity !== '0');
613
+ };
614
+ // Find all scrollable elements
615
+ const allElements = Array.from(document.querySelectorAll('*'));
616
+ const scrollableElements = allElements.filter(el => isScrollable(el) && isVisible(el));
617
+ // Sort by size (prefer larger scrollable areas) and scroll position
618
+ scrollableElements.sort((a, b) => {
619
+ const aArea = a.clientHeight * a.clientWidth;
620
+ const bArea = b.clientHeight * b.clientWidth;
621
+ return bArea - aArea;
622
+ });
623
+ let scrolled = false;
624
+ let scrolledElement = 'none';
625
+ const scrollAmount = 600; // Match viewport height
626
+ // Try to scroll the best candidate
627
+ if (scrollableElements.length > 0) {
628
+ for (const element of scrollableElements) {
629
+ const beforeScroll = element.scrollTop;
630
+ const maxScroll = element.scrollHeight - element.clientHeight;
631
+ if (beforeScroll < maxScroll) {
632
+ element.scrollBy({ top: scrollAmount, behavior: 'auto' });
633
+ const afterScroll = element.scrollTop;
634
+ if (afterScroll > beforeScroll) {
635
+ scrolled = true;
636
+ scrolledElement = element.tagName + (element.id ? `#${element.id}` : '') +
637
+ (element.className ? `.${Array.from(element.classList).join('.')}` : '');
638
+ break;
639
+ }
640
+ }
641
+ }
642
+ }
643
+ // Fallback to window scrolling if no scrollable element found or scroll failed
644
+ if (!scrolled) {
645
+ const beforeScroll = window.scrollY || document.documentElement.scrollTop;
646
+ window.scrollBy({ top: scrollAmount, behavior: 'auto' });
647
+ const afterScroll = window.scrollY || document.documentElement.scrollTop;
648
+ if (afterScroll > beforeScroll) {
649
+ scrolled = true;
650
+ scrolledElement = 'window';
651
+ }
652
+ }
653
+ return {
654
+ success: scrolled,
655
+ element: scrolledElement,
656
+ foundScrollableElements: scrollableElements.length
657
+ };
658
+ });
369
659
  await new Promise((resolve) => setTimeout(resolve, 300));
370
- return 'Scroll down action performed!';
660
+ if (scrollResult.success) {
661
+ return `Scroll down action performed! Scrolled element: ${scrollResult.element} (found ${scrollResult.foundScrollableElements} scrollable elements)`;
662
+ }
663
+ else {
664
+ return `Scroll down attempted but page may be at bottom or no scrollable content found (checked ${scrollResult.foundScrollableElements} elements)`;
665
+ }
371
666
  });
372
667
  }
373
668
  async scrollUp() {
374
669
  return this.doAction(async () => {
375
670
  if (!this.page)
376
671
  throw new Error('Page not available');
377
- await this.page.evaluate("window.scrollBy({top: -600, behavior: 'auto'})");
672
+ // Try to find and scroll the most appropriate scrollable element
673
+ const scrollResult = await this.page.evaluate(() => {
674
+ // Helper function to check if element is scrollable
675
+ const isScrollable = (el) => {
676
+ const style = window.getComputedStyle(el);
677
+ const overflowY = style.overflowY;
678
+ const hasScrollableContent = el.scrollHeight > el.clientHeight;
679
+ const isScrollableStyle = overflowY === 'auto' || overflowY === 'scroll';
680
+ return hasScrollableContent && (isScrollableStyle || el === document.documentElement);
681
+ };
682
+ // Helper function to check if element is visible
683
+ const isVisible = (el) => {
684
+ const rect = el.getBoundingClientRect();
685
+ const style = window.getComputedStyle(el);
686
+ return (rect.width > 0 &&
687
+ rect.height > 0 &&
688
+ style.display !== 'none' &&
689
+ style.visibility !== 'hidden' &&
690
+ style.opacity !== '0');
691
+ };
692
+ // Find all scrollable elements
693
+ const allElements = Array.from(document.querySelectorAll('*'));
694
+ const scrollableElements = allElements.filter(el => isScrollable(el) && isVisible(el));
695
+ // Sort by size (prefer larger scrollable areas)
696
+ scrollableElements.sort((a, b) => {
697
+ const aArea = a.clientHeight * a.clientWidth;
698
+ const bArea = b.clientHeight * b.clientWidth;
699
+ return bArea - aArea;
700
+ });
701
+ let scrolled = false;
702
+ let scrolledElement = 'none';
703
+ const scrollAmount = 600; // Match viewport height
704
+ // Try to scroll the best candidate
705
+ if (scrollableElements.length > 0) {
706
+ for (const element of scrollableElements) {
707
+ const beforeScroll = element.scrollTop;
708
+ if (beforeScroll > 0) {
709
+ element.scrollBy({ top: -scrollAmount, behavior: 'auto' });
710
+ const afterScroll = element.scrollTop;
711
+ if (afterScroll < beforeScroll) {
712
+ scrolled = true;
713
+ scrolledElement = element.tagName + (element.id ? `#${element.id}` : '') +
714
+ (element.className ? `.${Array.from(element.classList).join('.')}` : '');
715
+ break;
716
+ }
717
+ }
718
+ }
719
+ }
720
+ // Fallback to window scrolling if no scrollable element found or scroll failed
721
+ if (!scrolled) {
722
+ const beforeScroll = window.scrollY || document.documentElement.scrollTop;
723
+ window.scrollBy({ top: -scrollAmount, behavior: 'auto' });
724
+ const afterScroll = window.scrollY || document.documentElement.scrollTop;
725
+ if (afterScroll < beforeScroll) {
726
+ scrolled = true;
727
+ scrolledElement = 'window';
728
+ }
729
+ }
730
+ return {
731
+ success: scrolled,
732
+ element: scrolledElement,
733
+ foundScrollableElements: scrollableElements.length
734
+ };
735
+ });
378
736
  await new Promise((resolve) => setTimeout(resolve, 300));
379
- return 'Scroll up action performed!';
737
+ if (scrollResult.success) {
738
+ return `Scroll up action performed! Scrolled element: ${scrollResult.element} (found ${scrollResult.foundScrollableElements} scrollable elements)`;
739
+ }
740
+ else {
741
+ return `Scroll up attempted but page may be at top or no scrollable content found (checked ${scrollResult.foundScrollableElements} elements)`;
742
+ }
380
743
  });
381
744
  }
382
745
  }
@@ -389,7 +752,11 @@ class BrowserLaunchToolInvocation extends BaseToolInvocation {
389
752
  }
390
753
  async execute() {
391
754
  const session = ServerBrowserSession.getInstance();
392
- const result = await session.launchBrowser();
755
+ const result = await session.launchBrowser({
756
+ record: this.params.record,
757
+ videoDir: this.params.videoDir,
758
+ videoName: this.params.videoName,
759
+ });
393
760
  console.log('[BrowserLaunchTool] Browser launch completed', {
394
761
  success: result.execution_success,
395
762
  });
@@ -414,9 +781,22 @@ export class BrowserLaunchTool extends BaseDeclarativeTool {
414
781
  static Name = ToolNames.BROWSER_LAUNCH;
415
782
  // @ts-expect-error - Required by base class pattern
416
783
  constructor(config) {
417
- super(BrowserLaunchTool.Name, 'BrowserLaunch', 'Launches a Puppeteer-controlled browser instance with a 900x600 viewport. This must always be the first browser action before any other browser operations.', Kind.Execute, {
784
+ super(BrowserLaunchTool.Name, 'BrowserLaunch', 'Launches a Playwright-controlled browser instance with a 900x600 viewport. Optionally records the session to a .webm video.', Kind.Execute, {
418
785
  type: 'object',
419
- properties: {},
786
+ properties: {
787
+ record: {
788
+ type: 'boolean',
789
+ description: 'Enable screen recording for the session (video saved on close)',
790
+ },
791
+ videoDir: {
792
+ type: 'string',
793
+ description: 'Directory to save the recording (absolute path preferred). Defaults to "<cwd>/videos".',
794
+ },
795
+ videoName: {
796
+ type: 'string',
797
+ description: 'Filename for the recording (".webm" appended if missing). If omitted, Playwright default name is used.',
798
+ },
799
+ },
420
800
  required: [],
421
801
  }, false);
422
802
  this.config = config;
@@ -499,7 +879,7 @@ export class BrowserNavigateTool extends BaseDeclarativeTool {
499
879
  try {
500
880
  new URL(params.url);
501
881
  }
502
- catch (error) {
882
+ catch (_error) {
503
883
  // Check if it's a file path
504
884
  if (!params.url.startsWith('file://') && !params.url.startsWith('http')) {
505
885
  return `Invalid URL format: ${params.url}. Must be a valid URL (http://, https://, or file://)`;
@@ -843,6 +1223,7 @@ class BrowserCloseToolInvocation extends BaseToolInvocation {
843
1223
  return {
844
1224
  llmContent: result.execution_logs || 'Browser closed successfully',
845
1225
  returnDisplay: result.logs || 'Browser closed successfully',
1226
+ videoPath: result.videoPath, // Surface the saved video path
846
1227
  };
847
1228
  }
848
1229
  }