@hybridaione/hybridclaw 0.2.2 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +70 -0
- package/.husky/pre-commit +1 -0
- package/CHANGELOG.md +85 -0
- package/CONTRIBUTING.md +33 -0
- package/README.md +41 -16
- package/SECURITY.md +17 -0
- package/biome.json +35 -0
- package/config.example.json +71 -8
- package/container/package-lock.json +2 -2
- package/container/package.json +1 -1
- package/container/src/approval-policy.ts +1303 -0
- package/container/src/browser-tools.ts +431 -136
- package/container/src/extensions.ts +36 -12
- package/container/src/hybridai-client.ts +34 -13
- package/container/src/index.ts +451 -109
- package/container/src/ipc.ts +5 -3
- package/container/src/token-usage.ts +20 -10
- package/container/src/tools.ts +599 -225
- package/container/src/types.ts +32 -2
- package/container/src/web-fetch.ts +89 -32
- package/dist/agent.d.ts.map +1 -1
- package/dist/agent.js +10 -2
- package/dist/agent.js.map +1 -1
- package/dist/audit-cli.d.ts.map +1 -1
- package/dist/audit-cli.js +4 -2
- package/dist/audit-cli.js.map +1 -1
- package/dist/audit-events.d.ts.map +1 -1
- package/dist/audit-events.js +53 -3
- package/dist/audit-events.js.map +1 -1
- package/dist/audit-trail.d.ts.map +1 -1
- package/dist/audit-trail.js +17 -8
- package/dist/audit-trail.js.map +1 -1
- package/dist/channels/discord/attachments.d.ts.map +1 -1
- package/dist/channels/discord/attachments.js +14 -7
- package/dist/channels/discord/attachments.js.map +1 -1
- package/dist/channels/discord/debounce.d.ts +9 -0
- package/dist/channels/discord/debounce.d.ts.map +1 -0
- package/dist/channels/discord/debounce.js +20 -0
- package/dist/channels/discord/debounce.js.map +1 -0
- package/dist/channels/discord/delivery.d.ts +4 -1
- package/dist/channels/discord/delivery.d.ts.map +1 -1
- package/dist/channels/discord/delivery.js +19 -3
- package/dist/channels/discord/delivery.js.map +1 -1
- package/dist/channels/discord/human-delay.d.ts +16 -0
- package/dist/channels/discord/human-delay.d.ts.map +1 -0
- package/dist/channels/discord/human-delay.js +29 -0
- package/dist/channels/discord/human-delay.js.map +1 -0
- package/dist/channels/discord/inbound.d.ts +4 -0
- package/dist/channels/discord/inbound.d.ts.map +1 -1
- package/dist/channels/discord/inbound.js +45 -4
- package/dist/channels/discord/inbound.js.map +1 -1
- package/dist/channels/discord/mentions.d.ts.map +1 -1
- package/dist/channels/discord/mentions.js +16 -4
- package/dist/channels/discord/mentions.js.map +1 -1
- package/dist/channels/discord/presence.d.ts +33 -0
- package/dist/channels/discord/presence.d.ts.map +1 -0
- package/dist/channels/discord/presence.js +111 -0
- package/dist/channels/discord/presence.js.map +1 -0
- package/dist/channels/discord/rate-limiter.d.ts +14 -0
- package/dist/channels/discord/rate-limiter.d.ts.map +1 -0
- package/dist/channels/discord/rate-limiter.js +49 -0
- package/dist/channels/discord/rate-limiter.js.map +1 -0
- package/dist/channels/discord/reactions.d.ts +38 -0
- package/dist/channels/discord/reactions.d.ts.map +1 -0
- package/dist/channels/discord/reactions.js +151 -0
- package/dist/channels/discord/reactions.js.map +1 -0
- package/dist/channels/discord/runtime.d.ts +6 -3
- package/dist/channels/discord/runtime.d.ts.map +1 -1
- package/dist/channels/discord/runtime.js +621 -125
- package/dist/channels/discord/runtime.js.map +1 -1
- package/dist/channels/discord/stream.d.ts +4 -1
- package/dist/channels/discord/stream.d.ts.map +1 -1
- package/dist/channels/discord/stream.js +16 -8
- package/dist/channels/discord/stream.js.map +1 -1
- package/dist/channels/discord/tool-actions.d.ts.map +1 -1
- package/dist/channels/discord/tool-actions.js +24 -12
- package/dist/channels/discord/tool-actions.js.map +1 -1
- package/dist/channels/discord/typing.d.ts +15 -0
- package/dist/channels/discord/typing.d.ts.map +1 -0
- package/dist/channels/discord/typing.js +106 -0
- package/dist/channels/discord/typing.js.map +1 -0
- package/dist/chunk.d.ts.map +1 -1
- package/dist/chunk.js +4 -2
- package/dist/chunk.js.map +1 -1
- package/dist/cli.js +47 -22
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +19 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +103 -18
- package/dist/config.js.map +1 -1
- package/dist/container-runner.d.ts.map +1 -1
- package/dist/container-runner.js +58 -26
- package/dist/container-runner.js.map +1 -1
- package/dist/container-setup.d.ts.map +1 -1
- package/dist/container-setup.js +10 -9
- package/dist/container-setup.js.map +1 -1
- package/dist/conversation.d.ts +2 -2
- package/dist/conversation.d.ts.map +1 -1
- package/dist/conversation.js +1 -1
- package/dist/conversation.js.map +1 -1
- package/dist/db.d.ts +118 -2
- package/dist/db.d.ts.map +1 -1
- package/dist/db.js +1568 -50
- package/dist/db.js.map +1 -1
- package/dist/delegation-manager.d.ts.map +1 -1
- package/dist/delegation-manager.js +3 -2
- package/dist/delegation-manager.js.map +1 -1
- package/dist/gateway-client.d.ts +2 -2
- package/dist/gateway-client.d.ts.map +1 -1
- package/dist/gateway-client.js +10 -4
- package/dist/gateway-client.js.map +1 -1
- package/dist/gateway-service.d.ts +3 -3
- package/dist/gateway-service.d.ts.map +1 -1
- package/dist/gateway-service.js +563 -73
- package/dist/gateway-service.js.map +1 -1
- package/dist/gateway-types.d.ts +24 -0
- package/dist/gateway-types.d.ts.map +1 -1
- package/dist/gateway-types.js.map +1 -1
- package/dist/gateway.js +179 -24
- package/dist/gateway.js.map +1 -1
- package/dist/health.d.ts.map +1 -1
- package/dist/health.js +20 -10
- package/dist/health.js.map +1 -1
- package/dist/heartbeat.d.ts +4 -0
- package/dist/heartbeat.d.ts.map +1 -1
- package/dist/heartbeat.js +48 -20
- package/dist/heartbeat.js.map +1 -1
- package/dist/hybridai-bots.d.ts.map +1 -1
- package/dist/hybridai-bots.js +4 -2
- package/dist/hybridai-bots.js.map +1 -1
- package/dist/instruction-approval-audit.d.ts.map +1 -1
- package/dist/instruction-approval-audit.js.map +1 -1
- package/dist/instruction-integrity.d.ts.map +1 -1
- package/dist/instruction-integrity.js +8 -2
- package/dist/instruction-integrity.js.map +1 -1
- package/dist/ipc.d.ts.map +1 -1
- package/dist/ipc.js +6 -1
- package/dist/ipc.js.map +1 -1
- package/dist/logger.js.map +1 -1
- package/dist/memory-consolidation.d.ts +17 -0
- package/dist/memory-consolidation.d.ts.map +1 -0
- package/dist/memory-consolidation.js +25 -0
- package/dist/memory-consolidation.js.map +1 -0
- package/dist/memory-service.d.ts +200 -0
- package/dist/memory-service.d.ts.map +1 -0
- package/dist/memory-service.js +294 -0
- package/dist/memory-service.js.map +1 -0
- package/dist/mount-security.d.ts.map +1 -1
- package/dist/mount-security.js +31 -7
- package/dist/mount-security.js.map +1 -1
- package/dist/observability-ingest.d.ts.map +1 -1
- package/dist/observability-ingest.js +32 -11
- package/dist/observability-ingest.js.map +1 -1
- package/dist/onboarding.d.ts.map +1 -1
- package/dist/onboarding.js +32 -9
- package/dist/onboarding.js.map +1 -1
- package/dist/proactive-policy.d.ts.map +1 -1
- package/dist/proactive-policy.js +2 -1
- package/dist/proactive-policy.js.map +1 -1
- package/dist/prompt-hooks.d.ts.map +1 -1
- package/dist/prompt-hooks.js +9 -7
- package/dist/prompt-hooks.js.map +1 -1
- package/dist/runtime-config.d.ts +98 -1
- package/dist/runtime-config.d.ts.map +1 -1
- package/dist/runtime-config.js +477 -23
- package/dist/runtime-config.js.map +1 -1
- package/dist/scheduled-task-runner.d.ts +1 -0
- package/dist/scheduled-task-runner.d.ts.map +1 -1
- package/dist/scheduled-task-runner.js +29 -10
- package/dist/scheduled-task-runner.js.map +1 -1
- package/dist/scheduler.d.ts +43 -4
- package/dist/scheduler.d.ts.map +1 -1
- package/dist/scheduler.js +530 -56
- package/dist/scheduler.js.map +1 -1
- package/dist/session-export.d.ts +26 -0
- package/dist/session-export.d.ts.map +1 -0
- package/dist/session-export.js +149 -0
- package/dist/session-export.js.map +1 -0
- package/dist/session-maintenance.d.ts.map +1 -1
- package/dist/session-maintenance.js +75 -13
- package/dist/session-maintenance.js.map +1 -1
- package/dist/session-transcripts.d.ts.map +1 -1
- package/dist/session-transcripts.js.map +1 -1
- package/dist/side-effects.d.ts.map +1 -1
- package/dist/side-effects.js +14 -2
- package/dist/side-effects.js.map +1 -1
- package/dist/skills-guard.d.ts.map +1 -1
- package/dist/skills-guard.js +893 -130
- package/dist/skills-guard.js.map +1 -1
- package/dist/skills.d.ts +5 -0
- package/dist/skills.d.ts.map +1 -1
- package/dist/skills.js +29 -15
- package/dist/skills.js.map +1 -1
- package/dist/token-efficiency.d.ts.map +1 -1
- package/dist/token-efficiency.js.map +1 -1
- package/dist/tui.js +92 -11
- package/dist/tui.js.map +1 -1
- package/dist/types.d.ts +146 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +24 -1
- package/dist/types.js.map +1 -1
- package/dist/update.d.ts.map +1 -1
- package/dist/update.js +42 -14
- package/dist/update.js.map +1 -1
- package/dist/workspace.d.ts.map +1 -1
- package/dist/workspace.js +49 -9
- package/dist/workspace.js.map +1 -1
- package/docs/chat.html +9 -3
- package/docs/index.html +37 -13
- package/package.json +8 -2
- package/src/agent.ts +16 -3
- package/src/audit-cli.ts +44 -16
- package/src/audit-events.ts +69 -5
- package/src/audit-trail.ts +41 -15
- package/src/channels/discord/attachments.ts +81 -27
- package/src/channels/discord/debounce.ts +25 -0
- package/src/channels/discord/delivery.ts +57 -13
- package/src/channels/discord/human-delay.ts +48 -0
- package/src/channels/discord/inbound.ts +66 -7
- package/src/channels/discord/mentions.ts +42 -18
- package/src/channels/discord/presence.ts +148 -0
- package/src/channels/discord/rate-limiter.ts +58 -0
- package/src/channels/discord/reactions.ts +211 -0
- package/src/channels/discord/runtime.ts +1048 -182
- package/src/channels/discord/stream.ts +73 -27
- package/src/channels/discord/tool-actions.ts +78 -37
- package/src/channels/discord/typing.ts +140 -0
- package/src/chunk.ts +12 -4
- package/src/cli.ts +141 -56
- package/src/config.ts +192 -34
- package/src/container-runner.ts +132 -42
- package/src/container-setup.ts +57 -22
- package/src/conversation.ts +9 -7
- package/src/db.ts +2217 -84
- package/src/delegation-manager.ts +6 -2
- package/src/gateway-client.ts +41 -17
- package/src/gateway-service.ts +1019 -201
- package/src/gateway-types.ts +33 -0
- package/src/gateway.ts +321 -48
- package/src/health.ts +66 -26
- package/src/heartbeat.ts +84 -22
- package/src/hybridai-bots.ts +14 -5
- package/src/instruction-approval-audit.ts +4 -1
- package/src/instruction-integrity.ts +30 -9
- package/src/ipc.ts +23 -5
- package/src/logger.ts +4 -1
- package/src/memory-consolidation.ts +41 -0
- package/src/memory-service.ts +606 -0
- package/src/mount-security.ts +58 -13
- package/src/observability-ingest.ts +134 -35
- package/src/onboarding.ts +126 -35
- package/src/proactive-policy.ts +3 -1
- package/src/prompt-hooks.ts +40 -17
- package/src/runtime-config.ts +1114 -99
- package/src/scheduled-task-runner.ts +63 -11
- package/src/scheduler.ts +683 -60
- package/src/session-export.ts +196 -0
- package/src/session-maintenance.ts +125 -22
- package/src/session-transcripts.ts +12 -3
- package/src/side-effects.ts +28 -5
- package/src/skills-guard.ts +1067 -219
- package/src/skills.ts +163 -65
- package/src/token-efficiency.ts +31 -9
- package/src/tui.ts +166 -25
- package/src/types.ts +195 -2
- package/src/update.ts +79 -23
- package/src/workspace.ts +63 -11
- package/tests/approval-policy.test.ts +224 -0
- package/tests/discord.basic.test.ts +82 -2
- package/tests/discord.human-presence.test.ts +85 -0
- package/tests/gateway-service.media-routing.test.ts +8 -2
- package/tests/memory-service.test.ts +1114 -0
- package/tests/token-efficiency.basic.test.ts +8 -2
- package/vitest.e2e.config.ts +3 -1
- package/vitest.integration.config.ts +3 -1
- package/vitest.live.config.ts +3 -1
- package/vitest.unit.config.ts +9 -0
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { execFile, spawnSync } from 'child_process';
|
|
2
|
-
import { createHash } from 'crypto';
|
|
3
|
-
import { lookup } from 'dns/promises';
|
|
4
|
-
import fs from 'fs';
|
|
5
|
-
import net from 'net';
|
|
6
|
-
import path from 'path';
|
|
7
|
-
import { promisify } from 'util';
|
|
1
|
+
import { execFile, spawnSync } from 'node:child_process';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { lookup } from 'node:dns/promises';
|
|
4
|
+
import fs from 'node:fs';
|
|
5
|
+
import net from 'node:net';
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
import { promisify } from 'node:util';
|
|
8
8
|
|
|
9
9
|
import type { ToolDefinition } from './types.js';
|
|
10
10
|
|
|
@@ -19,8 +19,14 @@ const BROWSER_RUNTIME_ROOT = path.join(WORKSPACE_ROOT, '.hybridclaw-runtime');
|
|
|
19
19
|
const BROWSER_TMP_HOME = path.join(BROWSER_RUNTIME_ROOT, 'home');
|
|
20
20
|
const BROWSER_NPM_CACHE = path.join(BROWSER_RUNTIME_ROOT, 'npm-cache');
|
|
21
21
|
const BROWSER_XDG_CACHE = path.join(BROWSER_RUNTIME_ROOT, 'cache');
|
|
22
|
-
const BROWSER_PLAYWRIGHT_CACHE = path.join(
|
|
23
|
-
|
|
22
|
+
const BROWSER_PLAYWRIGHT_CACHE = path.join(
|
|
23
|
+
BROWSER_RUNTIME_ROOT,
|
|
24
|
+
'ms-playwright',
|
|
25
|
+
);
|
|
26
|
+
const BROWSER_PROFILE_ROOT = path.join(
|
|
27
|
+
BROWSER_RUNTIME_ROOT,
|
|
28
|
+
'browser-profiles',
|
|
29
|
+
);
|
|
24
30
|
const ENV_FALSEY = new Set(['0', 'false', 'no', 'off']);
|
|
25
31
|
const BOT_DETECTION_PATTERNS = [
|
|
26
32
|
'access denied',
|
|
@@ -134,7 +140,9 @@ export function setBrowserModelContext(
|
|
|
134
140
|
chatbotId: string,
|
|
135
141
|
): void {
|
|
136
142
|
currentBrowserModelContext = {
|
|
137
|
-
baseUrl: String(baseUrl || '')
|
|
143
|
+
baseUrl: String(baseUrl || '')
|
|
144
|
+
.trim()
|
|
145
|
+
.replace(/\/+$/, ''),
|
|
138
146
|
apiKey: String(apiKey || '').trim(),
|
|
139
147
|
model: String(model || '').trim(),
|
|
140
148
|
chatbotId: String(chatbotId || '').trim(),
|
|
@@ -178,7 +186,9 @@ function shouldPersistSessionState(): boolean {
|
|
|
178
186
|
function resolveProfileRoot(): string {
|
|
179
187
|
const configured = String(process.env.BROWSER_PROFILE_ROOT || '').trim();
|
|
180
188
|
if (!configured) return ensureWritableDir(BROWSER_PROFILE_ROOT);
|
|
181
|
-
const resolved = path.isAbsolute(configured)
|
|
189
|
+
const resolved = path.isAbsolute(configured)
|
|
190
|
+
? configured
|
|
191
|
+
: path.resolve(WORKSPACE_ROOT, configured);
|
|
182
192
|
return ensureWritableDir(resolved);
|
|
183
193
|
}
|
|
184
194
|
|
|
@@ -206,7 +216,9 @@ function resolveRunner(): BrowserRunner | null {
|
|
|
206
216
|
return cachedRunner;
|
|
207
217
|
}
|
|
208
218
|
|
|
209
|
-
const whichAgentBrowser = spawnSync('which', ['agent-browser'], {
|
|
219
|
+
const whichAgentBrowser = spawnSync('which', ['agent-browser'], {
|
|
220
|
+
encoding: 'utf-8',
|
|
221
|
+
});
|
|
210
222
|
if (whichAgentBrowser.status === 0 && whichAgentBrowser.stdout.trim()) {
|
|
211
223
|
cachedRunner = { cmd: whichAgentBrowser.stdout.trim(), prefixArgs: [] };
|
|
212
224
|
return cachedRunner;
|
|
@@ -214,7 +226,10 @@ function resolveRunner(): BrowserRunner | null {
|
|
|
214
226
|
|
|
215
227
|
const whichNpx = spawnSync('which', ['npx'], { encoding: 'utf-8' });
|
|
216
228
|
if (whichNpx.status === 0 && whichNpx.stdout.trim()) {
|
|
217
|
-
cachedRunner = {
|
|
229
|
+
cachedRunner = {
|
|
230
|
+
cmd: whichNpx.stdout.trim(),
|
|
231
|
+
prefixArgs: ['--yes', 'agent-browser'],
|
|
232
|
+
};
|
|
218
233
|
return cachedRunner;
|
|
219
234
|
}
|
|
220
235
|
|
|
@@ -238,14 +253,18 @@ function getSession(sessionId: string): BrowserSession {
|
|
|
238
253
|
let profileDir: string | undefined;
|
|
239
254
|
if (shouldPersistProfiles()) {
|
|
240
255
|
try {
|
|
241
|
-
profileDir = ensureWritableDir(
|
|
256
|
+
profileDir = ensureWritableDir(
|
|
257
|
+
path.join(resolveProfileRoot(), runtimeKey),
|
|
258
|
+
);
|
|
242
259
|
} catch {
|
|
243
260
|
// Fallback to ephemeral browser context if profile dir cannot be created.
|
|
244
261
|
profileDir = undefined;
|
|
245
262
|
}
|
|
246
263
|
}
|
|
247
264
|
|
|
248
|
-
const stateName = shouldPersistSessionState()
|
|
265
|
+
const stateName = shouldPersistSessionState()
|
|
266
|
+
? deriveStableId(sessionKey, 48)
|
|
267
|
+
: undefined;
|
|
249
268
|
|
|
250
269
|
const session: BrowserSession = {
|
|
251
270
|
sessionKey,
|
|
@@ -304,7 +323,10 @@ function removeSession(sessionId: string): void {
|
|
|
304
323
|
|
|
305
324
|
function isPrivateIpv4(ip: string): boolean {
|
|
306
325
|
const parts = ip.split('.').map((part) => Number.parseInt(part, 10));
|
|
307
|
-
if (
|
|
326
|
+
if (
|
|
327
|
+
parts.length !== 4 ||
|
|
328
|
+
parts.some((part) => Number.isNaN(part) || part < 0 || part > 255)
|
|
329
|
+
) {
|
|
308
330
|
return false;
|
|
309
331
|
}
|
|
310
332
|
const [a, b] = parts;
|
|
@@ -340,7 +362,12 @@ function isPrivateIp(ip: string): boolean {
|
|
|
340
362
|
async function isPrivateHost(hostname: string): Promise<boolean> {
|
|
341
363
|
const host = hostname.trim().toLowerCase();
|
|
342
364
|
if (!host) return true;
|
|
343
|
-
if (
|
|
365
|
+
if (
|
|
366
|
+
host === 'localhost' ||
|
|
367
|
+
host.endsWith('.localhost') ||
|
|
368
|
+
host.endsWith('.local')
|
|
369
|
+
)
|
|
370
|
+
return true;
|
|
344
371
|
if (net.isIP(host) > 0) return isPrivateIp(host);
|
|
345
372
|
try {
|
|
346
373
|
const resolved = await lookup(host, { all: true, verbatim: true });
|
|
@@ -371,7 +398,9 @@ async function assertNavigationUrl(raw: unknown): Promise<URL> {
|
|
|
371
398
|
throw new Error(`Unsupported URL protocol: ${parsed.protocol}`);
|
|
372
399
|
}
|
|
373
400
|
|
|
374
|
-
const allowPrivate =
|
|
401
|
+
const allowPrivate =
|
|
402
|
+
String(process.env.BROWSER_ALLOW_PRIVATE_NETWORK || '').toLowerCase() ===
|
|
403
|
+
'true';
|
|
375
404
|
if (!allowPrivate && (await isPrivateHost(parsed.hostname))) {
|
|
376
405
|
throw new Error(
|
|
377
406
|
`Navigation blocked by SSRF guard: private or loopback host (${parsed.hostname}). ` +
|
|
@@ -409,7 +438,9 @@ function resolveOutputPath(rawPath: unknown, extension: 'png' | 'pdf'): string {
|
|
|
409
438
|
}
|
|
410
439
|
|
|
411
440
|
if (path.isAbsolute(requested)) {
|
|
412
|
-
throw new Error(
|
|
441
|
+
throw new Error(
|
|
442
|
+
'Absolute output paths are not allowed. Use a relative path.',
|
|
443
|
+
);
|
|
413
444
|
}
|
|
414
445
|
const normalized = requested.replace(/\\/g, '/');
|
|
415
446
|
const clean = path.posix.normalize(normalized);
|
|
@@ -417,7 +448,9 @@ function resolveOutputPath(rawPath: unknown, extension: 'png' | 'pdf'): string {
|
|
|
417
448
|
throw new Error('Output path escapes browser artifacts directory.');
|
|
418
449
|
}
|
|
419
450
|
|
|
420
|
-
const withExt = clean.endsWith(`.${extension}`)
|
|
451
|
+
const withExt = clean.endsWith(`.${extension}`)
|
|
452
|
+
? clean
|
|
453
|
+
: `${clean}.${extension}`;
|
|
421
454
|
const resolved = path.resolve(BROWSER_ARTIFACT_ROOT, withExt);
|
|
422
455
|
const root = path.resolve(BROWSER_ARTIFACT_ROOT);
|
|
423
456
|
if (resolved !== root && !resolved.startsWith(root + path.sep)) {
|
|
@@ -430,7 +463,10 @@ function resolveOutputPath(rawPath: unknown, extension: 'png' | 'pdf'): string {
|
|
|
430
463
|
function createTempScreenshotPath(prefix: string): string {
|
|
431
464
|
fs.mkdirSync(BROWSER_ARTIFACT_ROOT, { recursive: true });
|
|
432
465
|
const nonce = Math.random().toString(36).slice(2, 10);
|
|
433
|
-
return path.join(
|
|
466
|
+
return path.join(
|
|
467
|
+
BROWSER_ARTIFACT_ROOT,
|
|
468
|
+
`${prefix}-${Date.now()}-${nonce}.png`,
|
|
469
|
+
);
|
|
434
470
|
}
|
|
435
471
|
|
|
436
472
|
function asRecord(value: unknown): Record<string, unknown> | null {
|
|
@@ -441,7 +477,8 @@ function asRecord(value: unknown): Record<string, unknown> | null {
|
|
|
441
477
|
function normalizeSnapshotMode(rawMode: unknown): SnapshotMode {
|
|
442
478
|
if (rawMode == null || String(rawMode).trim() === '') return 'default';
|
|
443
479
|
const mode = String(rawMode).trim().toLowerCase();
|
|
444
|
-
if (mode === 'default' || mode === 'interactive' || mode === 'full')
|
|
480
|
+
if (mode === 'default' || mode === 'interactive' || mode === 'full')
|
|
481
|
+
return mode;
|
|
445
482
|
throw new Error('mode must be one of "default", "interactive", or "full"');
|
|
446
483
|
}
|
|
447
484
|
|
|
@@ -455,12 +492,17 @@ function parseOptionalFrame(raw: unknown): FrameTarget | null {
|
|
|
455
492
|
};
|
|
456
493
|
}
|
|
457
494
|
|
|
458
|
-
async function applyFrameTarget(
|
|
495
|
+
async function applyFrameTarget(
|
|
496
|
+
sessionId: string,
|
|
497
|
+
target: FrameTarget | null,
|
|
498
|
+
): Promise<void> {
|
|
459
499
|
if (!target) return;
|
|
460
500
|
const commandArgs = target.isMain ? ['main'] : [target.raw];
|
|
461
501
|
const frameResult = await runAgentBrowser(sessionId, 'frame', commandArgs);
|
|
462
502
|
if (!frameResult.success) {
|
|
463
|
-
throw new Error(
|
|
503
|
+
throw new Error(
|
|
504
|
+
frameResult.error || `failed to switch to frame "${target.raw}"`,
|
|
505
|
+
);
|
|
464
506
|
}
|
|
465
507
|
}
|
|
466
508
|
|
|
@@ -469,7 +511,9 @@ async function runBrowserEval(
|
|
|
469
511
|
script: string,
|
|
470
512
|
timeoutMs = 30_000,
|
|
471
513
|
): Promise<{ success: boolean; result?: unknown; error?: string }> {
|
|
472
|
-
const response = await runAgentBrowser(sessionId, 'eval', [script], {
|
|
514
|
+
const response = await runAgentBrowser(sessionId, 'eval', [script], {
|
|
515
|
+
timeoutMs,
|
|
516
|
+
});
|
|
473
517
|
if (!response.success) {
|
|
474
518
|
return { success: false, error: response.error || 'browser eval failed' };
|
|
475
519
|
}
|
|
@@ -483,7 +527,10 @@ function normalizeFrameMetadata(raw: unknown): Record<string, unknown>[] {
|
|
|
483
527
|
for (const item of raw) {
|
|
484
528
|
const entry = asRecord(item);
|
|
485
529
|
if (!entry) continue;
|
|
486
|
-
const index =
|
|
530
|
+
const index =
|
|
531
|
+
typeof entry.index === 'number' && Number.isFinite(entry.index)
|
|
532
|
+
? entry.index
|
|
533
|
+
: null;
|
|
487
534
|
const id = typeof entry.id === 'string' ? entry.id : null;
|
|
488
535
|
const name = typeof entry.name === 'string' ? entry.name : null;
|
|
489
536
|
const title = typeof entry.title === 'string' ? entry.title : null;
|
|
@@ -503,8 +550,14 @@ function normalizeImageList(raw: unknown): Record<string, unknown>[] {
|
|
|
503
550
|
const src = typeof entry.src === 'string' ? entry.src : '';
|
|
504
551
|
if (!src || src.startsWith('data:')) continue;
|
|
505
552
|
const alt = typeof entry.alt === 'string' ? entry.alt : '';
|
|
506
|
-
const width =
|
|
507
|
-
|
|
553
|
+
const width =
|
|
554
|
+
typeof entry.width === 'number' && Number.isFinite(entry.width)
|
|
555
|
+
? entry.width
|
|
556
|
+
: null;
|
|
557
|
+
const height =
|
|
558
|
+
typeof entry.height === 'number' && Number.isFinite(entry.height)
|
|
559
|
+
? entry.height
|
|
560
|
+
: null;
|
|
508
561
|
images.push({ src, alt, width, height });
|
|
509
562
|
}
|
|
510
563
|
return images;
|
|
@@ -519,8 +572,12 @@ function normalizeTrackedRequests(raw: unknown): Record<string, unknown>[] {
|
|
|
519
572
|
const url = typeof entry.url === 'string' ? entry.url : '';
|
|
520
573
|
if (!url) continue;
|
|
521
574
|
const method = typeof entry.method === 'string' ? entry.method : null;
|
|
522
|
-
const type =
|
|
523
|
-
|
|
575
|
+
const type =
|
|
576
|
+
typeof entry.resourceType === 'string' ? entry.resourceType : null;
|
|
577
|
+
const timestamp =
|
|
578
|
+
typeof entry.timestamp === 'number' && Number.isFinite(entry.timestamp)
|
|
579
|
+
? entry.timestamp
|
|
580
|
+
: null;
|
|
524
581
|
requests.push({
|
|
525
582
|
url,
|
|
526
583
|
method,
|
|
@@ -534,7 +591,10 @@ function normalizeTrackedRequests(raw: unknown): Record<string, unknown>[] {
|
|
|
534
591
|
return requests;
|
|
535
592
|
}
|
|
536
593
|
|
|
537
|
-
function normalizePerformanceRequests(
|
|
594
|
+
function normalizePerformanceRequests(
|
|
595
|
+
raw: unknown,
|
|
596
|
+
filter?: string,
|
|
597
|
+
): Record<string, unknown>[] {
|
|
538
598
|
if (!Array.isArray(raw)) return [];
|
|
539
599
|
const loweredFilter = (filter || '').toLowerCase();
|
|
540
600
|
const requests: Record<string, unknown>[] = [];
|
|
@@ -545,10 +605,19 @@ function normalizePerformanceRequests(raw: unknown, filter?: string): Record<str
|
|
|
545
605
|
if (!url) continue;
|
|
546
606
|
if (loweredFilter && !url.toLowerCase().includes(loweredFilter)) continue;
|
|
547
607
|
const type = typeof entry.type === 'string' ? entry.type : null;
|
|
548
|
-
const duration =
|
|
608
|
+
const duration =
|
|
609
|
+
typeof entry.duration === 'number' && Number.isFinite(entry.duration)
|
|
610
|
+
? entry.duration
|
|
611
|
+
: null;
|
|
549
612
|
const transferSize =
|
|
550
|
-
typeof entry.transfer_size === 'number' &&
|
|
551
|
-
|
|
613
|
+
typeof entry.transfer_size === 'number' &&
|
|
614
|
+
Number.isFinite(entry.transfer_size)
|
|
615
|
+
? entry.transfer_size
|
|
616
|
+
: null;
|
|
617
|
+
const startTime =
|
|
618
|
+
typeof entry.start_time === 'number' && Number.isFinite(entry.start_time)
|
|
619
|
+
? entry.start_time
|
|
620
|
+
: null;
|
|
552
621
|
requests.push({
|
|
553
622
|
url,
|
|
554
623
|
method: 'GET',
|
|
@@ -563,11 +632,15 @@ function normalizePerformanceRequests(raw: unknown, filter?: string): Record<str
|
|
|
563
632
|
return requests;
|
|
564
633
|
}
|
|
565
634
|
|
|
566
|
-
function buildBotDetectionWarning(
|
|
635
|
+
function buildBotDetectionWarning(
|
|
636
|
+
titleValue: unknown,
|
|
637
|
+
): Record<string, unknown> | null {
|
|
567
638
|
const title = String(titleValue || '').trim();
|
|
568
639
|
if (!title) return null;
|
|
569
640
|
const lower = title.toLowerCase();
|
|
570
|
-
const matched = BOT_DETECTION_PATTERNS.find((pattern) =>
|
|
641
|
+
const matched = BOT_DETECTION_PATTERNS.find((pattern) =>
|
|
642
|
+
lower.includes(pattern),
|
|
643
|
+
);
|
|
571
644
|
if (!matched) return null;
|
|
572
645
|
const hintOverride = String(process.env.BROWSER_STEALTH_HINT || '').trim();
|
|
573
646
|
const hint =
|
|
@@ -611,22 +684,33 @@ function extractVisionTextContent(content: unknown): string {
|
|
|
611
684
|
return chunks.join('\n').trim();
|
|
612
685
|
}
|
|
613
686
|
|
|
614
|
-
async function callVisionModel(
|
|
687
|
+
async function callVisionModel(
|
|
688
|
+
question: string,
|
|
689
|
+
imageBase64: string,
|
|
690
|
+
): Promise<{ model: string; analysis: string }> {
|
|
615
691
|
const apiKey = currentBrowserModelContext.apiKey;
|
|
616
692
|
const baseUrl = currentBrowserModelContext.baseUrl;
|
|
617
693
|
const model = currentBrowserModelContext.model;
|
|
618
694
|
const chatbotId = currentBrowserModelContext.chatbotId;
|
|
619
695
|
if (!apiKey) {
|
|
620
|
-
throw new Error(
|
|
696
|
+
throw new Error(
|
|
697
|
+
'browser_vision is not configured: missing active request API key context.',
|
|
698
|
+
);
|
|
621
699
|
}
|
|
622
700
|
if (!baseUrl) {
|
|
623
|
-
throw new Error(
|
|
701
|
+
throw new Error(
|
|
702
|
+
'browser_vision is not configured: missing active request base URL context.',
|
|
703
|
+
);
|
|
624
704
|
}
|
|
625
705
|
if (!model) {
|
|
626
|
-
throw new Error(
|
|
706
|
+
throw new Error(
|
|
707
|
+
'browser_vision is not configured: missing active request model context.',
|
|
708
|
+
);
|
|
627
709
|
}
|
|
628
710
|
if (!chatbotId) {
|
|
629
|
-
throw new Error(
|
|
711
|
+
throw new Error(
|
|
712
|
+
'browser_vision is not configured: missing active request chatbot_id context.',
|
|
713
|
+
);
|
|
630
714
|
}
|
|
631
715
|
const endpoint = `${baseUrl}/v1/chat/completions`;
|
|
632
716
|
const payload = {
|
|
@@ -638,7 +722,10 @@ async function callVisionModel(question: string, imageBase64: string): Promise<{
|
|
|
638
722
|
role: 'user',
|
|
639
723
|
content: [
|
|
640
724
|
{ type: 'text', text: question },
|
|
641
|
-
{
|
|
725
|
+
{
|
|
726
|
+
type: 'image_url',
|
|
727
|
+
image_url: { url: `data:image/png;base64,${imageBase64}` },
|
|
728
|
+
},
|
|
642
729
|
],
|
|
643
730
|
},
|
|
644
731
|
],
|
|
@@ -655,8 +742,11 @@ async function callVisionModel(question: string, imageBase64: string): Promise<{
|
|
|
655
742
|
|
|
656
743
|
const bodyText = await response.text();
|
|
657
744
|
if (!response.ok) {
|
|
658
|
-
const details =
|
|
659
|
-
|
|
745
|
+
const details =
|
|
746
|
+
bodyText.length > 600 ? `${bodyText.slice(0, 600)}...` : bodyText;
|
|
747
|
+
throw new Error(
|
|
748
|
+
`vision API request failed (${response.status}): ${details}`,
|
|
749
|
+
);
|
|
660
750
|
}
|
|
661
751
|
|
|
662
752
|
let parsed: unknown;
|
|
@@ -697,7 +787,10 @@ async function runAgentBrowser(
|
|
|
697
787
|
};
|
|
698
788
|
}
|
|
699
789
|
|
|
700
|
-
const timeoutMs = Math.max(
|
|
790
|
+
const timeoutMs = Math.max(
|
|
791
|
+
1_000,
|
|
792
|
+
Math.min(options.timeoutMs ?? BROWSER_DEFAULT_TIMEOUT_MS, 180_000),
|
|
793
|
+
);
|
|
701
794
|
const session = getSession(sessionId);
|
|
702
795
|
const homeDir = resolveWritableHome();
|
|
703
796
|
const npmCacheDir = ensureWritableDir(BROWSER_NPM_CACHE);
|
|
@@ -742,27 +835,47 @@ async function runAgentBrowser(
|
|
|
742
835
|
return { success: true, data: {} };
|
|
743
836
|
}
|
|
744
837
|
|
|
745
|
-
let parsed:
|
|
838
|
+
let parsed: unknown;
|
|
746
839
|
try {
|
|
747
840
|
parsed = JSON.parse(output);
|
|
748
841
|
} catch {
|
|
749
842
|
return { success: true, data: { raw: output } };
|
|
750
843
|
}
|
|
751
844
|
|
|
752
|
-
if (parsed && typeof parsed === 'object' && parsed
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
845
|
+
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
|
846
|
+
const parsedRecord = parsed as Record<string, unknown>;
|
|
847
|
+
if (parsedRecord.success === false) {
|
|
848
|
+
return {
|
|
849
|
+
success: false,
|
|
850
|
+
error: String(parsedRecord.error || 'browser command failed'),
|
|
851
|
+
};
|
|
852
|
+
}
|
|
853
|
+
if ('data' in parsedRecord) {
|
|
854
|
+
return { success: true, data: parsedRecord.data };
|
|
855
|
+
}
|
|
757
856
|
}
|
|
758
857
|
return { success: true, data: parsed };
|
|
759
|
-
} catch (err:
|
|
760
|
-
const
|
|
761
|
-
|
|
858
|
+
} catch (err: unknown) {
|
|
859
|
+
const errorRecord = err as {
|
|
860
|
+
stderr?: unknown;
|
|
861
|
+
stdout?: unknown;
|
|
862
|
+
code?: unknown;
|
|
863
|
+
message?: unknown;
|
|
864
|
+
};
|
|
865
|
+
const stderr =
|
|
866
|
+
typeof errorRecord.stderr === 'string' ? errorRecord.stderr.trim() : '';
|
|
867
|
+
const stdout =
|
|
868
|
+
typeof errorRecord.stdout === 'string' ? errorRecord.stdout.trim() : '';
|
|
762
869
|
const timeoutHint =
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
870
|
+
errorRecord.code === 'ETIMEDOUT' ||
|
|
871
|
+
/timed out/i.test(String(errorRecord.message || ''))
|
|
872
|
+
? ` (timeout ${timeoutMs}ms)`
|
|
873
|
+
: '';
|
|
874
|
+
const msg = stderr || stdout || String(errorRecord.message || err);
|
|
875
|
+
return {
|
|
876
|
+
success: false,
|
|
877
|
+
error: `browser command failed${timeoutHint}: ${msg}`,
|
|
878
|
+
};
|
|
766
879
|
}
|
|
767
880
|
}
|
|
768
881
|
|
|
@@ -774,38 +887,63 @@ function failure(message: string): string {
|
|
|
774
887
|
return JSON.stringify({ success: false, error: message }, null, 2);
|
|
775
888
|
}
|
|
776
889
|
|
|
777
|
-
export async function executeBrowserTool(
|
|
890
|
+
export async function executeBrowserTool(
|
|
891
|
+
name: string,
|
|
892
|
+
args: Record<string, unknown>,
|
|
893
|
+
sessionId: string,
|
|
894
|
+
): Promise<string> {
|
|
778
895
|
try {
|
|
779
896
|
const effectiveSessionId = normalizeSessionKey(sessionId || 'default');
|
|
780
897
|
switch (name) {
|
|
781
898
|
case 'browser_navigate': {
|
|
782
899
|
const parsed = await assertNavigationUrl(args.url);
|
|
783
|
-
const result = await runAgentBrowser(
|
|
784
|
-
|
|
900
|
+
const result = await runAgentBrowser(
|
|
901
|
+
effectiveSessionId,
|
|
902
|
+
'open',
|
|
903
|
+
[parsed.toString()],
|
|
904
|
+
{ timeoutMs: 60_000 },
|
|
905
|
+
);
|
|
906
|
+
if (!result.success)
|
|
907
|
+
return failure(result.error || 'navigation failed');
|
|
785
908
|
const data = (result.data || {}) as Record<string, unknown>;
|
|
786
909
|
const title = String(data.title || '');
|
|
787
910
|
const botWarning = buildBotDetectionWarning(title);
|
|
788
|
-
const textEval = await runBrowserEval(
|
|
911
|
+
const textEval = await runBrowserEval(
|
|
912
|
+
effectiveSessionId,
|
|
913
|
+
EXTRACT_TEXT_PREVIEW_SCRIPT,
|
|
914
|
+
20_000,
|
|
915
|
+
);
|
|
789
916
|
const textData = textEval.success ? asRecord(textEval.result) : null;
|
|
790
|
-
const contentPreview =
|
|
917
|
+
const contentPreview =
|
|
918
|
+
typeof textData?.preview === 'string' ? textData.preview : '';
|
|
791
919
|
const contentLength =
|
|
792
|
-
typeof textData?.text_length === 'number' &&
|
|
920
|
+
typeof textData?.text_length === 'number' &&
|
|
921
|
+
Number.isFinite(textData.text_length)
|
|
793
922
|
? Math.max(0, Math.floor(textData.text_length))
|
|
794
923
|
: 0;
|
|
795
924
|
const contentPreviewTruncated = textData?.preview_truncated === true;
|
|
796
925
|
const hasNoscript = textData?.has_noscript === true;
|
|
797
926
|
const rootShell = textData?.root_shell === true;
|
|
798
|
-
const readyState =
|
|
799
|
-
|
|
927
|
+
const readyState =
|
|
928
|
+
typeof textData?.ready_state === 'string' ? textData.ready_state : '';
|
|
929
|
+
const extractionHint = buildReadExtractionHint({
|
|
930
|
+
contentLength,
|
|
931
|
+
hasNoscript,
|
|
932
|
+
rootShell,
|
|
933
|
+
});
|
|
800
934
|
// Best-effort priming so browser_network has request listeners active quickly.
|
|
801
|
-
await runAgentBrowser(effectiveSessionId, 'network', [
|
|
935
|
+
await runAgentBrowser(effectiveSessionId, 'network', [
|
|
936
|
+
'requests',
|
|
937
|
+
]).catch(() => undefined);
|
|
802
938
|
return success({
|
|
803
939
|
url: data.url || parsed.toString(),
|
|
804
940
|
title,
|
|
805
941
|
session_id: effectiveSessionId,
|
|
806
942
|
content_text_length: contentLength,
|
|
807
943
|
...(contentPreview ? { content_preview: contentPreview } : {}),
|
|
808
|
-
...(contentPreview
|
|
944
|
+
...(contentPreview
|
|
945
|
+
? { content_preview_truncated: contentPreviewTruncated }
|
|
946
|
+
: {}),
|
|
809
947
|
...(readyState ? { ready_state: readyState } : {}),
|
|
810
948
|
...(hasNoscript ? { has_noscript: true } : {}),
|
|
811
949
|
...(rootShell ? { root_shell: true } : {}),
|
|
@@ -822,18 +960,31 @@ export async function executeBrowserTool(name: string, args: Record<string, unkn
|
|
|
822
960
|
else if (mode === 'full') commandArgs = [];
|
|
823
961
|
else commandArgs = full ? [] : ['-i', '-c'];
|
|
824
962
|
|
|
825
|
-
const result = await runAgentBrowser(
|
|
963
|
+
const result = await runAgentBrowser(
|
|
964
|
+
effectiveSessionId,
|
|
965
|
+
'snapshot',
|
|
966
|
+
commandArgs,
|
|
967
|
+
{ timeoutMs: 45_000 },
|
|
968
|
+
);
|
|
826
969
|
if (!result.success) return failure(result.error || 'snapshot failed');
|
|
827
970
|
const data = (result.data || {}) as Record<string, unknown>;
|
|
828
971
|
const rawSnapshot = String(data.snapshot || '');
|
|
829
972
|
const truncated = truncateSnapshot(rawSnapshot);
|
|
830
|
-
const frameEval = await runBrowserEval(
|
|
831
|
-
|
|
973
|
+
const frameEval = await runBrowserEval(
|
|
974
|
+
effectiveSessionId,
|
|
975
|
+
EXTRACT_IFRAMES_SCRIPT,
|
|
976
|
+
15_000,
|
|
977
|
+
);
|
|
978
|
+
const frames = frameEval.success
|
|
979
|
+
? normalizeFrameMetadata(frameEval.result)
|
|
980
|
+
: [];
|
|
832
981
|
return success({
|
|
833
982
|
snapshot: truncated.text,
|
|
834
983
|
truncated: truncated.truncated,
|
|
835
984
|
element_count:
|
|
836
|
-
data.refs && typeof data.refs === 'object'
|
|
985
|
+
data.refs && typeof data.refs === 'object'
|
|
986
|
+
? Object.keys(data.refs as Record<string, unknown>).length
|
|
987
|
+
: 0,
|
|
837
988
|
url: data.url || data.origin || '',
|
|
838
989
|
mode,
|
|
839
990
|
...(frames.length > 0 ? { frames, frame_count: frames.length } : {}),
|
|
@@ -844,8 +995,11 @@ export async function executeBrowserTool(name: string, args: Record<string, unkn
|
|
|
844
995
|
const ref = ensureRef(args.ref);
|
|
845
996
|
const frame = parseOptionalFrame(args.frame);
|
|
846
997
|
await applyFrameTarget(effectiveSessionId, frame);
|
|
847
|
-
const result = await runAgentBrowser(effectiveSessionId, 'click', [
|
|
848
|
-
|
|
998
|
+
const result = await runAgentBrowser(effectiveSessionId, 'click', [
|
|
999
|
+
ref,
|
|
1000
|
+
]);
|
|
1001
|
+
if (!result.success)
|
|
1002
|
+
return failure(result.error || `failed to click ${ref}`);
|
|
849
1003
|
return success({
|
|
850
1004
|
clicked: ref,
|
|
851
1005
|
...(frame ? { frame: frame.raw } : {}),
|
|
@@ -858,8 +1012,12 @@ export async function executeBrowserTool(name: string, args: Record<string, unkn
|
|
|
858
1012
|
if (!text) return failure('text is required');
|
|
859
1013
|
const frame = parseOptionalFrame(args.frame);
|
|
860
1014
|
await applyFrameTarget(effectiveSessionId, frame);
|
|
861
|
-
const result = await runAgentBrowser(effectiveSessionId, 'fill', [
|
|
862
|
-
|
|
1015
|
+
const result = await runAgentBrowser(effectiveSessionId, 'fill', [
|
|
1016
|
+
ref,
|
|
1017
|
+
text,
|
|
1018
|
+
]);
|
|
1019
|
+
if (!result.success)
|
|
1020
|
+
return failure(result.error || `failed to fill ${ref}`);
|
|
863
1021
|
return success({
|
|
864
1022
|
element: ref,
|
|
865
1023
|
typed_chars: text.length,
|
|
@@ -872,8 +1030,11 @@ export async function executeBrowserTool(name: string, args: Record<string, unkn
|
|
|
872
1030
|
if (!key) return failure('key is required');
|
|
873
1031
|
const frame = parseOptionalFrame(args.frame);
|
|
874
1032
|
await applyFrameTarget(effectiveSessionId, frame);
|
|
875
|
-
const result = await runAgentBrowser(effectiveSessionId, 'press', [
|
|
876
|
-
|
|
1033
|
+
const result = await runAgentBrowser(effectiveSessionId, 'press', [
|
|
1034
|
+
key,
|
|
1035
|
+
]);
|
|
1036
|
+
if (!result.success)
|
|
1037
|
+
return failure(result.error || `failed to press ${key}`);
|
|
877
1038
|
return success({
|
|
878
1039
|
key,
|
|
879
1040
|
...(frame ? { frame: frame.raw } : {}),
|
|
@@ -881,20 +1042,30 @@ export async function executeBrowserTool(name: string, args: Record<string, unkn
|
|
|
881
1042
|
}
|
|
882
1043
|
|
|
883
1044
|
case 'browser_scroll': {
|
|
884
|
-
const direction = String(args.direction || '')
|
|
1045
|
+
const direction = String(args.direction || '')
|
|
1046
|
+
.trim()
|
|
1047
|
+
.toLowerCase();
|
|
885
1048
|
if (direction !== 'up' && direction !== 'down') {
|
|
886
1049
|
return failure('direction must be "up" or "down"');
|
|
887
1050
|
}
|
|
888
1051
|
const pixelsRaw = Number(args.pixels);
|
|
889
|
-
const pixels =
|
|
890
|
-
|
|
891
|
-
|
|
1052
|
+
const pixels =
|
|
1053
|
+
Number.isFinite(pixelsRaw) && pixelsRaw > 0
|
|
1054
|
+
? Math.floor(pixelsRaw)
|
|
1055
|
+
: 800;
|
|
1056
|
+
const result = await runAgentBrowser(effectiveSessionId, 'scroll', [
|
|
1057
|
+
direction,
|
|
1058
|
+
String(pixels),
|
|
1059
|
+
]);
|
|
1060
|
+
if (!result.success)
|
|
1061
|
+
return failure(result.error || `failed to scroll ${direction}`);
|
|
892
1062
|
return success({ direction, pixels });
|
|
893
1063
|
}
|
|
894
1064
|
|
|
895
1065
|
case 'browser_back': {
|
|
896
1066
|
const result = await runAgentBrowser(effectiveSessionId, 'back', []);
|
|
897
|
-
if (!result.success)
|
|
1067
|
+
if (!result.success)
|
|
1068
|
+
return failure(result.error || 'failed to navigate back');
|
|
898
1069
|
const data = (result.data || {}) as Record<string, unknown>;
|
|
899
1070
|
return success({ url: data.url || '' });
|
|
900
1071
|
}
|
|
@@ -903,8 +1074,14 @@ export async function executeBrowserTool(name: string, args: Record<string, unkn
|
|
|
903
1074
|
const outPath = resolveOutputPath(args.path, 'png');
|
|
904
1075
|
const fullPage = args.fullPage === true;
|
|
905
1076
|
const commandArgs = fullPage ? ['--full', outPath] : [outPath];
|
|
906
|
-
const result = await runAgentBrowser(
|
|
907
|
-
|
|
1077
|
+
const result = await runAgentBrowser(
|
|
1078
|
+
effectiveSessionId,
|
|
1079
|
+
'screenshot',
|
|
1080
|
+
commandArgs,
|
|
1081
|
+
{ timeoutMs: 60_000 },
|
|
1082
|
+
);
|
|
1083
|
+
if (!result.success)
|
|
1084
|
+
return failure(result.error || 'failed to capture screenshot');
|
|
908
1085
|
return success({
|
|
909
1086
|
path: path.relative(WORKSPACE_ROOT, outPath),
|
|
910
1087
|
full_page: fullPage,
|
|
@@ -913,8 +1090,14 @@ export async function executeBrowserTool(name: string, args: Record<string, unkn
|
|
|
913
1090
|
|
|
914
1091
|
case 'browser_pdf': {
|
|
915
1092
|
const outPath = resolveOutputPath(args.path, 'pdf');
|
|
916
|
-
const result = await runAgentBrowser(
|
|
917
|
-
|
|
1093
|
+
const result = await runAgentBrowser(
|
|
1094
|
+
effectiveSessionId,
|
|
1095
|
+
'pdf',
|
|
1096
|
+
[outPath],
|
|
1097
|
+
{ timeoutMs: 60_000 },
|
|
1098
|
+
);
|
|
1099
|
+
if (!result.success)
|
|
1100
|
+
return failure(result.error || 'failed to generate pdf');
|
|
918
1101
|
return success({ path: path.relative(WORKSPACE_ROOT, outPath) });
|
|
919
1102
|
}
|
|
920
1103
|
|
|
@@ -924,11 +1107,19 @@ export async function executeBrowserTool(name: string, args: Record<string, unkn
|
|
|
924
1107
|
|
|
925
1108
|
const tempPath = createTempScreenshotPath('browser-vision');
|
|
926
1109
|
try {
|
|
927
|
-
const screenshotResult = await runAgentBrowser(
|
|
928
|
-
|
|
929
|
-
|
|
1110
|
+
const screenshotResult = await runAgentBrowser(
|
|
1111
|
+
effectiveSessionId,
|
|
1112
|
+
'screenshot',
|
|
1113
|
+
[tempPath],
|
|
1114
|
+
{
|
|
1115
|
+
timeoutMs: 60_000,
|
|
1116
|
+
},
|
|
1117
|
+
);
|
|
930
1118
|
if (!screenshotResult.success) {
|
|
931
|
-
return failure(
|
|
1119
|
+
return failure(
|
|
1120
|
+
screenshotResult.error ||
|
|
1121
|
+
'failed to capture screenshot for vision analysis',
|
|
1122
|
+
);
|
|
932
1123
|
}
|
|
933
1124
|
|
|
934
1125
|
const imageBuffer = await fs.promises.readFile(tempPath);
|
|
@@ -944,8 +1135,13 @@ export async function executeBrowserTool(name: string, args: Record<string, unkn
|
|
|
944
1135
|
}
|
|
945
1136
|
|
|
946
1137
|
case 'browser_get_images': {
|
|
947
|
-
const evalResult = await runBrowserEval(
|
|
948
|
-
|
|
1138
|
+
const evalResult = await runBrowserEval(
|
|
1139
|
+
effectiveSessionId,
|
|
1140
|
+
EXTRACT_IMAGES_SCRIPT,
|
|
1141
|
+
20_000,
|
|
1142
|
+
);
|
|
1143
|
+
if (!evalResult.success)
|
|
1144
|
+
return failure(evalResult.error || 'failed to extract images');
|
|
949
1145
|
const images = normalizeImageList(evalResult.result);
|
|
950
1146
|
return success({ count: images.length, images });
|
|
951
1147
|
}
|
|
@@ -953,8 +1149,14 @@ export async function executeBrowserTool(name: string, args: Record<string, unkn
|
|
|
953
1149
|
case 'browser_console': {
|
|
954
1150
|
const clear = args.clear === true;
|
|
955
1151
|
const commandArgs = clear ? ['--clear'] : [];
|
|
956
|
-
const result = await runAgentBrowser(
|
|
957
|
-
|
|
1152
|
+
const result = await runAgentBrowser(
|
|
1153
|
+
effectiveSessionId,
|
|
1154
|
+
'console',
|
|
1155
|
+
commandArgs,
|
|
1156
|
+
{ timeoutMs: 20_000 },
|
|
1157
|
+
);
|
|
1158
|
+
if (!result.success)
|
|
1159
|
+
return failure(result.error || 'failed to read console logs');
|
|
958
1160
|
const data = asRecord(result.data) || {};
|
|
959
1161
|
if (clear) {
|
|
960
1162
|
return success({ cleared: true, count: 0, messages: [] });
|
|
@@ -967,11 +1169,22 @@ export async function executeBrowserTool(name: string, args: Record<string, unkn
|
|
|
967
1169
|
const text = typeof entry.text === 'string' ? entry.text : '';
|
|
968
1170
|
const level = typeof entry.type === 'string' ? entry.type : 'log';
|
|
969
1171
|
const timestamp =
|
|
970
|
-
typeof entry.timestamp === 'number' &&
|
|
1172
|
+
typeof entry.timestamp === 'number' &&
|
|
1173
|
+
Number.isFinite(entry.timestamp)
|
|
1174
|
+
? entry.timestamp
|
|
1175
|
+
: null;
|
|
971
1176
|
if (!text) return null;
|
|
972
1177
|
return { level, text, timestamp };
|
|
973
1178
|
})
|
|
974
|
-
.filter(
|
|
1179
|
+
.filter(
|
|
1180
|
+
(
|
|
1181
|
+
item,
|
|
1182
|
+
): item is {
|
|
1183
|
+
level: string;
|
|
1184
|
+
text: string;
|
|
1185
|
+
timestamp: number | null;
|
|
1186
|
+
} => item !== null,
|
|
1187
|
+
);
|
|
975
1188
|
return success({
|
|
976
1189
|
messages,
|
|
977
1190
|
count: messages.length,
|
|
@@ -983,39 +1196,70 @@ export async function executeBrowserTool(name: string, args: Record<string, unkn
|
|
|
983
1196
|
const clear = args.clear === true;
|
|
984
1197
|
const filter = String(args.filter || '').trim();
|
|
985
1198
|
if (clear) {
|
|
986
|
-
const clearRequestsResult = await runAgentBrowser(
|
|
987
|
-
|
|
988
|
-
|
|
1199
|
+
const clearRequestsResult = await runAgentBrowser(
|
|
1200
|
+
effectiveSessionId,
|
|
1201
|
+
'network',
|
|
1202
|
+
['requests', '--clear'],
|
|
1203
|
+
{
|
|
1204
|
+
timeoutMs: 20_000,
|
|
1205
|
+
},
|
|
1206
|
+
);
|
|
989
1207
|
if (!clearRequestsResult.success) {
|
|
990
|
-
return failure(
|
|
1208
|
+
return failure(
|
|
1209
|
+
clearRequestsResult.error ||
|
|
1210
|
+
'failed to clear network request history',
|
|
1211
|
+
);
|
|
991
1212
|
}
|
|
992
|
-
await runBrowserEval(
|
|
1213
|
+
await runBrowserEval(
|
|
1214
|
+
effectiveSessionId,
|
|
1215
|
+
CLEAR_NETWORK_TIMINGS_SCRIPT,
|
|
1216
|
+
10_000,
|
|
1217
|
+
).catch(() => undefined);
|
|
993
1218
|
return success({ cleared: true, count: 0, requests: [] });
|
|
994
1219
|
}
|
|
995
1220
|
|
|
996
1221
|
const networkArgs = ['requests'];
|
|
997
1222
|
if (filter) networkArgs.push('--filter', filter);
|
|
998
|
-
const trackedResult = await runAgentBrowser(
|
|
1223
|
+
const trackedResult = await runAgentBrowser(
|
|
1224
|
+
effectiveSessionId,
|
|
1225
|
+
'network',
|
|
1226
|
+
networkArgs,
|
|
1227
|
+
{ timeoutMs: 20_000 },
|
|
1228
|
+
);
|
|
999
1229
|
const trackedData = asRecord(trackedResult.data);
|
|
1000
|
-
const trackedRequests = trackedResult.success
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1230
|
+
const trackedRequests = trackedResult.success
|
|
1231
|
+
? normalizeTrackedRequests(trackedData?.requests)
|
|
1232
|
+
: [];
|
|
1233
|
+
|
|
1234
|
+
const timingsEval = await runBrowserEval(
|
|
1235
|
+
effectiveSessionId,
|
|
1236
|
+
NETWORK_TIMINGS_SCRIPT,
|
|
1237
|
+
20_000,
|
|
1238
|
+
);
|
|
1239
|
+
const perfRequests = timingsEval.success
|
|
1240
|
+
? normalizePerformanceRequests(timingsEval.result, filter)
|
|
1241
|
+
: [];
|
|
1004
1242
|
|
|
1005
1243
|
if (!trackedResult.success && !timingsEval.success) {
|
|
1006
|
-
return failure(
|
|
1244
|
+
return failure(
|
|
1245
|
+
trackedResult.error ||
|
|
1246
|
+
timingsEval.error ||
|
|
1247
|
+
'failed to read network requests',
|
|
1248
|
+
);
|
|
1007
1249
|
}
|
|
1008
1250
|
|
|
1009
1251
|
const dedupe = new Set<string>();
|
|
1010
|
-
const requests = [...trackedRequests, ...perfRequests].filter(
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1252
|
+
const requests = [...trackedRequests, ...perfRequests].filter(
|
|
1253
|
+
(entry) => {
|
|
1254
|
+
const url = typeof entry.url === 'string' ? entry.url : '';
|
|
1255
|
+
const method = typeof entry.method === 'string' ? entry.method : '';
|
|
1256
|
+
const type = typeof entry.type === 'string' ? entry.type : '';
|
|
1257
|
+
const key = `${method}|${type}|${url}`;
|
|
1258
|
+
if (!url || dedupe.has(key)) return false;
|
|
1259
|
+
dedupe.add(key);
|
|
1260
|
+
return true;
|
|
1261
|
+
},
|
|
1262
|
+
);
|
|
1019
1263
|
|
|
1020
1264
|
return success({
|
|
1021
1265
|
count: requests.length,
|
|
@@ -1054,7 +1298,10 @@ export const BROWSER_TOOL_DEFINITIONS: ToolDefinition[] = [
|
|
|
1054
1298
|
parameters: {
|
|
1055
1299
|
type: 'object',
|
|
1056
1300
|
properties: {
|
|
1057
|
-
url: {
|
|
1301
|
+
url: {
|
|
1302
|
+
type: 'string',
|
|
1303
|
+
description: 'URL to open (http:// or https://)',
|
|
1304
|
+
},
|
|
1058
1305
|
},
|
|
1059
1306
|
required: ['url'],
|
|
1060
1307
|
},
|
|
@@ -1069,7 +1316,11 @@ export const BROWSER_TOOL_DEFINITIONS: ToolDefinition[] = [
|
|
|
1069
1316
|
parameters: {
|
|
1070
1317
|
type: 'object',
|
|
1071
1318
|
properties: {
|
|
1072
|
-
full: {
|
|
1319
|
+
full: {
|
|
1320
|
+
type: 'boolean',
|
|
1321
|
+
description:
|
|
1322
|
+
'If true, request fuller snapshot output (default: false).',
|
|
1323
|
+
},
|
|
1073
1324
|
mode: {
|
|
1074
1325
|
type: 'string',
|
|
1075
1326
|
enum: ['default', 'interactive', 'full'],
|
|
@@ -1089,10 +1340,14 @@ export const BROWSER_TOOL_DEFINITIONS: ToolDefinition[] = [
|
|
|
1089
1340
|
parameters: {
|
|
1090
1341
|
type: 'object',
|
|
1091
1342
|
properties: {
|
|
1092
|
-
ref: {
|
|
1343
|
+
ref: {
|
|
1344
|
+
type: 'string',
|
|
1345
|
+
description: 'Element reference from browser_snapshot.',
|
|
1346
|
+
},
|
|
1093
1347
|
frame: {
|
|
1094
1348
|
type: 'string',
|
|
1095
|
-
description:
|
|
1349
|
+
description:
|
|
1350
|
+
'Optional frame selector. Use "main" to target the main document again.',
|
|
1096
1351
|
},
|
|
1097
1352
|
},
|
|
1098
1353
|
required: ['ref'],
|
|
@@ -1103,15 +1358,20 @@ export const BROWSER_TOOL_DEFINITIONS: ToolDefinition[] = [
|
|
|
1103
1358
|
type: 'function',
|
|
1104
1359
|
function: {
|
|
1105
1360
|
name: 'browser_type',
|
|
1106
|
-
description:
|
|
1361
|
+
description:
|
|
1362
|
+
'Type text into an input element by snapshot ref (clears then fills).',
|
|
1107
1363
|
parameters: {
|
|
1108
1364
|
type: 'object',
|
|
1109
1365
|
properties: {
|
|
1110
|
-
ref: {
|
|
1366
|
+
ref: {
|
|
1367
|
+
type: 'string',
|
|
1368
|
+
description: 'Element reference from browser_snapshot.',
|
|
1369
|
+
},
|
|
1111
1370
|
text: { type: 'string', description: 'Text to type.' },
|
|
1112
1371
|
frame: {
|
|
1113
1372
|
type: 'string',
|
|
1114
|
-
description:
|
|
1373
|
+
description:
|
|
1374
|
+
'Optional frame selector. Use "main" to target the main document again.',
|
|
1115
1375
|
},
|
|
1116
1376
|
},
|
|
1117
1377
|
required: ['ref', 'text'],
|
|
@@ -1122,14 +1382,16 @@ export const BROWSER_TOOL_DEFINITIONS: ToolDefinition[] = [
|
|
|
1122
1382
|
type: 'function',
|
|
1123
1383
|
function: {
|
|
1124
1384
|
name: 'browser_press',
|
|
1125
|
-
description:
|
|
1385
|
+
description:
|
|
1386
|
+
'Press a keyboard key in the active page (Enter, Tab, Escape, etc.).',
|
|
1126
1387
|
parameters: {
|
|
1127
1388
|
type: 'object',
|
|
1128
1389
|
properties: {
|
|
1129
1390
|
key: { type: 'string', description: 'Keyboard key name.' },
|
|
1130
1391
|
frame: {
|
|
1131
1392
|
type: 'string',
|
|
1132
|
-
description:
|
|
1393
|
+
description:
|
|
1394
|
+
'Optional frame selector. Use "main" to target the main document again.',
|
|
1133
1395
|
},
|
|
1134
1396
|
},
|
|
1135
1397
|
required: ['key'],
|
|
@@ -1144,8 +1406,14 @@ export const BROWSER_TOOL_DEFINITIONS: ToolDefinition[] = [
|
|
|
1144
1406
|
parameters: {
|
|
1145
1407
|
type: 'object',
|
|
1146
1408
|
properties: {
|
|
1147
|
-
direction: {
|
|
1148
|
-
|
|
1409
|
+
direction: {
|
|
1410
|
+
type: 'string',
|
|
1411
|
+
description: 'Scroll direction: "up" or "down".',
|
|
1412
|
+
},
|
|
1413
|
+
pixels: {
|
|
1414
|
+
type: 'number',
|
|
1415
|
+
description: 'Optional pixel amount (default: 800).',
|
|
1416
|
+
},
|
|
1149
1417
|
},
|
|
1150
1418
|
required: ['direction'],
|
|
1151
1419
|
},
|
|
@@ -1172,8 +1440,15 @@ export const BROWSER_TOOL_DEFINITIONS: ToolDefinition[] = [
|
|
|
1172
1440
|
parameters: {
|
|
1173
1441
|
type: 'object',
|
|
1174
1442
|
properties: {
|
|
1175
|
-
path: {
|
|
1176
|
-
|
|
1443
|
+
path: {
|
|
1444
|
+
type: 'string',
|
|
1445
|
+
description:
|
|
1446
|
+
'Optional relative output path under .browser-artifacts.',
|
|
1447
|
+
},
|
|
1448
|
+
fullPage: {
|
|
1449
|
+
type: 'boolean',
|
|
1450
|
+
description: 'Capture full page when true.',
|
|
1451
|
+
},
|
|
1177
1452
|
},
|
|
1178
1453
|
required: [],
|
|
1179
1454
|
},
|
|
@@ -1188,7 +1463,11 @@ export const BROWSER_TOOL_DEFINITIONS: ToolDefinition[] = [
|
|
|
1188
1463
|
parameters: {
|
|
1189
1464
|
type: 'object',
|
|
1190
1465
|
properties: {
|
|
1191
|
-
path: {
|
|
1466
|
+
path: {
|
|
1467
|
+
type: 'string',
|
|
1468
|
+
description:
|
|
1469
|
+
'Optional relative output path under .browser-artifacts.',
|
|
1470
|
+
},
|
|
1192
1471
|
},
|
|
1193
1472
|
required: [],
|
|
1194
1473
|
},
|
|
@@ -1203,7 +1482,10 @@ export const BROWSER_TOOL_DEFINITIONS: ToolDefinition[] = [
|
|
|
1203
1482
|
parameters: {
|
|
1204
1483
|
type: 'object',
|
|
1205
1484
|
properties: {
|
|
1206
|
-
question: {
|
|
1485
|
+
question: {
|
|
1486
|
+
type: 'string',
|
|
1487
|
+
description: 'Question to ask about the current page screenshot.',
|
|
1488
|
+
},
|
|
1207
1489
|
},
|
|
1208
1490
|
required: ['question'],
|
|
1209
1491
|
},
|
|
@@ -1225,11 +1507,16 @@ export const BROWSER_TOOL_DEFINITIONS: ToolDefinition[] = [
|
|
|
1225
1507
|
type: 'function',
|
|
1226
1508
|
function: {
|
|
1227
1509
|
name: 'browser_console',
|
|
1228
|
-
description:
|
|
1510
|
+
description:
|
|
1511
|
+
'Return console messages captured from the current page; optionally clear them.',
|
|
1229
1512
|
parameters: {
|
|
1230
1513
|
type: 'object',
|
|
1231
1514
|
properties: {
|
|
1232
|
-
clear: {
|
|
1515
|
+
clear: {
|
|
1516
|
+
type: 'boolean',
|
|
1517
|
+
description:
|
|
1518
|
+
'When true, clear stored console messages before returning.',
|
|
1519
|
+
},
|
|
1233
1520
|
},
|
|
1234
1521
|
required: [],
|
|
1235
1522
|
},
|
|
@@ -1244,8 +1531,15 @@ export const BROWSER_TOOL_DEFINITIONS: ToolDefinition[] = [
|
|
|
1244
1531
|
parameters: {
|
|
1245
1532
|
type: 'object',
|
|
1246
1533
|
properties: {
|
|
1247
|
-
filter: {
|
|
1248
|
-
|
|
1534
|
+
filter: {
|
|
1535
|
+
type: 'string',
|
|
1536
|
+
description: 'Optional URL substring filter.',
|
|
1537
|
+
},
|
|
1538
|
+
clear: {
|
|
1539
|
+
type: 'boolean',
|
|
1540
|
+
description:
|
|
1541
|
+
'When true, clear recorded network request history first.',
|
|
1542
|
+
},
|
|
1249
1543
|
},
|
|
1250
1544
|
required: [],
|
|
1251
1545
|
},
|
|
@@ -1255,7 +1549,8 @@ export const BROWSER_TOOL_DEFINITIONS: ToolDefinition[] = [
|
|
|
1255
1549
|
type: 'function',
|
|
1256
1550
|
function: {
|
|
1257
1551
|
name: 'browser_close',
|
|
1258
|
-
description:
|
|
1552
|
+
description:
|
|
1553
|
+
'Close the current browser session and release associated resources.',
|
|
1259
1554
|
parameters: {
|
|
1260
1555
|
type: 'object',
|
|
1261
1556
|
properties: {},
|