@browserbridge/bbx 1.12.5 → 1.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +4 -0
- package/README.md +2 -0
- package/SECURITY.md +10 -0
- package/package.json +6 -6
- package/packages/agent-client/src/cli-output.js +8 -2
- package/packages/agent-client/src/cli.js +132 -3
- package/packages/agent-client/src/command-registry.js +6 -0
- package/packages/agent-client/src/file-transfer.js +88 -0
- package/packages/agent-client/src/record-session.js +54 -0
- package/packages/agent-client/src/replay.js +124 -0
- package/packages/mcp-server/src/handlers-capture.js +9 -0
- package/packages/mcp-server/src/handlers-page.js +6 -0
- package/packages/mcp-server/src/handlers-utils.js +96 -0
- package/packages/mcp-server/src/server.js +22 -2
- package/packages/native-host/src/artifact-store.js +42 -4
- package/packages/native-host/src/auth-token.js +12 -0
- package/packages/native-host/src/daemon.js +142 -6
- package/packages/native-host/src/native-host.js +10 -0
- package/packages/native-host/src/recording-store.js +136 -0
- package/packages/protocol/src/downloads.js +35 -0
- package/packages/protocol/src/emulation.js +155 -0
- package/packages/protocol/src/files.js +94 -0
- package/packages/protocol/src/index.js +1 -0
- package/packages/protocol/src/protocol.js +44 -0
- package/packages/protocol/src/recording.js +70 -0
- package/packages/protocol/src/registry.js +224 -0
- package/packages/protocol/src/replay.js +144 -0
- package/packages/protocol/src/summary.js +49 -0
- package/packages/protocol/src/types.ts +37 -1
- package/packages/protocol/src/workflows.ts +22 -0
- package/skills/browser-bridge/SKILL.md +10 -0
- package/skills/browser-bridge/references/capture-context.md +9 -0
- package/skills/browser-bridge/references/emulation.md +18 -0
- package/skills/browser-bridge/references/files.md +50 -0
- package/skills/browser-bridge/references/playbooks.md +48 -0
- package/skills/browser-bridge/references/recording.md +54 -0
package/PRIVACY.md
CHANGED
|
@@ -89,6 +89,10 @@ Browser Bridge keeps data only as long as needed for the current local session a
|
|
|
89
89
|
- Optional CDP network capture is bounded in memory, explicitly started/stopped, expires after a safety interval, and is cleared on relevant debugger, tab, or access teardown.
|
|
90
90
|
- HAR export does not start, stop, or clear capture. Inline size limits remove whole oldest entries; artifact delivery uses opaque owner-scoped handles with no browser-host path. Artifacts expire after five minutes, are removed on daemon/extension teardown, and use private daemon directory/file modes on POSIX systems.
|
|
91
91
|
- Recovery telemetry is held in fixed five-second buckets over a five-minute process-local window. Extension-side summaries reset with the extension service worker, daemon summaries reset with the daemon, and neither is durable history.
|
|
92
|
+
- Opt-in activity recording captures event-time labels/selectors, route changes, and timing correlations. Input values are redacted. Optional network response bodies and messages receive best-effort text redaction; binary content and renderer pixels are not redacted. Child-tab capture is limited to opener descendants inside the enabled window.
|
|
93
|
+
- Opt-in durable recording writes private, bounded journals on the daemon machine. A secret journal capability permits retrieval after client, extension, or daemon restart. Journals expire 24 hours after the last write and are pruned during journal operations, or can be explicitly deleted. Revocation stops capture but preserves already-written evidence. Files exported by the client persist until you delete them.
|
|
94
|
+
- The `downloads` permission supports browser-native downloads, progress, pause/resume, and cancellation for downloads BBX starts. Downloaded files and browser download history follow Chrome's retention rules. Releasing a BBX handle does not delete either. BBX does not expose general download-history search.
|
|
95
|
+
- Explicit trusted image paste overwrites the system clipboard with the supplied PNG and dispatches native paste into a focused editor. It does not read existing clipboard content.
|
|
92
96
|
- Dialog message/default prompt text is returned to the requesting caller when inspected but is excluded from persisted action-log summaries and size diagnostics.
|
|
93
97
|
- Exact sensitive values traverse the selected client in memory but are excluded from action summaries, diagnostic size counters, daemon logs, and automatic artifact storage.
|
|
94
98
|
- Patch rollback records and remembered element references live in the current document. Browser Bridge attempts patch rollback when access is disabled or switched, while page replacement/navigation removes document-local state.
|
package/README.md
CHANGED
|
@@ -127,6 +127,8 @@ MCP mode is self-contained: the dual-era stdio server supports legacy MCP client
|
|
|
127
127
|
- Native pointer/text input, dialog handling, all-resource network capture, accessibility trees, screenshots, and raw CDP reads use the existing debugger permission only when requested
|
|
128
128
|
- HAR export is a read of an already armed CDP capture: start capture, reproduce, export, then stop. Export never starts, stops, or clears capture, and omits headers, cookies, bodies, authorization values, and unsupported size/timing detail
|
|
129
129
|
- HAR and screenshot artifacts are opaque, owner-scoped daemon handles with a five-minute lifetime and no browser-host path; CLI capture commands download, verify, delete, and write them on the CLI host
|
|
130
|
+
- [File workflows](https://github.com/koltyakov/browser-bridge/blob/main/skills/browser-bridge/references/files.md) include staged uploads, trusted PNG paste, and browser-native large downloads with progress, pause/resume, and cancellation
|
|
131
|
+
- [Opt-in recording](https://github.com/koltyakov/browser-bridge/blob/main/skills/browser-bridge/references/recording.md) supports child-tab watch, timing correlations, durable crash-recovery journals, continuous renderer playback, and local MP4 export
|
|
130
132
|
- Element references are document-local and strict by default; stale input recovery is opt-in, same-document, requires one strong unique semantic match, and fails safely when its bounded scan cannot prove uniqueness
|
|
131
133
|
- Semantic DOM baselines provide explicit create/compare/describe/release verification without modifying the page; compact snapshots stay memory-only, expire after five minutes, and are invalidated by navigation or access teardown
|
|
132
134
|
- Patches keep per-document rollback history and Browser Bridge attempts best-effort rollback when window access is disabled or switched; committing the patch baseline keeps current changes but discards their rollback history
|
package/SECURITY.md
CHANGED
|
@@ -61,6 +61,16 @@ Out of scope:
|
|
|
61
61
|
preventing SSH port forwarding from turning remote access into local control.
|
|
62
62
|
- Daemon connections, registration time, malformed input, pending requests, and
|
|
63
63
|
per-client pending requests are bounded to limit resource exhaustion.
|
|
64
|
+
- File staging and artifact access use daemon-issued connection identities.
|
|
65
|
+
Copying a client ID or agent session label does not grant access to another
|
|
66
|
+
connection's file data. Artifacts are deleted on disconnect. Staged uploads
|
|
67
|
+
become inaccessible on disconnect and expire within five minutes.
|
|
68
|
+
- Artifact ingress checks the requesting connection, routed extension, request
|
|
69
|
+
ID, artifact kind, chunk order, declared size, and SHA-256 checksum. File writes
|
|
70
|
+
account for partial writes and abort on failure before acknowledging progress.
|
|
71
|
+
- TCP authentication does not encrypt traffic. Use SSH or another encrypted
|
|
72
|
+
tunnel for remote connections. The default proxy listener is loopback-only;
|
|
73
|
+
exposing plaintext TCP requires an explicit unsafe option.
|
|
64
74
|
- Proxy mode (LAN exposure) is opt-in via `bbx proxy enable` and always provisions a
|
|
65
75
|
token; an invalid `bindHost` in the config rejects the config rather than widening
|
|
66
76
|
the bind address.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@browserbridge/bbx",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.14.1",
|
|
4
4
|
"private": false,
|
|
5
5
|
"keywords": [
|
|
6
6
|
"agent-tools",
|
|
@@ -78,18 +78,18 @@
|
|
|
78
78
|
"@modelcontextprotocol/server": "^2.3.1",
|
|
79
79
|
"@mozilla/readability": "^0.6.0",
|
|
80
80
|
"linkedom": "^0.18.13",
|
|
81
|
-
"zod": "^4.6.
|
|
81
|
+
"zod": "^4.6.7"
|
|
82
82
|
},
|
|
83
83
|
"devDependencies": {
|
|
84
84
|
"@modelcontextprotocol/client": "^2.3.1",
|
|
85
|
-
"@types/chrome": "^0.3.
|
|
86
|
-
"@types/node": "^26.6.
|
|
85
|
+
"@types/chrome": "^0.3.5",
|
|
86
|
+
"@types/node": "^26.6.5",
|
|
87
87
|
"ajv": "^8.20.0",
|
|
88
88
|
"ajv-formats": "^3.0.1",
|
|
89
89
|
"c8": "^12.0.0",
|
|
90
90
|
"har-schema": "^2.0.0",
|
|
91
|
-
"oxfmt": "^0.
|
|
92
|
-
"oxlint": "^1.
|
|
91
|
+
"oxfmt": "^0.74.0",
|
|
92
|
+
"oxlint": "^1.89.0",
|
|
93
93
|
"tsx": "^4.23.15",
|
|
94
94
|
"typescript": "^7.0.2"
|
|
95
95
|
},
|
|
@@ -52,7 +52,7 @@ export function printCallResponse(response, method) {
|
|
|
52
52
|
}
|
|
53
53
|
|
|
54
54
|
/**
|
|
55
|
-
* @returns {void}
|
|
55
|
+
* @returns {Promise<void>}
|
|
56
56
|
*/
|
|
57
57
|
export function printUsage() {
|
|
58
58
|
const blocks = ['Usage: bbx [--remote <name>] <command> [args]'];
|
|
@@ -60,5 +60,11 @@ export function printUsage() {
|
|
|
60
60
|
blocks.push('', `${section.title}:`);
|
|
61
61
|
blocks.push(...section.lines.map((line) => ` ${line}`));
|
|
62
62
|
}
|
|
63
|
-
|
|
63
|
+
// Wait for piped output to flush before the CLI exits.
|
|
64
|
+
return new Promise((resolve, reject) => {
|
|
65
|
+
process.stdout.write(`${blocks.join('\n')}\n`, (error) => {
|
|
66
|
+
if (error) reject(error);
|
|
67
|
+
else resolve();
|
|
68
|
+
});
|
|
69
|
+
});
|
|
64
70
|
}
|
|
@@ -47,6 +47,9 @@ import { formatMcpConfig, isMcpClientName, MCP_CLIENT_NAMES } from './mcp-config
|
|
|
47
47
|
import { getDoctorReport, requestBridge, resolveRef } from './runtime.js';
|
|
48
48
|
import { createBridgeClientForDestination } from './remotes.js';
|
|
49
49
|
import { atomicWriteFile } from './atomic-write.js';
|
|
50
|
+
import { uploadFiles } from './file-transfer.js';
|
|
51
|
+
import { recordSession } from './record-session.js';
|
|
52
|
+
import { exportReplay } from './replay.js';
|
|
50
53
|
|
|
51
54
|
/** @typedef {import('./types.js').BridgeMethod} BridgeMethod */
|
|
52
55
|
/** @typedef {import('./types.js').ScreenshotResult} ScreenshotResult */
|
|
@@ -98,7 +101,7 @@ const remoteDestinationId =
|
|
|
98
101
|
: (remoteFlag.remoteId ?? process.env[REMOTE_ENV] ?? null);
|
|
99
102
|
|
|
100
103
|
if (!command || ['help', '--help', '-h'].includes(command)) {
|
|
101
|
-
printUsage();
|
|
104
|
+
await printUsage();
|
|
102
105
|
process.exit(0);
|
|
103
106
|
}
|
|
104
107
|
|
|
@@ -403,6 +406,130 @@ async function main() {
|
|
|
403
406
|
return;
|
|
404
407
|
}
|
|
405
408
|
|
|
409
|
+
if (command === 'replay') {
|
|
410
|
+
if (rest.length !== 2)
|
|
411
|
+
throw new Error(
|
|
412
|
+
'Usage: bbx replay <journalId|recording-file> <output.html|output.json|output.mp4>'
|
|
413
|
+
);
|
|
414
|
+
const result = await exportReplay(
|
|
415
|
+
(method, params) => requestBridge(client, method, params, { source: REQUEST_SOURCE }),
|
|
416
|
+
rest[0],
|
|
417
|
+
rest[1]
|
|
418
|
+
);
|
|
419
|
+
printJson({ ok: true, ...result });
|
|
420
|
+
return;
|
|
421
|
+
}
|
|
422
|
+
if (command === 'upload' || command === 'paste-image') {
|
|
423
|
+
const action = command === 'paste-image' ? 'input.paste_image' : 'input.set_files';
|
|
424
|
+
const args = [...rest];
|
|
425
|
+
const flags = [];
|
|
426
|
+
for (const flag of ['--tab', '--extension', '--browser', '--profile']) {
|
|
427
|
+
const index = args.indexOf(flag);
|
|
428
|
+
if (index < 0) continue;
|
|
429
|
+
flags.push(...args.splice(index, 2));
|
|
430
|
+
}
|
|
431
|
+
const [selector, ...files] = args;
|
|
432
|
+
if (!selector || !files.length)
|
|
433
|
+
throw new Error(
|
|
434
|
+
`Usage: bbx ${command} --tab <id> [--extension <id>] <ref|selector> <local-file>...`
|
|
435
|
+
);
|
|
436
|
+
const {
|
|
437
|
+
tabId,
|
|
438
|
+
method: _method,
|
|
439
|
+
params: _params,
|
|
440
|
+
...route
|
|
441
|
+
} = await parseCallCommand([...flags, 'input.set_files', '{}']);
|
|
442
|
+
if (tabId === null)
|
|
443
|
+
throw new Error('upload requires --tab so every chunk targets the same tab.');
|
|
444
|
+
const response = await uploadFiles(
|
|
445
|
+
(method, params) =>
|
|
446
|
+
requestBridge(client, method, params, { ...route, tabId, source: REQUEST_SOURCE }),
|
|
447
|
+
isElementRef(selector) ? { elementRef: selector } : { selector },
|
|
448
|
+
files,
|
|
449
|
+
action
|
|
450
|
+
);
|
|
451
|
+
await printSummary(response, action);
|
|
452
|
+
return;
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
if (command === 'download') {
|
|
456
|
+
const args = [...rest];
|
|
457
|
+
const flags = [];
|
|
458
|
+
for (const flag of ['--tab', '--extension', '--browser', '--profile']) {
|
|
459
|
+
const index = args.indexOf(flag);
|
|
460
|
+
if (index >= 0) flags.push(...args.splice(index, 2));
|
|
461
|
+
}
|
|
462
|
+
const [url, output, ...extra] = args;
|
|
463
|
+
if (!url || !output || extra.length)
|
|
464
|
+
throw new Error('Usage: bbx download --tab <id> [--extension <id>] <url> <outPath>');
|
|
465
|
+
const {
|
|
466
|
+
tabId,
|
|
467
|
+
method: _method,
|
|
468
|
+
params: _params,
|
|
469
|
+
...route
|
|
470
|
+
} = await parseCallCommand([...flags, 'download.resource', '{}']);
|
|
471
|
+
if (tabId === null) throw new Error('download requires --tab.');
|
|
472
|
+
const response = await requestBridge(
|
|
473
|
+
client,
|
|
474
|
+
'download.resource',
|
|
475
|
+
{ url },
|
|
476
|
+
{ ...route, tabId, source: REQUEST_SOURCE }
|
|
477
|
+
);
|
|
478
|
+
if (!response.ok) {
|
|
479
|
+
await printSummary(response);
|
|
480
|
+
return;
|
|
481
|
+
}
|
|
482
|
+
const result = /** @type {{ artifact: ArtifactDescriptor }} */ (response.result);
|
|
483
|
+
const bytes = await downloadArtifact(client, result.artifact, 'resource');
|
|
484
|
+
await atomicWriteFile(output, bytes);
|
|
485
|
+
printJson({ ok: true, path: output, byteLength: bytes.length });
|
|
486
|
+
return;
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
if (command === 'record') {
|
|
490
|
+
const args = [...rest];
|
|
491
|
+
const flags = [];
|
|
492
|
+
for (const flag of ['--tab', '--extension', '--browser', '--profile']) {
|
|
493
|
+
const index = args.indexOf(flag);
|
|
494
|
+
if (index >= 0) flags.push(...args.splice(index, 2));
|
|
495
|
+
}
|
|
496
|
+
const [config, output, ...extra] = args;
|
|
497
|
+
if (!config || !output || extra.length)
|
|
498
|
+
throw new Error(
|
|
499
|
+
'Usage: bbx record --tab <id> [--extension <id>] <options-json> <out.json|out.html>'
|
|
500
|
+
);
|
|
501
|
+
const {
|
|
502
|
+
tabId,
|
|
503
|
+
params,
|
|
504
|
+
method: _method,
|
|
505
|
+
...route
|
|
506
|
+
} = await parseCallCommand([...flags, 'session.start', config]);
|
|
507
|
+
if (tabId === null) throw new Error('record requires --tab.');
|
|
508
|
+
if (!/\.(?:json|html|mp4)$/iu.test(output))
|
|
509
|
+
throw new Error('Recording output must be .json, .html, or .mp4.');
|
|
510
|
+
if (output.toLowerCase().endsWith('.mp4')) {
|
|
511
|
+
params.durable = true;
|
|
512
|
+
params.replay = true;
|
|
513
|
+
}
|
|
514
|
+
const result = await recordSession(
|
|
515
|
+
(method, params) =>
|
|
516
|
+
requestBridge(client, method, params, { ...route, tabId, source: REQUEST_SOURCE }),
|
|
517
|
+
params,
|
|
518
|
+
output.toLowerCase().endsWith('.html') ? 'html' : 'json',
|
|
519
|
+
async (artifact) =>
|
|
520
|
+
atomicWriteFile(output, await downloadArtifact(client, artifact, 'resource')),
|
|
521
|
+
(started) => printJson({ ok: true, recording: started }),
|
|
522
|
+
(journalId) =>
|
|
523
|
+
exportReplay(
|
|
524
|
+
(method, params) => requestBridge(client, method, params, { source: REQUEST_SOURCE }),
|
|
525
|
+
journalId,
|
|
526
|
+
output
|
|
527
|
+
)
|
|
528
|
+
);
|
|
529
|
+
printJson({ ok: true, path: output, ...result });
|
|
530
|
+
return;
|
|
531
|
+
}
|
|
532
|
+
|
|
406
533
|
if (command === 'batch') {
|
|
407
534
|
const { preset, rest: batchArgs } = extractPresetFlag(rest);
|
|
408
535
|
const results = await runBatchCalls(client, batchArgs[0], REQUEST_SOURCE, { preset });
|
|
@@ -528,6 +655,8 @@ async function main() {
|
|
|
528
655
|
evidence: {
|
|
529
656
|
savedTo: filePath,
|
|
530
657
|
rect: screenshotResult.rect,
|
|
658
|
+
dimensions: screenshotResult.dimensions,
|
|
659
|
+
...(screenshotResult.context ? { context: screenshotResult.context } : {}),
|
|
531
660
|
format: screenshotResult.format,
|
|
532
661
|
complete: screenshotResult.complete,
|
|
533
662
|
clipped: screenshotResult.clipped,
|
|
@@ -665,7 +794,7 @@ async function main() {
|
|
|
665
794
|
}
|
|
666
795
|
|
|
667
796
|
process.stderr.write(`Unknown command: ${command}\n`);
|
|
668
|
-
printUsage();
|
|
797
|
+
await printUsage();
|
|
669
798
|
process.exitCode = 1;
|
|
670
799
|
} catch (error) {
|
|
671
800
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -712,7 +841,7 @@ function decodeScreenshotDataUrl(value) {
|
|
|
712
841
|
/**
|
|
713
842
|
* @param {import('./client.js').BridgeClient} client
|
|
714
843
|
* @param {ArtifactDescriptor} artifact
|
|
715
|
-
* @param {'
|
|
844
|
+
* @param {import('../../protocol/src/types.js').ArtifactKind} [expectedKind]
|
|
716
845
|
*/
|
|
717
846
|
async function downloadArtifact(client, artifact, expectedKind) {
|
|
718
847
|
const chunks = [];
|
|
@@ -300,6 +300,12 @@ export const CLI_HELP_SECTIONS = Object.freeze([
|
|
|
300
300
|
'--preset fills budget gaps (maxNodes, maxDepth, textBudget, limit) from the shared quick|normal|deep presets; explicit params always win.',
|
|
301
301
|
'Every bridge command accepts --remote <name> (or BBX_REMOTE env) to target a remote destination from `bbx remote list`.',
|
|
302
302
|
'Advanced bridge params stay available through `bbx call`, even when shortcuts expose only the common case.',
|
|
303
|
+
'bbx upload --tab <id> [--extension <id>] <ref|selector> <local-file>... Stream local files and assign a file input',
|
|
304
|
+
'bbx paste-image --tab <id> [--extension <id>] <ref|selector> <image.png> Write clipboard and dispatch trusted paste into a focused editor',
|
|
305
|
+
'bbx download --tab <id> [--extension <id>] <url> <outPath> Save a browser-authenticated resource locally',
|
|
306
|
+
'bbx record --tab <id> [--extension <id>] <options-json> <out.json|out.html|out.mp4> Record and export on one connection',
|
|
307
|
+
'bbx replay <journalId|recording-file> <out.html|out.json|out.mp4> Recover or render a replay; MP4 requires ffmpeg',
|
|
308
|
+
'bbx call download.start \'{"url":"https://example.com/export"}\' Start a browser-native large download; use download.status/pause/resume/cancel with its handle',
|
|
303
309
|
'For open-ended investigation, start with `bbx batch` on `page.get_state`, `dom.query`, and `page.get_text` before any screenshot or CDP call.',
|
|
304
310
|
'',
|
|
305
311
|
'Interaction methods need a target wrapper (not a bare ref):',
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import { open } from 'node:fs/promises';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { FILE_CHUNK_BYTES, FILE_MAX_BYTES, FILE_TOTAL_BYTES } from '../../protocol/src/files.js';
|
|
6
|
+
|
|
7
|
+
/** Upload agent-local files through bounded chunks, then assign them once. No byte payload enters output.
|
|
8
|
+
* @param {(method: import('../../protocol/src/types.js').BridgeMethod, params: Record<string, unknown>) => Promise<import('../../protocol/src/types.js').BridgeResponse>} call
|
|
9
|
+
* @param {Record<string, unknown>} target
|
|
10
|
+
* @param {string[]} paths
|
|
11
|
+
* @param {'input.set_files' | 'input.paste_image'} [action]
|
|
12
|
+
*/
|
|
13
|
+
export async function uploadFiles(call, target, paths, action = 'input.set_files') {
|
|
14
|
+
if (action === 'input.paste_image' && paths.length !== 1)
|
|
15
|
+
throw new Error('Paste requires exactly one PNG.');
|
|
16
|
+
if (!paths.length || paths.length > 10) throw new Error('Provide 1..10 local files.');
|
|
17
|
+
/** @type {Array<{ file: import('node:fs/promises').FileHandle, name: string, size: number }>} */
|
|
18
|
+
const opened = [];
|
|
19
|
+
/** @type {string[]} */
|
|
20
|
+
const handles = [];
|
|
21
|
+
try {
|
|
22
|
+
for (const filename of paths) {
|
|
23
|
+
const file = await open(filename, 'r');
|
|
24
|
+
opened.push({ file, name: path.basename(filename), size: 0 });
|
|
25
|
+
const stat = await file.stat();
|
|
26
|
+
if (!stat.isFile() || stat.size > FILE_MAX_BYTES)
|
|
27
|
+
throw new Error('Each upload must be a regular file of at most 16 MiB.');
|
|
28
|
+
opened[opened.length - 1].size = stat.size;
|
|
29
|
+
}
|
|
30
|
+
if (opened.reduce((sum, item) => sum + item.size, 0) > FILE_TOTAL_BYTES)
|
|
31
|
+
throw new Error('Combined upload size exceeds 32 MiB.');
|
|
32
|
+
for (const { file, name, size } of opened) {
|
|
33
|
+
let offset = 0;
|
|
34
|
+
let transferId;
|
|
35
|
+
do {
|
|
36
|
+
const buffer = Buffer.alloc(Math.min(FILE_CHUNK_BYTES, size - offset));
|
|
37
|
+
let used = 0;
|
|
38
|
+
while (used < buffer.length) {
|
|
39
|
+
const { bytesRead } = await file.read(buffer, used, buffer.length - used, offset + used);
|
|
40
|
+
if (!bytesRead) throw new Error('Upload file changed while reading.');
|
|
41
|
+
used += bytesRead;
|
|
42
|
+
}
|
|
43
|
+
const response = await call('files.stage', {
|
|
44
|
+
...(transferId ? { transferId } : { name, mimeType: mimeForName(name), size }),
|
|
45
|
+
offset,
|
|
46
|
+
data: buffer.toString('base64'),
|
|
47
|
+
});
|
|
48
|
+
if (!response.ok) throw new Error(response.error.message);
|
|
49
|
+
const result = /** @type {{ transferId: string, nextOffset: number }} */ (response.result);
|
|
50
|
+
if (!transferId) {
|
|
51
|
+
transferId = result.transferId;
|
|
52
|
+
handles.push(transferId);
|
|
53
|
+
}
|
|
54
|
+
if (result.transferId !== transferId || result.nextOffset !== offset + used)
|
|
55
|
+
throw new Error('Invalid file staging acknowledgement.');
|
|
56
|
+
offset = result.nextOffset;
|
|
57
|
+
} while (offset < size);
|
|
58
|
+
}
|
|
59
|
+
return await call(
|
|
60
|
+
action,
|
|
61
|
+
action === 'input.paste_image'
|
|
62
|
+
? { target, transferId: handles[0] }
|
|
63
|
+
: { target, files: handles }
|
|
64
|
+
);
|
|
65
|
+
} finally {
|
|
66
|
+
await Promise.allSettled(opened.map(({ file }) => file.close()));
|
|
67
|
+
await Promise.allSettled(handles.map((transferId) => call('files.release', { transferId })));
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** @param {string} name */
|
|
72
|
+
function mimeForName(name) {
|
|
73
|
+
/** @type {Record<string, string>} */
|
|
74
|
+
const types = {
|
|
75
|
+
'.txt': 'text/plain',
|
|
76
|
+
'.csv': 'text/csv',
|
|
77
|
+
'.json': 'application/json',
|
|
78
|
+
'.pdf': 'application/pdf',
|
|
79
|
+
'.png': 'image/png',
|
|
80
|
+
'.jpg': 'image/jpeg',
|
|
81
|
+
'.jpeg': 'image/jpeg',
|
|
82
|
+
'.webp': 'image/webp',
|
|
83
|
+
'.svg': 'image/svg+xml',
|
|
84
|
+
'.html': 'text/html',
|
|
85
|
+
'.zip': 'application/zip',
|
|
86
|
+
};
|
|
87
|
+
return types[path.extname(name).toLowerCase()] ?? 'application/octet-stream';
|
|
88
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
/** Retain one connection for capture, export, and artifact retrieval.
|
|
4
|
+
* @param {(method: import('../../protocol/src/types.js').BridgeMethod, params: Record<string, unknown>) => Promise<import('../../protocol/src/types.js').BridgeResponse>} call
|
|
5
|
+
* @param {Record<string, unknown>} options
|
|
6
|
+
* @param {'json' | 'html'} format
|
|
7
|
+
* @param {(artifact: import('../../protocol/src/types.js').ArtifactDescriptor) => Promise<void>} save
|
|
8
|
+
* @param {(result: Record<string, unknown>) => void} [started]
|
|
9
|
+
* @param {(journalId: string) => Promise<unknown>} [saveJournal]
|
|
10
|
+
*/
|
|
11
|
+
export async function recordSession(call, options, format, save, started = () => {}, saveJournal) {
|
|
12
|
+
/** @param {import('../../protocol/src/types.js').BridgeMethod} method @param {Record<string, unknown>} params */
|
|
13
|
+
async function invoke(method, params) {
|
|
14
|
+
const response = await call(method, params);
|
|
15
|
+
if (!response.ok) throw new Error(response.error.message);
|
|
16
|
+
return /** @type {Record<string, unknown>} */ (response.result);
|
|
17
|
+
}
|
|
18
|
+
const initial = await invoke('session.start', options);
|
|
19
|
+
const sessionId = String(initial.sessionId);
|
|
20
|
+
started(initial);
|
|
21
|
+
try {
|
|
22
|
+
let cursor = Number(initial.nextCursor);
|
|
23
|
+
while (true) {
|
|
24
|
+
const next = await invoke('session.read', { sessionId, cursor, limit: 100, waitMs: 25000 });
|
|
25
|
+
if (next.journalError)
|
|
26
|
+
throw new Error(
|
|
27
|
+
`Recording journal failed: ${next.journalError}. Recover acknowledged evidence with journal ${initial.journalId}.`
|
|
28
|
+
);
|
|
29
|
+
cursor = Number(next.nextCursor);
|
|
30
|
+
if (!next.active) break;
|
|
31
|
+
}
|
|
32
|
+
if (typeof initial.journalId === 'string' && saveJournal) {
|
|
33
|
+
const stopped = await invoke('session.stop', { sessionId });
|
|
34
|
+
if (stopped.journalError)
|
|
35
|
+
throw new Error(
|
|
36
|
+
`Recording journal failed: ${stopped.journalError}. Recover journal ${initial.journalId}.`
|
|
37
|
+
);
|
|
38
|
+
await saveJournal(initial.journalId);
|
|
39
|
+
return {
|
|
40
|
+
sessionId,
|
|
41
|
+
journalId: initial.journalId,
|
|
42
|
+
gaps: stopped.gaps,
|
|
43
|
+
dropped: stopped.dropped,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
const exported = await invoke('session.export', { sessionId, format });
|
|
47
|
+
await save(
|
|
48
|
+
/** @type {import('../../protocol/src/types.js').ArtifactDescriptor} */ (exported.artifact)
|
|
49
|
+
);
|
|
50
|
+
return { sessionId, gaps: exported.gaps, dropped: exported.dropped };
|
|
51
|
+
} finally {
|
|
52
|
+
await call('session.release', { sessionId }).catch(() => {});
|
|
53
|
+
}
|
|
54
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
import fs from 'node:fs/promises';
|
|
3
|
+
import os from 'node:os';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { execFile } from 'node:child_process';
|
|
6
|
+
import { promisify } from 'node:util';
|
|
7
|
+
import { parseRecordingJournal, renderReplay } from '../../protocol/src/replay.js';
|
|
8
|
+
|
|
9
|
+
/** @param {import('../../protocol/src/replay.js').Replay} recording @param {string} output */
|
|
10
|
+
async function writeMp4(recording, output) {
|
|
11
|
+
if (!recording.frames.length) throw new Error('Recording has no frames to encode.');
|
|
12
|
+
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'bbx-replay-'));
|
|
13
|
+
try {
|
|
14
|
+
const frames = [...recording.frames].sort((a, b) => a.at - b.at);
|
|
15
|
+
const lines = ['ffconcat version 1.0'];
|
|
16
|
+
for (let index = 0; index < frames.length; index++) {
|
|
17
|
+
const frame = frames[index];
|
|
18
|
+
if (!Number.isFinite(frame.at) || !/^[A-Za-z0-9+/]+={0,2}$/u.test(frame.data))
|
|
19
|
+
throw new Error('Invalid recorded frame.');
|
|
20
|
+
const filename = `frame-${index}.jpg`;
|
|
21
|
+
await fs.writeFile(path.join(directory, filename), Buffer.from(frame.data, 'base64'), {
|
|
22
|
+
mode: 0o600,
|
|
23
|
+
});
|
|
24
|
+
lines.push(
|
|
25
|
+
`file '${filename}'`,
|
|
26
|
+
`duration ${Math.max(0.02, Math.min(1800, ((frames[index + 1]?.at ?? frame.at + 100) - frame.at) / 1000))}`
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
lines.push(`file 'frame-${frames.length - 1}.jpg'`);
|
|
30
|
+
await fs.writeFile(path.join(directory, 'frames.txt'), lines.join('\n'), { mode: 0o600 });
|
|
31
|
+
await promisify(execFile)(
|
|
32
|
+
'ffmpeg',
|
|
33
|
+
[
|
|
34
|
+
'-nostdin',
|
|
35
|
+
'-n',
|
|
36
|
+
'-loglevel',
|
|
37
|
+
'error',
|
|
38
|
+
'-f',
|
|
39
|
+
'concat',
|
|
40
|
+
'-safe',
|
|
41
|
+
'1',
|
|
42
|
+
'-i',
|
|
43
|
+
path.join(directory, 'frames.txt'),
|
|
44
|
+
'-vf',
|
|
45
|
+
'scale=1280:900:force_original_aspect_ratio=decrease,pad=1280:900:(ow-iw)/2:(oh-ih)/2',
|
|
46
|
+
'-c:v',
|
|
47
|
+
'libx264',
|
|
48
|
+
'-pix_fmt',
|
|
49
|
+
'yuv420p',
|
|
50
|
+
'-movflags',
|
|
51
|
+
'+faststart',
|
|
52
|
+
path.resolve(output),
|
|
53
|
+
],
|
|
54
|
+
{ timeout: 300000, maxBuffer: 1024 * 1024 }
|
|
55
|
+
);
|
|
56
|
+
} finally {
|
|
57
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Download an opaque journal capability or render a local recording without exposing frame bytes to the model.
|
|
62
|
+
* @param {(method: import('../../protocol/src/types.js').BridgeMethod, params: Record<string, unknown>) => Promise<import('../../protocol/src/types.js').BridgeResponse>} call
|
|
63
|
+
* @param {string} source @param {string} output */
|
|
64
|
+
export async function exportReplay(call, source, output) {
|
|
65
|
+
let text;
|
|
66
|
+
if (/^journal_[a-f0-9]{64}$/u.test(source)) {
|
|
67
|
+
const chunks = [];
|
|
68
|
+
let offset = 0;
|
|
69
|
+
for (;;) {
|
|
70
|
+
const response = await call('session.journal_read', {
|
|
71
|
+
journalId: source,
|
|
72
|
+
offset,
|
|
73
|
+
maxBytes: 196608,
|
|
74
|
+
});
|
|
75
|
+
if (!response.ok) throw new Error(response.error.message);
|
|
76
|
+
const chunk =
|
|
77
|
+
/** @type {{ offset: number, byteLength: number, totalBytes: number, nextOffset: number | null, data: string }} */ (
|
|
78
|
+
response.result
|
|
79
|
+
);
|
|
80
|
+
const bytes = Buffer.from(chunk.data, 'base64');
|
|
81
|
+
if (
|
|
82
|
+
chunk.offset !== offset ||
|
|
83
|
+
chunk.byteLength !== bytes.length ||
|
|
84
|
+
offset + bytes.length > 128 * 1024 * 1024 ||
|
|
85
|
+
(chunk.nextOffset !== null && chunk.nextOffset !== offset + bytes.length)
|
|
86
|
+
)
|
|
87
|
+
throw new Error('Invalid journal chunk acknowledgement.');
|
|
88
|
+
chunks.push(bytes);
|
|
89
|
+
offset += bytes.length;
|
|
90
|
+
if (chunk.nextOffset === null) break;
|
|
91
|
+
if (!bytes.length) throw new Error('Journal read made no progress.');
|
|
92
|
+
}
|
|
93
|
+
text = Buffer.concat(chunks).toString('utf8');
|
|
94
|
+
} else {
|
|
95
|
+
const stat = await fs.stat(source);
|
|
96
|
+
if (!stat.isFile() || stat.size > 128 * 1024 * 1024)
|
|
97
|
+
throw new Error('Recording must be a file of at most 128 MiB.');
|
|
98
|
+
text = await fs.readFile(source, 'utf8');
|
|
99
|
+
}
|
|
100
|
+
let recording;
|
|
101
|
+
try {
|
|
102
|
+
const parsed = /** @type {import('../../protocol/src/replay.js').Replay} */ (JSON.parse(text));
|
|
103
|
+
if (!Array.isArray(parsed.frames) || !Array.isArray(parsed.events))
|
|
104
|
+
throw new Error('Not a recording');
|
|
105
|
+
recording = parsed;
|
|
106
|
+
} catch {
|
|
107
|
+
recording = parseRecordingJournal(text);
|
|
108
|
+
}
|
|
109
|
+
const extension = path.extname(output).toLowerCase();
|
|
110
|
+
if (extension === '.mp4') await writeMp4(recording, output);
|
|
111
|
+
else if (extension === '.html' || extension === '.json')
|
|
112
|
+
await fs.writeFile(
|
|
113
|
+
output,
|
|
114
|
+
extension === '.html' ? renderReplay(recording) : JSON.stringify(recording),
|
|
115
|
+
{ mode: 0o600 }
|
|
116
|
+
);
|
|
117
|
+
else throw new Error('Replay output must be .html, .json, or .mp4. MP4 requires ffmpeg on PATH.');
|
|
118
|
+
return {
|
|
119
|
+
path: path.resolve(output),
|
|
120
|
+
frames: recording.frames.length,
|
|
121
|
+
gaps: recording.gaps,
|
|
122
|
+
dropped: recording.dropped,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
@@ -162,6 +162,9 @@ export function createScreenshotResult(response, method) {
|
|
|
162
162
|
mimeType: result.mimeType,
|
|
163
163
|
byteLength: result.byteLength,
|
|
164
164
|
dimensions: result.dimensions,
|
|
165
|
+
...(result.context && typeof result.context === 'object'
|
|
166
|
+
? { context: boundStructuredValue(result.context).value }
|
|
167
|
+
: {}),
|
|
165
168
|
rect: boundedRect(result.rect),
|
|
166
169
|
complete: result.complete === true,
|
|
167
170
|
clipped: result.clipped === true,
|
|
@@ -186,6 +189,12 @@ export function createScreenshotResult(response, method) {
|
|
|
186
189
|
mimeType: image.mimeType,
|
|
187
190
|
byteLength: image.byteLength,
|
|
188
191
|
...(rect ? { rect } : {}),
|
|
192
|
+
...(result.dimensions && typeof result.dimensions === 'object'
|
|
193
|
+
? { dimensions: boundStructuredValue(result.dimensions).value }
|
|
194
|
+
: {}),
|
|
195
|
+
...(result.context && typeof result.context === 'object'
|
|
196
|
+
? { context: boundStructuredValue(result.context).value }
|
|
197
|
+
: {}),
|
|
189
198
|
format: result.format ?? image.mimeType.replace('image/', ''),
|
|
190
199
|
complete: result.complete === true,
|
|
191
200
|
clipped: result.clipped === true,
|
|
@@ -547,6 +547,12 @@ export async function handleRawCallTool(args) {
|
|
|
547
547
|
if (method === 'artifact.read') {
|
|
548
548
|
return createArtifactReadResult(response);
|
|
549
549
|
}
|
|
550
|
+
if (method === 'session.journal_read' && response.ok) {
|
|
551
|
+
return createToolResult('Read exact journal bytes; page until nextOffset is null.', {
|
|
552
|
+
ok: true,
|
|
553
|
+
evidence: response.result,
|
|
554
|
+
});
|
|
555
|
+
}
|
|
550
556
|
return summarizeToolResponse(response, method, params);
|
|
551
557
|
},
|
|
552
558
|
{ destinationId: args.destinationId ?? null }
|