@vdoninja/mcp 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -661
- package/README.md +59 -272
- package/assets/browser-agent/audio-worklet.js +17 -0
- package/assets/browser-agent/client.js +124 -0
- package/assets/browser-agent/index.html +50 -0
- package/assets/cloudflare/worker-proxy.js +3 -0
- package/docs/.nojekyll +0 -0
- package/docs/404.html +28 -0
- package/docs/advanced-diagnostics.html +101 -0
- package/docs/assets/apple-touch-icon.png +0 -0
- package/docs/assets/favicon.svg +7 -0
- package/docs/assets/how-it-works.svg +62 -0
- package/docs/assets/icon-192.png +0 -0
- package/docs/assets/icon-512.png +0 -0
- package/docs/assets/icon-maskable-512.png +0 -0
- package/docs/assets/icons.svg +42 -0
- package/docs/assets/logo.svg +7 -0
- package/docs/assets/og-image.png +0 -0
- package/docs/assets/site.css +735 -0
- package/docs/assets/site.js +289 -0
- package/docs/browser-agent.html +122 -0
- package/docs/browser-worker.html +165 -0
- package/docs/client-config-examples.html +254 -0
- package/docs/direction.html +175 -0
- package/docs/favicon.ico +0 -0
- package/docs/guide.html +872 -0
- package/docs/http-relay.html +154 -0
- package/docs/index.html +834 -0
- package/docs/mcp-tool-contract.html +226 -0
- package/docs/quickstart-and-compat.html +222 -0
- package/docs/site.webmanifest +16 -0
- package/docs/sitemap.xml +63 -0
- package/docs/ssapp-integration.html +594 -0
- package/docs/vdo-ninja-p2p-data-notes.html +225 -0
- package/package.json +51 -9
- package/references/advanced-diagnostics.md +2 -0
- package/references/browser-agent.md +53 -0
- package/references/browser-worker.md +142 -0
- package/references/client-config-examples.md +2 -0
- package/references/direction.md +88 -0
- package/references/http-relay.md +107 -0
- package/references/mcp-tool-contract.md +11 -1
- package/references/quickstart-and-compat.md +4 -1
- package/references/ssapp-integration.md +371 -0
- package/references/vdo-ninja-p2p-data-notes.md +2 -0
- package/scripts/demo-file-workflow.js +3 -0
- package/scripts/demo-message-workflow.js +3 -0
- package/scripts/install-mcp.js +5 -2
- package/scripts/lib/bridge-session.js +143 -26
- package/scripts/lib/bridge-utils.js +3 -0
- package/scripts/lib/browser-agent.js +255 -0
- package/scripts/lib/browser-media.js +79 -0
- package/scripts/lib/browser-routes.js +119 -0
- package/scripts/lib/browser-rpc.js +214 -0
- package/scripts/lib/browser-setup.js +133 -0
- package/scripts/lib/browser-worker.js +216 -0
- package/scripts/lib/fake-network-sdk.js +3 -0
- package/scripts/lib/http-relay-transport.js +294 -0
- package/scripts/lib/load-vdo-sdk.js +3 -0
- package/scripts/lib/mcp-stdio-parser.js +3 -0
- package/scripts/lib/ssapp-worker.js +198 -0
- package/scripts/lib/tool-profiles.js +3 -0
- package/scripts/vdo-data-bridge.js +3 -0
- package/scripts/vdo-mcp-browser-agent.js +132 -0
- package/scripts/vdo-mcp-browser.js +218 -0
- package/scripts/vdo-mcp-relay.js +207 -0
- package/scripts/vdo-mcp-server.js +50 -7
- package/scripts/vdo-mcp-ssapp.js +208 -0
- package/scripts/vdo-mcp-streamable-http.js +23 -4
- package/server.json +22 -4
- package/LICENSE-SDK-EXCEPTION +0 -54
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
// Copyright (c) 2026 Steve Seguin
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
'use strict';
|
|
4
|
+
|
|
5
|
+
const http = require('node:http');
|
|
6
|
+
const crypto = require('node:crypto');
|
|
7
|
+
const path = require('node:path');
|
|
8
|
+
const { spawn } = require('node:child_process');
|
|
9
|
+
const { McpStdioParser } = require('./mcp-stdio-parser');
|
|
10
|
+
|
|
11
|
+
const RPC_OPTIONS = { magic: 'ninja_ssapp_v1', maxResult: 10 * 1024 * 1024, parallel: true, backpressure: true, errorCodes: true,
|
|
12
|
+
timeoutMessage: 'Inspect ninja_ssapp_result and the current app before repeating an action.' };
|
|
13
|
+
const error = (code, message) => Object.assign(new Error(message || code), { code });
|
|
14
|
+
|
|
15
|
+
function controlOrigin(value = 'http://127.0.0.1:17777') {
|
|
16
|
+
const url = new URL(value);
|
|
17
|
+
if (url.protocol !== 'http:' || url.hostname !== '127.0.0.1' || url.username || url.password || url.pathname !== '/' || url.search || url.hash) throw error('invalid_control_url', 'SSApp must use an http://127.0.0.1:PORT origin');
|
|
18
|
+
return url.origin;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function appInstance(origin) {
|
|
22
|
+
return new Promise((resolve, reject) => {
|
|
23
|
+
const req = http.get(`${origin}/api/v1/instance`, { timeout: 2000 }, (res) => {
|
|
24
|
+
let data = '';
|
|
25
|
+
res.setEncoding('utf8');
|
|
26
|
+
res.on('data', (part) => { data += part; if (data.length > 16000) res.destroy(error('invalid_instance')); });
|
|
27
|
+
res.on('error', reject);
|
|
28
|
+
res.on('end', () => {
|
|
29
|
+
try {
|
|
30
|
+
const value = JSON.parse(data);
|
|
31
|
+
if (res.statusCode !== 200 || !/^[a-f0-9-]{36}$/i.test(value.instanceId)) throw error('instance_binding_required', 'Start SSApp with API 1.4.1 or newer and enable Local AI / Automation');
|
|
32
|
+
resolve(value);
|
|
33
|
+
} catch (failure) { reject(failure); }
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
req.on('timeout', () => req.destroy(error('SSAPP_UNREACHABLE')));
|
|
37
|
+
req.on('error', reject);
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function validateCommand(settings) {
|
|
42
|
+
if (!settings || !path.isAbsolute(settings.command || '') || !Array.isArray(settings.args) || settings.args.some((arg) => typeof arg !== 'string')) throw error('invalid_mcp_command', 'Set an absolute host MCP executable and its argument array');
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
class LocalMcp {
|
|
46
|
+
constructor(settings, instanceId) {
|
|
47
|
+
validateCommand(settings);
|
|
48
|
+
this.pending = new Map(); this.nextId = 0; this.failed = false;
|
|
49
|
+
this.child = spawn(settings.command, settings.args, { windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'], cwd: settings.cwd,
|
|
50
|
+
env: { ...process.env, ...(settings.env || {}), UV_THREADPOOL_SIZE: '2', SSAPP_CONTROL_URL: controlOrigin(settings.controlUrl),
|
|
51
|
+
SSAPP_EXPECTED_INSTANCE_ID: instanceId || 'offline', SSAPP_MCP_REQUEST_TIMEOUT_MS: '35000' } });
|
|
52
|
+
this.child.stderr.on('data', () => {});
|
|
53
|
+
const parser = new McpStdioParser({ maxMessageBytes: RPC_OPTIONS.maxResult });
|
|
54
|
+
parser.on('message', ({ message }) => {
|
|
55
|
+
if (message.method && message.id === undefined) { if (this.onNotification) this.onNotification(message); return; }
|
|
56
|
+
const pending = this.pending.get(message.id);
|
|
57
|
+
if (!pending) return;
|
|
58
|
+
clearTimeout(pending.timer); this.pending.delete(message.id);
|
|
59
|
+
if (message.error) pending.reject(error('local_mcp_error', message.error.message));
|
|
60
|
+
else pending.resolve(message.result);
|
|
61
|
+
});
|
|
62
|
+
parser.on('error', () => this.fail(error('invalid_mcp_response')));
|
|
63
|
+
this.child.stdout.on('data', (chunk) => { parser.push(chunk); if (parser.buffer.length > RPC_OPTIONS.maxResult) this.fail(error('mcp_response_too_large')); });
|
|
64
|
+
this.child.on('error', (failure) => this.fail(failure));
|
|
65
|
+
this.child.on('exit', () => this.fail(error('local_mcp_closed')));
|
|
66
|
+
this.child.stdin.on('error', (failure) => this.fail(failure));
|
|
67
|
+
}
|
|
68
|
+
fail(failure) {
|
|
69
|
+
this.failed = true;
|
|
70
|
+
for (const p of this.pending.values()) { clearTimeout(p.timer); p.reject(failure); }
|
|
71
|
+
this.pending.clear();
|
|
72
|
+
}
|
|
73
|
+
request(method, params = {}) {
|
|
74
|
+
if (this.failed) return Promise.reject(error('local_mcp_closed'));
|
|
75
|
+
const id = ++this.nextId;
|
|
76
|
+
return new Promise((resolve, reject) => {
|
|
77
|
+
const timer = setTimeout(() => { this.fail(error('local_mcp_timeout', 'Local MCP outcome unknown. Inspect SSApp before resuming control.')); this.child.kill(); }, 38000);
|
|
78
|
+
this.pending.set(id, { resolve, reject, timer });
|
|
79
|
+
this.child.stdin.write(JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n');
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
async start() {
|
|
83
|
+
this.info = await this.request('initialize', { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: { name: 'ninja-ssapp-host', version: '0.4.2' } });
|
|
84
|
+
if (this.info.serverInfo?.name !== 'social-stream-ninja') throw error('wrong_mcp_backend', 'Configure SSApp\'s own MCP connector');
|
|
85
|
+
const version = String(this.info.serverInfo.version).split('.').map(Number);
|
|
86
|
+
if (!(version[0] > 1 || version[0] === 1 && (version[1] > 3 || version[1] === 3 && version[2] >= 1))) throw error('instance_binding_required', 'Use SSApp MCP adapter 1.3.1 or newer for remote access');
|
|
87
|
+
this.child.stdin.write(JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' }) + '\n');
|
|
88
|
+
this.catalog = await this.request('tools/list');
|
|
89
|
+
if (!Array.isArray(this.catalog.tools) || this.catalog.tools.some((tool) => !tool.name.startsWith('ssapp_'))) throw error('invalid_ssapp_catalog');
|
|
90
|
+
}
|
|
91
|
+
async close() {
|
|
92
|
+
this.fail(error('local_mcp_closed'));
|
|
93
|
+
if (!this.child.stdin.destroyed) this.child.stdin.end();
|
|
94
|
+
if (this.child.exitCode !== null) return;
|
|
95
|
+
await new Promise((resolve) => {
|
|
96
|
+
const timer = setTimeout(() => { this.child.kill(); resolve(); }, 1500);
|
|
97
|
+
this.child.once('exit', () => { clearTimeout(timer); resolve(); });
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
class SsappController {
|
|
103
|
+
constructor(config) {
|
|
104
|
+
this.config = config;
|
|
105
|
+
this.origin = controlOrigin(config.ssapp.controlUrl);
|
|
106
|
+
this.instance = null; this.lastInstanceId = null; this.local = null; this.checking = null; this.lease = null;
|
|
107
|
+
this.generation = 0; this.mutations = Promise.resolve(); this.queued = 0; this.closed = false; this.uncertain = false;
|
|
108
|
+
this.discovered = new Map();
|
|
109
|
+
}
|
|
110
|
+
revoke() { this.lease = null; this.generation++; }
|
|
111
|
+
async ensure() {
|
|
112
|
+
if (this.checking) return this.checking;
|
|
113
|
+
this.checking = (async () => {
|
|
114
|
+
let instance = null;
|
|
115
|
+
try { instance = await appInstance(this.origin); } catch (_) { /* Offline discovery still exposes the local tool catalog. */ }
|
|
116
|
+
if (this.closed) throw error('worker_closed');
|
|
117
|
+
if (this.instance?.instanceId !== instance?.instanceId) { this.revoke(); this.discovered.clear(); }
|
|
118
|
+
// A temporary API outage does not resolve a previously dispatched action.
|
|
119
|
+
if (instance && instance.instanceId !== this.lastInstanceId) { this.uncertain = false; this.lastInstanceId = instance.instanceId; }
|
|
120
|
+
if (!this.local || this.local.failed || this.instance?.instanceId !== instance?.instanceId) {
|
|
121
|
+
if (this.local) await this.local.close();
|
|
122
|
+
this.instance = instance;
|
|
123
|
+
this.local = new LocalMcp(this.config.ssapp, instance?.instanceId);
|
|
124
|
+
try { await this.local.start(); } catch (failure) { await this.local.close(); throw failure; }
|
|
125
|
+
}
|
|
126
|
+
this.instance = instance;
|
|
127
|
+
return instance;
|
|
128
|
+
})().finally(() => { this.checking = null; });
|
|
129
|
+
return this.checking;
|
|
130
|
+
}
|
|
131
|
+
allowed(client, tool) {
|
|
132
|
+
const identity = this.config.clients[client];
|
|
133
|
+
return (!identity.tools || identity.tools.includes(tool.name)) && (identity.role === 'controller' || tool.annotations?.readOnlyHint === true);
|
|
134
|
+
}
|
|
135
|
+
async discovery(client) {
|
|
136
|
+
await this.ensure();
|
|
137
|
+
if (this.instance) this.discovered.set(client, this.instance.instanceId);
|
|
138
|
+
const identity = this.config.clients[client];
|
|
139
|
+
return { backend: 'ssapp', contract: 1, instance: this.instance, serverInfo: this.local.info.serverInfo,
|
|
140
|
+
instructions: this.local.info.instructions, role: identity.role, allowHostFiles: identity.allowHostFiles === true,
|
|
141
|
+
catalog: { ...this.local.catalog, tools: this.local.catalog.tools.filter((tool) => this.allowed(client, tool)) },
|
|
142
|
+
lease: this.lease && this.lease.expires > Date.now() ? { owner: this.lease.owner, expires: this.lease.expires } : null,
|
|
143
|
+
uncertain: this.uncertain, limits: { result_bytes: RPC_OPTIONS.maxResult, request_envelope_bytes: 65000, timeout_ms: 60000 }, live_frames: false };
|
|
144
|
+
}
|
|
145
|
+
checkLease(client, args) {
|
|
146
|
+
if (!this.lease || this.lease.owner !== client || this.lease.token !== args.lease || this.lease.expires <= Date.now()) throw error('valid_app_lease_required');
|
|
147
|
+
}
|
|
148
|
+
async execute(client, role, op, args, request) {
|
|
149
|
+
if (op === 'discover') return this.discovery(client);
|
|
150
|
+
if (op === 'release') { this.checkLease(client, args); this.revoke(); return { released: true }; }
|
|
151
|
+
await this.ensure();
|
|
152
|
+
if (this.closed || Date.now() >= request.deadline) throw error('request_expired');
|
|
153
|
+
if (this.instance && (args.instance !== this.instance.instanceId || this.discovered.get(client) !== this.instance.instanceId)) throw error('fresh_app_discovery_required', 'Call ninja_ssapp_status, discover targets, and claim control again');
|
|
154
|
+
if (op === 'claim') {
|
|
155
|
+
if (!this.instance) throw error('SSAPP_UNREACHABLE', 'Start SSApp with API 1.4.1 or newer and enable Local AI / Automation');
|
|
156
|
+
if (role !== 'controller') throw error('observer_cannot_control');
|
|
157
|
+
const ttl = args.ttl_ms === undefined ? 60000 : args.ttl_ms;
|
|
158
|
+
if (!Number.isInteger(ttl) || ttl < 1000 || ttl > 120000) throw error('invalid_lease_duration');
|
|
159
|
+
if (this.uncertain && args.reconciled !== true) throw error('execution_uncertain', 'Inspect the app and operation result, then claim with reconciled: true to resume');
|
|
160
|
+
if (this.lease && this.lease.expires > Date.now() && (this.lease.owner !== client || this.lease.token !== args.lease)) throw error('app_already_claimed');
|
|
161
|
+
this.uncertain = false;
|
|
162
|
+
this.lease = { owner: client, token: this.lease?.expires > Date.now() ? this.lease.token : crypto.randomUUID(), expires: Date.now() + ttl };
|
|
163
|
+
return { lease: this.lease.token, expires: this.lease.expires, instance: this.instance.instanceId };
|
|
164
|
+
}
|
|
165
|
+
if (op !== 'call') throw error('unknown_ssapp_operation');
|
|
166
|
+
const tool = this.local.catalog.tools.find((item) => item.name === args.name);
|
|
167
|
+
if (!tool || !this.allowed(client, tool)) throw error('tool_not_permitted');
|
|
168
|
+
const mutation = tool.annotations?.readOnlyHint !== true;
|
|
169
|
+
if (mutation) { if (!this.instance) throw error('SSAPP_UNREACHABLE'); this.checkLease(client, args); }
|
|
170
|
+
if (args.name === 'ssapp_browser_frame_stream') throw error('remote_frames_unavailable', 'Use ssapp_browser_screenshot; local stream URLs cannot be used remotely');
|
|
171
|
+
if ((args.name === 'ssapp_browser_choose_files' || args.name === 'ssapp_respond_to_app_dialog' && args.arguments?.paths?.length) && this.config.clients[client].allowHostFiles !== true) throw error('host_file_access_disabled');
|
|
172
|
+
const generation = this.generation;
|
|
173
|
+
const dispatch = async () => {
|
|
174
|
+
if (this.closed || generation !== this.generation || Date.now() >= request.deadline) throw error('request_revoked_or_expired');
|
|
175
|
+
if (mutation) this.checkLease(client, args);
|
|
176
|
+
// The local adapter sends the app-instance condition with every HTTP request,
|
|
177
|
+
// including capability checks. A restart between checks cannot admit input.
|
|
178
|
+
let result;
|
|
179
|
+
try { result = await this.local.request('tools/call', { name: args.name, arguments: args.arguments || {} }); }
|
|
180
|
+
catch (failure) { if (mutation) { this.uncertain = true; this.revoke(); } throw failure; }
|
|
181
|
+
const code = result.structuredContent?.error?.code;
|
|
182
|
+
if (mutation && ['SSAPP_UNREACHABLE', 'SSAPP_TIMEOUT'].includes(code)) { this.uncertain = true; this.revoke(); }
|
|
183
|
+
if (code === 'SSAPP_INSTANCE_CHANGED') { this.revoke(); this.discovered.clear(); }
|
|
184
|
+
return { result };
|
|
185
|
+
};
|
|
186
|
+
// Observations and prompt recovery must remain usable during long waits or a
|
|
187
|
+
// blocked renderer. Ordinary mutations retain one queue and one controller.
|
|
188
|
+
if (!mutation || args.name === 'ssapp_respond_to_app_dialog') return dispatch();
|
|
189
|
+
if (this.queued >= 8) throw error('mutation_queue_full');
|
|
190
|
+
this.queued++;
|
|
191
|
+
const pending = this.mutations.then(dispatch, dispatch);
|
|
192
|
+
this.mutations = pending.catch(() => {});
|
|
193
|
+
try { return await pending; } finally { this.queued--; }
|
|
194
|
+
}
|
|
195
|
+
async close() { this.closed = true; this.revoke(); if (this.checking) await this.checking.catch(() => {}); if (this.local) await this.local.close(); }
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
module.exports = { RPC_OPTIONS, LocalMcp, SsappController, controlOrigin, appInstance, validateCommand };
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Copyright (c) 2026 Steve Seguin
|
|
3
|
+
// SPDX-License-Identifier: MIT
|
|
4
|
+
'use strict';
|
|
5
|
+
|
|
6
|
+
const http = require('node:http');
|
|
7
|
+
const fs = require('node:fs');
|
|
8
|
+
const path = require('node:path');
|
|
9
|
+
const crypto = require('node:crypto');
|
|
10
|
+
const { WebSocketServer } = require('ws');
|
|
11
|
+
const { loadConfig, makeSession } = require('./vdo-mcp-browser');
|
|
12
|
+
const { BrowserRpcClient } = require('./lib/browser-rpc');
|
|
13
|
+
const { BrowserVoiceAgent, MODEL } = require('./lib/browser-agent');
|
|
14
|
+
|
|
15
|
+
async function startOperator(config, options = {}) {
|
|
16
|
+
if (config.clients) throw new Error('Use an individual agent JSON file, not worker.json');
|
|
17
|
+
const token = crypto.randomBytes(32).toString('hex');
|
|
18
|
+
const session = options.session || makeSession(config, false);
|
|
19
|
+
const rpc = options.rpc || new BrowserRpcClient(session, config);
|
|
20
|
+
const assets = path.join(__dirname, '..', 'assets', 'browser-agent');
|
|
21
|
+
const files = new Map([
|
|
22
|
+
['/', [path.join(assets, 'index.html'), 'text/html; charset=utf-8']],
|
|
23
|
+
['/client.js', [path.join(assets, 'client.js'), 'text/javascript; charset=utf-8']],
|
|
24
|
+
['/audio-worklet.js', [path.join(assets, 'audio-worklet.js'), 'text/javascript; charset=utf-8']],
|
|
25
|
+
['/sdk.js', [require.resolve('@vdoninja/sdk/browser'), 'text/javascript; charset=utf-8']]
|
|
26
|
+
]);
|
|
27
|
+
let origin, client, agent, connecting, microphone = false, closed = false;
|
|
28
|
+
const send = (event) => {
|
|
29
|
+
if (!client || client.readyState !== 1) return;
|
|
30
|
+
if (client.bufferedAmount > 4 * 1024 * 1024) { client.close(1013, 'Operator connection too slow'); return; }
|
|
31
|
+
client.send(JSON.stringify(event));
|
|
32
|
+
};
|
|
33
|
+
const ensureWorker = () => {
|
|
34
|
+
if (!connecting) connecting = session.start().then(() => rpc.connect()).catch((error) => { connecting = null; throw error; });
|
|
35
|
+
return connecting;
|
|
36
|
+
};
|
|
37
|
+
const server = http.createServer((req, res) => {
|
|
38
|
+
if (req.headers.host !== new URL(origin).host || req.method !== 'GET') { res.writeHead(403); res.end(); return; }
|
|
39
|
+
const file = files.get(req.url);
|
|
40
|
+
if (!file) { res.writeHead(404); res.end(); return; }
|
|
41
|
+
res.writeHead(200, { 'content-type': file[1], 'cache-control': 'no-store', 'referrer-policy': 'no-referrer',
|
|
42
|
+
'content-security-policy': "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; connect-src 'self' wss: https:; img-src 'self' data:; media-src 'self' blob:; object-src 'none'; frame-ancestors 'none'" });
|
|
43
|
+
fs.createReadStream(file[0]).on('error', () => res.destroy()).pipe(res);
|
|
44
|
+
});
|
|
45
|
+
const sockets = new Set();
|
|
46
|
+
server.on('connection', (socket) => { sockets.add(socket); socket.on('close', () => sockets.delete(socket)); });
|
|
47
|
+
const wss = new WebSocketServer({ noServer: true, maxPayload: 100000, perMessageDeflate: false });
|
|
48
|
+
server.on('upgrade', (req, socket, head) => {
|
|
49
|
+
let url;
|
|
50
|
+
try { url = new URL(req.url, origin); } catch (_) { socket.destroy(); return; }
|
|
51
|
+
if (req.headers.host !== new URL(origin).host || req.headers.origin !== origin || url.pathname !== '/operator' || url.searchParams.get('token') !== token || client) { socket.destroy(); return; }
|
|
52
|
+
wss.handleUpgrade(req, socket, head, (ws) => wss.emit('connection', ws));
|
|
53
|
+
});
|
|
54
|
+
wss.on('connection', (ws) => {
|
|
55
|
+
client = ws;
|
|
56
|
+
const voice = new BrowserVoiceAgent(rpc, { apiKey: options.apiKey || process.env.OPENAI_API_KEY, model: options.model || process.env.VDON_BROWSER_MODEL || MODEL,
|
|
57
|
+
emit: (event) => { if (client === ws) send(event); }, connectSocket: options.connectSocket });
|
|
58
|
+
agent = voice;
|
|
59
|
+
send({ type: 'hello', worker: config.worker, provider: 'OpenAI', model: agent.model, key_available: !!agent.apiKey, transport: config.connection.transport || 'webrtc' });
|
|
60
|
+
let commands = Promise.resolve();
|
|
61
|
+
let count = 0;
|
|
62
|
+
let commandEpoch = 0;
|
|
63
|
+
ws.on('message', (raw) => {
|
|
64
|
+
let event;
|
|
65
|
+
try { event = JSON.parse(raw.toString()); } catch (_) { return; }
|
|
66
|
+
if (!event || typeof event.type !== 'string') return;
|
|
67
|
+
if (event.type === 'audio') { if (microphone) voice.audio(event.data); return; }
|
|
68
|
+
if (event.type === 'playback') { voice.truncate(event); return; }
|
|
69
|
+
// Stop and sharing controls take effect immediately, even during a tool call.
|
|
70
|
+
if (event.type === 'interrupt') { commandEpoch++; voice.interrupt(event); return; }
|
|
71
|
+
if (event.type === 'stop') { commandEpoch++; microphone = false; voice.stop(); return; }
|
|
72
|
+
if (event.type === 'permissions') {
|
|
73
|
+
microphone = !!event.microphone;
|
|
74
|
+
if (!microphone) voice.send({ type: 'input_audio_buffer.clear' });
|
|
75
|
+
voice.permissions({ share: event.share, control: event.control });
|
|
76
|
+
send({ type: 'sharing', microphone, frames: voice.share, control: voice.control, recipient: 'OpenAI' });
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
if (count >= 8) { send({ type: 'error', message: 'Too many pending operator actions' }); return; }
|
|
80
|
+
count++;
|
|
81
|
+
const epoch = commandEpoch;
|
|
82
|
+
commands = commands.then(async () => {
|
|
83
|
+
if (client !== ws || closed || epoch !== commandEpoch) return;
|
|
84
|
+
if (event.type === 'start') { await ensureWorker(); if (client === ws && epoch === commandEpoch) await voice.start(); }
|
|
85
|
+
else if (event.type === 'text') voice.text(event.text);
|
|
86
|
+
else if (event.type === 'frame') { await ensureWorker(); if (client === ws && epoch === commandEpoch) await voice.captureFrame(); }
|
|
87
|
+
else if (event.type === 'watch') {
|
|
88
|
+
await ensureWorker();
|
|
89
|
+
if (client !== ws || epoch !== commandEpoch) return;
|
|
90
|
+
const media = await rpc.request('media');
|
|
91
|
+
if (client === ws && epoch === commandEpoch) send({ type: 'media', ...media });
|
|
92
|
+
} else if (event.type === 'check') {
|
|
93
|
+
const hello = await ensureWorker();
|
|
94
|
+
if (client === ws && epoch === commandEpoch) send({ type: 'worker', ok: true, role: hello.role, route: hello.route });
|
|
95
|
+
}
|
|
96
|
+
}).catch((error) => { if (client === ws) send({ type: 'error', message: error.message }); }).finally(() => { count--; });
|
|
97
|
+
});
|
|
98
|
+
ws.on('close', () => {
|
|
99
|
+
commandEpoch++; voice.stop(); microphone = false;
|
|
100
|
+
if (client === ws) client = null;
|
|
101
|
+
});
|
|
102
|
+
ws.on('error', () => {});
|
|
103
|
+
});
|
|
104
|
+
await new Promise((resolve, reject) => { server.once('error', reject); server.listen(options.port || 0, '127.0.0.1', resolve); });
|
|
105
|
+
origin = `http://127.0.0.1:${server.address().port}`;
|
|
106
|
+
let closePromise;
|
|
107
|
+
const close = () => {
|
|
108
|
+
if (!closePromise) closePromise = (async () => {
|
|
109
|
+
closed = true;
|
|
110
|
+
if (agent) { await agent.stop(); await agent.toolChain; }
|
|
111
|
+
if (client) client.terminate();
|
|
112
|
+
wss.close(); rpc.close();
|
|
113
|
+
await session.stop();
|
|
114
|
+
await new Promise((resolve) => { server.close(resolve); for (const socket of sockets) socket.destroy(); });
|
|
115
|
+
})();
|
|
116
|
+
return closePromise;
|
|
117
|
+
};
|
|
118
|
+
return { url: `${origin}/#${token}`, origin, close, rpc, session };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
async function main() {
|
|
122
|
+
const args = process.argv.slice(2);
|
|
123
|
+
if (args.includes('--help')) { console.log('vdon-mcp-browser-agent --config <agent.json>\nOPENAI_API_KEY stays in this local process. Open the printed private loopback URL.'); return; }
|
|
124
|
+
if (args.length !== 2 || args[0] !== '--config') throw new Error('Use --config <agent.json>');
|
|
125
|
+
const service = await startOperator(loadConfig(args[1]));
|
|
126
|
+
console.log(`Open this private operator page on this computer:\n${service.url}`);
|
|
127
|
+
const shutdown = () => service.close().then(() => process.exit(0), (error) => { console.error(error.message); process.exit(1); });
|
|
128
|
+
process.once('SIGINT', shutdown); process.once('SIGTERM', shutdown);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
if (require.main === module) main().catch((error) => { console.error(error.message); process.exitCode = 1; });
|
|
132
|
+
module.exports = { startOperator };
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Copyright (c) 2026 Steve Seguin
|
|
3
|
+
// SPDX-License-Identifier: MIT
|
|
4
|
+
'use strict';
|
|
5
|
+
|
|
6
|
+
const fs = require('node:fs');
|
|
7
|
+
const path = require('node:path');
|
|
8
|
+
const crypto = require('node:crypto');
|
|
9
|
+
const readline = require('node:readline');
|
|
10
|
+
const { BridgeSession } = require('./lib/bridge-session');
|
|
11
|
+
const { BrowserRpcWorker, BrowserRpcClient, keyBytes } = require('./lib/browser-rpc');
|
|
12
|
+
const { BrowserController, launchBrowser } = require('./lib/browser-worker');
|
|
13
|
+
const { McpStdioParser, writeMcpMessage } = require('./lib/mcp-stdio-parser');
|
|
14
|
+
|
|
15
|
+
const pageId = { type: 'integer', minimum: 0 };
|
|
16
|
+
const lease = { type: 'string' };
|
|
17
|
+
function tool(name, description, properties, required = []) {
|
|
18
|
+
return { name, description, inputSchema: { type: 'object', properties, required, additionalProperties: false } };
|
|
19
|
+
}
|
|
20
|
+
const TOOLS = [
|
|
21
|
+
tool('browser_tabs', 'List remote tabs and current control leases.', {}),
|
|
22
|
+
tool('browser_media', 'Get live tab audio/video status and private viewer credentials when the host enables media. Use screenshots when WebRTC media is unavailable. Protect the returned viewing link.', {}),
|
|
23
|
+
tool('browser_observe', 'Read a fresh accessibility snapshot with element UIDs, or a JPEG screenshot. Take a new snapshot after claiming control and after each action.', { pageId, screenshot: { type: 'boolean' } }, ['pageId']),
|
|
24
|
+
tool('browser_claim', 'Claim one tab for up to 120 seconds. Pass the current lease to renew it. Other agents may observe while you control.', { pageId, ttl_ms: { type: 'integer', minimum: 1000, maximum: 120000 }, lease }, ['pageId']),
|
|
25
|
+
tool('browser_release', 'Release a tab for another controller.', { pageId, lease }, ['pageId', 'lease']),
|
|
26
|
+
tool('browser_call', 'Navigate, click, fill, or press a page key. Requires a tab lease; input actions also require a fresh snapshot token. Never repeat an uncertain action before reconciling its result.', {
|
|
27
|
+
pageId, lease, snapshot: { type: 'string' }, action: { type: 'string', enum: ['navigate', 'click', 'fill', 'key'] },
|
|
28
|
+
url: { type: 'string' }, uid: { type: 'string' }, value: { type: 'string' }, key: { type: 'string' }
|
|
29
|
+
}, ['pageId', 'lease', 'action']),
|
|
30
|
+
tool('browser_result', 'Look up a recent request by its request_id. Unknown is not evidence that an action did not execute; inspect the page before deciding what to do next.', { request_id: { type: 'string' } }, ['request_id'])
|
|
31
|
+
];
|
|
32
|
+
|
|
33
|
+
function loadConfig(file) {
|
|
34
|
+
if (!file) throw new Error('Use --config <file>, or set VDON_BROWSER_CONFIG. Run --init <directory> to create credentials.');
|
|
35
|
+
const config = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
36
|
+
if (!/^[a-zA-Z0-9_-]{1,80}$/.test(config.worker || '')) throw new Error('Invalid worker ID');
|
|
37
|
+
if (!config.connection || !/^[a-zA-Z0-9_-]{1,128}$/.test(config.connection.room || '')) throw new Error('A connection.room is required');
|
|
38
|
+
if (config.clients) {
|
|
39
|
+
if (!Object.keys(config.clients).length || Object.keys(config.clients).length > 32) throw new Error('Configure 1 to 32 browser clients');
|
|
40
|
+
const keys = new Set();
|
|
41
|
+
for (const [id, value] of Object.entries(config.clients)) {
|
|
42
|
+
if (!/^[a-zA-Z0-9_-]{1,80}$/.test(id) || !['controller', 'observer'].includes(value.role)) throw new Error('Invalid browser client');
|
|
43
|
+
keyBytes(value.key);
|
|
44
|
+
if (keys.has(value.key.toLowerCase())) throw new Error('Each browser client needs a unique key');
|
|
45
|
+
keys.add(value.key.toLowerCase());
|
|
46
|
+
}
|
|
47
|
+
} else {
|
|
48
|
+
if (!/^[a-zA-Z0-9_-]{1,80}$/.test(config.client || '')) throw new Error('Invalid browser client ID');
|
|
49
|
+
keyBytes(config.key);
|
|
50
|
+
}
|
|
51
|
+
return config;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function makeSession(config, worker) {
|
|
55
|
+
const sessionConfig = {
|
|
56
|
+
...config.connection,
|
|
57
|
+
relayUrl: process.env.VDON_MCP_RELAY_URL || config.connection.relayUrl,
|
|
58
|
+
relayToken: process.env.VDON_MCP_RELAY_TOKEN || config.connection.relayToken,
|
|
59
|
+
relayKey: process.env.VDON_MCP_RELAY_KEY || config.connection.relayKey,
|
|
60
|
+
streamID: worker ? config.worker : `${config.client}_${crypto.randomBytes(6).toString('hex')}`,
|
|
61
|
+
targetStreamID: worker ? null : config.worker,
|
|
62
|
+
maxQueueSize: 32
|
|
63
|
+
};
|
|
64
|
+
if (sessionConfig.transport === 'auto') {
|
|
65
|
+
const { BrowserRoutes } = require('./lib/browser-routes');
|
|
66
|
+
return new BrowserRoutes(sessionConfig);
|
|
67
|
+
}
|
|
68
|
+
return new BridgeSession(sessionConfig);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function initConfigs(directory) {
|
|
72
|
+
fs.mkdirSync(directory, { recursive: true });
|
|
73
|
+
const connection = { room: `browser_${crypto.randomBytes(12).toString('hex')}`, transport: 'webrtc' };
|
|
74
|
+
const worker = `chrome_${crypto.randomBytes(6).toString('hex')}`;
|
|
75
|
+
const clients = {};
|
|
76
|
+
const files = [];
|
|
77
|
+
for (const [client, role] of [['agent1', 'controller'], ['agent2', 'controller'], ['viewer', 'observer']]) {
|
|
78
|
+
const key = crypto.randomBytes(32).toString('hex');
|
|
79
|
+
clients[client] = { role, key };
|
|
80
|
+
files.push([`${client}.json`, { worker, client, key, connection }]);
|
|
81
|
+
}
|
|
82
|
+
files.push(['worker.json', { worker, clients, connection }]);
|
|
83
|
+
for (const [file] of files) if (fs.existsSync(path.join(directory, file))) throw new Error(`${file} already exists; choose a new directory`);
|
|
84
|
+
for (const [file, value] of files) fs.writeFileSync(path.join(directory, file), JSON.stringify(value, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
|
|
85
|
+
process.stderr.write(`Created worker.json, agent1.json, agent2.json, and viewer.json in ${path.resolve(directory)}. Keep these credentials private.\n`);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async function startWorker(config) {
|
|
89
|
+
if (!config.clients) throw new Error('Worker mode requires worker.json');
|
|
90
|
+
const session = makeSession(config, true);
|
|
91
|
+
const runtime = await launchBrowser(config);
|
|
92
|
+
const controller = new BrowserController(runtime.backend, runtime.pages);
|
|
93
|
+
runtime.target.on('framenavigated', (frame) => { if (frame === runtime.target.mainFrame()) controller.invalidate(runtime.pages[0]); });
|
|
94
|
+
const rpc = new BrowserRpcWorker(session, config, (client, role, op, args) => {
|
|
95
|
+
if (op === 'media') return runtime.mediaStatus();
|
|
96
|
+
return controller.execute(client, role, op, args);
|
|
97
|
+
});
|
|
98
|
+
let closePromise;
|
|
99
|
+
const close = () => {
|
|
100
|
+
if (!closePromise) closePromise = (async () => {
|
|
101
|
+
rpc.close();
|
|
102
|
+
try { await session.stop(); } finally { await runtime.close(); }
|
|
103
|
+
})();
|
|
104
|
+
return closePromise;
|
|
105
|
+
};
|
|
106
|
+
try { await session.start(); } catch (error) { await close(); throw error; }
|
|
107
|
+
return { session, rpc, controller, runtime, close };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
async function startClient(config, input = process.stdin, output = process.stdout) {
|
|
111
|
+
if (config.clients) throw new Error('Client mode requires an individual agent credential file');
|
|
112
|
+
const session = makeSession(config, false);
|
|
113
|
+
const rpc = new BrowserRpcClient(session, config);
|
|
114
|
+
const parser = new McpStdioParser({ maxMessageBytes: 65536 });
|
|
115
|
+
let connecting;
|
|
116
|
+
let closed = false;
|
|
117
|
+
let closePromise;
|
|
118
|
+
let inFlight = 0;
|
|
119
|
+
const close = () => {
|
|
120
|
+
if (!closePromise) {
|
|
121
|
+
closed = true;
|
|
122
|
+
rpc.close();
|
|
123
|
+
closePromise = session.stop();
|
|
124
|
+
}
|
|
125
|
+
return closePromise;
|
|
126
|
+
};
|
|
127
|
+
const dispatch = async (message, mode) => {
|
|
128
|
+
if (closed) return;
|
|
129
|
+
if (!message || message.jsonrpc !== '2.0' || typeof message.method !== 'string') return;
|
|
130
|
+
if (message.id === undefined) return;
|
|
131
|
+
const reply = (value) => writeMcpMessage(output, { jsonrpc: '2.0', id: message.id, ...value }, mode);
|
|
132
|
+
if (inFlight >= 16) return reply({ error: { code: -32000, message: 'Too many pending requests' } });
|
|
133
|
+
inFlight++;
|
|
134
|
+
try {
|
|
135
|
+
let result;
|
|
136
|
+
if (message.method === 'initialize') result = { protocolVersion: '2025-06-18', capabilities: { tools: {} }, serverInfo: { name: 'ninja-browser', version: '0.4.2' } };
|
|
137
|
+
else if (message.method === 'ping') result = {};
|
|
138
|
+
else if (message.method === 'tools/list') result = { tools: TOOLS };
|
|
139
|
+
else if (message.method === 'tools/call') {
|
|
140
|
+
const { name, arguments: args = {} } = message.params || {};
|
|
141
|
+
const definition = TOOLS.find((entry) => entry.name === name);
|
|
142
|
+
if (!definition) throw new Error('Unknown browser tool');
|
|
143
|
+
if (!connecting) connecting = session.start().then(() => rpc.connect()).catch((error) => { connecting = null; throw error; });
|
|
144
|
+
await connecting;
|
|
145
|
+
const response = await rpc.request(name.slice('browser_'.length), args);
|
|
146
|
+
const { result: contentResult, ...metadata } = response;
|
|
147
|
+
const hasContent = contentResult && Array.isArray(contentResult.content);
|
|
148
|
+
result = { content: [{ type: 'text', text: JSON.stringify(hasContent ? metadata : response) }, ...(hasContent ? contentResult.content : [])],
|
|
149
|
+
isError: !!response.error || !!(contentResult && contentResult.isError) };
|
|
150
|
+
} else return reply({ error: { code: -32601, message: 'Method not found' } });
|
|
151
|
+
reply({ result });
|
|
152
|
+
} catch (error) { reply({ result: { isError: true, content: [{ type: 'text', text: error.message }] } }); }
|
|
153
|
+
finally { inFlight--; }
|
|
154
|
+
};
|
|
155
|
+
parser.on('message', ({ message, mode }) => { dispatch(message, mode).catch(() => {}); });
|
|
156
|
+
parser.on('error', () => writeMcpMessage(output, { jsonrpc: '2.0', id: null, error: { code: -32700, message: 'Invalid MCP request' } }, parser.mode));
|
|
157
|
+
input.on('data', (chunk) => { parser.push(chunk); if (parser.buffer.length > 65536) { parser.buffer = Buffer.alloc(0); close().catch(() => {}); } });
|
|
158
|
+
input.on('end', () => { close().catch(() => {}); });
|
|
159
|
+
input.on('close', () => { close().catch(() => {}); });
|
|
160
|
+
return { session, rpc, close };
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
async function main() {
|
|
164
|
+
const args = process.argv.slice(2);
|
|
165
|
+
if (args[0] === '--init' && args.length === 2) return initConfigs(args[1]);
|
|
166
|
+
if (args[0] === '--setup') {
|
|
167
|
+
const directory = args[1];
|
|
168
|
+
if (!directory || directory.startsWith('--')) throw new Error('--setup requires a private directory');
|
|
169
|
+
const options = {};
|
|
170
|
+
for (let i = 2; i < args.length; i++) {
|
|
171
|
+
if (args[i] === '--media') options.media = true;
|
|
172
|
+
else if (args[i] === '--transport' && args[i + 1]) options.transport = args[++i];
|
|
173
|
+
else if (args[i] === '--relay-config' && args[i + 1]) options.relayConfig = args[++i];
|
|
174
|
+
else throw new Error(`Unknown setup option: ${args[i]}`);
|
|
175
|
+
}
|
|
176
|
+
return require('./lib/browser-setup').guidedSetup(directory, options);
|
|
177
|
+
}
|
|
178
|
+
const diagnostic = args[0] === '--doctor';
|
|
179
|
+
if (diagnostic) args.shift();
|
|
180
|
+
const worker = args[0] === '--worker';
|
|
181
|
+
if (worker) args.shift();
|
|
182
|
+
if (args.includes('--help')) {
|
|
183
|
+
process.stdout.write('vdon-mcp-browser --setup <private-directory> [--transport webrtc|http|auto] [--relay-config <relay.json>] [--media]\nvdon-mcp-browser --doctor --config <worker-or-agent.json>\nvdon-mcp-browser --worker --config <worker.json>\nvdon-mcp-browser --config <agent1.json>\n');
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
if (args.length && (args.length !== 2 || args[0] !== '--config')) throw new Error('Expected --config <file>');
|
|
187
|
+
const config = loadConfig(args[1] || process.env.VDON_BROWSER_CONFIG);
|
|
188
|
+
if (diagnostic) {
|
|
189
|
+
const result = await require('./lib/browser-setup').doctor(config, makeSession);
|
|
190
|
+
process.stdout.write(JSON.stringify(result, null, 2) + '\n', () => process.exit(result.ok ? 0 : 1));
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
const service = worker ? await startWorker(config) : await startClient(config);
|
|
194
|
+
const shutdown = () => service.close().then(() => process.exit(0), (error) => { process.stderr.write(error.message + '\n'); process.exit(1); });
|
|
195
|
+
process.once('SIGINT', shutdown);
|
|
196
|
+
process.once('SIGTERM', shutdown);
|
|
197
|
+
if (!worker) {
|
|
198
|
+
process.stdin.once('end', shutdown);
|
|
199
|
+
process.stdin.once('close', shutdown);
|
|
200
|
+
}
|
|
201
|
+
if (worker) {
|
|
202
|
+
process.stderr.write(`Browser worker ${config.worker} is running over ${config.connection.transport || 'webrtc'}. Enter "revoke <pageId>" to revoke a tab lease; "stop" or Ctrl+C stops the browser.\n`);
|
|
203
|
+
{
|
|
204
|
+
const terminal = readline.createInterface({ input: process.stdin });
|
|
205
|
+
terminal.on('line', (line) => {
|
|
206
|
+
const match = /^revoke (\d+)$/.exec(line.trim());
|
|
207
|
+
if (match) { service.controller.revoke(Number(match[1])); process.stderr.write('Lease revoked. An action already executing may finish.\n'); }
|
|
208
|
+
if (line.trim() === 'stop') { terminal.close(); shutdown(); }
|
|
209
|
+
});
|
|
210
|
+
terminal.on('SIGINT', () => { terminal.close(); shutdown(); });
|
|
211
|
+
process.once('SIGINT', () => terminal.close());
|
|
212
|
+
process.once('SIGTERM', () => terminal.close());
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
if (require.main === module) main().catch((error) => { process.stderr.write(error.message + '\n'); process.exitCode = 1; });
|
|
218
|
+
module.exports = { TOOLS, loadConfig, makeSession, initConfigs, startWorker, startClient };
|