@mario.andreschak/mcp-browser 3.42.0 → 3.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audioTap.d.ts +23 -0
- package/dist/audioTap.js +147 -0
- package/dist/audioTap.js.map +1 -0
- package/dist/capture.d.ts +94 -0
- package/dist/capture.js +302 -0
- package/dist/capture.js.map +1 -0
- package/dist/gateway.d.ts +28 -0
- package/dist/gateway.js +722 -0
- package/dist/gateway.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +9 -2
- package/dist/index.js.map +1 -1
- package/dist/recording.d.ts +21 -0
- package/dist/recording.js +391 -0
- package/dist/recording.js.map +1 -0
- package/dist/resources.d.ts +19 -1
- package/dist/resources.js +258 -37
- package/dist/resources.js.map +1 -1
- package/dist/runtime.d.ts +58 -2
- package/dist/runtime.js +653 -79
- package/dist/runtime.js.map +1 -1
- package/dist/tools.js +579 -33
- package/dist/tools.js.map +1 -1
- package/dist/viewHtml.d.ts +11 -0
- package/dist/viewHtml.js +469 -0
- package/dist/viewHtml.js.map +1 -0
- package/package.json +1 -1
package/dist/gateway.js
ADDED
|
@@ -0,0 +1,722 @@
|
|
|
1
|
+
import { createServer } from 'node:http';
|
|
2
|
+
import { randomBytes, timingSafeEqual } from 'node:crypto';
|
|
3
|
+
import { enabledEnv, getSession, integerEnv, } from './runtime.js';
|
|
4
|
+
import { renderBrowserViewHtml } from './viewHtml.js';
|
|
5
|
+
import { audioTapSource } from './audioTap.js';
|
|
6
|
+
/**
|
|
7
|
+
* Loopback media gateway for the browser MCP App.
|
|
8
|
+
*
|
|
9
|
+
* The MCP tool channel can only carry one screenshot per JSON-RPC round trip,
|
|
10
|
+
* which is why the app used to look like a slideshow and why anything that
|
|
11
|
+
* actually moves (video, canvas, CSS animation) never rendered. This gateway
|
|
12
|
+
* moves pixels and input off that channel:
|
|
13
|
+
*
|
|
14
|
+
* GET /view the browser UI itself, served from this origin
|
|
15
|
+
* GET /stream multipart/x-mixed-replace MJPEG fed by CDP Page.startScreencast
|
|
16
|
+
* GET /audio chunked PCM tapped out of the page's Web Audio graph
|
|
17
|
+
* GET /events Server-Sent Events carrying url/title/loading transitions
|
|
18
|
+
* POST /input low-latency mouse, wheel, keyboard, and viewport events
|
|
19
|
+
*
|
|
20
|
+
* `/view` is what the MCP App iframes (via `_meta.ui.csp.frameDomains`), the
|
|
21
|
+
* same pattern the VS Code MCP App uses to embed OpenVSCode. Because the UI
|
|
22
|
+
* then runs on a real origin rather than inside the host's app sandbox, it is
|
|
23
|
+
* not bound by the app CSP and can behave like an actual browser window.
|
|
24
|
+
*
|
|
25
|
+
* It binds to loopback only and requires a per-process bearer token that is
|
|
26
|
+
* templated straight into the app HTML, so the token never reaches the model.
|
|
27
|
+
*/
|
|
28
|
+
const MJPEG_BOUNDARY = 'flujoframe';
|
|
29
|
+
const MAX_INPUT_BYTES = 16_384;
|
|
30
|
+
const SSE_HEARTBEAT_MS = 15_000;
|
|
31
|
+
/** Drop frames instead of buffering when a client cannot keep up. */
|
|
32
|
+
const MAX_STREAM_BACKLOG_BYTES = 4_000_000;
|
|
33
|
+
/** Audio is small, but a stalled listener must not grow the heap either. */
|
|
34
|
+
const MAX_AUDIO_BACKLOG_BYTES = 1_000_000;
|
|
35
|
+
/** CDP binding the in-page audio tap posts its PCM chunks through. */
|
|
36
|
+
const AUDIO_BINDING = '__flujoAudioChunk';
|
|
37
|
+
const CORS_HEADERS = {
|
|
38
|
+
'access-control-allow-origin': '*',
|
|
39
|
+
'access-control-allow-headers': 'content-type',
|
|
40
|
+
'access-control-allow-methods': 'GET,POST,OPTIONS',
|
|
41
|
+
'access-control-max-age': '600',
|
|
42
|
+
};
|
|
43
|
+
let httpServer;
|
|
44
|
+
let endpoint;
|
|
45
|
+
let startPromise;
|
|
46
|
+
const channels = new Map();
|
|
47
|
+
function streamEnabled() {
|
|
48
|
+
const raw = process.env.FLUJO_BROWSER_STREAM_ENABLED?.trim();
|
|
49
|
+
if (!raw)
|
|
50
|
+
return true;
|
|
51
|
+
return /^(1|true|yes|on)$/i.test(raw);
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Escape hatch for hosted deployments behind a reverse proxy that rewrites
|
|
55
|
+
* `Host`/`Referer` headers. Mirrors the MCP Apps sandbox escape hatch: when the
|
|
56
|
+
* persisted `network.allowAllMcpAppContent` setting is enabled (propagated here
|
|
57
|
+
* by scripts/exposure-mode.mjs as `FLUJO_MCP_APP_SANDBOX_ALLOW_ALL`), the
|
|
58
|
+
* browser live-view gateway accepts any `Host` header and widens its CSP grants
|
|
59
|
+
* so the app can frame the gateway regardless of origin. This disables the
|
|
60
|
+
* DNS-rebinding guard and is intended only as a temporary escape hatch; the
|
|
61
|
+
* long-term fix is correct `FLUJO_BROWSER_STREAM_PUBLIC_ORIGIN` config.
|
|
62
|
+
*/
|
|
63
|
+
function sandboxAllowAll() {
|
|
64
|
+
const value = process.env.FLUJO_MCP_APP_SANDBOX_ALLOW_ALL;
|
|
65
|
+
if (!value)
|
|
66
|
+
return false;
|
|
67
|
+
return value === '1' || value === 'true' || value === 'yes' || value === 'on';
|
|
68
|
+
}
|
|
69
|
+
function bindHost() {
|
|
70
|
+
const configured = process.env.FLUJO_BROWSER_STREAM_HOST?.trim();
|
|
71
|
+
if (configured)
|
|
72
|
+
return configured;
|
|
73
|
+
// Under the escape hatch, bind all interfaces so a hosted reverse proxy can
|
|
74
|
+
// reach the gateway; otherwise stay on loopback.
|
|
75
|
+
return sandboxAllowAll() ? '0.0.0.0' : '127.0.0.1';
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Advertised origin. Defaults to the bound loopback address; operators running
|
|
79
|
+
* FLUJO behind a reverse proxy can point the app somewhere reachable instead.
|
|
80
|
+
*/
|
|
81
|
+
function publicOrigin(port) {
|
|
82
|
+
const configured = process.env.FLUJO_BROWSER_STREAM_PUBLIC_ORIGIN?.trim();
|
|
83
|
+
if (configured) {
|
|
84
|
+
try {
|
|
85
|
+
const url = new URL(configured);
|
|
86
|
+
if (url.protocol === 'http:' || url.protocol === 'https:')
|
|
87
|
+
return url.origin;
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
// A malformed override never replaces the safe loopback default.
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
const host = bindHost();
|
|
94
|
+
const literal = host === '0.0.0.0' || host === '::' ? '127.0.0.1' : host;
|
|
95
|
+
return `http://${literal.includes(':') ? `[${literal}]` : literal}:${port}`;
|
|
96
|
+
}
|
|
97
|
+
/** Exported for tests: whether the escape hatch is active. */
|
|
98
|
+
export function browserSandboxAllowAll() {
|
|
99
|
+
return sandboxAllowAll();
|
|
100
|
+
}
|
|
101
|
+
function tokenMatches(provided) {
|
|
102
|
+
if (!endpoint || !provided)
|
|
103
|
+
return false;
|
|
104
|
+
const expected = Buffer.from(endpoint.token, 'utf8');
|
|
105
|
+
const actual = Buffer.from(provided, 'utf8');
|
|
106
|
+
return expected.length === actual.length && timingSafeEqual(expected, actual);
|
|
107
|
+
}
|
|
108
|
+
/** Reject DNS-rebinding attempts that resolve some public name to our port. */
|
|
109
|
+
function hostHeaderAllowed(req) {
|
|
110
|
+
// Escape hatch: accept any Host header so a hosted reverse proxy that
|
|
111
|
+
// rewrites Host can forward to the gateway. Disables the DNS-rebinding guard.
|
|
112
|
+
if (sandboxAllowAll())
|
|
113
|
+
return true;
|
|
114
|
+
const header = (req.headers.host ?? '').toLowerCase();
|
|
115
|
+
if (!header)
|
|
116
|
+
return false;
|
|
117
|
+
const hostname = header.startsWith('[')
|
|
118
|
+
? header.slice(1, header.indexOf(']'))
|
|
119
|
+
: header.split(':')[0];
|
|
120
|
+
if (hostname === '127.0.0.1' || hostname === 'localhost' || hostname === '::1')
|
|
121
|
+
return true;
|
|
122
|
+
const configured = process.env.FLUJO_BROWSER_STREAM_PUBLIC_ORIGIN?.trim();
|
|
123
|
+
if (!configured)
|
|
124
|
+
return false;
|
|
125
|
+
try {
|
|
126
|
+
return new URL(configured).hostname.toLowerCase().replace(/^\[|\]$/g, '') === hostname;
|
|
127
|
+
}
|
|
128
|
+
catch {
|
|
129
|
+
return false;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
function respondJson(res, status, body) {
|
|
133
|
+
res.writeHead(status, {
|
|
134
|
+
'content-type': 'application/json; charset=utf-8',
|
|
135
|
+
'cache-control': 'no-store',
|
|
136
|
+
...CORS_HEADERS,
|
|
137
|
+
});
|
|
138
|
+
res.end(JSON.stringify(body));
|
|
139
|
+
}
|
|
140
|
+
function resolveChannel(sessionId) {
|
|
141
|
+
const existing = channels.get(sessionId);
|
|
142
|
+
if (existing && !existing.disposed) {
|
|
143
|
+
// Re-validate so the idle reaper sees the session as active while streaming.
|
|
144
|
+
getSession(sessionId);
|
|
145
|
+
return existing;
|
|
146
|
+
}
|
|
147
|
+
const session = getSession(sessionId);
|
|
148
|
+
const channel = {
|
|
149
|
+
sessionId: session.id,
|
|
150
|
+
session,
|
|
151
|
+
frameClients: new Set(),
|
|
152
|
+
eventClients: new Set(),
|
|
153
|
+
audioClients: new Set(),
|
|
154
|
+
screencasting: false,
|
|
155
|
+
audioTapped: false,
|
|
156
|
+
audioExecutionContexts: new Set(),
|
|
157
|
+
audioSignal: false,
|
|
158
|
+
inputChain: Promise.resolve(),
|
|
159
|
+
disposed: false,
|
|
160
|
+
dispose: () => undefined,
|
|
161
|
+
};
|
|
162
|
+
const onClose = () => disposeChannel(channel);
|
|
163
|
+
const onNavigated = (frame) => {
|
|
164
|
+
if (frame === session.page.mainFrame()) {
|
|
165
|
+
channel.audioSignal = false;
|
|
166
|
+
void emitState(channel, 'loading');
|
|
167
|
+
}
|
|
168
|
+
};
|
|
169
|
+
const onLoad = () => {
|
|
170
|
+
void emitState(channel, 'idle');
|
|
171
|
+
if (channel.audioClients.size > 0)
|
|
172
|
+
void setAudioMuted(channel, false);
|
|
173
|
+
};
|
|
174
|
+
session.page.once('close', onClose);
|
|
175
|
+
session.page.on('framenavigated', onNavigated);
|
|
176
|
+
session.page.on('domcontentloaded', onLoad);
|
|
177
|
+
session.page.on('load', onLoad);
|
|
178
|
+
channel.dispose = () => {
|
|
179
|
+
session.page.off('framenavigated', onNavigated);
|
|
180
|
+
session.page.off('domcontentloaded', onLoad);
|
|
181
|
+
session.page.off('load', onLoad);
|
|
182
|
+
};
|
|
183
|
+
channels.set(session.id, channel);
|
|
184
|
+
return channel;
|
|
185
|
+
}
|
|
186
|
+
function disposeChannel(channel) {
|
|
187
|
+
if (channel.disposed)
|
|
188
|
+
return;
|
|
189
|
+
channel.disposed = true;
|
|
190
|
+
channels.delete(channel.sessionId);
|
|
191
|
+
channel.dispose();
|
|
192
|
+
for (const client of channel.frameClients)
|
|
193
|
+
client.end();
|
|
194
|
+
for (const client of channel.eventClients)
|
|
195
|
+
client.end();
|
|
196
|
+
for (const client of channel.audioClients)
|
|
197
|
+
client.end();
|
|
198
|
+
channel.frameClients.clear();
|
|
199
|
+
channel.eventClients.clear();
|
|
200
|
+
channel.audioClients.clear();
|
|
201
|
+
const cdp = channel.cdp;
|
|
202
|
+
channel.cdp = undefined;
|
|
203
|
+
channel.screencasting = false;
|
|
204
|
+
channel.audioTapped = false;
|
|
205
|
+
channel.audioPreparePromise = undefined;
|
|
206
|
+
channel.audioExecutionContexts.clear();
|
|
207
|
+
void cdp?.detach().catch(() => undefined);
|
|
208
|
+
}
|
|
209
|
+
async function emitState(channel, phase) {
|
|
210
|
+
if (channel.disposed || channel.eventClients.size === 0)
|
|
211
|
+
return;
|
|
212
|
+
// The app is a user-facing surface behind a loopback token, so unlike the
|
|
213
|
+
// model-facing tool payloads it may show the real URL including its query.
|
|
214
|
+
const url = channel.session.page.url();
|
|
215
|
+
const title = await channel.session.page.title().catch(() => '');
|
|
216
|
+
const payload = JSON.stringify({
|
|
217
|
+
sessionId: channel.sessionId,
|
|
218
|
+
url,
|
|
219
|
+
title,
|
|
220
|
+
phase,
|
|
221
|
+
viewport: channel.session.page.viewportSize(),
|
|
222
|
+
audio: audioEnabled(),
|
|
223
|
+
audioSignal: channel.audioSignal,
|
|
224
|
+
});
|
|
225
|
+
for (const client of channel.eventClients) {
|
|
226
|
+
client.write(`event: state\ndata: ${payload}\n\n`);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
/** One CDP session per browser session, shared by the screencast and audio tap. */
|
|
230
|
+
async function ensureCdp(channel) {
|
|
231
|
+
if (channel.cdp)
|
|
232
|
+
return channel.cdp;
|
|
233
|
+
const cdp = await channel.session.context.newCDPSession(channel.session.page);
|
|
234
|
+
channel.cdp = cdp;
|
|
235
|
+
return cdp;
|
|
236
|
+
}
|
|
237
|
+
async function startScreencast(channel) {
|
|
238
|
+
if (channel.screencasting || channel.disposed)
|
|
239
|
+
return;
|
|
240
|
+
channel.screencasting = true;
|
|
241
|
+
try {
|
|
242
|
+
const cdp = await ensureCdp(channel);
|
|
243
|
+
cdp.on('Page.screencastFrame', (frame) => {
|
|
244
|
+
void cdp.send('Page.screencastFrameAck', { sessionId: frame.sessionId }).catch(() => undefined);
|
|
245
|
+
const buffer = Buffer.from(frame.data, 'base64');
|
|
246
|
+
channel.lastFrame = buffer;
|
|
247
|
+
broadcastFrame(channel, buffer);
|
|
248
|
+
});
|
|
249
|
+
// Headless Chromium throttles rendering (and therefore video) on a page it
|
|
250
|
+
// believes is unfocused; emulating focus is what keeps playback running.
|
|
251
|
+
await cdp.send('Emulation.setFocusEmulationEnabled', { enabled: true }).catch(() => undefined);
|
|
252
|
+
await cdp.send('Page.startScreencast', {
|
|
253
|
+
format: 'jpeg',
|
|
254
|
+
quality: integerEnv('FLUJO_BROWSER_STREAM_QUALITY', 55, 10, 95),
|
|
255
|
+
maxWidth: integerEnv('FLUJO_BROWSER_STREAM_MAX_WIDTH', 1600, 320, 3840),
|
|
256
|
+
maxHeight: integerEnv('FLUJO_BROWSER_STREAM_MAX_HEIGHT', 1200, 240, 2160),
|
|
257
|
+
everyNthFrame: 1,
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
catch {
|
|
261
|
+
channel.screencasting = false;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
async function stopScreencast(channel) {
|
|
265
|
+
if (!channel.screencasting)
|
|
266
|
+
return;
|
|
267
|
+
channel.screencasting = false;
|
|
268
|
+
await channel.cdp?.send('Page.stopScreencast').catch(() => undefined);
|
|
269
|
+
}
|
|
270
|
+
function broadcastFrame(channel, jpeg) {
|
|
271
|
+
const header = Buffer.from(`--${MJPEG_BOUNDARY}\r\nContent-Type: image/jpeg\r\nContent-Length: ${jpeg.length}\r\n\r\n`, 'ascii');
|
|
272
|
+
for (const client of channel.frameClients) {
|
|
273
|
+
// Never queue frames for a client that already fell behind: a stalled
|
|
274
|
+
// socket must degrade to a lower frame rate, not to unbounded memory.
|
|
275
|
+
if (client.writableLength > MAX_STREAM_BACKLOG_BYTES)
|
|
276
|
+
continue;
|
|
277
|
+
client.write(header);
|
|
278
|
+
client.write(jpeg);
|
|
279
|
+
client.write('\r\n');
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
/** Audio capture is on by default; operators can drop it to save bandwidth. */
|
|
283
|
+
function audioEnabled() {
|
|
284
|
+
const raw = process.env.FLUJO_BROWSER_STREAM_AUDIO?.trim();
|
|
285
|
+
if (!raw)
|
|
286
|
+
return true;
|
|
287
|
+
return /^(1|true|yes|on)$/i.test(raw);
|
|
288
|
+
}
|
|
289
|
+
async function evaluateInAudioContexts(channel, expression) {
|
|
290
|
+
const cdp = channel.cdp;
|
|
291
|
+
if (!cdp)
|
|
292
|
+
return;
|
|
293
|
+
const contextIds = [...channel.audioExecutionContexts];
|
|
294
|
+
if (contextIds.length === 0) {
|
|
295
|
+
await cdp.send('Runtime.evaluate', { expression }).catch(() => undefined);
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
await Promise.all(contextIds.map(async (contextId) => {
|
|
299
|
+
const result = await cdp.send('Runtime.evaluate', { expression, contextId }).catch(() => undefined);
|
|
300
|
+
if (result && 'exceptionDetails' in result && result.exceptionDetails) {
|
|
301
|
+
channel.audioExecutionContexts.delete(contextId);
|
|
302
|
+
}
|
|
303
|
+
}));
|
|
304
|
+
}
|
|
305
|
+
async function prepareAudio(channel) {
|
|
306
|
+
if (channel.disposed || !audioEnabled() || channel.audioTapped)
|
|
307
|
+
return;
|
|
308
|
+
if (channel.audioPreparePromise)
|
|
309
|
+
return channel.audioPreparePromise;
|
|
310
|
+
channel.audioPreparePromise = (async () => {
|
|
311
|
+
const cdp = await ensureCdp(channel);
|
|
312
|
+
cdp.on('Runtime.executionContextCreated', (event) => {
|
|
313
|
+
const isDefault = event.context.auxData?.isDefault;
|
|
314
|
+
if (isDefault !== true && isDefault !== 'true')
|
|
315
|
+
return;
|
|
316
|
+
channel.audioExecutionContexts.add(event.context.id);
|
|
317
|
+
if (channel.audioClients.size > 0) {
|
|
318
|
+
void cdp.send('Runtime.evaluate', {
|
|
319
|
+
expression: 'window.__flujoAudioMuted = false;',
|
|
320
|
+
contextId: event.context.id,
|
|
321
|
+
}).catch(() => undefined);
|
|
322
|
+
}
|
|
323
|
+
});
|
|
324
|
+
cdp.on('Runtime.executionContextDestroyed', (event) => {
|
|
325
|
+
channel.audioExecutionContexts.delete(event.executionContextId);
|
|
326
|
+
});
|
|
327
|
+
cdp.on('Runtime.executionContextsCleared', () => channel.audioExecutionContexts.clear());
|
|
328
|
+
cdp.on('Runtime.bindingCalled', (event) => {
|
|
329
|
+
if (event.name !== AUDIO_BINDING)
|
|
330
|
+
return;
|
|
331
|
+
broadcastAudio(channel, event.payload);
|
|
332
|
+
});
|
|
333
|
+
// The hook must exist before the first page navigation. It stays muted until
|
|
334
|
+
// /audio has a listener, so an idle live session pays no base64/CDP cost.
|
|
335
|
+
const source = audioTapSource(AUDIO_BINDING, true);
|
|
336
|
+
await cdp.send('Runtime.enable');
|
|
337
|
+
await cdp.send('Runtime.addBinding', { name: AUDIO_BINDING });
|
|
338
|
+
await cdp.send('Page.addScriptToEvaluateOnNewDocument', { source });
|
|
339
|
+
// Also recover the current document (for reused sessions and about:blank).
|
|
340
|
+
await evaluateInAudioContexts(channel, source);
|
|
341
|
+
channel.audioTapped = true;
|
|
342
|
+
})();
|
|
343
|
+
try {
|
|
344
|
+
await channel.audioPreparePromise;
|
|
345
|
+
}
|
|
346
|
+
finally {
|
|
347
|
+
channel.audioPreparePromise = undefined;
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
async function setAudioMuted(channel, muted) {
|
|
351
|
+
if (!channel.audioTapped)
|
|
352
|
+
return;
|
|
353
|
+
await evaluateInAudioContexts(channel, `window.__flujoAudioMuted = ${muted ? 'true' : 'false'};`);
|
|
354
|
+
}
|
|
355
|
+
async function startAudio(channel) {
|
|
356
|
+
if (channel.disposed || !audioEnabled())
|
|
357
|
+
return;
|
|
358
|
+
await prepareAudio(channel);
|
|
359
|
+
await setAudioMuted(channel, false);
|
|
360
|
+
}
|
|
361
|
+
/** Leave the tap installed but stop paying for chunks nobody is listening to. */
|
|
362
|
+
async function stopAudio(channel) {
|
|
363
|
+
if (!channel.audioTapped)
|
|
364
|
+
return;
|
|
365
|
+
await setAudioMuted(channel, true);
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* Frame one PCM chunk for the wire.
|
|
369
|
+
*
|
|
370
|
+
* Header is little-endian `[sampleRate u32, channels u32, byteLength u32]`,
|
|
371
|
+
* followed by interleaved 16-bit samples. Self-describing per chunk so a client
|
|
372
|
+
* can join the stream at any point and survive a sample-rate change.
|
|
373
|
+
*/
|
|
374
|
+
function broadcastAudio(channel, payload) {
|
|
375
|
+
if (channel.audioClients.size === 0)
|
|
376
|
+
return;
|
|
377
|
+
let rate;
|
|
378
|
+
let pcm;
|
|
379
|
+
try {
|
|
380
|
+
const parsed = JSON.parse(payload);
|
|
381
|
+
if (typeof parsed.rate !== 'number' || typeof parsed.pcm !== 'string')
|
|
382
|
+
return;
|
|
383
|
+
rate = Math.trunc(parsed.rate);
|
|
384
|
+
pcm = Buffer.from(parsed.pcm, 'base64');
|
|
385
|
+
}
|
|
386
|
+
catch {
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
if (!pcm.length || rate < 8_000 || rate > 192_000)
|
|
390
|
+
return;
|
|
391
|
+
if (!channel.audioSignal) {
|
|
392
|
+
channel.audioSignal = true;
|
|
393
|
+
void emitState(channel, 'idle');
|
|
394
|
+
}
|
|
395
|
+
const header = Buffer.alloc(12);
|
|
396
|
+
header.writeUInt32LE(rate, 0);
|
|
397
|
+
header.writeUInt32LE(2, 4);
|
|
398
|
+
header.writeUInt32LE(pcm.length, 8);
|
|
399
|
+
for (const client of channel.audioClients) {
|
|
400
|
+
if (client.writableLength > MAX_AUDIO_BACKLOG_BYTES)
|
|
401
|
+
continue;
|
|
402
|
+
client.write(header);
|
|
403
|
+
client.write(pcm);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
function handleAudio(channel, req, res) {
|
|
407
|
+
res.writeHead(200, {
|
|
408
|
+
'content-type': 'application/octet-stream',
|
|
409
|
+
'cache-control': 'no-store',
|
|
410
|
+
connection: 'close',
|
|
411
|
+
...CORS_HEADERS,
|
|
412
|
+
});
|
|
413
|
+
res.socket?.setNoDelay(true);
|
|
414
|
+
channel.audioClients.add(res);
|
|
415
|
+
const detach = () => {
|
|
416
|
+
channel.audioClients.delete(res);
|
|
417
|
+
if (channel.audioClients.size === 0)
|
|
418
|
+
void stopAudio(channel);
|
|
419
|
+
};
|
|
420
|
+
req.once('close', detach);
|
|
421
|
+
res.once('close', detach);
|
|
422
|
+
void startAudio(channel);
|
|
423
|
+
}
|
|
424
|
+
function handleStream(channel, req, res) {
|
|
425
|
+
res.writeHead(200, {
|
|
426
|
+
'content-type': `multipart/x-mixed-replace; boundary=${MJPEG_BOUNDARY}`,
|
|
427
|
+
'cache-control': 'no-store, no-cache, must-revalidate',
|
|
428
|
+
pragma: 'no-cache',
|
|
429
|
+
connection: 'close',
|
|
430
|
+
...CORS_HEADERS,
|
|
431
|
+
});
|
|
432
|
+
res.socket?.setNoDelay(true);
|
|
433
|
+
channel.frameClients.add(res);
|
|
434
|
+
if (channel.lastFrame)
|
|
435
|
+
broadcastFrame(channel, channel.lastFrame);
|
|
436
|
+
const detach = () => {
|
|
437
|
+
channel.frameClients.delete(res);
|
|
438
|
+
if (channel.frameClients.size === 0)
|
|
439
|
+
void stopScreencast(channel);
|
|
440
|
+
};
|
|
441
|
+
req.once('close', detach);
|
|
442
|
+
res.once('close', detach);
|
|
443
|
+
void startScreencast(channel);
|
|
444
|
+
}
|
|
445
|
+
function handleEvents(channel, req, res) {
|
|
446
|
+
res.writeHead(200, {
|
|
447
|
+
'content-type': 'text/event-stream; charset=utf-8',
|
|
448
|
+
'cache-control': 'no-store',
|
|
449
|
+
connection: 'keep-alive',
|
|
450
|
+
...CORS_HEADERS,
|
|
451
|
+
});
|
|
452
|
+
res.socket?.setNoDelay(true);
|
|
453
|
+
channel.eventClients.add(res);
|
|
454
|
+
const heartbeat = setInterval(() => res.write(': ping\n\n'), SSE_HEARTBEAT_MS);
|
|
455
|
+
heartbeat.unref();
|
|
456
|
+
const detach = () => {
|
|
457
|
+
clearInterval(heartbeat);
|
|
458
|
+
channel.eventClients.delete(res);
|
|
459
|
+
};
|
|
460
|
+
req.once('close', detach);
|
|
461
|
+
res.once('close', detach);
|
|
462
|
+
void emitState(channel, 'idle');
|
|
463
|
+
}
|
|
464
|
+
async function readBody(req) {
|
|
465
|
+
const chunks = [];
|
|
466
|
+
let size = 0;
|
|
467
|
+
for await (const chunk of req) {
|
|
468
|
+
const buffer = chunk;
|
|
469
|
+
size += buffer.length;
|
|
470
|
+
if (size > MAX_INPUT_BYTES)
|
|
471
|
+
throw new Error('Input payload is too large.');
|
|
472
|
+
chunks.push(buffer);
|
|
473
|
+
}
|
|
474
|
+
return Buffer.concat(chunks).toString('utf8');
|
|
475
|
+
}
|
|
476
|
+
function finiteNumber(value, fallback = 0) {
|
|
477
|
+
return typeof value === 'number' && Number.isFinite(value) ? value : fallback;
|
|
478
|
+
}
|
|
479
|
+
function mouseButton(value) {
|
|
480
|
+
return value === 'right' || value === 'middle' ? value : 'left';
|
|
481
|
+
}
|
|
482
|
+
/**
|
|
483
|
+
* Dispatch one app input event. Everything is funnelled through a per-session
|
|
484
|
+
* promise chain so a fast typist can never interleave two Playwright input
|
|
485
|
+
* calls on the same page.
|
|
486
|
+
*/
|
|
487
|
+
async function dispatchInput(channel, event) {
|
|
488
|
+
const { page } = channel.session;
|
|
489
|
+
const type = String(event.type ?? '');
|
|
490
|
+
if (type === 'mousemove') {
|
|
491
|
+
await page.mouse.move(finiteNumber(event.x), finiteNumber(event.y));
|
|
492
|
+
return;
|
|
493
|
+
}
|
|
494
|
+
if (type === 'mousedown' || type === 'mouseup') {
|
|
495
|
+
const button = mouseButton(event.button);
|
|
496
|
+
const clickCount = Math.min(3, Math.max(1, Math.trunc(finiteNumber(event.clickCount, 1))));
|
|
497
|
+
await page.mouse.move(finiteNumber(event.x), finiteNumber(event.y));
|
|
498
|
+
if (type === 'mousedown')
|
|
499
|
+
await page.mouse.down({ button, clickCount });
|
|
500
|
+
else
|
|
501
|
+
await page.mouse.up({ button, clickCount });
|
|
502
|
+
return;
|
|
503
|
+
}
|
|
504
|
+
if (type === 'wheel') {
|
|
505
|
+
await page.mouse.wheel(finiteNumber(event.deltaX), finiteNumber(event.deltaY));
|
|
506
|
+
return;
|
|
507
|
+
}
|
|
508
|
+
if (type === 'keydown' || type === 'keyup') {
|
|
509
|
+
const key = typeof event.key === 'string' ? event.key : '';
|
|
510
|
+
if (!key || key.length > 64)
|
|
511
|
+
return;
|
|
512
|
+
if (type === 'keydown')
|
|
513
|
+
await page.keyboard.down(key);
|
|
514
|
+
else
|
|
515
|
+
await page.keyboard.up(key);
|
|
516
|
+
return;
|
|
517
|
+
}
|
|
518
|
+
if (type === 'text') {
|
|
519
|
+
const text = typeof event.text === 'string' ? event.text : '';
|
|
520
|
+
if (text)
|
|
521
|
+
await page.keyboard.insertText(text.slice(0, 4_096));
|
|
522
|
+
return;
|
|
523
|
+
}
|
|
524
|
+
if (type === 'viewport') {
|
|
525
|
+
const width = Math.trunc(finiteNumber(event.width));
|
|
526
|
+
const height = Math.trunc(finiteNumber(event.height));
|
|
527
|
+
if (width < 320 || height < 240 || width > 3840 || height > 2160)
|
|
528
|
+
return;
|
|
529
|
+
const current = page.viewportSize();
|
|
530
|
+
if (current && current.width === width && current.height === height)
|
|
531
|
+
return;
|
|
532
|
+
await page.setViewportSize({ width, height });
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
async function handleInput(channel, req, res) {
|
|
536
|
+
let events;
|
|
537
|
+
try {
|
|
538
|
+
events = JSON.parse(await readBody(req));
|
|
539
|
+
}
|
|
540
|
+
catch {
|
|
541
|
+
respondJson(res, 400, { error: 'Malformed input payload.' });
|
|
542
|
+
return;
|
|
543
|
+
}
|
|
544
|
+
const list = Array.isArray(events) ? events : [events];
|
|
545
|
+
if (list.length > 64) {
|
|
546
|
+
respondJson(res, 400, { error: 'Too many input events in one batch.' });
|
|
547
|
+
return;
|
|
548
|
+
}
|
|
549
|
+
const run = channel.inputChain.then(async () => {
|
|
550
|
+
for (const event of list) {
|
|
551
|
+
if (!event || typeof event !== 'object' || Array.isArray(event))
|
|
552
|
+
continue;
|
|
553
|
+
await dispatchInput(channel, event);
|
|
554
|
+
}
|
|
555
|
+
});
|
|
556
|
+
// Keep the chain alive even when one event throws, otherwise a single failed
|
|
557
|
+
// dispatch would deadlock every later keystroke for this session.
|
|
558
|
+
channel.inputChain = run.catch(() => undefined);
|
|
559
|
+
try {
|
|
560
|
+
await run;
|
|
561
|
+
respondJson(res, 200, { ok: true });
|
|
562
|
+
}
|
|
563
|
+
catch (error) {
|
|
564
|
+
respondJson(res, 200, { ok: false, error: error instanceof Error ? error.message : 'Input failed.' });
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
async function handleRequest(req, res) {
|
|
568
|
+
if (req.method === 'OPTIONS') {
|
|
569
|
+
res.writeHead(204, CORS_HEADERS);
|
|
570
|
+
res.end();
|
|
571
|
+
return;
|
|
572
|
+
}
|
|
573
|
+
if (!hostHeaderAllowed(req)) {
|
|
574
|
+
respondJson(res, 403, { error: 'Forbidden host.' });
|
|
575
|
+
return;
|
|
576
|
+
}
|
|
577
|
+
const url = new URL(req.url ?? '/', 'http://127.0.0.1');
|
|
578
|
+
if (url.pathname === '/health') {
|
|
579
|
+
respondJson(res, 200, { ok: true });
|
|
580
|
+
return;
|
|
581
|
+
}
|
|
582
|
+
if (!tokenMatches(url.searchParams.get('t'))) {
|
|
583
|
+
respondJson(res, 403, { error: 'Invalid gateway token.' });
|
|
584
|
+
return;
|
|
585
|
+
}
|
|
586
|
+
if (req.method === 'GET' && url.pathname === '/view') {
|
|
587
|
+
// No X-Frame-Options and no frame-ancestors: the MCP App sandbox origin is
|
|
588
|
+
// unknown here, and the bearer token is what actually gates access.
|
|
589
|
+
res.writeHead(200, {
|
|
590
|
+
'content-type': 'text/html; charset=utf-8',
|
|
591
|
+
'cache-control': 'no-store',
|
|
592
|
+
'referrer-policy': 'no-referrer',
|
|
593
|
+
'x-content-type-options': 'nosniff',
|
|
594
|
+
});
|
|
595
|
+
res.end(renderBrowserViewHtml());
|
|
596
|
+
return;
|
|
597
|
+
}
|
|
598
|
+
const sessionId = url.searchParams.get('s') ?? '';
|
|
599
|
+
let channel;
|
|
600
|
+
try {
|
|
601
|
+
channel = resolveChannel(sessionId);
|
|
602
|
+
}
|
|
603
|
+
catch (error) {
|
|
604
|
+
respondJson(res, 404, { error: error instanceof Error ? error.message : 'Unknown session.' });
|
|
605
|
+
return;
|
|
606
|
+
}
|
|
607
|
+
if (req.method === 'GET' && url.pathname === '/stream') {
|
|
608
|
+
handleStream(channel, req, res);
|
|
609
|
+
return;
|
|
610
|
+
}
|
|
611
|
+
if (req.method === 'GET' && url.pathname === '/audio') {
|
|
612
|
+
if (!audioEnabled()) {
|
|
613
|
+
respondJson(res, 404, { error: 'Audio capture is disabled.' });
|
|
614
|
+
return;
|
|
615
|
+
}
|
|
616
|
+
handleAudio(channel, req, res);
|
|
617
|
+
return;
|
|
618
|
+
}
|
|
619
|
+
if (req.method === 'GET' && url.pathname === '/events') {
|
|
620
|
+
handleEvents(channel, req, res);
|
|
621
|
+
return;
|
|
622
|
+
}
|
|
623
|
+
if (req.method === 'POST' && url.pathname === '/input') {
|
|
624
|
+
await handleInput(channel, req, res);
|
|
625
|
+
return;
|
|
626
|
+
}
|
|
627
|
+
respondJson(res, 404, { error: 'Unknown gateway endpoint.' });
|
|
628
|
+
}
|
|
629
|
+
/**
|
|
630
|
+
* Start (or reuse) the loopback gateway. Resolves to `undefined` when streaming
|
|
631
|
+
* is disabled or the listener cannot bind, so the MCP App can fall back to the
|
|
632
|
+
* screenshot poll loop instead of failing to render at all.
|
|
633
|
+
*/
|
|
634
|
+
export async function ensureBrowserGateway() {
|
|
635
|
+
if (!streamEnabled())
|
|
636
|
+
return undefined;
|
|
637
|
+
if (endpoint)
|
|
638
|
+
return endpoint;
|
|
639
|
+
if (startPromise)
|
|
640
|
+
return startPromise;
|
|
641
|
+
startPromise = (async () => {
|
|
642
|
+
try {
|
|
643
|
+
const server = createServer((req, res) => {
|
|
644
|
+
void handleRequest(req, res).catch(() => {
|
|
645
|
+
if (!res.headersSent)
|
|
646
|
+
respondJson(res, 500, { error: 'Gateway failure.' });
|
|
647
|
+
else
|
|
648
|
+
res.end();
|
|
649
|
+
});
|
|
650
|
+
});
|
|
651
|
+
server.on('error', () => undefined);
|
|
652
|
+
// Screencast sockets are long-lived by design; the per-session idle
|
|
653
|
+
// reaper in runtime.ts is what bounds their lifetime.
|
|
654
|
+
server.headersTimeout = 0;
|
|
655
|
+
server.requestTimeout = 0;
|
|
656
|
+
server.keepAliveTimeout = 0;
|
|
657
|
+
server.timeout = 0;
|
|
658
|
+
const port = integerEnv('FLUJO_BROWSER_STREAM_PORT', 0, 0, 65_535);
|
|
659
|
+
await new Promise((resolve, reject) => {
|
|
660
|
+
server.once('error', reject);
|
|
661
|
+
server.listen(port, bindHost(), () => {
|
|
662
|
+
server.removeListener('error', reject);
|
|
663
|
+
resolve();
|
|
664
|
+
});
|
|
665
|
+
});
|
|
666
|
+
server.unref();
|
|
667
|
+
httpServer = server;
|
|
668
|
+
endpoint = {
|
|
669
|
+
origin: publicOrigin(server.address().port),
|
|
670
|
+
token: randomBytes(32).toString('base64url'),
|
|
671
|
+
};
|
|
672
|
+
return endpoint;
|
|
673
|
+
}
|
|
674
|
+
catch {
|
|
675
|
+
httpServer = undefined;
|
|
676
|
+
return undefined;
|
|
677
|
+
}
|
|
678
|
+
finally {
|
|
679
|
+
startPromise = undefined;
|
|
680
|
+
}
|
|
681
|
+
})();
|
|
682
|
+
return startPromise;
|
|
683
|
+
}
|
|
684
|
+
/** Current endpoint, or `undefined` when the gateway has not started. */
|
|
685
|
+
export function browserGatewayEndpoint() {
|
|
686
|
+
return endpoint;
|
|
687
|
+
}
|
|
688
|
+
/**
|
|
689
|
+
* Install the main-world audio interception before a navigation can create an
|
|
690
|
+
* AudioContext or fire a media element's play event. Capture remains muted
|
|
691
|
+
* until a client opens /audio. Failure is intentionally non-fatal: browser
|
|
692
|
+
* navigation and the screenshot stream must continue when audio is unavailable.
|
|
693
|
+
*/
|
|
694
|
+
export async function prepareBrowserAudioStream(sessionId) {
|
|
695
|
+
if (!streamEnabled() || !audioEnabled())
|
|
696
|
+
return;
|
|
697
|
+
try {
|
|
698
|
+
await prepareAudio(resolveChannel(sessionId));
|
|
699
|
+
}
|
|
700
|
+
catch {
|
|
701
|
+
// Audio is an optional live-view capability.
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
export async function shutdownBrowserGateway() {
|
|
705
|
+
for (const channel of [...channels.values()])
|
|
706
|
+
disposeChannel(channel);
|
|
707
|
+
const server = httpServer;
|
|
708
|
+
httpServer = undefined;
|
|
709
|
+
endpoint = undefined;
|
|
710
|
+
if (!server)
|
|
711
|
+
return;
|
|
712
|
+
await new Promise((resolve) => server.close(() => resolve()));
|
|
713
|
+
}
|
|
714
|
+
/** Exported for tests: host speaker output stays muted unless opted in. */
|
|
715
|
+
export function browserAudioEnabled() {
|
|
716
|
+
return enabledEnv('FLUJO_BROWSER_AUDIO');
|
|
717
|
+
}
|
|
718
|
+
/** Exported for tests: whether the page audio tap streams to the app. */
|
|
719
|
+
export function browserAudioStreamEnabled() {
|
|
720
|
+
return audioEnabled();
|
|
721
|
+
}
|
|
722
|
+
//# sourceMappingURL=gateway.js.map
|