neoagent 2.4.0 → 2.4.1-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +619 -21
- package/README.md +1 -1
- package/extensions/chrome-browser/background.mjs +19 -7
- package/extensions/chrome-browser/icons/icon128.png +0 -0
- package/extensions/chrome-browser/icons/icon16.png +0 -0
- package/extensions/chrome-browser/icons/icon48.png +0 -0
- package/extensions/chrome-browser/icons/logo.svg +12 -0
- package/extensions/chrome-browser/manifest.json +13 -2
- package/extensions/chrome-browser/popup.css +5 -0
- package/extensions/chrome-browser/popup.html +7 -5
- package/extensions/chrome-browser/popup.js +16 -7
- package/flutter_app/lib/features/onboarding/onboarding_companion_step.dart +391 -0
- package/flutter_app/lib/features/onboarding/onboarding_shell.dart +6 -0
- package/flutter_app/lib/features/onboarding/onboarding_welcome_step.dart +1 -1
- package/flutter_app/lib/main.dart +1 -0
- package/flutter_app/lib/main_controller.dart +156 -3
- package/flutter_app/lib/main_devices.dart +485 -119
- package/flutter_app/lib/main_settings.dart +289 -30
- package/flutter_app/lib/src/backend_client.dart +89 -0
- package/flutter_app/lib/src/desktop_companion_actions.dart +144 -0
- package/flutter_app/lib/src/desktop_companion_io.dart +145 -4
- package/flutter_app/lib/src/desktop_native_bridge.dart +13 -0
- package/flutter_app/lib/src/stream_renderer.dart +286 -0
- package/flutter_app/macos/Runner/AppDelegate.swift +56 -1
- package/package.json +2 -2
- package/server/guest_agent.js +19 -1
- package/server/http/routes.js +191 -0
- package/server/http/socket.js +1 -1
- package/server/index.js +4 -1
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/flutter_bootstrap.js +1 -1
- package/server/public/main.dart.js +73834 -72596
- package/server/routes/browser.js +14 -0
- package/server/routes/browser_extension.js +21 -4
- package/server/routes/desktop.js +10 -0
- package/server/routes/settings.js +4 -0
- package/server/routes/stream.js +187 -0
- package/server/services/ai/tools.js +40 -29
- package/server/services/android/controller.js +41 -2
- package/server/services/browser/controller.js +34 -0
- package/server/services/browser/extension/manifest.js +33 -0
- package/server/services/browser/extension/provider.js +12 -6
- package/server/services/browser/extension/registry.js +188 -18
- package/server/services/desktop/gateway.js +28 -3
- package/server/services/desktop/protocol.js +34 -0
- package/server/services/desktop/provider.js +25 -0
- package/server/services/desktop/registry.js +92 -10
- package/server/services/manager.js +19 -2
- package/server/services/runtime/backends/local-vm.js +6 -0
- package/server/services/runtime/docker-vm-manager.js +26 -3
- package/server/services/runtime/manager.js +36 -5
- package/server/services/runtime/settings.js +17 -0
- package/server/services/streaming/android-stream.js +298 -0
- package/server/services/streaming/browser-stream.js +87 -0
- package/server/services/streaming/stream-hub.js +231 -0
- package/server/services/websocket.js +73 -0
|
@@ -383,8 +383,14 @@ function createScreenRecorder(app) {
|
|
|
383
383
|
|
|
384
384
|
const extensionRegistry = app.locals.browserExtensionRegistry;
|
|
385
385
|
if (extensionRegistry?.connectionsByUser instanceof Map) {
|
|
386
|
-
for (const
|
|
387
|
-
if (
|
|
386
|
+
for (const value of extensionRegistry.connectionsByUser.values()) {
|
|
387
|
+
if (value instanceof Map) {
|
|
388
|
+
for (const connection of value.values()) {
|
|
389
|
+
if (typeof connection?.isOpen === 'function' && connection.isOpen()) {
|
|
390
|
+
return true;
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
} else if (typeof value?.isOpen === 'function' && value.isOpen()) {
|
|
388
394
|
return true;
|
|
389
395
|
}
|
|
390
396
|
}
|
|
@@ -438,6 +444,7 @@ function configureRealtime(app, io, services) {
|
|
|
438
444
|
recordingManager: services.recordingManager,
|
|
439
445
|
memoryManager: services.memoryManager,
|
|
440
446
|
voiceRuntimeManager: services.voiceRuntimeManager,
|
|
447
|
+
streamHub: app.locals.streamHub || services.streamHub || null,
|
|
441
448
|
app,
|
|
442
449
|
});
|
|
443
450
|
app.locals.io = io;
|
|
@@ -516,6 +523,7 @@ async function startServices(app, io) {
|
|
|
516
523
|
recordingManager,
|
|
517
524
|
memoryManager,
|
|
518
525
|
voiceRuntimeManager,
|
|
526
|
+
streamHub: app.locals.streamHub || null,
|
|
519
527
|
});
|
|
520
528
|
|
|
521
529
|
resumePendingRecordingSessions(recordingManager);
|
|
@@ -541,6 +549,15 @@ async function stopServices(app) {
|
|
|
541
549
|
}
|
|
542
550
|
}
|
|
543
551
|
|
|
552
|
+
if (app.locals.streamHub) {
|
|
553
|
+
try {
|
|
554
|
+
await app.locals.streamHub.shutdown();
|
|
555
|
+
logServiceReady('Stream hub stopped');
|
|
556
|
+
} catch (err) {
|
|
557
|
+
console.error('[StreamHub] Shutdown error:', getErrorMessage(err));
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
|
|
544
561
|
if (app.locals.memoryIngestionService) {
|
|
545
562
|
try {
|
|
546
563
|
app.locals.memoryIngestionService.stop();
|
|
@@ -253,6 +253,12 @@ class VmBrowserProvider {
|
|
|
253
253
|
extract(selector, attribute, all = false) { return this.client.request('POST', '/browser/extract', { selector, attribute, all }); }
|
|
254
254
|
evaluate(script) { return this.client.request('POST', '/browser/execute', { code: script }); }
|
|
255
255
|
async screenshot(options = {}) { return this.#materialize(await this.client.request('POST', '/browser/screenshot', options)); }
|
|
256
|
+
async screenshotJpeg(quality = 80, options = {}) {
|
|
257
|
+
const result = await this.client.request('POST', '/browser/screenshot-jpeg', { ...options, quality });
|
|
258
|
+
const content = String(result?.contentBase64 || '');
|
|
259
|
+
if (!content) throw new Error('VM browser screenshot-jpeg returned no data.');
|
|
260
|
+
return Buffer.from(content, 'base64');
|
|
261
|
+
}
|
|
256
262
|
launch(options = {}) { return this.client.request('POST', '/browser/launch', options); }
|
|
257
263
|
closeBrowser() { return this.client.request('POST', '/browser/close'); }
|
|
258
264
|
fill(selector, value) { return this.type(selector, value); }
|
|
@@ -102,17 +102,38 @@ const server = http.createServer(async (req, res) => {
|
|
|
102
102
|
|
|
103
103
|
// ── CLI execution ──────────────────────────────────────────────────────
|
|
104
104
|
if (req.method === 'POST' && url === '/exec') {
|
|
105
|
+
const timeoutMs = Math.min(Number(b.timeout) || 15 * 60 * 1000, 20 * 60 * 1000);
|
|
105
106
|
const child = spawn('sh', ['-c', b.command || 'true'], {
|
|
106
107
|
cwd: b.cwd || '/tmp',
|
|
107
108
|
env: { ...process.env, ...b.env },
|
|
108
109
|
});
|
|
109
110
|
const pid = child.pid;
|
|
110
|
-
let stdout = '', stderr = '';
|
|
111
|
+
let stdout = '', stderr = '', settled = false;
|
|
111
112
|
procs.set(pid, child);
|
|
112
113
|
child.stdout.on('data', d => { stdout += d; });
|
|
113
114
|
child.stderr.on('data', d => { stderr += d; });
|
|
114
|
-
|
|
115
|
-
|
|
115
|
+
const timer = setTimeout(() => {
|
|
116
|
+
if (settled) return;
|
|
117
|
+
settled = true;
|
|
118
|
+
procs.delete(pid);
|
|
119
|
+
try { child.kill('SIGKILL'); } catch {}
|
|
120
|
+
json(res, { stdout, stderr, exitCode: null, code: null, pid, timedOut: true, killed: true });
|
|
121
|
+
}, timeoutMs);
|
|
122
|
+
child.on('close', code => {
|
|
123
|
+
if (settled) return;
|
|
124
|
+
settled = true;
|
|
125
|
+
clearTimeout(timer);
|
|
126
|
+
procs.delete(pid);
|
|
127
|
+
const exitCode = code ?? 1;
|
|
128
|
+
json(res, { stdout, stderr, exitCode, code: exitCode, pid, timedOut: false, killed: false });
|
|
129
|
+
});
|
|
130
|
+
child.on('error', err => {
|
|
131
|
+
if (settled) return;
|
|
132
|
+
settled = true;
|
|
133
|
+
clearTimeout(timer);
|
|
134
|
+
procs.delete(pid);
|
|
135
|
+
json(res, { stdout, stderr, exitCode: 1, code: 1, pid, error: err.message, timedOut: false, killed: false });
|
|
136
|
+
});
|
|
116
137
|
return;
|
|
117
138
|
}
|
|
118
139
|
|
|
@@ -268,6 +289,7 @@ class DockerVMManager {
|
|
|
268
289
|
this.image = options.image || CONTAINER_IMAGE;
|
|
269
290
|
this.memoryMb = options.memoryMb || 2048;
|
|
270
291
|
this.cpus = options.cpus || 2;
|
|
292
|
+
this.guestToken = String(options.guestToken || process.env.NEOAGENT_VM_GUEST_TOKEN || '').trim();
|
|
271
293
|
this.#cleanupOrphans();
|
|
272
294
|
}
|
|
273
295
|
|
|
@@ -309,6 +331,7 @@ class DockerVMManager {
|
|
|
309
331
|
'--cpus', String(this.cpus),
|
|
310
332
|
'-p', `127.0.0.1:${port}:${port}`,
|
|
311
333
|
'-e', `AGENT_PORT=${port}`,
|
|
334
|
+
...(this.guestToken ? ['-e', `NEOAGENT_VM_GUEST_TOKEN=${this.guestToken}`] : []),
|
|
312
335
|
'--shm-size=2g',
|
|
313
336
|
'--security-opt', 'no-new-privileges',
|
|
314
337
|
'--label', CONTAINER_LABEL,
|
|
@@ -31,10 +31,11 @@ class RuntimeManager {
|
|
|
31
31
|
this.artifactStore = options.artifactStore || null;
|
|
32
32
|
|
|
33
33
|
this.getExtensionBrowserProvider = options.getExtensionBrowserProvider
|
|
34
|
-
|| ((userId) => new ExtensionBrowserProvider({
|
|
34
|
+
|| ((userId, providerOptions = {}) => new ExtensionBrowserProvider({
|
|
35
35
|
registry: options.browserExtensionRegistry,
|
|
36
36
|
artifactStore: options.artifactStore,
|
|
37
37
|
userId,
|
|
38
|
+
tokenId: providerOptions.tokenId || null,
|
|
38
39
|
}));
|
|
39
40
|
|
|
40
41
|
this.getDesktopCliProvider = options.getDesktopCliProvider
|
|
@@ -51,10 +52,11 @@ class RuntimeManager {
|
|
|
51
52
|
}
|
|
52
53
|
|
|
53
54
|
hasActiveExtensionBrowser(userId) {
|
|
55
|
+
const settings = this.getSettings(userId);
|
|
54
56
|
return Boolean(
|
|
55
57
|
this.browserExtensionRegistry
|
|
56
58
|
&& typeof this.browserExtensionRegistry.isConnected === 'function'
|
|
57
|
-
&& this.browserExtensionRegistry.isConnected(userId)
|
|
59
|
+
&& this.browserExtensionRegistry.isConnected(userId, settings.browser_extension_token_id)
|
|
58
60
|
);
|
|
59
61
|
}
|
|
60
62
|
|
|
@@ -92,7 +94,9 @@ class RuntimeManager {
|
|
|
92
94
|
async getBrowserProviderForUser(userId) {
|
|
93
95
|
const settings = this.getSettings(userId);
|
|
94
96
|
if (settings.browser_backend === 'extension' && this.hasActiveExtensionBrowser(userId)) {
|
|
95
|
-
return this.getExtensionBrowserProvider(userId
|
|
97
|
+
return this.getExtensionBrowserProvider(userId, {
|
|
98
|
+
tokenId: settings.browser_extension_token_id,
|
|
99
|
+
});
|
|
96
100
|
}
|
|
97
101
|
return this.browserBackend.getBrowserProviderForUser(userId);
|
|
98
102
|
}
|
|
@@ -100,9 +104,36 @@ class RuntimeManager {
|
|
|
100
104
|
async getCliProviderForUser(userId) {
|
|
101
105
|
const settings = this.getSettings(userId);
|
|
102
106
|
if (settings.cli_backend === 'desktop' && this.hasActiveDesktopCompanion(userId)) {
|
|
103
|
-
|
|
107
|
+
const desktopProvider = this.getDesktopCliProvider(userId);
|
|
108
|
+
const preferredDeviceId = settings.cli_desktop_device_id;
|
|
109
|
+
if (preferredDeviceId) {
|
|
110
|
+
desktopProvider.selectDevice(preferredDeviceId);
|
|
111
|
+
}
|
|
112
|
+
return {
|
|
113
|
+
backend: 'desktop-companion',
|
|
114
|
+
execute: (command, options = {}) => desktopProvider.executeCommand(command, options),
|
|
115
|
+
executeInteractive: (command, inputs = [], options = {}) => desktopProvider.executeCommand(command, { ...options, inputs }),
|
|
116
|
+
kill: () => Promise.resolve(false),
|
|
117
|
+
};
|
|
104
118
|
}
|
|
105
|
-
|
|
119
|
+
const executor = await this.browserBackend.getCommandExecutorForUser(userId);
|
|
120
|
+
return { ...executor, backend: 'vm' };
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
async executeCliCommand(userId, command, options = {}) {
|
|
124
|
+
const provider = await this.getCliProviderForUser(userId);
|
|
125
|
+
const result = await (options.pty === true && provider.executeInteractive
|
|
126
|
+
? provider.executeInteractive(command, options.inputs || [], options)
|
|
127
|
+
: provider.execute(command, options));
|
|
128
|
+
return { ...result, backend: provider.backend };
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
getActiveBrowserBackend(userId) {
|
|
132
|
+
const settings = this.getSettings(userId);
|
|
133
|
+
if (settings.browser_backend === 'extension' && this.hasActiveExtensionBrowser(userId)) {
|
|
134
|
+
return 'extension';
|
|
135
|
+
}
|
|
136
|
+
return 'vm';
|
|
106
137
|
}
|
|
107
138
|
|
|
108
139
|
async getAndroidProviderForUser(userId) {
|
|
@@ -10,6 +10,9 @@ function createDefaultRuntimeSettings() {
|
|
|
10
10
|
android_backend: policy.runtimeDefaults.android_backend,
|
|
11
11
|
mcp_backend: policy.runtimeDefaults.mcp_backend,
|
|
12
12
|
cli_backend: policy.runtimeDefaults.cli_backend ?? 'vm',
|
|
13
|
+
browser_extension_token_id: null,
|
|
14
|
+
selected_browser_extension_token_id: null,
|
|
15
|
+
cli_desktop_device_id: null,
|
|
13
16
|
};
|
|
14
17
|
}
|
|
15
18
|
|
|
@@ -26,6 +29,9 @@ const BASE_FALLBACK_SETTINGS = Object.freeze({
|
|
|
26
29
|
android_backend: 'host',
|
|
27
30
|
mcp_backend: 'host-remote',
|
|
28
31
|
cli_backend: 'vm',
|
|
32
|
+
browser_extension_token_id: null,
|
|
33
|
+
selected_browser_extension_token_id: null,
|
|
34
|
+
cli_desktop_device_id: null,
|
|
29
35
|
});
|
|
30
36
|
|
|
31
37
|
const RUNTIME_SETTING_KEYS = Object.freeze(Object.keys(DEFAULT_RUNTIME_SETTINGS));
|
|
@@ -35,6 +41,11 @@ function normalizeChoice(value, allowed, fallback) {
|
|
|
35
41
|
return allowed.includes(normalized) ? normalized : fallback;
|
|
36
42
|
}
|
|
37
43
|
|
|
44
|
+
function normalizeOptionalString(value) {
|
|
45
|
+
const normalized = String(value || '').trim();
|
|
46
|
+
return normalized || null;
|
|
47
|
+
}
|
|
48
|
+
|
|
38
49
|
function deriveDefaultsForProfile(profile) {
|
|
39
50
|
switch (profile) {
|
|
40
51
|
case 'secure-vm':
|
|
@@ -57,6 +68,9 @@ function normalizeRuntimeSettings(raw = {}) {
|
|
|
57
68
|
const runtimeBackend = normalizeChoice(raw.runtime_backend, ['vm'], derived.runtime_backend);
|
|
58
69
|
const browserBackend = normalizeChoice(raw.browser_backend, ['vm', 'extension'], derived.browser_backend);
|
|
59
70
|
const cliBackend = normalizeChoice(raw.cli_backend, ['vm', 'desktop'], derived.cli_backend ?? 'vm');
|
|
71
|
+
const selectedExtensionTokenId = normalizeOptionalString(
|
|
72
|
+
raw.browser_extension_token_id || raw.selected_browser_extension_token_id,
|
|
73
|
+
);
|
|
60
74
|
const androidBackend = 'host';
|
|
61
75
|
return {
|
|
62
76
|
runtime_profile: profile === 'trusted-host' ? 'secure-vm' : profile,
|
|
@@ -65,6 +79,9 @@ function normalizeRuntimeSettings(raw = {}) {
|
|
|
65
79
|
android_backend: androidBackend,
|
|
66
80
|
mcp_backend: 'host-remote',
|
|
67
81
|
cli_backend: cliBackend === 'desktop' ? 'desktop' : 'vm',
|
|
82
|
+
browser_extension_token_id: selectedExtensionTokenId,
|
|
83
|
+
selected_browser_extension_token_id: selectedExtensionTokenId,
|
|
84
|
+
cli_desktop_device_id: normalizeOptionalString(raw.cli_desktop_device_id),
|
|
68
85
|
};
|
|
69
86
|
}
|
|
70
87
|
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const path = require('path');
|
|
4
|
+
const { spawn } = require('child_process');
|
|
5
|
+
const { EventEmitter } = require('events');
|
|
6
|
+
const sharp = require('sharp');
|
|
7
|
+
|
|
8
|
+
// Derive the full path to the `adb` binary the same way the Android controller does.
|
|
9
|
+
function resolveAdbBin(sdkDir) {
|
|
10
|
+
if (sdkDir) {
|
|
11
|
+
return path.join(sdkDir, 'platform-tools', process.platform === 'win32' ? 'adb.exe' : 'adb');
|
|
12
|
+
}
|
|
13
|
+
return process.platform === 'win32' ? 'adb.exe' : 'adb'; // fall back to PATH
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function clampInt(value, fallback, min, max) {
|
|
17
|
+
const parsed = Number(value);
|
|
18
|
+
if (!Number.isFinite(parsed)) return fallback;
|
|
19
|
+
return Math.min(max, Math.max(min, Math.floor(parsed)));
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// ---------------------------------------------------------------------------
|
|
23
|
+
// JpegFrameParser — extracts complete JPEG images from a raw byte stream.
|
|
24
|
+
// ffmpeg writes MJPEG to stdout as a continuous stream of JPEG frames, each
|
|
25
|
+
// delimited by SOI (FF D8) and EOI (FF D9) markers. We buffer incoming
|
|
26
|
+
// chunks and emit a 'frame' event for every complete JPEG found.
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
class JpegFrameParser extends EventEmitter {
|
|
29
|
+
constructor() {
|
|
30
|
+
super();
|
|
31
|
+
this._buf = null; // Buffer | null
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
push(chunk) {
|
|
35
|
+
if (!chunk?.length) return;
|
|
36
|
+
this._buf = this._buf ? Buffer.concat([this._buf, chunk]) : Buffer.from(chunk);
|
|
37
|
+
let start = 0;
|
|
38
|
+
|
|
39
|
+
while (start < this._buf.length - 1) {
|
|
40
|
+
// Locate SOI marker (0xFF 0xD8).
|
|
41
|
+
let soiIdx = -1;
|
|
42
|
+
for (let i = start; i < this._buf.length - 1; i++) {
|
|
43
|
+
if (this._buf[i] === 0xff && this._buf[i + 1] === 0xd8) {
|
|
44
|
+
soiIdx = i;
|
|
45
|
+
break;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
if (soiIdx === -1) {
|
|
49
|
+
// No SOI found – discard everything.
|
|
50
|
+
this._buf = null;
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Locate EOI marker (0xFF 0xD9) that comes after SOI.
|
|
55
|
+
let eoiIdx = -1;
|
|
56
|
+
for (let i = soiIdx + 2; i < this._buf.length - 1; i++) {
|
|
57
|
+
if (this._buf[i] === 0xff && this._buf[i + 1] === 0xd9) {
|
|
58
|
+
eoiIdx = i;
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
if (eoiIdx === -1) {
|
|
63
|
+
// SOI found but no matching EOI yet – keep buffered from SOI onward.
|
|
64
|
+
this._buf = soiIdx > 0 ? this._buf.slice(soiIdx) : this._buf;
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Emit the complete JPEG (inclusive of both markers).
|
|
69
|
+
this.emit('frame', this._buf.slice(soiIdx, eoiIdx + 2));
|
|
70
|
+
start = eoiIdx + 2;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// Retain any leftover bytes that haven't formed a complete frame yet.
|
|
74
|
+
this._buf = start < this._buf.length ? this._buf.slice(start) : null;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// ---------------------------------------------------------------------------
|
|
79
|
+
// AndroidStream
|
|
80
|
+
//
|
|
81
|
+
// Streams the Android emulator screen by:
|
|
82
|
+
// 1. Running `adb exec-out screenrecord --output-format=h264 -` to get a
|
|
83
|
+
// continuous H.264 bitstream at up to 30 fps with no per-frame process
|
|
84
|
+
// overhead (replacing the previous one-shot screencap-per-frame approach).
|
|
85
|
+
// 2. Piping the bitstream through `ffmpeg` which decodes H.264 and re-encodes
|
|
86
|
+
// each frame as MJPEG at the requested fps / quality.
|
|
87
|
+
// 3. Parsing complete JPEG frames from ffmpeg stdout via JpegFrameParser and
|
|
88
|
+
// forwarding them to StreamHub.
|
|
89
|
+
// 4. Auto-restarting when Android's built-in 3-minute screenrecord limit
|
|
90
|
+
// expires (~170 s to give a small safety margin).
|
|
91
|
+
// 5. Falling back to screencap polling when H.264 streaming cannot start
|
|
92
|
+
// (e.g. older device without screenrecord stdout support).
|
|
93
|
+
// ---------------------------------------------------------------------------
|
|
94
|
+
class AndroidStream {
|
|
95
|
+
constructor({ userId, deviceId, controller, streamHub, fps = 12, quality = 75 }) {
|
|
96
|
+
this.userId = String(userId || '');
|
|
97
|
+
this.deviceId = String(deviceId || '');
|
|
98
|
+
this.controller = controller;
|
|
99
|
+
this.streamHub = streamHub;
|
|
100
|
+
this.fps = clampInt(fps, 12, 1, 30);
|
|
101
|
+
this.quality = clampInt(quality, 75, 30, 95);
|
|
102
|
+
|
|
103
|
+
this._stopped = true;
|
|
104
|
+
this._adbProc = null;
|
|
105
|
+
this._ffmpegProc = null;
|
|
106
|
+
this._restartTimer = null;
|
|
107
|
+
this._seq = 0;
|
|
108
|
+
this._lastErrorLogAt = 0;
|
|
109
|
+
this._usePollingFallback = false; // set true when H.264 fails to start
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
start() {
|
|
113
|
+
if (!this._stopped) return;
|
|
114
|
+
this._stopped = false;
|
|
115
|
+
if (this._usePollingFallback) {
|
|
116
|
+
this._startPollingFallback();
|
|
117
|
+
} else {
|
|
118
|
+
this._launchH264();
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
stop() {
|
|
123
|
+
this._stopped = true;
|
|
124
|
+
this._killProcesses();
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// ── H.264 streaming (primary path) ─────────────────────────────────────
|
|
128
|
+
|
|
129
|
+
_launchH264() {
|
|
130
|
+
if (this._stopped) return;
|
|
131
|
+
|
|
132
|
+
const adb = resolveAdbBin(this.controller?.sdkDir);
|
|
133
|
+
// deviceId is the ADB serial (emulator-5554, etc.)
|
|
134
|
+
const serial = this.deviceId;
|
|
135
|
+
const adbArgs = [
|
|
136
|
+
'-s', serial,
|
|
137
|
+
'exec-out',
|
|
138
|
+
'screenrecord',
|
|
139
|
+
'--output-format=h264',
|
|
140
|
+
'--bit-rate=2000000',
|
|
141
|
+
'--size=1280x720',
|
|
142
|
+
'-',
|
|
143
|
+
];
|
|
144
|
+
|
|
145
|
+
try {
|
|
146
|
+
this._adbProc = spawn(adb, adbArgs, { stdio: ['ignore', 'pipe', 'ignore'] });
|
|
147
|
+
} catch (err) {
|
|
148
|
+
this._logError('Failed to spawn adb for H.264 streaming, falling back to screencap', err);
|
|
149
|
+
this._fallback();
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// Map our 30–95 quality range to ffmpeg q:v 2–20 (lower = better quality).
|
|
154
|
+
const ffmpegQ = Math.round(2 + ((95 - this.quality) / (95 - 30)) * 18);
|
|
155
|
+
const ffmpegArgs = [
|
|
156
|
+
'-loglevel', 'error',
|
|
157
|
+
'-f', 'h264',
|
|
158
|
+
'-i', 'pipe:0',
|
|
159
|
+
'-f', 'image2pipe',
|
|
160
|
+
'-vcodec', 'mjpeg',
|
|
161
|
+
'-q:v', String(ffmpegQ),
|
|
162
|
+
'-vf', `fps=${this.fps}`,
|
|
163
|
+
'pipe:1',
|
|
164
|
+
];
|
|
165
|
+
|
|
166
|
+
try {
|
|
167
|
+
this._ffmpegProc = spawn('ffmpeg', ffmpegArgs, { stdio: ['pipe', 'pipe', 'ignore'] });
|
|
168
|
+
} catch (err) {
|
|
169
|
+
this._logError('ffmpeg not found, falling back to screencap', err);
|
|
170
|
+
try { this._adbProc?.kill('SIGTERM'); } catch {}
|
|
171
|
+
this._adbProc = null;
|
|
172
|
+
this._fallback();
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// Wire adb stdout → ffmpeg stdin.
|
|
177
|
+
this._adbProc.stdout.pipe(this._ffmpegProc.stdin);
|
|
178
|
+
|
|
179
|
+
// Parse JPEG frames from ffmpeg stdout and forward to StreamHub.
|
|
180
|
+
const parser = new JpegFrameParser();
|
|
181
|
+
this._ffmpegProc.stdout.on('data', (chunk) => parser.push(chunk));
|
|
182
|
+
parser.on('frame', (jpeg) => {
|
|
183
|
+
if (this._stopped) return;
|
|
184
|
+
this.streamHub.handleFrame(this.userId, this.deviceId, {
|
|
185
|
+
jpeg,
|
|
186
|
+
platform: 'android',
|
|
187
|
+
seq: this._seq++ >>> 0,
|
|
188
|
+
ts: Date.now() >>> 0,
|
|
189
|
+
flags: 1,
|
|
190
|
+
});
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
this._adbProc.on('error', (err) => {
|
|
194
|
+
this._logError('adb process error', err);
|
|
195
|
+
});
|
|
196
|
+
this._ffmpegProc.on('error', (err) => {
|
|
197
|
+
this._logError('ffmpeg process error', err);
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
// Android screenrecord's hard limit is 180 s — restart at 170 s so there
|
|
201
|
+
// is no gap in the stream.
|
|
202
|
+
this._restartTimer = setTimeout(() => {
|
|
203
|
+
if (!this._stopped) {
|
|
204
|
+
this._killProcesses();
|
|
205
|
+
this._launchH264();
|
|
206
|
+
}
|
|
207
|
+
}, 170_000);
|
|
208
|
+
this._restartTimer.unref?.();
|
|
209
|
+
|
|
210
|
+
// If adb exits early (e.g. emulator restart), attempt recovery.
|
|
211
|
+
this._adbProc.on('close', (code) => {
|
|
212
|
+
if (this._stopped) return;
|
|
213
|
+
// If it exited immediately with a bad code, the device likely does not
|
|
214
|
+
// support stdout screenrecord — fall back to screencap polling.
|
|
215
|
+
if (code !== 0 && this._seq === 0) {
|
|
216
|
+
this._logError(`screenrecord exited (code ${code}) before producing any frames — using screencap fallback`);
|
|
217
|
+
this._killProcesses();
|
|
218
|
+
this._fallback();
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
this._logError(`adb screenrecord exited (code ${code}), restarting in 2 s`);
|
|
222
|
+
this._killProcesses();
|
|
223
|
+
this._scheduleRestart(2000, () => this._launchH264());
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
_fallback() {
|
|
228
|
+
this._usePollingFallback = true;
|
|
229
|
+
if (!this._stopped) this._startPollingFallback();
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// ── Screencap polling fallback ──────────────────────────────────────────
|
|
233
|
+
// Used when H.264 streaming is unavailable. Runs a tight continuous loop
|
|
234
|
+
// (no fixed interval) so there is never idle time waiting between captures.
|
|
235
|
+
|
|
236
|
+
_startPollingFallback() {
|
|
237
|
+
// Fire and forget — the loop runs until this._stopped is set.
|
|
238
|
+
void this._pollLoop();
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
async _pollLoop() {
|
|
242
|
+
while (!this._stopped) {
|
|
243
|
+
try {
|
|
244
|
+
if (!this.controller || typeof this.controller.capturePng !== 'function') {
|
|
245
|
+
throw new Error('Android streaming requires a controller with capturePng().');
|
|
246
|
+
}
|
|
247
|
+
const png = await this.controller.capturePng({ deviceId: this.deviceId });
|
|
248
|
+
if (this._stopped || !png?.length) continue;
|
|
249
|
+
const jpeg = await sharp(png)
|
|
250
|
+
.jpeg({ quality: this.quality, mozjpeg: false })
|
|
251
|
+
.toBuffer();
|
|
252
|
+
if (this._stopped) return;
|
|
253
|
+
this.streamHub.handleFrame(this.userId, this.deviceId, {
|
|
254
|
+
jpeg,
|
|
255
|
+
platform: 'android',
|
|
256
|
+
seq: this._seq++ >>> 0,
|
|
257
|
+
ts: Date.now() >>> 0,
|
|
258
|
+
flags: 1,
|
|
259
|
+
});
|
|
260
|
+
} catch (error) {
|
|
261
|
+
this._logError('screencap poll failed', error);
|
|
262
|
+
// Brief pause before retrying to avoid a tight spin on persistent errors.
|
|
263
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// ── Helpers ─────────────────────────────────────────────────────────────
|
|
269
|
+
|
|
270
|
+
_scheduleRestart(delayMs, fn) {
|
|
271
|
+
this._restartTimer = setTimeout(fn, delayMs);
|
|
272
|
+
this._restartTimer.unref?.();
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
_killProcesses() {
|
|
276
|
+
clearTimeout(this._restartTimer);
|
|
277
|
+
this._restartTimer = null;
|
|
278
|
+
// Kill ffmpeg first so it stops reading; then kill adb.
|
|
279
|
+
try { this._ffmpegProc?.kill('SIGTERM'); } catch {}
|
|
280
|
+
try { this._adbProc?.kill('SIGTERM'); } catch {}
|
|
281
|
+
this._ffmpegProc = null;
|
|
282
|
+
this._adbProc = null;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
_logError(msg, err) {
|
|
286
|
+
const now = Date.now();
|
|
287
|
+
if (now - this._lastErrorLogAt > 10_000) {
|
|
288
|
+
this._lastErrorLogAt = now;
|
|
289
|
+
console.warn(`[AndroidStream] ${msg}`, {
|
|
290
|
+
userId: this.userId,
|
|
291
|
+
deviceId: this.deviceId,
|
|
292
|
+
error: err ? String(err?.message || err) : undefined,
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
module.exports = { AndroidStream };
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const sharp = require('sharp');
|
|
5
|
+
|
|
6
|
+
function clampInt(value, fallback, min, max) {
|
|
7
|
+
const parsed = Number(value);
|
|
8
|
+
if (!Number.isFinite(parsed)) return fallback;
|
|
9
|
+
return Math.min(max, Math.max(min, Math.floor(parsed)));
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
class BrowserStream {
|
|
13
|
+
constructor({ userId, deviceId = 'browser', controller, streamHub, fps = 15, quality = 80 }) {
|
|
14
|
+
this.userId = String(userId || '');
|
|
15
|
+
this.deviceId = String(deviceId || 'browser');
|
|
16
|
+
this.controller = controller;
|
|
17
|
+
this.streamHub = streamHub;
|
|
18
|
+
this.fps = clampInt(fps, 15, 1, 20);
|
|
19
|
+
this.quality = clampInt(quality, 80, 30, 95);
|
|
20
|
+
this._timer = null;
|
|
21
|
+
this._capturing = false;
|
|
22
|
+
this._seq = 0;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
start() {
|
|
26
|
+
if (this._timer) return;
|
|
27
|
+
const interval = Math.max(1, Math.floor(1000 / this.fps));
|
|
28
|
+
this._timer = setInterval(() => {
|
|
29
|
+
void this._captureOnce();
|
|
30
|
+
}, interval);
|
|
31
|
+
this._timer.unref?.();
|
|
32
|
+
void this._captureOnce();
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
stop() {
|
|
36
|
+
if (this._timer) {
|
|
37
|
+
clearInterval(this._timer);
|
|
38
|
+
this._timer = null;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async _captureOnce() {
|
|
43
|
+
if (this._capturing) return;
|
|
44
|
+
this._capturing = true;
|
|
45
|
+
try {
|
|
46
|
+
const jpeg = await this._captureJpeg();
|
|
47
|
+
if (!jpeg?.length) return;
|
|
48
|
+
this.streamHub.handleFrame(this.userId, this.deviceId, {
|
|
49
|
+
jpeg,
|
|
50
|
+
platform: 'browser',
|
|
51
|
+
seq: this._seq++ >>> 0,
|
|
52
|
+
ts: Date.now() >>> 0,
|
|
53
|
+
flags: 1,
|
|
54
|
+
});
|
|
55
|
+
} catch (error) {
|
|
56
|
+
console.warn('[BrowserStream] frame capture failed', {
|
|
57
|
+
userId: this.userId,
|
|
58
|
+
deviceId: this.deviceId,
|
|
59
|
+
error: String(error?.message || error),
|
|
60
|
+
});
|
|
61
|
+
} finally {
|
|
62
|
+
this._capturing = false;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
async _captureJpeg() {
|
|
67
|
+
if (!this.controller) {
|
|
68
|
+
throw new Error('Browser controller is unavailable.');
|
|
69
|
+
}
|
|
70
|
+
if (typeof this.controller.screenshotJpeg === 'function') {
|
|
71
|
+
return this.controller.screenshotJpeg(this.quality);
|
|
72
|
+
}
|
|
73
|
+
if (typeof this.controller.screenshot !== 'function') {
|
|
74
|
+
throw new Error('Browser streaming requires a screenshot-capable controller.');
|
|
75
|
+
}
|
|
76
|
+
const result = await this.controller.screenshot({ fullPage: false });
|
|
77
|
+
if (!result?.fullPath || !fs.existsSync(result.fullPath)) {
|
|
78
|
+
throw new Error('Browser screenshot did not produce a readable file.');
|
|
79
|
+
}
|
|
80
|
+
const png = fs.readFileSync(result.fullPath);
|
|
81
|
+
return sharp(png).jpeg({ quality: this.quality }).toBuffer();
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
module.exports = {
|
|
86
|
+
BrowserStream,
|
|
87
|
+
};
|