neoagent 2.4.0 → 2.4.1-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/LICENSE +619 -21
  2. package/README.md +1 -1
  3. package/extensions/chrome-browser/background.mjs +19 -7
  4. package/extensions/chrome-browser/icons/icon128.png +0 -0
  5. package/extensions/chrome-browser/icons/icon16.png +0 -0
  6. package/extensions/chrome-browser/icons/icon48.png +0 -0
  7. package/extensions/chrome-browser/icons/logo.svg +12 -0
  8. package/extensions/chrome-browser/manifest.json +13 -2
  9. package/extensions/chrome-browser/popup.css +5 -0
  10. package/extensions/chrome-browser/popup.html +7 -5
  11. package/extensions/chrome-browser/popup.js +16 -7
  12. package/flutter_app/lib/features/onboarding/onboarding_companion_step.dart +391 -0
  13. package/flutter_app/lib/features/onboarding/onboarding_shell.dart +6 -0
  14. package/flutter_app/lib/features/onboarding/onboarding_welcome_step.dart +1 -1
  15. package/flutter_app/lib/main.dart +1 -0
  16. package/flutter_app/lib/main_controller.dart +156 -3
  17. package/flutter_app/lib/main_devices.dart +485 -119
  18. package/flutter_app/lib/main_settings.dart +289 -30
  19. package/flutter_app/lib/src/backend_client.dart +89 -0
  20. package/flutter_app/lib/src/desktop_companion_actions.dart +144 -0
  21. package/flutter_app/lib/src/desktop_companion_io.dart +145 -4
  22. package/flutter_app/lib/src/desktop_native_bridge.dart +13 -0
  23. package/flutter_app/lib/src/stream_renderer.dart +286 -0
  24. package/flutter_app/macos/Runner/AppDelegate.swift +56 -1
  25. package/package.json +2 -2
  26. package/server/guest_agent.js +19 -1
  27. package/server/http/routes.js +191 -0
  28. package/server/http/socket.js +1 -1
  29. package/server/index.js +4 -1
  30. package/server/public/.last_build_id +1 -1
  31. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  32. package/server/public/flutter_bootstrap.js +1 -1
  33. package/server/public/main.dart.js +73834 -72596
  34. package/server/routes/browser.js +14 -0
  35. package/server/routes/browser_extension.js +21 -4
  36. package/server/routes/desktop.js +10 -0
  37. package/server/routes/settings.js +4 -0
  38. package/server/routes/stream.js +187 -0
  39. package/server/services/ai/tools.js +40 -29
  40. package/server/services/android/controller.js +41 -2
  41. package/server/services/browser/controller.js +34 -0
  42. package/server/services/browser/extension/manifest.js +33 -0
  43. package/server/services/browser/extension/provider.js +12 -6
  44. package/server/services/browser/extension/registry.js +188 -18
  45. package/server/services/desktop/gateway.js +28 -3
  46. package/server/services/desktop/protocol.js +34 -0
  47. package/server/services/desktop/provider.js +25 -0
  48. package/server/services/desktop/registry.js +92 -10
  49. package/server/services/manager.js +19 -2
  50. package/server/services/runtime/backends/local-vm.js +6 -0
  51. package/server/services/runtime/docker-vm-manager.js +26 -3
  52. package/server/services/runtime/manager.js +36 -5
  53. package/server/services/runtime/settings.js +17 -0
  54. package/server/services/streaming/android-stream.js +298 -0
  55. package/server/services/streaming/browser-stream.js +87 -0
  56. package/server/services/streaming/stream-hub.js +231 -0
  57. package/server/services/websocket.js +73 -0
@@ -383,8 +383,14 @@ function createScreenRecorder(app) {
383
383
 
384
384
  const extensionRegistry = app.locals.browserExtensionRegistry;
385
385
  if (extensionRegistry?.connectionsByUser instanceof Map) {
386
- for (const connection of extensionRegistry.connectionsByUser.values()) {
387
- if (typeof connection?.isOpen === 'function' && connection.isOpen()) {
386
+ for (const value of extensionRegistry.connectionsByUser.values()) {
387
+ if (value instanceof Map) {
388
+ for (const connection of value.values()) {
389
+ if (typeof connection?.isOpen === 'function' && connection.isOpen()) {
390
+ return true;
391
+ }
392
+ }
393
+ } else if (typeof value?.isOpen === 'function' && value.isOpen()) {
388
394
  return true;
389
395
  }
390
396
  }
@@ -438,6 +444,7 @@ function configureRealtime(app, io, services) {
438
444
  recordingManager: services.recordingManager,
439
445
  memoryManager: services.memoryManager,
440
446
  voiceRuntimeManager: services.voiceRuntimeManager,
447
+ streamHub: app.locals.streamHub || services.streamHub || null,
441
448
  app,
442
449
  });
443
450
  app.locals.io = io;
@@ -516,6 +523,7 @@ async function startServices(app, io) {
516
523
  recordingManager,
517
524
  memoryManager,
518
525
  voiceRuntimeManager,
526
+ streamHub: app.locals.streamHub || null,
519
527
  });
520
528
 
521
529
  resumePendingRecordingSessions(recordingManager);
@@ -541,6 +549,15 @@ async function stopServices(app) {
541
549
  }
542
550
  }
543
551
 
552
+ if (app.locals.streamHub) {
553
+ try {
554
+ await app.locals.streamHub.shutdown();
555
+ logServiceReady('Stream hub stopped');
556
+ } catch (err) {
557
+ console.error('[StreamHub] Shutdown error:', getErrorMessage(err));
558
+ }
559
+ }
560
+
544
561
  if (app.locals.memoryIngestionService) {
545
562
  try {
546
563
  app.locals.memoryIngestionService.stop();
@@ -253,6 +253,12 @@ class VmBrowserProvider {
253
253
  extract(selector, attribute, all = false) { return this.client.request('POST', '/browser/extract', { selector, attribute, all }); }
254
254
  evaluate(script) { return this.client.request('POST', '/browser/execute', { code: script }); }
255
255
  async screenshot(options = {}) { return this.#materialize(await this.client.request('POST', '/browser/screenshot', options)); }
256
+ async screenshotJpeg(quality = 80, options = {}) {
257
+ const result = await this.client.request('POST', '/browser/screenshot-jpeg', { ...options, quality });
258
+ const content = String(result?.contentBase64 || '');
259
+ if (!content) throw new Error('VM browser screenshot-jpeg returned no data.');
260
+ return Buffer.from(content, 'base64');
261
+ }
256
262
  launch(options = {}) { return this.client.request('POST', '/browser/launch', options); }
257
263
  closeBrowser() { return this.client.request('POST', '/browser/close'); }
258
264
  fill(selector, value) { return this.type(selector, value); }
@@ -102,17 +102,38 @@ const server = http.createServer(async (req, res) => {
102
102
 
103
103
  // ── CLI execution ──────────────────────────────────────────────────────
104
104
  if (req.method === 'POST' && url === '/exec') {
105
+ const timeoutMs = Math.min(Number(b.timeout) || 15 * 60 * 1000, 20 * 60 * 1000);
105
106
  const child = spawn('sh', ['-c', b.command || 'true'], {
106
107
  cwd: b.cwd || '/tmp',
107
108
  env: { ...process.env, ...b.env },
108
109
  });
109
110
  const pid = child.pid;
110
- let stdout = '', stderr = '';
111
+ let stdout = '', stderr = '', settled = false;
111
112
  procs.set(pid, child);
112
113
  child.stdout.on('data', d => { stdout += d; });
113
114
  child.stderr.on('data', d => { stderr += d; });
114
- child.on('close', code => { procs.delete(pid); json(res, { stdout, stderr, code: code ?? 1, pid }); });
115
- child.on('error', err => { procs.delete(pid); json(res, { stdout, stderr, code: 1, pid, error: err.message }); });
115
+ const timer = setTimeout(() => {
116
+ if (settled) return;
117
+ settled = true;
118
+ procs.delete(pid);
119
+ try { child.kill('SIGKILL'); } catch {}
120
+ json(res, { stdout, stderr, exitCode: null, code: null, pid, timedOut: true, killed: true });
121
+ }, timeoutMs);
122
+ child.on('close', code => {
123
+ if (settled) return;
124
+ settled = true;
125
+ clearTimeout(timer);
126
+ procs.delete(pid);
127
+ const exitCode = code ?? 1;
128
+ json(res, { stdout, stderr, exitCode, code: exitCode, pid, timedOut: false, killed: false });
129
+ });
130
+ child.on('error', err => {
131
+ if (settled) return;
132
+ settled = true;
133
+ clearTimeout(timer);
134
+ procs.delete(pid);
135
+ json(res, { stdout, stderr, exitCode: 1, code: 1, pid, error: err.message, timedOut: false, killed: false });
136
+ });
116
137
  return;
117
138
  }
118
139
 
@@ -268,6 +289,7 @@ class DockerVMManager {
268
289
  this.image = options.image || CONTAINER_IMAGE;
269
290
  this.memoryMb = options.memoryMb || 2048;
270
291
  this.cpus = options.cpus || 2;
292
+ this.guestToken = String(options.guestToken || process.env.NEOAGENT_VM_GUEST_TOKEN || '').trim();
271
293
  this.#cleanupOrphans();
272
294
  }
273
295
 
@@ -309,6 +331,7 @@ class DockerVMManager {
309
331
  '--cpus', String(this.cpus),
310
332
  '-p', `127.0.0.1:${port}:${port}`,
311
333
  '-e', `AGENT_PORT=${port}`,
334
+ ...(this.guestToken ? ['-e', `NEOAGENT_VM_GUEST_TOKEN=${this.guestToken}`] : []),
312
335
  '--shm-size=2g',
313
336
  '--security-opt', 'no-new-privileges',
314
337
  '--label', CONTAINER_LABEL,
@@ -31,10 +31,11 @@ class RuntimeManager {
31
31
  this.artifactStore = options.artifactStore || null;
32
32
 
33
33
  this.getExtensionBrowserProvider = options.getExtensionBrowserProvider
34
- || ((userId) => new ExtensionBrowserProvider({
34
+ || ((userId, providerOptions = {}) => new ExtensionBrowserProvider({
35
35
  registry: options.browserExtensionRegistry,
36
36
  artifactStore: options.artifactStore,
37
37
  userId,
38
+ tokenId: providerOptions.tokenId || null,
38
39
  }));
39
40
 
40
41
  this.getDesktopCliProvider = options.getDesktopCliProvider
@@ -51,10 +52,11 @@ class RuntimeManager {
51
52
  }
52
53
 
53
54
  hasActiveExtensionBrowser(userId) {
55
+ const settings = this.getSettings(userId);
54
56
  return Boolean(
55
57
  this.browserExtensionRegistry
56
58
  && typeof this.browserExtensionRegistry.isConnected === 'function'
57
- && this.browserExtensionRegistry.isConnected(userId)
59
+ && this.browserExtensionRegistry.isConnected(userId, settings.browser_extension_token_id)
58
60
  );
59
61
  }
60
62
 
@@ -92,7 +94,9 @@ class RuntimeManager {
92
94
  async getBrowserProviderForUser(userId) {
93
95
  const settings = this.getSettings(userId);
94
96
  if (settings.browser_backend === 'extension' && this.hasActiveExtensionBrowser(userId)) {
95
- return this.getExtensionBrowserProvider(userId);
97
+ return this.getExtensionBrowserProvider(userId, {
98
+ tokenId: settings.browser_extension_token_id,
99
+ });
96
100
  }
97
101
  return this.browserBackend.getBrowserProviderForUser(userId);
98
102
  }
@@ -100,9 +104,36 @@ class RuntimeManager {
100
104
  async getCliProviderForUser(userId) {
101
105
  const settings = this.getSettings(userId);
102
106
  if (settings.cli_backend === 'desktop' && this.hasActiveDesktopCompanion(userId)) {
103
- return this.getDesktopCliProvider(userId);
107
+ const desktopProvider = this.getDesktopCliProvider(userId);
108
+ const preferredDeviceId = settings.cli_desktop_device_id;
109
+ if (preferredDeviceId) {
110
+ desktopProvider.selectDevice(preferredDeviceId);
111
+ }
112
+ return {
113
+ backend: 'desktop-companion',
114
+ execute: (command, options = {}) => desktopProvider.executeCommand(command, options),
115
+ executeInteractive: (command, inputs = [], options = {}) => desktopProvider.executeCommand(command, { ...options, inputs }),
116
+ kill: () => Promise.resolve(false),
117
+ };
104
118
  }
105
- return this.browserBackend.getCommandExecutorForUser(userId);
119
+ const executor = await this.browserBackend.getCommandExecutorForUser(userId);
120
+ return { ...executor, backend: 'vm' };
121
+ }
122
+
123
+ async executeCliCommand(userId, command, options = {}) {
124
+ const provider = await this.getCliProviderForUser(userId);
125
+ const result = await (options.pty === true && provider.executeInteractive
126
+ ? provider.executeInteractive(command, options.inputs || [], options)
127
+ : provider.execute(command, options));
128
+ return { ...result, backend: provider.backend };
129
+ }
130
+
131
+ getActiveBrowserBackend(userId) {
132
+ const settings = this.getSettings(userId);
133
+ if (settings.browser_backend === 'extension' && this.hasActiveExtensionBrowser(userId)) {
134
+ return 'extension';
135
+ }
136
+ return 'vm';
106
137
  }
107
138
 
108
139
  async getAndroidProviderForUser(userId) {
@@ -10,6 +10,9 @@ function createDefaultRuntimeSettings() {
10
10
  android_backend: policy.runtimeDefaults.android_backend,
11
11
  mcp_backend: policy.runtimeDefaults.mcp_backend,
12
12
  cli_backend: policy.runtimeDefaults.cli_backend ?? 'vm',
13
+ browser_extension_token_id: null,
14
+ selected_browser_extension_token_id: null,
15
+ cli_desktop_device_id: null,
13
16
  };
14
17
  }
15
18
 
@@ -26,6 +29,9 @@ const BASE_FALLBACK_SETTINGS = Object.freeze({
26
29
  android_backend: 'host',
27
30
  mcp_backend: 'host-remote',
28
31
  cli_backend: 'vm',
32
+ browser_extension_token_id: null,
33
+ selected_browser_extension_token_id: null,
34
+ cli_desktop_device_id: null,
29
35
  });
30
36
 
31
37
  const RUNTIME_SETTING_KEYS = Object.freeze(Object.keys(DEFAULT_RUNTIME_SETTINGS));
@@ -35,6 +41,11 @@ function normalizeChoice(value, allowed, fallback) {
35
41
  return allowed.includes(normalized) ? normalized : fallback;
36
42
  }
37
43
 
44
+ function normalizeOptionalString(value) {
45
+ const normalized = String(value || '').trim();
46
+ return normalized || null;
47
+ }
48
+
38
49
  function deriveDefaultsForProfile(profile) {
39
50
  switch (profile) {
40
51
  case 'secure-vm':
@@ -57,6 +68,9 @@ function normalizeRuntimeSettings(raw = {}) {
57
68
  const runtimeBackend = normalizeChoice(raw.runtime_backend, ['vm'], derived.runtime_backend);
58
69
  const browserBackend = normalizeChoice(raw.browser_backend, ['vm', 'extension'], derived.browser_backend);
59
70
  const cliBackend = normalizeChoice(raw.cli_backend, ['vm', 'desktop'], derived.cli_backend ?? 'vm');
71
+ const selectedExtensionTokenId = normalizeOptionalString(
72
+ raw.browser_extension_token_id || raw.selected_browser_extension_token_id,
73
+ );
60
74
  const androidBackend = 'host';
61
75
  return {
62
76
  runtime_profile: profile === 'trusted-host' ? 'secure-vm' : profile,
@@ -65,6 +79,9 @@ function normalizeRuntimeSettings(raw = {}) {
65
79
  android_backend: androidBackend,
66
80
  mcp_backend: 'host-remote',
67
81
  cli_backend: cliBackend === 'desktop' ? 'desktop' : 'vm',
82
+ browser_extension_token_id: selectedExtensionTokenId,
83
+ selected_browser_extension_token_id: selectedExtensionTokenId,
84
+ cli_desktop_device_id: normalizeOptionalString(raw.cli_desktop_device_id),
68
85
  };
69
86
  }
70
87
 
@@ -0,0 +1,298 @@
1
+ 'use strict';
2
+
3
+ const path = require('path');
4
+ const { spawn } = require('child_process');
5
+ const { EventEmitter } = require('events');
6
+ const sharp = require('sharp');
7
+
8
+ // Derive the full path to the `adb` binary the same way the Android controller does.
9
+ function resolveAdbBin(sdkDir) {
10
+ if (sdkDir) {
11
+ return path.join(sdkDir, 'platform-tools', process.platform === 'win32' ? 'adb.exe' : 'adb');
12
+ }
13
+ return process.platform === 'win32' ? 'adb.exe' : 'adb'; // fall back to PATH
14
+ }
15
+
16
+ function clampInt(value, fallback, min, max) {
17
+ const parsed = Number(value);
18
+ if (!Number.isFinite(parsed)) return fallback;
19
+ return Math.min(max, Math.max(min, Math.floor(parsed)));
20
+ }
21
+
22
+ // ---------------------------------------------------------------------------
23
+ // JpegFrameParser — extracts complete JPEG images from a raw byte stream.
24
+ // ffmpeg writes MJPEG to stdout as a continuous stream of JPEG frames, each
25
+ // delimited by SOI (FF D8) and EOI (FF D9) markers. We buffer incoming
26
+ // chunks and emit a 'frame' event for every complete JPEG found.
27
+ // ---------------------------------------------------------------------------
28
+ class JpegFrameParser extends EventEmitter {
29
+ constructor() {
30
+ super();
31
+ this._buf = null; // Buffer | null
32
+ }
33
+
34
+ push(chunk) {
35
+ if (!chunk?.length) return;
36
+ this._buf = this._buf ? Buffer.concat([this._buf, chunk]) : Buffer.from(chunk);
37
+ let start = 0;
38
+
39
+ while (start < this._buf.length - 1) {
40
+ // Locate SOI marker (0xFF 0xD8).
41
+ let soiIdx = -1;
42
+ for (let i = start; i < this._buf.length - 1; i++) {
43
+ if (this._buf[i] === 0xff && this._buf[i + 1] === 0xd8) {
44
+ soiIdx = i;
45
+ break;
46
+ }
47
+ }
48
+ if (soiIdx === -1) {
49
+ // No SOI found – discard everything.
50
+ this._buf = null;
51
+ return;
52
+ }
53
+
54
+ // Locate EOI marker (0xFF 0xD9) that comes after SOI.
55
+ let eoiIdx = -1;
56
+ for (let i = soiIdx + 2; i < this._buf.length - 1; i++) {
57
+ if (this._buf[i] === 0xff && this._buf[i + 1] === 0xd9) {
58
+ eoiIdx = i;
59
+ break;
60
+ }
61
+ }
62
+ if (eoiIdx === -1) {
63
+ // SOI found but no matching EOI yet – keep buffered from SOI onward.
64
+ this._buf = soiIdx > 0 ? this._buf.slice(soiIdx) : this._buf;
65
+ return;
66
+ }
67
+
68
+ // Emit the complete JPEG (inclusive of both markers).
69
+ this.emit('frame', this._buf.slice(soiIdx, eoiIdx + 2));
70
+ start = eoiIdx + 2;
71
+ }
72
+
73
+ // Retain any leftover bytes that haven't formed a complete frame yet.
74
+ this._buf = start < this._buf.length ? this._buf.slice(start) : null;
75
+ }
76
+ }
77
+
78
+ // ---------------------------------------------------------------------------
79
+ // AndroidStream
80
+ //
81
+ // Streams the Android emulator screen by:
82
+ // 1. Running `adb exec-out screenrecord --output-format=h264 -` to get a
83
+ // continuous H.264 bitstream at up to 30 fps with no per-frame process
84
+ // overhead (replacing the previous one-shot screencap-per-frame approach).
85
+ // 2. Piping the bitstream through `ffmpeg` which decodes H.264 and re-encodes
86
+ // each frame as MJPEG at the requested fps / quality.
87
+ // 3. Parsing complete JPEG frames from ffmpeg stdout via JpegFrameParser and
88
+ // forwarding them to StreamHub.
89
+ // 4. Auto-restarting when Android's built-in 3-minute screenrecord limit
90
+ // expires (~170 s to give a small safety margin).
91
+ // 5. Falling back to screencap polling when H.264 streaming cannot start
92
+ // (e.g. older device without screenrecord stdout support).
93
+ // ---------------------------------------------------------------------------
94
+ class AndroidStream {
95
+ constructor({ userId, deviceId, controller, streamHub, fps = 12, quality = 75 }) {
96
+ this.userId = String(userId || '');
97
+ this.deviceId = String(deviceId || '');
98
+ this.controller = controller;
99
+ this.streamHub = streamHub;
100
+ this.fps = clampInt(fps, 12, 1, 30);
101
+ this.quality = clampInt(quality, 75, 30, 95);
102
+
103
+ this._stopped = true;
104
+ this._adbProc = null;
105
+ this._ffmpegProc = null;
106
+ this._restartTimer = null;
107
+ this._seq = 0;
108
+ this._lastErrorLogAt = 0;
109
+ this._usePollingFallback = false; // set true when H.264 fails to start
110
+ }
111
+
112
+ start() {
113
+ if (!this._stopped) return;
114
+ this._stopped = false;
115
+ if (this._usePollingFallback) {
116
+ this._startPollingFallback();
117
+ } else {
118
+ this._launchH264();
119
+ }
120
+ }
121
+
122
+ stop() {
123
+ this._stopped = true;
124
+ this._killProcesses();
125
+ }
126
+
127
+ // ── H.264 streaming (primary path) ─────────────────────────────────────
128
+
129
+ _launchH264() {
130
+ if (this._stopped) return;
131
+
132
+ const adb = resolveAdbBin(this.controller?.sdkDir);
133
+ // deviceId is the ADB serial (emulator-5554, etc.)
134
+ const serial = this.deviceId;
135
+ const adbArgs = [
136
+ '-s', serial,
137
+ 'exec-out',
138
+ 'screenrecord',
139
+ '--output-format=h264',
140
+ '--bit-rate=2000000',
141
+ '--size=1280x720',
142
+ '-',
143
+ ];
144
+
145
+ try {
146
+ this._adbProc = spawn(adb, adbArgs, { stdio: ['ignore', 'pipe', 'ignore'] });
147
+ } catch (err) {
148
+ this._logError('Failed to spawn adb for H.264 streaming, falling back to screencap', err);
149
+ this._fallback();
150
+ return;
151
+ }
152
+
153
+ // Map our 30–95 quality range to ffmpeg q:v 2–20 (lower = better quality).
154
+ const ffmpegQ = Math.round(2 + ((95 - this.quality) / (95 - 30)) * 18);
155
+ const ffmpegArgs = [
156
+ '-loglevel', 'error',
157
+ '-f', 'h264',
158
+ '-i', 'pipe:0',
159
+ '-f', 'image2pipe',
160
+ '-vcodec', 'mjpeg',
161
+ '-q:v', String(ffmpegQ),
162
+ '-vf', `fps=${this.fps}`,
163
+ 'pipe:1',
164
+ ];
165
+
166
+ try {
167
+ this._ffmpegProc = spawn('ffmpeg', ffmpegArgs, { stdio: ['pipe', 'pipe', 'ignore'] });
168
+ } catch (err) {
169
+ this._logError('ffmpeg not found, falling back to screencap', err);
170
+ try { this._adbProc?.kill('SIGTERM'); } catch {}
171
+ this._adbProc = null;
172
+ this._fallback();
173
+ return;
174
+ }
175
+
176
+ // Wire adb stdout → ffmpeg stdin.
177
+ this._adbProc.stdout.pipe(this._ffmpegProc.stdin);
178
+
179
+ // Parse JPEG frames from ffmpeg stdout and forward to StreamHub.
180
+ const parser = new JpegFrameParser();
181
+ this._ffmpegProc.stdout.on('data', (chunk) => parser.push(chunk));
182
+ parser.on('frame', (jpeg) => {
183
+ if (this._stopped) return;
184
+ this.streamHub.handleFrame(this.userId, this.deviceId, {
185
+ jpeg,
186
+ platform: 'android',
187
+ seq: this._seq++ >>> 0,
188
+ ts: Date.now() >>> 0,
189
+ flags: 1,
190
+ });
191
+ });
192
+
193
+ this._adbProc.on('error', (err) => {
194
+ this._logError('adb process error', err);
195
+ });
196
+ this._ffmpegProc.on('error', (err) => {
197
+ this._logError('ffmpeg process error', err);
198
+ });
199
+
200
+ // Android screenrecord's hard limit is 180 s — restart at 170 s so there
201
+ // is no gap in the stream.
202
+ this._restartTimer = setTimeout(() => {
203
+ if (!this._stopped) {
204
+ this._killProcesses();
205
+ this._launchH264();
206
+ }
207
+ }, 170_000);
208
+ this._restartTimer.unref?.();
209
+
210
+ // If adb exits early (e.g. emulator restart), attempt recovery.
211
+ this._adbProc.on('close', (code) => {
212
+ if (this._stopped) return;
213
+ // If it exited immediately with a bad code, the device likely does not
214
+ // support stdout screenrecord — fall back to screencap polling.
215
+ if (code !== 0 && this._seq === 0) {
216
+ this._logError(`screenrecord exited (code ${code}) before producing any frames — using screencap fallback`);
217
+ this._killProcesses();
218
+ this._fallback();
219
+ return;
220
+ }
221
+ this._logError(`adb screenrecord exited (code ${code}), restarting in 2 s`);
222
+ this._killProcesses();
223
+ this._scheduleRestart(2000, () => this._launchH264());
224
+ });
225
+ }
226
+
227
+ _fallback() {
228
+ this._usePollingFallback = true;
229
+ if (!this._stopped) this._startPollingFallback();
230
+ }
231
+
232
+ // ── Screencap polling fallback ──────────────────────────────────────────
233
+ // Used when H.264 streaming is unavailable. Runs a tight continuous loop
234
+ // (no fixed interval) so there is never idle time waiting between captures.
235
+
236
+ _startPollingFallback() {
237
+ // Fire and forget — the loop runs until this._stopped is set.
238
+ void this._pollLoop();
239
+ }
240
+
241
+ async _pollLoop() {
242
+ while (!this._stopped) {
243
+ try {
244
+ if (!this.controller || typeof this.controller.capturePng !== 'function') {
245
+ throw new Error('Android streaming requires a controller with capturePng().');
246
+ }
247
+ const png = await this.controller.capturePng({ deviceId: this.deviceId });
248
+ if (this._stopped || !png?.length) continue;
249
+ const jpeg = await sharp(png)
250
+ .jpeg({ quality: this.quality, mozjpeg: false })
251
+ .toBuffer();
252
+ if (this._stopped) return;
253
+ this.streamHub.handleFrame(this.userId, this.deviceId, {
254
+ jpeg,
255
+ platform: 'android',
256
+ seq: this._seq++ >>> 0,
257
+ ts: Date.now() >>> 0,
258
+ flags: 1,
259
+ });
260
+ } catch (error) {
261
+ this._logError('screencap poll failed', error);
262
+ // Brief pause before retrying to avoid a tight spin on persistent errors.
263
+ await new Promise((r) => setTimeout(r, 500));
264
+ }
265
+ }
266
+ }
267
+
268
+ // ── Helpers ─────────────────────────────────────────────────────────────
269
+
270
+ _scheduleRestart(delayMs, fn) {
271
+ this._restartTimer = setTimeout(fn, delayMs);
272
+ this._restartTimer.unref?.();
273
+ }
274
+
275
+ _killProcesses() {
276
+ clearTimeout(this._restartTimer);
277
+ this._restartTimer = null;
278
+ // Kill ffmpeg first so it stops reading; then kill adb.
279
+ try { this._ffmpegProc?.kill('SIGTERM'); } catch {}
280
+ try { this._adbProc?.kill('SIGTERM'); } catch {}
281
+ this._ffmpegProc = null;
282
+ this._adbProc = null;
283
+ }
284
+
285
+ _logError(msg, err) {
286
+ const now = Date.now();
287
+ if (now - this._lastErrorLogAt > 10_000) {
288
+ this._lastErrorLogAt = now;
289
+ console.warn(`[AndroidStream] ${msg}`, {
290
+ userId: this.userId,
291
+ deviceId: this.deviceId,
292
+ error: err ? String(err?.message || err) : undefined,
293
+ });
294
+ }
295
+ }
296
+ }
297
+
298
+ module.exports = { AndroidStream };
@@ -0,0 +1,87 @@
1
+ 'use strict';
2
+
3
+ const fs = require('fs');
4
+ const sharp = require('sharp');
5
+
6
+ function clampInt(value, fallback, min, max) {
7
+ const parsed = Number(value);
8
+ if (!Number.isFinite(parsed)) return fallback;
9
+ return Math.min(max, Math.max(min, Math.floor(parsed)));
10
+ }
11
+
12
+ class BrowserStream {
13
+ constructor({ userId, deviceId = 'browser', controller, streamHub, fps = 15, quality = 80 }) {
14
+ this.userId = String(userId || '');
15
+ this.deviceId = String(deviceId || 'browser');
16
+ this.controller = controller;
17
+ this.streamHub = streamHub;
18
+ this.fps = clampInt(fps, 15, 1, 20);
19
+ this.quality = clampInt(quality, 80, 30, 95);
20
+ this._timer = null;
21
+ this._capturing = false;
22
+ this._seq = 0;
23
+ }
24
+
25
+ start() {
26
+ if (this._timer) return;
27
+ const interval = Math.max(1, Math.floor(1000 / this.fps));
28
+ this._timer = setInterval(() => {
29
+ void this._captureOnce();
30
+ }, interval);
31
+ this._timer.unref?.();
32
+ void this._captureOnce();
33
+ }
34
+
35
+ stop() {
36
+ if (this._timer) {
37
+ clearInterval(this._timer);
38
+ this._timer = null;
39
+ }
40
+ }
41
+
42
+ async _captureOnce() {
43
+ if (this._capturing) return;
44
+ this._capturing = true;
45
+ try {
46
+ const jpeg = await this._captureJpeg();
47
+ if (!jpeg?.length) return;
48
+ this.streamHub.handleFrame(this.userId, this.deviceId, {
49
+ jpeg,
50
+ platform: 'browser',
51
+ seq: this._seq++ >>> 0,
52
+ ts: Date.now() >>> 0,
53
+ flags: 1,
54
+ });
55
+ } catch (error) {
56
+ console.warn('[BrowserStream] frame capture failed', {
57
+ userId: this.userId,
58
+ deviceId: this.deviceId,
59
+ error: String(error?.message || error),
60
+ });
61
+ } finally {
62
+ this._capturing = false;
63
+ }
64
+ }
65
+
66
+ async _captureJpeg() {
67
+ if (!this.controller) {
68
+ throw new Error('Browser controller is unavailable.');
69
+ }
70
+ if (typeof this.controller.screenshotJpeg === 'function') {
71
+ return this.controller.screenshotJpeg(this.quality);
72
+ }
73
+ if (typeof this.controller.screenshot !== 'function') {
74
+ throw new Error('Browser streaming requires a screenshot-capable controller.');
75
+ }
76
+ const result = await this.controller.screenshot({ fullPage: false });
77
+ if (!result?.fullPath || !fs.existsSync(result.fullPath)) {
78
+ throw new Error('Browser screenshot did not produce a readable file.');
79
+ }
80
+ const png = fs.readFileSync(result.fullPath);
81
+ return sharp(png).jpeg({ quality: this.quality }).toBuffer();
82
+ }
83
+ }
84
+
85
+ module.exports = {
86
+ BrowserStream,
87
+ };