neoagent 2.4.1-beta.13 → 2.4.1-beta.15

package/server/routes/voice_assistant.js

@@ -4,7 +4,8 @@ const { requireAuth } = require('../middleware/auth');
 const { sanitizeError } = require('../utils/security');
 const { getAgentIdFromRequest, resolveAgentId } = require('../services/agents/manager');
 const { TurnCoordinator } = require('../services/voice/turnCoordinator');
-const { normalizeVoiceSynthesisOptions } = require('../services/voice/providers');
+const { normalizeVoiceSynthesisOptions, transcribeVoiceInput } = require('../services/voice/providers');
+const { writeTempAudioFile, removeTempFile } = require('../services/voice/liveAudio');
 const { runVoiceTranscriptTurn } = require('../services/voice/turnRunner');
 
 const router = express.Router();
@@ -135,4 +136,38 @@ router.post('/respond', async (req, res) => {
   }
 });
 
+router.post('/transcribe', async (req, res) => {
+  try {
+    const audioBase64 = String(req.body?.audioBase64 || '').trim();
+    const mimeType = String(req.body?.mimeType || 'audio/pcm;rate=16000;channels=1').trim();
+
+    if (!audioBase64) {
+      return res.status(400).json({ error: 'audioBase64 is required.' });
+    }
+
+    const approxBytes = (audioBase64.length * 3) / 4;
+    if (approxBytes > 25 * 1024 * 1024) {
+      return res.status(400).json({ error: 'Audio exceeds maximum size of 25MB.' });
+    }
+
+    const audioBytes = Buffer.from(audioBase64, 'base64');
+    const { filePath, mimeType: fileMimeType } = await writeTempAudioFile(audioBytes, mimeType);
+    let transcript = '';
+    try {
+      transcript = await transcribeVoiceInput(filePath, {
+        mimeType: fileMimeType,
+        userId: req.session.userId,
+        timeoutMs: 30000,
+      });
+    } finally {
+      await removeTempFile(filePath);
+    }
+
+    return res.json({ transcript: String(transcript || '').trim() });
+  } catch (err) {
+    const message = sanitizeError(err);
+    return res.status(500).json({ error: message });
+  }
+});
+
 module.exports = router;

package/server/services/browser/extension/registry.js

@@ -10,6 +10,7 @@ const DEFAULT_PAIRING_TTL_MS = 10 * 60 * 1000;
 const DEFAULT_COMMAND_TIMEOUT_MS = 30 * 1000;
 const DEFAULT_HEARTBEAT_INTERVAL_MS = 25 * 1000;
 const DEFAULT_HEARTBEAT_TIMEOUT_MS = 75 * 1000;
+const DEFAULT_PRESENCE_TOUCH_INTERVAL_MS = 15 * 1000;
 
 function sha256(value) {
   return crypto.createHash('sha256').update(String(value || '')).digest('hex');
@@ -46,6 +47,7 @@ class BrowserExtensionRegistry {
     this.pairingTtlMs = Number(options.pairingTtlMs || process.env.NEOAGENT_BROWSER_EXTENSION_PAIRING_TTL_MS || DEFAULT_PAIRING_TTL_MS);
     this.heartbeatIntervalMs = Number(options.heartbeatIntervalMs || process.env.NEOAGENT_BROWSER_EXTENSION_HEARTBEAT_INTERVAL_MS || DEFAULT_HEARTBEAT_INTERVAL_MS);
     this.heartbeatTimeoutMs = Number(options.heartbeatTimeoutMs || process.env.NEOAGENT_BROWSER_EXTENSION_HEARTBEAT_TIMEOUT_MS || DEFAULT_HEARTBEAT_TIMEOUT_MS);
+    this.presenceTouchIntervalMs = Number(options.presenceTouchIntervalMs || process.env.NEOAGENT_BROWSER_EXTENSION_PRESENCE_TOUCH_INTERVAL_MS || DEFAULT_PRESENCE_TOUCH_INTERVAL_MS);
     this.connectionsByUser = new Map();
   }
 
@@ -218,6 +220,7 @@ class BrowserExtensionRegistry {
       timeoutMs: this.commandTimeoutMs,
       heartbeatIntervalMs: this.heartbeatIntervalMs,
       heartbeatTimeoutMs: this.heartbeatTimeoutMs,
+      presenceTouchIntervalMs: this.presenceTouchIntervalMs,
     });
     userMap.set(tokenRow.id, connection);
     this.db.prepare(
@@ -315,6 +318,15 @@ class BrowserExtensionRegistry {
     return Boolean(this.getConnection(userId, tokenId)?.isOpen());
   }
 
+  touchPresence(userId, tokenId) {
+    const userMap = this.#getUserConnections(userId);
+    const connection = userMap?.get(String(tokenId));
+    if (!connection?.isOpen()) return;
+    this.db.prepare(
+      `UPDATE browser_extension_tokens SET last_seen_at = datetime('now') WHERE id = ?`
+    ).run(tokenId);
+  }
+
   async dispatch(userId, command, payload = {}, options = {}) {
     const connection = this.getConnection(userId, options.tokenId || payload.tokenId || null);
     if (!connection || !connection.isOpen()) {
@@ -329,10 +341,16 @@ class BrowserExtensionRegistry {
 
   getStatus(userId) {
     const userMap = this.#getUserConnections(userId);
-    const selectedTokenId = this.getSelectedTokenId(userId);
-    const connected = selectedTokenId
+    let selectedTokenId = this.getSelectedTokenId(userId);
+    let connected = selectedTokenId
       ? userMap?.get(selectedTokenId)
-      : this.getConnection(userId);
+      : null;
+    if (!connected?.isOpen()) {
+      connected = this.getConnection(userId);
+      if (connected?.isOpen()) {
+        selectedTokenId = connected.tokenId;
+      }
+    }
     const effectiveSelectedTokenId = selectedTokenId || connected?.tokenId || null;
     const tokens = this.db.prepare(
       `SELECT id, name, status, last_connected_at, last_seen_at, revoked_at, created_at, metadata_json
@@ -400,7 +418,7 @@ class BrowserExtensionRegistry {
 }
 
 class ExtensionBrowserConnection {
-  constructor({ registry, ws, userId, tokenId, meta, timeoutMs, heartbeatIntervalMs, heartbeatTimeoutMs }) {
+  constructor({ registry, ws, userId, tokenId, meta, timeoutMs, heartbeatIntervalMs, heartbeatTimeoutMs, presenceTouchIntervalMs }) {
     this.registry = registry;
     this.ws = ws;
     this.userId = userId;
@@ -409,13 +427,18 @@ class ExtensionBrowserConnection {
     this.timeoutMs = timeoutMs;
     this.heartbeatIntervalMs = heartbeatIntervalMs;
     this.heartbeatTimeoutMs = heartbeatTimeoutMs;
+    this.presenceTouchIntervalMs = presenceTouchIntervalMs;
     this.pending = new Map();
     this.connectedAt = new Date().toISOString();
     this.lastPongAt = Date.now();
+    this.lastPresenceTouchAt = 0;
     this.heartbeatTimer = null;
 
     ws.on('message', (data) => this.#handleMessage(data));
-    ws.on('pong', () => { this.lastPongAt = Date.now(); });
+    ws.on('pong', () => {
+      this.lastPongAt = Date.now();
+      this.touchPresence();
+    });
     ws.on('close', () => this.#closePending(new ExtensionBrowserUnavailableError('Extension browser disconnected.')));
     ws.on('error', (error) => this.#closePending(error));
     this.#startHeartbeat();
@@ -435,6 +458,21 @@ class ExtensionBrowserConnection {
     this.#closePending(new ExtensionBrowserUnavailableError('Extension browser disconnected.'));
   }
 
+  touchPresence({ force = false } = {}) {
+    const now = Date.now();
+    const intervalMs = Number(this.presenceTouchIntervalMs);
+    if (
+      !force
+      && Number.isFinite(intervalMs)
+      && intervalMs > 0
+      && now - this.lastPresenceTouchAt < intervalMs
+    ) {
+      return;
+    }
+    this.lastPresenceTouchAt = now;
+    this.registry.touchPresence(this.userId, this.tokenId);
+  }
+
   sendCommand(command, payload = {}, options = {}) {
     if (!this.isOpen()) {
       return Promise.reject(new ExtensionBrowserUnavailableError());
@@ -460,6 +498,7 @@ class ExtensionBrowserConnection {
 
   #handleMessage(data) {
     this.lastPongAt = Date.now();
+    this.touchPresence();
     let message;
     try {
       message = parseExtensionMessage(data);
@@ -497,6 +536,7 @@ class ExtensionBrowserConnection {
     const intervalMs = Number(this.heartbeatIntervalMs);
     const timeoutMs = Number(this.heartbeatTimeoutMs);
     if (!Number.isFinite(intervalMs) || intervalMs <= 0) return;
+    this.touchPresence({ force: true });
     this.heartbeatTimer = setInterval(() => {
       if (!this.isOpen()) {
         this.#closePending(new ExtensionBrowserUnavailableError('Extension browser disconnected.'));
@@ -507,6 +547,7 @@ class ExtensionBrowserConnection {
         this.#closePending(new ExtensionBrowserUnavailableError('Extension browser heartbeat timed out.'));
         return;
       }
+      this.touchPresence();
       try {
         this.ws.ping();
       } catch (error) {

package/server/services/desktop/registry.js

@@ -10,6 +10,9 @@ const {
 } = require('./protocol');
 
 const DEFAULT_COMMAND_TIMEOUT_MS = 30 * 1000;
+const DEFAULT_HEARTBEAT_INTERVAL_MS = 25 * 1000;
+const DEFAULT_HEARTBEAT_TIMEOUT_MS = 75 * 1000;
+const DEFAULT_PRESENCE_TOUCH_INTERVAL_MS = 15 * 1000;
 
 function safeJson(value) {
   try {
@@ -47,6 +50,21 @@ class DesktopCompanionRegistry {
       || process.env.NEOAGENT_DESKTOP_COMMAND_TIMEOUT_MS
       || DEFAULT_COMMAND_TIMEOUT_MS,
     );
+    this.heartbeatIntervalMs = Number(
+      options.heartbeatIntervalMs
+      || process.env.NEOAGENT_DESKTOP_HEARTBEAT_INTERVAL_MS
+      || DEFAULT_HEARTBEAT_INTERVAL_MS,
+    );
+    this.heartbeatTimeoutMs = Number(
+      options.heartbeatTimeoutMs
+      || process.env.NEOAGENT_DESKTOP_HEARTBEAT_TIMEOUT_MS
+      || DEFAULT_HEARTBEAT_TIMEOUT_MS,
+    );
+    this.presenceTouchIntervalMs = Number(
+      options.presenceTouchIntervalMs
+      || process.env.NEOAGENT_DESKTOP_PRESENCE_TOUCH_INTERVAL_MS
+      || DEFAULT_PRESENCE_TOUCH_INTERVAL_MS,
+    );
     this.connectionsByUser = new Map();
   }
 
@@ -200,6 +218,9 @@ class DesktopCompanionRegistry {
         platform: record.platform,
       },
       timeoutMs: this.commandTimeoutMs,
+      heartbeatIntervalMs: this.heartbeatIntervalMs,
+      heartbeatTimeoutMs: this.heartbeatTimeoutMs,
+      presenceTouchIntervalMs: this.presenceTouchIntervalMs,
     });
     // Install the new connection in the map BEFORE closing the old one.
     // This ensures that when the old socket's async 'close' event fires and
@@ -275,6 +296,19 @@ class DesktopCompanionRegistry {
     return this.getDeviceRecordByDeviceId(userId, deviceId);
   }
 
+  touchPresence(userId, deviceId) {
+    const userMap = this._getUserMap(userId);
+    const connection = userMap?.get(String(deviceId));
+    if (!connection?.isOpen()) return;
+    this.db.prepare(
+      `UPDATE desktop_companion_devices
+       SET status = 'online',
+           last_seen_at = datetime('now'),
+           updated_at = datetime('now')
+       WHERE user_id = ? AND device_id = ? AND revoked_at IS NULL`
+    ).run(userId, deviceId);
+  }
+
   isConnected(userId) {
     const userMap = this._getUserMap(userId);
     return userMap != null && userMap.size > 0;
@@ -493,7 +527,19 @@ class DesktopCompanionRegistry {
 }
 
 class DesktopCompanionConnection {
-  constructor({ registry, ws, userId, sessionId, deviceId, recordId, meta, timeoutMs }) {
+  constructor({
+    registry,
+    ws,
+    userId,
+    sessionId,
+    deviceId,
+    recordId,
+    meta,
+    timeoutMs,
+    heartbeatIntervalMs,
+    heartbeatTimeoutMs,
+    presenceTouchIntervalMs,
+  }) {
     this.registry = registry;
     this.ws = ws;
     this.userId = userId;
@@ -502,11 +548,22 @@ class DesktopCompanionConnection {
     this.recordId = recordId;
     this.meta = meta || {};
     this.timeoutMs = timeoutMs;
+    this.heartbeatIntervalMs = heartbeatIntervalMs;
+    this.heartbeatTimeoutMs = heartbeatTimeoutMs;
+    this.presenceTouchIntervalMs = presenceTouchIntervalMs;
     this.pending = new Map();
+    this.lastPongAt = Date.now();
+    this.lastPresenceTouchAt = 0;
+    this.heartbeatTimer = null;
 
     ws.on('message', (data) => this._handleMessage(data));
+    ws.on('pong', () => {
+      this.lastPongAt = Date.now();
+      this.touchPresence();
+    });
     ws.on('close', () => this._closePending(new DesktopCompanionUnavailableError('Desktop companion disconnected.')));
     ws.on('error', (error) => this._closePending(error));
+    this._startHeartbeat();
   }
 
   isOpen() {
@@ -525,6 +582,21 @@ class DesktopCompanionConnection {
     this._closePending(new DesktopCompanionUnavailableError('Desktop companion disconnected.'));
   }
 
+  touchPresence({ force = false } = {}) {
+    const now = Date.now();
+    const intervalMs = Number(this.presenceTouchIntervalMs);
+    if (
+      !force
+      && Number.isFinite(intervalMs)
+      && intervalMs > 0
+      && now - this.lastPresenceTouchAt < intervalMs
+    ) {
+      return;
+    }
+    this.lastPresenceTouchAt = now;
+    this.registry.touchPresence(this.userId, this.deviceId);
+  }
+
   sendCommand(command, payload = {}, options = {}) {
     if (!this.isOpen()) {
       return Promise.reject(new DesktopCompanionUnavailableError());
@@ -549,6 +621,8 @@ class DesktopCompanionConnection {
   }
 
   _handleMessage(data) {
+    this.lastPongAt = Date.now();
+    this.touchPresence();
     if (Buffer.isBuffer(data) && data.length > 0 && data[0] === FRAME_TYPE_VIDEO) {
       return;
     }
@@ -585,6 +659,10 @@ class DesktopCompanionConnection {
   }
 
   _closePending(error) {
+    if (this.heartbeatTimer) {
+      clearInterval(this.heartbeatTimer);
+      this.heartbeatTimer = null;
+    }
     for (const pending of this.pending.values()) {
       clearTimeout(pending.timer);
       pending.reject(error);
@@ -596,6 +674,31 @@ class DesktopCompanionConnection {
     // mark the device offline.
     this.registry.unregisterConnection(this);
   }
+
+  _startHeartbeat() {
+    const intervalMs = Number(this.heartbeatIntervalMs);
+    const timeoutMs = Number(this.heartbeatTimeoutMs);
+    if (!Number.isFinite(intervalMs) || intervalMs <= 0) return;
+    this.touchPresence({ force: true });
+    this.heartbeatTimer = setInterval(() => {
+      if (!this.isOpen()) {
+        this._closePending(new DesktopCompanionUnavailableError('Desktop companion disconnected.'));
+        return;
+      }
+      if (Number.isFinite(timeoutMs) && timeoutMs > 0 && Date.now() - this.lastPongAt > timeoutMs) {
+        try { this.ws.terminate(); } catch {}
+        this._closePending(new DesktopCompanionUnavailableError('Desktop companion heartbeat timed out.'));
+        return;
+      }
+      this.touchPresence();
+      try {
+        this.ws.ping();
+      } catch (error) {
+        this._closePending(error);
+      }
+    }, intervalMs);
+    this.heartbeatTimer.unref?.();
+  }
 }
 
 module.exports = {

package/server/services/streaming/android-stream.js

@@ -5,6 +5,18 @@ const { spawn } = require('child_process');
 const { EventEmitter } = require('events');
 const sharp = require('sharp');
 
+let H264_FIRST_FRAME_TIMEOUT_MS = 6_000;
+const envTimeout = process.env.NEOAGENT_ANDROID_STREAM_FIRST_FRAME_TIMEOUT_MS;
+
+if (envTimeout) {
+  const parsed = parseInt(envTimeout, 10);
+  if (!isNaN(parsed) && parsed > 0) {
+    H264_FIRST_FRAME_TIMEOUT_MS = parsed;
+  } else {
+    console.warn('[AndroidStream] Invalid NEOAGENT_ANDROID_STREAM_FIRST_FRAME_TIMEOUT_MS; using default 6000ms');
+  }
+}
+
 // Derive the full path to the `adb` binary the same way the Android controller does.
 function resolveAdbBin(sdkDir) {
   if (sdkDir) {
@@ -104,6 +116,7 @@ class AndroidStream {
     this._adbProc = null;
     this._ffmpegProc = null;
     this._restartTimer = null;
+    this._h264StartupTimer = null;
     this._seq = 0;
     this._lastErrorLogAt = 0;
     this._usePollingFallback = false; // set true when H.264 fails to start
@@ -181,6 +194,7 @@ class AndroidStream {
     this._ffmpegProc.stdout.on('data', (chunk) => parser.push(chunk));
     parser.on('frame', (jpeg) => {
       if (this._stopped) return;
+      this._clearH264StartupTimer();
       this.streamHub.handleFrame(this.userId, this.deviceId, {
         jpeg,
         platform: 'android',
@@ -197,6 +211,15 @@ class AndroidStream {
       this._logError('ffmpeg process error', err);
     });
 
+    if (H264_FIRST_FRAME_TIMEOUT_MS > 0) {
+      this._h264StartupTimer = setTimeout(() => {
+        if (this._stopped || this._usePollingFallback || this._seq > 0) return;
+        this._logError('H.264 stream produced no frames before timeout — using screencap fallback');
+        this._fallback();
+      }, H264_FIRST_FRAME_TIMEOUT_MS);
+      this._h264StartupTimer.unref?.();
+    }
+
     // Android screenrecord's hard limit is 180 s — restart at 170 s so there
     // is no gap in the stream.
     this._restartTimer = setTimeout(() => {
@@ -209,7 +232,7 @@ class AndroidStream {
 
     // If adb exits early (e.g. emulator restart), attempt recovery.
     this._adbProc.on('close', (code) => {
-      if (this._stopped) return;
+      if (this._stopped || this._usePollingFallback) return;
       // If it exited immediately with a bad code, the device likely does not
       // support stdout screenrecord — fall back to screencap polling.
       if (code !== 0 && this._seq === 0) {
@@ -222,9 +245,18 @@ class AndroidStream {
       this._killProcesses();
       this._scheduleRestart(2000, () => this._launchH264());
     });
+
+    this._ffmpegProc.on('close', (code) => {
+      if (this._stopped || this._usePollingFallback) return;
+      if (this._seq === 0) {
+        this._logError(`ffmpeg exited (code ${code}) before producing any frames — using screencap fallback`);
+        this._fallback();
+      }
+    });
   }
 
   _fallback() {
+    this._killProcesses();
     this._usePollingFallback = true;
     if (!this._stopped) this._startPollingFallback();
   }
@@ -275,6 +307,7 @@ class AndroidStream {
   _killProcesses() {
     clearTimeout(this._restartTimer);
     this._restartTimer = null;
+    this._clearH264StartupTimer();
     // Kill ffmpeg first so it stops reading; then kill adb.
     try { this._ffmpegProc?.kill('SIGTERM'); } catch {}
     try { this._adbProc?.kill('SIGTERM'); } catch {}
@@ -282,6 +315,11 @@ class AndroidStream {
     this._adbProc = null;
   }
 
+  _clearH264StartupTimer() {
+    clearTimeout(this._h264StartupTimer);
+    this._h264StartupTimer = null;
+  }
+
   _logError(msg, err) {
     const now = Date.now();
     if (now - this._lastErrorLogAt > 10_000) {

package/server/utils/version.js

@@ -12,6 +12,11 @@ const {
 const { getDeploymentInfo } = require('./deployment');
 
 const PACKAGE_JSON_PATH = path.join(APP_DIR, 'package.json');
+const GIT_COMMAND_TIMEOUT_MS = 750;
+const VERSION_CACHE_TTL_MS = 30 * 1000;
+
+let cachedVersionInfo = null;
+let cachedVersionInfoAt = 0;
 
 function readPackageVersion() {
   try {
@@ -22,7 +27,16 @@ function readPackageVersion() {
   }
 }
 
-function getVersionInfo() {
+function readGitValue(command) {
+  return execSync(command, {
+    cwd: APP_DIR,
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'ignore'],
+    timeout: GIT_COMMAND_TIMEOUT_MS,
+  }).trim();
+}
+
+function buildVersionInfo() {
   const packageVersion = readPackageVersion() || '0.0.0';
   const releaseChannel = readConfiguredReleaseChannel();
   const deployment = getDeploymentInfo();
@@ -32,24 +46,9 @@ function getVersionInfo() {
   let gitBranch = null;
 
   try {
-    gitVersion =
-      execSync('git describe --tags --always --dirty', {
-        cwd: APP_DIR,
-        encoding: 'utf8',
-        stdio: ['ignore', 'pipe', 'ignore']
-      })
-        .trim()
-        .replace(/^v/, '') || null;
-    gitSha = execSync('git rev-parse --short HEAD', {
-      cwd: APP_DIR,
-      encoding: 'utf8',
-      stdio: ['ignore', 'pipe', 'ignore']
-    }).trim();
-    gitBranch = execSync('git rev-parse --abbrev-ref HEAD', {
-      cwd: APP_DIR,
-      encoding: 'utf8',
-      stdio: ['ignore', 'pipe', 'ignore']
-    }).trim();
+    gitVersion = readGitValue('git describe --tags --always --dirty').replace(/^v/, '') || null;
+    gitSha = readGitValue('git rev-parse --short HEAD') || null;
+    gitBranch = readGitValue('git rev-parse --abbrev-ref HEAD') || null;
   } catch {
     gitSha = process.env.GIT_SHA || null;
   }
@@ -80,4 +79,15 @@ function getVersionInfo() {
   };
 }
 
+function getVersionInfo() {
+  const now = Date.now();
+  if (cachedVersionInfo && now - cachedVersionInfoAt < VERSION_CACHE_TTL_MS) {
+    return { ...cachedVersionInfo };
+  }
+
+  cachedVersionInfo = buildVersionInfo();
+  cachedVersionInfoAt = now;
+  return { ...cachedVersionInfo };
+}
+
 module.exports = { getVersionInfo };

package/server/utils/version.test.js

@@ -0,0 +1,39 @@
+'use strict';
+
+const assert = require('assert');
+const { test } = require('node:test');
+const childProcess = require('child_process');
+
+const VERSION_MODULE_PATH = require.resolve('./version');
+
+function loadVersionModuleWithExecSync(execSync) {
+  const originalExecSync = childProcess.execSync;
+  delete require.cache[VERSION_MODULE_PATH];
+  childProcess.execSync = execSync;
+  try {
+    return require('./version');
+  } finally {
+    childProcess.execSync = originalExecSync;
+  }
+}
+
+test('getVersionInfo caches git metadata reads', () => {
+  let callCount = 0;
+  const { getVersionInfo } = loadVersionModuleWithExecSync((command) => {
+    callCount += 1;
+    if (command.includes('describe')) return 'v1.2.3\n';
+    if (command.includes('rev-parse --short')) return 'abc123\n';
+    if (command.includes('abbrev-ref')) return 'main\n';
+    throw new Error(`unexpected command: ${command}`);
+  });
+
+  const first = getVersionInfo();
+  const second = getVersionInfo();
+
+  assert.equal(first.gitVersion, '1.2.3');
+  assert.equal(first.gitSha, 'abc123');
+  assert.equal(first.gitBranch, 'main');
+  assert.equal(second.gitVersion, '1.2.3');
+  assert.equal(callCount, 3);
+});
+