neoagent 2.4.1-beta.8 → 2.4.1

package/server/services/browser/extension/registry.js

@@ -8,6 +8,9 @@ const {
 
 const DEFAULT_PAIRING_TTL_MS = 10 * 60 * 1000;
 const DEFAULT_COMMAND_TIMEOUT_MS = 30 * 1000;
+const DEFAULT_HEARTBEAT_INTERVAL_MS = 25 * 1000;
+const DEFAULT_HEARTBEAT_TIMEOUT_MS = 75 * 1000;
+const DEFAULT_PRESENCE_TOUCH_INTERVAL_MS = 15 * 1000;
 
 function sha256(value) {
   return crypto.createHash('sha256').update(String(value || '')).digest('hex');
@@ -42,9 +45,21 @@ class BrowserExtensionRegistry {
     this.db = options.db || db;
     this.commandTimeoutMs = Number(options.commandTimeoutMs || process.env.NEOAGENT_BROWSER_EXTENSION_COMMAND_TIMEOUT_MS || DEFAULT_COMMAND_TIMEOUT_MS);
     this.pairingTtlMs = Number(options.pairingTtlMs || process.env.NEOAGENT_BROWSER_EXTENSION_PAIRING_TTL_MS || DEFAULT_PAIRING_TTL_MS);
+    this.heartbeatIntervalMs = Number(options.heartbeatIntervalMs || process.env.NEOAGENT_BROWSER_EXTENSION_HEARTBEAT_INTERVAL_MS || DEFAULT_HEARTBEAT_INTERVAL_MS);
+    this.heartbeatTimeoutMs = Number(options.heartbeatTimeoutMs || process.env.NEOAGENT_BROWSER_EXTENSION_HEARTBEAT_TIMEOUT_MS || DEFAULT_HEARTBEAT_TIMEOUT_MS);
+    this.presenceTouchIntervalMs = Number(options.presenceTouchIntervalMs || process.env.NEOAGENT_BROWSER_EXTENSION_PRESENCE_TOUCH_INTERVAL_MS || DEFAULT_PRESENCE_TOUCH_INTERVAL_MS);
     this.connectionsByUser = new Map();
   }
 
+  #getUserConnections(userId, create = false) {
+    const key = String(userId || '').trim();
+    if (!key) return null;
+    if (!this.connectionsByUser.has(key) && create) {
+      this.connectionsByUser.set(key, new Map());
+    }
+    return this.connectionsByUser.get(key) || null;
+  }
+
   createPairingRequest(options = {}) {
     const pairingId = crypto.randomUUID();
     const pairingSecret = randomSecret(48);
@@ -129,16 +144,42 @@ class BrowserExtensionRegistry {
     };
 
     const tx = this.db.transaction(() => {
+      // Find other active tokens with the same name for this user.
+      // If they are currently offline, revoke them to clean up old installs/duplicates.
+      const duplicates = this.db.prepare(
+        `SELECT id FROM browser_extension_tokens
+         WHERE user_id = ? AND name = ? AND status = 'active'`
+      ).all(row.user_id, extensionName);
+
+      for (const dup of duplicates) {
+        if (!this.isConnected(row.user_id, dup.id)) {
+          this.db.prepare(
+            `UPDATE browser_extension_tokens
+             SET status = 'revoked', revoked_at = datetime('now')
+             WHERE id = ?`
+          ).run(dup.id);
+        }
+      }
+
       this.db.prepare(
         `UPDATE browser_extension_pairing_requests
          SET status = 'claimed', claimed_at = datetime('now')
          WHERE id = ?`
       ).run(row.id);
+      
       this.db.prepare(
         `INSERT INTO browser_extension_tokens (
            id, user_id, token_hash, name, status, metadata_json
          ) VALUES (?, ?, ?, ?, 'active', ?)`
       ).run(tokenId, row.user_id, sha256(token), extensionName, safeJson(metadata));
+
+      // Auto-select the newly paired token on claim
+      const write = this.db.prepare(
+        `INSERT INTO user_settings (user_id, key, value) VALUES (?, ?, ?)
+         ON CONFLICT(user_id, key) DO UPDATE SET value = excluded.value`
+      );
+      write.run(row.user_id, 'browser_extension_token_id', tokenId);
+      write.run(row.user_id, 'selected_browser_extension_token_id', tokenId);
     });
     tx();
 
@@ -164,7 +205,8 @@ class BrowserExtensionRegistry {
 
   registerConnection(tokenRow, ws, meta = {}) {
     const userId = String(tokenRow.user_id);
-    const existing = this.connectionsByUser.get(userId);
+    const userMap = this.#getUserConnections(userId, true);
+    const existing = userMap.get(tokenRow.id);
     if (existing && existing.ws !== ws) {
       existing.close('replaced by a newer extension connection');
     }
@@ -176,8 +218,11 @@ class BrowserExtensionRegistry {
       tokenId: tokenRow.id,
       meta: { ...tokenRow.metadata, ...meta },
       timeoutMs: this.commandTimeoutMs,
+      heartbeatIntervalMs: this.heartbeatIntervalMs,
+      heartbeatTimeoutMs: this.heartbeatTimeoutMs,
+      presenceTouchIntervalMs: this.presenceTouchIntervalMs,
     });
-    this.connectionsByUser.set(userId, connection);
+    userMap.set(tokenRow.id, connection);
     this.db.prepare(
       `UPDATE browser_extension_tokens
        SET last_connected_at = datetime('now'), last_seen_at = datetime('now')
@@ -188,21 +233,102 @@ class BrowserExtensionRegistry {
 
   unregisterConnection(connection) {
     const userId = String(connection.userId);
-    if (this.connectionsByUser.get(userId) === connection) {
-      this.connectionsByUser.delete(userId);
+    const userMap = this.#getUserConnections(userId);
+    if (userMap?.get(connection.tokenId) === connection) {
+      userMap.delete(connection.tokenId);
+      if (userMap.size === 0) {
+        this.connectionsByUser.delete(userId);
+      }
+    }
+  }
+
+  getSelectedTokenId(userId) {
+    const value = this.db.prepare(
+      `SELECT value FROM user_settings WHERE user_id = ? AND key = ?`
+    ).get(userId, 'browser_extension_token_id')?.value || null;
+    const normalized = String(value || '').trim();
+    if (!normalized || normalized === 'null') return null;
+    const existing = this.db.prepare(
+      `SELECT id FROM browser_extension_tokens WHERE user_id = ? AND id = ? AND status = 'active'`
+    ).get(userId, normalized);
+    if (!existing) return null;
+    return normalized;
+  }
+
+  setSelectedTokenId(userId, tokenId) {
+    const normalized = String(tokenId || '').trim();
+    if (!normalized) {
+      const error = new Error('Browser extension token id is required.');
+      error.status = 400;
+      throw error;
+    }
+    const existing = this.db.prepare(
+      `SELECT id FROM browser_extension_tokens WHERE user_id = ? AND id = ? AND status = 'active'`
+    ).get(userId, normalized);
+    if (!existing) {
+      const error = new Error('Browser extension device not found.');
+      error.status = 404;
+      throw error;
+    }
+    const write = this.db.prepare(
+      `INSERT INTO user_settings (user_id, key, value) VALUES (?, ?, ?)
+       ON CONFLICT(user_id, key) DO UPDATE SET value = excluded.value`
+    );
+    write.run(userId, 'browser_extension_token_id', normalized);
+    write.run(userId, 'selected_browser_extension_token_id', normalized);
+    return { success: true, selectedTokenId: normalized };
+  }
+
+  getConnection(userId, tokenId = null) {
+    const userMap = this.#getUserConnections(userId);
+    if (!userMap) return null;
+    const explicit = String(tokenId || '').trim();
+    if (explicit) return userMap.get(explicit) || null;
+
+    const selected = this.getSelectedTokenId(userId);
+    if (selected && userMap.get(selected)?.isOpen()) {
+      return userMap.get(selected);
+    }
+
+    // Auto-select online connection if selected is offline/unset
+    const online = Array.from(userMap.values()).filter((connection) => connection.isOpen());
+    if (online.length > 0) {
+      online.sort((a, b) => String(b.connectedAt || '').localeCompare(String(a.connectedAt || '')));
+      const activeConnection = online[0];
+      try {
+        const write = this.db.prepare(
+          `INSERT INTO user_settings (user_id, key, value) VALUES (?, ?, ?)
+           ON CONFLICT(user_id, key) DO UPDATE SET value = excluded.value`
+        );
+        write.run(userId, 'browser_extension_token_id', activeConnection.tokenId);
+        write.run(userId, 'selected_browser_extension_token_id', activeConnection.tokenId);
+      } catch (err) {
+        console.error('[Registry] failed to auto-select online extension', err);
+      }
+      return activeConnection;
     }
+
+    if (selected) {
+      return userMap.get(selected) || null;
+    }
+    return null;
   }
 
-  getConnection(userId) {
-    return this.connectionsByUser.get(String(userId));
+  isConnected(userId, tokenId = null) {
+    return Boolean(this.getConnection(userId, tokenId)?.isOpen());
   }
 
-  isConnected(userId) {
-    return Boolean(this.getConnection(userId)?.isOpen());
+  touchPresence(userId, tokenId) {
+    const userMap = this.#getUserConnections(userId);
+    const connection = userMap?.get(String(tokenId));
+    if (!connection?.isOpen()) return;
+    this.db.prepare(
+      `UPDATE browser_extension_tokens SET last_seen_at = datetime('now') WHERE id = ?`
+    ).run(tokenId);
   }
 
   async dispatch(userId, command, payload = {}, options = {}) {
-    const connection = this.getConnection(userId);
+    const connection = this.getConnection(userId, options.tokenId || payload.tokenId || null);
     if (!connection || !connection.isOpen()) {
       throw new ExtensionBrowserUnavailableError();
     }
@@ -214,20 +340,41 @@ class BrowserExtensionRegistry {
   }
 
   getStatus(userId) {
-    const connected = this.getConnection(userId);
+    const userMap = this.#getUserConnections(userId);
+    let selectedTokenId = this.getSelectedTokenId(userId);
+    let connected = selectedTokenId
+      ? userMap?.get(selectedTokenId)
+      : null;
+    if (!connected?.isOpen()) {
+      connected = this.getConnection(userId);
+      if (connected?.isOpen()) {
+        selectedTokenId = connected.tokenId;
+      }
+    }
+    const effectiveSelectedTokenId = selectedTokenId || connected?.tokenId || null;
     const tokens = this.db.prepare(
       `SELECT id, name, status, last_connected_at, last_seen_at, revoked_at, created_at, metadata_json
        FROM browser_extension_tokens
        WHERE user_id = ?
        ORDER BY created_at DESC`
-    ).all(userId).map((row) => ({
-      ...row,
-      metadata: parseJson(row.metadata_json),
-      metadata_json: undefined,
-    }));
+    ).all(userId).map((row) => {
+      const connection = userMap?.get(row.id) || null;
+      return {
+        ...row,
+        tokenId: row.id,
+        deviceId: row.id,
+        connected: Boolean(connection?.isOpen()),
+        online: Boolean(connection?.isOpen()),
+        selected: row.id === effectiveSelectedTokenId,
+        metadata: parseJson(row.metadata_json),
+        connectedMeta: connection?.meta || null,
+        metadata_json: undefined,
+      };
+    });
     return {
       connected: Boolean(connected?.isOpen()),
       activeTokenId: connected?.tokenId || null,
+      selectedTokenId: effectiveSelectedTokenId,
       tokens,
       connectedMeta: connected?.meta || null,
     };
@@ -249,7 +396,7 @@ class BrowserExtensionRegistry {
       ).run(userId);
     }
 
-    const connection = this.getConnection(userId);
+    const connection = this.getConnection(userId, targetTokenId);
     if (connection && (!targetTokenId || connection.tokenId === targetTokenId)) {
       connection.close('extension token revoked');
     }
@@ -258,25 +405,43 @@ class BrowserExtensionRegistry {
 
   closeAll() {
     for (const connection of this.connectionsByUser.values()) {
-      connection.close('server shutdown');
+      if (connection instanceof Map) {
+        for (const nested of connection.values()) {
+          nested.close('server shutdown');
+        }
+      } else {
+        connection.close('server shutdown');
+      }
     }
     this.connectionsByUser.clear();
   }
 }
 
 class ExtensionBrowserConnection {
-  constructor({ registry, ws, userId, tokenId, meta, timeoutMs }) {
+  constructor({ registry, ws, userId, tokenId, meta, timeoutMs, heartbeatIntervalMs, heartbeatTimeoutMs, presenceTouchIntervalMs }) {
     this.registry = registry;
     this.ws = ws;
     this.userId = userId;
     this.tokenId = tokenId;
     this.meta = meta || {};
     this.timeoutMs = timeoutMs;
+    this.heartbeatIntervalMs = heartbeatIntervalMs;
+    this.heartbeatTimeoutMs = heartbeatTimeoutMs;
+    this.presenceTouchIntervalMs = presenceTouchIntervalMs;
     this.pending = new Map();
+    this.connectedAt = new Date().toISOString();
+    this.lastPongAt = Date.now();
+    this.lastPresenceTouchAt = 0;
+    this.heartbeatTimer = null;
 
     ws.on('message', (data) => this.#handleMessage(data));
+    ws.on('pong', () => {
+      this.lastPongAt = Date.now();
+      this.touchPresence();
+    });
     ws.on('close', () => this.#closePending(new ExtensionBrowserUnavailableError('Extension browser disconnected.')));
     ws.on('error', (error) => this.#closePending(error));
+    this.#startHeartbeat();
   }
 
   isOpen() {
@@ -293,6 +458,21 @@ class ExtensionBrowserConnection {
     this.#closePending(new ExtensionBrowserUnavailableError('Extension browser disconnected.'));
   }
 
+  touchPresence({ force = false } = {}) {
+    const now = Date.now();
+    const intervalMs = Number(this.presenceTouchIntervalMs);
+    if (
+      !force
+      && Number.isFinite(intervalMs)
+      && intervalMs > 0
+      && now - this.lastPresenceTouchAt < intervalMs
+    ) {
+      return;
+    }
+    this.lastPresenceTouchAt = now;
+    this.registry.touchPresence(this.userId, this.tokenId);
+  }
+
   sendCommand(command, payload = {}, options = {}) {
     if (!this.isOpen()) {
       return Promise.reject(new ExtensionBrowserUnavailableError());
@@ -317,6 +497,8 @@ class ExtensionBrowserConnection {
   }
 
   #handleMessage(data) {
+    this.lastPongAt = Date.now();
+    this.touchPresence();
     let message;
     try {
       message = parseExtensionMessage(data);
@@ -339,12 +521,41 @@ class ExtensionBrowserConnection {
 
   #closePending(error) {
     this.registry.unregisterConnection(this);
+    if (this.heartbeatTimer) {
+      clearInterval(this.heartbeatTimer);
+      this.heartbeatTimer = null;
+    }
     for (const pending of this.pending.values()) {
       clearTimeout(pending.timer);
       pending.reject(error);
     }
     this.pending.clear();
   }
+
+  #startHeartbeat() {
+    const intervalMs = Number(this.heartbeatIntervalMs);
+    const timeoutMs = Number(this.heartbeatTimeoutMs);
+    if (!Number.isFinite(intervalMs) || intervalMs <= 0) return;
+    this.touchPresence({ force: true });
+    this.heartbeatTimer = setInterval(() => {
+      if (!this.isOpen()) {
+        this.#closePending(new ExtensionBrowserUnavailableError('Extension browser disconnected.'));
+        return;
+      }
+      if (Number.isFinite(timeoutMs) && timeoutMs > 0 && Date.now() - this.lastPongAt > timeoutMs) {
+        try { this.ws.terminate(); } catch {}
+        this.#closePending(new ExtensionBrowserUnavailableError('Extension browser heartbeat timed out.'));
+        return;
+      }
+      this.touchPresence();
+      try {
+        this.ws.ping();
+      } catch (error) {
+        this.#closePending(error);
+      }
+    }, intervalMs);
+    this.heartbeatTimer.unref?.();
+  }
 }
 
 module.exports = {

package/server/services/desktop/protocol.js

@@ -18,6 +18,7 @@ const DESKTOP_COMMANDS = Object.freeze({
   PAUSE_CONTROL: 'pauseControl',
   EXECUTE_COMMAND: 'executeCommand',
   PING: 'ping',
+  MOUSE_MOVE: 'mouseMove',
 });
 
 const FRAME_TYPE_VIDEO = 0x01;

package/server/services/desktop/provider.js

@@ -148,6 +148,10 @@ class DesktopProvider {
     return this._dispatch(DESKTOP_COMMANDS.CLICK, { ...options, x, y });
   }
 
+  mouseMove(x, y, options = {}) {
+    return this._dispatch(DESKTOP_COMMANDS.MOUSE_MOVE, { ...options, x, y });
+  }
+
   drag(options = {}) {
     return this._dispatch(DESKTOP_COMMANDS.DRAG, options);
   }

package/server/services/desktop/registry.js

@@ -10,6 +10,9 @@ const {
 } = require('./protocol');
 
 const DEFAULT_COMMAND_TIMEOUT_MS = 30 * 1000;
+const DEFAULT_HEARTBEAT_INTERVAL_MS = 25 * 1000;
+const DEFAULT_HEARTBEAT_TIMEOUT_MS = 75 * 1000;
+const DEFAULT_PRESENCE_TOUCH_INTERVAL_MS = 15 * 1000;
 
 function safeJson(value) {
   try {
@@ -47,6 +50,21 @@ class DesktopCompanionRegistry {
       || process.env.NEOAGENT_DESKTOP_COMMAND_TIMEOUT_MS
       || DEFAULT_COMMAND_TIMEOUT_MS,
     );
+    this.heartbeatIntervalMs = Number(
+      options.heartbeatIntervalMs
+      || process.env.NEOAGENT_DESKTOP_HEARTBEAT_INTERVAL_MS
+      || DEFAULT_HEARTBEAT_INTERVAL_MS,
+    );
+    this.heartbeatTimeoutMs = Number(
+      options.heartbeatTimeoutMs
+      || process.env.NEOAGENT_DESKTOP_HEARTBEAT_TIMEOUT_MS
+      || DEFAULT_HEARTBEAT_TIMEOUT_MS,
+    );
+    this.presenceTouchIntervalMs = Number(
+      options.presenceTouchIntervalMs
+      || process.env.NEOAGENT_DESKTOP_PRESENCE_TOUCH_INTERVAL_MS
+      || DEFAULT_PRESENCE_TOUCH_INTERVAL_MS,
+    );
     this.connectionsByUser = new Map();
   }
 
@@ -200,6 +218,9 @@ class DesktopCompanionRegistry {
         platform: record.platform,
       },
       timeoutMs: this.commandTimeoutMs,
+      heartbeatIntervalMs: this.heartbeatIntervalMs,
+      heartbeatTimeoutMs: this.heartbeatTimeoutMs,
+      presenceTouchIntervalMs: this.presenceTouchIntervalMs,
     });
     // Install the new connection in the map BEFORE closing the old one.
     // This ensures that when the old socket's async 'close' event fires and
@@ -275,6 +296,19 @@ class DesktopCompanionRegistry {
     return this.getDeviceRecordByDeviceId(userId, deviceId);
   }
 
+  touchPresence(userId, deviceId) {
+    const userMap = this._getUserMap(userId);
+    const connection = userMap?.get(String(deviceId));
+    if (!connection?.isOpen()) return;
+    this.db.prepare(
+      `UPDATE desktop_companion_devices
+       SET status = 'online',
+           last_seen_at = datetime('now'),
+           updated_at = datetime('now')
+       WHERE user_id = ? AND device_id = ? AND revoked_at IS NULL`
+    ).run(userId, deviceId);
+  }
+
   isConnected(userId) {
     const userMap = this._getUserMap(userId);
     return userMap != null && userMap.size > 0;
@@ -493,7 +527,19 @@ class DesktopCompanionRegistry {
 }
 
 class DesktopCompanionConnection {
-  constructor({ registry, ws, userId, sessionId, deviceId, recordId, meta, timeoutMs }) {
+  constructor({
+    registry,
+    ws,
+    userId,
+    sessionId,
+    deviceId,
+    recordId,
+    meta,
+    timeoutMs,
+    heartbeatIntervalMs,
+    heartbeatTimeoutMs,
+    presenceTouchIntervalMs,
+  }) {
     this.registry = registry;
     this.ws = ws;
     this.userId = userId;
@@ -502,11 +548,22 @@ class DesktopCompanionConnection {
     this.recordId = recordId;
     this.meta = meta || {};
     this.timeoutMs = timeoutMs;
+    this.heartbeatIntervalMs = heartbeatIntervalMs;
+    this.heartbeatTimeoutMs = heartbeatTimeoutMs;
+    this.presenceTouchIntervalMs = presenceTouchIntervalMs;
     this.pending = new Map();
+    this.lastPongAt = Date.now();
+    this.lastPresenceTouchAt = 0;
+    this.heartbeatTimer = null;
 
     ws.on('message', (data) => this._handleMessage(data));
+    ws.on('pong', () => {
+      this.lastPongAt = Date.now();
+      this.touchPresence();
+    });
     ws.on('close', () => this._closePending(new DesktopCompanionUnavailableError('Desktop companion disconnected.')));
     ws.on('error', (error) => this._closePending(error));
+    this._startHeartbeat();
   }
 
   isOpen() {
@@ -525,6 +582,21 @@ class DesktopCompanionConnection {
     this._closePending(new DesktopCompanionUnavailableError('Desktop companion disconnected.'));
   }
 
+  touchPresence({ force = false } = {}) {
+    const now = Date.now();
+    const intervalMs = Number(this.presenceTouchIntervalMs);
+    if (
+      !force
+      && Number.isFinite(intervalMs)
+      && intervalMs > 0
+      && now - this.lastPresenceTouchAt < intervalMs
+    ) {
+      return;
+    }
+    this.lastPresenceTouchAt = now;
+    this.registry.touchPresence(this.userId, this.deviceId);
+  }
+
   sendCommand(command, payload = {}, options = {}) {
     if (!this.isOpen()) {
       return Promise.reject(new DesktopCompanionUnavailableError());
@@ -549,6 +621,8 @@ class DesktopCompanionConnection {
   }
 
   _handleMessage(data) {
+    this.lastPongAt = Date.now();
+    this.touchPresence();
     if (Buffer.isBuffer(data) && data.length > 0 && data[0] === FRAME_TYPE_VIDEO) {
       return;
     }
@@ -585,6 +659,10 @@ class DesktopCompanionConnection {
   }
 
   _closePending(error) {
+    if (this.heartbeatTimer) {
+      clearInterval(this.heartbeatTimer);
+      this.heartbeatTimer = null;
+    }
     for (const pending of this.pending.values()) {
       clearTimeout(pending.timer);
       pending.reject(error);
@@ -596,6 +674,31 @@ class DesktopCompanionConnection {
     // mark the device offline.
     this.registry.unregisterConnection(this);
   }
+
+  _startHeartbeat() {
+    const intervalMs = Number(this.heartbeatIntervalMs);
+    const timeoutMs = Number(this.heartbeatTimeoutMs);
+    if (!Number.isFinite(intervalMs) || intervalMs <= 0) return;
+    this.touchPresence({ force: true });
+    this.heartbeatTimer = setInterval(() => {
+      if (!this.isOpen()) {
+        this._closePending(new DesktopCompanionUnavailableError('Desktop companion disconnected.'));
+        return;
+      }
+      if (Number.isFinite(timeoutMs) && timeoutMs > 0 && Date.now() - this.lastPongAt > timeoutMs) {
+        try { this.ws.terminate(); } catch {}
+        this._closePending(new DesktopCompanionUnavailableError('Desktop companion heartbeat timed out.'));
+        return;
+      }
+      this.touchPresence();
+      try {
+        this.ws.ping();
+      } catch (error) {
+        this._closePending(error);
+      }
+    }, intervalMs);
+    this.heartbeatTimer.unref?.();
+  }
 }
 
 module.exports = {