neoagent 2.4.1-beta.8 → 2.4.1

package/server/services/messaging/automation.js

@@ -18,9 +18,12 @@ const {
   buildVoiceMessagingRunOptions,
   isVoiceLikeMessage,
 } = require('../voice/runtime');
+const { getErrorMessage } = require('../bootstrap_helpers');
+const { processInboundQueue } = require('./inbound_queue');
+const { startTypingKeepalive } = require('./typing_keepalive');
 
 function registerMessagingAutomation({ app, io, messagingManager, agentEngine }) {
-  const userQueues = {};
+  const userQueues = Object.create(null);
   app.locals.userQueues = userQueues;
 
   messagingManager.registerHandler(async (userId, msg) => {
@@ -100,62 +103,85 @@ function registerMessagingAutomation({ app, io, messagingManager, agentEngine })
       messagingManager,
       agentEngine,
       userId,
-      msg
+      msg,
+      onProcessingError: ({ error, runId, failedMessage }) => {
+        const errorMessage = getErrorMessage(error);
+        console.error(
+          `[MessagingAutomation] Agent run failed platform=${failedMessage.platform} user=${userId}:`,
+          errorMessage
+        );
+        io.to(`user:${userId}`).emit('messaging:error', {
+          error: `NeoAgent could not finish the ${failedMessage.platform} request. Check the run history for details.`,
+          platform: failedMessage.platform,
+          chatId: failedMessage.chatId,
+          runId
+        });
+      }
     });
   });
 }
 
 async function processQueuedMessage({
   userQueues,
+  messagingManager,
+  agentEngine,
+  userId,
+  msg,
+  onProcessingError = null
+}) {
+  return processInboundQueue({
+    userQueues,
+    userId,
+    msg,
+    executeMessage: (queuedMessage) =>
+      executeQueuedMessage({
+        messagingManager,
+        agentEngine,
+        userId,
+        msg: queuedMessage
+      }),
+    onProcessingError
+  });
+}
+
+async function executeQueuedMessage({
   messagingManager,
   agentEngine,
   userId,
   msg
 }) {
   const agentId = msg.agentId || null;
-  const queueKey = `${userId}:${agentId || 'main'}`;
-  if (!userQueues[queueKey]) {
-    userQueues[queueKey] = { running: false, pending: [], cancelRequested: false };
-  }
-  const queue = userQueues[queueKey];
+  const runId = randomUUID();
+  const reportSideEffectError = (operation, error) => {
+    console.warn(
+      `[MessagingAutomation] ${operation} failed platform=${msg.platform} user=${userId}:`,
+      getErrorMessage(error)
+    );
+  };
 
-  if (queue.cancelRequested && !queue.running) {
-    queue.pending = [];
-    queue.cancelRequested = false;
+  try {
+    await messagingManager.markRead(
+      userId,
+      msg.platform,
+      msg.chatId,
+      msg.messageId,
+      { agentId }
+    );
+  } catch (error) {
+    reportSideEffectError('mark read', error);
   }
 
-  if (queue.running) {
-    const last = queue.pending[queue.pending.length - 1];
-    if (
-      last
-      && last.platform === msg.platform
-      && last.chatId === msg.chatId
-      && String(last.sender || '') === String(msg.sender || '')
-    ) {
-      last.content += `\n${msg.content}`;
-      last.messageId = msg.messageId;
-    } else {
-      queue.pending.push({ ...msg });
-    }
-    return;
-  }
+  const stopTypingKeepalive = startTypingKeepalive({
+    messagingManager,
+    userId,
+    agentId,
+    runId,
+    platform: msg.platform,
+    chatId: msg.chatId,
+    onError: reportSideEffectError
+  });
 
-  queue.running = true;
-  let stopTypingKeepalive = async () => {};
   try {
-    const runId = randomUUID();
-    await messagingManager
-      .markRead(userId, msg.platform, msg.chatId, msg.messageId, { agentId })
-      .catch(() => {});
-    stopTypingKeepalive = startTypingKeepalive({
-      messagingManager,
-      userId,
-      agentId,
-      runId,
-      platform: msg.platform,
-      chatId: msg.chatId
-    });
-
     const prompt = buildIncomingPrompt(msg);
     const conversationId = ensureConversation(userId, msg);
     const runOptions = isVoiceLikeMessage(msg)
@@ -182,105 +208,15 @@ async function processQueuedMessage({
       ];
     }
 
-    await agentEngine.run(userId, prompt, runOptions);
+    const result = await agentEngine.run(userId, prompt, runOptions);
+    return { runId, result, error: null };
+  } catch (error) {
+    return { runId, result: null, error };
   } finally {
     await stopTypingKeepalive();
-    if (queue.cancelRequested) {
-      queue.pending = [];
-      queue.running = false;
-      queue.cancelRequested = false;
-      return;
-    }
-    queue.running = false;
-    if (queue.pending.length > 0) {
-      const next = queue.pending.shift();
-      await processQueuedMessage({
-        userQueues,
-        messagingManager,
-        agentEngine,
-        userId,
-        msg: next
-      });
-    }
   }
 }
 
-function startTypingKeepalive({
-  messagingManager,
-  userId,
-  agentId,
-  runId,
-  platform,
-  chatId,
-  intervalMs = 4000
-}) {
-  let stopped = false;
-  let timer = null;
-  let releaseWait = null;
-  let stopPromise = null;
-
-  const matchesRunDelivery = (event) => (
-    event?.runId
-    && runId
-    && event.runId === runId
-    && event.userId === userId
-    && event.platform === platform
-    && event.to === chatId
-  );
-
-  const onMessageSent = (event) => {
-    if (matchesRunDelivery(event) && event.deliveryKind !== 'interim') {
-      stop().catch(() => {});
-    }
-  };
-
-  if (typeof messagingManager?.on === 'function' && typeof messagingManager?.off === 'function') {
-    messagingManager.on('message_sent', onMessageSent);
-  }
-
-  const wait = () =>
-    new Promise((resolve) => {
-      releaseWait = resolve;
-      timer = setTimeout(resolve, intervalMs);
-    });
-
-  const loop = (async () => {
-    while (!stopped) {
-      await messagingManager
-        .sendTyping(userId, platform, chatId, true, { agentId })
-        .catch(() => {});
-
-      if (stopped) break;
-      await wait();
-    }
-  })();
-
-  const stop = async () => {
-    if (stopPromise) return stopPromise;
-    stopPromise = (async () => {
-      if (typeof messagingManager?.off === 'function') {
-        messagingManager.off('message_sent', onMessageSent);
-      }
-      stopped = true;
-      if (timer) {
-        clearTimeout(timer);
-        timer = null;
-      }
-      if (releaseWait) {
-        releaseWait();
-        releaseWait = null;
-      }
-      await loop.catch(() => {});
-      await messagingManager
-        .sendTyping(userId, platform, chatId, false, { agentId })
-        .catch(() => {});
-    })();
-    return stopPromise;
-  };
-
-  return stop;
-}
-
 function ensureConversation(userId, msg) {
   const agentId = msg.agentId || null;
   let conversation = db
@@ -419,6 +355,7 @@ function emitBlockedSenderSuggestion({ io, userId, msg }) {
 module.exports = {
   buildIncomingPrompt,
   buildSenderIdentityBlock,
+  executeQueuedMessage,
   isAllowedMessagingSender,
   processQueuedMessage,
   registerMessagingAutomation,

package/server/services/messaging/formatting_guides.js

@@ -99,9 +99,34 @@ function adaptSpokenFormatting(text) {
     .trim();
 }
 
+function coerceMessageContent(content) {
+  if (typeof content === 'string') return content;
+  if (content == null) return '';
+  if (typeof content === 'number' || typeof content === 'boolean' || typeof content === 'bigint') {
+    return String(content);
+  }
+  if (typeof content !== 'object') return String(content || '');
+
+  for (const key of ['text', 'content', 'message', 'summary']) {
+    if (typeof content[key] === 'string' && content[key].trim()) {
+      return content[key];
+    }
+  }
+
+  if (content.result != null && content.result !== content) {
+    return coerceMessageContent(content.result);
+  }
+
+  try {
+    return JSON.stringify(content, null, 2);
+  } catch {
+    return String(content || '');
+  }
+}
+
 function normalizeOutgoingMessageForPlatform(platform, content, options = {}) {
   const profile = getPlatformFormattingProfile(platform);
-  let text = String(content || '');
+  let text = coerceMessageContent(content);
 
   if (options.stripNoResponseMarker !== false) {
     text = text.replace(/\[NO RESPONSE\]/gi, '');

package/server/services/messaging/inbound_queue.js

@@ -0,0 +1,111 @@
+'use strict';
+
+const { getErrorMessage } = require('../bootstrap_helpers');
+
+async function processInboundQueue({
+  userQueues,
+  userId,
+  msg,
+  executeMessage,
+  onProcessingError = null
+}) {
+  const agentId = msg.agentId || null;
+  const queueKey = `${userId}:${agentId || 'main'}`;
+  if (!userQueues[queueKey]) {
+    userQueues[queueKey] = { running: false, pending: [], cancelRequested: false };
+  }
+  const queue = userQueues[queueKey];
+
+  if (queue.cancelRequested && !queue.running) {
+    queue.pending = [];
+    queue.cancelRequested = false;
+  }
+
+  if (queue.running) {
+    const last = queue.pending[queue.pending.length - 1];
+    if (
+      last
+      && last.platform === msg.platform
+      && last.chatId === msg.chatId
+      && String(last.sender || '') === String(msg.sender || '')
+    ) {
+      last.content += `\n${msg.content}`;
+      last.messageId = msg.messageId;
+    } else {
+      queue.pending.push({ ...msg });
+    }
+    return { queued: true };
+  }
+
+  queue.running = true;
+  let currentMessage = msg;
+  let processedCount = 0;
+  let failedCount = 0;
+  let cancelled = false;
+
+  try {
+    while (currentMessage) {
+      let outcome;
+      try {
+        outcome = await executeMessage(currentMessage);
+      } catch (error) {
+        outcome = { runId: null, result: null, error };
+      }
+      processedCount += 1;
+
+      if (outcome?.error) {
+        failedCount += 1;
+        await notifyProcessingError(onProcessingError, {
+          error: outcome.error,
+          runId: outcome.runId,
+          userId,
+          failedMessage: currentMessage
+        });
+      }
+
+      if (queue.cancelRequested) {
+        queue.pending = [];
+        cancelled = true;
+        break;
+      }
+
+      currentMessage = queue.pending.shift() || null;
+    }
+  } finally {
+    queue.running = false;
+    queue.pending = [];
+    queue.cancelRequested = false;
+    if (userQueues[queueKey] === queue) {
+      delete userQueues[queueKey];
+    }
+  }
+
+  return {
+    processedCount,
+    failedCount,
+    cancelled
+  };
+}
+
+async function notifyProcessingError(handler, details) {
+  if (typeof handler !== 'function') {
+    console.error(
+      `[MessagingAutomation] Agent run failed platform=${details.failedMessage.platform} user=${details.userId}:`,
+      getErrorMessage(details.error)
+    );
+    return;
+  }
+
+  try {
+    await handler(details);
+  } catch (error) {
+    console.error(
+      '[MessagingAutomation] Failed to report an agent run error:',
+      getErrorMessage(error)
+    );
+  }
+}
+
+module.exports = {
+  processInboundQueue
+};

package/server/services/messaging/manager.js

@@ -104,6 +104,9 @@ class MessagingManager extends EventEmitter {
       return null;
     }
 
+    const normalizedIncomingContent = normalizeOutgoingMessageForPlatform(platformName, msg?.content, {
+      stripNoResponseMarker: false
+    });
     const agentId = this._agentId(userId, {
       ...options,
       agentId: options?.agentId ?? msg?.agentId ?? null,
@@ -135,7 +138,7 @@ class MessagingManager extends EventEmitter {
         userId,
         agentId,
         'user',
-        msg.content,
+        normalizedIncomingContent,
         platformName,
         msg.messageId,
         msg.chatId,
@@ -143,7 +146,7 @@ class MessagingManager extends EventEmitter {
         msg.timestamp,
       );
 
-    const enrichedMsg = { ...msg, agentId, platform: platformName };
+    const enrichedMsg = { ...msg, content: normalizedIncomingContent, agentId, platform: platformName };
 
     if (this.isShuttingDown) {
       return enrichedMsg;
@@ -317,6 +320,9 @@ class MessagingManager extends EventEmitter {
     config.userId = userId;
     config.agentId = agentId;
     config.accessPolicy = this._loadAccessPolicy(userId, agentId, platformName);
+    const existingConnection = db
+      .prepare('SELECT id, status FROM platform_connections WHERE user_id = ? AND agent_id = ? AND platform = ?')
+      .get(userId, agentId, platformName);
     const PlatformClass = this.platformTypes[platformName];
     if (!PlatformClass) throw new Error(`Unknown platform: ${platformName}`);
     if (platformName === 'meshtastic' && !readMeshtasticEnabled()) {
@@ -325,7 +331,18 @@ class MessagingManager extends EventEmitter {
 
     if (platformName === 'whatsapp' && !config.authDir) {
       config.authDir = this._scopedPlatformAuthDir(userId, agentId, platformName);
-      this._maybeMigrateLegacyWhatsAppAuth(config.authDir);
+      let shouldMigrateLegacyAuth = true;
+      if (
+        existingConnection &&
+        existingConnection.status !== 'connected' &&
+        existingConnection.status !== 'awaiting_qr'
+      ) {
+        fs.rmSync(config.authDir, { recursive: true, force: true });
+        shouldMigrateLegacyAuth = false;
+      }
+      if (shouldMigrateLegacyAuth) {
+        this._maybeMigrateLegacyWhatsAppAuth(config.authDir);
+      }
     }
 
     // For Telnyx, inject saved whitelist and voice secret into config before constructing
@@ -443,8 +460,7 @@ class MessagingManager extends EventEmitter {
       await this.ingestMessage(userId, platformName, msg, { agentId });
     });
 
-    const existing = db.prepare('SELECT id FROM platform_connections WHERE user_id = ? AND agent_id = ? AND platform = ?').get(userId, agentId, platformName);
-    if (!existing) {
+    if (!existingConnection) {
       db.prepare('INSERT INTO platform_connections (user_id, agent_id, platform, config, status) VALUES (?, ?, ?, ?, ?)')
         .run(userId, agentId, platformName, storedConfig, 'connecting');
     } else {
@@ -460,10 +476,11 @@ class MessagingManager extends EventEmitter {
     const agentId = this._agentId(userId, options);
     const key = this._key(userId, agentId, platformName);
     const platform = this.platforms.get(key);
-    if (!platform) return { status: 'not_connected' };
 
-    await platform.disconnect();
-    this.platforms.delete(key);
+    if (platform) {
+      await platform.disconnect();
+      this.platforms.delete(key);
+    }
 
     db.prepare('UPDATE platform_connections SET status = ? WHERE user_id = ? AND agent_id = ? AND platform = ?')
       .run('disconnected', userId, agentId, platformName);

package/server/services/messaging/typing_keepalive.js

@@ -0,0 +1,110 @@
+'use strict';
+
+const { getErrorMessage } = require('../bootstrap_helpers');
+
+function startTypingKeepalive({
+  messagingManager,
+  userId,
+  agentId,
+  runId,
+  platform,
+  chatId,
+  intervalMs = 4000,
+  onError = null
+}) {
+  let stopped = false;
+  let timer = null;
+  let releaseWait = null;
+  let stopPromise = null;
+  const reportedFailures = new Set();
+
+  const reportFailure = (operation, error) => {
+    if (reportedFailures.has(operation)) return;
+    reportedFailures.add(operation);
+    if (typeof onError === 'function') {
+      try {
+        onError(operation, error);
+      } catch (reportError) {
+        console.error(
+          '[MessagingAutomation] Typing failure reporter failed:',
+          getErrorMessage(reportError)
+        );
+      }
+    }
+  };
+
+  const matchesRunDelivery = (event) => (
+    event?.runId
+    && runId
+    && event.runId === runId
+    && event.userId === userId
+    && event.platform === platform
+    && event.to === chatId
+  );
+
+  const onMessageSent = (event) => {
+    if (matchesRunDelivery(event) && event.deliveryKind !== 'interim') {
+      stop().catch((error) => reportFailure('stop typing keepalive', error));
+    }
+  };
+
+  if (typeof messagingManager?.on === 'function' && typeof messagingManager?.off === 'function') {
+    messagingManager.on('message_sent', onMessageSent);
+  }
+
+  const wait = () =>
+    new Promise((resolve) => {
+      releaseWait = resolve;
+      timer = setTimeout(resolve, intervalMs);
+    });
+
+  const sendTyping = async (isTyping, operation) => {
+    try {
+      await messagingManager.sendTyping(
+        userId,
+        platform,
+        chatId,
+        isTyping,
+        { agentId }
+      );
+    } catch (error) {
+      reportFailure(operation, error);
+    }
+  };
+
+  const loop = (async () => {
+    while (!stopped) {
+      await sendTyping(true, 'send typing indicator');
+
+      if (stopped) break;
+      await wait();
+    }
+  })();
+
+  const stop = async () => {
+    if (stopPromise) return stopPromise;
+    stopPromise = (async () => {
+      if (typeof messagingManager?.off === 'function') {
+        messagingManager.off('message_sent', onMessageSent);
+      }
+      stopped = true;
+      if (timer) {
+        clearTimeout(timer);
+        timer = null;
+      }
+      if (releaseWait) {
+        releaseWait();
+        releaseWait = null;
+      }
+      await loop.catch((error) => reportFailure('typing keepalive loop', error));
+      await sendTyping(false, 'clear typing indicator');
+    })();
+    return stopPromise;
+  };
+
+  return stop;
+}
+
+module.exports = {
+  startTypingKeepalive
+};

package/server/services/runtime/manager.js

@@ -31,10 +31,11 @@ class RuntimeManager {
     this.artifactStore = options.artifactStore || null;
 
     this.getExtensionBrowserProvider = options.getExtensionBrowserProvider
-      || ((userId) => new ExtensionBrowserProvider({
+      || ((userId, providerOptions = {}) => new ExtensionBrowserProvider({
         registry: options.browserExtensionRegistry,
         artifactStore: options.artifactStore,
         userId,
+        tokenId: providerOptions.tokenId || null,
       }));
 
     this.getDesktopCliProvider = options.getDesktopCliProvider
@@ -51,10 +52,11 @@ class RuntimeManager {
   }
 
   hasActiveExtensionBrowser(userId) {
+    const settings = this.getSettings(userId);
     return Boolean(
       this.browserExtensionRegistry
       && typeof this.browserExtensionRegistry.isConnected === 'function'
-      && this.browserExtensionRegistry.isConnected(userId)
+      && this.browserExtensionRegistry.isConnected(userId, settings.browser_extension_token_id)
     );
   }
 
@@ -92,7 +94,9 @@ class RuntimeManager {
   async getBrowserProviderForUser(userId) {
     const settings = this.getSettings(userId);
     if (settings.browser_backend === 'extension' && this.hasActiveExtensionBrowser(userId)) {
-      return this.getExtensionBrowserProvider(userId);
+      return this.getExtensionBrowserProvider(userId, {
+        tokenId: settings.browser_extension_token_id,
+      });
     }
     return this.browserBackend.getBrowserProviderForUser(userId);
   }

package/server/services/runtime/settings.js

@@ -10,6 +10,9 @@ function createDefaultRuntimeSettings() {
     android_backend: policy.runtimeDefaults.android_backend,
     mcp_backend: policy.runtimeDefaults.mcp_backend,
     cli_backend: policy.runtimeDefaults.cli_backend ?? 'vm',
+    browser_extension_token_id: null,
+    selected_browser_extension_token_id: null,
+    cli_desktop_device_id: null,
   };
 }
 
@@ -26,6 +29,9 @@ const BASE_FALLBACK_SETTINGS = Object.freeze({
   android_backend: 'host',
   mcp_backend: 'host-remote',
   cli_backend: 'vm',
+  browser_extension_token_id: null,
+  selected_browser_extension_token_id: null,
+  cli_desktop_device_id: null,
 });
 
 const RUNTIME_SETTING_KEYS = Object.freeze(Object.keys(DEFAULT_RUNTIME_SETTINGS));
@@ -35,6 +41,11 @@ function normalizeChoice(value, allowed, fallback) {
   return allowed.includes(normalized) ? normalized : fallback;
 }
 
+function normalizeOptionalString(value) {
+  const normalized = String(value || '').trim();
+  return normalized || null;
+}
+
 function deriveDefaultsForProfile(profile) {
   switch (profile) {
     case 'secure-vm':
@@ -57,6 +68,9 @@ function normalizeRuntimeSettings(raw = {}) {
   const runtimeBackend = normalizeChoice(raw.runtime_backend, ['vm'], derived.runtime_backend);
   const browserBackend = normalizeChoice(raw.browser_backend, ['vm', 'extension'], derived.browser_backend);
   const cliBackend = normalizeChoice(raw.cli_backend, ['vm', 'desktop'], derived.cli_backend ?? 'vm');
+  const selectedExtensionTokenId = normalizeOptionalString(
+    raw.browser_extension_token_id || raw.selected_browser_extension_token_id,
+  );
   const androidBackend = 'host';
   return {
     runtime_profile: profile === 'trusted-host' ? 'secure-vm' : profile,
@@ -65,6 +79,9 @@ function normalizeRuntimeSettings(raw = {}) {
     android_backend: androidBackend,
     mcp_backend: 'host-remote',
     cli_backend: cliBackend === 'desktop' ? 'desktop' : 'vm',
+    browser_extension_token_id: selectedExtensionTokenId,
+    selected_browser_extension_token_id: selectedExtensionTokenId,
+    cli_desktop_device_id: normalizeOptionalString(raw.cli_desktop_device_id),
   };
 }