neoagent 3.2.1-beta.0 → 3.2.1-beta.10

package/server/services/messaging/http_platforms.js

@@ -4,6 +4,7 @@ const crypto = require('crypto');
 const net = require('net');
 const tls = require('tls');
 const { BasePlatform } = require('./base');
+const { fetchResponseText } = require('../network/http');
 
 function requireText(value, label) {
   const text = String(value || '').trim();
@@ -51,14 +52,18 @@ function parseHeaders(value) {
 }
 
 async function fetchJson(url, options = {}, serviceName = 'Messaging platform') {
-  const response = await fetch(url, {
+  const { response, text } = await fetchResponseText(url, {
     ...options,
     headers: {
       ...(options.body == null ? {} : { 'content-type': 'application/json' }),
       ...(options.headers || {}),
     },
+    maxResponseBytes: Number(options.maxResponseBytes) > 0
+      ? Number(options.maxResponseBytes)
+      : 2 * 1024 * 1024,
+    serviceName,
+    timeoutMs: Number(options.timeoutMs) > 0 ? Number(options.timeoutMs) : 15000,
   });
-  const text = await response.text();
   let body = null;
   if (text) {
     try { body = JSON.parse(text); } catch { body = text; }
@@ -71,7 +76,13 @@ async function fetchJson(url, options = {}, serviceName = 'Messaging platform')
       .replace(/https?:\/\/[^\s"'<>]+/gi, '[redacted-url]')
       .replace(/\b(token|access_token|refresh_token|authorization)\b\s*[:=]\s*"[^"]*"/gi, '$1=[redacted]')
       .replace(/\b(token|access_token|refresh_token|authorization)\b\s*[:=]\s*[^,\s;"'}\]]+/gi, '$1=[redacted]');
-    throw new Error(`${serviceName} request failed (${response.status}): ${detail || response.statusText}`);
+    const error = new Error(
+      `${serviceName} request failed (${response.status}): ${detail || response.statusText}`,
+    );
+    error.status = response.status;
+    error.headers = response.headers;
+    error.safeToRetry = response.status === 429;
+    throw error;
   }
   return body;
 }
@@ -173,7 +184,7 @@ class ConfigurableHttpPlatform extends BasePlatform {
     };
   }
 
-  async sendMessage(to, content) {
+  async sendMessage(to, content, options = {}) {
     const urlTemplate = this.config.outboundUrl || this.config.webhookUrl || this.defaults.outboundUrl || this.defaults.webhookUrl;
     if (!urlTemplate) throw new Error(`${this.defaults.label || this.name} outbound URL is not configured`);
 
@@ -209,6 +220,7 @@ class ConfigurableHttpPlatform extends BasePlatform {
       method: this.config.method || this.defaults.method || 'POST',
       headers,
       body: JSON.stringify(body),
+      signal: options.signal,
     }, this.defaults.label || this.name);
     return { success: true };
   }
@@ -284,12 +296,13 @@ class SlackPlatform extends BasePlatform {
     return this._botUserId ? { username: this._botUserId } : null;
   }
 
-  async sendMessage(to, content) {
+  async sendMessage(to, content, options = {}) {
     if (!this.botToken) throw new Error('Slack bot token is required for outbound messages');
     const result = await fetchJson('https://slack.com/api/chat.postMessage', {
       method: 'POST',
       headers: { Authorization: `Bearer ${this.botToken}` },
       body: JSON.stringify({ channel: to, text: content }),
+      signal: options.signal,
     }, 'Slack');
     if (result && result.ok === false) throw new Error(`Slack post failed: ${result.error || 'unknown error'}`);
     return { success: true, ts: result?.ts };
@@ -593,12 +606,13 @@ class MatrixPlatform extends BasePlatform {
     }
   }
 
-  async sendMessage(to, content) {
+  async sendMessage(to, content, options = {}) {
     if (this.status !== 'connected') throw new Error('Matrix not connected');
     const txnId = encodeURIComponent(crypto.randomUUID());
     await this.#matrix(`/rooms/${encodeURIComponent(to)}/send/m.room.message/${txnId}`, {
       method: 'PUT',
       body: JSON.stringify({ msgtype: 'm.text', body: content }),
+      signal: options.signal,
     });
     return { success: true };
   }
@@ -618,9 +632,9 @@ class BlueBubblesPlatform extends ConfigurableHttpPlatform {
     return { status: 'connected' };
   }
 
-  async sendMessage(to, content) {
+  async sendMessage(to, content, options = {}) {
     if (this.config.outboundUrl || this.config.webhookUrl) {
-      return super.sendMessage(to, content);
+      return super.sendMessage(to, content, options);
     }
     const url = new URL(`${this.serverUrl}${this.config.sendPath || '/api/v1/message/text'}`);
     url.searchParams.set('guid', to);
@@ -629,6 +643,7 @@ class BlueBubblesPlatform extends ConfigurableHttpPlatform {
     await fetchJson(url.toString(), {
       method: 'POST',
       body: JSON.stringify({ message: content }),
+      signal: options.signal,
     }, this.defaults.label);
     return { success: true };
   }
@@ -713,9 +728,9 @@ class SignalPlatform extends ConfigurableHttpPlatform {
     }
   }
 
-  async sendMessage(to, content) {
+  async sendMessage(to, content, options = {}) {
     if (this.config.outboundUrl || this.config.webhookUrl) {
-      return super.sendMessage(to, content);
+      return super.sendMessage(to, content, options);
     }
     await fetchJson(`${this.restUrl}/v2/send`, {
       method: 'POST',
@@ -724,6 +739,7 @@ class SignalPlatform extends ConfigurableHttpPlatform {
         number: this.account,
         recipients: [to],
       }),
+      signal: options.signal,
     }, 'Signal');
     return { success: true };
   }
@@ -743,7 +759,7 @@ class LinePlatform extends ConfigurableHttpPlatform {
     return { status: 'connected' };
   }
 
-  async sendMessage(to, content) {
+  async sendMessage(to, content, options = {}) {
     const token = requireText(this.config.channelAccessToken || this.config.token, 'LINE channel access token');
     await fetchJson('https://api.line.me/v2/bot/message/push', {
       method: 'POST',
@@ -752,6 +768,7 @@ class LinePlatform extends ConfigurableHttpPlatform {
         to,
         messages: [{ type: 'text', text: content }],
       }),
+      signal: options.signal,
     }, 'LINE');
     return { success: true };
   }
@@ -815,14 +832,17 @@ class MattermostPlatform extends ConfigurableHttpPlatform {
     return { status: 'connected' };
   }
 
-  async sendMessage(to, content) {
-    if (this.config.webhookUrl || this.config.outboundUrl) return super.sendMessage(to, content);
+  async sendMessage(to, content, options = {}) {
+    if (this.config.webhookUrl || this.config.outboundUrl) {
+      return super.sendMessage(to, content, options);
+    }
     const baseUrl = trimTrailingSlash(this.config.baseUrl);
     const token = requireText(this.config.token, 'Mattermost token');
     await fetchJson(`${baseUrl}/api/v4/posts`, {
       method: 'POST',
       headers: { Authorization: `Bearer ${token}` },
       body: JSON.stringify({ channel_id: to, message: content }),
+      signal: options.signal,
     }, 'Mattermost');
     return { success: true };
   }

package/server/services/messaging/inbound_queue.js

@@ -2,6 +2,56 @@
 
 const { getErrorMessage } = require('../bootstrap_helpers');
 
+function completionResult(outcome, cancelled = false) {
+  return {
+    runId: outcome?.runId || null,
+    result: outcome?.result || null,
+    error: outcome?.error || null,
+    cancelled,
+  };
+}
+
+function settleWaiters(item, result) {
+  for (const resolve of item?.waiters || []) resolve(result);
+  if (item?.waiters) item.waiters = [];
+}
+
+function cancelPending(queue) {
+  const result = completionResult(null, true);
+  for (const item of queue.pending.splice(0)) settleWaiters(item, result);
+}
+
+function queuedResult(queue, msg) {
+  let resolveCompletion;
+  const completion = new Promise((resolve) => {
+    resolveCompletion = resolve;
+  });
+  const last = queue.pending[queue.pending.length - 1];
+  if (
+    last
+    && last.message.platform === msg.platform
+    && last.message.chatId === msg.chatId
+    && String(last.message.sender || '') === String(msg.sender || '')
+  ) {
+    last.message.content += `\n${msg.content}`;
+    last.message.messageId = msg.messageId;
+    last.message.inboundJobIds = Array.from(new Set([
+      ...(last.message.inboundJobIds || []),
+      ...(msg.inboundJobIds || []),
+      msg.inboundJobId,
+    ].filter(Boolean)));
+    last.waiters.push(resolveCompletion);
+  } else {
+    queue.pending.push({
+      message: { ...msg },
+      waiters: [resolveCompletion],
+    });
+  }
+  const result = { queued: true };
+  Object.defineProperty(result, 'completion', { value: completion });
+  return result;
+}
+
 async function processInboundQueue({
   userQueues,
   userId,
@@ -12,79 +62,83 @@ async function processInboundQueue({
   const agentId = msg.agentId || null;
   const queueKey = `${userId}:${agentId || 'main'}`;
   if (!userQueues[queueKey]) {
-    userQueues[queueKey] = { running: false, pending: [], cancelRequested: false };
+    userQueues[queueKey] = {
+      running: false,
+      pending: [],
+      cancelRequested: false,
+      cancelPending() {
+        cancelPending(this);
+      },
+    };
   }
   const queue = userQueues[queueKey];
 
   if (queue.cancelRequested && !queue.running) {
-    queue.pending = [];
+    cancelPending(queue);
     queue.cancelRequested = false;
   }
 
   if (queue.running) {
-    const last = queue.pending[queue.pending.length - 1];
-    if (
-      last
-      && last.platform === msg.platform
-      && last.chatId === msg.chatId
-      && String(last.sender || '') === String(msg.sender || '')
-    ) {
-      last.content += `\n${msg.content}`;
-      last.messageId = msg.messageId;
-    } else {
-      queue.pending.push({ ...msg });
-    }
-    return { queued: true };
+    return queuedResult(queue, msg);
   }
 
   queue.running = true;
-  let currentMessage = msg;
+  let currentItem = { message: msg, waiters: [] };
   let processedCount = 0;
   let failedCount = 0;
   let cancelled = false;
+  let initialOutcome = null;
 
   try {
-    while (currentMessage) {
+    while (currentItem) {
       let outcome;
       try {
-        outcome = await executeMessage(currentMessage);
+        outcome = await executeMessage(currentItem.message);
       } catch (error) {
         outcome = { runId: null, result: null, error };
       }
       processedCount += 1;
+      const itemCancelled = queue.cancelRequested;
+      const itemResult = completionResult(outcome, itemCancelled);
+      if (!initialOutcome) initialOutcome = itemResult;
+      settleWaiters(currentItem, itemResult);
 
-      if (outcome?.error) {
+      if (outcome?.error && !queue.cancelRequested) {
         failedCount += 1;
         await notifyProcessingError(onProcessingError, {
           error: outcome.error,
           runId: outcome.runId,
           userId,
-          failedMessage: currentMessage
+          failedMessage: currentItem.message
         });
       }
 
       if (queue.cancelRequested) {
-        queue.pending = [];
+        cancelPending(queue);
         cancelled = true;
         break;
       }
 
-      currentMessage = queue.pending.shift() || null;
+      currentItem = queue.pending.shift() || null;
     }
   } finally {
     queue.running = false;
-    queue.pending = [];
+    cancelPending(queue);
     queue.cancelRequested = false;
     if (userQueues[queueKey] === queue) {
       delete userQueues[queueKey];
     }
   }
 
-  return {
+  const result = {
     processedCount,
     failedCount,
     cancelled
   };
+  Object.defineProperty(result, 'outcome', {
+    value: initialOutcome || completionResult(null, cancelled),
+  });
+  return result;
 }
 
 async function notifyProcessingError(handler, details) {

package/server/services/messaging/inbound_store.js

@@ -0,0 +1,224 @@
+'use strict';
+
+const { randomUUID } = require('node:crypto');
+const db = require('../../db/database');
+
+const MAX_INBOUND_PAYLOAD_BYTES = 1024 * 1024;
+const JOB_STATUSES = new Set(['pending', 'processing', 'completed', 'failed']);
+
+function encodePayload(payload) {
+  const encoded = JSON.stringify(payload);
+  if (Buffer.byteLength(encoded, 'utf8') > MAX_INBOUND_PAYLOAD_BYTES) {
+    const error = new Error('Inbound messaging payload exceeds the 1 MiB durability limit.');
+    error.code = 'MESSAGING_INBOUND_PAYLOAD_TOO_LARGE';
+    throw error;
+  }
+  return encoded;
+}
+
+function decodePayload(value) {
+  try {
+    const payload = JSON.parse(String(value || ''));
+    return payload && typeof payload === 'object' && !Array.isArray(payload)
+      ? payload
+      : null;
+  } catch {
+    return null;
+  }
+}
+
+function getJobByMessageId(messageId) {
+  return db.prepare(
+    'SELECT * FROM messaging_inbound_jobs WHERE message_id = ?',
+  ).get(messageId) || null;
+}
+
+function enqueueInboundMessage({
+  userId,
+  agentId,
+  platform,
+  platformMessageId,
+  chatId,
+  content,
+  metadata,
+  createdAt,
+  payload,
+}) {
+  return db.transaction(() => {
+    if (platformMessageId) {
+      const existing = db.prepare(
+        `SELECT id
+         FROM messages
+         WHERE user_id = ? AND platform = ? AND platform_msg_id = ? AND role = 'user'
+         LIMIT 1`,
+      ).get(userId, platform, platformMessageId);
+      if (existing) {
+        return {
+          created: false,
+          job: getJobByMessageId(existing.id),
+          messageId: existing.id,
+        };
+      }
+    }
+
+    const jobId = randomUUID();
+    const durablePayload = {
+      ...payload,
+      inboundJobId: jobId,
+      inboundJobIds: [jobId],
+    };
+    const insert = db.prepare(
+      `INSERT INTO messages (
+        user_id, agent_id, role, content, platform, platform_msg_id,
+        platform_chat_id, metadata, created_at
+      ) VALUES (?, ?, 'user', ?, ?, ?, ?, ?, ?)`,
+    ).run(
+      userId,
+      agentId,
+      content,
+      platform,
+      platformMessageId || null,
+      chatId,
+      metadata ? JSON.stringify(metadata) : null,
+      createdAt || new Date().toISOString(),
+    );
+    const messageId = Number(insert.lastInsertRowid);
+    db.prepare(
+      `INSERT INTO messaging_inbound_jobs (
+        id, message_id, user_id, agent_id, platform, platform_msg_id,
+        platform_chat_id, payload_json, status
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'pending')`,
+    ).run(
+      jobId,
+      messageId,
+      userId,
+      agentId,
+      platform,
+      platformMessageId || null,
+      chatId,
+      encodePayload(durablePayload),
+    );
+    return {
+      created: true,
+      job: getJobByMessageId(messageId),
+      messageId,
+      payload: durablePayload,
+    };
+  })();
+}
+
+function claimInboundJob(jobId) {
+  const result = db.prepare(
+    `UPDATE messaging_inbound_jobs
+     SET status = 'processing',
+         attempts = attempts + 1,
+         last_error = NULL,
+         updated_at = datetime('now')
+     WHERE id = ? AND status = 'pending'`,
+  ).run(jobId);
+  return result.changes === 1;
+}
+
+function settleInboundJob(jobId, status, error = null) {
+  if (!JOB_STATUSES.has(status) || status === 'processing') {
+    throw new Error(`Invalid inbound messaging job status: ${status}`);
+  }
+  db.prepare(
+    `UPDATE messaging_inbound_jobs
+     SET status = ?,
+         last_error = ?,
+         completed_at = CASE WHEN ? IN ('completed', 'failed') THEN datetime('now') ELSE NULL END,
+         updated_at = datetime('now')
+     WHERE id = ?`,
+  ).run(
+    status,
+    error ? String(error).slice(0, 4000) : null,
+    status,
+    jobId,
+  );
+}
+
+function attachRunToInboundJobs(jobIds, runId) {
+  const ids = Array.from(new Set(
+    (Array.isArray(jobIds) ? jobIds : [jobIds])
+      .map((value) => String(value || '').trim())
+      .filter(Boolean),
+  ));
+  if (!ids.length || !runId) return 0;
+  const update = db.prepare(
+    `UPDATE messaging_inbound_jobs
+     SET run_id = ?, updated_at = datetime('now')
+     WHERE id = ? AND status = 'processing'`,
+  );
+  return db.transaction(() => ids.reduce(
+    (count, id) => count + update.run(runId, id).changes,
+    0,
+  ))();
+}
+
+function reconcileInterruptedInboundJobs() {
+  return db.transaction(() => {
+    const completed = db.prepare(
+      `UPDATE messaging_inbound_jobs
+       SET status = 'completed', completed_at = datetime('now'), updated_at = datetime('now')
+       WHERE status = 'processing'
+         AND run_id IN (SELECT id FROM agent_runs WHERE status = 'completed')`,
+    ).run().changes;
+    const failed = db.prepare(
+      `UPDATE messaging_inbound_jobs
+       SET status = 'failed',
+           last_error = COALESCE(last_error, 'The server restarted after this agent run began; it will not be replayed automatically.'),
+           completed_at = datetime('now'),
+           updated_at = datetime('now')
+       WHERE status = 'processing'
+         AND run_id IN (SELECT id FROM agent_runs)`,
+    ).run().changes;
+    const pending = db.prepare(
+      `UPDATE messaging_inbound_jobs
+       SET status = 'pending',
+           last_error = NULL,
+           updated_at = datetime('now')
+       WHERE status = 'processing'
+         AND (run_id IS NULL OR run_id NOT IN (SELECT id FROM agent_runs))`,
+    ).run().changes;
+    return { completed, failed, pending };
+  })();
+}
+
+function listPendingInboundJobs(filters = {}) {
+  const clauses = ["status = 'pending'"];
+  const values = [];
+  for (const [column, value] of [
+    ['user_id', filters.userId],
+    ['agent_id', filters.agentId],
+    ['platform', filters.platform],
+  ]) {
+    if (value === undefined) continue;
+    if (value === null) clauses.push(`${column} IS NULL`);
+    else {
+      clauses.push(`${column} = ?`);
+      values.push(value);
+    }
+  }
+  const limit = Math.max(1, Math.min(500, Number(filters.limit) || 100));
+  return db.prepare(
+    `SELECT * FROM messaging_inbound_jobs
+     WHERE ${clauses.join(' AND ')}
+     ORDER BY created_at ASC, id ASC
+     LIMIT ?`,
+  ).all(...values, limit);
+}
+
+function payloadForInboundJob(job) {
+  return decodePayload(job?.payload_json);
+}
+
+module.exports = {
+  attachRunToInboundJobs,
+  claimInboundJob,
+  enqueueInboundMessage,
+  listPendingInboundJobs,
+  payloadForInboundJob,
+  reconcileInterruptedInboundJobs,
+  settleInboundJob,
+};