neoagent 2.4.1-beta.34 → 2.4.1-beta.36

package/server/services/memory/ingestion.js

@@ -3,292 +3,136 @@
 const { v4: uuidv4 } = require('uuid');
 const db = require('../../db/database');
 const { resolveAgentId } = require('../agents/manager');
-const { compactTextPayload } = require('../ai/preModelCompaction');
-
-const SOURCE_TYPES = Object.freeze([
-  'email',
-  'calendar',
-  'chat',
-  'docs',
-  'tickets',
-  'repos',
-  'files',
-  'crm',
-  'payments',
-  'notes',
-]);
-
-const FRESHNESS_POLICIES = Object.freeze({
-  email: Object.freeze({ intervalMinutes: 60, staleAfterDays: 14 }),
-  calendar: Object.freeze({ intervalMinutes: 180, staleAfterDays: 30 }),
-  chat: Object.freeze({ intervalMinutes: 60, staleAfterDays: 14 }),
-  docs: Object.freeze({ intervalMinutes: 360, staleAfterDays: 60 }),
-  tickets: Object.freeze({ intervalMinutes: 120, staleAfterDays: 30 }),
-  repos: Object.freeze({ intervalMinutes: 180, staleAfterDays: 45 }),
-  files: Object.freeze({ intervalMinutes: 360, staleAfterDays: 60 }),
-  crm: Object.freeze({ intervalMinutes: 240, staleAfterDays: 45 }),
-  payments: Object.freeze({ intervalMinutes: 360, staleAfterDays: 90 }),
-  notes: Object.freeze({ intervalMinutes: 360, staleAfterDays: 90 }),
-});
-
-const SOURCE_MEMORY_CATEGORIES = Object.freeze({
-  email: 'episodic',
-  calendar: 'events',
-  chat: 'episodic',
-  docs: 'projects',
-  tickets: 'tasks',
-  repos: 'projects',
-  files: 'episodic',
-  crm: 'contacts',
-  payments: 'events',
-  notes: 'episodic',
-});
-
-const INTEGRATION_SOURCE_TYPES = Object.freeze({
-  google_workspace: Object.freeze({
-    gmail: Object.freeze(['email']),
-    calendar: Object.freeze(['calendar']),
-    drive: Object.freeze(['files']),
-    docs: Object.freeze(['docs']),
-    sheets: Object.freeze(['docs', 'files']),
-  }),
-  microsoft_365: Object.freeze({
-    outlook: Object.freeze(['email']),
-    calendar: Object.freeze(['calendar']),
-    onedrive: Object.freeze(['files']),
-    teams: Object.freeze(['chat']),
-  }),
-  github: Object.freeze({
-    repos: Object.freeze(['repos', 'tickets']),
-  }),
-  slack: Object.freeze({
-    slack: Object.freeze(['chat']),
-  }),
-  whatsapp: Object.freeze({
-    personal: Object.freeze(['chat']),
-  }),
-  notion: Object.freeze({
-    notion: Object.freeze(['notes', 'docs']),
-  }),
-  trello: Object.freeze({
-    trello: Object.freeze(['tickets']),
-  }),
-});
-
-function safeTrim(value, maxLength = 240) {
-  return String(value || '').trim().slice(0, maxLength);
-}
-
-function parseJsonObject(value, fallback = {}) {
-  if (!value) return { ...fallback };
-  if (typeof value === 'object' && !Array.isArray(value)) return { ...value };
-  try {
-    const parsed = JSON.parse(String(value));
-    return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
-      ? parsed
-      : { ...fallback };
-  } catch {
-    return { ...fallback };
-  }
-}
-
-function normalizeSourceType(value) {
-  const normalized = String(value || '').trim().toLowerCase();
-  return SOURCE_TYPES.includes(normalized) ? normalized : 'notes';
-}
-
-function getFreshnessPolicy(sourceType) {
-  const normalized = normalizeSourceType(sourceType);
-  return { ...FRESHNESS_POLICIES[normalized] };
-}
-
-function nextSyncFromPolicy(policy, now = Date.now()) {
-  const intervalMinutes = Math.max(15, Number(policy?.intervalMinutes) || 360);
-  return new Date(now + intervalMinutes * 60 * 1000).toISOString();
-}
-
-function normalizeDocument(raw = {}, defaults = {}) {
-  const sourceType = normalizeSourceType(raw.sourceType || defaults.sourceType);
-  const externalObjectId = safeTrim(raw.externalObjectId || raw.id || raw.sourceId, 180);
-  const contentSource = raw.content ?? raw.text ?? raw.body ?? '';
-  const content = safeTrim(contentSource, 12000);
-  if (!externalObjectId || !content) {
-    throw new Error('Normalized memory documents require externalObjectId and content.');
-  }
-  const compacted = compactTextPayload(content, { maxChars: 2400, maxLines: 50 });
-  return {
-    sourceType,
-    normalizedType: normalizeSourceType(raw.normalizedType || raw.memoryType || sourceType),
-    providerKey: safeTrim(raw.providerKey || defaults.providerKey, 80),
-    connectionId: Number.isInteger(Number(raw.connectionId ?? defaults.connectionId))
-      ? Number(raw.connectionId ?? defaults.connectionId)
-      : 0,
-    externalObjectId,
-    sourceAccount: safeTrim(raw.sourceAccount || defaults.sourceAccount, 180),
-    title: safeTrim(raw.title || raw.subject || raw.name || externalObjectId, 220),
-    content,
-    summary: safeTrim(raw.summary || compacted.text, 1200),
-    salience: Math.max(1, Math.min(10, Number(raw.salience) || 5)),
-    sourceTimestamp: safeTrim(raw.sourceTimestamp || raw.updatedAt || raw.createdAt, 80) || null,
-    metadata: {
-      ...parseJsonObject(raw.metadata, {}),
-      compaction: compacted.metrics,
-    },
-    payload: parseJsonObject(raw.payload, {}),
-  };
-}
-
-function sourceTypesForConnection(providerKey, appKey) {
-  const providerMap = INTEGRATION_SOURCE_TYPES[String(providerKey || '').trim()] || {};
-  return Array.from(providerMap[String(appKey || '').trim()] || []);
-}
+const { getErrorMessage } = require('../bootstrap_helpers');
+const { ingestDocuments } = require('./ingestion_documents');
+const {
+  decorateProviderSnapshot,
+  listConnectionStatuses,
+} = require('./ingestion_coverage');
+const {
+  FRESHNESS_POLICIES,
+  SOURCE_TYPES,
+  getFreshnessPolicy,
+  nextSyncFromPolicy,
+  normalizeSourceType,
+  sourceTypesForConnection,
+} = require('./ingestion_support');
 
-function buildCoverageForConnection(connection, latestJob = null) {
-  const dataDomains = sourceTypesForConnection(connection.provider_key, connection.app_key);
-  const supported = dataDomains.length > 0;
-  return {
-    supported,
-    contributesToMemory: supported,
-    contributesToTaskExecution: String(connection.status || '') === 'connected',
-    status: latestJob?.status || (supported ? 'ready' : 'not_supported'),
-    dataDomains,
-    lastRefreshAt: latestJob?.completedAt || latestJob?.updatedAt || null,
-    nextRefreshAt: latestJob?.nextSyncAt || null,
-    documentCount: Number(latestJob?.documentCount || 0),
-    error: latestJob?.error || null,
-  };
-}
+const DEFAULT_REFRESH_INTERVAL_MS = 10 * 60 * 1000;
 
 class MemoryIngestionService {
-  constructor({ memoryManager, integrationManager, intervalMs = 10 * 60 * 1000 } = {}) {
+  constructor({
+    memoryManager,
+    integrationManager,
+    intervalMs = DEFAULT_REFRESH_INTERVAL_MS,
+    database = db,
+    setIntervalFn = setInterval,
+    clearIntervalFn = clearInterval,
+  } = {}) {
     this.memoryManager = memoryManager;
     this.integrationManager = integrationManager;
-    this.intervalMs = intervalMs;
+    this.intervalMs = Number(intervalMs);
+    if (!Number.isSafeInteger(this.intervalMs) || this.intervalMs < 1000) {
+      throw new Error('Memory ingestion intervalMs must be an integer greater than or equal to 1000.');
+    }
+    this.db = database;
+    this.setInterval = setIntervalFn;
+    this.clearInterval = clearIntervalFn;
     this.timer = null;
-    this.running = false;
+    this.activeBatches = new Map();
+    this.activeConnections = new Map();
+    this.stopping = false;
+    this.stopPromise = null;
+    this.state = 'idle';
+    this.lastRunAt = null;
+    this.lastCompletedAt = null;
+    this.lastError = null;
+  }
+
+  getStatus() {
+    return {
+      state: this.state,
+      intervalMs: this.intervalMs,
+      activeBatchCount: this.activeBatches.size,
+      activeConnectionCount: this.activeConnections.size,
+      lastRunAt: this.lastRunAt,
+      lastCompletedAt: this.lastCompletedAt,
+      lastError: this.lastError,
+    };
   }
 
   start() {
-    if (this.timer) return;
-    this.timer = setInterval(() => {
-      this.refreshDueConnections().catch((err) => {
-        console.warn('[MemoryIngestion] Background refresh failed:', err.message);
-      });
+    if (this.timer) return this.getStatus();
+    if (this.stopPromise) {
+      throw new Error('Memory ingestion cannot start while shutdown is in progress.');
+    }
+
+    this.stopping = false;
+    this.state = 'running';
+    this.lastError = null;
+    this.timer = this.setInterval(() => {
+      void this._runBackgroundRefresh();
     }, this.intervalMs);
     this.timer.unref?.();
+    void this._runBackgroundRefresh();
+    return this.getStatus();
   }
 
-  stop() {
-    if (this.timer) clearInterval(this.timer);
-    this.timer = null;
+  async _runBackgroundRefresh() {
+    try {
+      await this.refreshDueConnections();
+    } catch (err) {
+      this.lastError = getErrorMessage(err);
+      console.warn('[MemoryIngestion] Background refresh failed:', this.lastError);
+    }
   }
 
-  getFreshnessPolicy(sourceType) {
-    return getFreshnessPolicy(sourceType);
+  async stop() {
+    if (this.stopPromise) return this.stopPromise;
+    this.stopping = true;
+    this.state = 'stopping';
+    if (this.timer) this.clearInterval(this.timer);
+    this.timer = null;
+
+    this.stopPromise = (async () => {
+      while (this.activeBatches.size > 0 || this.activeConnections.size > 0) {
+        await Promise.allSettled([
+          ...this.activeBatches.values(),
+          ...this.activeConnections.values(),
+        ]);
+      }
+      this.stopping = false;
+      this.state = 'stopped';
+      return this.getStatus();
+    })().finally(() => {
+      this.stopPromise = null;
+    });
+    return this.stopPromise;
   }
 
   async ingestDocuments(userId, documents = [], options = {}) {
-    const agentId = resolveAgentId(userId, options.agentId || options.agent_id || null);
-    const normalizedDocs = (Array.isArray(documents) ? documents : [])
-      .map((document) => normalizeDocument(document, {
-        sourceType: options.sourceType,
-        providerKey: options.providerKey,
-        connectionId: options.connectionId,
-        sourceAccount: options.sourceAccount,
-      }));
-    const sourceType = normalizeSourceType(options.sourceType || normalizedDocs[0]?.sourceType);
-    const policy = getFreshnessPolicy(sourceType);
-    const jobId = uuidv4();
+    return ingestDocuments(this, userId, documents, options);
+  }
 
-    this.memoryManager.recordIngestionJob(userId, {
-      id: jobId,
-      sourceType,
-      providerKey: safeTrim(options.providerKey, 80),
-      connectionId: Number.isInteger(Number(options.connectionId)) ? Number(options.connectionId) : 0,
-      status: 'running',
-      freshnessPolicy: policy,
-      metadata: parseJsonObject(options.metadata, {}),
-      documentCount: 0,
-      nextSyncAt: nextSyncFromPolicy(policy),
-    }, { agentId });
+  refreshDueConnections(userId = null) {
+    const scopeKey = userId == null ? 'all' : `user:${userId}`;
+    if (this.stopping) {
+      return Promise.resolve({ skipped: true, reason: 'service_stopping' });
+    }
+    const active = this.activeBatches.get(scopeKey);
+    if (active) return active;
 
-    const documentIds = [];
-    const memoryIds = [];
-    try {
-      for (const document of normalizedDocs) {
-        const documentId = this.memoryManager.upsertIngestionDocument(userId, document, { agentId });
-        documentIds.push(documentId);
-        const memoryId = await this.memoryManager.saveMemory(
-          userId,
-          `${document.title}: ${document.summary || document.content}`,
-          SOURCE_MEMORY_CATEGORIES[document.normalizedType] || SOURCE_MEMORY_CATEGORIES[document.sourceType] || 'episodic',
-          document.salience,
-          {
-            agentId,
-            staleAfterDays: getFreshnessPolicy(document.sourceType).staleAfterDays,
-            sourceRef: {
-              sourceType: 'memory_ingestion',
-              sourceId: document.externalObjectId,
-              sourceLabel: document.title,
-            },
-            scope: {
-              scopeType: 'agent',
-              scopeId: agentId,
-            },
-            metadata: {
-              ingestionJobId: jobId,
-              ingestionDocumentId: documentId,
-              providerKey: document.providerKey || null,
-              connectionId: document.connectionId || null,
-              sourceType: document.sourceType,
-            },
-          },
-        );
-        if (memoryId) memoryIds.push(memoryId);
+    const promise = this._refreshDueConnections(userId).finally(() => {
+      if (this.activeBatches.get(scopeKey) === promise) {
+        this.activeBatches.delete(scopeKey);
       }
-
-      this.memoryManager.recordIngestionJob(userId, {
-        id: jobId,
-        sourceType,
-        providerKey: safeTrim(options.providerKey, 80),
-        connectionId: Number.isInteger(Number(options.connectionId)) ? Number(options.connectionId) : 0,
-        status: 'completed',
-        freshnessPolicy: policy,
-        summary: { documentIds, memoryIds },
-        metadata: parseJsonObject(options.metadata, {}),
-        documentCount: documentIds.length,
-        completedAt: new Date().toISOString(),
-        nextSyncAt: nextSyncFromPolicy(policy),
-      }, { agentId });
-
-      const knowledgeViews = this.memoryManager.materializeKnowledgeViews(userId, { agentId });
-      return {
-        jobId,
-        status: 'completed',
-        documentIds,
-        memoryIds,
-        knowledgeViews,
-      };
-    } catch (err) {
-      this.memoryManager.recordIngestionJob(userId, {
-        id: jobId,
-        sourceType,
-        providerKey: safeTrim(options.providerKey, 80),
-        connectionId: Number.isInteger(Number(options.connectionId)) ? Number(options.connectionId) : 0,
-        status: 'failed',
-        freshnessPolicy: policy,
-        documentCount: documentIds.length,
-        error: err.message,
-        completedAt: new Date().toISOString(),
-        nextSyncAt: nextSyncFromPolicy(policy),
-      }, { agentId });
-      throw err;
-    }
+    });
+    this.activeBatches.set(scopeKey, promise);
+    return promise;
   }
 
-  async refreshDueConnections(userId = null) {
-    if (this.running) return { skipped: true };
-    this.running = true;
+  async _refreshDueConnections(userId = null) {
+    this.lastRunAt = new Date().toISOString();
     try {
       const params = [];
       let sql = `SELECT *
@@ -298,14 +142,75 @@ class MemoryIngestionService {
         sql += ' AND user_id = ?';
         params.push(userId);
       }
-      const connections = db.prepare(sql).all(...params);
+      const connections = this.db.prepare(sql).all(...params);
       const results = [];
       for (const connection of connections) {
-        results.push(await this.refreshConnection(connection));
+        if (this.stopping) break;
+        results.push(await this._refreshConnectionSafely(connection));
+      }
+      if (!results.some((result) => result?.status === 'failed')) {
+        this.lastError = null;
       }
+      this.lastCompletedAt = new Date().toISOString();
       return { refreshed: results.length, results };
-    } finally {
-      this.running = false;
+    } catch (err) {
+      this.lastError = getErrorMessage(err);
+      throw err;
+    }
+  }
+
+  _refreshConnectionSafely(connection) {
+    const connectionKey = `${connection.user_id}:${connection.id}`;
+    const active = this.activeConnections.get(connectionKey);
+    if (active) return active;
+
+    const promise = Promise.resolve()
+      .then(() => this.refreshConnection(connection))
+      .catch((err) => {
+        const error = getErrorMessage(err);
+        this.lastError = error;
+        console.warn(
+          `[MemoryIngestion] Connection ${connection.id} refresh failed:`,
+          error,
+        );
+        return {
+          connectionId: connection.id,
+          status: 'failed',
+          error,
+        };
+      })
+      .finally(() => {
+        if (this.activeConnections.get(connectionKey) === promise) {
+          this.activeConnections.delete(connectionKey);
+        }
+      });
+    this.activeConnections.set(connectionKey, promise);
+    return promise;
+  }
+
+  _recordCollectorFailure(connection, sourceType, policy, agentId, err) {
+    try {
+      this.memoryManager.recordIngestionJob(connection.user_id, {
+        id: uuidv4(),
+        sourceType,
+        providerKey: connection.provider_key,
+        connectionId: connection.id,
+        status: 'failed',
+        freshnessPolicy: policy,
+        documentCount: 0,
+        error: getErrorMessage(err),
+        completedAt: new Date().toISOString(),
+        nextSyncAt: nextSyncFromPolicy(policy),
+        metadata: {
+          appKey: connection.app_key,
+          sourceTypes: sourceTypesForConnection(connection.provider_key, connection.app_key),
+        },
+      }, { agentId });
+    } catch (recordError) {
+      console.warn(
+        `[MemoryIngestion] Could not persist failure for connection ${connection.id}:`,
+        getErrorMessage(recordError),
+      );
     }
   }
 
@@ -331,11 +236,17 @@ class MemoryIngestionService {
     }
 
     if (typeof provider?.collectMemoryDocuments === 'function') {
-      const collected = await provider.collectMemoryDocuments({
-        connection,
-        sourceTypes,
-        cursor: latestJob?.cursor || {},
-      });
+      let collected;
+      try {
+        collected = await provider.collectMemoryDocuments({
+          connection,
+          sourceTypes,
+          cursor: latestJob?.cursor || {},
+        });
+      } catch (err) {
+        this._recordCollectorFailure(connection, primarySource, policy, agentId, err);
+        throw err;
+      }
       return this.ingestDocuments(connection.user_id, collected.documents || [], {
         agentId,
         sourceType: primarySource,
@@ -373,107 +284,11 @@ class MemoryIngestionService {
   }
 
   listConnectionStatuses(userId, { agentId = null } = {}) {
-    const scopedAgentId = resolveAgentId(userId, agentId);
-    const connections = db.prepare(
-      `SELECT *
-       FROM integration_connections
-       WHERE user_id = ? AND agent_id = ?
-       ORDER BY updated_at DESC, id DESC`,
-    ).all(userId, scopedAgentId);
-    const jobs = this.memoryManager.listIngestionJobs(userId, { agentId: scopedAgentId, limit: 100 });
-    return connections.map((connection) => {
-      const latestJob = jobs.find((job) => Number(job.connectionId || 0) === Number(connection.id));
-      return {
-        connectionId: connection.id,
-        providerKey: connection.provider_key,
-        appKey: connection.app_key,
-        accountEmail: connection.account_email || null,
-        ...buildCoverageForConnection(connection, latestJob),
-      };
-    });
+    return listConnectionStatuses(this, userId, { agentId });
   }
 
   decorateProviderSnapshot(snapshot, userId, agentId = null) {
-    if (!snapshot || typeof snapshot !== 'object') return snapshot;
-    const scopedAgentId = resolveAgentId(userId, agentId);
-    const connections = db.prepare(
-      `SELECT *
-       FROM integration_connections
-       WHERE user_id = ? AND agent_id = ? AND provider_key = ?`,
-    ).all(userId, scopedAgentId, snapshot.provider);
-    const jobs = this.memoryManager.listIngestionJobs(userId, {
-      agentId: scopedAgentId,
-      providerKey: snapshot.provider,
-      limit: 100,
-    });
-    const latestJobForConnection = (connectionId) =>
-      jobs.find((job) => Number(job.connectionId || 0) === Number(connectionId));
-
-    const connectionCoverage = connections.map((connection) => ({
-      connectionId: connection.id,
-      appKey: connection.app_key,
-      accountEmail: connection.account_email || null,
-      ...buildCoverageForConnection(connection, latestJobForConnection(connection.id)),
-    }));
-    const providerDomains = Array.from(new Set(connectionCoverage.flatMap((item) => item.dataDomains || [])));
-    const providerJob = jobs[0] || null;
-    const decorated = {
-      ...snapshot,
-      memoryCoverage: {
-        supported: providerDomains.length > 0,
-        contributesToMemory: providerDomains.length > 0,
-        contributesToTaskExecution: Boolean(snapshot.connection?.connected),
-        status: providerJob?.status || (providerDomains.length > 0 ? 'ready' : 'not_supported'),
-        dataDomains: providerDomains,
-        documentCount: connectionCoverage.reduce((sum, item) => sum + Number(item.documentCount || 0), 0),
-        lastRefreshAt: providerJob?.completedAt || providerJob?.updatedAt || null,
-        nextRefreshAt: providerJob?.nextSyncAt || null,
-        error: providerJob?.error || null,
-      },
-    };
-
-    decorated.apps = (snapshot.apps || []).map((app) => {
-      const appConnections = connectionCoverage.filter((item) => item.appKey === app.id);
-      const dataDomains = Array.from(new Set([
-        ...sourceTypesForConnection(snapshot.provider, app.id),
-        ...appConnections.flatMap((item) => item.dataDomains || []),
-      ]));
-      const latest = appConnections[0] || null;
-      return {
-        ...app,
-        memoryCoverage: {
-          supported: dataDomains.length > 0,
-          contributesToMemory: dataDomains.length > 0 && app.connection?.connected === true,
-          contributesToTaskExecution: app.connection?.connected === true,
-          status: latest?.status || (dataDomains.length > 0 ? 'ready' : 'not_supported'),
-          dataDomains,
-          documentCount: appConnections.reduce((sum, item) => sum + Number(item.documentCount || 0), 0),
-          lastRefreshAt: latest?.lastRefreshAt || null,
-          nextRefreshAt: latest?.nextRefreshAt || null,
-          error: latest?.error || null,
-        },
-        accounts: (app.accounts || []).map((account) => {
-          const accountCoverage = appConnections.find((item) =>
-            Number(item.connectionId) === Number(account.connectionId),
-          );
-          return {
-            ...account,
-            memoryCoverage: accountCoverage || {
-              supported: dataDomains.length > 0,
-              contributesToMemory: false,
-              contributesToTaskExecution: app.connection?.connected === true,
-              status: dataDomains.length > 0 ? 'ready' : 'not_supported',
-              dataDomains,
-              documentCount: 0,
-              lastRefreshAt: null,
-              nextRefreshAt: null,
-              error: null,
-            },
-          };
-        }),
-      };
-    });
-    return decorated;
+    return decorateProviderSnapshot(this, snapshot, userId, agentId);
   }
 }