neoagent 2.4.1-beta.9 → 2.4.1

package/server/services/memory/ingestion_coverage.js

@@ -0,0 +1,129 @@
+'use strict';
+
+const { resolveAgentId } = require('../agents/manager');
+const {
+  buildCoverageForConnection,
+  sourceTypesForConnection,
+} = require('./ingestion_support');
+
+function listConnectionStatuses(service, userId, { agentId = null } = {}) {
+  const scopedAgentId = resolveAgentId(userId, agentId);
+  const connections = service.db.prepare(
+    `SELECT *
+     FROM integration_connections
+     WHERE user_id = ? AND agent_id = ?
+     ORDER BY updated_at DESC, id DESC`,
+  ).all(userId, scopedAgentId);
+  const jobs = service.memoryManager.listIngestionJobs(
+    userId,
+    { agentId: scopedAgentId, limit: 100 },
+  );
+  return connections.map((connection) => {
+    const latestJob = jobs.find(
+      (job) => Number(job.connectionId || 0) === Number(connection.id),
+    );
+    return {
+      connectionId: connection.id,
+      providerKey: connection.provider_key,
+      appKey: connection.app_key,
+      accountEmail: connection.account_email || null,
+      ...buildCoverageForConnection(connection, latestJob),
+    };
+  });
+}
+
+function decorateProviderSnapshot(service, snapshot, userId, agentId = null) {
+  if (!snapshot || typeof snapshot !== 'object') return snapshot;
+  const scopedAgentId = resolveAgentId(userId, agentId);
+  const connections = service.db.prepare(
+    `SELECT *
+     FROM integration_connections
+     WHERE user_id = ? AND agent_id = ? AND provider_key = ?`,
+  ).all(userId, scopedAgentId, snapshot.provider);
+  const jobs = service.memoryManager.listIngestionJobs(userId, {
+    agentId: scopedAgentId,
+    providerKey: snapshot.provider,
+    limit: 100,
+  });
+  const latestJobForConnection = (connectionId) =>
+    jobs.find((job) => Number(job.connectionId || 0) === Number(connectionId));
+
+  const connectionCoverage = connections.map((connection) => ({
+    connectionId: connection.id,
+    appKey: connection.app_key,
+    accountEmail: connection.account_email || null,
+    ...buildCoverageForConnection(connection, latestJobForConnection(connection.id)),
+  }));
+  const providerDomains = Array.from(
+    new Set(connectionCoverage.flatMap((item) => item.dataDomains || [])),
+  );
+  const providerJob = jobs[0] || null;
+  const decorated = {
+    ...snapshot,
+    memoryCoverage: {
+      supported: providerDomains.length > 0,
+      contributesToMemory: providerDomains.length > 0,
+      contributesToTaskExecution: Boolean(snapshot.connection?.connected),
+      status: providerJob?.status || (providerDomains.length > 0 ? 'ready' : 'not_supported'),
+      dataDomains: providerDomains,
+      documentCount: connectionCoverage.reduce(
+        (sum, item) => sum + Number(item.documentCount || 0),
+        0,
+      ),
+      lastRefreshAt: providerJob?.completedAt || providerJob?.updatedAt || null,
+      nextRefreshAt: providerJob?.nextSyncAt || null,
+      error: providerJob?.error || null,
+    },
+  };
+
+  decorated.apps = (snapshot.apps || []).map((app) => {
+    const appConnections = connectionCoverage.filter((item) => item.appKey === app.id);
+    const dataDomains = Array.from(new Set([
+      ...sourceTypesForConnection(snapshot.provider, app.id),
+      ...appConnections.flatMap((item) => item.dataDomains || []),
+    ]));
+    const latest = appConnections[0] || null;
+    return {
+      ...app,
+      memoryCoverage: {
+        supported: dataDomains.length > 0,
+        contributesToMemory: dataDomains.length > 0 && app.connection?.connected === true,
+        contributesToTaskExecution: app.connection?.connected === true,
+        status: latest?.status || (dataDomains.length > 0 ? 'ready' : 'not_supported'),
+        dataDomains,
+        documentCount: appConnections.reduce(
+          (sum, item) => sum + Number(item.documentCount || 0),
+          0,
+        ),
+        lastRefreshAt: latest?.lastRefreshAt || null,
+        nextRefreshAt: latest?.nextRefreshAt || null,
+        error: latest?.error || null,
+      },
+      accounts: (app.accounts || []).map((account) => {
+        const accountCoverage = appConnections.find(
+          (item) => Number(item.connectionId) === Number(account.connectionId),
+        );
+        return {
+          ...account,
+          memoryCoverage: accountCoverage || {
+            supported: dataDomains.length > 0,
+            contributesToMemory: false,
+            contributesToTaskExecution: app.connection?.connected === true,
+            status: dataDomains.length > 0 ? 'ready' : 'not_supported',
+            dataDomains,
+            documentCount: 0,
+            lastRefreshAt: null,
+            nextRefreshAt: null,
+            error: null,
+          },
+        };
+      }),
+    };
+  });
+  return decorated;
+}
+
+module.exports = {
+  decorateProviderSnapshot,
+  listConnectionStatuses,
+};

package/server/services/memory/ingestion_documents.js

@@ -0,0 +1,123 @@
+'use strict';
+
+const { v4: uuidv4 } = require('uuid');
+const { resolveAgentId } = require('../agents/manager');
+const {
+  SOURCE_MEMORY_CATEGORIES,
+  getFreshnessPolicy,
+  nextSyncFromPolicy,
+  normalizeDocument,
+  normalizeSourceType,
+  parseJsonObject,
+  safeTrim,
+} = require('./ingestion_support');
+
+async function ingestDocuments(service, userId, documents = [], options = {}) {
+  const agentId = resolveAgentId(userId, options.agentId || options.agent_id || null);
+  const normalizedDocs = (Array.isArray(documents) ? documents : [])
+    .map((document) => normalizeDocument(document, {
+      sourceType: options.sourceType,
+      providerKey: options.providerKey,
+      connectionId: options.connectionId,
+      sourceAccount: options.sourceAccount,
+    }));
+  const sourceType = normalizeSourceType(options.sourceType || normalizedDocs[0]?.sourceType);
+  const policy = getFreshnessPolicy(sourceType);
+  const jobId = uuidv4();
+  const connectionId = Number.isInteger(Number(options.connectionId))
+    ? Number(options.connectionId)
+    : 0;
+
+  service.memoryManager.recordIngestionJob(userId, {
+    id: jobId,
+    sourceType,
+    providerKey: safeTrim(options.providerKey, 80),
+    connectionId,
+    status: 'running',
+    freshnessPolicy: policy,
+    metadata: parseJsonObject(options.metadata, {}),
+    documentCount: 0,
+    nextSyncAt: nextSyncFromPolicy(policy),
+  }, { agentId });
+
+  const documentIds = [];
+  const memoryIds = [];
+  try {
+    for (const document of normalizedDocs) {
+      const documentId = service.memoryManager.upsertIngestionDocument(
+        userId,
+        document,
+        { agentId },
+      );
+      documentIds.push(documentId);
+      const memoryId = await service.memoryManager.saveMemory(
+        userId,
+        `${document.title}: ${document.summary || document.content}`,
+        SOURCE_MEMORY_CATEGORIES[document.normalizedType]
+          || SOURCE_MEMORY_CATEGORIES[document.sourceType]
+          || 'episodic',
+        document.salience,
+        {
+          agentId,
+          staleAfterDays: getFreshnessPolicy(document.sourceType).staleAfterDays,
+          sourceRef: {
+            sourceType: 'memory_ingestion',
+            sourceId: document.externalObjectId,
+            sourceLabel: document.title,
+          },
+          scope: {
+            scopeType: 'agent',
+            scopeId: agentId,
+          },
+          metadata: {
+            ingestionJobId: jobId,
+            ingestionDocumentId: documentId,
+            providerKey: document.providerKey || null,
+            connectionId: document.connectionId || null,
+            sourceType: document.sourceType,
+          },
+        },
+      );
+      if (memoryId) memoryIds.push(memoryId);
+    }
+
+    service.memoryManager.recordIngestionJob(userId, {
+      id: jobId,
+      sourceType,
+      providerKey: safeTrim(options.providerKey, 80),
+      connectionId,
+      status: 'completed',
+      freshnessPolicy: policy,
+      summary: { documentIds, memoryIds },
+      metadata: parseJsonObject(options.metadata, {}),
+      documentCount: documentIds.length,
+      completedAt: new Date().toISOString(),
+      nextSyncAt: nextSyncFromPolicy(policy),
+    }, { agentId });
+
+    const knowledgeViews = service.memoryManager.materializeKnowledgeViews(userId, { agentId });
+    return {
+      jobId,
+      status: 'completed',
+      documentIds,
+      memoryIds,
+      knowledgeViews,
+    };
+  } catch (err) {
+    service.memoryManager.recordIngestionJob(userId, {
+      id: jobId,
+      sourceType,
+      providerKey: safeTrim(options.providerKey, 80),
+      connectionId,
+      status: 'failed',
+      freshnessPolicy: policy,
+      documentCount: documentIds.length,
+      error: err.message,
+      completedAt: new Date().toISOString(),
+      nextSyncAt: nextSyncFromPolicy(policy),
+    }, { agentId });
+    throw err;
+  }
+}
+
+module.exports = { ingestDocuments };

package/server/services/memory/ingestion_support.js

@@ -0,0 +1,171 @@
+'use strict';
+
+const { compactTextPayload } = require('../ai/preModelCompaction');
+
+const SOURCE_TYPES = Object.freeze([
+  'email',
+  'calendar',
+  'chat',
+  'docs',
+  'tickets',
+  'repos',
+  'files',
+  'crm',
+  'payments',
+  'notes',
+]);
+
+const FRESHNESS_POLICIES = Object.freeze({
+  email: Object.freeze({ intervalMinutes: 60, staleAfterDays: 14 }),
+  calendar: Object.freeze({ intervalMinutes: 180, staleAfterDays: 30 }),
+  chat: Object.freeze({ intervalMinutes: 60, staleAfterDays: 14 }),
+  docs: Object.freeze({ intervalMinutes: 360, staleAfterDays: 60 }),
+  tickets: Object.freeze({ intervalMinutes: 120, staleAfterDays: 30 }),
+  repos: Object.freeze({ intervalMinutes: 180, staleAfterDays: 45 }),
+  files: Object.freeze({ intervalMinutes: 360, staleAfterDays: 60 }),
+  crm: Object.freeze({ intervalMinutes: 240, staleAfterDays: 45 }),
+  payments: Object.freeze({ intervalMinutes: 360, staleAfterDays: 90 }),
+  notes: Object.freeze({ intervalMinutes: 360, staleAfterDays: 90 }),
+});
+
+const SOURCE_MEMORY_CATEGORIES = Object.freeze({
+  email: 'episodic',
+  calendar: 'events',
+  chat: 'episodic',
+  docs: 'projects',
+  tickets: 'tasks',
+  repos: 'projects',
+  files: 'episodic',
+  crm: 'contacts',
+  payments: 'events',
+  notes: 'episodic',
+});
+
+const INTEGRATION_SOURCE_TYPES = Object.freeze({
+  google_workspace: Object.freeze({
+    gmail: Object.freeze(['email']),
+    calendar: Object.freeze(['calendar']),
+    drive: Object.freeze(['files']),
+    docs: Object.freeze(['docs']),
+    sheets: Object.freeze(['docs', 'files']),
+  }),
+  microsoft_365: Object.freeze({
+    outlook: Object.freeze(['email']),
+    calendar: Object.freeze(['calendar']),
+    onedrive: Object.freeze(['files']),
+    teams: Object.freeze(['chat']),
+  }),
+  github: Object.freeze({
+    repos: Object.freeze(['repos', 'tickets']),
+  }),
+  slack: Object.freeze({
+    slack: Object.freeze(['chat']),
+  }),
+  whatsapp: Object.freeze({
+    personal: Object.freeze(['chat']),
+  }),
+  notion: Object.freeze({
+    notion: Object.freeze(['notes', 'docs']),
+  }),
+  trello: Object.freeze({
+    trello: Object.freeze(['tickets']),
+  }),
+});
+
+function safeTrim(value, maxLength = 240) {
+  return String(value || '').trim().slice(0, maxLength);
+}
+
+function parseJsonObject(value, fallback = {}) {
+  if (!value) return { ...fallback };
+  if (typeof value === 'object' && !Array.isArray(value)) return { ...value };
+  try {
+    const parsed = JSON.parse(String(value));
+    return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
+      ? parsed
+      : { ...fallback };
+  } catch {
+    return { ...fallback };
+  }
+}
+
+function normalizeSourceType(value) {
+  const normalized = String(value || '').trim().toLowerCase();
+  return SOURCE_TYPES.includes(normalized) ? normalized : 'notes';
+}
+
+function getFreshnessPolicy(sourceType) {
+  const normalized = normalizeSourceType(sourceType);
+  return { ...FRESHNESS_POLICIES[normalized] };
+}
+
+function nextSyncFromPolicy(policy, now = Date.now()) {
+  const intervalMinutes = Math.max(15, Number(policy?.intervalMinutes) || 360);
+  return new Date(now + intervalMinutes * 60 * 1000).toISOString();
+}
+
+function normalizeDocument(raw = {}, defaults = {}) {
+  const sourceType = normalizeSourceType(raw.sourceType || defaults.sourceType);
+  const externalObjectId = safeTrim(raw.externalObjectId || raw.id || raw.sourceId, 180);
+  const contentSource = raw.content ?? raw.text ?? raw.body ?? '';
+  const content = safeTrim(contentSource, 12000);
+  if (!externalObjectId || !content) {
+    throw new Error('Normalized memory documents require externalObjectId and content.');
+  }
+  const compacted = compactTextPayload(content, { maxChars: 2400, maxLines: 50 });
+  return {
+    sourceType,
+    normalizedType: normalizeSourceType(raw.normalizedType || raw.memoryType || sourceType),
+    providerKey: safeTrim(raw.providerKey || defaults.providerKey, 80),
+    connectionId: Number.isInteger(Number(raw.connectionId ?? defaults.connectionId))
+      ? Number(raw.connectionId ?? defaults.connectionId)
+      : 0,
+    externalObjectId,
+    sourceAccount: safeTrim(raw.sourceAccount || defaults.sourceAccount, 180),
+    title: safeTrim(raw.title || raw.subject || raw.name || externalObjectId, 220),
+    content,
+    summary: safeTrim(raw.summary || compacted.text, 1200),
+    salience: Math.max(1, Math.min(10, Number(raw.salience) || 5)),
+    sourceTimestamp: safeTrim(raw.sourceTimestamp || raw.updatedAt || raw.createdAt, 80) || null,
+    metadata: {
+      ...parseJsonObject(raw.metadata, {}),
+      compaction: compacted.metrics,
+    },
+    payload: parseJsonObject(raw.payload, {}),
+  };
+}
+
+function sourceTypesForConnection(providerKey, appKey) {
+  const providerMap = INTEGRATION_SOURCE_TYPES[String(providerKey || '').trim()] || {};
+  return Array.from(providerMap[String(appKey || '').trim()] || []);
+}
+
+function buildCoverageForConnection(connection, latestJob = null) {
+  const dataDomains = sourceTypesForConnection(connection.provider_key, connection.app_key);
+  const supported = dataDomains.length > 0;
+  return {
+    supported,
+    contributesToMemory: supported,
+    contributesToTaskExecution: String(connection.status || '') === 'connected',
+    status: latestJob?.status || (supported ? 'ready' : 'not_supported'),
+    dataDomains,
+    lastRefreshAt: latestJob?.completedAt || latestJob?.updatedAt || null,
+    nextRefreshAt: latestJob?.nextSyncAt || null,
+    documentCount: Number(latestJob?.documentCount || 0),
+    error: latestJob?.error || null,
+  };
+}
+
+module.exports = {
+  FRESHNESS_POLICIES,
+  SOURCE_MEMORY_CATEGORIES,
+  SOURCE_TYPES,
+  buildCoverageForConnection,
+  getFreshnessPolicy,
+  nextSyncFromPolicy,
+  normalizeDocument,
+  normalizeSourceType,
+  parseJsonObject,
+  safeTrim,
+  sourceTypesForConnection,
+};

package/server/services/memory/intelligence.js

@@ -0,0 +1,181 @@
+'use strict';
+
+const crypto = require('crypto');
+
+const ENTITY_KIND_PATTERNS = [
+  ['email', /^[^\s@]+@[^\s@]+\.[^\s@]+$/],
+  ['url', /^https?:\/\//i],
+  ['version', /^v?\d+(?:\.\d+){1,3}$/i],
+  ['identifier', /^[a-z0-9][a-z0-9_-]{2,}$/i],
+];
+
+function clamp(value, min, max, fallback) {
+  const number = Number(value);
+  return Number.isFinite(number) ? Math.max(min, Math.min(max, number)) : fallback;
+}
+
+function stableHash(value) {
+  return crypto
+    .createHash('sha256')
+    .update(String(value || '').trim().toLowerCase())
+    .digest('hex');
+}
+
+function canonicalEntityKey(name) {
+  return String(name || '')
+    .trim()
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}._@:/+-]+/gu, ' ')
+    .replace(/\s+/g, ' ')
+    .slice(0, 160);
+}
+
+function classifyEntity(name) {
+  const value = String(name || '').trim();
+  for (const [kind, pattern] of ENTITY_KIND_PATTERNS) {
+    if (pattern.test(value)) return kind;
+  }
+  if (/^[A-Z][A-Z0-9_-]{2,}$/.test(value)) return 'acronym';
+  if (/\.(js|ts|dart|py|json|md|yaml|yml|sql|rs|go|java|kt|swift|css|html)$/i.test(value)) {
+    return 'file';
+  }
+  return 'concept';
+}
+
+function uniqueByKey(items, getKey) {
+  const seen = new Set();
+  const result = [];
+  for (const item of items) {
+    const key = getKey(item);
+    if (!key || seen.has(key)) continue;
+    seen.add(key);
+    result.push(item);
+  }
+  return result;
+}
+
+function extractEntities(text, { maxEntities = 16 } = {}) {
+  const raw = String(text || '');
+  const candidates = [];
+
+  for (const match of raw.matchAll(/[^\s@]+@[^\s@]+\.[^\s@]+/g)) {
+    candidates.push(match[0]);
+  }
+  for (const match of raw.matchAll(/https?:\/\/[^\s)]+/gi)) {
+    candidates.push(match[0].replace(/[.,;]+$/g, ''));
+  }
+  for (const match of raw.matchAll(/\b[A-Z][\p{L}\p{N}_+./:-]*(?:\s+[A-Z][\p{L}\p{N}_+./:-]*){0,4}\b/gu)) {
+    const value = match[0].trim();
+    if (value.length > 2) candidates.push(value);
+  }
+  for (const match of raw.matchAll(/\b[\p{L}\p{N}_./-]+\.(?:js|ts|dart|py|json|md|yaml|yml|sql|rs|go|java|kt|swift|css|html)\b/giu)) {
+    candidates.push(match[0]);
+  }
+
+  return uniqueByKey(
+    candidates
+      .map((name) => {
+        const normalized = String(name || '').trim().replace(/\s+/g, ' ');
+        const key = canonicalEntityKey(normalized);
+        if (!key || key.length < 2) return null;
+        return {
+          key,
+          name: normalized.slice(0, 160),
+          kind: classifyEntity(normalized),
+        };
+      })
+      .filter(Boolean),
+    (entity) => entity.key,
+  ).slice(0, maxEntities);
+}
+
+function extractKeywords(text, { maxKeywords = 24 } = {}) {
+  const counts = new Map();
+  const tokens = String(text || '')
+    .toLowerCase()
+    .match(/[\p{L}\p{N}_-]{5,}/gu) || [];
+  for (const token of tokens) {
+    if (/^\d+$/.test(token)) continue;
+    counts.set(token, (counts.get(token) || 0) + 1);
+  }
+  return [...counts.entries()]
+    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+    .slice(0, maxKeywords)
+    .map(([keyword]) => keyword);
+}
+
+function splitSentences(text) {
+  return String(text || '')
+    .split(/(?<=[.!?])\s+|\n+/u)
+    .map((part) => part.trim())
+    .filter(Boolean);
+}
+
+function buildFacts({ content, category, sourceRef, metadata } = {}) {
+  const sentences = splitSentences(content);
+  const entities = extractEntities(content);
+  const keywords = extractKeywords(content, { maxKeywords: 10 });
+  const subject = entities[0]?.name || String(category || 'memory').replace(/_/g, ' ');
+  const predicate = String(category || 'episodic').replace(/_/g, ' ');
+  const sourceType = sourceRef?.sourceType || metadata?.sourceType || null;
+
+  const factTexts = sentences.length
+    ? sentences.slice(0, 6)
+    : [String(content || '').trim()].filter(Boolean);
+
+  return factTexts.map((text, index) => ({
+    subject: String(subject || 'memory').slice(0, 180),
+    predicate: index === 0 ? predicate : 'detail',
+    object: text.slice(0, 900),
+    category,
+    confidence: sourceType === 'llm_import' ? 0.74 : 0.68,
+    metadata: {
+      keywords,
+      sourceType,
+      extractedBy: 'local_memory_intelligence',
+    },
+  }));
+}
+
+function summarizeForPrompt(memory) {
+  const entities = Array.isArray(memory.entities) ? memory.entities : [];
+  const content = String(memory.content || '').replace(/\s+/g, ' ').trim();
+  const entitySuffix = entities.length
+    ? ` (${entities.slice(0, 4).map((entity) => entity.name || entity).join(', ')})`
+    : '';
+  return `${content}${entitySuffix}`.slice(0, 900);
+}
+
+function rankFuse(rank, weight = 1) {
+  if (!Number.isFinite(rank) || rank < 0) return 0;
+  return weight / (60 + rank + 1);
+}
+
+function scoreMemoryCandidate({
+  semanticRank = -1,
+  lexicalRank = -1,
+  entityRank = -1,
+  baseScore = 0,
+  importance = 5,
+  accessCount = 0,
+  freshness = 1,
+} = {}) {
+  const fused = (
+    rankFuse(semanticRank, 1.0) +
+    rankFuse(lexicalRank, 0.85) +
+    rankFuse(entityRank, 0.95)
+  ) * 20;
+  const quality = 0.15 + clamp(importance, 1, 10, 5) / 22;
+  const usage = Math.min(0.08, Math.log1p(Math.max(0, Number(accessCount) || 0)) / 50);
+  return Math.max(baseScore, fused + quality + usage) * freshness;
+}
+
+module.exports = {
+  buildFacts,
+  canonicalEntityKey,
+  extractEntities,
+  extractKeywords,
+  scoreMemoryCandidate,
+  stableHash,
+  summarizeForPrompt,
+};