neoagent 2.4.4-beta.0 → 2.4.4-beta.11

package/server/services/memory/embeddings.js

@@ -141,12 +141,31 @@ async function getOpenAIEmbedding(text) {
  * @returns {Float32Array|null}
  */
 async function getEmbedding(text, provider) {
+  const result = await getEmbeddingWithMetadata(text, provider);
+  return result?.vector || null;
+}
+
+async function getEmbeddingWithMetadata(text, provider) {
   if (!text || !text.trim()) return null;
   if (provider === 'google' && process.env.GOOGLE_AI_KEY) {
     const vec = await getGeminiEmbedding(text);
-    if (vec) return vec;
+    if (vec) {
+      return {
+        vector: vec,
+        provider: 'google',
+        model: GOOGLE_MODEL,
+        dimensions: vec.length,
+      };
+    }
   }
-  return getOpenAIEmbedding(text);
+  const vec = await getOpenAIEmbedding(text);
+  if (!vec) return null;
+  return {
+    vector: vec,
+    provider: 'openai',
+    model: OPENAI_MODEL,
+    dimensions: vec.length,
+  };
 }
 
 /**
@@ -203,6 +222,7 @@ function keywordSimilarity(query, text) {
 
 module.exports = {
   getEmbedding,
+  getEmbeddingWithMetadata,
   cosineSimilarity,
   serializeEmbedding,
   deserializeEmbedding,

package/server/services/memory/evaluation.js

@@ -0,0 +1,187 @@
+'use strict';
+
+const { performance } = require('node:perf_hooks');
+
+function mean(values) {
+  if (!values.length) return 0;
+  return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+
+function percentile(values, fraction) {
+  if (!values.length) return 0;
+  const sorted = [...values].sort((left, right) => left - right);
+  const index = Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1);
+  return sorted[Math.max(0, index)];
+}
+
+function calculateRetrievalMetrics(results, relevantKeys, k) {
+  const returned = results.slice(0, k);
+  const relevance = returned.map((result) => (
+    relevantKeys.has(`memory:${result.id}`)
+    || relevantKeys.has(`source:${result.sourceRef?.sourceId || ''}`)
+      ? 1
+      : 0
+  ));
+  const relevantRetrieved = relevance.reduce((sum, value) => sum + value, 0);
+  const totalRelevant = relevantKeys.size;
+  const precisionAtK = returned.length ? relevantRetrieved / returned.length : 0;
+  const recallAtK = totalRelevant ? relevantRetrieved / totalRelevant : 0;
+  const firstRelevant = relevance.indexOf(1);
+  const dcg = relevance.reduce(
+    (sum, value, index) => sum + value / Math.log2(index + 2),
+    0,
+  );
+  let idealDcg = 0;
+  for (let index = 0; index < Math.min(totalRelevant, returned.length); index += 1) {
+    idealDcg += 1 / Math.log2(index + 2);
+  }
+
+  return {
+    hitAtK: relevantRetrieved > 0 ? 1 : 0,
+    precisionAtK,
+    recallAtK,
+    f1AtK: precisionAtK + recallAtK
+      ? (2 * precisionAtK * recallAtK) / (precisionAtK + recallAtK)
+      : 0,
+    mrr: firstRelevant >= 0 ? 1 / (firstRelevant + 1) : 0,
+    ndcg: idealDcg ? dcg / idealDcg : 0,
+    relevantRetrieved,
+    totalRelevant,
+  };
+}
+
+function normalizeQuestions(dataset) {
+  const questions = Array.isArray(dataset)
+    ? dataset
+    : dataset?.questions || dataset?.queries;
+  if (!Array.isArray(questions) || !questions.length) {
+    throw new Error('Dataset must contain a non-empty questions or queries array.');
+  }
+
+  return questions.map((question, index) => {
+    const query = String(question.query || question.question || '').trim();
+    const relevantMemoryIds = Array.isArray(question.relevantMemoryIds)
+      ? question.relevantMemoryIds.map(String).filter(Boolean)
+      : [];
+    const relevantSourceIds = Array.isArray(question.relevantSourceIds)
+      ? question.relevantSourceIds.map(String).filter(Boolean)
+      : Array.isArray(question.haystackSessionIds)
+        ? question.haystackSessionIds.map(String).filter(Boolean)
+        : [];
+    if (!query) throw new Error(`Question ${index + 1} has no query.`);
+    if (!relevantMemoryIds.length && !relevantSourceIds.length) {
+      throw new Error(`Question ${index + 1} has no relevant memory or source IDs.`);
+    }
+    return {
+      id: String(question.id || question.questionId || index + 1),
+      query,
+      category: String(question.category || question.questionType || 'uncategorized'),
+      relevantMemoryIds,
+      relevantSourceIds,
+      validAt: question.validAt || null,
+      knownAt: question.knownAt || null,
+    };
+  });
+}
+
+function aggregateEvaluations(evaluations, k) {
+  const metricNames = ['hitAtK', 'precisionAtK', 'recallAtK', 'f1AtK', 'mrr', 'ndcg'];
+  const retrieval = { k };
+  for (const metric of metricNames) {
+    retrieval[metric] = mean(evaluations.map((evaluation) => evaluation.metrics[metric]));
+  }
+  const latencies = evaluations.map((evaluation) => evaluation.latencyMs);
+  const contextTokens = evaluations.map((evaluation) => evaluation.contextTokensEstimate);
+  const candidateCounts = evaluations.map((evaluation) => evaluation.candidateCount);
+  return {
+    questions: evaluations.length,
+    retrieval,
+    latencyMs: {
+      mean: mean(latencies),
+      median: percentile(latencies, 0.5),
+      p95: percentile(latencies, 0.95),
+      p99: percentile(latencies, 0.99),
+      max: latencies.length ? Math.max(...latencies) : 0,
+    },
+    contextTokensEstimate: {
+      mean: mean(contextTokens),
+      p95: percentile(contextTokens, 0.95),
+      max: contextTokens.length ? Math.max(...contextTokens) : 0,
+    },
+    candidateCount: {
+      mean: mean(candidateCounts),
+      p95: percentile(candidateCounts, 0.95),
+      max: candidateCounts.length ? Math.max(...candidateCounts) : 0,
+    },
+  };
+}
+
+async function runRetrievalBenchmark({
+  memoryManager,
+  userId,
+  agentId,
+  dataset,
+  k = 15,
+}) {
+  const limit = Math.max(1, Math.min(Number(k) || 15, 50));
+  const questions = normalizeQuestions(dataset);
+  const evaluations = [];
+
+  for (const question of questions) {
+    const relevantKeys = new Set([
+      ...question.relevantMemoryIds.map((id) => `memory:${id}`),
+      ...question.relevantSourceIds.map((id) => `source:${id}`),
+    ]);
+    const startedAt = performance.now();
+    const results = await memoryManager.recallMemory(userId, question.query, limit, {
+      agentId,
+      validAt: question.validAt,
+      knownAt: question.knownAt,
+    });
+    const latencyMs = performance.now() - startedAt;
+    const contextText = results
+      .map((result) => result.summary || result.content || '')
+      .join('\n');
+
+    evaluations.push({
+      id: question.id,
+      category: question.category,
+      query: question.query,
+      latencyMs,
+      contextTokensEstimate: Math.ceil(contextText.length / 4),
+      candidateCount: Math.max(
+        0,
+        ...results.map((result) => Number(result.scoreBreakdown?.candidateCount) || 0),
+      ),
+      retrievedMemoryIds: results.map((result) => result.id),
+      retrievedSourceIds: results
+        .map((result) => result.sourceRef?.sourceId)
+        .filter(Boolean),
+      metrics: calculateRetrievalMetrics(results, relevantKeys, limit),
+    });
+  }
+
+  const categories = {};
+  for (const category of new Set(evaluations.map((evaluation) => evaluation.category))) {
+    categories[category] = aggregateEvaluations(
+      evaluations.filter((evaluation) => evaluation.category === category),
+      limit,
+    );
+  }
+
+  return {
+    generatedAt: new Date().toISOString(),
+    dataset: String(dataset?.name || 'unnamed'),
+    userId,
+    agentId,
+    ...aggregateEvaluations(evaluations, limit),
+    categories,
+    evaluations,
+  };
+}
+
+module.exports = {
+  calculateRetrievalMetrics,
+  normalizeQuestions,
+  runRetrievalBenchmark,
+};

package/server/services/memory/ingestion_chunking.js

@@ -0,0 +1,191 @@
+'use strict';
+
+const crypto = require('node:crypto');
+
+const TARGET_CHARS = 1400;
+const MAX_CHARS = 2200;
+
+function contentHash(content) {
+  return crypto.createHash('sha256').update(content).digest('hex');
+}
+
+function messageBlocks(document) {
+  const messages = Array.isArray(document?.payload?.messages)
+    ? document.payload.messages
+    : [];
+  return messages
+    .map((message) => {
+      const speaker = String(message.speaker || message.role || message.author || '').trim();
+      const timestamp = String(message.timestamp || message.createdAt || '').trim();
+      const content = String(message.content || message.text || message.body || '').trim();
+      if (!content) return null;
+      const prefix = [timestamp, speaker].filter(Boolean).join(' ');
+      return {
+        content,
+        rendered: prefix ? `${prefix}\n${content}` : content,
+        messageId: String(message.id || message.messageId || '').trim() || null,
+      };
+    })
+    .filter(Boolean);
+}
+
+function rawStructuralBlocks(text) {
+  return String(text || '')
+    .replace(/\r\n?/g, '\n')
+    .split(/\n{2,}|(?=^#{1,6}\s)/gm)
+    .map((block) => block.trim())
+    .filter(Boolean);
+}
+
+function rawEmailBlocks(text) {
+  return String(text || '')
+    .replace(/\r\n?/g, '\n')
+    .split(/\n(?=On\s.*wrote:|\s*>\s|-----Original Message-----|________________________________)/g)
+    .map((block) => block.trim())
+    .filter(Boolean);
+}
+
+function rawCodeBlocks(text) {
+  return String(text || '')
+    .replace(/\r\n?/g, '\n')
+    .split(/\n(?=(?:export\s+)?(?:class|function|const|let|var|import)\s)/g)
+    .map((block) => block.trim())
+    .filter(Boolean);
+}
+
+function findBlockPositions(originalContent, blocks) {
+  let searchFrom = 0;
+  return blocks.map((block) => {
+    const pos = originalContent.indexOf(block, searchFrom);
+    const charStart = pos >= 0 ? pos : searchFrom;
+    const charEnd = charStart + block.length;
+    searchFrom = charEnd;
+    return { content: block, charStart, charEnd };
+  });
+}
+
+function splitOversizedWithPos({ content, charStart }) {
+  if (content.length <= MAX_CHARS) {
+    return [{ content, charStart, charEnd: charStart + content.length }];
+  }
+  const parts = [];
+  let offset = 0;
+  while (offset < content.length) {
+    const slice = content.slice(offset, offset + MAX_CHARS);
+    parts.push({
+      content: slice,
+      charStart: charStart + offset,
+      charEnd: charStart + offset + slice.length,
+    });
+    offset += MAX_CHARS;
+  }
+  return parts;
+}
+
+function packBlocksWithPositions(blocksWithPos) {
+  const chunks = [];
+  let group = [];
+  let groupLen = 0;
+
+  for (const bwp of blocksWithPos.flatMap(splitOversizedWithPos)) {
+    const addLen = (groupLen ? 2 : 0) + bwp.content.length;
+    if (group.length && groupLen + addLen > TARGET_CHARS) {
+      chunks.push(group);
+      group = [bwp];
+      groupLen = bwp.content.length;
+    } else {
+      group.push(bwp);
+      groupLen += addLen;
+    }
+  }
+  if (group.length) chunks.push(group);
+
+  return chunks.map((g) => ({
+    content: g.map((b) => b.content).join('\n\n'),
+    charStart: g[0].charStart,
+    charEnd: g[g.length - 1].charEnd,
+  }));
+}
+
+function chunkDocument(document) {
+  const content = String(document?.content || '').trim();
+  if (!content) return [];
+
+  const messages = messageBlocks(document);
+  if (messages.length) {
+    let searchOffset = 0;
+    return messages.map((message, chunkIndex) => {
+      let charStart = content.indexOf(message.content, searchOffset);
+      if (charStart < 0) charStart = searchOffset;
+      const charEnd = Math.min(content.length, charStart + message.content.length);
+      searchOffset = charEnd;
+      return {
+        chunkIndex,
+        charStart,
+        charEnd,
+        content: message.rendered,
+        contentHash: contentHash(message.rendered),
+        metadata: {
+          boundary: 'message',
+          messageId: message.messageId,
+        },
+      };
+    });
+  }
+
+  const sourceType = String(document?.sourceType || document?.normalizedType || '').toLowerCase();
+  const title = String(document?.title || '').toLowerCase();
+
+  let rawBlocks;
+  let boundaryType;
+
+  if (sourceType.includes('email') || title.endsWith('.eml') || title.includes('message')) {
+    rawBlocks = rawEmailBlocks(content);
+    boundaryType = 'email_thread';
+  } else if (
+    sourceType.includes('code') ||
+    title.endsWith('.js') ||
+    title.endsWith('.ts') ||
+    title.endsWith('.py') ||
+    title.endsWith('.java')
+  ) {
+    rawBlocks = rawCodeBlocks(content);
+    boundaryType = 'code_structure';
+  } else {
+    rawBlocks = rawStructuralBlocks(content);
+    boundaryType = 'structural';
+  }
+
+  const blocksWithPos = findBlockPositions(content, rawBlocks);
+  const packed = packBlocksWithPositions(blocksWithPos);
+
+  return packed.map((chunk, chunkIndex) => ({
+    chunkIndex,
+    charStart: chunk.charStart,
+    charEnd: chunk.charEnd,
+    content: chunk.content,
+    contentHash: contentHash(chunk.content),
+    metadata: { boundary: boundaryType },
+  }));
+}
+
+function overlapWindowChunks(chunks) {
+  if (chunks.length < 2) return [];
+  return chunks.slice(0, -1).map((chunk, i) => {
+    const next = chunks[i + 1];
+    const combined = `${chunk.content}\n\n${next.content}`;
+    return {
+      chunkIndex: -(i + 1),
+      charStart: chunk.charStart,
+      charEnd: next.charEnd,
+      content: combined.slice(0, MAX_CHARS * 2),
+      contentHash: contentHash(combined),
+      metadata: { boundary: 'overlap_window', baseChunkIndex: i },
+    };
+  });
+}
+
+module.exports = {
+  chunkDocument,
+  overlapWindowChunks,
+};

package/server/services/memory/ingestion_documents.js

@@ -11,6 +11,7 @@ const {
   parseJsonObject,
   safeTrim,
 } = require('./ingestion_support');
+const { chunkDocument, overlapWindowChunks } = require('./ingestion_chunking');
 
 async function ingestDocuments(service, userId, documents = [], options = {}) {
   const agentId = resolveAgentId(userId, options.agentId || options.agent_id || null);
@@ -50,35 +51,104 @@ async function ingestDocuments(service, userId, documents = [], options = {}) {
         { agentId },
       );
       documentIds.push(documentId);
-      const memoryId = await service.memoryManager.saveMemory(
-        userId,
-        `${document.title}: ${document.summary || document.content}`,
-        SOURCE_MEMORY_CATEGORIES[document.normalizedType]
-          || SOURCE_MEMORY_CATEGORIES[document.sourceType]
-          || 'episodic',
-        document.salience,
-        {
-          agentId,
-          staleAfterDays: getFreshnessPolicy(document.sourceType).staleAfterDays,
-          sourceRef: {
-            sourceType: 'memory_ingestion',
-            sourceId: document.externalObjectId,
-            sourceLabel: document.title,
+      const retainedChunkIds = [];
+      const chunks = chunkDocument(document);
+      const savedChunkMemoryIds = [];
+      for (const chunk of chunks) {
+        const memoryId = await service.memoryManager.saveMemory(
+          userId,
+          `${document.title}\n${chunk.content}`,
+          SOURCE_MEMORY_CATEGORIES[document.normalizedType]
+            || SOURCE_MEMORY_CATEGORIES[document.sourceType]
+            || 'episodic',
+          document.salience,
+          {
+            agentId,
+            staleAfterDays: getFreshnessPolicy(document.sourceType).staleAfterDays,
+            sourceRef: {
+              sourceType: 'memory_ingestion',
+              sourceId: document.externalObjectId,
+              sourceLabel: document.title,
+            },
+            scope: {
+              scopeType: 'agent',
+              scopeId: agentId,
+            },
+            metadata: {
+              ingestionJobId: jobId,
+              ingestionDocumentId: documentId,
+              providerKey: document.providerKey || null,
+              connectionId: document.connectionId || null,
+              sourceType: document.sourceType,
+              trustLevel: 'external_source',
+              chunkIndex: chunk.chunkIndex,
+              sourceSpan: {
+                charStart: chunk.charStart,
+                charEnd: chunk.charEnd,
+              },
+            },
           },
-          scope: {
-            scopeType: 'agent',
-            scopeId: agentId,
+        );
+        if (!memoryId) continue;
+        memoryIds.push(memoryId);
+        savedChunkMemoryIds.push(memoryId);
+        retainedChunkIds.push(service.memoryManager.replaceSourceChunk(
+          documentId,
+          chunk,
+          memoryId,
+          {
+            userId,
+            agentId,
+            sourceTimestamp: document.sourceTimestamp,
+            metadata: {
+              ingestionJobId: jobId,
+              externalObjectId: document.externalObjectId,
+            },
           },
-          metadata: {
-            ingestionJobId: jobId,
-            ingestionDocumentId: documentId,
-            providerKey: document.providerKey || null,
-            connectionId: document.connectionId || null,
-            sourceType: document.sourceType,
+        ));
+      }
+
+      const overlapChunks = overlapWindowChunks(chunks);
+      for (const window of overlapChunks) {
+        const windowMemoryId = await service.memoryManager.saveMemory(
+          userId,
+          `${document.title}\n${window.content}`,
+          SOURCE_MEMORY_CATEGORIES[document.normalizedType]
+            || SOURCE_MEMORY_CATEGORIES[document.sourceType]
+            || 'episodic',
+          document.salience,
+          {
+            agentId,
+            staleAfterDays: getFreshnessPolicy(document.sourceType).staleAfterDays,
+            sourceRef: {
+              sourceType: 'memory_ingestion',
+              sourceId: document.externalObjectId,
+              sourceLabel: document.title,
+            },
+            scope: {
+              scopeType: 'agent',
+              scopeId: agentId,
+            },
+            metadata: {
+              ingestionJobId: jobId,
+              ingestionDocumentId: documentId,
+              providerKey: document.providerKey || null,
+              connectionId: document.connectionId || null,
+              sourceType: document.sourceType,
+              trustLevel: 'external_source',
+              chunkIndex: window.chunkIndex,
+              sourceSpan: {
+                charStart: window.charStart,
+                charEnd: window.charEnd,
+              },
+              isOverlapWindow: true,
+            },
           },
-        },
-      );
-      if (memoryId) memoryIds.push(memoryId);
+        );
+        if (windowMemoryId) savedChunkMemoryIds.push(windowMemoryId);
+      }
+
+      service.memoryManager.pruneSourceChunks(documentId, retainedChunkIds);
     }
 
     service.memoryManager.recordIngestionJob(userId, {

package/server/services/memory/intelligence.js

@@ -157,6 +157,7 @@ function scoreMemoryCandidate({
   entityRank = -1,
   baseScore = 0,
   importance = 5,
+  confidence = 0.7,
   accessCount = 0,
   freshness = 1,
 } = {}) {
@@ -166,8 +167,9 @@ function scoreMemoryCandidate({
     rankFuse(entityRank, 0.95)
   ) * 20;
   const quality = 0.15 + clamp(importance, 1, 10, 5) / 22;
+  const confidenceMultiplier = 0.65 + clamp(confidence, 0, 1, 0.7) * 0.35;
   const usage = Math.min(0.08, Math.log1p(Math.max(0, Number(accessCount) || 0)) / 50);
-  return Math.max(baseScore, fused + quality + usage) * freshness;
+  return Math.max(baseScore, fused + quality + usage) * freshness * confidenceMultiplier;
 }
 
 module.exports = {