@lov3kaizen/agentsea-memory 0.5.1

package/dist/processing/index.cjs

@@ -0,0 +1,1575 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/processing/index.ts
+var processing_exports = {};
+__export(processing_exports, {
+  Compressor: () => Compressor,
+  Consolidator: () => Consolidator,
+  Extractor: () => Extractor,
+  Forgetter: () => Forgetter,
+  Summarizer: () => Summarizer,
+  createCompressor: () => createCompressor,
+  createConsolidator: () => createConsolidator,
+  createExtractor: () => createExtractor,
+  createForgetter: () => createForgetter,
+  createSummarizer: () => createSummarizer
+});
+module.exports = __toCommonJS(processing_exports);
+
+// src/processing/Summarizer.ts
+var Summarizer = class {
+  config;
+  summaryFn;
+  constructor(config = {}) {
+    this.config = {
+      provider: config.provider ?? void 0,
+      model: config.model ?? "default",
+      strategy: config.strategy ?? "abstractive",
+      maxLength: config.maxLength ?? 500,
+      preserveEntities: config.preserveEntities ?? true,
+      focusPrompt: config.focusPrompt ?? "",
+      maxSummaryLength: config.maxSummaryLength ?? 500,
+      minEntriesForSummary: config.minEntriesForSummary ?? 3,
+      preserveKeyEntities: config.preserveKeyEntities ?? true,
+      summaryStyle: config.summaryStyle ?? "concise"
+    };
+  }
+  /**
+   * Set custom summary function (for LLM integration)
+   */
+  setSummaryFunction(fn) {
+    this.summaryFn = fn;
+  }
+  /**
+   * Summarize a collection of memories
+   */
+  async summarize(entries) {
+    if (entries.length < this.config.minEntriesForSummary) {
+      return Promise.resolve(this.createSimpleSummary(entries));
+    }
+    if (this.summaryFn) {
+      const summary = await this.summaryFn(entries, {
+        maxLength: this.config.maxSummaryLength,
+        style: this.config.summaryStyle
+      });
+      return {
+        summary,
+        keyPoints: this.extractKeyPoints(entries),
+        sourceCount: entries.length,
+        compressionRatio: this.calculateCompressionRatio(entries, summary),
+        metadata: {
+          method: "llm",
+          style: this.config.summaryStyle
+        }
+      };
+    }
+    return Promise.resolve(this.heuristicSummarize(entries));
+  }
+  /**
+   * Summarize memories by time period
+   */
+  async summarizeByPeriod(entries, period) {
+    const grouped = this.groupByPeriod(entries, period);
+    const results = /* @__PURE__ */ new Map();
+    for (const [key, groupEntries] of grouped) {
+      const summary = await this.summarize(groupEntries);
+      results.set(key, summary);
+    }
+    return results;
+  }
+  /**
+   * Summarize memories by topic/type
+   */
+  async summarizeByTopic(entries) {
+    const grouped = this.groupByTopic(entries);
+    const results = /* @__PURE__ */ new Map();
+    for (const [topic, topicEntries] of grouped) {
+      const summary = await this.summarize(topicEntries);
+      results.set(topic, summary);
+    }
+    return results;
+  }
+  /**
+   * Create incremental summary (add to existing summary)
+   */
+  async incrementalSummarize(existingSummary, newEntries) {
+    if (this.summaryFn) {
+      const contextEntry = {
+        id: "context",
+        content: `Previous summary: ${existingSummary}`,
+        type: "summary",
+        importance: 0.8,
+        metadata: {
+          source: "system",
+          confidence: 1
+        },
+        timestamp: Date.now(),
+        accessCount: 0,
+        createdAt: Date.now(),
+        updatedAt: Date.now()
+      };
+      const summary = await this.summaryFn([contextEntry, ...newEntries], {
+        maxLength: this.config.maxSummaryLength,
+        style: this.config.summaryStyle
+      });
+      return {
+        summary,
+        keyPoints: this.extractKeyPoints(newEntries),
+        sourceCount: newEntries.length,
+        compressionRatio: this.calculateCompressionRatio(newEntries, summary),
+        metadata: {
+          method: "incremental",
+          hadExistingSummary: true
+        }
+      };
+    }
+    const newPoints = this.extractKeyPoints(newEntries);
+    const combinedSummary = `${existingSummary} Additionally: ${newPoints.join("; ")}.`;
+    return {
+      summary: combinedSummary.slice(0, this.config.maxSummaryLength),
+      keyPoints: newPoints,
+      sourceCount: newEntries.length,
+      compressionRatio: this.calculateCompressionRatio(
+        newEntries,
+        combinedSummary
+      ),
+      metadata: {
+        method: "heuristic-incremental"
+      }
+    };
+  }
+  /**
+   * Heuristic-based summarization
+   */
+  heuristicSummarize(entries) {
+    const sorted = [...entries].sort((a, b) => b.importance - a.importance);
+    const keyPoints = this.extractKeyPoints(sorted);
+    const typeGroups = /* @__PURE__ */ new Map();
+    for (const entry of entries) {
+      typeGroups.set(entry.type, (typeGroups.get(entry.type) ?? 0) + 1);
+    }
+    const typesSummary = Array.from(typeGroups.entries()).map(([type, count]) => `${count} ${type}(s)`).join(", ");
+    const timeRange = this.getTimeRange(entries);
+    const summary = `Summary of ${entries.length} memories (${typesSummary}) from ${timeRange}: ${keyPoints.slice(0, 3).join(". ")}.`;
+    return {
+      summary: summary.slice(0, this.config.maxSummaryLength),
+      keyPoints,
+      sourceCount: entries.length,
+      compressionRatio: this.calculateCompressionRatio(entries, summary),
+      metadata: {
+        method: "heuristic",
+        typeDistribution: Object.fromEntries(typeGroups)
+      }
+    };
+  }
+  /**
+   * Create simple summary for few entries
+   */
+  createSimpleSummary(entries) {
+    const summary = entries.map((e) => e.content.slice(0, 100)).join("; ");
+    return {
+      summary: summary.slice(0, this.config.maxSummaryLength),
+      keyPoints: entries.map((e) => e.content.slice(0, 50)),
+      sourceCount: entries.length,
+      compressionRatio: 1,
+      metadata: {
+        method: "simple"
+      }
+    };
+  }
+  /**
+   * Extract key points from entries
+   */
+  extractKeyPoints(entries) {
+    const points = [];
+    for (const entry of entries.slice(0, 10)) {
+      const firstSentence = entry.content.split(/[.!?]/)[0];
+      if (firstSentence && firstSentence.length > 10) {
+        points.push(firstSentence.trim());
+      }
+    }
+    return this.deduplicateStrings(points);
+  }
+  /**
+   * Group entries by time period
+   */
+  groupByPeriod(entries, period) {
+    const groups = /* @__PURE__ */ new Map();
+    for (const entry of entries) {
+      const date = new Date(entry.timestamp);
+      let key;
+      switch (period) {
+        case "hour":
+          key = `${date.toISOString().slice(0, 13)}:00`;
+          break;
+        case "day":
+          key = date.toISOString().slice(0, 10);
+          break;
+        case "week": {
+          const weekStart = new Date(date);
+          weekStart.setDate(date.getDate() - date.getDay());
+          key = `week-${weekStart.toISOString().slice(0, 10)}`;
+          break;
+        }
+      }
+      if (!groups.has(key)) {
+        groups.set(key, []);
+      }
+      groups.get(key).push(entry);
+    }
+    return groups;
+  }
+  /**
+   * Group entries by topic/type
+   */
+  groupByTopic(entries) {
+    const groups = /* @__PURE__ */ new Map();
+    for (const entry of entries) {
+      const topic = String(entry.metadata.topic ?? entry.type);
+      if (!groups.has(topic)) {
+        groups.set(topic, []);
+      }
+      groups.get(topic).push(entry);
+    }
+    return groups;
+  }
+  /**
+   * Get time range description
+   */
+  getTimeRange(entries) {
+    if (entries.length === 0) return "unknown period";
+    const timestamps = entries.map((e) => e.timestamp);
+    const start = new Date(Math.min(...timestamps));
+    const end = new Date(Math.max(...timestamps));
+    const diffMs = end.getTime() - start.getTime();
+    const diffHours = diffMs / (1e3 * 60 * 60);
+    if (diffHours < 1) {
+      return "the last hour";
+    } else if (diffHours < 24) {
+      return "today";
+    } else if (diffHours < 168) {
+      return "this week";
+    } else {
+      return `${start.toLocaleDateString()} to ${end.toLocaleDateString()}`;
+    }
+  }
+  /**
+   * Calculate compression ratio
+   */
+  calculateCompressionRatio(entries, summary) {
+    const originalLength = entries.reduce(
+      (sum, e) => sum + e.content.length,
+      0
+    );
+    return originalLength > 0 ? originalLength / summary.length : 1;
+  }
+  /**
+   * Deduplicate similar strings
+   */
+  deduplicateStrings(strings) {
+    const result = [];
+    for (const str of strings) {
+      const isDuplicate = result.some(
+        (existing) => this.stringSimilarity(str, existing) > 0.8
+      );
+      if (!isDuplicate) {
+        result.push(str);
+      }
+    }
+    return result;
+  }
+  /**
+   * Simple string similarity (Jaccard)
+   */
+  stringSimilarity(a, b) {
+    const setA = new Set(a.toLowerCase().split(/\s+/));
+    const setB = new Set(b.toLowerCase().split(/\s+/));
+    const intersection = new Set([...setA].filter((x) => setB.has(x)));
+    const union = /* @__PURE__ */ new Set([...setA, ...setB]);
+    return intersection.size / union.size;
+  }
+  /**
+   * Update configuration
+   */
+  configure(config) {
+    this.config = { ...this.config, ...config };
+  }
+};
+function createSummarizer(config) {
+  return new Summarizer(config);
+}
+
+// src/processing/Compressor.ts
+var Compressor = class {
+  config;
+  constructor(config = {}) {
+    this.config = {
+      targetRatio: config.targetRatio ?? 0.5,
+      preserveImportant: config.preserveImportant ?? true,
+      strategy: config.strategy ?? "importance-weighted",
+      minImportance: config.minImportance ?? 0.3,
+      minContentLength: config.minContentLength ?? 50,
+      removeEmbeddings: config.removeEmbeddings ?? false,
+      truncateMetadata: config.truncateMetadata ?? true
+    };
+  }
+  /**
+   * Compress a single memory entry
+   */
+  compress(entry) {
+    const originalSize = this.calculateSize(entry);
+    const preservedFields = [];
+    const compressed = { ...entry };
+    if (entry.content.length > this.config.minContentLength) {
+      compressed.content = this.truncateContent(
+        entry.content,
+        entry.importance
+      );
+      preservedFields.push("content (truncated)");
+    } else {
+      preservedFields.push("content");
+    }
+    if (this.config.removeEmbeddings && entry.embedding) {
+      delete compressed.embedding;
+    } else if (entry.embedding) {
+      preservedFields.push("embedding");
+    }
+    if (this.config.truncateMetadata) {
+      compressed.metadata = this.truncateMetadata(entry.metadata);
+      preservedFields.push("metadata (truncated)");
+    } else {
+      preservedFields.push("metadata");
+    }
+    const compressedSize = this.calculateSize(compressed);
+    return {
+      compressed,
+      originalSize,
+      compressedSize,
+      ratio: compressedSize / originalSize,
+      preservedFields
+    };
+  }
+  /**
+   * Compress multiple entries
+   */
+  compressBatch(entries) {
+    let totalOriginalSize = 0;
+    let totalCompressedSize = 0;
+    let removedCount = 0;
+    const compressedEntries = [];
+    for (const entry of entries) {
+      const originalSize = this.calculateSize(entry);
+      totalOriginalSize += originalSize;
+      if (this.config.preserveImportant && entry.importance < 0.2) {
+        removedCount++;
+        continue;
+      }
+      const result = this.compress(entry);
+      compressedEntries.push(result.compressed);
+      totalCompressedSize += result.compressedSize;
+    }
+    return {
+      entries: compressedEntries,
+      totalOriginalSize,
+      totalCompressedSize,
+      avgRatio: totalOriginalSize > 0 ? totalCompressedSize / totalOriginalSize : 1,
+      removedCount
+    };
+  }
+  /**
+   * Compress entries to target size
+   */
+  compressToSize(entries, targetSize) {
+    const sorted = [...entries].sort((a, b) => b.importance - a.importance);
+    let currentSize = 0;
+    const result = [];
+    let removedCount = 0;
+    for (const entry of sorted) {
+      const compressed = this.compress(entry);
+      if (currentSize + compressed.compressedSize <= targetSize) {
+        result.push(compressed.compressed);
+        currentSize += compressed.compressedSize;
+      } else {
+        const aggressive = this.aggressiveCompress(entry);
+        if (currentSize + this.calculateSize(aggressive) <= targetSize) {
+          result.push(aggressive);
+          currentSize += this.calculateSize(aggressive);
+        } else {
+          removedCount++;
+        }
+      }
+    }
+    return {
+      entries: result,
+      totalOriginalSize: entries.reduce(
+        (sum, e) => sum + this.calculateSize(e),
+        0
+      ),
+      totalCompressedSize: currentSize,
+      avgRatio: currentSize / Math.max(
+        entries.reduce((sum, e) => sum + this.calculateSize(e), 0),
+        1
+      ),
+      removedCount
+    };
+  }
+  /**
+   * Deduplicate and compress similar entries
+   */
+  deduplicateAndCompress(entries) {
+    const seen = /* @__PURE__ */ new Map();
+    let removedCount = 0;
+    for (const entry of entries) {
+      const key = this.getDedupeKey(entry);
+      const existing = seen.get(key);
+      if (!existing) {
+        seen.set(key, entry);
+      } else {
+        if (entry.importance > existing.importance || entry.importance === existing.importance && entry.timestamp > existing.timestamp) {
+          seen.set(key, entry);
+        }
+        removedCount++;
+      }
+    }
+    const unique = Array.from(seen.values());
+    const batchResult = this.compressBatch(unique);
+    return {
+      ...batchResult,
+      removedCount: removedCount + batchResult.removedCount
+    };
+  }
+  /**
+   * Truncate content based on importance
+   */
+  truncateContent(content, importance) {
+    const keepRatio = 0.3 + importance * 0.5;
+    const targetLength = Math.max(
+      this.config.minContentLength,
+      Math.floor(content.length * keepRatio)
+    );
+    if (content.length <= targetLength) {
+      return content;
+    }
+    const truncated = content.slice(0, targetLength);
+    const lastSentenceEnd = Math.max(
+      truncated.lastIndexOf("."),
+      truncated.lastIndexOf("!"),
+      truncated.lastIndexOf("?")
+    );
+    if (lastSentenceEnd > targetLength * 0.5) {
+      return truncated.slice(0, lastSentenceEnd + 1);
+    }
+    const lastSpace = truncated.lastIndexOf(" ");
+    if (lastSpace > targetLength * 0.8) {
+      return truncated.slice(0, lastSpace) + "...";
+    }
+    return truncated + "...";
+  }
+  /**
+   * Truncate metadata
+   */
+  truncateMetadata(metadata) {
+    const result = {
+      source: metadata.source,
+      confidence: metadata.confidence
+    };
+    const essentialFields = [
+      "userId",
+      "agentId",
+      "conversationId",
+      "namespace",
+      "tags"
+    ];
+    for (const field of essentialFields) {
+      if (field in metadata) {
+        result[field] = metadata[field];
+      }
+    }
+    return result;
+  }
+  /**
+   * Aggressive compression for tight space constraints
+   */
+  aggressiveCompress(entry) {
+    return {
+      ...entry,
+      content: entry.content.slice(0, 100) + (entry.content.length > 100 ? "..." : ""),
+      embedding: void 0,
+      metadata: {
+        source: entry.metadata.source,
+        confidence: entry.metadata.confidence,
+        namespace: entry.metadata.namespace
+      }
+    };
+  }
+  /**
+   * Get deduplication key for entry
+   */
+  getDedupeKey(entry) {
+    const contentKey = entry.content.toLowerCase().replace(/[^\w\s]/g, "").slice(0, 100);
+    return `${entry.type}:${contentKey}`;
+  }
+  /**
+   * Calculate approximate size of entry in bytes
+   */
+  calculateSize(entry) {
+    let size = 0;
+    size += entry.content.length * 2;
+    if (entry.embedding) {
+      size += entry.embedding.length * 8;
+    }
+    size += JSON.stringify(entry.metadata).length * 2;
+    size += 200;
+    return size;
+  }
+  /**
+   * Update configuration
+   */
+  configure(config) {
+    this.config = { ...this.config, ...config };
+  }
+};
+function createCompressor(config) {
+  return new Compressor(config);
+}
+
+// src/processing/Consolidator.ts
+var Consolidator = class {
+  config;
+  summarizer;
+  embedFn;
+  constructor(config = {}) {
+    this.config = {
+      similarityThreshold: config.similarityThreshold ?? 0.8,
+      mergeStrategy: config.mergeStrategy ?? "confidence-weighted",
+      extractRelations: config.extractRelations ?? false,
+      maxBatchSize: config.maxBatchSize ?? 100,
+      minGroupSize: config.minGroupSize ?? 2,
+      maxGroupSize: config.maxGroupSize ?? 20,
+      groupingStrategy: config.groupingStrategy ?? "semantic",
+      preserveOriginals: config.preserveOriginals ?? false
+    };
+    this.summarizer = new Summarizer();
+  }
+  /**
+   * Set embedding function for semantic grouping
+   */
+  setEmbeddingFunction(fn) {
+    this.embedFn = fn;
+  }
+  /**
+   * Set summarizer function
+   */
+  setSummarizerFunction(fn) {
+    this.summarizer.setSummaryFunction(fn);
+  }
+  /**
+   * Find and consolidate similar memories
+   */
+  async consolidate(entries, store) {
+    const groups = await this.groupSimilar(entries);
+    const results = [];
+    for (const group of groups) {
+      if (group.entries.length < this.config.minGroupSize) {
+        continue;
+      }
+      const consolidated = await this.consolidateGroup(group);
+      results.push(consolidated);
+      if (store) {
+        await store.add(consolidated.consolidated);
+        if (!this.config.preserveOriginals) {
+          for (const id of consolidated.sourceIds) {
+            await store.delete(id);
+          }
+        }
+      }
+    }
+    return results;
+  }
+  /**
+   * Group similar entries
+   */
+  async groupSimilar(entries) {
+    switch (this.config.groupingStrategy) {
+      case "semantic":
+        return Promise.resolve(this.groupBySemantic(entries));
+      case "temporal":
+        return Promise.resolve(this.groupByTemporal(entries));
+      case "type":
+        return Promise.resolve(this.groupByType(entries));
+      default:
+        return Promise.resolve(this.groupBySemantic(entries));
+    }
+  }
+  /**
+   * Group by semantic similarity
+   */
+  async groupBySemantic(entries) {
+    if (!this.embedFn) {
+      return Promise.resolve(this.groupByTextSimilarity(entries));
+    }
+    const withEmbeddings = await Promise.all(
+      entries.map(async (entry) => {
+        if (entry.embedding) return entry;
+        return {
+          ...entry,
+          embedding: await this.embedFn(entry.content)
+        };
+      })
+    );
+    const groups = [];
+    const assigned = /* @__PURE__ */ new Set();
+    for (let i = 0; i < withEmbeddings.length; i++) {
+      if (assigned.has(withEmbeddings[i].id)) continue;
+      const group = [withEmbeddings[i]];
+      assigned.add(withEmbeddings[i].id);
+      for (let j = i + 1; j < withEmbeddings.length; j++) {
+        if (assigned.has(withEmbeddings[j].id)) continue;
+        if (group.length >= this.config.maxGroupSize) break;
+        const similarity = this.cosineSimilarity(
+          withEmbeddings[i].embedding,
+          withEmbeddings[j].embedding
+        );
+        if (similarity >= this.config.similarityThreshold) {
+          group.push(withEmbeddings[j]);
+          assigned.add(withEmbeddings[j].id);
+        }
+      }
+      if (group.length >= this.config.minGroupSize) {
+        groups.push({
+          id: this.generateId(),
+          entries: group,
+          similarity: this.calculateGroupSimilarity(group),
+          groupKey: `semantic-${i}`
+        });
+      }
+    }
+    return groups;
+  }
+  /**
+   * Group by text similarity (fallback)
+   */
+  groupByTextSimilarity(entries) {
+    const groups = [];
+    const assigned = /* @__PURE__ */ new Set();
+    for (let i = 0; i < entries.length; i++) {
+      if (assigned.has(entries[i].id)) continue;
+      const group = [entries[i]];
+      assigned.add(entries[i].id);
+      for (let j = i + 1; j < entries.length; j++) {
+        if (assigned.has(entries[j].id)) continue;
+        if (group.length >= this.config.maxGroupSize) break;
+        const similarity = this.textSimilarity(
+          entries[i].content,
+          entries[j].content
+        );
+        if (similarity >= this.config.similarityThreshold) {
+          group.push(entries[j]);
+          assigned.add(entries[j].id);
+        }
+      }
+      if (group.length >= this.config.minGroupSize) {
+        groups.push({
+          id: this.generateId(),
+          entries: group,
+          similarity: 0.8,
+          groupKey: `text-${i}`
+        });
+      }
+    }
+    return groups;
+  }
+  /**
+   * Group by temporal proximity
+   */
+  groupByTemporal(entries) {
+    const sorted = [...entries].sort((a, b) => a.timestamp - b.timestamp);
+    const groups = [];
+    const windowMs = 60 * 60 * 1e3;
+    let currentGroup = [];
+    let windowStart = sorted[0]?.timestamp ?? 0;
+    for (const entry of sorted) {
+      if (entry.timestamp - windowStart <= windowMs && currentGroup.length < this.config.maxGroupSize) {
+        currentGroup.push(entry);
+      } else {
+        if (currentGroup.length >= this.config.minGroupSize) {
+          groups.push({
+            id: this.generateId(),
+            entries: currentGroup,
+            similarity: 1,
+            groupKey: `temporal-${new Date(windowStart).toISOString()}`
+          });
+        }
+        currentGroup = [entry];
+        windowStart = entry.timestamp;
+      }
+    }
+    if (currentGroup.length >= this.config.minGroupSize) {
+      groups.push({
+        id: this.generateId(),
+        entries: currentGroup,
+        similarity: 1,
+        groupKey: `temporal-${new Date(windowStart).toISOString()}`
+      });
+    }
+    return groups;
+  }
+  /**
+   * Group by memory type
+   */
+  groupByType(entries) {
+    const typeGroups = /* @__PURE__ */ new Map();
+    for (const entry of entries) {
+      if (!typeGroups.has(entry.type)) {
+        typeGroups.set(entry.type, []);
+      }
+      typeGroups.get(entry.type).push(entry);
+    }
+    const groups = [];
+    for (const [type, typeEntries] of typeGroups) {
+      for (let i = 0; i < typeEntries.length; i += this.config.maxGroupSize) {
+        const chunk = typeEntries.slice(i, i + this.config.maxGroupSize);
+        if (chunk.length >= this.config.minGroupSize) {
+          groups.push({
+            id: this.generateId(),
+            entries: chunk,
+            similarity: 1,
+            groupKey: `type-${type}-${i}`
+          });
+        }
+      }
+    }
+    return groups;
+  }
+  /**
+   * Consolidate a group into a single entry
+   */
+  async consolidateGroup(group) {
+    const entries = group.entries;
+    const summaryResult = await this.summarizer.summarize(entries);
+    const avgImportance = entries.reduce((sum, e) => sum + e.importance, 0) / entries.length;
+    const maxImportance = Math.max(...entries.map((e) => e.importance));
+    const combinedImportance = avgImportance * 0.7 + maxImportance * 0.3;
+    const timestamps = entries.map((e) => e.timestamp);
+    const timeSpan = {
+      start: Math.min(...timestamps),
+      end: Math.max(...timestamps)
+    };
+    const allTags = /* @__PURE__ */ new Set();
+    for (const entry of entries) {
+      const tags = entry.metadata.tags;
+      if (tags) {
+        tags.forEach((t) => allTags.add(t));
+      }
+    }
+    const consolidated = {
+      id: `consolidated-${group.id}`,
+      content: summaryResult.summary,
+      type: "summary",
+      importance: Math.min(combinedImportance + 0.1, 1),
+      // Boost for consolidation
+      metadata: {
+        source: "system",
+        confidence: 0.9,
+        consolidated: true,
+        sourceCount: entries.length,
+        groupKey: group.groupKey,
+        tags: Array.from(allTags),
+        keyPoints: summaryResult.keyPoints
+      },
+      timestamp: timeSpan.start,
+      accessCount: entries.reduce((sum, e) => sum + e.accessCount, 0),
+      createdAt: Date.now(),
+      updatedAt: Date.now()
+    };
+    if (this.embedFn) {
+      consolidated.embedding = await this.embedFn(consolidated.content);
+    }
+    return {
+      consolidated,
+      sourceIds: entries.map((e) => e.id),
+      groupKey: group.groupKey,
+      avgImportance,
+      timeSpan
+    };
+  }
+  /**
+   * Calculate cosine similarity
+   */
+  cosineSimilarity(a, b) {
+    if (a.length !== b.length) return 0;
+    let dotProduct = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+      dotProduct += a[i] * b[i];
+      normA += a[i] * a[i];
+      normB += b[i] * b[i];
+    }
+    const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
+    return magnitude === 0 ? 0 : dotProduct / magnitude;
+  }
+  /**
+   * Calculate text similarity (Jaccard)
+   */
+  textSimilarity(a, b) {
+    const wordsA = new Set(a.toLowerCase().split(/\s+/));
+    const wordsB = new Set(b.toLowerCase().split(/\s+/));
+    const intersection = new Set([...wordsA].filter((x) => wordsB.has(x)));
+    const union = /* @__PURE__ */ new Set([...wordsA, ...wordsB]);
+    return intersection.size / union.size;
+  }
+  /**
+   * Calculate average similarity within a group
+   */
+  calculateGroupSimilarity(entries) {
+    if (entries.length < 2) return 1;
+    if (!entries[0].embedding) return 0.8;
+    let totalSimilarity = 0;
+    let comparisons = 0;
+    for (let i = 0; i < entries.length; i++) {
+      for (let j = i + 1; j < entries.length; j++) {
+        if (entries[i].embedding && entries[j].embedding) {
+          totalSimilarity += this.cosineSimilarity(
+            entries[i].embedding,
+            entries[j].embedding
+          );
+          comparisons++;
+        }
+      }
+    }
+    return comparisons > 0 ? totalSimilarity / comparisons : 0.8;
+  }
+  /**
+   * Generate unique ID
+   */
+  generateId() {
+    return `group-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
+  }
+  /**
+   * Update configuration
+   */
+  configure(config) {
+    this.config = { ...this.config, ...config };
+  }
+};
+function createConsolidator(config) {
+  return new Consolidator(config);
+}
+
+// src/processing/Forgetter.ts
+var Forgetter = class {
+  config;
+  constructor(config = {}) {
+    this.config = {
+      retentionPolicy: config.retentionPolicy ?? {},
+      importanceThreshold: config.importanceThreshold ?? 0.3,
+      maxAge: config.maxAge ?? 30 * 24 * 60 * 60 * 1e3,
+      // 30 days
+      preserveTypes: config.preserveTypes ?? [],
+      curve: config.curve ?? "exponential",
+      halfLife: config.halfLife ?? 7 * 24 * 60 * 60 * 1e3,
+      // 7 days
+      minRetention: config.minRetention ?? 0.1,
+      accessBoost: config.accessBoost ?? 0.1,
+      importanceWeight: config.importanceWeight ?? 0.5,
+      forgetThreshold: config.forgetThreshold ?? 0.05
+    };
+  }
+  /**
+   * Calculate retention score for a memory
+   */
+  calculateRetention(entry, now = Date.now()) {
+    const ageMs = now - entry.timestamp;
+    const baseRetention = this.calculateBaseRetention(ageMs);
+    const accessBoost = Math.min(
+      entry.accessCount * this.config.accessBoost,
+      0.5
+    );
+    const importanceBoost = entry.importance * this.config.importanceWeight;
+    let retention = baseRetention + accessBoost + importanceBoost;
+    if (entry.importance >= 0.8) {
+      retention = Math.max(retention, 0.5);
+    }
+    retention = Math.min(Math.max(retention, this.config.minRetention), 1);
+    return {
+      entryId: entry.id,
+      retention,
+      ageMs,
+      accessCount: entry.accessCount,
+      importance: entry.importance,
+      shouldForget: retention < this.config.forgetThreshold
+    };
+  }
+  /**
+   * Calculate base retention using the configured curve
+   */
+  calculateBaseRetention(ageMs) {
+    const halfLife = this.config.halfLife;
+    switch (this.config.curve) {
+      case "exponential":
+        return Math.exp(-Math.LN2 * ageMs / halfLife);
+      case "power": {
+        const c = halfLife;
+        const b = 0.5;
+        return Math.pow(1 + ageMs / c, -b);
+      }
+      case "ebbinghaus": {
+        const stability = halfLife;
+        return Math.exp(-ageMs / stability);
+      }
+      default:
+        return Math.exp(-Math.LN2 * ageMs / halfLife);
+    }
+  }
+  /**
+   * Apply forgetting to memories in a store
+   */
+  async applyForgetting(store) {
+    const { entries } = await store.query({ limit: 1e4 });
+    const now = Date.now();
+    const forgotten = [];
+    const decayed = [];
+    const retained = [];
+    let totalRetention = 0;
+    for (const entry of entries) {
+      const score = this.calculateRetention(entry, now);
+      totalRetention += score.retention;
+      if (score.shouldForget) {
+        await store.delete(entry.id);
+        forgotten.push(entry.id);
+      } else if (score.retention < 0.5 && entry.importance > 0.3) {
+        const newImportance = Math.max(
+          entry.importance * score.retention,
+          this.config.minRetention
+        );
+        if (newImportance < entry.importance) {
+          await store.update(entry.id, { importance: newImportance });
+          decayed.push({
+            id: entry.id,
+            oldImportance: entry.importance,
+            newImportance
+          });
+        }
+        retained.push(entry.id);
+      } else {
+        retained.push(entry.id);
+      }
+    }
+    return {
+      forgotten,
+      decayed,
+      retained,
+      avgRetention: entries.length > 0 ? totalRetention / entries.length : 1
+    };
+  }
+  /**
+   * Simulate forgetting over time (for testing/visualization)
+   */
+  simulateForgetting(entries, timePeriodMs, steps = 10) {
+    const results = [];
+    const stepMs = timePeriodMs / steps;
+    const baseTime = Date.now();
+    for (let i = 0; i <= steps; i++) {
+      const simulatedNow = baseTime + i * stepMs;
+      let totalRetention = 0;
+      let forgottenCount = 0;
+      for (const entry of entries) {
+        const score = this.calculateRetention(entry, simulatedNow);
+        totalRetention += score.retention;
+        if (score.shouldForget) forgottenCount++;
+      }
+      results.push({
+        time: i * stepMs,
+        avgRetention: entries.length > 0 ? totalRetention / entries.length : 1,
+        forgottenCount
+      });
+    }
+    return results;
+  }
+  /**
+   * Get memories at risk of being forgotten
+   */
+  async getAtRiskMemories(store, threshold = 0.2) {
+    const { entries } = await store.query({ limit: 1e3 });
+    const now = Date.now();
+    const atRisk = [];
+    for (const entry of entries) {
+      const score = this.calculateRetention(entry, now);
+      if (score.retention < threshold && score.retention >= this.config.forgetThreshold) {
+        atRisk.push({ entry, retention: score.retention });
+      }
+    }
+    return Promise.resolve(atRisk.sort((a, b) => a.retention - b.retention));
+  }
+  /**
+   * Reinforce a memory (simulate rehearsal)
+   */
+  async reinforce(store, id, boost = 0.2) {
+    const entry = await store.get(id);
+    if (!entry) return false;
+    const newImportance = Math.min(entry.importance + boost, 1);
+    return store.update(id, {
+      importance: newImportance,
+      lastAccessedAt: Date.now()
+    });
+  }
+  /**
+   * Batch reinforce multiple memories
+   */
+  async reinforceBatch(store, ids, boost = 0.1) {
+    let count = 0;
+    for (const id of ids) {
+      if (await this.reinforce(store, id, boost)) {
+        count++;
+      }
+    }
+    return count;
+  }
+  /**
+   * Calculate optimal review schedule for a memory
+   */
+  calculateReviewSchedule(entry, targetRetention = 0.8) {
+    const baseInterval = this.config.halfLife / 10;
+    const intervals = [];
+    let _currentInterval = baseInterval;
+    let simulatedTime = 0;
+    const maxReviews = 10;
+    for (let i = 0; i < maxReviews; i++) {
+      let retention = 1;
+      let checkTime = simulatedTime;
+      while (retention > targetRetention) {
+        checkTime += baseInterval / 10;
+        const ageMs = checkTime - entry.timestamp;
+        retention = this.calculateBaseRetention(ageMs);
+      }
+      intervals.push(checkTime);
+      simulatedTime = checkTime;
+      _currentInterval *= 2;
+    }
+    return intervals;
+  }
+  /**
+   * Get statistics about memory retention
+   */
+  async getRetentionStats(store) {
+    const { entries } = await store.query({ limit: 1e4 });
+    const now = Date.now();
+    let totalRetention = 0;
+    let atRiskCount = 0;
+    let forgettableCount = 0;
+    let healthyCount = 0;
+    const distribution = {
+      "0-20%": 0,
+      "20-40%": 0,
+      "40-60%": 0,
+      "60-80%": 0,
+      "80-100%": 0
+    };
+    for (const entry of entries) {
+      const score = this.calculateRetention(entry, now);
+      totalRetention += score.retention;
+      if (score.shouldForget) {
+        forgettableCount++;
+      } else if (score.retention < 0.3) {
+        atRiskCount++;
+      } else {
+        healthyCount++;
+      }
+      if (score.retention < 0.2) distribution["0-20%"]++;
+      else if (score.retention < 0.4) distribution["20-40%"]++;
+      else if (score.retention < 0.6) distribution["40-60%"]++;
+      else if (score.retention < 0.8) distribution["60-80%"]++;
+      else distribution["80-100%"]++;
+    }
+    return {
+      avgRetention: entries.length > 0 ? totalRetention / entries.length : 1,
+      atRiskCount,
+      forgettableCount,
+      healthyCount,
+      distribution
+    };
+  }
+  /**
+   * Update configuration
+   */
+  configure(config) {
+    this.config = { ...this.config, ...config };
+  }
+};
+function createForgetter(config) {
+  return new Forgetter(config);
+}
+
+// src/processing/Extractor.ts
+var Extractor = class {
+  config;
+  extractFn;
+  constructor(config = {}) {
+    this.config = {
+      provider: config.provider ?? void 0,
+      model: config.model ?? "default",
+      extractTypes: config.extractTypes ?? [],
+      customPrompt: config.customPrompt ?? "",
+      confidence: config.confidence ?? 0.5,
+      extractEntities: config.extractEntities ?? true,
+      extractRelations: config.extractRelations ?? true,
+      extractKeywords: config.extractKeywords ?? true,
+      extractSentiment: config.extractSentiment ?? false,
+      minConfidence: config.minConfidence ?? 0.5,
+      maxEntitiesPerEntry: config.maxEntitiesPerEntry ?? 20
+    };
+  }
+  /**
+   * Set custom extraction function (for LLM integration)
+   */
+  setExtractionFunction(fn) {
+    this.extractFn = fn;
+  }
+  /**
+   * Extract information from a memory entry
+   */
+  async extract(entry) {
+    if (this.extractFn) {
+      return this.extractFn(entry.content, {
+        extractRelations: this.config.extractRelations,
+        extractSentiment: this.config.extractSentiment
+      });
+    }
+    return Promise.resolve(this.heuristicExtract(entry.content));
+  }
+  /**
+   * Extract from multiple entries
+   */
+  async extractBatch(entries) {
+    const results = /* @__PURE__ */ new Map();
+    for (const entry of entries) {
+      const result = await this.extract(entry);
+      results.set(entry.id, result);
+    }
+    return results;
+  }
+  /**
+   * Extract and aggregate across multiple entries
+   */
+  async extractAggregate(entries) {
+    const entityMap = /* @__PURE__ */ new Map();
+    const allRelations = [];
+    const keywordCounts = /* @__PURE__ */ new Map();
+    let sentimentSum = 0;
+    let sentimentCount = 0;
+    for (const entry of entries) {
+      const result = await this.extract(entry);
+      for (const entity of result.entities) {
+        const key = `${entity.type}:${entity.text.toLowerCase()}`;
+        const existing = entityMap.get(key);
+        if (existing) {
+          existing.count++;
+          existing.entity.confidence = Math.max(
+            existing.entity.confidence,
+            entity.confidence
+          );
+        } else {
+          entityMap.set(key, { entity, count: 1 });
+        }
+      }
+      allRelations.push(...result.relations);
+      for (const keyword of result.keywords) {
+        const lower = keyword.toLowerCase();
+        keywordCounts.set(lower, (keywordCounts.get(lower) ?? 0) + 1);
+      }
+      if (result.sentiment) {
+        sentimentSum += result.sentiment.score;
+        sentimentCount++;
+      }
+    }
+    const topKeywords = Array.from(keywordCounts.entries()).map(([keyword, count]) => ({ keyword, count })).sort((a, b) => b.count - a.count).slice(0, 20);
+    return {
+      allEntities: entityMap,
+      allRelations,
+      topKeywords,
+      avgSentiment: sentimentCount > 0 ? sentimentSum / sentimentCount : null
+    };
+  }
+  /**
+   * Heuristic-based extraction (fallback)
+   */
+  heuristicExtract(content) {
+    const entities = [];
+    const relations = [];
+    const keywords = [];
+    if (this.config.extractEntities) {
+      entities.push(...this.extractEntities(content));
+    }
+    if (this.config.extractRelations) {
+      relations.push(...this.extractRelations(content));
+    }
+    if (this.config.extractKeywords) {
+      keywords.push(...this.extractKeywords(content));
+    }
+    const sentiment = this.config.extractSentiment ? this.extractSentiment(content) : void 0;
+    return {
+      entities: entities.slice(0, this.config.maxEntitiesPerEntry),
+      relations,
+      keywords,
+      sentiment
+    };
+  }
+  /**
+   * Extract entities using patterns
+   */
+  extractEntities(content) {
+    const entities = [];
+    const datePatterns = [
+      /\b\d{1,2}\/\d{1,2}\/\d{2,4}\b/g,
+      /\b\d{4}-\d{2}-\d{2}\b/g,
+      /\b(?:January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{1,2}(?:,\s*\d{4})?\b/gi,
+      /\b(?:today|yesterday|tomorrow|last\s+week|next\s+month)\b/gi
+    ];
+    for (const pattern of datePatterns) {
+      let match2;
+      while ((match2 = pattern.exec(content)) !== null) {
+        entities.push({
+          text: match2[0],
+          type: "date",
+          confidence: 0.9,
+          position: { start: match2.index, end: match2.index + match2[0].length }
+        });
+      }
+    }
+    const numberPattern = /\b\d+(?:\.\d+)?(?:\s*(?:percent|%|dollars|\$|euros|€|pounds|£))?\b/gi;
+    let match;
+    while ((match = numberPattern.exec(content)) !== null) {
+      entities.push({
+        text: match[0],
+        type: "number",
+        confidence: 0.8,
+        position: { start: match.index, end: match.index + match[0].length }
+      });
+    }
+    const capitalizedPattern = /\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)+\b/g;
+    while ((match = capitalizedPattern.exec(content)) !== null) {
+      const text = match[0];
+      if (!this.isSentenceStarter(text, content, match.index)) {
+        const type = this.guessEntityType(text);
+        entities.push({
+          text,
+          type,
+          confidence: 0.6,
+          position: { start: match.index, end: match.index + text.length }
+        });
+      }
+    }
+    const emailPattern = /\b[\w.-]+@[\w.-]+\.\w+\b/g;
+    while ((match = emailPattern.exec(content)) !== null) {
+      entities.push({
+        text: match[0],
+        type: "other",
+        confidence: 0.95,
+        position: { start: match.index, end: match.index + match[0].length },
+        metadata: { subtype: "email" }
+      });
+    }
+    const urlPattern = /https?:\/\/[^\s]+/g;
+    while ((match = urlPattern.exec(content)) !== null) {
+      entities.push({
+        text: match[0],
+        type: "other",
+        confidence: 0.95,
+        position: { start: match.index, end: match.index + match[0].length },
+        metadata: { subtype: "url" }
+      });
+    }
+    return entities.filter((e) => e.confidence >= this.config.minConfidence);
+  }
+  /**
+   * Extract relations using patterns
+   */
+  extractRelations(content) {
+    const relations = [];
+    const patterns = [
+      // "X is a/an Y"
+      {
+        regex: /(\w+(?:\s+\w+)?)\s+is\s+a(?:n)?\s+(\w+(?:\s+\w+)?)/gi,
+        predicate: "is_a"
+      },
+      // "X works at Y"
+      {
+        regex: /(\w+(?:\s+\w+)?)\s+works\s+at\s+(\w+(?:\s+\w+)?)/gi,
+        predicate: "works_at"
+      },
+      // "X is located in Y"
+      {
+        regex: /(\w+(?:\s+\w+)?)\s+is\s+located\s+in\s+(\w+(?:\s+\w+)?)/gi,
+        predicate: "located_in"
+      },
+      // "X belongs to Y"
+      {
+        regex: /(\w+(?:\s+\w+)?)\s+belongs\s+to\s+(\w+(?:\s+\w+)?)/gi,
+        predicate: "belongs_to"
+      },
+      // "X created Y"
+      {
+        regex: /(\w+(?:\s+\w+)?)\s+created\s+(\w+(?:\s+\w+)?)/gi,
+        predicate: "created"
+      },
+      // "X contains Y"
+      {
+        regex: /(\w+(?:\s+\w+)?)\s+contains\s+(\w+(?:\s+\w+)?)/gi,
+        predicate: "contains"
+      }
+    ];
+    for (const { regex, predicate } of patterns) {
+      let match;
+      while ((match = regex.exec(content)) !== null) {
+        relations.push({
+          subject: match[1].trim(),
+          predicate,
+          object: match[2].trim(),
+          confidence: 0.7,
+          sourceText: match[0]
+        });
+      }
+    }
+    return relations.filter((r) => r.confidence >= this.config.minConfidence);
+  }
+  /**
+   * Extract keywords
+   */
+  extractKeywords(content) {
+    const stopWords = /* @__PURE__ */ new Set([
+      "the",
+      "a",
+      "an",
+      "and",
+      "or",
+      "but",
+      "in",
+      "on",
+      "at",
+      "to",
+      "for",
+      "of",
+      "with",
+      "by",
+      "from",
+      "as",
+      "is",
+      "was",
+      "are",
+      "were",
+      "been",
+      "be",
+      "have",
+      "has",
+      "had",
+      "do",
+      "does",
+      "did",
+      "will",
+      "would",
+      "could",
+      "should",
+      "may",
+      "might",
+      "must",
+      "shall",
+      "can",
+      "need",
+      "it",
+      "its",
+      "this",
+      "that",
+      "these",
+      "those",
+      "i",
+      "you",
+      "he",
+      "she",
+      "we",
+      "they",
+      "what",
+      "which",
+      "who",
+      "when",
+      "where",
+      "why",
+      "how",
+      "all",
+      "each",
+      "every",
+      "both",
+      "few",
+      "more",
+      "most",
+      "other",
+      "some",
+      "such",
+      "no",
+      "not",
+      "only",
+      "own",
+      "same",
+      "so",
+      "than",
+      "too",
+      "very"
+    ]);
+    const words = content.toLowerCase().replace(/[^\w\s]/g, " ").split(/\s+/).filter((word) => word.length > 3 && !stopWords.has(word));
+    const counts = /* @__PURE__ */ new Map();
+    for (const word of words) {
+      counts.set(word, (counts.get(word) ?? 0) + 1);
+    }
+    return Array.from(counts.entries()).sort((a, b) => b[1] - a[1]).slice(0, 10).map(([word]) => word);
+  }
+  /**
+   * Extract sentiment
+   */
+  extractSentiment(content) {
+    const positiveWords = /* @__PURE__ */ new Set([
+      "good",
+      "great",
+      "excellent",
+      "amazing",
+      "wonderful",
+      "fantastic",
+      "happy",
+      "love",
+      "best",
+      "awesome",
+      "nice",
+      "beautiful",
+      "perfect",
+      "success",
+      "successful",
+      "pleased",
+      "delighted",
+      "enjoy",
+      "enjoyed",
+      "helpful",
+      "thanks",
+      "thank"
+    ]);
+    const negativeWords = /* @__PURE__ */ new Set([
+      "bad",
+      "terrible",
+      "awful",
+      "horrible",
+      "poor",
+      "worst",
+      "hate",
+      "sad",
+      "angry",
+      "disappointed",
+      "frustrating",
+      "annoying",
+      "problem",
+      "issue",
+      "error",
+      "fail",
+      "failed",
+      "failure",
+      "wrong",
+      "broken",
+      "difficult",
+      "hard"
+    ]);
+    const words = content.toLowerCase().split(/\s+/);
+    let positiveCount = 0;
+    let negativeCount = 0;
+    for (const word of words) {
+      if (positiveWords.has(word)) positiveCount++;
+      if (negativeWords.has(word)) negativeCount++;
+    }
+    const total = positiveCount + negativeCount;
+    if (total === 0) {
+      return { score: 0, label: "neutral" };
+    }
+    const score = (positiveCount - negativeCount) / total;
+    let label;
+    if (score > 0.2) label = "positive";
+    else if (score < -0.2) label = "negative";
+    else label = "neutral";
+    return { score, label };
+  }
+  /**
+   * Check if text is a sentence starter
+   */
+  isSentenceStarter(_text, content, index) {
+    if (index === 0) return true;
+    const prevChar = content[index - 1];
+    return prevChar === "." || prevChar === "!" || prevChar === "?" || prevChar === "\n";
+  }
+  /**
+   * Guess entity type from text
+   */
+  guessEntityType(text) {
+    const lowerText = text.toLowerCase();
+    const locationIndicators = [
+      "city",
+      "state",
+      "country",
+      "street",
+      "avenue",
+      "road"
+    ];
+    if (locationIndicators.some((ind) => lowerText.includes(ind))) {
+      return "location";
+    }
+    const orgIndicators = [
+      "inc",
+      "corp",
+      "company",
+      "organization",
+      "foundation",
+      "institute"
+    ];
+    if (orgIndicators.some((ind) => lowerText.includes(ind))) {
+      return "organization";
+    }
+    const wordCount = text.split(/\s+/).length;
+    if (wordCount >= 2 && wordCount <= 3) {
+      return "person";
+    }
+    return "concept";
+  }
+  /**
+   * Update configuration
+   */
+  configure(config) {
+    this.config = { ...this.config, ...config };
+  }
+};
+function createExtractor(config) {
+  return new Extractor(config);
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  Compressor,
+  Consolidator,
+  Extractor,
+  Forgetter,
+  Summarizer,
+  createCompressor,
+  createConsolidator,
+  createExtractor,
+  createForgetter,
+  createSummarizer
+});