@stackmemoryai/stackmemory 0.3.1 → 0.3.5

package/dist/core/retrieval/hierarchical-retrieval.js

@@ -0,0 +1,652 @@
+import { logger } from "../monitoring/logger.js";
+import * as zlib from "zlib";
+import { promisify } from "util";
+import crypto from "crypto";
+const gzipAsync = promisify(zlib.gzip);
+const gunzipAsync = promisify(zlib.gunzip);
+const DEFAULT_HIERARCHY_CONFIG = {
+  maxEncyclopediaSize: 5e4,
+  maxChapterSize: 6e3,
+  maxSectionSize: 250,
+  maxParagraphSize: 20,
+  compressionThreshold: 1e3,
+  semanticThreshold: 0.7
+};
+class HierarchicalRetrieval {
+  db;
+  config;
+  hierarchyCache = /* @__PURE__ */ new Map();
+  summaryCache = /* @__PURE__ */ new Map();
+  constructor(db, config = {}) {
+    this.db = db;
+    this.config = { ...DEFAULT_HIERARCHY_CONFIG, ...config };
+    this.initializeSchema();
+  }
+  initializeSchema() {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS hierarchy_nodes (
+        id TEXT PRIMARY KEY,
+        level TEXT NOT NULL,
+        parent_id TEXT,
+        title TEXT NOT NULL,
+        summary TEXT,
+        embeddings BLOB,
+        child_count INTEGER DEFAULT 0,
+        token_count INTEGER DEFAULT 0,
+        score REAL DEFAULT 0,
+        time_start INTEGER,
+        time_end INTEGER,
+        compression_ratio REAL,
+        semantic_density REAL,
+        access_pattern TEXT DEFAULT 'cold',
+        last_accessed INTEGER,
+        created_at INTEGER DEFAULT (unixepoch() * 1000),
+        FOREIGN KEY (parent_id) REFERENCES hierarchy_nodes(id) ON DELETE CASCADE
+      )
+    `);
+    this.db.exec(`
+      CREATE INDEX IF NOT EXISTS idx_hierarchy_level ON hierarchy_nodes(level);
+      CREATE INDEX IF NOT EXISTS idx_hierarchy_parent ON hierarchy_nodes(parent_id);
+      CREATE INDEX IF NOT EXISTS idx_hierarchy_score ON hierarchy_nodes(score DESC);
+      CREATE INDEX IF NOT EXISTS idx_hierarchy_time ON hierarchy_nodes(time_start, time_end);
+    `);
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS hierarchy_content (
+        node_id TEXT PRIMARY KEY,
+        content TEXT,
+        compressed BOOLEAN DEFAULT 0,
+        original_size INTEGER,
+        compressed_size INTEGER,
+        FOREIGN KEY (node_id) REFERENCES hierarchy_nodes(id) ON DELETE CASCADE
+      )
+    `);
+  }
+  /**
+   * Build hierarchical structure from traces
+   */
+  async buildHierarchy(traces) {
+    logger.info("Building hierarchical retrieval structure", {
+      traceCount: traces.length
+    });
+    traces.sort((a, b) => {
+      const timeDiff = b.metadata.startTime - a.metadata.startTime;
+      return timeDiff !== 0 ? timeDiff : b.score - a.score;
+    });
+    const encyclopediaId = this.generateId("encyclopedia");
+    const encyclopedia = {
+      id: encyclopediaId,
+      level: {
+        level: "encyclopedia",
+        id: encyclopediaId,
+        title: "Knowledge Base",
+        summary: await this.generateSummary(traces, "encyclopedia"),
+        childCount: 0,
+        tokenCount: traces.reduce((sum, t) => sum + (t.tokenCount || 0), 0),
+        score: Math.max(...traces.map((t) => t.score)),
+        timeRange: {
+          start: Math.min(...traces.map((t) => t.metadata.startTime)),
+          end: Math.max(...traces.map((t) => t.metadata.endTime))
+        },
+        metadata: {
+          semanticDensity: 1,
+          accessPattern: "hot"
+        }
+      },
+      children: []
+    };
+    const chapters = await this.partitionIntoChapters(traces);
+    for (const chapterTraces of chapters) {
+      const chapter = await this.buildChapter(chapterTraces, encyclopediaId);
+      encyclopedia.children.push(chapter);
+    }
+    encyclopedia.level.childCount = encyclopedia.children.length;
+    await this.storeNode(encyclopedia);
+    return encyclopedia;
+  }
+  /**
+   * Partition traces into chapters based on semantic similarity and time
+   */
+  async partitionIntoChapters(traces) {
+    const chapters = [];
+    let currentChapter = [];
+    for (const trace of traces) {
+      if (currentChapter.length >= this.config.maxChapterSize) {
+        chapters.push(currentChapter);
+        currentChapter = [trace];
+      } else if (currentChapter.length > 0) {
+        const similarity = await this.calculateSimilarity(
+          trace,
+          currentChapter[currentChapter.length - 1]
+        );
+        if (similarity < this.config.semanticThreshold) {
+          chapters.push(currentChapter);
+          currentChapter = [trace];
+        } else {
+          currentChapter.push(trace);
+        }
+      } else {
+        currentChapter.push(trace);
+      }
+    }
+    if (currentChapter.length > 0) {
+      chapters.push(currentChapter);
+    }
+    return chapters;
+  }
+  /**
+   * Build a chapter node
+   */
+  async buildChapter(traces, parentId) {
+    const chapterId = this.generateId("chapter");
+    const chapter = {
+      id: chapterId,
+      level: {
+        level: "chapter",
+        id: chapterId,
+        parentId,
+        title: this.generateChapterTitle(traces),
+        summary: await this.generateSummary(traces, "chapter"),
+        childCount: 0,
+        tokenCount: traces.reduce((sum, t) => sum + (t.tokenCount || 0), 0),
+        score: Math.max(...traces.map((t) => t.score)),
+        timeRange: {
+          start: Math.min(...traces.map((t) => t.metadata.startTime)),
+          end: Math.max(...traces.map((t) => t.metadata.endTime))
+        },
+        metadata: {
+          compressionRatio: 0.8,
+          semanticDensity: 0.8,
+          accessPattern: "warm"
+        }
+      },
+      children: []
+    };
+    const sections = await this.partitionIntoSections(traces);
+    for (const sectionTraces of sections) {
+      const section = await this.buildSection(sectionTraces, chapterId);
+      chapter.children.push(section);
+    }
+    chapter.level.childCount = chapter.children.length;
+    return chapter;
+  }
+  /**
+   * Build a section node
+   */
+  async buildSection(traces, parentId) {
+    const sectionId = this.generateId("section");
+    const section = {
+      id: sectionId,
+      level: {
+        level: "section",
+        id: sectionId,
+        parentId,
+        title: this.generateSectionTitle(traces),
+        summary: await this.generateSummary(traces, "section"),
+        childCount: 0,
+        tokenCount: traces.reduce((sum, t) => sum + (t.tokenCount || 0), 0),
+        score: Math.max(...traces.map((t) => t.score)),
+        timeRange: {
+          start: Math.min(...traces.map((t) => t.metadata.startTime)),
+          end: Math.max(...traces.map((t) => t.metadata.endTime))
+        },
+        metadata: {
+          compressionRatio: 0.6,
+          semanticDensity: 0.6,
+          accessPattern: "cold"
+        }
+      },
+      children: []
+    };
+    const paragraphs = await this.partitionIntoParagraphs(traces);
+    for (const paragraphTraces of paragraphs) {
+      const paragraph = await this.buildParagraph(paragraphTraces, sectionId);
+      section.children.push(paragraph);
+    }
+    section.level.childCount = section.children.length;
+    return section;
+  }
+  /**
+   * Build a paragraph (leaf) node
+   */
+  async buildParagraph(traces, parentId) {
+    const paragraphId = this.generateId("paragraph");
+    const content = traces.map((t) => {
+      return `[${new Date(t.metadata.startTime).toISOString()}] ${t.type}: ${t.summary}`;
+    }).join("\n\n");
+    let storedContent = content;
+    let compressed = false;
+    if (content.length > this.config.compressionThreshold) {
+      const compressedData = await gzipAsync(content);
+      storedContent = compressedData.toString("base64");
+      compressed = true;
+    }
+    const paragraph = {
+      id: paragraphId,
+      level: {
+        level: "paragraph",
+        id: paragraphId,
+        parentId,
+        title: this.generateParagraphTitle(traces),
+        summary: await this.generateSummary(traces, "paragraph"),
+        childCount: traces.length,
+        tokenCount: traces.reduce((sum, t) => sum + (t.tokenCount || 0), 0),
+        score: Math.max(...traces.map((t) => t.score)),
+        timeRange: {
+          start: Math.min(...traces.map((t) => t.metadata.startTime)),
+          end: Math.max(...traces.map((t) => t.metadata.endTime))
+        },
+        metadata: {
+          compressionRatio: compressed ? 0.3 : 1,
+          semanticDensity: 0.4,
+          accessPattern: "cold"
+        }
+      },
+      content: storedContent,
+      compressed
+    };
+    return paragraph;
+  }
+  /**
+   * Partition traces into sections
+   */
+  async partitionIntoSections(traces) {
+    const sections = [];
+    const sectionSize = Math.ceil(
+      traces.length / Math.ceil(traces.length / this.config.maxSectionSize)
+    );
+    for (let i = 0; i < traces.length; i += sectionSize) {
+      sections.push(traces.slice(i, i + sectionSize));
+    }
+    return sections;
+  }
+  /**
+   * Partition traces into paragraphs
+   */
+  async partitionIntoParagraphs(traces) {
+    const paragraphs = [];
+    const paragraphSize = Math.ceil(
+      traces.length / Math.ceil(traces.length / this.config.maxParagraphSize)
+    );
+    for (let i = 0; i < traces.length; i += paragraphSize) {
+      paragraphs.push(traces.slice(i, i + paragraphSize));
+    }
+    return paragraphs;
+  }
+  /**
+   * Traverse hierarchy to retrieve relevant content
+   */
+  async retrieve(query, maxDepth = 4, tokenBudget = 4e3) {
+    logger.info("Hierarchical retrieval", { query, maxDepth, tokenBudget });
+    const encyclopedia = await this.loadRootNode();
+    if (!encyclopedia) {
+      return "No content available";
+    }
+    const path = [encyclopedia];
+    let currentNode = encyclopedia;
+    let tokensUsed = 0;
+    for (let depth = 1; depth < maxDepth && tokensUsed < tokenBudget; depth++) {
+      if (!currentNode.children || currentNode.children.length === 0) {
+        break;
+      }
+      const bestChild = await this.selectBestChild(
+        currentNode.children,
+        query,
+        tokenBudget - tokensUsed
+      );
+      if (!bestChild) break;
+      path.push(bestChild);
+      currentNode = bestChild;
+      tokensUsed += bestChild.level.tokenCount;
+      await this.updateAccessPattern(bestChild.id);
+    }
+    return this.buildContextFromPath(path, tokenBudget);
+  }
+  /**
+   * Select best matching child node
+   */
+  async selectBestChild(children, query, remainingBudget) {
+    let bestChild = null;
+    let bestScore = 0;
+    for (const child of children) {
+      if (child.level.tokenCount > remainingBudget) {
+        continue;
+      }
+      const score = await this.calculateRelevance(child, query);
+      if (score > bestScore) {
+        bestScore = score;
+        bestChild = child;
+      }
+    }
+    return bestChild;
+  }
+  /**
+   * Calculate relevance of node to query
+   */
+  async calculateRelevance(node, query) {
+    const queryWords = query.toLowerCase().split(/\s+/);
+    const nodeText = `${node.level.title} ${node.level.summary}`.toLowerCase();
+    let matches = 0;
+    for (const word of queryWords) {
+      if (nodeText.includes(word)) {
+        matches++;
+      }
+    }
+    const keywordScore = matches / queryWords.length;
+    const recencyScore = 1 / (1 + (Date.now() - node.level.timeRange.end) / (1e3 * 60 * 60 * 24));
+    const importanceScore = node.level.score;
+    return keywordScore * 0.5 + recencyScore * 0.3 + importanceScore * 0.2;
+  }
+  /**
+   * Build context string from retrieval path
+   */
+  async buildContextFromPath(path, tokenBudget) {
+    const sections = [];
+    sections.push("## Retrieval Path");
+    sections.push(path.map((n) => n.level.title).join(" \u2192 "));
+    sections.push("");
+    for (const node of path) {
+      sections.push(`### ${node.level.level}: ${node.level.title}`);
+      sections.push(node.level.summary);
+      if (node.content) {
+        let content = node.content;
+        if (node.compressed) {
+          const compressed = Buffer.from(content, "base64");
+          const decompressed = await gunzipAsync(compressed);
+          content = decompressed.toString();
+        }
+        sections.push("");
+        sections.push("**Content:**");
+        sections.push(content);
+      }
+      sections.push("");
+    }
+    sections.push("## Retrieval Statistics");
+    sections.push(`- Levels traversed: ${path.length}`);
+    sections.push(
+      `- Search space reduction: ${this.calculateReduction(path)}x`
+    );
+    sections.push(
+      `- Semantic density: ${this.calculateDensity(path).toFixed(2)}`
+    );
+    return sections.join("\n");
+  }
+  /**
+   * Calculate search space reduction
+   */
+  calculateReduction(path) {
+    if (path.length < 2) return 1;
+    const initial = path[0].level.childCount;
+    const final = path[path.length - 1].level.childCount || 1;
+    return Math.round(initial / final);
+  }
+  /**
+   * Calculate semantic density along path
+   */
+  calculateDensity(path) {
+    const densities = path.map((n) => n.level.metadata.semanticDensity || 1);
+    return densities.reduce((sum, d) => sum + d, 0) / densities.length;
+  }
+  /**
+   * Generate summary for a level
+   */
+  async generateSummary(traces, level) {
+    const cacheKey = `${level}:${traces.map((t) => t.id).join(",")}`;
+    if (this.summaryCache.has(cacheKey)) {
+      return this.summaryCache.get(cacheKey);
+    }
+    let summary;
+    switch (level) {
+      case "encyclopedia":
+        summary = `Complete knowledge base with ${traces.length} traces covering ${this.getTopics(traces).join(", ")}`;
+        break;
+      case "chapter":
+        summary = `${traces.length} operations focused on ${this.getDominantOperation(traces)}`;
+        break;
+      case "section":
+        summary = `${traces.length} traces: ${this.getKeyActivities(traces).join(", ")}`;
+        break;
+      case "paragraph":
+        summary = traces.slice(0, 3).map((t) => t.summary).join(". ");
+        break;
+      default:
+        summary = `${traces.length} items`;
+    }
+    this.summaryCache.set(cacheKey, summary);
+    return summary;
+  }
+  /**
+   * Extract topics from traces
+   */
+  getTopics(traces) {
+    const topics = /* @__PURE__ */ new Set();
+    for (const trace of traces) {
+      topics.add(trace.type);
+    }
+    return Array.from(topics).slice(0, 5);
+  }
+  /**
+   * Get dominant operation type
+   */
+  getDominantOperation(traces) {
+    const counts = {};
+    for (const trace of traces) {
+      counts[trace.type] = (counts[trace.type] || 0) + 1;
+    }
+    const sorted = Object.entries(counts).sort((a, b) => b[1] - a[1]);
+    return sorted[0]?.[0] || "mixed operations";
+  }
+  /**
+   * Get key activities
+   */
+  getKeyActivities(traces) {
+    return traces.slice(0, 3).map((t) => t.type);
+  }
+  /**
+   * Generate titles
+   */
+  generateChapterTitle(traces) {
+    const start = new Date(traces[0].metadata.startTime);
+    const operation = this.getDominantOperation(traces);
+    return `${operation} (${start.toLocaleDateString()})`;
+  }
+  generateSectionTitle(traces) {
+    const start = new Date(traces[0].metadata.startTime);
+    return `Section ${start.toLocaleTimeString()}`;
+  }
+  generateParagraphTitle(traces) {
+    return `${traces.length} traces`;
+  }
+  /**
+   * Calculate similarity between traces
+   */
+  async calculateSimilarity(a, b) {
+    const typeSimilarity = a.type === b.type ? 1 : 0;
+    const timeDiff = Math.abs(a.metadata.startTime - b.metadata.startTime);
+    const timeSimilarity = 1 / (1 + timeDiff / (1e3 * 60 * 60));
+    return typeSimilarity * 0.5 + timeSimilarity * 0.5;
+  }
+  /**
+   * Store node in database
+   */
+  async storeNode(node) {
+    const stmt = this.db.prepare(`
+      INSERT OR REPLACE INTO hierarchy_nodes (
+        id, level, parent_id, title, summary,
+        child_count, token_count, score,
+        time_start, time_end,
+        compression_ratio, semantic_density, access_pattern
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+    stmt.run(
+      node.id,
+      node.level.level,
+      node.level.parentId || null,
+      node.level.title,
+      node.level.summary,
+      node.level.childCount,
+      node.level.tokenCount,
+      node.level.score,
+      node.level.timeRange.start,
+      node.level.timeRange.end,
+      node.level.metadata.compressionRatio || 1,
+      node.level.metadata.semanticDensity || 1,
+      node.level.metadata.accessPattern || "cold"
+    );
+    if (node.content) {
+      this.db.prepare(
+        `
+        INSERT OR REPLACE INTO hierarchy_content (
+          node_id, content, compressed, original_size, compressed_size
+        ) VALUES (?, ?, ?, ?, ?)
+      `
+      ).run(
+        node.id,
+        node.content,
+        node.compressed ? 1 : 0,
+        node.compressed ? node.content.length * 2 : node.content.length,
+        node.content.length
+      );
+    }
+    if (node.children) {
+      for (const child of node.children) {
+        await this.storeNode(child);
+      }
+    }
+  }
+  /**
+   * Load root encyclopedia node
+   */
+  async loadRootNode() {
+    const row = this.db.prepare(
+      `
+      SELECT * FROM hierarchy_nodes
+      WHERE level = 'encyclopedia'
+      ORDER BY created_at DESC
+      LIMIT 1
+    `
+    ).get();
+    if (!row) return null;
+    return this.loadNode(row.id);
+  }
+  /**
+   * Load node from database
+   */
+  async loadNode(nodeId) {
+    if (this.hierarchyCache.has(nodeId)) {
+      return this.hierarchyCache.get(nodeId);
+    }
+    const nodeRow = this.db.prepare(
+      `
+      SELECT * FROM hierarchy_nodes WHERE id = ?
+    `
+    ).get(nodeId);
+    if (!nodeRow) return null;
+    const contentRow = this.db.prepare(
+      `
+      SELECT * FROM hierarchy_content WHERE node_id = ?
+    `
+    ).get(nodeId);
+    const childRows = this.db.prepare(
+      `
+      SELECT id FROM hierarchy_nodes WHERE parent_id = ?
+    `
+    ).all(nodeId);
+    const children = [];
+    for (const childRow of childRows) {
+      const child = await this.loadNode(childRow.id);
+      if (child) children.push(child);
+    }
+    const node = {
+      id: nodeRow.id,
+      level: {
+        level: nodeRow.level,
+        id: nodeRow.id,
+        parentId: nodeRow.parent_id,
+        title: nodeRow.title,
+        summary: nodeRow.summary,
+        childCount: nodeRow.child_count,
+        tokenCount: nodeRow.token_count,
+        score: nodeRow.score,
+        timeRange: {
+          start: nodeRow.time_start,
+          end: nodeRow.time_end
+        },
+        metadata: {
+          compressionRatio: nodeRow.compression_ratio,
+          semanticDensity: nodeRow.semantic_density,
+          accessPattern: nodeRow.access_pattern,
+          lastAccessed: nodeRow.last_accessed
+        }
+      },
+      children: children.length > 0 ? children : void 0,
+      content: contentRow?.content,
+      compressed: contentRow?.compressed === 1
+    };
+    this.hierarchyCache.set(nodeId, node);
+    return node;
+  }
+  /**
+   * Update access pattern for a node
+   */
+  async updateAccessPattern(nodeId) {
+    this.db.prepare(
+      `
+      UPDATE hierarchy_nodes
+      SET last_accessed = ?, access_pattern = 'hot'
+      WHERE id = ?
+    `
+    ).run(Date.now(), nodeId);
+  }
+  /**
+   * Generate unique ID
+   */
+  generateId(prefix) {
+    return `${prefix}_${crypto.randomBytes(8).toString("hex")}`;
+  }
+  /**
+   * Get hierarchy statistics
+   */
+  getStatistics() {
+    const stats = this.db.prepare(
+      `
+      SELECT 
+        level,
+        COUNT(*) as count,
+        AVG(token_count) as avg_tokens,
+        AVG(child_count) as avg_children,
+        AVG(compression_ratio) as avg_compression,
+        AVG(semantic_density) as avg_density
+      FROM hierarchy_nodes
+      GROUP BY level
+    `
+    ).all();
+    const totalNodes = this.db.prepare(
+      `
+      SELECT COUNT(*) as count FROM hierarchy_nodes
+    `
+    ).get();
+    const totalContent = this.db.prepare(
+      `
+      SELECT 
+        SUM(original_size) as original,
+        SUM(compressed_size) as compressed
+      FROM hierarchy_content
+    `
+    ).get();
+    return {
+      nodesByLevel: stats,
+      totalNodes: totalNodes.count,
+      totalContent: {
+        original: totalContent?.original || 0,
+        compressed: totalContent?.compressed || 0,
+        ratio: totalContent?.original ? (1 - totalContent.compressed / totalContent.original).toFixed(2) : 0
+      },
+      cacheSize: this.hierarchyCache.size
+    };
+  }
+}
+export {
+  DEFAULT_HIERARCHY_CONFIG,
+  HierarchicalRetrieval
+};
+//# sourceMappingURL=hierarchical-retrieval.js.map