@stackmemoryai/stackmemory 0.3.0 → 0.3.3

package/dist/core/retrieval/graph-retrieval.js

@@ -0,0 +1,658 @@
+import { logger } from "../monitoring/logger.js";
+import crypto from "crypto";
+const DEFAULT_GRAPH_CONFIG = {
+  maxNodes: 1e4,
+  maxEdges: 5e4,
+  minEdgeWeight: 0.1,
+  importanceThreshold: 0.3,
+  traversalTimeout: 5e3,
+  enableBidirectional: true
+};
+class GraphRetrieval {
+  db;
+  config;
+  nodeIndex = /* @__PURE__ */ new Map();
+  adjacencyList = /* @__PURE__ */ new Map();
+  reverseAdjacencyList = /* @__PURE__ */ new Map();
+  constructor(db, config = {}) {
+    this.db = db;
+    this.config = { ...DEFAULT_GRAPH_CONFIG, ...config };
+    this.initializeSchema();
+  }
+  initializeSchema() {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS graph_nodes (
+        id TEXT PRIMARY KEY,
+        type TEXT NOT NULL,
+        label TEXT NOT NULL,
+        content TEXT,
+        importance REAL DEFAULT 0.5,
+        created INTEGER DEFAULT (unixepoch() * 1000),
+        modified INTEGER DEFAULT (unixepoch() * 1000),
+        accessed INTEGER DEFAULT (unixepoch() * 1000),
+        access_count INTEGER DEFAULT 0,
+        trace_ids TEXT,
+        frame_ids TEXT,
+        tags TEXT,
+        embeddings BLOB
+      )
+    `);
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS graph_edges (
+        id TEXT PRIMARY KEY,
+        source TEXT NOT NULL,
+        target TEXT NOT NULL,
+        type TEXT NOT NULL,
+        weight REAL DEFAULT 0.5,
+        created INTEGER DEFAULT (unixepoch() * 1000),
+        evidence TEXT,
+        bidirectional BOOLEAN DEFAULT 0,
+        FOREIGN KEY (source) REFERENCES graph_nodes(id) ON DELETE CASCADE,
+        FOREIGN KEY (target) REFERENCES graph_nodes(id) ON DELETE CASCADE,
+        UNIQUE(source, target, type)
+      )
+    `);
+    this.db.exec(`
+      CREATE INDEX IF NOT EXISTS idx_graph_nodes_type ON graph_nodes(type);
+      CREATE INDEX IF NOT EXISTS idx_graph_nodes_importance ON graph_nodes(importance DESC);
+      CREATE INDEX IF NOT EXISTS idx_graph_edges_source ON graph_edges(source);
+      CREATE INDEX IF NOT EXISTS idx_graph_edges_target ON graph_edges(target);
+      CREATE INDEX IF NOT EXISTS idx_graph_edges_type ON graph_edges(type);
+      CREATE INDEX IF NOT EXISTS idx_graph_edges_weight ON graph_edges(weight DESC);
+    `);
+  }
+  /**
+   * Build graph from traces and frames
+   */
+  async buildGraph(traces, frames) {
+    logger.info("Building semantic graph", {
+      traceCount: traces.length,
+      frameCount: frames.length
+    });
+    for (const trace of traces) {
+      await this.createDocumentNode(trace);
+    }
+    for (const frame of frames) {
+      await this.createConceptNode(frame);
+    }
+    await this.establishTemporalEdges(traces);
+    await this.establishCausalEdges(traces);
+    await this.establishSemanticEdges(traces);
+    await this.establishTopicEdges(traces);
+    await this.loadGraphIntoMemory();
+    logger.info("Graph built successfully", {
+      nodes: this.nodeIndex.size,
+      edges: this.adjacencyList.size
+    });
+  }
+  /**
+   * Create document node from trace
+   */
+  async createDocumentNode(trace) {
+    const node = {
+      id: `doc_${trace.id}`,
+      type: "document",
+      label: trace.summary.substring(0, 100),
+      content: JSON.stringify(trace),
+      importance: trace.score,
+      metadata: {
+        created: trace.metadata.startTime,
+        modified: trace.metadata.endTime,
+        accessed: Date.now(),
+        accessCount: 0,
+        traceIds: [trace.id],
+        tags: [trace.type, ...trace.metadata.filesModified.slice(0, 3)]
+      }
+    };
+    await this.insertNode(node);
+    return node;
+  }
+  /**
+   * Create concept node from frame
+   */
+  async createConceptNode(frame) {
+    const node = {
+      id: `concept_${frame.id}`,
+      type: "concept",
+      label: frame.name,
+      importance: frame.score,
+      metadata: {
+        created: frame.created_at,
+        modified: frame.updated_at || frame.created_at,
+        accessed: Date.now(),
+        accessCount: 0,
+        frameIds: [frame.id],
+        tags: [frame.type]
+      }
+    };
+    await this.insertNode(node);
+    return node;
+  }
+  /**
+   * Establish temporal edges between traces
+   */
+  async establishTemporalEdges(traces) {
+    const sorted = [...traces].sort(
+      (a, b) => a.metadata.startTime - b.metadata.startTime
+    );
+    for (let i = 0; i < sorted.length - 1; i++) {
+      const current = sorted[i];
+      const next = sorted[i + 1];
+      const timeDiff = next.metadata.startTime - current.metadata.endTime;
+      if (timeDiff < 60 * 60 * 1e3) {
+        const weight = 1 / (1 + timeDiff / (1e3 * 60));
+        await this.createEdge({
+          id: this.generateId("edge"),
+          source: `doc_${current.id}`,
+          target: `doc_${next.id}`,
+          type: "temporal",
+          weight,
+          metadata: {
+            created: Date.now(),
+            evidence: [`${timeDiff}ms gap`]
+          }
+        });
+      }
+    }
+  }
+  /**
+   * Establish causal edges based on trace relationships
+   */
+  async establishCausalEdges(traces) {
+    for (const trace of traces) {
+      if (trace.metadata.causalChain && trace.metadata.causalChain.length > 0) {
+        for (const parentId of trace.metadata.causalChain) {
+          const parentExists = traces.find((t) => t.id === parentId);
+          if (parentExists) {
+            await this.createEdge({
+              id: this.generateId("edge"),
+              source: `doc_${parentId}`,
+              target: `doc_${trace.id}`,
+              type: "causal",
+              weight: 0.9,
+              // Strong causal relationship
+              metadata: {
+                created: Date.now(),
+                evidence: ["explicit causal chain"]
+              }
+            });
+          }
+        }
+      }
+    }
+  }
+  /**
+   * Establish semantic edges based on similarity
+   */
+  async establishSemanticEdges(traces) {
+    for (let i = 0; i < traces.length - 1; i++) {
+      for (let j = i + 1; j < traces.length; j++) {
+        const similarity = this.calculateSimilarity(traces[i], traces[j]);
+        if (similarity > this.config.minEdgeWeight) {
+          await this.createEdge({
+            id: this.generateId("edge"),
+            source: `doc_${traces[i].id}`,
+            target: `doc_${traces[j].id}`,
+            type: "semantic",
+            weight: similarity,
+            metadata: {
+              created: Date.now(),
+              evidence: [`similarity: ${similarity.toFixed(2)}`],
+              bidirectional: true
+            }
+          });
+        }
+      }
+    }
+  }
+  /**
+   * Establish topic overlap edges
+   */
+  async establishTopicEdges(traces) {
+    const topicGroups = /* @__PURE__ */ new Map();
+    for (const trace of traces) {
+      const topic = trace.type;
+      if (!topicGroups.has(topic)) {
+        topicGroups.set(topic, []);
+      }
+      topicGroups.get(topic).push(trace);
+    }
+    for (const [topic, group] of topicGroups) {
+      if (group.length < 2) continue;
+      const topicNode = {
+        id: `topic_${topic}`,
+        type: "topic",
+        label: topic,
+        importance: 0.7,
+        metadata: {
+          created: Date.now(),
+          modified: Date.now(),
+          accessed: Date.now(),
+          accessCount: 0,
+          tags: [topic]
+        }
+      };
+      await this.insertNode(topicNode);
+      for (const trace of group) {
+        await this.createEdge({
+          id: this.generateId("edge"),
+          source: `doc_${trace.id}`,
+          target: topicNode.id,
+          type: "topic_overlap",
+          weight: 0.6,
+          metadata: {
+            created: Date.now(),
+            bidirectional: true
+          }
+        });
+      }
+    }
+  }
+  /**
+   * Traverse graph to find relevant paths
+   */
+  async traverse(query, config) {
+    const startTime = Date.now();
+    const queryConfig = config || {};
+    const maxHops = queryConfig.maxHops || 3;
+    const paths = [];
+    const startNodes = await this.findStartNodes(query, queryConfig);
+    if (startNodes.length === 0) {
+      logger.warn("No starting nodes found for query", { query });
+      return [];
+    }
+    for (const startNode of startNodes) {
+      const nodePaths = await this.traverseFromNode(
+        startNode,
+        query,
+        maxHops,
+        queryConfig
+      );
+      paths.push(...nodePaths);
+    }
+    paths.sort((a, b) => b.score - a.score);
+    const topPaths = paths.slice(0, 10);
+    logger.info("Graph traversal complete", {
+      query: query.substring(0, 50),
+      startNodes: startNodes.length,
+      pathsFound: paths.length,
+      timeMs: Date.now() - startTime
+    });
+    return topPaths;
+  }
+  /**
+   * Find starting nodes for traversal
+   */
+  async findStartNodes(query, config) {
+    if (config.startNodes) {
+      return config.startNodes.map((id) => this.nodeIndex.get(id)).filter((n) => n !== void 0);
+    }
+    const queryWords = query.toLowerCase().split(/\s+/);
+    const candidates = [];
+    for (const node of this.nodeIndex.values()) {
+      if (config.nodeTypes && !config.nodeTypes.includes(node.type)) {
+        continue;
+      }
+      const label = node.label.toLowerCase();
+      const tags = (node.metadata.tags || []).join(" ").toLowerCase();
+      let score = 0;
+      for (const word of queryWords) {
+        if (label.includes(word)) score += 2;
+        if (tags.includes(word)) score += 1;
+      }
+      if (score > 0) {
+        score *= node.importance;
+        candidates.push({ node, score });
+      }
+    }
+    candidates.sort((a, b) => b.score - a.score);
+    return candidates.slice(0, 5).map((c) => c.node);
+  }
+  /**
+   * Traverse from a specific node
+   */
+  async traverseFromNode(startNode, query, maxHops, config) {
+    const paths = [];
+    const visited = /* @__PURE__ */ new Set();
+    const queue = [
+      {
+        node: startNode,
+        path: {
+          nodes: [startNode],
+          edges: [],
+          score: startNode.importance,
+          reasoning: `Starting from ${startNode.type}: ${startNode.label}`
+        },
+        depth: 0
+      }
+    ];
+    while (queue.length > 0) {
+      const current = queue.shift();
+      if (current.depth >= maxHops) {
+        paths.push(current.path);
+        continue;
+      }
+      if (visited.has(current.node.id)) {
+        continue;
+      }
+      visited.add(current.node.id);
+      const edges = this.adjacencyList.get(current.node.id) || [];
+      for (const edge of edges) {
+        if (config.edgeTypes && !config.edgeTypes.includes(edge.type)) {
+          continue;
+        }
+        if (config.minWeight && edge.weight < config.minWeight) {
+          continue;
+        }
+        const targetNode = this.nodeIndex.get(edge.target);
+        if (!targetNode) continue;
+        const pathScore = this.calculatePathScore(
+          current.path,
+          edge,
+          targetNode,
+          query
+        );
+        const newPath = {
+          nodes: [...current.path.nodes, targetNode],
+          edges: [...current.path.edges, edge],
+          score: pathScore,
+          reasoning: `${current.path.reasoning} \u2192 ${edge.type} \u2192 ${targetNode.label}`
+        };
+        queue.push({
+          node: targetNode,
+          path: newPath,
+          depth: current.depth + 1
+        });
+      }
+    }
+    return paths;
+  }
+  /**
+   * Calculate path score
+   */
+  calculatePathScore(currentPath, edge, targetNode, query) {
+    let score = currentPath.score;
+    score *= edge.weight;
+    score *= targetNode.importance;
+    const queryWords = query.toLowerCase().split(/\s+/);
+    const targetLabel = targetNode.label.toLowerCase();
+    let relevance = 0;
+    for (const word of queryWords) {
+      if (targetLabel.includes(word)) relevance += 1;
+    }
+    score *= 1 + relevance * 0.2;
+    score *= Math.pow(0.9, currentPath.nodes.length);
+    return score;
+  }
+  /**
+   * Calculate similarity between traces
+   */
+  calculateSimilarity(a, b) {
+    const typeSim = a.type === b.type ? 0.3 : 0;
+    const filesA = new Set(a.metadata.filesModified);
+    const filesB = new Set(b.metadata.filesModified);
+    const intersection = new Set([...filesA].filter((x) => filesB.has(x)));
+    const union = /* @__PURE__ */ new Set([...filesA, ...filesB]);
+    const fileSim = union.size > 0 ? intersection.size / union.size * 0.3 : 0;
+    const toolsA = new Set(a.tools.map((t) => t.tool));
+    const toolsB = new Set(b.tools.map((t) => t.tool));
+    const toolIntersection = new Set([...toolsA].filter((x) => toolsB.has(x)));
+    const toolUnion = /* @__PURE__ */ new Set([...toolsA, ...toolsB]);
+    const toolSim = toolUnion.size > 0 ? toolIntersection.size / toolUnion.size * 0.2 : 0;
+    const wordsA = new Set(a.summary.toLowerCase().split(/\s+/));
+    const wordsB = new Set(b.summary.toLowerCase().split(/\s+/));
+    const wordIntersection = new Set([...wordsA].filter((x) => wordsB.has(x)));
+    const wordUnion = /* @__PURE__ */ new Set([...wordsA, ...wordsB]);
+    const textSim = wordUnion.size > 0 ? wordIntersection.size / wordUnion.size * 0.2 : 0;
+    return typeSim + fileSim + toolSim + textSim;
+  }
+  /**
+   * Insert node into database
+   */
+  async insertNode(node) {
+    this.db.prepare(
+      `
+      INSERT OR REPLACE INTO graph_nodes (
+        id, type, label, content, importance,
+        created, modified, accessed, access_count,
+        trace_ids, frame_ids, tags, embeddings
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `
+    ).run(
+      node.id,
+      node.type,
+      node.label,
+      node.content || null,
+      node.importance,
+      node.metadata.created,
+      node.metadata.modified,
+      node.metadata.accessed,
+      node.metadata.accessCount,
+      JSON.stringify(node.metadata.traceIds || []),
+      JSON.stringify(node.metadata.frameIds || []),
+      JSON.stringify(node.metadata.tags || []),
+      node.metadata.embeddings ? Buffer.from(node.metadata.embeddings) : null
+    );
+  }
+  /**
+   * Create edge in database
+   */
+  async createEdge(edge) {
+    try {
+      this.db.prepare(
+        `
+        INSERT OR IGNORE INTO graph_edges (
+          id, source, target, type, weight,
+          created, evidence, bidirectional
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+      `
+      ).run(
+        edge.id,
+        edge.source,
+        edge.target,
+        edge.type,
+        edge.weight,
+        edge.metadata.created,
+        JSON.stringify(edge.metadata.evidence || []),
+        edge.metadata.bidirectional ? 1 : 0
+      );
+    } catch (error) {
+    }
+  }
+  /**
+   * Load graph into memory for fast traversal
+   */
+  async loadGraphIntoMemory() {
+    const nodes = this.db.prepare("SELECT * FROM graph_nodes").all();
+    for (const row of nodes) {
+      const node = {
+        id: row.id,
+        type: row.type,
+        label: row.label,
+        content: row.content,
+        importance: row.importance,
+        metadata: {
+          created: row.created,
+          modified: row.modified,
+          accessed: row.accessed,
+          accessCount: row.access_count,
+          traceIds: JSON.parse(row.trace_ids || "[]"),
+          frameIds: JSON.parse(row.frame_ids || "[]"),
+          tags: JSON.parse(row.tags || "[]")
+        }
+      };
+      this.nodeIndex.set(node.id, node);
+    }
+    const edges = this.db.prepare("SELECT * FROM graph_edges").all();
+    for (const row of edges) {
+      const edge = {
+        id: row.id,
+        source: row.source,
+        target: row.target,
+        type: row.type,
+        weight: row.weight,
+        metadata: {
+          created: row.created,
+          evidence: JSON.parse(row.evidence || "[]"),
+          bidirectional: row.bidirectional === 1
+        }
+      };
+      if (!this.adjacencyList.has(edge.source)) {
+        this.adjacencyList.set(edge.source, []);
+      }
+      this.adjacencyList.get(edge.source).push(edge);
+      if (!this.reverseAdjacencyList.has(edge.target)) {
+        this.reverseAdjacencyList.set(edge.target, []);
+      }
+      this.reverseAdjacencyList.get(edge.target).push(edge);
+      if (edge.metadata.bidirectional) {
+        const reverseEdge = {
+          ...edge,
+          source: edge.target,
+          target: edge.source
+        };
+        if (!this.adjacencyList.has(reverseEdge.source)) {
+          this.adjacencyList.set(reverseEdge.source, []);
+        }
+        this.adjacencyList.get(reverseEdge.source).push(reverseEdge);
+      }
+    }
+  }
+  /**
+   * Generate unique ID
+   */
+  generateId(prefix) {
+    return `${prefix}_${crypto.randomBytes(8).toString("hex")}`;
+  }
+  /**
+   * Get graph statistics
+   */
+  getStatistics() {
+    const nodeStats = this.db.prepare(
+      `
+      SELECT 
+        type,
+        COUNT(*) as count,
+        AVG(importance) as avg_importance,
+        MAX(importance) as max_importance
+      FROM graph_nodes
+      GROUP BY type
+    `
+    ).all();
+    const edgeStats = this.db.prepare(
+      `
+      SELECT 
+        type,
+        COUNT(*) as count,
+        AVG(weight) as avg_weight,
+        MAX(weight) as max_weight
+      FROM graph_edges
+      GROUP BY type
+    `
+    ).all();
+    return {
+      nodes: {
+        total: this.nodeIndex.size,
+        byType: nodeStats,
+        inMemory: this.nodeIndex.size
+      },
+      edges: {
+        total: edgeStats.reduce((sum, e) => sum + e.count, 0),
+        byType: edgeStats,
+        adjacencyListSize: this.adjacencyList.size
+      },
+      connectivity: {
+        avgDegree: this.calculateAverageDegree(),
+        maxDegree: this.calculateMaxDegree()
+      }
+    };
+  }
+  /**
+   * Calculate average node degree
+   */
+  calculateAverageDegree() {
+    if (this.nodeIndex.size === 0) return 0;
+    let totalDegree = 0;
+    for (const nodeId of this.nodeIndex.keys()) {
+      const outgoing = this.adjacencyList.get(nodeId)?.length || 0;
+      const incoming = this.reverseAdjacencyList.get(nodeId)?.length || 0;
+      totalDegree += outgoing + incoming;
+    }
+    return totalDegree / this.nodeIndex.size;
+  }
+  /**
+   * Calculate maximum node degree
+   */
+  calculateMaxDegree() {
+    let maxDegree = 0;
+    for (const nodeId of this.nodeIndex.keys()) {
+      const outgoing = this.adjacencyList.get(nodeId)?.length || 0;
+      const incoming = this.reverseAdjacencyList.get(nodeId)?.length || 0;
+      maxDegree = Math.max(maxDegree, outgoing + incoming);
+    }
+    return maxDegree;
+  }
+  /**
+   * Export graph for visualization
+   */
+  exportForVisualization() {
+    const nodes = Array.from(this.nodeIndex.values()).map((node) => ({
+      id: node.id,
+      label: node.label,
+      type: node.type,
+      size: node.importance * 10,
+      color: this.getNodeColor(node.type)
+    }));
+    const edges = [];
+    for (const edgeList of this.adjacencyList.values()) {
+      for (const edge of edgeList) {
+        edges.push({
+          source: edge.source,
+          target: edge.target,
+          type: edge.type,
+          weight: edge.weight,
+          color: this.getEdgeColor(edge.type)
+        });
+      }
+    }
+    return { nodes, edges };
+  }
+  /**
+   * Get node color for visualization
+   */
+  getNodeColor(type) {
+    const colors = {
+      document: "#4A90E2",
+      concept: "#7ED321",
+      entity: "#F5A623",
+      event: "#D0021B",
+      decision: "#9013FE",
+      constraint: "#50E3C2",
+      topic: "#B8E986"
+    };
+    return colors[type] || "#CCCCCC";
+  }
+  /**
+   * Get edge color for visualization
+   */
+  getEdgeColor(type) {
+    const colors = {
+      citation: "#4A90E2",
+      topic_overlap: "#7ED321",
+      temporal: "#F5A623",
+      causal: "#D0021B",
+      semantic: "#9013FE",
+      structural: "#50E3C2",
+      dependency: "#B8E986",
+      evolution: "#417505",
+      contradiction: "#FF0000",
+      implementation: "#0099FF"
+    };
+    return colors[type] || "#999999";
+  }
+}
+export {
+  DEFAULT_GRAPH_CONFIG,
+  GraphRetrieval
+};
+//# sourceMappingURL=graph-retrieval.js.map