@stackmemoryai/stackmemory 0.5.59 → 0.5.61

package/dist/core/retrieval/llm-context-retrieval.js

@@ -1,613 +0,0 @@
-import { fileURLToPath as __fileURLToPath } from 'url';
-import { dirname as __pathDirname } from 'path';
-const __filename = __fileURLToPath(import.meta.url);
-const __dirname = __pathDirname(__filename);
-import { QueryParser } from "../query/query-parser.js";
-import { CompressedSummaryGenerator } from "./summary-generator.js";
-import {
-  DEFAULT_RETRIEVAL_CONFIG
-} from "./types.js";
-import { logger } from "../monitoring/logger.js";
-import { LazyContextLoader } from "../performance/lazy-context-loader.js";
-import { ContextCache } from "../performance/context-cache.js";
-import { createLLMProvider } from "./llm-provider.js";
-import { RetrievalAuditStore } from "./retrieval-audit.js";
-class HeuristicAnalyzer {
-  analyze(query, summary, parsedQuery) {
-    const framesToRetrieve = [];
-    const recommendations = [];
-    const matchedPatterns = [];
-    const queryLower = query.toLowerCase();
-    const queryWords = queryLower.split(/\W+/).filter((w) => w.length > 2);
-    for (const frame of summary.recentSession.frames) {
-      let priority = 5;
-      const reasons = [];
-      const ageHours = (Date.now() - frame.createdAt) / (1e3 * 60 * 60);
-      if (ageHours < 1) {
-        priority += 3;
-        reasons.push("very recent");
-      } else if (ageHours < 6) {
-        priority += 2;
-        reasons.push("recent");
-      }
-      priority += Math.floor(frame.score * 3);
-      const nameLower = frame.name.toLowerCase();
-      const nameMatches = queryWords.filter((w) => nameLower.includes(w));
-      if (nameMatches.length > 0) {
-        priority += nameMatches.length * 2;
-        reasons.push(`matches: ${nameMatches.join(", ")}`);
-        matchedPatterns.push(`name_match:${nameMatches.join(",")}`);
-      }
-      if (parsedQuery?.frame?.type) {
-        const frameType = frame.type.toLowerCase();
-        if (parsedQuery.frame.type.some((t) => t.toLowerCase() === frameType)) {
-          priority += 2;
-          reasons.push("type match");
-        }
-      }
-      if (parsedQuery?.content?.topic) {
-        const topics = parsedQuery.content.topic;
-        const topicMatches = topics.filter(
-          (t) => nameLower.includes(t.toLowerCase()) || frame.digestPreview && frame.digestPreview.toLowerCase().includes(t.toLowerCase())
-        );
-        if (topicMatches.length > 0) {
-          priority += topicMatches.length;
-          reasons.push(`topic: ${topicMatches.join(", ")}`);
-        }
-      }
-      priority = Math.min(priority, 10);
-      if (priority >= 5) {
-        framesToRetrieve.push({
-          frameId: frame.frameId,
-          priority,
-          reason: reasons.length > 0 ? reasons.join("; ") : "relevant context",
-          includeEvents: priority >= 7,
-          includeAnchors: true,
-          includeDigest: true,
-          estimatedTokens: this.estimateFrameTokens(frame)
-        });
-      }
-    }
-    framesToRetrieve.sort((a, b) => b.priority - a.priority);
-    if (summary.recentSession.errorsEncountered.length > 0) {
-      recommendations.push({
-        type: "include",
-        target: "error_context",
-        reason: `${summary.recentSession.errorsEncountered.length} errors encountered recently`,
-        impact: "medium"
-      });
-    }
-    if (queryLower.includes("decision") || queryLower.includes("why") || queryLower.includes("chose")) {
-      recommendations.push({
-        type: "include",
-        target: "decisions",
-        reason: "Query appears to be about past decisions",
-        impact: "high"
-      });
-    }
-    const avgPriority = framesToRetrieve.length > 0 ? framesToRetrieve.reduce((sum, f) => sum + f.priority, 0) / framesToRetrieve.length : 0;
-    const confidenceScore = Math.min(avgPriority / 10, 0.95);
-    const reasoning = this.generateReasoning(
-      query,
-      framesToRetrieve,
-      summary,
-      matchedPatterns
-    );
-    return {
-      reasoning,
-      framesToRetrieve: framesToRetrieve.slice(0, 10),
-      // Limit to top 10
-      confidenceScore,
-      recommendations,
-      metadata: {
-        analysisTimeMs: 0,
-        // Will be set by caller
-        summaryTokens: this.estimateSummaryTokens(summary),
-        queryComplexity: this.assessQueryComplexity(query, parsedQuery),
-        matchedPatterns,
-        fallbackUsed: true
-      }
-    };
-  }
-  estimateFrameTokens(frame) {
-    let tokens = 50;
-    tokens += frame.eventCount * 30;
-    tokens += frame.anchorCount * 40;
-    if (frame.digestPreview) tokens += frame.digestPreview.length / 4;
-    return Math.floor(tokens);
-  }
-  estimateSummaryTokens(summary) {
-    return Math.floor(JSON.stringify(summary).length / 4);
-  }
-  assessQueryComplexity(query, parsedQuery) {
-    const wordCount = query.split(/\s+/).length;
-    const hasTimeFilter = !!parsedQuery?.time;
-    const hasContentFilter = !!parsedQuery?.content;
-    const hasPeopleFilter = !!parsedQuery?.people;
-    const hasFrameFilter = !!parsedQuery?.frame;
-    const filterCount = [
-      hasTimeFilter,
-      hasContentFilter,
-      hasPeopleFilter,
-      hasFrameFilter
-    ].filter(Boolean).length;
-    if (wordCount <= 5 && filterCount <= 1) return "simple";
-    if (wordCount <= 15 && filterCount <= 2) return "moderate";
-    return "complex";
-  }
-  generateReasoning(query, frames, summary, matchedPatterns) {
-    const parts = [];
-    parts.push(`Query: "${query}"`);
-    parts.push(
-      `Analyzed ${summary.recentSession.frames.length} recent frames.`
-    );
-    if (matchedPatterns.length > 0) {
-      parts.push(`Matched patterns: ${matchedPatterns.join(", ")}`);
-    }
-    if (frames.length > 0) {
-      parts.push(`Selected ${frames.length} frames for retrieval.`);
-      const topFrames = frames.slice(0, 3);
-      parts.push(
-        `Top frames: ${topFrames.map((f) => `${f.frameId} (priority: ${f.priority})`).join(", ")}`
-      );
-    } else {
-      parts.push("No highly relevant frames found. Using general context.");
-    }
-    return parts.join(" ");
-  }
-}
-class LLMContextRetrieval {
-  db;
-  frameManager;
-  summaryGenerator;
-  queryParser;
-  heuristicAnalyzer;
-  llmProvider;
-  config;
-  projectId;
-  lazyLoader;
-  contextCache;
-  auditStore;
-  enableAudit;
-  constructor(db, frameManager, projectId, config = {}, llmProvider) {
-    this.db = db;
-    this.frameManager = frameManager;
-    this.projectId = projectId;
-    this.config = { ...DEFAULT_RETRIEVAL_CONFIG, ...config };
-    this.llmProvider = llmProvider ?? createLLMProvider();
-    if (this.llmProvider) {
-      logger.info("LLM provider configured for context retrieval", {
-        projectId,
-        provider: this.config.llmConfig.provider
-      });
-    }
-    this.summaryGenerator = new CompressedSummaryGenerator(
-      db,
-      frameManager,
-      projectId,
-      config
-    );
-    this.queryParser = new QueryParser();
-    this.heuristicAnalyzer = new HeuristicAnalyzer();
-    this.auditStore = new RetrievalAuditStore(db, projectId);
-    this.enableAudit = true;
-    this.lazyLoader = new LazyContextLoader(db, projectId);
-    this.contextCache = new ContextCache({
-      maxSize: 50 * 1024 * 1024,
-      // 50MB for context cache
-      maxItems: 100,
-      defaultTTL: 6e5
-      // 10 minutes
-    });
-    this.contextCache.startCleanup(6e4);
-  }
-  /**
-   * Get the audit store for external access
-   */
-  getAuditStore() {
-    return this.auditStore;
-  }
-  /**
-   * Check if LLM provider is available
-   */
-  hasLLMProvider() {
-    return !!this.llmProvider;
-  }
-  /**
-   * Retrieve context based on query using LLM analysis (with caching)
-   */
-  async retrieveContext(query, options = {}) {
-    const startTime = Date.now();
-    const tokenBudget = options.tokenBudget || this.config.defaultTokenBudget;
-    if (!options.forceRefresh) {
-      const cacheKey = `${query}:${tokenBudget}:${JSON.stringify(options.hints || {})}`;
-      const cached = this.contextCache.get(cacheKey);
-      if (cached) {
-        logger.debug("Context cache hit", {
-          query: query.substring(0, 50),
-          cacheStats: this.contextCache.getStats()
-        });
-        return cached;
-      }
-    }
-    logger.info("Starting context retrieval", {
-      projectId: this.projectId,
-      query: query.substring(0, 100),
-      tokenBudget
-    });
-    const parsedQuery = this.queryParser.parseNaturalLanguage(query);
-    const summary = this.summaryGenerator.generateSummary({
-      forceRefresh: options.forceRefresh
-    });
-    const analysis = await this.analyzeWithLLM({
-      currentQuery: query,
-      parsedQuery,
-      compressedSummary: summary,
-      tokenBudget,
-      hints: options.hints
-    });
-    const { frames, anchors, events, tokensUsed } = await this.retrieveFrames(
-      analysis,
-      tokenBudget
-    );
-    const context = this.assembleContext(frames, anchors, events, analysis);
-    const metadata = {
-      retrievalTimeMs: Date.now() - startTime,
-      cacheHit: false,
-      // Would need cache tracking
-      framesScanned: summary.recentSession.frames.length,
-      framesIncluded: frames.length,
-      compressionRatio: tokensUsed > 0 ? tokenBudget / tokensUsed : 1
-    };
-    logger.info("Context retrieval complete", {
-      projectId: this.projectId,
-      framesIncluded: frames.length,
-      tokensUsed,
-      retrievalTimeMs: metadata.retrievalTimeMs,
-      confidence: analysis.confidenceScore
-    });
-    const result = {
-      context,
-      frames,
-      anchors,
-      events,
-      analysis,
-      tokenUsage: {
-        budget: tokenBudget,
-        used: tokensUsed,
-        remaining: tokenBudget - tokensUsed
-      },
-      metadata
-    };
-    if (this.enableAudit) {
-      const provider = analysis.metadata.fallbackUsed ? "heuristic" : this.llmProvider ? "anthropic" : "heuristic";
-      this.auditStore.record(query, analysis, {
-        tokensUsed,
-        tokenBudget,
-        provider
-      });
-    }
-    if (!options.forceRefresh) {
-      const cacheKey = `${query}:${tokenBudget}:${JSON.stringify(options.hints || {})}`;
-      this.contextCache.set(cacheKey, result, {
-        ttl: 6e5
-        // 10 minutes
-      });
-    }
-    return result;
-  }
-  /**
-   * Perform LLM analysis or fall back to heuristics
-   */
-  async analyzeWithLLM(request) {
-    const startTime = Date.now();
-    if (this.llmProvider) {
-      try {
-        const prompt = this.buildAnalysisPrompt(request);
-        const response = await this.llmProvider.analyze(
-          prompt,
-          this.config.llmConfig.maxTokens
-        );
-        const analysis = this.parseAnalysisResponse(response, request);
-        analysis.metadata.analysisTimeMs = Date.now() - startTime;
-        analysis.metadata.fallbackUsed = false;
-        if (analysis.confidenceScore >= this.config.minConfidenceThreshold) {
-          return analysis;
-        }
-        logger.warn("LLM confidence below threshold, using fallback", {
-          confidence: analysis.confidenceScore,
-          threshold: this.config.minConfidenceThreshold
-        });
-      } catch (error) {
-        logger.error(
-          "LLM analysis failed, using fallback",
-          error instanceof Error ? error : new Error(String(error))
-        );
-      }
-    }
-    if (this.config.enableFallback) {
-      const analysis = this.heuristicAnalyzer.analyze(
-        request.currentQuery,
-        request.compressedSummary,
-        request.parsedQuery
-      );
-      analysis.metadata.analysisTimeMs = Date.now() - startTime;
-      return analysis;
-    }
-    return {
-      reasoning: "Unable to perform analysis - LLM unavailable and fallback disabled",
-      framesToRetrieve: [],
-      confidenceScore: 0,
-      recommendations: [],
-      metadata: {
-        analysisTimeMs: Date.now() - startTime,
-        summaryTokens: 0,
-        queryComplexity: "simple",
-        matchedPatterns: [],
-        fallbackUsed: false
-      }
-    };
-  }
-  /**
-   * Build the prompt for LLM analysis
-   */
-  buildAnalysisPrompt(request) {
-    const summary = request.compressedSummary;
-    return `You are analyzing a code project's memory to retrieve relevant context.
-
-## Current Query
-"${request.currentQuery}"
-
-## Token Budget
-${request.tokenBudget} tokens available
-
-## Recent Session Summary
-- Frames: ${summary.recentSession.frames.length}
-- Time range: ${new Date(summary.recentSession.timeRange.start).toISOString()} to ${new Date(summary.recentSession.timeRange.end).toISOString()}
-- Dominant operations: ${summary.recentSession.dominantOperations.map((o) => `${o.operation}(${o.count})`).join(", ")}
-- Files touched: ${summary.recentSession.filesTouched.slice(0, 5).map((f) => f.path).join(", ")}
-- Errors: ${summary.recentSession.errorsEncountered.length}
-
-## Available Frames
-${summary.recentSession.frames.slice(0, 15).map(
-      (f) => `- ${f.frameId}: "${f.name}" (${f.type}, score: ${f.score.toFixed(2)}, events: ${f.eventCount})`
-    ).join("\n")}
-
-## Key Decisions
-${summary.historicalPatterns.keyDecisions.slice(0, 5).map((d) => `- ${d.text.substring(0, 80)}...`).join("\n")}
-
-## Task
-Analyze the query and select the most relevant frames to retrieve.
-Return a JSON object with:
-{
-  "reasoning": "Your analysis of why these frames are relevant",
-  "framesToRetrieve": [
-    {"frameId": "...", "priority": 1-10, "reason": "...", "includeEvents": true/false, "includeAnchors": true/false}
-  ],
-  "confidenceScore": 0.0-1.0,
-  "recommendations": [{"type": "include/exclude/summarize", "target": "...", "reason": "...", "impact": "low/medium/high"}]
-}
-
-${request.hints ? `
-## Hints
-${JSON.stringify(request.hints)}` : ""}
-
-Respond with only the JSON object, no other text.`;
-  }
-  /**
-   * Parse LLM response into structured analysis
-   */
-  parseAnalysisResponse(response, request) {
-    try {
-      let jsonStr = response;
-      const jsonMatch = response.match(/```(?:json)?\s*([\s\S]*?)```/);
-      if (jsonMatch) {
-        jsonStr = jsonMatch[1];
-      }
-      const parsed = JSON.parse(jsonStr.trim());
-      return {
-        reasoning: parsed.reasoning || "No reasoning provided",
-        framesToRetrieve: (parsed.framesToRetrieve || []).map((f) => ({
-          frameId: f.frameId,
-          priority: Math.min(10, Math.max(1, f.priority || 5)),
-          reason: f.reason || "Selected by LLM",
-          includeEvents: f.includeEvents ?? true,
-          includeAnchors: f.includeAnchors ?? true,
-          includeDigest: f.includeDigest ?? true,
-          estimatedTokens: f.estimatedTokens || 100
-        })),
-        confidenceScore: Math.min(
-          1,
-          Math.max(0, parsed.confidenceScore || 0.5)
-        ),
-        recommendations: (parsed.recommendations || []).map((r) => ({
-          type: r.type || "include",
-          target: r.target || "",
-          reason: r.reason || "",
-          impact: r.impact || "medium"
-        })),
-        metadata: {
-          analysisTimeMs: 0,
-          summaryTokens: Math.floor(
-            JSON.stringify(request.compressedSummary).length / 4
-          ),
-          queryComplexity: this.assessQueryComplexity(request.currentQuery),
-          matchedPatterns: [],
-          fallbackUsed: false
-        }
-      };
-    } catch (error) {
-      logger.warn("Failed to parse LLM response, using fallback", {
-        error,
-        response
-      });
-      return this.heuristicAnalyzer.analyze(
-        request.currentQuery,
-        request.compressedSummary,
-        request.parsedQuery
-      );
-    }
-  }
-  assessQueryComplexity(query) {
-    const wordCount = query.split(/\s+/).length;
-    if (wordCount <= 5) return "simple";
-    if (wordCount <= 15) return "moderate";
-    return "complex";
-  }
-  /**
-   * Retrieve frames based on analysis (with lazy loading)
-   */
-  async retrieveFrames(analysis, tokenBudget) {
-    const frames = [];
-    const anchors = [];
-    const events = [];
-    let tokensUsed = 0;
-    const frameIds = analysis.framesToRetrieve.map((p) => p.frameId);
-    await this.lazyLoader.preloadContext(frameIds, {
-      parallel: true,
-      depth: 2
-      // Load frames, anchors, and events
-    });
-    for (const plan of analysis.framesToRetrieve) {
-      if (tokensUsed + plan.estimatedTokens > tokenBudget) {
-        logger.debug("Token budget exceeded, stopping retrieval", {
-          tokensUsed,
-          budget: tokenBudget
-        });
-        break;
-      }
-      try {
-        const frame = await this.lazyLoader.lazyFrame(plan.frameId).get();
-        frames.push(frame);
-        tokensUsed += 50;
-        if (plan.includeAnchors) {
-          const frameAnchors = await this.lazyLoader.lazyAnchors(plan.frameId).get();
-          anchors.push(...frameAnchors);
-          tokensUsed += frameAnchors.length * 40;
-        }
-        if (plan.includeEvents) {
-          const frameEvents = await this.lazyLoader.lazyEvents(plan.frameId, 10).get();
-          events.push(...frameEvents);
-          tokensUsed += frameEvents.length * 30;
-        }
-      } catch (error) {
-        logger.warn("Failed to retrieve frame", {
-          frameId: plan.frameId,
-          error
-        });
-      }
-    }
-    return { frames, anchors, events, tokensUsed };
-  }
-  getFrameAnchors(frameId) {
-    try {
-      const rows = this.db.prepare(
-        `
-        SELECT * FROM anchors WHERE frame_id = ?
-        ORDER BY priority DESC, created_at DESC
-      `
-      ).all(frameId);
-      return rows.map((row) => ({
-        ...row,
-        metadata: JSON.parse(row.metadata || "{}")
-      }));
-    } catch {
-      return [];
-    }
-  }
-  /**
-   * Assemble final context string
-   */
-  assembleContext(frames, anchors, events, analysis) {
-    const sections = [];
-    sections.push("## Context Retrieval Analysis");
-    sections.push(
-      `*Confidence: ${(analysis.confidenceScore * 100).toFixed(0)}%*`
-    );
-    sections.push(analysis.reasoning);
-    sections.push("");
-    if (frames.length > 0) {
-      sections.push("## Relevant Frames");
-      for (const frame of frames) {
-        sections.push(`### ${frame.name} (${frame.type})`);
-        if (frame.digest_text) {
-          sections.push(frame.digest_text);
-        }
-        sections.push("");
-      }
-    }
-    const decisions = anchors.filter((a) => a.type === "DECISION");
-    const constraints = anchors.filter((a) => a.type === "CONSTRAINT");
-    const facts = anchors.filter((a) => a.type === "FACT");
-    if (decisions.length > 0) {
-      sections.push("## Key Decisions");
-      for (const d of decisions.slice(0, 5)) {
-        sections.push(`- ${d.text}`);
-      }
-      sections.push("");
-    }
-    if (constraints.length > 0) {
-      sections.push("## Active Constraints");
-      for (const c of constraints.slice(0, 5)) {
-        sections.push(`- ${c.text}`);
-      }
-      sections.push("");
-    }
-    if (facts.length > 0) {
-      sections.push("## Important Facts");
-      for (const f of facts.slice(0, 5)) {
-        sections.push(`- ${f.text}`);
-      }
-      sections.push("");
-    }
-    if (events.length > 0) {
-      sections.push("## Recent Activity");
-      const eventSummary = this.summarizeEvents(events);
-      sections.push(eventSummary);
-      sections.push("");
-    }
-    if (analysis.recommendations.length > 0) {
-      sections.push("## Recommendations");
-      for (const rec of analysis.recommendations) {
-        const icon = rec.type === "include" ? "+" : rec.type === "exclude" ? "-" : "~";
-        sections.push(`${icon} [${rec.impact.toUpperCase()}] ${rec.reason}`);
-      }
-    }
-    return sections.join("\n");
-  }
-  summarizeEvents(events) {
-    const byType = {};
-    for (const event of events) {
-      byType[event.event_type] = (byType[event.event_type] || 0) + 1;
-    }
-    return Object.entries(byType).map(([type, count]) => `- ${type}: ${count} occurrences`).join("\n");
-  }
-  /**
-   * Get just the compressed summary (useful for external analysis)
-   */
-  getSummary(forceRefresh = false) {
-    return this.summaryGenerator.generateSummary({ forceRefresh });
-  }
-  /**
-   * Set LLM provider
-   */
-  setLLMProvider(provider) {
-    this.llmProvider = provider;
-  }
-  /**
-   * Clear all caches
-   */
-  clearCache() {
-    this.summaryGenerator.clearCache();
-    this.lazyLoader.clearCache();
-    this.contextCache.clear();
-    logger.info("Cleared all caches", {
-      projectId: this.projectId,
-      cacheStats: this.contextCache.getStats()
-    });
-  }
-}
-export {
-  LLMContextRetrieval
-};
-//# sourceMappingURL=llm-context-retrieval.js.map