forge-server 0.1.0

package/src/query/hybrid-search.ts

@@ -0,0 +1,337 @@
+/**
+ * Hybrid search engine combining vector similarity + graph neighbor boost.
+ * Final scoring: 0.6 * vector_score + 0.3 * graph_centrality + 0.1 * recency_boost
+ *
+ * Phase 2: Full graph neighbor expansion.
+ * For each vector hit, traverse graph neighbors (IMPORTS, CALLS, TESTS, CO_MODIFIED)
+ * up to pivot_depth hops, collect neighbor file paths, and boost their scores.
+ * Graph centrality = number of times a file appears as a neighbor / max neighbor appearances.
+ */
+
+import type { VectorStore, GraphStore } from '../storage/interfaces.js';
+import type { CodeChunkPayload, QueryIntent } from '../util/types.js';
+import { embedText } from '../ingestion/embedder.js';
+import { computeScore, deduplicateByFile, buildMatchReason, type ScoredResult } from './ranking.js';
+import { QUERY_FILE_NEIGHBORS, QUERY_FILE_HOTSPOTS } from './graph-queries.js';
+import { logger } from '../util/logger.js';
+
+export interface HybridSearchOptions {
+  query: string;
+  repoId?: string;
+  limit?: number;
+  includeTests?: boolean;
+  pivotDepth?: number;
+  vectorWeight?: number;
+  graphWeight?: number;
+  /** Query intent — affects whether hotspot scoring is applied */
+  intent?: QueryIntent;
+}
+
+export interface HybridSearchResult {
+  results: ScoredResult[];
+  cacheHits: number;
+  executionTimeMs: number;
+}
+
+/**
+ * Perform hybrid search over indexed code chunks.
+ * Returns deduplicated, ranked results by file.
+ */
+export async function hybridSearch(
+  vectorStore: VectorStore,
+  graphStore: GraphStore,
+  options: HybridSearchOptions
+): Promise<HybridSearchResult> {
+  const startTime = Date.now();
+  const {
+    query,
+    repoId,
+    limit = 20,
+    includeTests = false,
+    pivotDepth = 2,
+    vectorWeight = 0.55,
+    graphWeight = 0.20,
+    intent,
+  } = options;
+
+  try {
+    // Step 1: Embed query
+    const queryVector = await embedText(query);
+
+    // Step 2: Vector search
+    const filter: Record<string, unknown> = {};
+    if (repoId) filter['repo_id'] = repoId;
+
+    const vectorResults = await vectorStore.searchCodeChunks(queryVector, {
+      limit: limit * 2, // Get extra to allow deduplication
+      filter: Object.keys(filter).length > 0 ? filter : undefined,
+      scoreThreshold: 0.1,
+    });
+
+    // Step 3: Filter test files if needed
+    const filteredResults = vectorResults.filter(r => {
+      const payload = r.payload as CodeChunkPayload;
+      if (!includeTests) {
+        const fp = payload.file_path ?? '';
+        if (fp.includes('.spec.') || fp.includes('.test.') || fp.includes('__tests__')) {
+          return false;
+        }
+      }
+      return true;
+    });
+
+    // Step 4: Graph neighbor expansion
+    // For each unique file in vector results, traverse graph neighbors and
+    // collect how often each neighboring file appears (centrality count).
+    const neighborCounts = new Map<string, number>(); // filePath -> appearance count
+    const vectorFileSet = new Set(
+      filteredResults.map(r => (r.payload as CodeChunkPayload).file_path)
+    );
+
+    const graphHealthy = await graphStore.isHealthy();
+    if (!graphHealthy && repoId) {
+      logger.warn('Graph store is unhealthy — skipping graph expansion (vector-only mode)', {
+        query: query.slice(0, 50),
+        repoId,
+      });
+    }
+    if (graphHealthy && repoId) {
+      const uniqueFiles = Array.from(vectorFileSet);
+
+      // Query graph neighbors for each vector hit file
+      await Promise.all(uniqueFiles.map(async (filePath) => {
+        try {
+          const result = await graphStore.query(
+            QUERY_FILE_NEIGHBORS(filePath, repoId, pivotDepth)
+          );
+
+          // Parse neighbor paths from raw result
+          const rows = result.raw ?? [];
+          for (const row of rows) {
+            const neighborPath = extractNeighborPath(row);
+            if (neighborPath && neighborPath !== filePath) {
+              neighborCounts.set(neighborPath, (neighborCounts.get(neighborPath) ?? 0) + 1);
+            }
+          }
+        } catch {
+          // Graph query failure is non-fatal — degrade to vector-only
+        }
+      }));
+    }
+
+    // Step 5: Compute normalized graph centrality scores
+    const maxNeighborCount = neighborCounts.size > 0
+      ? Math.max(...neighborCounts.values())
+      : 1;
+
+    // Step 5b: Fetch file hotspot scores from FalkorDB File node commit_count
+    const hotspotMap = new Map<string, number>(); // filePath -> normalized hotspot score
+    const useHotspot = intent === 'modify' || intent === 'debug' || intent === 'refactor';
+
+    if (useHotspot && graphHealthy && repoId) {
+      try {
+        const hotspotResult = await graphStore.query(
+          QUERY_FILE_HOTSPOTS(repoId, 100)
+        );
+        const rows = hotspotResult.raw ?? [];
+        let maxCommitCount = 1;
+
+        // First pass: find max commit count
+        for (const row of rows) {
+          const commitCount = extractHotspotCommitCount(row);
+          if (commitCount > maxCommitCount) maxCommitCount = commitCount;
+        }
+
+        // Second pass: normalize
+        for (const row of rows) {
+          const path = extractHotspotPath(row);
+          const commitCount = extractHotspotCommitCount(row);
+          if (path && commitCount > 0) {
+            hotspotMap.set(path, commitCount / maxCommitCount);
+          }
+        }
+      } catch {
+        // Hotspot query failure is non-fatal
+        logger.debug('Hotspot query failed — skipping hotspot scoring');
+      }
+    }
+
+    // Step 6: Convert to ScoredResult with combined scores
+    const scored: ScoredResult[] = filteredResults.map(r => {
+      const payload = r.payload as CodeChunkPayload;
+      const filePath = payload.file_path;
+
+      // Graph score: normalized neighbor appearance count
+      // Files that appear as neighbors of many vector hits get boosted
+      const rawGraphScore = neighborCounts.get(filePath) ?? 0;
+      const graphCentrality = maxNeighborCount > 0 ? rawGraphScore / maxNeighborCount : 0;
+
+      // Hotspot score: normalized commit count (0 for read intent)
+      const hotspotScore = useHotspot ? (hotspotMap.get(filePath) ?? 0) : 0;
+
+      const { finalScore, recencyScore } = computeScore(
+        r.score,
+        graphCentrality,
+        payload.indexed_at ?? Date.now(),
+        vectorWeight,
+        graphWeight,
+        undefined, // recencyWeight — use default
+        hotspotScore,
+        undefined, // hotspotWeight — use default
+        0,         // observationScore — Phase 4
+        undefined  // observationWeight — use default
+      );
+
+      return {
+        filePath,
+        repoId: payload.repo_id,
+        vectorScore: r.score,
+        graphScore: graphCentrality,
+        recencyScore,
+        hotspotScore,
+        observationScore: 0,
+        finalScore,
+        entityName: payload.entity_name,
+        entityType: payload.entity_type,
+        startLine: payload.start_line,
+        endLine: payload.end_line,
+        matchReason: buildMatchReason(r.score, graphCentrality, payload.entity_name, payload.entity_type, hotspotScore),
+        indexedAt: payload.indexed_at,
+      } satisfies ScoredResult;
+    });
+
+    // Step 7: Add neighbor files that scored above threshold but weren't in vector results
+    // These are files boosted purely by graph centrality (indirect relevance)
+    for (const [neighborPath, count] of neighborCounts) {
+      if (vectorFileSet.has(neighborPath)) continue; // Already in vector results
+
+      // Skip test files if not requested
+      if (!includeTests) {
+        if (
+          neighborPath.includes('.spec.') ||
+          neighborPath.includes('.test.') ||
+          neighborPath.includes('__tests__')
+        ) continue;
+      }
+
+      const normalizedCount = count / maxNeighborCount;
+      if (normalizedCount < 0.3) continue; // Only include strongly-connected neighbors
+
+      const neighborHotspot = useHotspot ? (hotspotMap.get(neighborPath) ?? 0) : 0;
+
+      const { finalScore, recencyScore } = computeScore(
+        0, // No direct vector score
+        normalizedCount,
+        Date.now(), // Unknown indexed_at for neighbor-only results
+        vectorWeight,
+        graphWeight,
+        undefined,
+        neighborHotspot,
+        undefined,
+        0,
+        undefined
+      );
+
+      scored.push({
+        filePath: neighborPath,
+        repoId: repoId ?? '',
+        vectorScore: 0,
+        graphScore: normalizedCount,
+        recencyScore,
+        hotspotScore: neighborHotspot,
+        observationScore: 0,
+        finalScore,
+        entityName: null,
+        entityType: 'file_level',
+        startLine: 0,
+        endLine: 0,
+        matchReason: buildMatchReason(0, normalizedCount, null, 'file_level', neighborHotspot),
+        indexedAt: undefined,
+      });
+    }
+
+    // Step 8: Deduplicate by file, keep highest score per file
+    const deduplicated = deduplicateByFile(scored);
+
+    logger.debug('Hybrid search completed', {
+      query: query.slice(0, 50),
+      vectorHits: vectorResults.length,
+      filteredHits: filteredResults.length,
+      neighborFiles: neighborCounts.size,
+      uniqueFiles: deduplicated.length,
+      durationMs: Date.now() - startTime,
+    });
+
+    return {
+      results: deduplicated.slice(0, limit),
+      cacheHits: 0, // Tracked by FileContentCache in tool handler
+      executionTimeMs: Date.now() - startTime,
+    };
+  } catch (err) {
+    logger.error('Hybrid search failed', { query: query.slice(0, 50), error: String(err) });
+    return {
+      results: [],
+      cacheHits: 0,
+      executionTimeMs: Date.now() - startTime,
+    };
+  }
+}
+
+/**
+ * Extract neighbor file path from a raw FalkorDB query result row.
+ * QUERY_FILE_NEIGHBORS returns: RETURN neighbor.path as path, ...
+ * FalkorDB non-compact format: [header, [rows], stats]
+ * Each row cell for an alias like `path` is a scalar string.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function extractNeighborPath(row: any): string | null {
+  if (!row) return null;
+
+  // Row is an array of cell values (one per column in SELECT)
+  // Column order: [path, rel_type, depth]
+  if (Array.isArray(row)) {
+    const first = row[0];
+    if (typeof first === 'string') return first;
+  }
+
+  // Object row format (some parsers return keyed objects)
+  if (typeof row === 'object' && row !== null) {
+    if (typeof row.path === 'string') return row.path;
+  }
+
+  return null;
+}
+
+/**
+ * Extract file path from a QUERY_FILE_HOTSPOTS result row.
+ * Column order: [path, commit_count, stability_score, change_velocity]
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function extractHotspotPath(row: any): string | null {
+  if (!row) return null;
+  if (Array.isArray(row)) {
+    const first = row[0];
+    if (typeof first === 'string') return first;
+  }
+  if (typeof row === 'object' && row !== null) {
+    if (typeof row.path === 'string') return row.path;
+  }
+  return null;
+}
+
+/**
+ * Extract commit_count from a QUERY_FILE_HOTSPOTS result row.
+ * Column order: [path, commit_count, stability_score, change_velocity]
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function extractHotspotCommitCount(row: any): number {
+  if (!row) return 0;
+  if (Array.isArray(row)) {
+    const second = row[1];
+    if (typeof second === 'number') return second;
+  }
+  if (typeof row === 'object' && row !== null) {
+    if (typeof row.commit_count === 'number') return row.commit_count;
+  }
+  return 0;
+}

package/src/query/intent-detector.ts

@@ -0,0 +1,131 @@
+/**
+ * Query intent classifier using keyword matching (Phase 1).
+ * Classifies queries as: read | modify | debug | refactor
+ * Phase 4 will enhance with ML-based classification.
+ */
+
+import type { QueryIntent } from '../util/types.js';
+
+const INTENT_KEYWORDS: Record<QueryIntent, string[]> = {
+  debug: [
+    'error', 'bug', 'fix', 'broken', 'failing', 'crash', 'exception',
+    'undefined', 'null', 'TypeError', 'undefined is not', 'cannot read',
+    'stack trace', 'stacktrace', 'traceback', 'debug', 'diagnose', 'why is',
+    'not working', 'breaks', 'issue', 'problem', 'wrong', 'incorrect',
+  ],
+  refactor: [
+    'refactor', 'restructure', 'reorganize', 'extract', 'move', 'rename',
+    'split', 'merge', 'clean up', 'cleanup', 'simplify', 'optimize',
+    'performance', 'improve', 'better', 'smell', 'technical debt', 'dead code',
+    'duplicate', 'dry', 'abstract',
+  ],
+  modify: [
+    'add', 'implement', 'create', 'build', 'write', 'update', 'change',
+    'modify', 'edit', 'extend', 'enhance', 'feature', 'new', 'support',
+    'integrate', 'connect', 'wire', 'hook up', 'plug in', 'configure',
+  ],
+  read: [
+    'how', 'what', 'where', 'explain', 'understand', 'show', 'list',
+    'find', 'search', 'look', 'describe', 'overview', 'structure', 'schema',
+    'architecture', 'flow', 'logic', 'context', 'about', 'definition',
+  ],
+};
+
+/**
+ * Detect the intent of a query string.
+ * Returns the intent category with highest keyword match count.
+ * Default: 'read' (safest, most common).
+ */
+export function detectIntent(query: string): QueryIntent {
+  const lowerQuery = query.toLowerCase();
+  const scores: Record<QueryIntent, number> = { debug: 0, refactor: 0, modify: 0, read: 0 };
+
+  for (const [intent, keywords] of Object.entries(INTENT_KEYWORDS) as [QueryIntent, string[]][]) {
+    for (const keyword of keywords) {
+      if (lowerQuery.includes(keyword)) {
+        scores[intent]++;
+      }
+    }
+  }
+
+  // Find intent with highest score
+  let maxScore = 0;
+  let detectedIntent: QueryIntent = 'read';
+
+  // Priority: debug > refactor > modify > read (more specific wins on tie)
+  for (const intent of ['debug', 'refactor', 'modify', 'read'] as QueryIntent[]) {
+    if (scores[intent] > maxScore) {
+      maxScore = scores[intent];
+      detectedIntent = intent;
+    }
+  }
+
+  return detectedIntent;
+}
+
+/**
+ * Get retrieval strategy adjustments based on intent.
+ * These affect ranking weights and result composition.
+ *
+ * Phase 4: 5-signal weights (vector, graph, recency, hotspot, observation).
+ *
+ * | Intent   | Vector | Graph | Recency | Hotspot | Observation |
+ * |----------|--------|-------|---------|---------|-------------|
+ * | read     | 0.55   | 0.20  | 0.05    | 0.10    | 0.10        |
+ * | modify   | 0.50   | 0.20  | 0.08    | 0.12    | 0.10        |
+ * | debug    | 0.40   | 0.25  | 0.05    | 0.10    | 0.20        |
+ * | refactor | 0.35   | 0.30  | 0.05    | 0.15    | 0.15        |
+ */
+export function getIntentStrategy(intent: QueryIntent): {
+  vectorWeight: number;
+  graphWeight: number;
+  recencyWeight: number;
+  hotspotWeight: number;
+  observationWeight: number;
+  includeTests: boolean;
+  expandDepth: number;
+} {
+  switch (intent) {
+    case 'debug':
+      return {
+        vectorWeight: 0.40,
+        graphWeight: 0.25,
+        recencyWeight: 0.05,
+        hotspotWeight: 0.10,
+        observationWeight: 0.20,  // Observations are critical for debugging
+        includeTests: true,       // Tests often show error scenarios
+        expandDepth: 3,
+      };
+    case 'refactor':
+      return {
+        vectorWeight: 0.35,
+        graphWeight: 0.30,   // Blast radius is critical for refactoring
+        recencyWeight: 0.05,
+        hotspotWeight: 0.15, // Hotspots indicate high-churn areas
+        observationWeight: 0.15,
+        includeTests: true,
+        expandDepth: 3,
+      };
+    case 'modify':
+      return {
+        vectorWeight: 0.50,
+        graphWeight: 0.20,
+        recencyWeight: 0.08,
+        hotspotWeight: 0.12,
+        observationWeight: 0.10,
+        includeTests: false,
+        expandDepth: 2,
+      };
+    case 'read':
+    default:
+      return {
+        vectorWeight: 0.55,
+        graphWeight: 0.20,
+        recencyWeight: 0.05,
+        hotspotWeight: 0.10,
+        observationWeight: 0.10,
+        includeTests: false,
+        expandDepth: 2,
+      };
+  }
+}

package/src/query/ranking.ts

@@ -0,0 +1,161 @@
+/**
+ * Result ranking and scoring for hybrid search.
+ * Phase 1: Vector score only (graph_centrality added in Phase 2).
+ * Phase 3: 4-signal scoring with hotspot.
+ * Phase 4: 5-signal scoring with observation relevance.
+ *
+ * Default weights: 0.55 vector + 0.20 graph + 0.05 recency + 0.10 hotspot + 0.10 observation
+ * All new weights are backward-compatible (default to 0).
+ */
+
+const VECTOR_WEIGHT = 0.55;
+const GRAPH_WEIGHT = 0.20;
+const RECENCY_WEIGHT = 0.05;
+const HOTSPOT_WEIGHT = 0.10;
+const OBSERVATION_WEIGHT = 0.10;
+
+// Recency boost: files indexed within the last hour get a small boost
+const RECENCY_WINDOW_MS = 60 * 60 * 1000; // 1 hour
+
+export interface ScoredResult {
+  filePath: string;
+  repoId: string;
+  vectorScore: number;
+  graphScore: number;
+  recencyScore: number;
+  hotspotScore: number;
+  observationScore: number;
+  finalScore: number;
+  entityName: string | null;
+  entityType: string;
+  startLine: number;
+  endLine: number;
+  matchReason: string;
+  indexedAt?: number;
+}
+
+/**
+ * Compute final ranking score combining up to 5 signals:
+ *   vector, graph, recency, hotspot, observation.
+ *
+ * Backward-compatible: hotspotScore and observationScore default to 0,
+ * with their weights redistributed to maintain consistent scoring
+ * when callers haven't been updated to supply them.
+ */
+export function computeScore(
+  vectorScore: number,
+  graphScore: number,
+  indexedAt: number,
+  vectorWeight = VECTOR_WEIGHT,
+  graphWeight = GRAPH_WEIGHT,
+  recencyWeight = RECENCY_WEIGHT,
+  hotspotScore: number = 0,
+  hotspotWeight: number = HOTSPOT_WEIGHT,
+  observationScore: number = 0,
+  observationWeight: number = OBSERVATION_WEIGHT
+): { finalScore: number; recencyScore: number } {
+  const now = Date.now();
+  const age = now - indexedAt;
+  const recencyScore = age < RECENCY_WINDOW_MS ? 1.0 - (age / RECENCY_WINDOW_MS) : 0;
+
+  const finalScore =
+    vectorWeight * vectorScore +
+    graphWeight * graphScore +
+    recencyWeight * recencyScore +
+    hotspotWeight * hotspotScore +
+    observationWeight * observationScore;
+
+  return { finalScore, recencyScore };
+}
+
+/**
+ * Deduplicate results by file path, keeping the highest-scored entry per file.
+ */
+export function deduplicateByFile(results: ScoredResult[]): ScoredResult[] {
+  const byFile = new Map<string, ScoredResult>();
+
+  for (const result of results) {
+    const existing = byFile.get(result.filePath);
+    if (!existing || result.finalScore > existing.finalScore) {
+      byFile.set(result.filePath, result);
+    }
+  }
+
+  return Array.from(byFile.values()).sort((a, b) => b.finalScore - a.finalScore);
+}
+
+/**
+ * Split results into pivot files (full content) and skeletons (summaries) using actual token sizes.
+ * If fileSizes is provided (map of filePath -> estimated tokens), uses real sizes.
+ * Otherwise falls back to avgFileTokens estimate.
+ */
+export function splitPivotsAndSkeletons(
+  results: ScoredResult[],
+  tokenBudget: number,
+  fileSizes?: Map<string, number>,
+  avgFileTokens: number = 500
+): { pivots: ScoredResult[]; skeletons: ScoredResult[] } {
+  const pivots: ScoredResult[] = [];
+  const skeletons: ScoredResult[] = [];
+  let usedTokens = 0;
+
+  for (const result of results) {
+    const fileTokens = fileSizes?.get(result.filePath) ?? avgFileTokens;
+    if (usedTokens + fileTokens <= tokenBudget) {
+      pivots.push(result);
+      usedTokens += fileTokens;
+    } else {
+      skeletons.push(result);
+    }
+  }
+
+  return { pivots, skeletons };
+}
+
+/**
+ * Generate a human-readable match reason string.
+ * Phase 3: includes hotspot info.
+ * Phase 4: includes observation info.
+ */
+export function buildMatchReason(
+  vectorScore: number,
+  graphScore: number,
+  entityName: string | null,
+  entityType: string,
+  hotspotScore: number = 0,
+  observationScore: number = 0
+): string {
+  const parts: string[] = [];
+
+  if (vectorScore > 0.8) {
+    parts.push('high semantic similarity');
+  } else if (vectorScore > 0.5) {
+    parts.push('semantic match');
+  } else {
+    parts.push('partial semantic match');
+  }
+
+  if (graphScore > 0.5) {
+    parts.push('high graph centrality');
+  } else if (graphScore > 0) {
+    parts.push('graph neighbor');
+  }
+
+  if (hotspotScore > 0.7) {
+    parts.push('change hotspot');
+  } else if (hotspotScore > 0.3) {
+    parts.push('frequently changed');
+  }
+
+  if (observationScore > 0.5) {
+    parts.push('has relevant observations');
+  } else if (observationScore > 0) {
+    parts.push('has observations');
+  }
+
+  if (entityName) {
+    parts.push(`entity: ${entityName} (${entityType})`);
+  }
+
+  return parts.join(', ');
+}