forge-server 0.1.0

package/src/knowledge/search.ts

@@ -0,0 +1,565 @@
+// KnowledgeSearch — semantic search over the Qdrant knowledge collection.
+//
+// Architecture notes (ADR-2):
+// - ALL knowledge search goes through Qdrant. There is no SQLite FTS fallback.
+// - When Qdrant is unavailable the search returns [] with a logged warning —
+//   it does NOT throw. The pipeline continues; agents just receive no gotchas.
+// - Cross-repo search is opt-in (include_cross_repo defaults to false at the
+//   tool layer). Results from the caller's repo get a small score boost to
+//   surface local knowledge first without completely filtering out cross-repo
+//   items when cross-repo is enabled.
+// - stack_tags filtering uses a Qdrant "should" condition (any tag matches)
+//   rather than "must" (all tags must match), because a gotcha tagged with
+//   ["nestjs", "swc"] is equally relevant to a query from a project that uses
+//   either nestjs or swc.
+//
+// Qdrant filter shape reference:
+//   must  → all conditions must be satisfied (AND)
+//   should→ at least one condition must be satisfied (OR)
+//   Values indexed as 'keyword' support match: { value: "..." } for exact match
+//   Values indexed as 'float' support range: { gte: 0.3 }
+
+import { logger } from '../util/logger.js';
+import { embedText } from '../ingestion/embedder.js';
+import type { QdrantVectorStore } from '../storage/qdrant-store.js';
+import type { KnowledgeCategory, KnowledgeItem, KnowledgePayload } from '../util/types.js';
+
+// ---------------------------------------------------------------------------
+// Public types (spec-matching interface)
+// ---------------------------------------------------------------------------
+
+/**
+ * Result shape returned from KnowledgeSearch.search().
+ * Uses snake_case field names to match the MCP tool response contract.
+ */
+export interface KnowledgeSearchResult {
+  id: string;
+  category: string;
+  title: string;
+  content: string;
+  confidence: number;
+  stack_tags: string[];
+  source_repo: string;
+  relevance_score: number;
+}
+
+// Legacy camelCase interface retained for backward compat with earlier callers.
+// New callers should use KnowledgeSearchResult (snake_case).
+export interface KnowledgeSearchOptions {
+  repoId?: string;
+  category?: KnowledgeCategory;
+  stackTags?: string[];
+  minConfidence?: number;
+  limit?: number;
+  scoreThreshold?: number;
+}
+
+// ---------------------------------------------------------------------------
+// Score boost applied to results from the same repo as the caller.
+// Set conservatively: local items should surface first but not crowd out
+// highly relevant cross-repo items.
+// ---------------------------------------------------------------------------
+const SAME_REPO_SCORE_BOOST = 0.05;
+
+// ---------------------------------------------------------------------------
+// KnowledgeSearch
+// ---------------------------------------------------------------------------
+
+export class KnowledgeSearch {
+  constructor(private readonly vectorStore: QdrantVectorStore) {}
+
+  // ---------------------------------------------------------------------------
+  // Primary search method (spec B4 signature)
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Semantic search across knowledge items stored in the Qdrant knowledge
+   * collection.
+   *
+   * Options:
+   *   category          — filter to a single KnowledgeCategory
+   *   stack_tags        — filter to items that share at least one tag (OR match)
+   *   repo_id           — used for same-repo scoring boost; also hard-filters
+   *                       when include_cross_repo is false
+   *   limit             — max results to return (default 10)
+   *   include_cross_repo— include items from other repos (default true)
+   *   min_confidence    — minimum confidence score (default 0.3)
+   *
+   * Returns results sorted descending by relevance_score.
+   * Returns [] when Qdrant is unavailable — never throws.
+   */
+  async search(
+    query: string,
+    options?: {
+      category?: KnowledgeCategory;
+      stack_tags?: string[];
+      repo_id?: string;
+      limit?: number;
+      include_cross_repo?: boolean;
+      min_confidence?: number;
+    },
+  ): Promise<KnowledgeSearchResult[]> {
+    const {
+      category,
+      stack_tags,
+      repo_id,
+      limit = 10,
+      include_cross_repo = true,
+      min_confidence = 0.3,
+    } = options ?? {};
+
+    // Check Qdrant availability first — avoid embedding model init overhead
+    // when Qdrant is down.
+    const healthy = await this.vectorStore.isHealthy();
+    if (!healthy) {
+      logger.warn('KnowledgeSearch.search: Qdrant unavailable, returning empty results', {
+        query: query.slice(0, 80),
+      });
+      return [];
+    }
+
+    let queryVector: number[];
+    try {
+      queryVector = await embedText(query);
+    } catch (err) {
+      logger.warn('KnowledgeSearch.search: embedding failed, returning empty results', {
+        error: String(err),
+      });
+      return [];
+    }
+
+    // Build Qdrant filter.
+    // When include_cross_repo is false, hard-filter to repo_id only.
+    // When include_cross_repo is true, only include cross-repo items that have
+    // sharing = 'team' or 'public' (private repos never leak into other repos'
+    // results). Items from the caller's own repo are always included regardless
+    // of sharing setting.
+    const qdrantFilter = this._buildQdrantFilter({
+      category,
+      stack_tags,
+      repo_id: include_cross_repo ? undefined : repo_id,
+      caller_repo_id: include_cross_repo ? repo_id : undefined,
+      min_confidence,
+    });
+
+    let rawResults: Array<{ id: string; score: number; payload: Record<string, unknown> }>;
+    try {
+      // Fetch more than requested to allow re-ranking after same-repo boost.
+      const fetchLimit = Math.min(limit * 3, 100);
+
+      const results = await this.vectorStore.searchKnowledge(queryVector, {
+        limit: fetchLimit,
+        // We handle min_confidence via the filter's range condition on the
+        // confidence payload field rather than scoreThreshold (which applies
+        // to the cosine similarity score, a different axis).
+        scoreThreshold: 0.0,
+        filter: qdrantFilter as Record<string, unknown>,
+      });
+
+      rawResults = results.map((r) => ({
+        id: r.id,
+        score: r.score,
+        payload: r.payload as unknown as Record<string, unknown>,
+      }));
+    } catch (err) {
+      logger.warn('KnowledgeSearch.search: Qdrant search failed, returning empty results', {
+        error: String(err),
+      });
+      return [];
+    }
+
+    // Apply same-repo boost, re-rank, trim to limit.
+    const boosted = rawResults.map((r) => {
+      const payload = r.payload as Partial<KnowledgePayload>;
+      const boost = repo_id && payload.repo_id === repo_id ? SAME_REPO_SCORE_BOOST : 0;
+      return { ...r, boostedScore: r.score + boost };
+    });
+
+    boosted.sort((a, b) => b.boostedScore - a.boostedScore);
+
+    const output: KnowledgeSearchResult[] = [];
+    for (const r of boosted) {
+      if (output.length >= limit) break;
+
+      const p = r.payload as Partial<KnowledgePayload>;
+      if (!p.id || !p.title || !p.content) continue;
+
+      output.push({
+        id: p.id,
+        category: p.category ?? 'pattern',
+        title: p.title,
+        content: p.content,
+        confidence: p.confidence ?? 0,
+        stack_tags: Array.isArray(p.stack_tags) ? p.stack_tags : [],
+        source_repo: p.repo_id ?? '',
+        relevance_score: r.boostedScore,
+      });
+    }
+
+    logger.debug('KnowledgeSearch.search: completed', {
+      query: query.slice(0, 80),
+      resultCount: output.length,
+      category,
+      hadSameRepoBoost: !!repo_id,
+    });
+
+    return output;
+  }
+
+  // ---------------------------------------------------------------------------
+  // Confidence boost
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Increment access_count and update accessed_at for a knowledge item.
+   * Optionally bumps the confidence score by boostAmount.
+   *
+   * Called when an agent actively consumes a knowledge item — access signals
+   * genuine utility and nudges confidence upward.
+   * Silent on Qdrant failure (non-fatal side effect).
+   */
+  async boostConfidence(knowledgeId: string, boostAmount = 0.01): Promise<void> {
+    const healthy = await this.vectorStore.isHealthy();
+    if (!healthy) {
+      logger.debug('KnowledgeSearch.boostConfidence: Qdrant unavailable, skip', { knowledgeId });
+      return;
+    }
+
+    try {
+      // Use setKnowledgePayload if the store exposes it; otherwise fall back
+      // to a no-op so callers never see an exception from a best-effort call.
+      const store = this.vectorStore as QdrantVectorStore & {
+        getKnowledge?: (id: string) => Promise<Record<string, unknown> | null>;
+        setKnowledgePayload?: (id: string, patch: Record<string, unknown>) => Promise<void>;
+      };
+
+      let currentConfidence = 0;
+      let currentAccessCount = 0;
+
+      if (typeof store.getKnowledge === 'function') {
+        const existing = await store.getKnowledge(knowledgeId);
+        if (existing) {
+          currentConfidence = (existing['confidence'] as number | undefined) ?? 0;
+          currentAccessCount = (existing['access_count'] as number | undefined) ?? 0;
+        }
+      }
+
+      const patch: Record<string, unknown> = {
+        accessed_at: Date.now(),
+        access_count: currentAccessCount + 1,
+        confidence: Math.min(1.0, currentConfidence + boostAmount),
+      };
+
+      if (typeof store.setKnowledgePayload === 'function') {
+        await store.setKnowledgePayload(knowledgeId, patch);
+        logger.debug('KnowledgeSearch.boostConfidence: boosted', {
+          knowledgeId,
+          newAccessCount: patch['access_count'],
+          newConfidence: patch['confidence'],
+        });
+      } else {
+        logger.debug(
+          'KnowledgeSearch.boostConfidence: setKnowledgePayload not available, skipping',
+          { knowledgeId },
+        );
+      }
+    } catch (err) {
+      logger.warn('KnowledgeSearch.boostConfidence: failed (non-fatal)', {
+        knowledgeId,
+        error: String(err),
+      });
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+  // Typed accessors
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Return gotcha items optionally filtered by stack tags.
+   * Uses Qdrant filter (category=gotcha) plus optional stack_tags OR match.
+   * Results sorted by confidence descending.
+   * Gracefully returns [] on Qdrant unavailability.
+   */
+  async getGotchas(
+    stack?: string[],
+    limit = 20,
+    minConfidence = 0.3,
+  ): Promise<KnowledgeItem[]> {
+    return this._getByCategory('gotcha', stack, limit, minConfidence);
+  }
+
+  /**
+   * Return pattern items optionally filtered by domain.
+   * `domain` is treated as a stack tag (single-element array) for consistency.
+   * Results sorted by confidence descending.
+   * Gracefully returns [] on Qdrant unavailability.
+   */
+  async getPatterns(
+    domain?: string,
+    limit = 20,
+    minConfidence = 0.3,
+  ): Promise<KnowledgeItem[]> {
+    const tags = domain ? [domain] : undefined;
+    return this._getByCategory('pattern', tags, limit, minConfidence);
+  }
+
+  // ---------------------------------------------------------------------------
+  // Backward-compat helpers (camelCase signature from the prior implementation)
+  // ---------------------------------------------------------------------------
+
+  /** Convenience: search with category=gotcha. Backward-compat camelCase API. */
+  async searchGotchas(
+    query: string,
+    opts?: Omit<KnowledgeSearchOptions, 'category'>,
+  ): Promise<KnowledgeSearchResult[]> {
+    return this.search(query, {
+      category: 'gotcha',
+      stack_tags: opts?.stackTags,
+      repo_id: opts?.repoId,
+      limit: opts?.limit,
+      min_confidence: opts?.minConfidence,
+    });
+  }
+
+  /** Convenience: search with category=decision. Backward-compat camelCase API. */
+  async searchDecisions(
+    query: string,
+    opts?: Omit<KnowledgeSearchOptions, 'category'>,
+  ): Promise<KnowledgeSearchResult[]> {
+    return this.search(query, {
+      category: 'decision',
+      stack_tags: opts?.stackTags,
+      repo_id: opts?.repoId,
+      limit: opts?.limit,
+      min_confidence: opts?.minConfidence,
+    });
+  }
+
+  /** Convenience: search with category=pattern. Backward-compat camelCase API. */
+  async searchPatterns(
+    query: string,
+    opts?: Omit<KnowledgeSearchOptions, 'category'>,
+  ): Promise<KnowledgeSearchResult[]> {
+    return this.search(query, {
+      category: 'pattern',
+      stack_tags: opts?.stackTags,
+      repo_id: opts?.repoId,
+      limit: opts?.limit,
+      min_confidence: opts?.minConfidence,
+    });
+  }
+
+  /** Convenience: search with category=convention. Backward-compat camelCase API. */
+  async searchConventions(
+    query: string,
+    opts?: Omit<KnowledgeSearchOptions, 'category'>,
+  ): Promise<KnowledgeSearchResult[]> {
+    return this.search(query, {
+      category: 'convention',
+      stack_tags: opts?.stackTags,
+      repo_id: opts?.repoId,
+      limit: opts?.limit,
+      min_confidence: opts?.minConfidence,
+    });
+  }
+
+  /** True when the vector store is configured and healthy. */
+  async isAvailable(): Promise<boolean> {
+    return this.vectorStore.isHealthy();
+  }
+
+  // ---------------------------------------------------------------------------
+  // Private helpers
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Retrieve knowledge items by category using Qdrant filter scroll (no
+   * semantic query). Falls back to semantic search when filterKnowledge is
+   * not yet available on the store.
+   */
+  private async _getByCategory(
+    category: KnowledgeCategory,
+    stack?: string[],
+    limit = 20,
+    minConfidence = 0.3,
+  ): Promise<KnowledgeItem[]> {
+    const healthy = await this.vectorStore.isHealthy();
+    if (!healthy) {
+      logger.warn(`KnowledgeSearch._getByCategory: Qdrant unavailable`, { category });
+      return [];
+    }
+
+    try {
+      const qdrantFilter = this._buildQdrantFilter({
+        category,
+        stack_tags: stack,
+        min_confidence: minConfidence,
+      });
+
+      // Use filterKnowledge (scroll-based, no semantic ranking) if available.
+      const store = this.vectorStore as QdrantVectorStore & {
+        filterKnowledge?: (
+          filter: Record<string, unknown>,
+          limit: number,
+        ) => Promise<Array<{ id: string; payload: Record<string, unknown> }>>;
+      };
+
+      if (typeof store.filterKnowledge === 'function') {
+        const rawItems = await store.filterKnowledge(
+          qdrantFilter as Record<string, unknown>,
+          limit,
+        );
+        return rawItems
+          .map((r) => this._payloadToKnowledgeItem(r.payload))
+          .filter((item): item is KnowledgeItem => item !== null)
+          .sort((a, b) => b.confidence - a.confidence);
+      }
+
+      // Fallback: semantic search using the category name as query.
+      // Less accurate but functional until filterKnowledge lands.
+      logger.debug(
+        `KnowledgeSearch._getByCategory: filterKnowledge unavailable, ` +
+        `falling back to semantic search`,
+        { category },
+      );
+      const results = await this.search(category, {
+        category: category as KnowledgeCategory,
+        stack_tags: stack,
+        limit,
+        min_confidence: minConfidence,
+      });
+      return results.map((r) => this._resultToKnowledgeItem(r));
+    } catch (err) {
+      logger.warn(`KnowledgeSearch._getByCategory: failed`, {
+        category,
+        error: String(err),
+      });
+      return [];
+    }
+  }
+
+  /**
+   * Build the Qdrant filter object from search option components.
+   *
+   * We construct the full Qdrant filter shape directly here (not relying on
+   * QdrantVectorStore.buildFilter) because we need both "must" (AND) and
+   * "should" (OR) conditions which the simple flat-map builder cannot express.
+   *
+   * Qdrant filter reference:
+   *   { must: [...], should: [...], must_not: [...] }
+   *   Each condition: { key: string, match: { value } } or { key, range: { gte } }
+   */
+  private _buildQdrantFilter(opts: {
+    category?: KnowledgeCategory;
+    stack_tags?: string[];
+    repo_id?: string;
+    caller_repo_id?: string;
+    min_confidence?: number;
+  }): Record<string, unknown> | undefined {
+    const must: unknown[] = [];
+    const should: unknown[] = [];
+
+    if (opts.category) {
+      must.push({ key: 'category', match: { value: opts.category } });
+    }
+
+    // Hard-filter to a single repo (used when include_cross_repo is false).
+    if (opts.repo_id) {
+      must.push({ key: 'repo_id', match: { value: opts.repo_id } });
+    }
+
+    // Cross-repo sharing enforcement: when searching across repos, only include
+    // items that are either (a) from the caller's own repo, or (b) have sharing
+    // set to 'team' or 'public'. Private repos never leak into other repos.
+    //
+    // Qdrant filter logic:
+    //   must: [{ should: [
+    //     { repo_id == caller_repo_id },   ← own repo: always included
+    //     { sharing == 'team' },            ← team-shared repos
+    //     { sharing == 'public' },          ← public repos
+    //   ]}]
+    if (opts.caller_repo_id && !opts.repo_id) {
+      must.push({
+        should: [
+          { key: 'repo_id', match: { value: opts.caller_repo_id } },
+          { key: 'sharing', match: { value: 'team' } },
+          { key: 'sharing', match: { value: 'public' } },
+        ],
+      });
+    } else if (!opts.repo_id && !opts.caller_repo_id) {
+      // No caller repo context — only allow team/public items to prevent
+      // accidental exposure of private repo knowledge.
+      must.push({
+        should: [
+          { key: 'sharing', match: { value: 'team' } },
+          { key: 'sharing', match: { value: 'public' } },
+        ],
+      });
+    }
+
+    if (opts.min_confidence !== undefined && opts.min_confidence > 0) {
+      must.push({ key: 'confidence', range: { gte: opts.min_confidence } });
+    }
+
+    // stack_tags: require at least one matching tag (OR semantics).
+    if (opts.stack_tags && opts.stack_tags.length > 0) {
+      for (const tag of opts.stack_tags) {
+        should.push({ key: 'stack_tags', match: { value: tag } });
+      }
+    }
+
+    if (must.length === 0 && should.length === 0) {
+      return undefined;
+    }
+
+    const filter: Record<string, unknown> = {};
+    if (must.length > 0) filter['must'] = must;
+    if (should.length > 0) filter['should'] = should;
+
+    return filter;
+  }
+
+  /**
+   * Convert a KnowledgePayload from Qdrant into a KnowledgeItem.
+   * Returns null if required fields are missing.
+   */
+  private _payloadToKnowledgeItem(
+    payload: Record<string, unknown>,
+  ): KnowledgeItem | null {
+    const p = payload as Partial<KnowledgePayload>;
+    if (!p.id || !p.title || !p.content) return null;
+
+    return {
+      id: p.id,
+      title: p.title,
+      content: p.content,
+      stack_tags: Array.isArray(p.stack_tags) ? p.stack_tags : [],
+      confidence: p.confidence ?? 0,
+      source: (p.source as KnowledgeItem['source']) ?? 'manual',
+      source_phase: (p.source_phase as KnowledgeItem['source_phase']) ?? null,
+      source_agent: p.source_agent ?? null,
+      created_at: p.created_at ?? Date.now(),
+      updated_at: p.updated_at ?? Date.now(),
+    };
+  }
+
+  /**
+   * Convert a KnowledgeSearchResult back to a KnowledgeItem shape.
+   * Used as a fallback when filterKnowledge is not available on the store.
+   */
+  private _resultToKnowledgeItem(result: KnowledgeSearchResult): KnowledgeItem {
+    return {
+      id: result.id,
+      title: result.title,
+      content: result.content,
+      stack_tags: result.stack_tags,
+      confidence: result.confidence,
+      source: 'manual',
+      source_phase: null,
+      source_agent: null,
+      created_at: Date.now(),
+      updated_at: Date.now(),
+    };
+  }
+}