codetrap 0.1.2 → 0.1.4

package/src/lib/search-policy.ts

@@ -0,0 +1,365 @@
+import type { RankingSignal, Trap, TrapSearchResult } from "../domain/trap";
+import type { TrapStatus } from "./constants";
+import { EmbeddingProviderUnavailableError } from "./embedder";
+import { parseTrapPathGlobs, parseTrapTags } from "./trap-json-fields";
+import {
+  hasSpecificPathMatch,
+  trapMatchesApplicability,
+  type ApplicabilityFilter,
+} from "./trap-scope-match";
+
+export interface SearchPolicyOptions extends ApplicabilityFilter {
+  category?: string;
+  scope?: string;
+  status?: TrapStatus | "all";
+  limit?: number;
+  rerank?: boolean;
+  includeRankingSignals?: boolean;
+}
+
+export type SearchStorageFilter = {
+  category?: string;
+  scope?: string;
+  status?: TrapStatus | "all";
+  module?: string;
+  owner?: string;
+  limit?: number;
+};
+
+export type SemanticStorageFilter = {
+  category?: string;
+  scope?: string;
+  status?: TrapStatus | "all";
+};
+
+export type SearchRetrievalSource = "fts" | "semantic";
+
+export interface SearchRetrievalPlan {
+  resultLimit: number;
+  candidateLimit: number;
+  ftsStorageFilter: SearchStorageFilter & { limit: number };
+  semanticStorageFilter: SemanticStorageFilter;
+  applicabilityFilter: ApplicabilityFilter;
+}
+
+export interface RankingConfig {
+  rrfK: number;
+  semanticMinScore: number;
+  lengthNormAnchor: number;
+  maxBoost: number;
+  titleTokenBoost: number;
+  tagTokenBoost: number;
+  identifierBoost: number;
+  severityBoost: Record<string, number>;
+  pathMatchBoost: number;
+  moduleMatchBoost: number;
+  ownerMatchBoost: number;
+}
+
+export const DEFAULT_RANKING_CONFIG: RankingConfig = {
+  rrfK: 60,
+  semanticMinScore: 0.3,
+  lengthNormAnchor: 500,
+  maxBoost: 0.45,
+  titleTokenBoost: 0.16,
+  tagTokenBoost: 0.2,
+  identifierBoost: 0.18,
+  severityBoost: {
+    warning: 0,
+    error: 0.04,
+    critical: 0.07,
+  },
+  pathMatchBoost: 0.12,
+  moduleMatchBoost: 0.08,
+  ownerMatchBoost: 0.04,
+};
+
+export class TrapSearchPolicy {
+  constructor(private readonly ranking: RankingConfig = DEFAULT_RANKING_CONFIG) {}
+
+  plan(opts: SearchPolicyOptions, defaultLimit: number): SearchRetrievalPlan {
+    const resultLimit = opts.limit ?? defaultLimit;
+    const candidateLimit = this.candidateLimit(opts, resultLimit);
+    return {
+      resultLimit,
+      candidateLimit,
+      ftsStorageFilter: {
+        category: opts.category,
+        scope: opts.scope,
+        status: opts.status,
+        module: opts.module,
+        owner: opts.owner,
+        limit: candidateLimit,
+      },
+      semanticStorageFilter: {
+        category: opts.category,
+        scope: opts.scope,
+        status: opts.status,
+      },
+      applicabilityFilter: {
+        path: opts.path,
+        module: opts.module,
+        owner: opts.owner,
+      },
+    };
+  }
+
+  candidateLimit(opts: SearchPolicyOptions, resultLimit: number): number {
+    return shouldOverfetch(opts) ? Math.max(resultLimit * 5, 50) : resultLimit;
+  }
+
+  semanticMinScore(): number {
+    return this.ranking.semanticMinScore;
+  }
+
+  filterResults(results: TrapSearchResult[], filter: ApplicabilityFilter): TrapSearchResult[] {
+    return results.filter((result) => trapMatchesApplicability(result.trap, filter));
+  }
+
+  matchesTrap(trap: Trap, filter: ApplicabilityFilter): boolean {
+    return trapMatchesApplicability(trap, filter);
+  }
+
+  filterTraps(traps: Trap[], filter: ApplicabilityFilter): Trap[] {
+    return traps.filter((trap) => this.matchesTrap(trap, filter));
+  }
+
+  prepareRetrievedResults(
+    results: TrapSearchResult[],
+    source: SearchRetrievalSource,
+    plan: SearchRetrievalPlan
+  ): TrapSearchResult[] {
+    const applicable = this.filterResults(results, plan.applicabilityFilter);
+    if (source === "semantic") {
+      return applicable
+        .filter((result) => (result.score ?? 0) >= this.semanticMinScore())
+        .sort((a, b) => (b.score ?? 0) - (a.score ?? 0))
+        .slice(0, plan.candidateLimit);
+    }
+    return applicable.slice(0, plan.candidateLimit);
+  }
+
+  finalizeResults(
+    results: TrapSearchResult[],
+    query: string,
+    opts: SearchPolicyOptions,
+    plan: SearchRetrievalPlan
+  ): TrapSearchResult[] {
+    return this.rankResults(results, query, opts, plan.resultLimit);
+  }
+
+  rankResults(
+    results: TrapSearchResult[],
+    query: string,
+    opts: SearchPolicyOptions,
+    limit: number
+  ): TrapSearchResult[] {
+    return applyReranking(results, query, opts, this.ranking).slice(0, limit);
+  }
+
+  fuse(
+    ftsResults: TrapSearchResult[],
+    semanticResults: TrapSearchResult[],
+    query: string,
+    opts: SearchPolicyOptions,
+    limit: number
+  ): TrapSearchResult[] {
+    const byId = new Map<number, TrapSearchResult & { score: number; sources: ("fts" | "semantic")[] }>();
+
+    addRankedResults(byId, ftsResults, "fts", this.ranking);
+    addRankedResults(byId, semanticResults, "semantic", this.ranking);
+
+    const fused = [...byId.values()]
+      .map((result) => ({
+        ...result,
+        score: applyLengthNormalization(result.score, result.trap, this.ranking),
+        rank: applyLengthNormalization(result.score, result.trap, this.ranking),
+      }))
+      .sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
+
+    return this.rankResults(fused, query, opts, limit);
+  }
+
+  fuseAndFinalize(
+    ftsResults: TrapSearchResult[],
+    semanticResults: TrapSearchResult[],
+    query: string,
+    opts: SearchPolicyOptions,
+    plan: SearchRetrievalPlan
+  ): TrapSearchResult[] {
+    return this.fuse(ftsResults, semanticResults, query, opts, plan.resultLimit);
+  }
+
+  withDiagnostics(
+    results: TrapSearchResult[],
+    diagnostic: { code: string; message: string }
+  ): TrapSearchResult[] {
+    return results.map((result) => ({
+      ...result,
+      diagnostics: [...(result.diagnostics ?? []), diagnostic],
+    }));
+  }
+
+  semanticDiagnostic(error: unknown): { code: string; message: string } {
+    if (error instanceof EmbeddingProviderUnavailableError) {
+      return {
+        code: "semantic_unavailable",
+        message: error.message,
+      };
+    }
+    return {
+      code: "semantic_failed",
+      message: error instanceof Error ? error.message : "Semantic search failed; hybrid search returned FTS results.",
+    };
+  }
+}
+
+function addRankedResults(
+  byId: Map<number, TrapSearchResult & { score: number; sources: ("fts" | "semantic")[] }>,
+  results: TrapSearchResult[],
+  source: "fts" | "semantic",
+  ranking: RankingConfig
+): void {
+  results.forEach((result, index) => {
+    const score = 1 / (ranking.rrfK + index + 1);
+    const existing = byId.get(result.trap.id);
+    if (existing) {
+      existing.score += score;
+      if (!existing.sources.includes(source)) existing.sources.push(source);
+      return;
+    }
+    byId.set(result.trap.id, {
+      ...result,
+      score,
+      sources: [source],
+    });
+  });
+}
+
+function applyLengthNormalization(score: number, trap: Trap, ranking: RankingConfig): number {
+  const length = `${trap.context}\n${trap.mistake}\n${trap.fix}`.length;
+  if (length <= ranking.lengthNormAnchor) return score;
+  return score * Math.sqrt(ranking.lengthNormAnchor / length);
+}
+
+function applyReranking(
+  results: TrapSearchResult[],
+  query: string,
+  opts: SearchPolicyOptions,
+  ranking: RankingConfig
+): TrapSearchResult[] {
+  if (opts.rerank === false) return stripRankingSignals(results, opts);
+
+  const queryInfo = analyzeQuery(query);
+  return results
+    .map((result) => {
+      const signals = rankingSignals(result.trap, queryInfo, opts, ranking);
+      const boost = Math.min(
+        ranking.maxBoost,
+        signals.reduce((sum, signal) => sum + signal.weight, 0)
+      );
+      const score = (result.score ?? result.rank ?? 0) * (1 + boost);
+      return {
+        ...result,
+        score,
+        rank: score,
+        ...(opts.includeRankingSignals && signals.length > 0 ? { ranking_signals: signals } : {}),
+      };
+    })
+    .sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
+}
+
+function stripRankingSignals(results: TrapSearchResult[], opts: SearchPolicyOptions): TrapSearchResult[] {
+  if (opts.includeRankingSignals) return results.map((result) => ({ ...result, ranking_signals: [] }));
+  return results.map(({ ranking_signals: _rankingSignals, ...result }) => result);
+}
+
+function rankingSignals(
+  trap: Trap,
+  query: QueryInfo,
+  filter: ApplicabilityFilter,
+  ranking: RankingConfig
+): RankingSignal[] {
+  const signals: RankingSignal[] = [];
+  const titleTokens = tokenize(trap.title);
+  const tags = parseTrapTags(trap.tags).map((tag) => tag.toLowerCase());
+  const allFieldTokens = tokenize([
+    trap.title,
+    trap.context,
+    trap.mistake,
+    trap.fix,
+    trap.before_code ?? "",
+    trap.after_code ?? "",
+    parseTrapPathGlobs(trap.path_globs).join(" "),
+    trap.module ?? "",
+    trap.owner ?? "",
+  ].join(" "));
+
+  for (const token of query.tokens) {
+    if (titleTokens.has(token)) {
+      signals.push({ code: "title_token_exact", weight: ranking.titleTokenBoost, detail: token });
+      break;
+    }
+  }
+
+  for (const token of query.tokens) {
+    if (tags.includes(token)) {
+      signals.push({ code: "tag_exact", weight: ranking.tagTokenBoost, detail: token });
+      break;
+    }
+  }
+
+  for (const token of query.identifierTokens) {
+    if (allFieldTokens.has(token)) {
+      signals.push({ code: "code_identifier_exact", weight: ranking.identifierBoost, detail: token });
+      break;
+    }
+  }
+
+  const severityBoost = ranking.severityBoost[trap.severity] ?? 0;
+  if (severityBoost > 0) {
+    signals.push({ code: "severity", weight: severityBoost, detail: trap.severity });
+  }
+
+  if (hasSpecificPathMatch(trap, filter.path)) {
+    signals.push({ code: "path_scope_match", weight: ranking.pathMatchBoost, detail: filter.path });
+  }
+  if (filter.module && trap.module === filter.module) {
+    signals.push({ code: "module_scope_match", weight: ranking.moduleMatchBoost, detail: filter.module });
+  }
+  if (filter.owner && trap.owner === filter.owner) {
+    signals.push({ code: "owner_scope_match", weight: ranking.ownerMatchBoost, detail: filter.owner });
+  }
+
+  return signals;
+}
+
+type QueryInfo = {
+  tokens: Set<string>;
+  identifierTokens: Set<string>;
+};
+
+function analyzeQuery(query: string): QueryInfo {
+  const tokens = tokenize(query);
+  return {
+    tokens,
+    identifierTokens: new Set([...tokens].filter(isIdentifierLike)),
+  };
+}
+
+function tokenize(value: string): Set<string> {
+  return new Set((value.match(/[A-Za-z0-9_.$/@:-]+/g) ?? []).map((token) => token.toLowerCase()));
+}
+
+function isIdentifierLike(token: string): boolean {
+  return (
+    /[_.$/@:-]/.test(token) ||
+    /\d/.test(token) ||
+    /[a-z][A-Z]/.test(token) ||
+    token.length >= 8
+  );
+}
+
+function shouldOverfetch(opts: SearchPolicyOptions): boolean {
+  return Boolean(opts.path || opts.module || opts.owner || opts.rerank !== false);
+}

package/src/lib/search-result-card.ts

@@ -15,13 +15,8 @@ export function toTrapActionCard(result: TrapSearchResult, scope: Scope): TrapAc
     severity: trap.severity,
     score: result.score ?? null,
     sources: result.sources ?? [],
-    next_action: {
-      details_tool: "get_trap",
-      details_args: {
-        id: trap.id,
-        scope,
-      },
-    },
+    ...(result.diagnostics ? { diagnostics: result.diagnostics } : {}),
+    ...(result.ranking_signals ? { ranking_signals: result.ranking_signals } : {}),
   };
 }
 

package/src/lib/search-service.ts

@@ -1,7 +1,6 @@
 import type { Database } from "bun:sqlite";
-import * as embeddingQueries from "../db/embedding-queries";
 import * as queries from "../db/queries";
-import type { Trap, TrapSearchResult } from "../domain/trap";
+import type { TrapSearchResult } from "../domain/trap";
 import type { SearchMode, TrapStatus } from "./constants";
 import {
   cosineSimilarity,
@@ -9,6 +8,13 @@ import {
   embeddingConfig,
   type EmbeddingProvider,
 } from "./embedder";
+import {
+  DEFAULT_RANKING_CONFIG,
+  TrapSearchPolicy,
+  type RankingConfig,
+  type SearchRetrievalPlan,
+} from "./search-policy";
+import { DatabaseEmbeddingIndex } from "./embedding-index";
 
 export interface SearchOptions {
   category?: string;
@@ -16,28 +22,27 @@ export interface SearchOptions {
   limit?: number;
   mode?: SearchMode;
   status?: TrapStatus | "all";
+  path?: string;
+  module?: string;
+  owner?: string;
+  rerank?: boolean;
+  includeRankingSignals?: boolean;
 }
 
-export interface RankingConfig {
-  rrfK: number;
-  semanticMinScore: number;
-  lengthNormAnchor: number;
-}
-
-export const DEFAULT_RANKING_CONFIG: RankingConfig = {
-  rrfK: 60,
-  semanticMinScore: 0.3,
-  lengthNormAnchor: 500,
-};
-
 const DEFAULT_LIMIT = 20;
 
 export class SearchService {
+  private readonly policy: TrapSearchPolicy;
+  private readonly embeddingIndex: DatabaseEmbeddingIndex;
+
   constructor(
     private readonly db: Database,
     private readonly embedder?: EmbeddingProvider,
-    private readonly ranking: RankingConfig = DEFAULT_RANKING_CONFIG
-  ) {}
+    ranking: RankingConfig = DEFAULT_RANKING_CONFIG
+  ) {
+    this.policy = new TrapSearchPolicy(ranking);
+    this.embeddingIndex = new DatabaseEmbeddingIndex(db);
+  }
 
   async search(query: string, opts: SearchOptions = {}): Promise<TrapSearchResult[]> {
     if (!query.trim()) return [];
@@ -56,14 +61,51 @@ export class SearchService {
   }
 
   ftsSearch(query: string, opts: SearchOptions = {}): TrapSearchResult[] {
-    return queries.searchTraps(this.db, query, opts).map((result) => ({
-      ...result,
-      sources: ["fts"],
-      score: ftsScore(result.rank),
-    }));
+    const plan = this.policy.plan(opts, DEFAULT_LIMIT);
+    return this.policy.finalizeResults(this.retrieveFtsCandidates(query, plan), query, opts, plan);
   }
 
   async semanticSearch(query: string, opts: SearchOptions = {}): Promise<TrapSearchResult[]> {
+    const plan = this.policy.plan(opts, DEFAULT_LIMIT);
+    return this.policy.finalizeResults(await this.retrieveSemanticCandidates(query, plan), query, opts, plan);
+  }
+
+  async hybridSearch(query: string, opts: SearchOptions = {}): Promise<TrapSearchResult[]> {
+    const plan = this.policy.plan(opts, DEFAULT_LIMIT);
+    const ftsCandidates = this.retrieveFtsCandidates(query, plan);
+
+    try {
+      const semanticCandidates = await this.retrieveSemanticCandidates(query, plan);
+      if (semanticCandidates.length === 0) {
+        return this.policy.withDiagnostics(this.policy.finalizeResults(ftsCandidates, query, opts, plan), {
+          code: "semantic_no_candidates",
+          message: "Hybrid search used FTS results because no fresh semantic candidates passed the score threshold.",
+        });
+      }
+      return this.policy.fuseAndFinalize(ftsCandidates, semanticCandidates, query, opts, plan);
+    } catch (error) {
+      return this.policy.withDiagnostics(
+        this.policy.finalizeResults(ftsCandidates, query, opts, plan),
+        this.policy.semanticDiagnostic(error)
+      );
+    }
+  }
+
+  private retrieveFtsCandidates(query: string, plan: SearchRetrievalPlan): TrapSearchResult[] {
+    const candidates = queries
+      .searchTraps(this.db, query, plan.ftsStorageFilter)
+      .map((result) => ({
+        ...result,
+        sources: ["fts"] as ("fts")[],
+        score: ftsScore(result.rank),
+      }));
+    return this.policy.prepareRetrievedResults(candidates, "fts", plan);
+  }
+
+  private async retrieveSemanticCandidates(
+    query: string,
+    plan: SearchRetrievalPlan
+  ): Promise<TrapSearchResult[]> {
     if (!this.embedder) {
       throw new EmbeddingProviderUnavailableError();
     }
@@ -72,13 +114,9 @@ export class SearchService {
     if (!queryEmbedding) return [];
 
     const config = embeddingConfig(this.embedder);
-    const candidates = embeddingQueries.getAllFreshEmbeddings(this.db, config, {
-      category: opts.category,
-      scope: opts.scope,
-      status: opts.status,
-    });
+    const candidates = this.embeddingIndex.freshEmbeddings(config, plan.semanticStorageFilter);
 
-    return candidates
+    const results = candidates
       .map(({ trap, embedding }) => {
         const score = cosineSimilarity(queryEmbedding, embedding);
         return {
@@ -88,102 +126,11 @@ export class SearchService {
           score,
         };
       })
-      .filter((result) => (result.score ?? 0) >= this.ranking.semanticMinScore)
-      .sort((a, b) => (b.score ?? 0) - (a.score ?? 0))
-      .slice(0, opts.limit ?? DEFAULT_LIMIT);
-  }
-
-  async hybridSearch(query: string, opts: SearchOptions = {}): Promise<TrapSearchResult[]> {
-    const limit = opts.limit ?? DEFAULT_LIMIT;
-    const ftsResults = this.ftsSearch(query, { ...opts, limit });
-
-    try {
-      const semanticResults = await this.semanticSearch(query, { ...opts, limit });
-      if (semanticResults.length === 0) {
-        return withDiagnostics(ftsResults, {
-          code: "semantic_no_candidates",
-          message: "Hybrid search used FTS results because no fresh semantic candidates passed the score threshold.",
-        });
-      }
-      return rrfFuse(ftsResults, semanticResults, limit, this.ranking);
-    } catch (error) {
-      return withDiagnostics(ftsResults, semanticDiagnostic(error));
-    }
+      .map((result) => result as TrapSearchResult);
+    return this.policy.prepareRetrievedResults(results, "semantic", plan);
   }
 }
 
-export function rrfFuse(
-  ftsResults: TrapSearchResult[],
-  semanticResults: TrapSearchResult[],
-  limit = DEFAULT_LIMIT,
-  ranking: RankingConfig = DEFAULT_RANKING_CONFIG
-): TrapSearchResult[] {
-  const byId = new Map<number, TrapSearchResult & { score: number; sources: ("fts" | "semantic")[] }>();
-
-  addRankedResults(byId, ftsResults, "fts", ranking);
-  addRankedResults(byId, semanticResults, "semantic", ranking);
-
-  return [...byId.values()]
-    .map((result) => ({
-      ...result,
-      score: applyLengthNormalization(result.score, result.trap, ranking),
-      rank: applyLengthNormalization(result.score, result.trap, ranking),
-    }))
-    .sort((a, b) => (b.score ?? 0) - (a.score ?? 0))
-    .slice(0, limit);
-}
-
-function addRankedResults(
-  byId: Map<number, TrapSearchResult & { score: number; sources: ("fts" | "semantic")[] }>,
-  results: TrapSearchResult[],
-  source: "fts" | "semantic",
-  ranking: RankingConfig
-): void {
-  results.forEach((result, index) => {
-    const score = 1 / (ranking.rrfK + index + 1);
-    const existing = byId.get(result.trap.id);
-    if (existing) {
-      existing.score += score;
-      if (!existing.sources.includes(source)) existing.sources.push(source);
-      return;
-    }
-    byId.set(result.trap.id, {
-      ...result,
-      score,
-      sources: [source],
-    });
-  });
-}
-
-function applyLengthNormalization(score: number, trap: Trap, ranking: RankingConfig): number {
-  const length = `${trap.context}\n${trap.mistake}\n${trap.fix}`.length;
-  if (length <= ranking.lengthNormAnchor) return score;
-  return score * Math.sqrt(ranking.lengthNormAnchor / length);
-}
-
 function ftsScore(rank: number): number {
   return Number.isFinite(rank) ? -rank : 0;
 }
-
-function withDiagnostics(
-  results: TrapSearchResult[],
-  diagnostic: { code: string; message: string }
-): TrapSearchResult[] {
-  return results.map((result) => ({
-    ...result,
-    diagnostics: [...(result.diagnostics ?? []), diagnostic],
-  }));
-}
-
-function semanticDiagnostic(error: unknown): { code: string; message: string } {
-  if (error instanceof EmbeddingProviderUnavailableError) {
-    return {
-      code: "semantic_unavailable",
-      message: error.message,
-    };
-  }
-  return {
-    code: "semantic_failed",
-    message: error instanceof Error ? error.message : "Semantic search failed; hybrid search returned FTS results.",
-  };
-}