codetrap 0.1.0

package/src/lib/embedder.ts

@@ -0,0 +1,133 @@
+import type { Trap } from "../domain/trap";
+import { PASSAGE_VERSION } from "./trap-search-document";
+
+export type EmbeddingTask = "retrieval.query" | "retrieval.passage";
+
+export interface EmbeddingProvider {
+  provider: string;
+  model: string;
+  dimensions: number;
+  embed(texts: string[], task: EmbeddingTask): Promise<Float32Array[]>;
+}
+
+export interface EmbeddingConfig {
+  provider: string;
+  model: string;
+  dimensions: number;
+  passageVersion: number;
+}
+
+export interface StoredEmbedding {
+  trap_id: number;
+  provider: string;
+  model: string;
+  dimensions: number;
+  passage_version: number;
+  passage_hash: string;
+  embedding: Float32Array;
+  updated_at?: string;
+}
+
+export interface FreshEmbedding {
+  trap: Trap;
+  embedding: Float32Array;
+}
+
+export class EmbeddingProviderUnavailableError extends Error {
+  constructor(message = "Embedding provider is unavailable. Set JINA_API_KEY or use --mode fts.") {
+    super(message);
+    this.name = "EmbeddingProviderUnavailableError";
+  }
+}
+
+export class JinaEmbedder implements EmbeddingProvider {
+  readonly provider = "jina";
+  readonly model = "jina-embeddings-v5-text-small";
+  readonly dimensions = 1024;
+
+  constructor(
+    private readonly apiKey: string,
+    private readonly baseURL = "https://api.jina.ai/v1"
+  ) {}
+
+  async embed(texts: string[], task: EmbeddingTask): Promise<Float32Array[]> {
+    if (texts.length === 0) return [];
+
+    const response = await fetch(`${this.baseURL}/embeddings`, {
+      method: "POST",
+      headers: {
+        "Authorization": `Bearer ${this.apiKey}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        model: this.model,
+        input: texts,
+        task,
+      }),
+    });
+
+    if (!response.ok) {
+      const body = await response.text();
+      throw new Error(`Jina embeddings request failed (${response.status}): ${body}`);
+    }
+
+    const payload = (await response.json()) as {
+      data?: { embedding?: number[]; index?: number }[];
+    };
+    const rows = payload.data ?? [];
+    if (rows.length !== texts.length) {
+      throw new Error(`Jina embeddings returned ${rows.length} vectors for ${texts.length} inputs.`);
+    }
+
+    return rows
+      .slice()
+      .sort((a, b) => (a.index ?? 0) - (b.index ?? 0))
+      .map((row) => {
+        if (!Array.isArray(row.embedding)) {
+          throw new Error("Jina embeddings response is missing an embedding vector.");
+        }
+        return Float32Array.from(row.embedding);
+      });
+  }
+}
+
+export function createDefaultEmbeddingProvider(): EmbeddingProvider | undefined {
+  const apiKey = process.env.JINA_API_KEY;
+  return apiKey ? new JinaEmbedder(apiKey) : undefined;
+}
+
+export function embeddingConfig(provider: EmbeddingProvider): EmbeddingConfig {
+  return {
+    provider: provider.provider,
+    model: provider.model,
+    dimensions: provider.dimensions,
+    passageVersion: PASSAGE_VERSION,
+  };
+}
+
+export function encodeEmbedding(embedding: Float32Array): Buffer {
+  return Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
+}
+
+export function decodeEmbedding(blob: Uint8Array | ArrayBuffer): Float32Array {
+  const bytes = blob instanceof Uint8Array ? blob : new Uint8Array(blob);
+  const copy = new Uint8Array(bytes.byteLength);
+  copy.set(bytes);
+  return new Float32Array(copy.buffer);
+}
+
+export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
+  if (a.length !== b.length || a.length === 0) return 0;
+
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+
+  if (normA === 0 || normB === 0) return 0;
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}

package/src/lib/embedding-job.ts

@@ -0,0 +1,68 @@
+import type { Trap } from "../domain/trap";
+import {
+  embeddingConfig,
+  type EmbeddingConfig,
+  type EmbeddingProvider,
+  type StoredEmbedding,
+} from "./embedder";
+import { buildTrapPassage, hashTrapPassage } from "./trap-search-document";
+
+export interface EmbeddingJobOptions {
+  scope?: string;
+  category?: string;
+  limit?: number;
+  force?: boolean;
+  batchSize?: number;
+}
+
+export interface EmbeddingJobResult {
+  generated: number;
+  skipped: number;
+  batches: number;
+}
+
+export interface EmbeddingJobAdapter {
+  countEmbeddable(opts: { scope?: string; category?: string }): number;
+  trapsNeedingEmbeddings(config: EmbeddingConfig, opts: EmbeddingJobOptions): Trap[];
+  saveEmbedding(record: StoredEmbedding): void;
+}
+
+const DEFAULT_BATCH_SIZE = 16;
+
+export async function runEmbeddingJob(
+  adapter: EmbeddingJobAdapter,
+  provider: EmbeddingProvider,
+  opts: EmbeddingJobOptions = {}
+): Promise<EmbeddingJobResult> {
+  const config = embeddingConfig(provider);
+  const total = adapter.countEmbeddable({ scope: opts.scope, category: opts.category });
+  const traps = adapter.trapsNeedingEmbeddings(config, opts);
+  if (traps.length === 0) return { generated: 0, skipped: total, batches: 0 };
+
+  const batchSize = opts.batchSize ?? DEFAULT_BATCH_SIZE;
+  let batches = 0;
+
+  for (let start = 0; start < traps.length; start += batchSize) {
+    const batch = traps.slice(start, start + batchSize);
+    const passages = batch.map(buildTrapPassage);
+    const embeddings = await provider.embed(passages, "retrieval.passage");
+    if (embeddings.length !== batch.length) {
+      throw new Error(`Embedding provider returned ${embeddings.length} vectors for ${batch.length} traps.`);
+    }
+
+    for (let i = 0; i < batch.length; i++) {
+      adapter.saveEmbedding({
+        trap_id: batch[i].id,
+        provider: config.provider,
+        model: config.model,
+        dimensions: config.dimensions,
+        passage_version: config.passageVersion,
+        passage_hash: hashTrapPassage(passages[i]),
+        embedding: embeddings[i],
+      });
+    }
+    batches++;
+  }
+
+  return { generated: traps.length, skipped: Math.max(0, total - traps.length), batches };
+}

package/src/lib/format.ts

@@ -0,0 +1,97 @@
+import { CATEGORY_LABELS, SEVERITY_ICONS, type Category, type Severity } from "./constants";
+import type { Trap, TrapActionCard, TrapDetails, TrapEvidence } from "../domain/trap";
+import { parseEvidenceRelatedFiles, parseTrapTags } from "./trap-json-fields";
+export type { Trap } from "../domain/trap";
+
+export function formatTrapShort(t: Trap, scopeLabel: string): string {
+  const sev = SEVERITY_ICONS[t.severity as Severity] ?? t.severity;
+  const cat = CATEGORY_LABELS[t.category as Category] ?? t.category;
+  return `[${scopeLabel}] [${sev}] [${cat}] #${t.id} ${t.title}`;
+}
+
+export function formatTrapActionCard(card: TrapActionCard): string {
+  const sev = SEVERITY_ICONS[card.severity as Severity] ?? card.severity;
+  const sourceLabel = card.sources.length > 0 ? card.sources.join("+") : "unknown";
+  return `\
+[${card.scope}] [${sev}] #${card.trap_id} ${card.title}
+Why relevant: ${card.why_relevant}
+Avoid: ${card.avoid}
+Do instead: ${card.do_instead}
+Score: ${formatScore(card.score)} (${sourceLabel})
+Next: get_trap id=${card.next_action.details_args.id} scope=${card.next_action.details_args.scope}`;
+}
+
+export function formatTrapDetail(t: Trap, scopeLabel: string): string {
+  const sev = SEVERITY_ICONS[t.severity as Severity] ?? t.severity;
+  const cat = CATEGORY_LABELS[t.category as Category] ?? t.category;
+  const tags = parseTrapTags(t.tags);
+  let out = `\
+══════════════════════════════════════════
+#${t.id}  ${t.title}
+══════════════════════════════════════════
+Scope:     ${scopeLabel} (${t.scope})
+Severity:  ${sev}
+Category:  ${cat}
+Tags:      ${tags.join(", ") || "-"}
+Hit count: ${t.hit_count}
+Created:   ${t.created_at}
+Updated:   ${t.updated_at}
+──────────────────────────────────────────
+Context:
+  ${t.context}
+
+Mistake (what AI tends to do wrong):
+  ${t.mistake}
+
+Fix (what should be done instead):
+  ${t.fix}`;
+
+  if (t.before_code) {
+    out += `\n\nBefore (wrong):\n${indent(t.before_code, 2)}`;
+  }
+  if (t.after_code) {
+    out += `\n\nAfter (correct):\n${indent(t.after_code, 2)}`;
+  }
+  return out;
+}
+
+export function formatTrapDetails(details: TrapDetails): string {
+  const base = formatTrapDetail(details.trap, details.scope);
+  const lifecycle = `\
+
+Lifecycle:
+  Status: ${details.trap.status}
+  State key: ${details.trap.state_key ?? "-"}
+  Supersedes: ${details.trap.supersedes_id ?? "-"}
+  Valid from: ${details.trap.valid_from}
+  Valid until: ${details.trap.valid_until ?? "-"}`;
+
+  if (details.evidence.length === 0) return `${base}${lifecycle}`;
+
+  return `${base}${lifecycle}
+
+Evidence:
+${details.evidence.map(formatEvidence).join("\n\n")}`;
+}
+
+function formatEvidence(evidence: TrapEvidence): string {
+  const relatedFiles = parseEvidenceRelatedFiles(evidence.related_files);
+  return `\
+  - #${evidence.id} ${evidence.source_type}
+    Observed: ${evidence.observed_at}
+    Source: ${evidence.source_ref ?? "-"}
+    Files: ${relatedFiles.join(", ") || "-"}
+    Note: ${evidence.note ?? "-"}`;
+}
+
+function formatScore(score: number | null): string {
+  return score === null ? "-" : score.toFixed(4);
+}
+
+function indent(text: string, spaces: number): string {
+  const pad = " ".repeat(spaces);
+  return text
+    .split("\n")
+    .map((l) => pad + l)
+    .join("\n");
+}

package/src/lib/fts-query.ts

@@ -0,0 +1,17 @@
+export function prepareFTSQuery(query: string): string {
+  const terms = tokenizeLiteralQuery(query);
+  if (terms.length === 0) return "";
+  return terms.map(quoteFTSTerm).join(" OR ");
+}
+
+function tokenizeLiteralQuery(query: string): string[] {
+  return query
+    .trim()
+    .split(/\s+/)
+    .map((term) => term.trim())
+    .filter(Boolean);
+}
+
+function quoteFTSTerm(term: string): string {
+  return `"${term.replaceAll('"', '""')}"`;
+}

package/src/lib/scope.ts

@@ -0,0 +1,30 @@
+import { existsSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+import { CODETRAP_DIR, TRAPS_DB_FILE } from "./constants";
+
+export function getGlobalDir(): string {
+  const dir = join(homedir(), CODETRAP_DIR);
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+export function getGlobalDB(): string {
+  return join(getGlobalDir(), TRAPS_DB_FILE);
+}
+
+export function findProjectRoot(cwd: string): string | null {
+  let dir = cwd;
+  while (true) {
+    if (existsSync(join(dir, CODETRAP_DIR))) return dir;
+    const parent = join(dir, "..");
+    if (parent === dir) return null;
+    dir = parent;
+  }
+}
+
+export function getProjectDB(root: string): string {
+  const dir = join(root, CODETRAP_DIR);
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  return join(dir, TRAPS_DB_FILE);
+}

package/src/lib/search-normalizer.ts

@@ -0,0 +1,92 @@
+const CJK_RUN = /[\u3400-\u9fff\uf900-\ufaff\u3040-\u30ff\uac00-\ud7af]+/gu;
+const ASCII_TOKEN = /[A-Za-z0-9_.$/@:-]+/g;
+
+const SYNONYMS: Record<string, string[]> = {
+  "请求": ["http", "https", "fetch", "request", "axios"],
+  "认证": ["auth", "authentication", "login", "session"],
+  "数据库": ["db", "sql", "sqlite", "migration"],
+  "配置": ["config", "env", "environment"],
+  "缓存": ["cache", "redis"],
+  "路由": ["route", "router"],
+};
+
+export type SearchTextFields = {
+  title?: string;
+  context?: string;
+  mistake?: string;
+  fix?: string;
+  tags?: string | string[];
+  before_code?: string | null;
+  after_code?: string | null;
+};
+
+export const SEARCH_TEXT_FIELD_NAMES = [
+  "title",
+  "context",
+  "mistake",
+  "fix",
+  "tags",
+  "before_code",
+  "after_code",
+] as const;
+
+export function bigramCJK(input: string): string[] {
+  const grams: string[] = [];
+  for (const run of input.matchAll(CJK_RUN)) {
+    const chars = Array.from(run[0]);
+    if (chars.length === 1) {
+      grams.push(chars[0]);
+      continue;
+    }
+    for (let i = 0; i < chars.length - 1; i++) {
+      grams.push(chars[i] + chars[i + 1]);
+    }
+  }
+  return grams;
+}
+
+export function buildSearchText(fields: SearchTextFields): string {
+  const source = fieldsToText(fields);
+  const tokens = new Set<string>();
+
+  for (const token of source.match(ASCII_TOKEN) ?? []) {
+    tokens.add(token.toLowerCase());
+  }
+
+  for (const gram of bigramCJK(source)) {
+    addTokenWithSynonyms(tokens, gram);
+  }
+
+  for (const [trigger, expansions] of Object.entries(SYNONYMS)) {
+    if (source.includes(trigger)) {
+      tokens.add(trigger);
+      for (const expansion of expansions) tokens.add(expansion);
+    }
+  }
+
+  return [...tokens].join(" ");
+}
+
+export function normalizeQuery(query: string): string {
+  return buildSearchText({ title: query });
+}
+
+function addTokenWithSynonyms(tokens: Set<string>, token: string): void {
+  tokens.add(token);
+  for (const expansion of SYNONYMS[token] ?? []) {
+    tokens.add(expansion);
+  }
+}
+
+function fieldsToText(fields: SearchTextFields): string {
+  const parts: string[] = [];
+  for (const field of SEARCH_TEXT_FIELD_NAMES) {
+    const value = fields[field];
+    if (Array.isArray(value)) {
+      parts.push(value.join(" "));
+    } else if (typeof value === "string") {
+      parts.push(value);
+    }
+  }
+  return parts.join(" ");
+}

package/src/lib/search-result-card.ts

@@ -0,0 +1,38 @@
+import type { TrapActionCard, TrapSearchResult } from "../domain/trap";
+import type { Scope } from "./constants";
+
+const MAX_CARD_FIELD_LENGTH = 220;
+
+export function toTrapActionCard(result: TrapSearchResult, scope: Scope): TrapActionCard {
+  const trap = result.trap;
+  return {
+    trap_id: trap.id,
+    scope,
+    title: trap.title,
+    why_relevant: compact(trap.context),
+    avoid: compact(trap.mistake),
+    do_instead: compact(trap.fix),
+    severity: trap.severity,
+    score: result.score ?? null,
+    sources: result.sources ?? [],
+    next_action: {
+      details_tool: "get_trap",
+      details_args: {
+        id: trap.id,
+        scope,
+      },
+    },
+  };
+}
+
+export function toTrapActionCards(groups: { results: TrapSearchResult[]; scope: string }[]): TrapActionCard[] {
+  return groups.flatMap((group) =>
+    group.results.map((result) => toTrapActionCard(result, group.scope as Scope))
+  );
+}
+
+function compact(value: string): string {
+  const normalized = value.replace(/\s+/g, " ").trim();
+  if (normalized.length <= MAX_CARD_FIELD_LENGTH) return normalized;
+  return `${normalized.slice(0, MAX_CARD_FIELD_LENGTH - 3).trimEnd()}...`;
+}

package/src/lib/search-service.ts

@@ -0,0 +1,189 @@
+import type { Database } from "bun:sqlite";
+import * as embeddingQueries from "../db/embedding-queries";
+import * as queries from "../db/queries";
+import type { Trap, TrapSearchResult } from "../domain/trap";
+import type { SearchMode, TrapStatus } from "./constants";
+import {
+  cosineSimilarity,
+  EmbeddingProviderUnavailableError,
+  embeddingConfig,
+  type EmbeddingProvider,
+} from "./embedder";
+
+export interface SearchOptions {
+  category?: string;
+  scope?: string;
+  limit?: number;
+  mode?: SearchMode;
+  status?: TrapStatus | "all";
+}
+
+export interface RankingConfig {
+  rrfK: number;
+  semanticMinScore: number;
+  lengthNormAnchor: number;
+}
+
+export const DEFAULT_RANKING_CONFIG: RankingConfig = {
+  rrfK: 60,
+  semanticMinScore: 0.3,
+  lengthNormAnchor: 500,
+};
+
+const DEFAULT_LIMIT = 20;
+
+export class SearchService {
+  constructor(
+    private readonly db: Database,
+    private readonly embedder?: EmbeddingProvider,
+    private readonly ranking: RankingConfig = DEFAULT_RANKING_CONFIG
+  ) {}
+
+  async search(query: string, opts: SearchOptions = {}): Promise<TrapSearchResult[]> {
+    if (!query.trim()) return [];
+
+    const mode = opts.mode ?? "fts";
+    switch (mode) {
+      case "fts":
+        return this.ftsSearch(query, opts);
+      case "semantic":
+        return this.semanticSearch(query, opts);
+      case "hybrid":
+        return this.hybridSearch(query, opts);
+      default:
+        throw new Error(`Invalid search mode: ${mode satisfies never}`);
+    }
+  }
+
+  ftsSearch(query: string, opts: SearchOptions = {}): TrapSearchResult[] {
+    return queries.searchTraps(this.db, query, opts).map((result) => ({
+      ...result,
+      sources: ["fts"],
+      score: ftsScore(result.rank),
+    }));
+  }
+
+  async semanticSearch(query: string, opts: SearchOptions = {}): Promise<TrapSearchResult[]> {
+    if (!this.embedder) {
+      throw new EmbeddingProviderUnavailableError();
+    }
+
+    const [queryEmbedding] = await this.embedder.embed([query], "retrieval.query");
+    if (!queryEmbedding) return [];
+
+    const config = embeddingConfig(this.embedder);
+    const candidates = embeddingQueries.getAllFreshEmbeddings(this.db, config, {
+      category: opts.category,
+      scope: opts.scope,
+      status: opts.status,
+    });
+
+    return candidates
+      .map(({ trap, embedding }) => {
+        const score = cosineSimilarity(queryEmbedding, embedding);
+        return {
+          trap,
+          rank: score,
+          sources: ["semantic"] as ("semantic")[],
+          score,
+        };
+      })
+      .filter((result) => (result.score ?? 0) >= this.ranking.semanticMinScore)
+      .sort((a, b) => (b.score ?? 0) - (a.score ?? 0))
+      .slice(0, opts.limit ?? DEFAULT_LIMIT);
+  }
+
+  async hybridSearch(query: string, opts: SearchOptions = {}): Promise<TrapSearchResult[]> {
+    const limit = opts.limit ?? DEFAULT_LIMIT;
+    const ftsResults = this.ftsSearch(query, { ...opts, limit });
+
+    try {
+      const semanticResults = await this.semanticSearch(query, { ...opts, limit });
+      if (semanticResults.length === 0) {
+        return withDiagnostics(ftsResults, {
+          code: "semantic_no_candidates",
+          message: "Hybrid search used FTS results because no fresh semantic candidates passed the score threshold.",
+        });
+      }
+      return rrfFuse(ftsResults, semanticResults, limit, this.ranking);
+    } catch (error) {
+      return withDiagnostics(ftsResults, semanticDiagnostic(error));
+    }
+  }
+}
+
+export function rrfFuse(
+  ftsResults: TrapSearchResult[],
+  semanticResults: TrapSearchResult[],
+  limit = DEFAULT_LIMIT,
+  ranking: RankingConfig = DEFAULT_RANKING_CONFIG
+): TrapSearchResult[] {
+  const byId = new Map<number, TrapSearchResult & { score: number; sources: ("fts" | "semantic")[] }>();
+
+  addRankedResults(byId, ftsResults, "fts", ranking);
+  addRankedResults(byId, semanticResults, "semantic", ranking);
+
+  return [...byId.values()]
+    .map((result) => ({
+      ...result,
+      score: applyLengthNormalization(result.score, result.trap, ranking),
+      rank: applyLengthNormalization(result.score, result.trap, ranking),
+    }))
+    .sort((a, b) => (b.score ?? 0) - (a.score ?? 0))
+    .slice(0, limit);
+}
+
+function addRankedResults(
+  byId: Map<number, TrapSearchResult & { score: number; sources: ("fts" | "semantic")[] }>,
+  results: TrapSearchResult[],
+  source: "fts" | "semantic",
+  ranking: RankingConfig
+): void {
+  results.forEach((result, index) => {
+    const score = 1 / (ranking.rrfK + index + 1);
+    const existing = byId.get(result.trap.id);
+    if (existing) {
+      existing.score += score;
+      if (!existing.sources.includes(source)) existing.sources.push(source);
+      return;
+    }
+    byId.set(result.trap.id, {
+      ...result,
+      score,
+      sources: [source],
+    });
+  });
+}
+
+function applyLengthNormalization(score: number, trap: Trap, ranking: RankingConfig): number {
+  const length = `${trap.context}\n${trap.mistake}\n${trap.fix}`.length;
+  if (length <= ranking.lengthNormAnchor) return score;
+  return score * Math.sqrt(ranking.lengthNormAnchor / length);
+}
+
+function ftsScore(rank: number): number {
+  return Number.isFinite(rank) ? -rank : 0;
+}
+
+function withDiagnostics(
+  results: TrapSearchResult[],
+  diagnostic: { code: string; message: string }
+): TrapSearchResult[] {
+  return results.map((result) => ({
+    ...result,
+    diagnostics: [...(result.diagnostics ?? []), diagnostic],
+  }));
+}
+
+function semanticDiagnostic(error: unknown): { code: string; message: string } {
+  if (error instanceof EmbeddingProviderUnavailableError) {
+    return {
+      code: "semantic_unavailable",
+      message: error.message,
+    };
+  }
+  return {
+    code: "semantic_failed",
+    message: error instanceof Error ? error.message : "Semantic search failed; hybrid search returned FTS results.",
+  };
+}