cognitive-core 0.0.2 → 0.1.2

package/references/minimem/src/search/__tests__/hybrid.test.ts

@@ -0,0 +1,247 @@
+import { describe, expect, it } from "vitest";
+
+import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "../hybrid.js";
+
+describe("buildFtsQuery", () => {
+  it("tokenizes and AND-joins", () => {
+    expect(buildFtsQuery("hello world")).toBe('"hello" AND "world"');
+    expect(buildFtsQuery("FOO_bar baz-1")).toBe('"FOO_bar" AND "baz" AND "1"');
+  });
+
+  it("returns null for empty/whitespace input", () => {
+    expect(buildFtsQuery("   ")).toBeNull();
+    expect(buildFtsQuery("")).toBeNull();
+  });
+
+  it("strips quotes from tokens", () => {
+    expect(buildFtsQuery('hello "world')).toBe('"hello" AND "world"');
+  });
+
+  it("handles single token", () => {
+    expect(buildFtsQuery("hello")).toBe('"hello"');
+  });
+
+  it("handles special characters", () => {
+    expect(buildFtsQuery("hello@world.com")).toBe('"hello" AND "world" AND "com"');
+  });
+});
+
+describe("bm25RankToScore", () => {
+  it("returns 1 for rank 0", () => {
+    expect(bm25RankToScore(0)).toBeCloseTo(1);
+  });
+
+  it("returns 0.5 for rank 1 or -1", () => {
+    // Both positive and negative 1 give same result (absolute value)
+    expect(bm25RankToScore(1)).toBeCloseTo(0.5);
+    expect(bm25RankToScore(-1)).toBeCloseTo(0.5);
+  });
+
+  it("is monotonically decreasing with absolute value", () => {
+    // Higher magnitude = lower score
+    expect(bm25RankToScore(10)).toBeLessThan(bm25RankToScore(1));
+    expect(bm25RankToScore(100)).toBeLessThan(bm25RankToScore(10));
+  });
+
+  it("treats negative ranks same as positive (uses absolute value)", () => {
+    // FTS5 BM25 ranks are negative, so -10 should give same score as 10
+    expect(bm25RankToScore(-10)).toBeCloseTo(bm25RankToScore(10));
+    expect(bm25RankToScore(-100)).toBeCloseTo(bm25RankToScore(100));
+    // -100 → abs = 100 → score = 1/101 ≈ 0.0099
+    expect(bm25RankToScore(-100)).toBeCloseTo(1 / 101);
+  });
+
+  it("handles infinity by returning 0", () => {
+    expect(bm25RankToScore(Infinity)).toBe(0);
+    expect(bm25RankToScore(-Infinity)).toBe(0);
+  });
+
+  it("handles NaN by returning 0", () => {
+    expect(bm25RankToScore(NaN)).toBe(0);
+  });
+});
+
+describe("mergeHybridResults", () => {
+  it("unions by id and combines weighted scores", () => {
+    const merged = mergeHybridResults({
+      vectorWeight: 0.7,
+      textWeight: 0.3,
+      vector: [
+        {
+          id: "a",
+          path: "memory/a.md",
+          startLine: 1,
+          endLine: 2,
+          source: "memory",
+          snippet: "vec-a",
+          vectorScore: 0.9,
+        },
+      ],
+      keyword: [
+        {
+          id: "b",
+          path: "memory/b.md",
+          startLine: 3,
+          endLine: 4,
+          source: "memory",
+          snippet: "kw-b",
+          textScore: 1.0,
+        },
+      ],
+    });
+
+    expect(merged).toHaveLength(2);
+    const a = merged.find((r) => r.path === "memory/a.md");
+    const b = merged.find((r) => r.path === "memory/b.md");
+    expect(a?.score).toBeCloseTo(0.7 * 0.9);
+    expect(b?.score).toBeCloseTo(0.3 * 1.0);
+  });
+
+  it("prefers keyword snippet when ids overlap", () => {
+    const merged = mergeHybridResults({
+      vectorWeight: 0.5,
+      textWeight: 0.5,
+      vector: [
+        {
+          id: "a",
+          path: "memory/a.md",
+          startLine: 1,
+          endLine: 2,
+          source: "memory",
+          snippet: "vec-a",
+          vectorScore: 0.2,
+        },
+      ],
+      keyword: [
+        {
+          id: "a",
+          path: "memory/a.md",
+          startLine: 1,
+          endLine: 2,
+          source: "memory",
+          snippet: "kw-a",
+          textScore: 1.0,
+        },
+      ],
+    });
+
+    expect(merged).toHaveLength(1);
+    expect(merged[0]?.snippet).toBe("kw-a");
+    expect(merged[0]?.score).toBeCloseTo(0.5 * 0.2 + 0.5 * 1.0);
+  });
+
+  it("sorts by score descending", () => {
+    const merged = mergeHybridResults({
+      vectorWeight: 1.0,
+      textWeight: 0.0,
+      vector: [
+        {
+          id: "low",
+          path: "memory/low.md",
+          startLine: 1,
+          endLine: 1,
+          source: "memory",
+          snippet: "low",
+          vectorScore: 0.3,
+        },
+        {
+          id: "high",
+          path: "memory/high.md",
+          startLine: 1,
+          endLine: 1,
+          source: "memory",
+          snippet: "high",
+          vectorScore: 0.9,
+        },
+      ],
+      keyword: [],
+    });
+
+    expect(merged[0]?.path).toBe("memory/high.md");
+    expect(merged[1]?.path).toBe("memory/low.md");
+  });
+
+  it("handles empty inputs", () => {
+    const merged = mergeHybridResults({
+      vectorWeight: 0.7,
+      textWeight: 0.3,
+      vector: [],
+      keyword: [],
+    });
+
+    expect(merged).toHaveLength(0);
+  });
+
+  it("handles vector-only results with normalized weights", () => {
+    const merged = mergeHybridResults({
+      vectorWeight: 0.7,
+      textWeight: 0.3,
+      vector: [
+        {
+          id: "a",
+          path: "memory/a.md",
+          startLine: 1,
+          endLine: 2,
+          source: "memory",
+          snippet: "vec-a",
+          vectorScore: 0.8,
+        },
+      ],
+      keyword: [],
+    });
+
+    expect(merged).toHaveLength(1);
+    // When keyword side is empty, vector weight normalizes to 1.0
+    expect(merged[0]?.score).toBeCloseTo(0.8);
+  });
+
+  it("handles keyword-only results with normalized weights", () => {
+    const merged = mergeHybridResults({
+      vectorWeight: 0.7,
+      textWeight: 0.3,
+      vector: [],
+      keyword: [
+        {
+          id: "a",
+          path: "memory/a.md",
+          startLine: 1,
+          endLine: 2,
+          source: "memory",
+          snippet: "kw-a",
+          textScore: 0.9,
+        },
+      ],
+    });
+
+    expect(merged).toHaveLength(1);
+    // When vector side is empty, text weight normalizes to 1.0
+    expect(merged[0]?.score).toBeCloseTo(0.9);
+  });
+
+  it("preserves all metadata fields", () => {
+    const merged = mergeHybridResults({
+      vectorWeight: 1.0,
+      textWeight: 0.0,
+      vector: [
+        {
+          id: "test",
+          path: "memory/test.md",
+          startLine: 5,
+          endLine: 10,
+          source: "sessions",
+          snippet: "test snippet",
+          vectorScore: 0.5,
+        },
+      ],
+      keyword: [],
+    });
+
+    expect(merged[0]).toMatchObject({
+      path: "memory/test.md",
+      startLine: 5,
+      endLine: 10,
+      source: "sessions",
+      snippet: "test snippet",
+    });
+  });
+});

package/references/minimem/src/search/graph.ts

@@ -0,0 +1,234 @@
+import type { DatabaseSync } from "node:sqlite";
+
+/**
+ * A knowledge graph link between two nodes
+ */
+export type GraphLink = {
+  fromId: string;
+  toId: string;
+  relation: string;
+  layer: string | null;
+  weight: number;
+  sourcePath: string | null;
+};
+
+/**
+ * A neighbor node discovered during graph traversal
+ */
+export type GraphNeighbor = {
+  id: string;
+  depth: number;
+  link: GraphLink;
+};
+
+/**
+ * Get all outgoing links from a node.
+ */
+export function getLinksFrom(
+  db: DatabaseSync,
+  fromId: string,
+  opts?: { relation?: string; layer?: string },
+): GraphLink[] {
+  let sql = `SELECT from_id, to_id, relation, layer, weight, source_path FROM knowledge_links WHERE from_id = ?`;
+  const params: (string | number)[] = [fromId];
+
+  if (opts?.relation) {
+    sql += ` AND relation = ?`;
+    params.push(opts.relation);
+  }
+  if (opts?.layer) {
+    sql += ` AND layer = ?`;
+    params.push(opts.layer);
+  }
+
+  const rows = db.prepare(sql).all(...params) as Array<{
+    from_id: string;
+    to_id: string;
+    relation: string;
+    layer: string | null;
+    weight: number;
+    source_path: string | null;
+  }>;
+
+  return rows.map(toGraphLink);
+}
+
+/**
+ * Get all incoming links to a node.
+ */
+export function getLinksTo(
+  db: DatabaseSync,
+  toId: string,
+  opts?: { relation?: string; layer?: string },
+): GraphLink[] {
+  let sql = `SELECT from_id, to_id, relation, layer, weight, source_path FROM knowledge_links WHERE to_id = ?`;
+  const params: (string | number)[] = [toId];
+
+  if (opts?.relation) {
+    sql += ` AND relation = ?`;
+    params.push(opts.relation);
+  }
+  if (opts?.layer) {
+    sql += ` AND layer = ?`;
+    params.push(opts.layer);
+  }
+
+  const rows = db.prepare(sql).all(...params) as Array<{
+    from_id: string;
+    to_id: string;
+    relation: string;
+    layer: string | null;
+    weight: number;
+    source_path: string | null;
+  }>;
+
+  return rows.map(toGraphLink);
+}
+
+/**
+ * BFS traversal to find neighbors up to a given depth.
+ *
+ * @param db - Database handle
+ * @param startId - The starting node ID
+ * @param depth - Maximum traversal depth (default: 1)
+ * @param opts - Optional filters for relation and layer
+ * @returns Array of neighbor nodes with their depth and connecting link
+ */
+export function getNeighbors(
+  db: DatabaseSync,
+  startId: string,
+  depth: number = 1,
+  opts?: { relation?: string; layer?: string },
+): GraphNeighbor[] {
+  const visited = new Set<string>([startId]);
+  const result: GraphNeighbor[] = [];
+  let frontier = [startId];
+
+  for (let d = 1; d <= depth; d++) {
+    const nextFrontier: string[] = [];
+
+    for (const nodeId of frontier) {
+      // Follow outgoing links
+      const outgoing = getLinksFrom(db, nodeId, opts);
+      for (const link of outgoing) {
+        if (!visited.has(link.toId)) {
+          visited.add(link.toId);
+          nextFrontier.push(link.toId);
+          result.push({ id: link.toId, depth: d, link });
+        }
+      }
+
+      // Follow incoming links
+      const incoming = getLinksTo(db, nodeId, opts);
+      for (const link of incoming) {
+        if (!visited.has(link.fromId)) {
+          visited.add(link.fromId);
+          nextFrontier.push(link.fromId);
+          result.push({ id: link.fromId, depth: d, link });
+        }
+      }
+    }
+
+    frontier = nextFrontier;
+    if (frontier.length === 0) break;
+  }
+
+  return result;
+}
+
+/**
+ * BFS shortest path between two nodes, max depth 3.
+ *
+ * @returns Array of links forming the path, or empty if no path found.
+ */
+export function getPathBetween(
+  db: DatabaseSync,
+  fromId: string,
+  toId: string,
+  maxDepth: number = 3,
+): GraphLink[] {
+  if (fromId === toId) return [];
+
+  // BFS with parent tracking
+  const visited = new Set<string>([fromId]);
+  // Maps each visited node to the link that discovered it
+  const parentLink = new Map<string, GraphLink>();
+  let frontier = [fromId];
+
+  for (let d = 0; d < maxDepth; d++) {
+    const nextFrontier: string[] = [];
+
+    for (const nodeId of frontier) {
+      // Follow outgoing links
+      const outgoing = getLinksFrom(db, nodeId);
+      for (const link of outgoing) {
+        if (!visited.has(link.toId)) {
+          visited.add(link.toId);
+          parentLink.set(link.toId, link);
+          if (link.toId === toId) {
+            return reconstructPath(parentLink, fromId, toId);
+          }
+          nextFrontier.push(link.toId);
+        }
+      }
+
+      // Follow incoming links
+      const incoming = getLinksTo(db, nodeId);
+      for (const link of incoming) {
+        if (!visited.has(link.fromId)) {
+          visited.add(link.fromId);
+          parentLink.set(link.fromId, link);
+          if (link.fromId === toId) {
+            return reconstructPath(parentLink, fromId, toId);
+          }
+          nextFrontier.push(link.fromId);
+        }
+      }
+    }
+
+    frontier = nextFrontier;
+    if (frontier.length === 0) break;
+  }
+
+  return []; // No path found
+}
+
+/**
+ * Reconstruct the path from BFS parent map.
+ */
+function reconstructPath(
+  parentLink: Map<string, GraphLink>,
+  fromId: string,
+  toId: string,
+): GraphLink[] {
+  const path: GraphLink[] = [];
+  let current = toId;
+
+  while (current !== fromId) {
+    const link = parentLink.get(current);
+    if (!link) break;
+    path.unshift(link);
+    // Determine which side led to `current`
+    current = link.toId === current ? link.fromId : link.toId;
+  }
+
+  return path;
+}
+
+function toGraphLink(row: {
+  from_id: string;
+  to_id: string;
+  relation: string;
+  layer: string | null;
+  weight: number;
+  source_path: string | null;
+}): GraphLink {
+  return {
+    fromId: row.from_id,
+    toId: row.to_id,
+    relation: row.relation,
+    layer: row.layer,
+    weight: row.weight,
+    sourcePath: row.source_path,
+  };
+}

package/references/minimem/src/search/hybrid.ts

@@ -0,0 +1,151 @@
+export type HybridSource = string;
+
+export type HybridVectorResult = {
+  id: string;
+  path: string;
+  startLine: number;
+  endLine: number;
+  source: HybridSource;
+  snippet: string;
+  vectorScore: number;
+};
+
+export type HybridKeywordResult = {
+  id: string;
+  path: string;
+  startLine: number;
+  endLine: number;
+  source: HybridSource;
+  snippet: string;
+  textScore: number;
+};
+
+export function buildFtsQuery(raw: string): string | null {
+  const tokens =
+    raw
+      .match(/[A-Za-z0-9_]+/g)
+      ?.map((t) => t.trim())
+      .filter(Boolean) ?? [];
+  if (tokens.length === 0) return null;
+  const quoted = tokens.map((t) => `"${t.replaceAll('"', "")}"`);
+  return quoted.join(" AND ");
+}
+
+/**
+ * Convert BM25 rank from SQLite FTS5 to a 0-1 score.
+ *
+ * FTS5 BM25 ranks are NEGATIVE numbers where more negative = better match.
+ * A rank of 0 means no match, -10 is better than -1.
+ *
+ * We use absolute value to convert to positive, then normalize to 0-1
+ * using the formula: score = 1 / (1 + absRank)
+ *
+ * Examples:
+ * - rank 0 (no match) -> score 1.0
+ * - rank -1 (weak match) -> score 0.5
+ * - rank -10 (strong match) -> score ~0.09
+ *
+ * Note: Higher absolute rank magnitude = better match = higher score after conversion.
+ */
+export function bm25RankToScore(rank: number): number {
+  // Handle non-finite values (NaN, Infinity)
+  if (!Number.isFinite(rank)) {
+    return 0;
+  }
+
+  // BM25 ranks from FTS5 are negative (more negative = better match)
+  // Use absolute value to get the magnitude
+  const absRank = Math.abs(rank);
+
+  // Convert to 0-1 score where higher magnitude = higher score
+  // Using 1/(1+x) gives us a nice 0-1 range that decreases smoothly
+  return 1 / (1 + absRank);
+}
+
+export function mergeHybridResults(params: {
+  vector: HybridVectorResult[];
+  keyword: HybridKeywordResult[];
+  vectorWeight: number;
+  textWeight: number;
+}): Array<{
+  path: string;
+  startLine: number;
+  endLine: number;
+  score: number;
+  snippet: string;
+  source: HybridSource;
+}> {
+  const byId = new Map<
+    string,
+    {
+      id: string;
+      path: string;
+      startLine: number;
+      endLine: number;
+      source: HybridSource;
+      snippet: string;
+      vectorScore: number;
+      textScore: number;
+    }
+  >();
+
+  for (const r of params.vector) {
+    byId.set(r.id, {
+      id: r.id,
+      path: r.path,
+      startLine: r.startLine,
+      endLine: r.endLine,
+      source: r.source,
+      snippet: r.snippet,
+      vectorScore: r.vectorScore,
+      textScore: 0,
+    });
+  }
+
+  for (const r of params.keyword) {
+    const existing = byId.get(r.id);
+    if (existing) {
+      existing.textScore = r.textScore;
+      if (r.snippet && r.snippet.length > 0) existing.snippet = r.snippet;
+    } else {
+      byId.set(r.id, {
+        id: r.id,
+        path: r.path,
+        startLine: r.startLine,
+        endLine: r.endLine,
+        source: r.source,
+        snippet: r.snippet,
+        vectorScore: 0,
+        textScore: r.textScore,
+      });
+    }
+  }
+
+  // When one side of the hybrid search has no results, normalize weights
+  // so the available side scores at full strength. Without this, BM25-only
+  // results would be scaled to 0.3 * textScore which is too low to pass
+  // the default minScore threshold.
+  let vw = params.vectorWeight;
+  let tw = params.textWeight;
+  if (params.vector.length === 0 && params.keyword.length > 0) {
+    vw = 0;
+    tw = 1;
+  } else if (params.keyword.length === 0 && params.vector.length > 0) {
+    vw = 1;
+    tw = 0;
+  }
+
+  const merged = Array.from(byId.values()).map((entry) => {
+    const score = vw * entry.vectorScore + tw * entry.textScore;
+    return {
+      path: entry.path,
+      startLine: entry.startLine,
+      endLine: entry.endLine,
+      score,
+      snippet: entry.snippet,
+      source: entry.source,
+    };
+  });
+
+  return merged.sort((a, b) => b.score - a.score);
+}