@context-vault/core 2.17.0 → 3.0.0

package/src/search.ts

@@ -0,0 +1,285 @@
+import type { BaseCtx, SearchResult, SearchOptions, VaultEntry } from "./types.js";
+
+const NEAR_DUP_THRESHOLD = 0.92;
+const RRF_K = 60;
+
+export function recencyDecayScore(updatedAt: string | null | undefined, decayRate = 0.05): number {
+  if (updatedAt == null) return 0.5;
+  const ageDays = (Date.now() - new Date(updatedAt).getTime()) / 86400000;
+  return Math.exp(-decayRate * ageDays);
+}
+
+export function dotProduct(a: Float32Array, b: Float32Array): number {
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) sum += a[i] * b[i];
+  return sum;
+}
+
+export function buildFtsQuery(query: string): string | null {
+  const words = query
+    .split(/[\s-]+/)
+    .map((w) => w.replace(/[*"():^~{}]/g, ""))
+    .filter((w) => w.length > 0);
+  if (!words.length) return null;
+  if (words.length === 1) return `"${words[0]}"`;
+  const phrase = `"${words.join(" ")}"`;
+  const near = `NEAR(${words.map((w) => `"${w}"`).join(" ")}, 10)`;
+  const and = words.map((w) => `"${w}"`).join(" AND ");
+  return `${phrase} OR ${near} OR ${and}`;
+}
+
+export function recencyBoost(createdAt: string, category: string, decayDays = 30): number {
+  if (category !== "event") return 1.0;
+  const ageDays = (Date.now() - new Date(createdAt).getTime()) / 86400000;
+  return 1 / (1 + ageDays / decayDays);
+}
+
+export function buildFilterClauses({
+  categoryFilter,
+  excludeEvents = false,
+  since,
+  until,
+  includeSuperseeded = false,
+}: {
+  categoryFilter?: string | null;
+  excludeEvents?: boolean;
+  since?: string | null;
+  until?: string | null;
+  includeSuperseeded?: boolean;
+}): { clauses: string[]; params: unknown[] } {
+  const clauses: string[] = [];
+  const params: unknown[] = [];
+  if (categoryFilter) {
+    clauses.push("e.category = ?");
+    params.push(categoryFilter);
+  }
+  if (excludeEvents && !categoryFilter) {
+    clauses.push("e.category != 'event'");
+  }
+  if (since) {
+    clauses.push("e.created_at >= ?");
+    params.push(since);
+  }
+  if (until) {
+    clauses.push("e.created_at <= ?");
+    params.push(until);
+  }
+  clauses.push("(e.expires_at IS NULL OR e.expires_at > datetime('now'))");
+  if (!includeSuperseeded) {
+    clauses.push("e.superseded_by IS NULL");
+  }
+  return { clauses, params };
+}
+
+export function reciprocalRankFusion(
+  rankedLists: string[][],
+  k: number = RRF_K,
+): Map<string, number> {
+  const scores = new Map<string, number>();
+  for (const list of rankedLists) {
+    for (let rank = 0; rank < list.length; rank++) {
+      const id = list[rank];
+      scores.set(id, (scores.get(id) ?? 0) + 1 / (k + rank + 1));
+    }
+  }
+  return scores;
+}
+
+export async function hybridSearch(
+  ctx: BaseCtx,
+  query: string,
+  opts: SearchOptions = {},
+): Promise<SearchResult[]> {
+  const {
+    kindFilter = null,
+    categoryFilter = null,
+    excludeEvents = false,
+    since = null,
+    until = null,
+    limit = 20,
+    offset = 0,
+    decayDays = 30,
+    includeSuperseeded = false,
+  } = opts;
+
+  const rowMap = new Map<string, VaultEntry>();
+  const idToRowid = new Map<string, number>();
+  let queryVec: Float32Array | null = null;
+
+  const extraFilters = buildFilterClauses({
+    categoryFilter,
+    excludeEvents,
+    since,
+    until,
+    includeSuperseeded,
+  });
+
+  const ftsRankedIds: string[] = [];
+
+  const ftsQuery = buildFtsQuery(query);
+  if (ftsQuery) {
+    try {
+      const whereParts = ["vault_fts MATCH ?"];
+      const ftsParams: unknown[] = [ftsQuery];
+
+      if (kindFilter) {
+        whereParts.push("e.kind = ?");
+        ftsParams.push(kindFilter);
+      }
+      whereParts.push(...extraFilters.clauses);
+      ftsParams.push(...extraFilters.params);
+
+      const ftsSQL = `SELECT e.*, rank FROM vault_fts f JOIN vault e ON f.rowid = e.rowid WHERE ${whereParts.join(" AND ")} ORDER BY rank LIMIT 15`;
+      // @ts-expect-error -- node:sqlite types are overly strict for dynamic SQL params
+      const rows = ctx.db.prepare(ftsSQL).all(...ftsParams) as unknown as (VaultEntry & { rank: number })[];
+
+      for (const { rank: _rank, ...row } of rows) {
+        ftsRankedIds.push(row.id);
+        if (!rowMap.has(row.id)) rowMap.set(row.id, row);
+      }
+    } catch (err) {
+      if (!(err as Error).message?.includes("fts5: syntax error")) {
+        console.error(`[retrieve] FTS search error: ${(err as Error).message}`);
+      }
+    }
+  }
+
+  const vecRankedIds: string[] = [];
+  const vecSimMap = new Map<string, number>();
+
+  try {
+    const vecCount = (ctx.db.prepare("SELECT COUNT(*) as c FROM vault_vec").get() as { c: number }).c;
+    if (vecCount > 0) {
+      queryVec = await ctx.embed(query);
+      if (queryVec) {
+        const vecLimit = kindFilter ? 30 : 15;
+        const vecRows = ctx.db
+          .prepare(
+            `SELECT v.rowid, v.distance FROM vault_vec v WHERE embedding MATCH ? ORDER BY distance LIMIT ?`,
+          )
+          .all(queryVec, vecLimit) as { rowid: number; distance: number }[];
+
+        if (vecRows.length) {
+          const rowids = vecRows.map((vr) => vr.rowid);
+          const placeholders = rowids.map(() => "?").join(",");
+          const hydrated = ctx.db
+            .prepare(
+              `SELECT rowid, * FROM vault WHERE rowid IN (${placeholders})`,
+            )
+            .all(...rowids) as unknown as (VaultEntry & { rowid: number })[];
+
+          const byRowid = new Map<number, VaultEntry & { rowid: number }>();
+          for (const row of hydrated) byRowid.set(row.rowid, row);
+
+          for (const vr of vecRows) {
+            const row = byRowid.get(vr.rowid);
+            if (!row) continue;
+            if (kindFilter && row.kind !== kindFilter) continue;
+            if (categoryFilter && row.category !== categoryFilter) continue;
+            if (excludeEvents && row.category === "event") continue;
+            if (since && row.created_at < since) continue;
+            if (until && row.created_at > until) continue;
+            if (row.expires_at && new Date(row.expires_at) <= new Date())
+              continue;
+
+            const { rowid: _rowid, ...cleanRow } = row;
+            idToRowid.set(cleanRow.id, Number(row.rowid));
+
+            const vecSim = Math.max(0, 1 - vr.distance / 2);
+            vecSimMap.set(cleanRow.id, vecSim);
+            vecRankedIds.push(cleanRow.id);
+
+            if (!rowMap.has(cleanRow.id)) rowMap.set(cleanRow.id, cleanRow);
+          }
+        }
+      }
+    }
+  } catch (err) {
+    if (!(err as Error).message?.includes("no such table")) {
+      console.error(`[retrieve] Vector search error: ${(err as Error).message}`);
+    }
+  }
+
+  if (rowMap.size === 0) return [];
+
+  const rrfScores = reciprocalRankFusion([ftsRankedIds, vecRankedIds]);
+
+  for (const [id, entry] of rowMap) {
+    const boost = recencyBoost(entry.created_at, entry.category, decayDays);
+    rrfScores.set(id, (rrfScores.get(id) ?? 0) * boost);
+  }
+
+  const candidates: SearchResult[] = [...rowMap.values()].map((entry) => ({
+    ...entry,
+    score: rrfScores.get(entry.id) ?? 0,
+  }));
+  candidates.sort((a, b) => b.score - a.score);
+
+  const embeddingMap = new Map<string, Float32Array>();
+  if (queryVec && idToRowid.size > 0) {
+    const rowidToId = new Map<number, string>();
+    for (const [id, rowid] of idToRowid) rowidToId.set(rowid, id);
+
+    const rowidsToFetch = [...idToRowid.values()];
+    try {
+      const placeholders = rowidsToFetch.map(() => "?").join(",");
+      const vecData = ctx.db
+        .prepare(
+          `SELECT rowid, embedding FROM vault_vec WHERE rowid IN (${placeholders})`,
+        )
+        .all(...rowidsToFetch) as { rowid: number; embedding: Buffer }[];
+      for (const row of vecData) {
+        const id = rowidToId.get(Number(row.rowid));
+        const buf = row.embedding;
+        if (id && buf) {
+          embeddingMap.set(
+            id,
+            new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4),
+          );
+        }
+      }
+    } catch {
+      // Embeddings unavailable
+    }
+  }
+
+  if (queryVec && embeddingMap.size > 0) {
+    const selected: SearchResult[] = [];
+    const selectedVecs: Float32Array[] = [];
+    for (const candidate of candidates) {
+      if (selected.length >= offset + limit) break;
+      const vec = embeddingMap.get(candidate.id);
+      if (vec && selectedVecs.length > 0) {
+        let maxSim = 0;
+        for (const sv of selectedVecs) {
+          const sim = dotProduct(sv, vec);
+          if (sim > maxSim) maxSim = sim;
+        }
+        if (maxSim > NEAR_DUP_THRESHOLD) continue;
+      }
+      selected.push(candidate);
+      if (vec) selectedVecs.push(vec);
+    }
+    const dedupedPage = selected.slice(offset, offset + limit);
+    trackAccess(ctx, dedupedPage);
+    return dedupedPage;
+  }
+
+  const finalPage = candidates.slice(offset, offset + limit);
+  trackAccess(ctx, finalPage);
+  return finalPage;
+}
+
+function trackAccess(ctx: BaseCtx, entries: SearchResult[]): void {
+  if (!entries.length) return;
+  try {
+    const placeholders = entries.map(() => "?").join(",");
+    ctx.db
+      .prepare(
+        `UPDATE vault SET hit_count = hit_count + 1, last_accessed_at = datetime('now') WHERE id IN (${placeholders})`,
+      )
+      .run(...entries.map((e) => e.id));
+  } catch {
+    // Non-fatal
+  }
+}

package/src/types.ts

@@ -0,0 +1,166 @@
+import type { DatabaseSync, StatementSync } from "node:sqlite";
+
+export interface VaultConfig {
+  vaultDir: string;
+  dataDir: string;
+  dbPath: string;
+  devDir: string;
+  eventDecayDays: number;
+  thresholds: GrowthThresholds;
+  telemetry: boolean;
+  resolvedFrom: string;
+  configPath?: string;
+  vaultDirExists?: boolean;
+  recall: RecallConfig;
+  consolidation: ConsolidationConfig;
+  lifecycle: Record<string, { archiveAfterDays?: number }>;
+}
+
+export interface RecallConfig {
+  maxResults: number;
+  maxOutputBytes: number;
+  minRelevanceScore: number;
+  excludeKinds: string[];
+  excludeCategories: string[];
+  bodyTruncateChars: number;
+}
+
+export interface ConsolidationConfig {
+  tagThreshold: number;
+  maxAgeDays: number;
+  autoConsolidate: boolean;
+}
+
+export interface GrowthThresholds {
+  totalEntries: { warn: number; critical: number };
+  eventEntries: { warn: number; critical: number };
+  vaultSizeBytes: { warn: number; critical: number };
+  eventsWithoutTtl: { warn: number };
+}
+
+export interface PreparedStatements {
+  insertEntry: StatementSync;
+  updateEntry: StatementSync;
+  deleteEntry: StatementSync;
+  getRowid: StatementSync;
+  getRowidByPath: StatementSync;
+  getEntryById: StatementSync;
+  getByIdentityKey: StatementSync;
+  upsertByIdentityKey: StatementSync;
+  updateSourceFiles: StatementSync;
+  updateRelatedTo: StatementSync;
+  insertVecStmt: StatementSync;
+  deleteVecStmt: StatementSync;
+  updateSupersededBy: StatementSync;
+  clearSupersededByRef: StatementSync;
+}
+
+export interface VaultEntry {
+  id: string;
+  kind: string;
+  category: string;
+  title: string | null;
+  body: string;
+  meta: string | null;
+  tags: string | null;
+  source: string | null;
+  file_path: string | null;
+  identity_key: string | null;
+  expires_at: string | null;
+  superseded_by: string | null;
+  created_at: string;
+  updated_at: string | null;
+  hit_count: number;
+  last_accessed_at: string | null;
+  source_files: string | null;
+  tier: string;
+  related_to: string | null;
+  rowid?: number;
+}
+
+export interface SearchResult extends VaultEntry {
+  score: number;
+  stale?: boolean;
+  stale_reason?: string;
+}
+
+export interface CaptureInput {
+  kind: string;
+  title?: string | null;
+  body: string;
+  meta?: Record<string, unknown> | null;
+  tags?: string[] | null;
+  source?: string | null;
+  folder?: string | null;
+  identity_key?: string | null;
+  expires_at?: string | null;
+  supersedes?: string[] | null;
+  related_to?: string[] | null;
+  source_files?: Array<{ path: string; hash: string }> | null;
+  tier?: string | null;
+}
+
+export interface CaptureResult {
+  id: string;
+  filePath: string;
+  kind: string;
+  category: string;
+  title: string | null;
+  body: string;
+  meta: Record<string, unknown> | undefined;
+  tags: string[] | null;
+  source: string | null;
+  createdAt: string;
+  updatedAt: string;
+  identity_key: string | null;
+  expires_at: string | null;
+  supersedes: string[] | null;
+  related_to: string[] | null;
+  source_files: Array<{ path: string; hash: string }> | null;
+  tier: string | null;
+}
+
+export interface IndexEntryInput {
+  id: string;
+  kind: string;
+  category: string;
+  title: string | null;
+  body: string;
+  meta: Record<string, unknown> | undefined;
+  tags: string[] | null;
+  source: string | null;
+  filePath: string;
+  createdAt: string;
+  identity_key: string | null;
+  expires_at: string | null;
+  source_files: Array<{ path: string; hash: string }> | null;
+  tier: string | null;
+}
+
+export interface ReindexStats {
+  added: number;
+  updated: number;
+  removed: number;
+  unchanged: number;
+}
+
+export interface BaseCtx {
+  db: DatabaseSync;
+  config: VaultConfig;
+  stmts: PreparedStatements;
+  embed: (text: string) => Promise<Float32Array | null>;
+  insertVec: (rowid: number, embedding: Float32Array) => void;
+  deleteVec: (rowid: number) => void;
+}
+
+export interface SearchOptions {
+  kindFilter?: string | null;
+  categoryFilter?: string | null;
+  excludeEvents?: boolean;
+  since?: string | null;
+  until?: string | null;
+  limit?: number;
+  offset?: number;
+  decayDays?: number;
+  includeSuperseeded?: boolean;
+}

package/src/capture/file-ops.js

@@ -1,97 +0,0 @@
-/**
- * file-ops.js — Capture-specific file operations
- *
- * Writes markdown entry files with frontmatter to the vault directory.
- */
-
-import { mkdirSync, writeFileSync } from "node:fs";
-import { resolve, relative } from "node:path";
-import { formatFrontmatter } from "../core/frontmatter.js";
-import { slugify, kindToPath } from "../core/files.js";
-import { formatBody } from "./formatters.js";
-
-export function safeFolderPath(vaultDir, kind, folder) {
-  const base = resolve(vaultDir, kindToPath(kind));
-  if (!folder) return base;
-  const resolved = resolve(base, folder);
-  const rel = relative(base, resolved);
-  if (rel.startsWith("..") || resolve(base, rel) !== resolved) {
-    throw new Error(`Folder path escapes vault: "${folder}"`);
-  }
-  return resolved;
-}
-
-export function writeEntryFile(
-  vaultDir,
-  kind,
-  {
-    id,
-    title,
-    body,
-    meta,
-    tags,
-    source,
-    createdAt,
-    updatedAt,
-    folder,
-    category,
-    identity_key,
-    expires_at,
-    supersedes,
-  },
-) {
-  // P5: folder is now a top-level param; also accept from meta for backward compat
-  const resolvedFolder = folder || meta?.folder || "";
-  const dir = safeFolderPath(vaultDir, kind, resolvedFolder);
-
-  try {
-    mkdirSync(dir, { recursive: true });
-  } catch (e) {
-    throw new Error(`Failed to create directory "${dir}": ${e.message}`);
-  }
-
-  const created = createdAt || new Date().toISOString();
-  const fmFields = { id };
-
-  // Add kind-specific meta fields to frontmatter (flattened, not nested)
-  if (meta) {
-    for (const [k, v] of Object.entries(meta)) {
-      if (k === "folder") continue;
-      if (v !== null && v !== undefined) fmFields[k] = v;
-    }
-  }
-
-  if (identity_key) fmFields.identity_key = identity_key;
-  if (expires_at) fmFields.expires_at = expires_at;
-  if (supersedes?.length) fmFields.supersedes = supersedes;
-  fmFields.tags = tags || [];
-  fmFields.source = source || "claude-code";
-  fmFields.created = created;
-  if (updatedAt && updatedAt !== created) fmFields.updated = updatedAt;
-
-  const mdBody = formatBody(kind, { title, body, meta });
-
-  // Entity kinds: deterministic filename from identity_key (no ULID suffix)
-  let filename;
-  if (category === "entity" && identity_key) {
-    const identitySlug = slugify(identity_key);
-    filename = identitySlug
-      ? `${identitySlug}.md`
-      : `${id.slice(-8).toLowerCase()}.md`;
-  } else {
-    const slug = slugify((title || body).slice(0, 40));
-    const shortId = id.slice(-8).toLowerCase();
-    filename = slug ? `${slug}-${shortId}.md` : `${shortId}.md`;
-  }
-
-  const filePath = resolve(dir, filename);
-  const md = formatFrontmatter(fmFields) + mdBody;
-
-  try {
-    writeFileSync(filePath, md);
-  } catch (e) {
-    throw new Error(`Failed to write entry file "${filePath}": ${e.message}`);
-  }
-
-  return filePath;
-}

package/src/capture/import-pipeline.js

@@ -1,46 +0,0 @@
-import { captureAndIndex } from "./index.js";
-
-export async function importEntries(ctx, entries, opts = {}) {
-  const { onProgress, source } = opts;
-  let imported = 0;
-  let failed = 0;
-  const errors = [];
-
-  for (let i = 0; i < entries.length; i++) {
-    const entry = entries[i];
-
-    if (onProgress) {
-      onProgress(i + 1, entries.length);
-    }
-
-    try {
-      if (!entry.body?.trim()) {
-        failed++;
-        errors.push({ index: i, title: entry.title, error: "Empty body" });
-        continue;
-      }
-
-      await captureAndIndex(ctx, {
-        kind: entry.kind || "insight",
-        title: entry.title || null,
-        body: entry.body,
-        meta: entry.meta,
-        tags: entry.tags,
-        source: entry.source || source || "import",
-        identity_key: entry.identity_key,
-        expires_at: entry.expires_at,
-        userId: ctx.userId || null,
-      });
-      imported++;
-    } catch (err) {
-      failed++;
-      errors.push({
-        index: i,
-        title: entry.title || null,
-        error: err.message,
-      });
-    }
-  }
-
-  return { imported, failed, errors };
-}