@context-vault/core 2.17.1 → 3.0.3

package/src/{retrieve/index.js → search.ts}

@@ -1,48 +1,21 @@
-/**
- * Retrieve Layer — Public API
- *
- * All read-path query logic: hybrid semantic search and any future
- * query patterns (scoped, recency-weighted, etc.).
- *
- * Agent Constraint: Read-only access to DB. Never writes.
- */
+import type { BaseCtx, SearchResult, SearchOptions, VaultEntry } from "./types.js";
 
 const NEAR_DUP_THRESHOLD = 0.92;
-
 const RRF_K = 60;
 
-/**
- * Exponential recency decay score based on updated_at timestamp.
- * Returns e^(-decayRate * ageDays) for valid dates, or 0.5 as a neutral
- * score when updatedAt is null/undefined.
- *
- * @param {string|null|undefined} updatedAt - ISO timestamp
- * @param {number} decayRate - Decay rate per day (default 0.05)
- * @returns {number} Score in [0, 1]
- */
-export function recencyDecayScore(updatedAt, decayRate = 0.05) {
+export function recencyDecayScore(updatedAt: string | null | undefined, decayRate = 0.05): number {
   if (updatedAt == null) return 0.5;
   const ageDays = (Date.now() - new Date(updatedAt).getTime()) / 86400000;
   return Math.exp(-decayRate * ageDays);
 }
 
-/**
- * Dot product of two Float32Array vectors (cosine similarity for unit vectors).
- */
-export function dotProduct(a, b) {
+export function dotProduct(a: Float32Array, b: Float32Array): number {
   let sum = 0;
   for (let i = 0; i < a.length; i++) sum += a[i] * b[i];
   return sum;
 }
 
-/**
- * Build a tiered FTS5 query that prioritises phrase match, then proximity,
- * then AND.  Multi-word queries become:
- *   "word1 word2" OR NEAR("word1" "word2", 10) OR "word1" AND "word2"
- * Single-word queries remain a simple quoted term.
- * Returns null if no valid words remain after stripping FTS5 metacharacters.
- */
-export function buildFtsQuery(query) {
+export function buildFtsQuery(query: string): string | null {
   const words = query
     .split(/[\s-]+/)
     .map((w) => w.replace(/[*"():^~{}]/g, ""))
@@ -55,40 +28,27 @@ export function buildFtsQuery(query) {
   return `${phrase} OR ${near} OR ${and}`;
 }
 
-/**
- * Category-aware recency decay:
- *   knowledge + entity: no decay (enduring)
- *   event: steeper decay (~0.5 at 30 days)
- */
-export function recencyBoost(createdAt, category, decayDays = 30) {
+export function recencyBoost(createdAt: string, category: string, decayDays = 30): number {
   if (category !== "event") return 1.0;
   const ageDays = (Date.now() - new Date(createdAt).getTime()) / 86400000;
   return 1 / (1 + ageDays / decayDays);
 }
 
-/**
- * Build additional WHERE clauses for category/time filtering.
- * Returns { clauses: string[], params: any[] }
- */
 export function buildFilterClauses({
   categoryFilter,
   excludeEvents = false,
   since,
   until,
-  userIdFilter,
-  teamIdFilter,
   includeSuperseeded = false,
-}) {
-  const clauses = [];
-  const params = [];
-  if (userIdFilter !== undefined) {
-    clauses.push("e.user_id = ?");
-    params.push(userIdFilter);
-  }
-  if (teamIdFilter) {
-    clauses.push("e.team_id = ?");
-    params.push(teamIdFilter);
-  }
+}: {
+  categoryFilter?: string | null;
+  excludeEvents?: boolean;
+  since?: string | null;
+  until?: string | null;
+  includeSuperseeded?: boolean;
+}): { clauses: string[]; params: unknown[] } {
+  const clauses: string[] = [];
+  const params: unknown[] = [];
   if (categoryFilter) {
     clauses.push("e.category = ?");
     params.push(categoryFilter);
@@ -111,16 +71,11 @@ export function buildFilterClauses({
   return { clauses, params };
 }
 
-/**
- * Reciprocal Rank Fusion: merge multiple ranked lists into a single score.
- * Each document receives 1/(k + rank) from each list it appears in.
- *
- * @param {Array<string[]>} rankedLists - Arrays of document IDs in rank order (best first).
- * @param {number} k - Smoothing constant (default RRF_K = 60).
- * @returns {Map<string, number>} Map of id -> RRF score.
- */
-export function reciprocalRankFusion(rankedLists, k = RRF_K) {
-  const scores = new Map();
+export function reciprocalRankFusion(
+  rankedLists: string[][],
+  k: number = RRF_K,
+): Map<string, number> {
+  const scores = new Map<string, number>();
   for (const list of rankedLists) {
     for (let rank = 0; rank < list.length; rank++) {
       const id = list[rank];
@@ -130,26 +85,12 @@ export function reciprocalRankFusion(rankedLists, k = RRF_K) {
   return scores;
 }
 
-/**
- * Hybrid search combining FTS5 text matching and vector similarity,
- * with RRF merging, recency decay, and near-duplicate suppression.
- *
- * Pipeline:
- *   1. FTS5 ranked list
- *   2. Vector (semantic) ranked list
- *   3. RRF: merge the two ranked lists into a single score
- *   4. Recency decay: penalise old events (knowledge/entity entries unaffected)
- *   5. Near-duplicate suppression (cosine similarity > 0.92 threshold)
- *
- * @param {import('../server/types.js').BaseCtx} ctx
- * @param {string} query
- * @param {{ kindFilter?: string|null, categoryFilter?: string|null, since?: string|null, until?: string|null, limit?: number, offset?: number }} opts
- * @returns {Promise<Array<{id, kind, category, title, body, meta, tags, source, file_path, created_at, score}>>}
- */
 export async function hybridSearch(
-  ctx,
-  query,
-  {
+  ctx: BaseCtx,
+  query: string,
+  opts: SearchOptions = {},
+): Promise<SearchResult[]> {
+  const {
     kindFilter = null,
     categoryFilter = null,
     excludeEvents = false,
@@ -158,33 +99,28 @@ export async function hybridSearch(
     limit = 20,
     offset = 0,
     decayDays = 30,
-    userIdFilter,
-    teamIdFilter = null,
     includeSuperseeded = false,
-  } = {},
-) {
-  const rowMap = new Map();
-  const idToRowid = new Map();
-  let queryVec = null;
+  } = opts;
+
+  const rowMap = new Map<string, VaultEntry>();
+  const idToRowid = new Map<string, number>();
+  let queryVec: Float32Array | null = null;
 
   const extraFilters = buildFilterClauses({
     categoryFilter,
     excludeEvents,
     since,
     until,
-    userIdFilter,
-    teamIdFilter,
     includeSuperseeded,
   });
 
-  const ftsRankedIds = [];
+  const ftsRankedIds: string[] = [];
 
-  // Stage 1a: FTS5 — collect ranked list of IDs
   const ftsQuery = buildFtsQuery(query);
   if (ftsQuery) {
     try {
       const whereParts = ["vault_fts MATCH ?"];
-      const ftsParams = [ftsQuery];
+      const ftsParams: unknown[] = [ftsQuery];
 
       if (kindFilter) {
         whereParts.push("e.kind = ?");
@@ -194,43 +130,34 @@ export async function hybridSearch(
       ftsParams.push(...extraFilters.params);
 
       const ftsSQL = `SELECT e.*, rank FROM vault_fts f JOIN vault e ON f.rowid = e.rowid WHERE ${whereParts.join(" AND ")} ORDER BY rank LIMIT 15`;
-      const rows = ctx.db.prepare(ftsSQL).all(...ftsParams);
+      // @ts-expect-error -- node:sqlite types are overly strict for dynamic SQL params
+      const rows = ctx.db.prepare(ftsSQL).all(...ftsParams) as unknown as (VaultEntry & { rank: number })[];
 
       for (const { rank: _rank, ...row } of rows) {
         ftsRankedIds.push(row.id);
         if (!rowMap.has(row.id)) rowMap.set(row.id, row);
       }
     } catch (err) {
-      if (!err.message?.includes("fts5: syntax error")) {
-        console.error(`[retrieve] FTS search error: ${err.message}`);
+      if (!(err as Error).message?.includes("fts5: syntax error")) {
+        console.error(`[retrieve] FTS search error: ${(err as Error).message}`);
       }
     }
   }
 
-  const vecRankedIds = [];
-  const vecSimMap = new Map();
+  const vecRankedIds: string[] = [];
+  const vecSimMap = new Map<string, number>();
 
-  // Stage 1b: Vector similarity — collect ranked list of IDs and raw similarity scores
   try {
-    const vecCount = ctx.db
-      .prepare("SELECT COUNT(*) as c FROM vault_vec")
-      .get().c;
+    const vecCount = (ctx.db.prepare("SELECT COUNT(*) as c FROM vault_vec").get() as { c: number }).c;
     if (vecCount > 0) {
       queryVec = await ctx.embed(query);
       if (queryVec) {
-        const hasPostFilter = userIdFilter !== undefined || teamIdFilter;
-        const vecLimit = hasPostFilter
-          ? kindFilter
-            ? 60
-            : 30
-          : kindFilter
-            ? 30
-            : 15;
+        const vecLimit = kindFilter ? 30 : 15;
         const vecRows = ctx.db
           .prepare(
             `SELECT v.rowid, v.distance FROM vault_vec v WHERE embedding MATCH ? ORDER BY distance LIMIT ?`,
           )
-          .all(queryVec, vecLimit);
+          .all(queryVec, vecLimit) as { rowid: number; distance: number }[];
 
         if (vecRows.length) {
           const rowids = vecRows.map((vr) => vr.rowid);
@@ -239,17 +166,14 @@ export async function hybridSearch(
             .prepare(
               `SELECT rowid, * FROM vault WHERE rowid IN (${placeholders})`,
             )
-            .all(...rowids);
+            .all(...rowids) as unknown as (VaultEntry & { rowid: number })[];
 
-          const byRowid = new Map();
+          const byRowid = new Map<number, VaultEntry & { rowid: number }>();
           for (const row of hydrated) byRowid.set(row.rowid, row);
 
           for (const vr of vecRows) {
             const row = byRowid.get(vr.rowid);
             if (!row) continue;
-            if (userIdFilter !== undefined && row.user_id !== userIdFilter)
-              continue;
-            if (teamIdFilter && row.team_id !== teamIdFilter) continue;
             if (kindFilter && row.kind !== kindFilter) continue;
             if (categoryFilter && row.category !== categoryFilter) continue;
             if (excludeEvents && row.category === "event") continue;
@@ -261,8 +185,6 @@ export async function hybridSearch(
             const { rowid: _rowid, ...cleanRow } = row;
             idToRowid.set(cleanRow.id, Number(row.rowid));
 
-            // sqlite-vec returns L2 distance [0, 2] for normalized vectors.
-            // Convert to similarity [0, 1]: 1 - distance/2
             const vecSim = Math.max(0, 1 - vr.distance / 2);
             vecSimMap.set(cleanRow.id, vecSim);
             vecRankedIds.push(cleanRow.id);
@@ -273,33 +195,29 @@ export async function hybridSearch(
       }
     }
   } catch (err) {
-    if (!err.message?.includes("no such table")) {
-      console.error(`[retrieve] Vector search error: ${err.message}`);
+    if (!(err as Error).message?.includes("no such table")) {
+      console.error(`[retrieve] Vector search error: ${(err as Error).message}`);
     }
   }
 
   if (rowMap.size === 0) return [];
 
-  // Stage 2: RRF — merge FTS and vector ranked lists into a single score
   const rrfScores = reciprocalRankFusion([ftsRankedIds, vecRankedIds]);
 
-  // Stage 3: Apply category-aware recency boost to RRF scores
   for (const [id, entry] of rowMap) {
     const boost = recencyBoost(entry.created_at, entry.category, decayDays);
     rrfScores.set(id, (rrfScores.get(id) ?? 0) * boost);
   }
 
-  // Attach final score to each entry and sort by RRF score descending
-  const candidates = [...rowMap.values()].map((entry) => ({
+  const candidates: SearchResult[] = [...rowMap.values()].map((entry) => ({
     ...entry,
     score: rrfScores.get(entry.id) ?? 0,
   }));
   candidates.sort((a, b) => b.score - a.score);
 
-  // Stage 4: Fetch embeddings for near-duplicate suppression
-  const embeddingMap = new Map();
+  const embeddingMap = new Map<string, Float32Array>();
   if (queryVec && idToRowid.size > 0) {
-    const rowidToId = new Map();
+    const rowidToId = new Map<number, string>();
     for (const [id, rowid] of idToRowid) rowidToId.set(rowid, id);
 
     const rowidsToFetch = [...idToRowid.values()];
@@ -309,7 +227,7 @@ export async function hybridSearch(
         .prepare(
           `SELECT rowid, embedding FROM vault_vec WHERE rowid IN (${placeholders})`,
         )
-        .all(...rowidsToFetch);
+        .all(...rowidsToFetch) as { rowid: number; embedding: Buffer }[];
       for (const row of vecData) {
         const id = rowidToId.get(Number(row.rowid));
         const buf = row.embedding;
@@ -320,15 +238,14 @@ export async function hybridSearch(
           );
         }
       }
-    } catch (_) {
-      // Embeddings unavailable — near-dup suppression skipped
+    } catch {
+      // Embeddings unavailable
     }
   }
 
-  // Stage 5: Near-duplicate suppression (cosine similarity > 0.92 threshold)
   if (queryVec && embeddingMap.size > 0) {
-    const selected = [];
-    const selectedVecs = [];
+    const selected: SearchResult[] = [];
+    const selectedVecs: Float32Array[] = [];
     for (const candidate of candidates) {
       if (selected.length >= offset + limit) break;
       const vec = embeddingMap.get(candidate.id);
@@ -344,30 +261,25 @@ export async function hybridSearch(
       if (vec) selectedVecs.push(vec);
     }
     const dedupedPage = selected.slice(offset, offset + limit);
-    trackAccess(ctx.db, dedupedPage);
+    trackAccess(ctx, dedupedPage);
     return dedupedPage;
   }
 
   const finalPage = candidates.slice(offset, offset + limit);
-  trackAccess(ctx.db, finalPage);
+  trackAccess(ctx, finalPage);
   return finalPage;
 }
 
-/**
- * Increment hit_count and set last_accessed_at for a batch of retrieved entries.
- * Single batched UPDATE for efficiency.
- *
- * @param {import('node:sqlite').DatabaseSync} db
- * @param {Array<{id: string}>} entries
- */
-function trackAccess(db, entries) {
+function trackAccess(ctx: BaseCtx, entries: SearchResult[]): void {
   if (!entries.length) return;
   try {
     const placeholders = entries.map(() => "?").join(",");
-    db.prepare(
-      `UPDATE vault SET hit_count = hit_count + 1, last_accessed_at = datetime('now') WHERE id IN (${placeholders})`,
-    ).run(...entries.map((e) => e.id));
-  } catch (_) {
-    // Non-fatal: frequency tracking is best-effort
+    ctx.db
+      .prepare(
+        `UPDATE vault SET hit_count = hit_count + 1, last_accessed_at = datetime('now') WHERE id IN (${placeholders})`,
+      )
+      .run(...entries.map((e) => e.id));
+  } catch {
+    // Non-fatal
   }
 }

package/src/types.ts

@@ -0,0 +1,166 @@
+import type { DatabaseSync, StatementSync } from "node:sqlite";
+
+export interface VaultConfig {
+  vaultDir: string;
+  dataDir: string;
+  dbPath: string;
+  devDir: string;
+  eventDecayDays: number;
+  thresholds: GrowthThresholds;
+  telemetry: boolean;
+  resolvedFrom: string;
+  configPath?: string;
+  vaultDirExists?: boolean;
+  recall: RecallConfig;
+  consolidation: ConsolidationConfig;
+  lifecycle: Record<string, { archiveAfterDays?: number }>;
+}
+
+export interface RecallConfig {
+  maxResults: number;
+  maxOutputBytes: number;
+  minRelevanceScore: number;
+  excludeKinds: string[];
+  excludeCategories: string[];
+  bodyTruncateChars: number;
+}
+
+export interface ConsolidationConfig {
+  tagThreshold: number;
+  maxAgeDays: number;
+  autoConsolidate: boolean;
+}
+
+export interface GrowthThresholds {
+  totalEntries: { warn: number; critical: number };
+  eventEntries: { warn: number; critical: number };
+  vaultSizeBytes: { warn: number; critical: number };
+  eventsWithoutTtl: { warn: number };
+}
+
+export interface PreparedStatements {
+  insertEntry: StatementSync;
+  updateEntry: StatementSync;
+  deleteEntry: StatementSync;
+  getRowid: StatementSync;
+  getRowidByPath: StatementSync;
+  getEntryById: StatementSync;
+  getByIdentityKey: StatementSync;
+  upsertByIdentityKey: StatementSync;
+  updateSourceFiles: StatementSync;
+  updateRelatedTo: StatementSync;
+  insertVecStmt: StatementSync;
+  deleteVecStmt: StatementSync;
+  updateSupersededBy: StatementSync;
+  clearSupersededByRef: StatementSync;
+}
+
+export interface VaultEntry {
+  id: string;
+  kind: string;
+  category: string;
+  title: string | null;
+  body: string;
+  meta: string | null;
+  tags: string | null;
+  source: string | null;
+  file_path: string | null;
+  identity_key: string | null;
+  expires_at: string | null;
+  superseded_by: string | null;
+  created_at: string;
+  updated_at: string | null;
+  hit_count: number;
+  last_accessed_at: string | null;
+  source_files: string | null;
+  tier: string;
+  related_to: string | null;
+  rowid?: number;
+}
+
+export interface SearchResult extends VaultEntry {
+  score: number;
+  stale?: boolean;
+  stale_reason?: string;
+}
+
+export interface CaptureInput {
+  kind: string;
+  title?: string | null;
+  body: string;
+  meta?: Record<string, unknown> | null;
+  tags?: string[] | null;
+  source?: string | null;
+  folder?: string | null;
+  identity_key?: string | null;
+  expires_at?: string | null;
+  supersedes?: string[] | null;
+  related_to?: string[] | null;
+  source_files?: Array<{ path: string; hash: string }> | null;
+  tier?: string | null;
+}
+
+export interface CaptureResult {
+  id: string;
+  filePath: string;
+  kind: string;
+  category: string;
+  title: string | null;
+  body: string;
+  meta: Record<string, unknown> | undefined;
+  tags: string[] | null;
+  source: string | null;
+  createdAt: string;
+  updatedAt: string;
+  identity_key: string | null;
+  expires_at: string | null;
+  supersedes: string[] | null;
+  related_to: string[] | null;
+  source_files: Array<{ path: string; hash: string }> | null;
+  tier: string | null;
+}
+
+export interface IndexEntryInput {
+  id: string;
+  kind: string;
+  category: string;
+  title: string | null;
+  body: string;
+  meta: Record<string, unknown> | undefined;
+  tags: string[] | null;
+  source: string | null;
+  filePath: string;
+  createdAt: string;
+  identity_key: string | null;
+  expires_at: string | null;
+  source_files: Array<{ path: string; hash: string }> | null;
+  tier: string | null;
+}
+
+export interface ReindexStats {
+  added: number;
+  updated: number;
+  removed: number;
+  unchanged: number;
+}
+
+export interface BaseCtx {
+  db: DatabaseSync;
+  config: VaultConfig;
+  stmts: PreparedStatements;
+  embed: (text: string) => Promise<Float32Array | null>;
+  insertVec: (rowid: number, embedding: Float32Array) => void;
+  deleteVec: (rowid: number) => void;
+}
+
+export interface SearchOptions {
+  kindFilter?: string | null;
+  categoryFilter?: string | null;
+  excludeEvents?: boolean;
+  since?: string | null;
+  until?: string | null;
+  limit?: number;
+  offset?: number;
+  decayDays?: number;
+  includeSuperseeded?: boolean;
+}

package/src/capture/file-ops.js

@@ -1,99 +0,0 @@
-/**
- * file-ops.js — Capture-specific file operations
- *
- * Writes markdown entry files with frontmatter to the vault directory.
- */
-
-import { mkdirSync, writeFileSync } from "node:fs";
-import { resolve, relative } from "node:path";
-import { formatFrontmatter } from "../core/frontmatter.js";
-import { slugify, kindToPath } from "../core/files.js";
-import { formatBody } from "./formatters.js";
-
-export function safeFolderPath(vaultDir, kind, folder) {
-  const base = resolve(vaultDir, kindToPath(kind));
-  if (!folder) return base;
-  const resolved = resolve(base, folder);
-  const rel = relative(base, resolved);
-  if (rel.startsWith("..") || resolve(base, rel) !== resolved) {
-    throw new Error(`Folder path escapes vault: "${folder}"`);
-  }
-  return resolved;
-}
-
-export function writeEntryFile(
-  vaultDir,
-  kind,
-  {
-    id,
-    title,
-    body,
-    meta,
-    tags,
-    source,
-    createdAt,
-    updatedAt,
-    folder,
-    category,
-    identity_key,
-    expires_at,
-    supersedes,
-    related_to,
-  },
-) {
-  // P5: folder is now a top-level param; also accept from meta for backward compat
-  const resolvedFolder = folder || meta?.folder || "";
-  const dir = safeFolderPath(vaultDir, kind, resolvedFolder);
-
-  try {
-    mkdirSync(dir, { recursive: true });
-  } catch (e) {
-    throw new Error(`Failed to create directory "${dir}": ${e.message}`);
-  }
-
-  const created = createdAt || new Date().toISOString();
-  const fmFields = { id };
-
-  // Add kind-specific meta fields to frontmatter (flattened, not nested)
-  if (meta) {
-    for (const [k, v] of Object.entries(meta)) {
-      if (k === "folder") continue;
-      if (v !== null && v !== undefined) fmFields[k] = v;
-    }
-  }
-
-  if (identity_key) fmFields.identity_key = identity_key;
-  if (expires_at) fmFields.expires_at = expires_at;
-  if (supersedes?.length) fmFields.supersedes = supersedes;
-  if (related_to?.length) fmFields.related_to = related_to;
-  fmFields.tags = tags || [];
-  fmFields.source = source || "claude-code";
-  fmFields.created = created;
-  if (updatedAt && updatedAt !== created) fmFields.updated = updatedAt;
-
-  const mdBody = formatBody(kind, { title, body, meta });
-
-  // Entity kinds: deterministic filename from identity_key (no ULID suffix)
-  let filename;
-  if (category === "entity" && identity_key) {
-    const identitySlug = slugify(identity_key);
-    filename = identitySlug
-      ? `${identitySlug}.md`
-      : `${id.slice(-8).toLowerCase()}.md`;
-  } else {
-    const slug = slugify((title || body).slice(0, 40));
-    const shortId = id.slice(-8).toLowerCase();
-    filename = slug ? `${slug}-${shortId}.md` : `${shortId}.md`;
-  }
-
-  const filePath = resolve(dir, filename);
-  const md = formatFrontmatter(fmFields) + mdBody;
-
-  try {
-    writeFileSync(filePath, md);
-  } catch (e) {
-    throw new Error(`Failed to write entry file "${filePath}": ${e.message}`);
-  }
-
-  return filePath;
-}

package/src/capture/import-pipeline.js

@@ -1,46 +0,0 @@
-import { captureAndIndex } from "./index.js";
-
-export async function importEntries(ctx, entries, opts = {}) {
-  const { onProgress, source } = opts;
-  let imported = 0;
-  let failed = 0;
-  const errors = [];
-
-  for (let i = 0; i < entries.length; i++) {
-    const entry = entries[i];
-
-    if (onProgress) {
-      onProgress(i + 1, entries.length);
-    }
-
-    try {
-      if (!entry.body?.trim()) {
-        failed++;
-        errors.push({ index: i, title: entry.title, error: "Empty body" });
-        continue;
-      }
-
-      await captureAndIndex(ctx, {
-        kind: entry.kind || "insight",
-        title: entry.title || null,
-        body: entry.body,
-        meta: entry.meta,
-        tags: entry.tags,
-        source: entry.source || source || "import",
-        identity_key: entry.identity_key,
-        expires_at: entry.expires_at,
-        userId: ctx.userId || null,
-      });
-      imported++;
-    } catch (err) {
-      failed++;
-      errors.push({
-        index: i,
-        title: entry.title || null,
-        error: err.message,
-      });
-    }
-  }
-
-  return { imported, failed, errors };
-}