@context-vault/core 2.8.3

package/src/retrieve/index.js

@@ -0,0 +1,219 @@
+/**
+ * Retrieve Layer — Public API
+ *
+ * All read-path query logic: hybrid semantic search and any future
+ * query patterns (scoped, recency-weighted, etc.).
+ *
+ * Agent Constraint: Read-only access to DB. Never writes.
+ */
+
+const FTS_WEIGHT = 0.4;
+const VEC_WEIGHT = 0.6;
+
+/**
+ * Strip FTS5 metacharacters from query words and build an AND query.
+ * Returns null if no valid words remain.
+ */
+export function buildFtsQuery(query) {
+  const words = query
+    .split(/\s+/)
+    .map((w) => w.replace(/[*"()\-:^~{}]/g, ""))
+    .filter((w) => w.length > 0);
+  if (!words.length) return null;
+  return words.map((w) => `"${w}"`).join(" AND ");
+}
+
+/**
+ * Category-aware recency decay:
+ *   knowledge + entity: no decay (enduring)
+ *   event: steeper decay (~0.5 at 30 days)
+ */
+export function recencyBoost(createdAt, category, decayDays = 30) {
+  if (category !== "event") return 1.0;
+  const ageDays = (Date.now() - new Date(createdAt).getTime()) / 86400000;
+  return 1 / (1 + ageDays / decayDays);
+}
+
+/**
+ * Build additional WHERE clauses for category/time filtering.
+ * Returns { clauses: string[], params: any[] }
+ */
+export function buildFilterClauses({
+  categoryFilter,
+  since,
+  until,
+  userIdFilter,
+  teamIdFilter,
+}) {
+  const clauses = [];
+  const params = [];
+  if (userIdFilter !== undefined) {
+    clauses.push("e.user_id = ?");
+    params.push(userIdFilter);
+  }
+  if (teamIdFilter) {
+    clauses.push("e.team_id = ?");
+    params.push(teamIdFilter);
+  }
+  if (categoryFilter) {
+    clauses.push("e.category = ?");
+    params.push(categoryFilter);
+  }
+  if (since) {
+    clauses.push("e.created_at >= ?");
+    params.push(since);
+  }
+  if (until) {
+    clauses.push("e.created_at <= ?");
+    params.push(until);
+  }
+  clauses.push("(e.expires_at IS NULL OR e.expires_at > datetime('now'))");
+  return { clauses, params };
+}
+
+/**
+ * Hybrid search combining FTS5 text matching and vector similarity.
+ *
+ * @param {import('../server/types.js').BaseCtx} ctx
+ * @param {string} query
+ * @param {{ kindFilter?: string|null, categoryFilter?: string|null, since?: string|null, until?: string|null, limit?: number, offset?: number }} opts
+ * @returns {Promise<Array<{id, kind, category, title, body, meta, tags, source, file_path, created_at, score}>>}
+ */
+export async function hybridSearch(
+  ctx,
+  query,
+  {
+    kindFilter = null,
+    categoryFilter = null,
+    since = null,
+    until = null,
+    limit = 20,
+    offset = 0,
+    decayDays = 30,
+    userIdFilter,
+    teamIdFilter = null,
+  } = {},
+) {
+  const results = new Map();
+  const extraFilters = buildFilterClauses({
+    categoryFilter,
+    since,
+    until,
+    userIdFilter,
+    teamIdFilter,
+  });
+
+  // FTS5 search
+  const ftsQuery = buildFtsQuery(query);
+  if (ftsQuery) {
+    try {
+      const whereParts = ["vault_fts MATCH ?"];
+      const ftsParams = [ftsQuery];
+
+      if (kindFilter) {
+        whereParts.push("e.kind = ?");
+        ftsParams.push(kindFilter);
+      }
+      whereParts.push(...extraFilters.clauses);
+      ftsParams.push(...extraFilters.params);
+
+      const ftsSQL = `SELECT e.*, rank FROM vault_fts f JOIN vault e ON f.rowid = e.rowid WHERE ${whereParts.join(" AND ")} ORDER BY rank LIMIT 15`;
+      const rows = ctx.db.prepare(ftsSQL).all(...ftsParams);
+
+      // Normalize FTS scores to [0, 1]
+      const ftsScores = rows.map((r) => Math.abs(r.rank || 0));
+      const maxFts = Math.max(...ftsScores, 1);
+
+      for (let i = 0; i < rows.length; i++) {
+        const { rank: _rank, ...row } = rows[i];
+        const normalized = ftsScores[i] / maxFts;
+        results.set(row.id, { ...row, score: normalized * FTS_WEIGHT });
+      }
+    } catch (err) {
+      if (err.message?.includes("fts5: syntax error")) {
+        // Expected: malformed query, fall through to vector search
+      } else {
+        console.error(`[retrieve] FTS search error: ${err.message}`);
+      }
+    }
+  }
+
+  // Vector similarity search (skipped if embedding unavailable)
+  try {
+    const vecCount = ctx.db
+      .prepare("SELECT COUNT(*) as c FROM vault_vec")
+      .get().c;
+    if (vecCount > 0) {
+      const queryVec = await ctx.embed(query);
+      if (queryVec) {
+        // Increase limits in hosted mode to compensate for post-filtering
+        const hasPostFilter = userIdFilter !== undefined || teamIdFilter;
+        const vecLimit = hasPostFilter
+          ? kindFilter
+            ? 60
+            : 30
+          : kindFilter
+            ? 30
+            : 15;
+        const vecRows = ctx.db
+          .prepare(
+            `SELECT v.rowid, v.distance FROM vault_vec v WHERE embedding MATCH ? ORDER BY distance LIMIT ${vecLimit}`,
+          )
+          .all(queryVec);
+
+        if (vecRows.length) {
+          // Batch hydration: single query instead of N+1
+          const rowids = vecRows.map((vr) => vr.rowid);
+          const placeholders = rowids.map(() => "?").join(",");
+          const hydrated = ctx.db
+            .prepare(
+              `SELECT rowid, * FROM vault WHERE rowid IN (${placeholders})`,
+            )
+            .all(...rowids);
+
+          const byRowid = new Map();
+          for (const row of hydrated) byRowid.set(row.rowid, row);
+
+          for (const vr of vecRows) {
+            const row = byRowid.get(vr.rowid);
+            if (!row) continue;
+            if (userIdFilter !== undefined && row.user_id !== userIdFilter)
+              continue;
+            if (teamIdFilter && row.team_id !== teamIdFilter) continue;
+            if (kindFilter && row.kind !== kindFilter) continue;
+            if (categoryFilter && row.category !== categoryFilter) continue;
+            if (since && row.created_at < since) continue;
+            if (until && row.created_at > until) continue;
+            if (row.expires_at && new Date(row.expires_at) <= new Date())
+              continue;
+
+            const { rowid: _rowid, ...cleanRow } = row;
+            // sqlite-vec returns L2 distance [0, 2] for normalized vectors.
+            // Convert to similarity [1, 0] with: 1 - distance/2
+            const vecScore = Math.max(0, 1 - vr.distance / 2) * VEC_WEIGHT;
+            const existing = results.get(cleanRow.id);
+            if (existing) {
+              existing.score += vecScore;
+            } else {
+              results.set(cleanRow.id, { ...cleanRow, score: vecScore });
+            }
+          }
+        }
+      }
+    }
+  } catch (err) {
+    if (err.message?.includes("no such table")) {
+      // Expected on fresh vaults with no vec table yet
+    } else {
+      console.error(`[retrieve] Vector search error: ${err.message}`);
+    }
+  }
+
+  // Apply category-aware recency boost
+  for (const [, entry] of results) {
+    entry.score *= recencyBoost(entry.created_at, entry.category, decayDays);
+  }
+
+  const sorted = [...results.values()].sort((a, b) => b.score - a.score);
+  return sorted.slice(offset, offset + limit);
+}

package/src/server/helpers.js

@@ -0,0 +1,31 @@
+/**
+ * helpers.js — Shared MCP response helpers and validation
+ */
+
+export function ok(text) {
+  return { content: [{ type: "text", text }] };
+}
+
+export function err(text, code = "UNKNOWN") {
+  return { content: [{ type: "text", text }], isError: true, code };
+}
+
+export function ensureVaultExists(config) {
+  if (!config.vaultDirExists) {
+    return err(
+      `Vault directory not found: ${config.vaultDir}. Run context_status for diagnostics.`,
+      "VAULT_NOT_FOUND",
+    );
+  }
+  return null;
+}
+
+export function ensureValidKind(kind) {
+  if (!/^[a-z][a-z0-9_-]*$/.test(kind)) {
+    return err(
+      "Required: kind (lowercase alphanumeric, e.g. 'insight', 'reference')",
+      "INVALID_KIND",
+    );
+  }
+  return null;
+}

package/src/server/tools/context-status.js

@@ -0,0 +1,104 @@
+import { gatherVaultStatus } from "../../core/status.js";
+import { ok } from "../helpers.js";
+
+export const name = "context_status";
+
+export const description =
+  "Show vault health: resolved config, file counts per kind, database size, and any issues. Use to verify setup or troubleshoot. Call this when a user asks about their vault or to debug search issues.";
+
+export const inputSchema = {};
+
+/**
+ * @param {object} _args
+ * @param {import('../types.js').BaseCtx & Partial<import('../types.js').HostedCtxExtensions>} ctx
+ */
+export function handler(_args, ctx) {
+  const { config } = ctx;
+  const userId = ctx.userId !== undefined ? ctx.userId : undefined;
+
+  const status = gatherVaultStatus(ctx, { userId });
+
+  const hasIssues = status.stalePaths || status.embeddingStatus?.missing > 0;
+  const healthIcon = hasIssues ? "⚠" : "✓";
+
+  const lines = [
+    `## ${healthIcon} Vault Status (connected)`,
+    ``,
+    `Vault:     ${config.vaultDir} (${config.vaultDirExists ? status.fileCount + " files" : "missing"})`,
+    `Database:  ${config.dbPath} (${status.dbSize})`,
+    `Dev dir:   ${config.devDir}`,
+    `Data dir:  ${config.dataDir}`,
+    `Config:    ${config.configPath}`,
+    `Resolved via: ${status.resolvedFrom}`,
+    `Schema:    v7 (teams)`,
+  ];
+
+  if (status.embeddingStatus) {
+    const { indexed, total, missing } = status.embeddingStatus;
+    const pct = total > 0 ? Math.round((indexed / total) * 100) : 100;
+    lines.push(`Embeddings: ${indexed}/${total} (${pct}%)`);
+  }
+  if (status.embedModelAvailable === false) {
+    lines.push(
+      `Embed model: unavailable (semantic search disabled, FTS still works)`,
+    );
+  } else if (status.embedModelAvailable === true) {
+    lines.push(`Embed model: loaded`);
+  }
+  lines.push(`Decay:     ${config.eventDecayDays} days (event recency window)`);
+  if (status.expiredCount > 0) {
+    lines.push(
+      `Expired:   ${status.expiredCount} entries (pruned on next reindex)`,
+    );
+  }
+
+  lines.push(``, `### Indexed`);
+
+  if (status.kindCounts.length) {
+    for (const { kind, c } of status.kindCounts) lines.push(`- ${c} ${kind}s`);
+  } else {
+    lines.push(`- (empty)`);
+  }
+
+  if (status.categoryCounts.length) {
+    lines.push(``);
+    lines.push(`### Categories`);
+    for (const { category, c } of status.categoryCounts)
+      lines.push(`- ${category}: ${c}`);
+  }
+
+  if (status.subdirs.length) {
+    lines.push(``);
+    lines.push(`### Disk Directories`);
+    for (const { name, count } of status.subdirs)
+      lines.push(`- ${name}/: ${count} files`);
+  }
+
+  if (status.stalePaths) {
+    lines.push(``);
+    lines.push(`### ⚠ Stale Paths`);
+    lines.push(
+      `DB contains ${status.staleCount} paths not matching current vault dir.`,
+    );
+    lines.push(`Auto-reindex will fix this on next search or save.`);
+  }
+
+  // Suggested actions
+  const actions = [];
+  if (status.stalePaths)
+    actions.push("- Run `context-vault reindex` to fix stale paths");
+  if (status.embeddingStatus?.missing > 0)
+    actions.push(
+      "- Run `context-vault reindex` to generate missing embeddings",
+    );
+  if (!config.vaultDirExists)
+    actions.push("- Run `context-vault setup` to create the vault directory");
+  if (status.kindCounts.length === 0 && config.vaultDirExists)
+    actions.push("- Use `save_context` to add your first entry");
+
+  if (actions.length) {
+    lines.push("", "### Suggested Actions", ...actions);
+  }
+
+  return ok(lines.join("\n"));
+}

package/src/server/tools/delete-context.js

@@ -0,0 +1,53 @@
+import { z } from "zod";
+import { unlinkSync } from "node:fs";
+import { ok, err } from "../helpers.js";
+
+export const name = "delete_context";
+
+export const description =
+  "Delete an entry from your vault by its ULID id. Removes the file from disk and cleans up the search index.";
+
+export const inputSchema = {
+  id: z.string().describe("The entry ULID to delete"),
+};
+
+/**
+ * @param {object} args
+ * @param {import('../types.js').BaseCtx & Partial<import('../types.js').HostedCtxExtensions>} ctx
+ * @param {import('../types.js').ToolShared} shared
+ */
+export async function handler({ id }, ctx, { ensureIndexed }) {
+  const userId = ctx.userId !== undefined ? ctx.userId : undefined;
+
+  if (!id?.trim())
+    return err("Required: id (non-empty string)", "INVALID_INPUT");
+  await ensureIndexed();
+
+  const entry = ctx.stmts.getEntryById.get(id);
+  if (!entry) return err(`Entry not found: ${id}`, "NOT_FOUND");
+
+  // Ownership check: don't leak existence across users
+  if (userId !== undefined && entry.user_id !== userId) {
+    return err(`Entry not found: ${id}`, "NOT_FOUND");
+  }
+
+  // Delete file from disk first (source of truth)
+  if (entry.file_path) {
+    try {
+      unlinkSync(entry.file_path);
+    } catch {}
+  }
+
+  // Delete vector embedding
+  const rowidResult = ctx.stmts.getRowid.get(id);
+  if (rowidResult?.rowid) {
+    try {
+      ctx.deleteVec(Number(rowidResult.rowid));
+    } catch {}
+  }
+
+  // Delete DB row (FTS trigger handles FTS cleanup)
+  ctx.stmts.deleteEntry.run(id);
+
+  return ok(`Deleted ${entry.kind}: ${entry.title || "(untitled)"} [${id}]`);
+}

package/src/server/tools/get-context.js

@@ -0,0 +1,235 @@
+import { z } from "zod";
+import { hybridSearch } from "../../retrieve/index.js";
+import { categoryFor } from "../../core/categories.js";
+import { normalizeKind } from "../../core/files.js";
+import { ok, err } from "../helpers.js";
+import { isEmbedAvailable } from "../../index/embed.js";
+
+export const name = "get_context";
+
+export const description =
+  "Search your knowledge vault. Returns entries ranked by relevance using hybrid full-text + semantic search. Use this to find insights, decisions, patterns, or any saved context. Each result includes an `id` you can use with save_context or delete_context.";
+
+export const inputSchema = {
+  query: z
+    .string()
+    .optional()
+    .describe(
+      "Search query (natural language or keywords). Optional if filters (tags, kind, category) are provided.",
+    ),
+  kind: z
+    .string()
+    .optional()
+    .describe("Filter by kind (e.g. 'insight', 'decision', 'pattern')"),
+  category: z
+    .enum(["knowledge", "entity", "event"])
+    .optional()
+    .describe("Filter by category"),
+  identity_key: z
+    .string()
+    .optional()
+    .describe("For entity lookup: exact match on identity key. Requires kind."),
+  tags: z
+    .array(z.string())
+    .optional()
+    .describe("Filter by tags (entries must match at least one)"),
+  since: z
+    .string()
+    .optional()
+    .describe("ISO date, return entries created after this"),
+  until: z
+    .string()
+    .optional()
+    .describe("ISO date, return entries created before this"),
+  limit: z.number().optional().describe("Max results to return (default 10)"),
+};
+
+/**
+ * @param {object} args
+ * @param {import('../types.js').BaseCtx & Partial<import('../types.js').HostedCtxExtensions>} ctx
+ * @param {import('../types.js').ToolShared} shared
+ */
+export async function handler(
+  { query, kind, category, identity_key, tags, since, until, limit },
+  ctx,
+  { ensureIndexed, reindexFailed },
+) {
+  const { config } = ctx;
+  const userId = ctx.userId !== undefined ? ctx.userId : undefined;
+
+  const hasQuery = query?.trim();
+  const hasFilters =
+    kind || category || tags?.length || since || until || identity_key;
+  if (!hasQuery && !hasFilters)
+    return err(
+      "Required: query or at least one filter (kind, category, tags, since, until, identity_key)",
+      "INVALID_INPUT",
+    );
+  await ensureIndexed();
+
+  const kindFilter = kind ? normalizeKind(kind) : null;
+
+  // Gap 1: Entity exact-match by identity_key
+  if (identity_key) {
+    if (!kindFilter)
+      return err("identity_key requires kind to be specified", "INVALID_INPUT");
+    const match = ctx.stmts.getByIdentityKey.get(
+      kindFilter,
+      identity_key,
+      userId !== undefined ? userId : null,
+    );
+    if (match) {
+      const entryTags = match.tags ? JSON.parse(match.tags) : [];
+      const tagStr = entryTags.length ? entryTags.join(", ") : "none";
+      const relPath =
+        match.file_path && config.vaultDir
+          ? match.file_path.replace(config.vaultDir + "/", "")
+          : match.file_path || "n/a";
+      const lines = [
+        `## Entity Match (exact)\n`,
+        `### ${match.title || "(untitled)"} [${match.kind}/${match.category}]`,
+        `1.000 · ${tagStr} · ${relPath} · id: \`${match.id}\``,
+        match.body?.slice(0, 300) + (match.body?.length > 300 ? "..." : ""),
+      ];
+      return ok(lines.join("\n"));
+    }
+    // Fall through to semantic search as fallback
+  }
+
+  // Gap 2: Event default time-window
+  const effectiveCategory =
+    category || (kindFilter ? categoryFor(kindFilter) : null);
+  let effectiveSince = since || null;
+  let effectiveUntil = until || null;
+  let autoWindowed = false;
+  if (effectiveCategory === "event" && !since && !until) {
+    const decayMs = (config.eventDecayDays || 30) * 86400000;
+    effectiveSince = new Date(Date.now() - decayMs).toISOString();
+    autoWindowed = true;
+  }
+
+  const effectiveLimit = limit || 10;
+  // When tag-filtering, over-fetch to compensate for post-filter reduction
+  const fetchLimit = tags?.length ? effectiveLimit * 10 : effectiveLimit;
+
+  let filtered;
+  if (hasQuery) {
+    // Hybrid search mode
+    const sorted = await hybridSearch(ctx, query, {
+      kindFilter,
+      categoryFilter: category || null,
+      since: effectiveSince,
+      until: effectiveUntil,
+      limit: fetchLimit,
+      decayDays: config.eventDecayDays || 30,
+      userIdFilter: userId,
+    });
+
+    // Post-filter by tags if provided, then apply requested limit
+    filtered = tags?.length
+      ? sorted
+          .filter((r) => {
+            const entryTags = r.tags ? JSON.parse(r.tags) : [];
+            return tags.some((t) => entryTags.includes(t));
+          })
+          .slice(0, effectiveLimit)
+      : sorted;
+  } else {
+    // Filter-only mode (no query, use SQL directly)
+    const clauses = [];
+    const params = [];
+    if (userId !== undefined) {
+      clauses.push("user_id = ?");
+      params.push(userId);
+    }
+    if (kindFilter) {
+      clauses.push("kind = ?");
+      params.push(kindFilter);
+    }
+    if (category) {
+      clauses.push("category = ?");
+      params.push(category);
+    }
+    if (effectiveSince) {
+      clauses.push("created_at >= ?");
+      params.push(effectiveSince);
+    }
+    if (effectiveUntil) {
+      clauses.push("created_at <= ?");
+      params.push(effectiveUntil);
+    }
+    clauses.push("(expires_at IS NULL OR expires_at > datetime('now'))");
+    const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
+    params.push(fetchLimit);
+    const rows = ctx.db
+      .prepare(`SELECT * FROM vault ${where} ORDER BY created_at DESC LIMIT ?`)
+      .all(...params);
+
+    // Post-filter by tags if provided, then apply requested limit
+    filtered = tags?.length
+      ? rows
+          .filter((r) => {
+            const entryTags = r.tags ? JSON.parse(r.tags) : [];
+            return tags.some((t) => entryTags.includes(t));
+          })
+          .slice(0, effectiveLimit)
+      : rows;
+
+    // Add score field for consistent output
+    for (const r of filtered) r.score = 0;
+  }
+
+  if (!filtered.length)
+    return ok(
+      hasQuery
+        ? "No results found for: " + query
+        : "No entries found matching the given filters.",
+    );
+
+  // Decrypt encrypted entries if ctx.decrypt is available
+  if (ctx.decrypt) {
+    for (const r of filtered) {
+      if (r.body_encrypted) {
+        const decrypted = await ctx.decrypt(r);
+        r.body = decrypted.body;
+        if (decrypted.title) r.title = decrypted.title;
+        if (decrypted.meta) r.meta = JSON.stringify(decrypted.meta);
+      }
+    }
+  }
+
+  const lines = [];
+  if (reindexFailed)
+    lines.push(
+      `> **Warning:** Auto-reindex failed. Results may be stale. Run \`context-vault reindex\` to fix.\n`,
+    );
+  if (hasQuery && isEmbedAvailable() === false)
+    lines.push(
+      `> **Note:** Semantic search unavailable — results ranked by keyword match only. Run \`context-vault setup\` to download the embedding model.\n`,
+    );
+  const heading = hasQuery ? `Results for "${query}"` : "Filtered entries";
+  lines.push(`## ${heading} (${filtered.length} matches)\n`);
+  for (let i = 0; i < filtered.length; i++) {
+    const r = filtered[i];
+    const entryTags = r.tags ? JSON.parse(r.tags) : [];
+    const tagStr = entryTags.length ? entryTags.join(", ") : "none";
+    const relPath =
+      r.file_path && config.vaultDir
+        ? r.file_path.replace(config.vaultDir + "/", "")
+        : r.file_path || "n/a";
+    lines.push(
+      `### [${i + 1}/${filtered.length}] ${r.title || "(untitled)"} [${r.kind}/${r.category}]`,
+    );
+    lines.push(
+      `${r.score.toFixed(3)} · ${tagStr} · ${relPath} · id: \`${r.id}\``,
+    );
+    lines.push(r.body?.slice(0, 300) + (r.body?.length > 300 ? "..." : ""));
+    lines.push("");
+  }
+  if (autoWindowed) {
+    lines.push(
+      `_Showing events from last ${config.eventDecayDays || 30} days. Use since/until for custom range._`,
+    );
+  }
+  return ok(lines.join("\n"));
+}