@context-vault/core 2.13.0 → 2.15.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@context-vault/core",
-  "version": "2.13.0",
+  "version": "2.15.0",
   "type": "module",
   "description": "Shared core: capture, index, retrieve, tools, and utilities for context-vault",
   "main": "src/index.js",
@@ -36,6 +36,7 @@
     "access": "public"
   },
   "dependencies": {
+    "@anthropic-ai/sdk": "^0.78.0",
     "@modelcontextprotocol/sdk": "^1.26.0",
     "sqlite-vec": "^0.1.0"
   },

package/src/capture/index.js

@@ -27,6 +27,8 @@ export function writeEntry(
     identity_key,
     expires_at,
     supersedes,
+    source_files,
+    tier,
     userId,
   },
 ) {
@@ -103,6 +105,8 @@ export function writeEntry(
     identity_key,
     expires_at,
     supersedes,
+    source_files: source_files || null,
+    tier: tier || null,
     userId: userId || null,
   };
 }
@@ -134,6 +138,12 @@ export function updateEntryFile(ctx, existing, updates) {
     updates.supersedes !== undefined
       ? updates.supersedes
       : fmMeta.supersedes || null;
+  const source_files =
+    updates.source_files !== undefined
+      ? updates.source_files
+      : existing.source_files
+        ? JSON.parse(existing.source_files)
+        : null;
 
   let mergedMeta;
   if (updates.meta !== undefined) {
@@ -179,6 +189,7 @@ export function updateEntryFile(ctx, existing, updates) {
     identity_key: existing.identity_key,
     expires_at,
     supersedes,
+    source_files: source_files || null,
     userId: existing.user_id || null,
   };
 }

package/src/consolidation/index.js

@@ -0,0 +1,112 @@
+/**
+ * Consolidation utilities — identifies tags and entries that warrant maintenance.
+ *
+ * These are pure DB queries with no LLM calls. The caller decides what to do
+ * with the results (e.g. run create_snapshot, archive entries, report to user).
+ */
+
+/**
+ * Identifies tags that have accumulated enough entries to warrant consolidation.
+ *
+ * A tag is "hot" when it has >= tagThreshold non-superseded entries AND no
+ * brief/snapshot was saved for it within the last maxSnapshotAgeDays days.
+ *
+ * @param {import('node:sqlite').DatabaseSync} db
+ * @param {{ tagThreshold?: number, maxSnapshotAgeDays?: number }} [opts]
+ * @returns {{ tag: string, entryCount: number, lastSnapshotAge: number | null }[]}
+ */
+export function findHotTags(
+  db,
+  { tagThreshold = 10, maxSnapshotAgeDays = 7 } = {},
+) {
+  const rows = db
+    .prepare(
+      `SELECT id, tags, kind FROM vault
+       WHERE superseded_by IS NULL
+         AND tags IS NOT NULL
+         AND tags != '[]'`,
+    )
+    .all();
+
+  const tagCounts = new Map();
+
+  for (const row of rows) {
+    let tags;
+    try {
+      tags = JSON.parse(row.tags);
+    } catch {
+      continue;
+    }
+    if (!Array.isArray(tags)) continue;
+
+    for (const tag of tags) {
+      if (typeof tag !== "string" || !tag) continue;
+      tagCounts.set(tag, (tagCounts.get(tag) ?? 0) + 1);
+    }
+  }
+
+  const hotTags = [];
+
+  for (const [tag, count] of tagCounts) {
+    if (count < tagThreshold) continue;
+
+    const snapshotRow = db
+      .prepare(
+        `SELECT created_at FROM vault
+         WHERE kind = 'brief'
+           AND tags LIKE ?
+           AND created_at > datetime('now', '-' || ? || ' days')
+         ORDER BY created_at DESC
+         LIMIT 1`,
+      )
+      .get(`%"${tag}"%`, String(maxSnapshotAgeDays));
+
+    if (snapshotRow) continue;
+
+    const lastSnapshotAny = db
+      .prepare(
+        `SELECT created_at FROM vault
+         WHERE kind = 'brief'
+           AND tags LIKE ?
+         ORDER BY created_at DESC
+         LIMIT 1`,
+      )
+      .get(`%"${tag}"%`);
+
+    let lastSnapshotAge = null;
+    if (lastSnapshotAny) {
+      const ms = Date.now() - new Date(lastSnapshotAny.created_at).getTime();
+      lastSnapshotAge = Math.floor(ms / (1000 * 60 * 60 * 24));
+    }
+
+    hotTags.push({ tag, entryCount: count, lastSnapshotAge });
+  }
+
+  hotTags.sort((a, b) => b.entryCount - a.entryCount);
+
+  return hotTags;
+}
+
+/**
+ * Identifies cold entries (old, never or rarely accessed) that can be archived.
+ *
+ * Returns IDs of entries that are old enough, have low hit counts, are not
+ * superseded, and are not in permanent kinds (decision, architecture, brief).
+ *
+ * @param {import('node:sqlite').DatabaseSync} db
+ * @param {{ maxAgeDays?: number, maxHitCount?: number }} [opts]
+ * @returns {string[]} Entry IDs eligible for archiving
+ */
+export function findColdEntries(db, { maxAgeDays = 90, maxHitCount = 0 } = {}) {
+  const rows = db
+    .prepare(
+      `SELECT id FROM vault
+       WHERE hit_count <= ?
+         AND created_at < datetime('now', '-' || ? || ' days')
+         AND superseded_by IS NULL
+         AND kind NOT IN ('decision', 'architecture', 'brief')`,
+    )
+    .all(maxHitCount, String(maxAgeDays));
+
+  return rows.map((r) => r.id);
+}

package/src/core/categories.js

@@ -21,6 +21,7 @@ const KIND_CATEGORY = {
   project: "entity",
   tool: "entity",
   source: "entity",
+  bucket: "entity",
   // Event — append-only, decaying
   conversation: "event",
   message: "event",
@@ -51,10 +52,19 @@ export const KIND_STALENESS_DAYS = {
   reference: 90,
 };
 
+const DURABLE_KINDS = new Set(["decision", "architecture", "pattern"]);
+const EPHEMERAL_KINDS = new Set(["session", "observation"]);
+
 export function categoryFor(kind) {
   return KIND_CATEGORY[kind] || "knowledge";
 }
 
+export function defaultTierFor(kind) {
+  if (DURABLE_KINDS.has(kind)) return "durable";
+  if (EPHEMERAL_KINDS.has(kind)) return "ephemeral";
+  return "working";
+}
+
 /** Returns the category directory name for a given kind (e.g. "insight" → "knowledge") */
 export function categoryDirFor(kind) {
   const cat = categoryFor(kind);

package/src/core/config.js

@@ -35,6 +35,19 @@ export function resolveConfig() {
     thresholds: { ...DEFAULT_GROWTH_THRESHOLDS },
     telemetry: false,
     resolvedFrom: "defaults",
+    recall: {
+      maxResults: 5,
+      maxOutputBytes: 2000,
+      minRelevanceScore: 0.3,
+      excludeKinds: [],
+      excludeCategories: ["event"],
+      bodyTruncateChars: 400,
+    },
+    consolidation: {
+      tagThreshold: 10,
+      maxAgeDays: 7,
+      autoConsolidate: false,
+    },
   };
 
   const configPath = join(dataDir, "config.json");
@@ -73,6 +86,30 @@ export function resolveConfig() {
           };
       }
       if (fc.telemetry != null) config.telemetry = fc.telemetry === true;
+      if (fc.recall && typeof fc.recall === "object") {
+        const r = fc.recall;
+        if (r.maxResults != null)
+          config.recall.maxResults = Number(r.maxResults);
+        if (r.maxOutputBytes != null)
+          config.recall.maxOutputBytes = Number(r.maxOutputBytes);
+        if (r.minRelevanceScore != null)
+          config.recall.minRelevanceScore = Number(r.minRelevanceScore);
+        if (Array.isArray(r.excludeKinds))
+          config.recall.excludeKinds = r.excludeKinds;
+        if (Array.isArray(r.excludeCategories))
+          config.recall.excludeCategories = r.excludeCategories;
+        if (r.bodyTruncateChars != null)
+          config.recall.bodyTruncateChars = Number(r.bodyTruncateChars);
+      }
+      if (fc.consolidation && typeof fc.consolidation === "object") {
+        const c = fc.consolidation;
+        if (c.tagThreshold != null)
+          config.consolidation.tagThreshold = Number(c.tagThreshold);
+        if (c.maxAgeDays != null)
+          config.consolidation.maxAgeDays = Number(c.maxAgeDays);
+        if (c.autoConsolidate != null)
+          config.consolidation.autoConsolidate = c.autoConsolidate === true;
+      }
       // Hosted account linking (Phase 4)
       if (fc.hostedUrl) config.hostedUrl = fc.hostedUrl;
       if (fc.apiKey) config.apiKey = fc.apiKey;

package/src/index/db.js

@@ -63,7 +63,11 @@ export const SCHEMA_DDL = `
     body_encrypted  BLOB,
     title_encrypted BLOB,
     meta_encrypted  BLOB,
-    iv              BLOB
+    iv              BLOB,
+    hit_count       INTEGER DEFAULT 0,
+    last_accessed_at TEXT,
+    source_files    TEXT,
+    tier            TEXT DEFAULT 'working' CHECK(tier IN ('ephemeral', 'working', 'durable'))
   );
 
   CREATE INDEX IF NOT EXISTS idx_vault_kind ON vault(kind);
@@ -74,6 +78,7 @@ export const SCHEMA_DDL = `
   CREATE INDEX IF NOT EXISTS idx_vault_team ON vault(team_id);
   CREATE UNIQUE INDEX IF NOT EXISTS idx_vault_identity ON vault(user_id, kind, identity_key) WHERE identity_key IS NOT NULL;
   CREATE INDEX IF NOT EXISTS idx_vault_superseded ON vault(superseded_by) WHERE superseded_by IS NOT NULL;
+  CREATE INDEX IF NOT EXISTS idx_vault_tier ON vault(tier);
 
   -- Single FTS5 table
   CREATE VIRTUAL TABLE IF NOT EXISTS vault_fts USING fts5(
@@ -151,13 +156,13 @@ export async function initDatabase(dbPath) {
 
     const freshDb = createDb(dbPath);
     freshDb.exec(SCHEMA_DDL);
-    freshDb.exec("PRAGMA user_version = 9");
+    freshDb.exec("PRAGMA user_version = 12");
     return freshDb;
   }
 
   if (version < 5) {
     db.exec(SCHEMA_DDL);
-    db.exec("PRAGMA user_version = 9");
+    db.exec("PRAGMA user_version = 12");
   } else if (version === 5) {
     // v5 -> v6 migration: add multi-tenancy + encryption columns
     // Wrapped in transaction with duplicate-column guards for idempotent retry
@@ -192,7 +197,9 @@ export async function initDatabase(dbPath) {
       db.exec(
         `CREATE INDEX IF NOT EXISTS idx_vault_superseded ON vault(superseded_by) WHERE superseded_by IS NOT NULL`,
       );
-      db.exec("PRAGMA user_version = 9");
+      addColumnSafe(`ALTER TABLE vault ADD COLUMN hit_count INTEGER DEFAULT 0`);
+      addColumnSafe(`ALTER TABLE vault ADD COLUMN last_accessed_at TEXT`);
+      db.exec("PRAGMA user_version = 10");
     });
   } else if (version === 6) {
     // v6 -> v7+v8+v9 migration: add team_id, updated_at, superseded_by columns
@@ -222,7 +229,17 @@ export async function initDatabase(dbPath) {
       db.exec(
         `CREATE INDEX IF NOT EXISTS idx_vault_superseded ON vault(superseded_by) WHERE superseded_by IS NOT NULL`,
       );
-      db.exec("PRAGMA user_version = 9");
+      try {
+        db.exec(`ALTER TABLE vault ADD COLUMN hit_count INTEGER DEFAULT 0`);
+      } catch (e) {
+        if (!e.message.includes("duplicate column")) throw e;
+      }
+      try {
+        db.exec(`ALTER TABLE vault ADD COLUMN last_accessed_at TEXT`);
+      } catch (e) {
+        if (!e.message.includes("duplicate column")) throw e;
+      }
+      db.exec("PRAGMA user_version = 10");
     });
   } else if (version === 7) {
     // v7 -> v8+v9 migration: add updated_at, superseded_by columns
@@ -246,7 +263,17 @@ export async function initDatabase(dbPath) {
       db.exec(
         `CREATE INDEX IF NOT EXISTS idx_vault_superseded ON vault(superseded_by) WHERE superseded_by IS NOT NULL`,
       );
-      db.exec("PRAGMA user_version = 9");
+      try {
+        db.exec(`ALTER TABLE vault ADD COLUMN hit_count INTEGER DEFAULT 0`);
+      } catch (e) {
+        if (!e.message.includes("duplicate column")) throw e;
+      }
+      try {
+        db.exec(`ALTER TABLE vault ADD COLUMN last_accessed_at TEXT`);
+      } catch (e) {
+        if (!e.message.includes("duplicate column")) throw e;
+      }
+      db.exec("PRAGMA user_version = 10");
     });
   } else if (version === 8) {
     // v8 -> v9 migration: add superseded_by column
@@ -261,6 +288,60 @@ export async function initDatabase(dbPath) {
       );
       db.exec("PRAGMA user_version = 9");
     });
+    // fall through to v9 migration
+    runTransaction(db, () => {
+      const addColumnSafe = (sql) => {
+        try {
+          db.exec(sql);
+        } catch (e) {
+          if (!e.message.includes("duplicate column")) throw e;
+        }
+      };
+      addColumnSafe(`ALTER TABLE vault ADD COLUMN hit_count INTEGER DEFAULT 0`);
+      addColumnSafe(`ALTER TABLE vault ADD COLUMN last_accessed_at TEXT`);
+      db.exec("PRAGMA user_version = 10");
+    });
+  } else if (version === 9) {
+    // v9 -> v10 migration: add hit_count + last_accessed_at columns
+    runTransaction(db, () => {
+      const addColumnSafe = (sql) => {
+        try {
+          db.exec(sql);
+        } catch (e) {
+          if (!e.message.includes("duplicate column")) throw e;
+        }
+      };
+      addColumnSafe(`ALTER TABLE vault ADD COLUMN hit_count INTEGER DEFAULT 0`);
+      addColumnSafe(`ALTER TABLE vault ADD COLUMN last_accessed_at TEXT`);
+      db.exec("PRAGMA user_version = 10");
+    });
+  }
+
+  if (version >= 5 && version <= 10) {
+    // v10 -> v11 migration: add source_files column for stale-linking
+    runTransaction(db, () => {
+      try {
+        db.exec(`ALTER TABLE vault ADD COLUMN source_files TEXT`);
+      } catch (e) {
+        if (!e.message.includes("duplicate column")) throw e;
+      }
+      db.exec("PRAGMA user_version = 11");
+    });
+  }
+
+  if (version >= 5 && version <= 11) {
+    // v11 -> v12 migration: add tier column for memory tiers
+    runTransaction(db, () => {
+      try {
+        db.exec(
+          `ALTER TABLE vault ADD COLUMN tier TEXT DEFAULT 'working' CHECK(tier IN ('ephemeral', 'working', 'durable'))`,
+        );
+      } catch (e) {
+        if (!e.message.includes("duplicate column")) throw e;
+      }
+      db.exec(`CREATE INDEX IF NOT EXISTS idx_vault_tier ON vault(tier)`);
+      db.exec("PRAGMA user_version = 12");
+    });
   }
 
   return db;
@@ -270,10 +351,10 @@ export function prepareStatements(db) {
   try {
     return {
       insertEntry: db.prepare(
-        `INSERT INTO vault (id, user_id, kind, category, title, body, meta, tags, source, file_path, identity_key, expires_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+        `INSERT INTO vault (id, user_id, kind, category, title, body, meta, tags, source, file_path, identity_key, expires_at, created_at, updated_at, source_files, tier) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
       ),
       insertEntryEncrypted: db.prepare(
-        `INSERT INTO vault (id, user_id, kind, category, title, body, meta, tags, source, file_path, identity_key, expires_at, created_at, updated_at, body_encrypted, title_encrypted, meta_encrypted, iv) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+        `INSERT INTO vault (id, user_id, kind, category, title, body, meta, tags, source, file_path, identity_key, expires_at, created_at, updated_at, body_encrypted, title_encrypted, meta_encrypted, iv, source_files, tier) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
       ),
       updateEntry: db.prepare(
         `UPDATE vault SET title = ?, body = ?, meta = ?, tags = ?, source = ?, category = ?, identity_key = ?, expires_at = ?, updated_at = datetime('now') WHERE file_path = ?`,
@@ -286,7 +367,10 @@ export function prepareStatements(db) {
         `SELECT * FROM vault WHERE kind = ? AND identity_key = ? AND user_id IS ?`,
       ),
       upsertByIdentityKey: db.prepare(
-        `UPDATE vault SET title = ?, body = ?, meta = ?, tags = ?, source = ?, category = ?, file_path = ?, expires_at = ?, updated_at = datetime('now') WHERE kind = ? AND identity_key = ? AND user_id IS ?`,
+        `UPDATE vault SET title = ?, body = ?, meta = ?, tags = ?, source = ?, category = ?, file_path = ?, expires_at = ?, source_files = ?, updated_at = datetime('now') WHERE kind = ? AND identity_key = ? AND user_id IS ?`,
+      ),
+      updateSourceFiles: db.prepare(
+        `UPDATE vault SET source_files = ? WHERE id = ?`,
       ),
       insertVecStmt: db.prepare(
         `INSERT INTO vault_vec (rowid, embedding) VALUES (?, ?)`,
@@ -321,3 +405,12 @@ export function deleteVec(stmts, rowid) {
   if (safeRowid < 1n) throw new Error(`Invalid rowid: ${rowid}`);
   stmts.deleteVecStmt.run(safeRowid);
 }
+
+export function testConnection(db) {
+  try {
+    db.prepare("SELECT 1").get();
+    return true;
+  } catch {
+    return false;
+  }
+}

package/src/index/index.js

@@ -10,7 +10,11 @@
 import { readFileSync, readdirSync, existsSync, unlinkSync } from "node:fs";
 import { join, basename } from "node:path";
 import { dirToKind, walkDir, ulid } from "../core/files.js";
-import { categoryFor, CATEGORY_DIRS } from "../core/categories.js";
+import {
+  categoryFor,
+  defaultTierFor,
+  CATEGORY_DIRS,
+} from "../core/categories.js";
 import {
   parseFrontmatter,
   parseEntryFromMarkdown,
@@ -46,6 +50,8 @@ export async function indexEntry(
     createdAt,
     identity_key,
     expires_at,
+    source_files,
+    tier,
     userId,
   },
 ) {
@@ -56,7 +62,9 @@ export async function indexEntry(
 
   const tagsJson = tags ? JSON.stringify(tags) : null;
   const metaJson = meta ? JSON.stringify(meta) : null;
+  const sourceFilesJson = source_files ? JSON.stringify(source_files) : null;
   const cat = category || categoryFor(kind);
+  const effectiveTier = tier || defaultTierFor(kind);
   const userIdVal = userId || null;
 
   let wasUpdate = false;
@@ -78,6 +86,7 @@ export async function indexEntry(
         cat,
         filePath,
         expires_at || null,
+        sourceFilesJson,
         kind,
         identity_key,
         userIdVal,
@@ -116,6 +125,8 @@ export async function indexEntry(
           encrypted.title_encrypted,
           encrypted.meta_encrypted,
           encrypted.iv,
+          sourceFilesJson,
+          effectiveTier,
         );
       } else {
         ctx.stmts.insertEntry.run(
@@ -133,6 +144,8 @@ export async function indexEntry(
           expires_at || null,
           createdAt,
           createdAt,
+          sourceFilesJson,
+          effectiveTier,
         );
       }
     } catch (e) {
@@ -148,6 +161,16 @@ export async function indexEntry(
           expires_at || null,
           filePath,
         );
+        if (sourceFilesJson !== null && ctx.stmts.updateSourceFiles) {
+          const entryRow = ctx.stmts.getRowidByPath.get(filePath);
+          if (entryRow) {
+            const idRow = ctx.db
+              .prepare("SELECT id FROM vault WHERE file_path = ?")
+              .get(filePath);
+            if (idRow)
+              ctx.stmts.updateSourceFiles.run(sourceFilesJson, idRow.id);
+          }
+        }
         wasUpdate = true;
       } else {
         throw e;

package/src/index.js

@@ -8,6 +8,7 @@
 export {
   categoryFor,
   categoryDirFor,
+  defaultTierFor,
   CATEGORY_DIRS,
 } from "./core/categories.js";
 export { parseArgs, resolveConfig } from "./core/config.js";
@@ -52,6 +53,9 @@ export { indexEntry, reindex, pruneExpired } from "./index/index.js";
 // Retrieve layer
 export { hybridSearch } from "./retrieve/index.js";
 
+// Consolidation utilities
+export { findHotTags, findColdEntries } from "./consolidation/index.js";
+
 // Server tools & helpers
 export { registerTools } from "./server/tools.js";
 export {