context-vault 3.18.0 → 3.20.0

package/node_modules/@context-vault/core/src/frontmatter.ts

@@ -23,12 +23,34 @@ export function parseFrontmatter(text: string): {
   const match = normalized.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
   if (!match) return { meta: {}, body: normalized.trim() };
   const meta: Record<string, unknown> = {};
-  for (const line of match[1].split('\n')) {
+  const lines = match[1].split('\n');
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
     const idx = line.indexOf(':');
     if (idx === -1) continue;
     const key = line.slice(0, idx).trim();
     let val: unknown = line.slice(idx + 1).trim() as string;
-    if (
+
+    // Handle YAML multiline list format (tags: followed by - items)
+    if (typeof val === 'string' && val === '' && key === 'tags') {
+      const items: string[] = [];
+      let nextIdx = i + 1;
+      while (nextIdx < lines.length) {
+        const nextLine = lines[nextIdx];
+        // Match lines that start with optional whitespace + dash + space
+        const dashMatch = nextLine.match(/^\s*-\s+(.+)$/);
+        if (!dashMatch) break;
+        items.push(dashMatch[1].trim().replace(/^"|"$/g, '').replace(/^'|'$/g, ''));
+        nextIdx++;
+      }
+      if (items.length > 0) {
+        val = items;
+        i = nextIdx - 1; // Skip processed lines
+      } else {
+        val = '';
+      }
+    } else if (
       typeof val === 'string' &&
       val.length >= 2 &&
       val.startsWith('"') &&
@@ -40,8 +62,7 @@ export function parseFrontmatter(text: string): {
       } catch {
         /* keep as-is */
       }
-    }
-    if (typeof val === 'string' && val.startsWith('[') && val.endsWith(']')) {
+    } else if (typeof val === 'string' && val.startsWith('[') && val.endsWith(']')) {
       try {
         val = JSON.parse(val);
       } catch {

package/node_modules/@context-vault/core/src/index.ts

@@ -7,6 +7,9 @@ import { embedBatch } from './embed.js';
 import type { BaseCtx, IndexEntryInput, IndexingConfig, ReindexStats } from './types.js';
 import { shouldIndex } from './indexing.js';
 import { DEFAULT_INDEXING } from './constants.js';
+import { generateSummaryTiers } from './summarize.js';
+import { buildEmbeddingText } from './search.js';
+import { withBusyRetry, isBusyError } from './db.js';
 
 const EXCLUDED_DIRS = new Set(['projects', '_archive']);
 const EXCLUDED_FILES = new Set(['context.md', 'memory.md', 'README.md']);
@@ -46,95 +49,117 @@ export async function indexEntry(
   const cat = category || categoryFor(kind);
   const effectiveTier = tier || defaultTierFor(kind);
 
-  let wasUpdate = false;
-
-  if (cat === 'entity' && identity_key) {
-    const existing = ctx.stmts.getByIdentityKey.get(kind, identity_key) as
-      | Record<string, unknown>
-      | undefined;
-    if (existing) {
-      ctx.stmts.upsertByIdentityKey.run(
-        title || null,
-        body,
-        metaJson,
-        tagsJson,
-        source || 'claude-code',
-        cat,
-        filePath,
-        expires_at || null,
-        sourceFilesJson,
-        kind,
-        identity_key
-      );
-      wasUpdate = true;
-    }
-  }
+  const rowid = await withBusyRetry(() => {
+    let wasUpdate = false;
 
-  if (!wasUpdate) {
-    try {
-      ctx.stmts.insertEntry.run(
-        id,
-        kind,
-        cat,
-        title || null,
-        body,
-        metaJson,
-        tagsJson,
-        source || 'claude-code',
-        filePath,
-        identity_key || null,
-        expires_at || null,
-        createdAt,
-        createdAt,
-        sourceFilesJson,
-        effectiveTier,
-        indexed ? 1 : 0
-      );
-    } catch (e) {
-      if ((e as Error).message.includes('UNIQUE constraint')) {
-        ctx.stmts.updateEntry.run(
+    if (cat === 'entity' && identity_key) {
+      const existing = ctx.stmts.getByIdentityKey.get(kind, identity_key) as
+        | Record<string, unknown>
+        | undefined;
+      if (existing) {
+        ctx.stmts.upsertByIdentityKey.run(
           title || null,
           body,
           metaJson,
           tagsJson,
           source || 'claude-code',
           cat,
+          filePath,
+          expires_at || null,
+          sourceFilesJson,
+          kind,
+          identity_key
+        );
+        wasUpdate = true;
+      }
+    }
+
+    if (!wasUpdate) {
+      try {
+        ctx.stmts.insertEntry.run(
+          id,
+          kind,
+          cat,
+          title || null,
+          body,
+          metaJson,
+          tagsJson,
+          source || 'claude-code',
+          filePath,
           identity_key || null,
           expires_at || null,
-          filePath
+          createdAt,
+          createdAt,
+          sourceFilesJson,
+          effectiveTier,
+          indexed ? 1 : 0
         );
-        if (sourceFilesJson !== null && ctx.stmts.updateSourceFiles) {
-          const entryRow = ctx.stmts.getRowidByPath.get(filePath) as { rowid: number } | undefined;
-          if (entryRow) {
-            const idRow = ctx.db
-              .prepare('SELECT id FROM vault WHERE file_path = ?')
-              .get(filePath) as { id: string } | undefined;
-            if (idRow) ctx.stmts.updateSourceFiles.run(sourceFilesJson, idRow.id);
+      } catch (e) {
+        if (isBusyError(e)) throw e;
+        if ((e as Error).message.includes('UNIQUE constraint')) {
+          ctx.stmts.updateEntry.run(
+            title || null,
+            body,
+            metaJson,
+            tagsJson,
+            source || 'claude-code',
+            cat,
+            identity_key || null,
+            expires_at || null,
+            filePath
+          );
+          if (sourceFilesJson !== null && ctx.stmts.updateSourceFiles) {
+            const entryRow = ctx.stmts.getRowidByPath.get(filePath) as { rowid: number } | undefined;
+            if (entryRow) {
+              const idRow = ctx.db
+                .prepare('SELECT id FROM vault WHERE file_path = ?')
+                .get(filePath) as { id: string } | undefined;
+              if (idRow) ctx.stmts.updateSourceFiles.run(sourceFilesJson, idRow.id);
+            }
           }
+          wasUpdate = true;
+        } else {
+          throw e;
         }
-        wasUpdate = true;
-      } else {
-        throw e;
       }
     }
-  }
 
-  const rowidResult = wasUpdate
-    ? (ctx.stmts.getRowidByPath.get(filePath) as { rowid: number } | undefined)
-    : (ctx.stmts.getRowid.get(id) as { rowid: number } | undefined);
+    const rowidResult = wasUpdate
+      ? (ctx.stmts.getRowidByPath.get(filePath) as { rowid: number } | undefined)
+      : (ctx.stmts.getRowid.get(id) as { rowid: number } | undefined);
 
-  if (!rowidResult || rowidResult.rowid == null) {
-    throw new Error(
-      `Could not find rowid for entry: ${wasUpdate ? `file_path=${filePath}` : `id=${id}`}`
-    );
-  }
+    if (!rowidResult || rowidResult.rowid == null) {
+      throw new Error(
+        `Could not find rowid for entry: ${wasUpdate ? `file_path=${filePath}` : `id=${id}`}`
+      );
+    }
 
-  const rowid = Number(rowidResult.rowid);
-  if (!Number.isFinite(rowid) || rowid < 1) {
-    throw new Error(
-      `Invalid rowid retrieved: ${rowidResult.rowid} (type: ${typeof rowidResult.rowid})`
-    );
-  }
+    const rowidNum = Number(rowidResult.rowid);
+    if (!Number.isFinite(rowidNum) || rowidNum < 1) {
+      throw new Error(
+        `Invalid rowid retrieved: ${rowidResult.rowid} (type: ${typeof rowidResult.rowid})`
+      );
+    }
+
+    // Generate and store precomputed summary tiers
+    try {
+      const { condensed, keypoint } = generateSummaryTiers(body);
+      ctx.db
+        .prepare('UPDATE vault SET summary_condensed = ?, summary_keypoint = ? WHERE id = ?')
+        .run(condensed || null, keypoint || null, id);
+    } catch (sumErr) {
+      if (isBusyError(sumErr)) throw sumErr;
+    }
+
+    return rowidNum;
+  });
+
+  // Observability: log the INSERT outcome so silent-wedge cases surface in error.log.
+  // Added 2026-04-19 after 44h of save_context returning ✓ without persisting (runtime/
+  // subprocess-state issue, see specs/reindex-and-save-pipeline-bugs.md).
+  console.warn(
+    `[context-vault] indexEntry: id=${id} rowid=${rowid} cat=${cat} indexed=${indexed ? 1 : 0}`
+  );
 
   if (indexed && cat !== 'event') {
     let embedding: Float32Array | null = null;
@@ -142,7 +167,7 @@ export async function indexEntry(
       embedding = precomputedEmbedding;
     } else {
       try {
-        embedding = await ctx.embed([title, body].filter(Boolean).join(' '));
+        embedding = await ctx.embed(buildEmbeddingText(title, body, tagsJson, kind));
       } catch (embedErr) {
         console.warn(
           `[context-vault] embed() failed for entry ${id} — skipping vec insert: ${(embedErr as Error).message}`
@@ -151,12 +176,15 @@ export async function indexEntry(
     }
 
     if (embedding) {
-      try {
-        ctx.deleteVec(rowid);
-      } catch {
-        /* no-op */
-      }
-      ctx.insertVec(rowid, embedding);
+      await withBusyRetry(() => {
+        try {
+          ctx.deleteVec(rowid);
+        } catch (delErr) {
+          if (isBusyError(delErr)) throw delErr;
+          /* no-op */
+        }
+        ctx.insertVec(rowid, embedding);
+      });
     }
   }
 }
@@ -261,6 +289,7 @@ export async function reindex(
     return stats;
   }
 
+  await withBusyRetry(() => {
   ctx.db.exec('BEGIN');
   try {
     for (const { kind, dir } of filteredKindEntries) {
@@ -336,10 +365,9 @@ export async function reindex(
             if (entryIndexed && category !== 'event') {
               const rowidResult = ctx.stmts.getRowid.get(id) as { rowid: number } | undefined;
               if (rowidResult?.rowid) {
-                const embeddingText = [parsed.title, parsed.body].filter(Boolean).join(' ');
                 pendingEmbeds.push({
                   rowid: rowidResult.rowid,
-                  text: embeddingText,
+                  text: buildEmbeddingText(parsed.title, parsed.body, tagsJson, kind),
                 });
               }
             }
@@ -384,13 +412,12 @@ export async function reindex(
                 try { ctx.deleteVec(rowid); stats.embeddingsCleared!++; } catch {}
               }
               stats.skippedIndexing!++;
-            } else if ((bodyChanged || titleChanged) && category !== 'event') {
+            } else if ((bodyChanged || titleChanged || tagsChanged) && category !== 'event') {
               const rowid = (
                 ctx.stmts.getRowid.get(existing.id as string) as { rowid: number } | undefined
               )?.rowid;
               if (rowid) {
-                const embeddingText = [parsed.title, parsed.body].filter(Boolean).join(' ');
-                pendingEmbeds.push({ rowid, text: embeddingText });
+                pendingEmbeds.push({ rowid, text: buildEmbeddingText(parsed.title, parsed.body, tagsJson, kind) });
               }
             }
             stats.updated++;
@@ -465,9 +492,21 @@ export async function reindex(
 
     ctx.db.exec('COMMIT');
   } catch (e) {
-    ctx.db.exec('ROLLBACK');
+    try { ctx.db.exec('ROLLBACK'); } catch {}
+    // On SQLITE_BUSY, reset stats so the retry is idempotent (counters
+    // accumulate inside the tx and would double-count otherwise).
+    if (isBusyError(e)) {
+      stats.added = 0;
+      stats.updated = 0;
+      stats.removed = 0;
+      stats.unchanged = 0;
+      stats.skippedIndexing = 0;
+      stats.embeddingsCleared = 0;
+      pendingEmbeds.length = 0;
+    }
     throw e;
   }
+  });
 
   if (!skipEmbeddings) {
     for (let i = 0; i < pendingEmbeds.length; i += EMBED_BATCH_SIZE) {
@@ -487,17 +526,17 @@ export async function reindex(
     if (fullSync) {
       const missingVec = ctx.db
         .prepare(
-          `SELECT v.rowid, v.title, v.body FROM vault v
+          `SELECT v.rowid, v.title, v.body, v.tags, v.kind FROM vault v
            WHERE v.category != 'event'
              AND v.indexed = 1
              AND v.rowid NOT IN (SELECT rowid FROM vault_vec)`
         )
-        .all() as { rowid: number; title: string | null; body: string }[];
+        .all() as { rowid: number; title: string | null; body: string; tags: string | null; kind: string }[];
 
       if (missingVec.length > 0) {
         const missingEmbeds = missingVec.map((r) => ({
           rowid: r.rowid,
-          text: [r.title, r.body].filter(Boolean).join(' '),
+          text: buildEmbeddingText(r.title, r.body, r.tags, r.kind),
         }));
 
         for (let i = 0; i < missingEmbeds.length; i += EMBED_BATCH_SIZE) {

package/node_modules/@context-vault/core/src/main.ts

@@ -117,3 +117,10 @@ export { htmlToMarkdown, extractHtmlContent, ingestUrl } from './ingest-url.js';
 // Watch
 export { startWatcher } from './watch.js';
 export type { WatcherOptions, VaultWatcher } from './watch.js';
+
+// Consolidation
+export { findDuplicates, mergeEntries, computeDecayScores } from './consolidation.js';
+export type { DuplicateGroup, MergeResult, DecayScore } from './consolidation.js';
+
+// Assemble
+export { assembleContext } from './assemble.js';

package/node_modules/@context-vault/core/src/search.test.ts

@@ -0,0 +1,59 @@
+import { describe, it, expect } from 'vitest';
+import { kindBoost, recencyBoost } from './search.js';
+
+describe('kindBoost', () => {
+  it('returns 1.5 for task kind', () => {
+    expect(kindBoost('task')).toBe(1.5);
+  });
+
+  it('returns 1.0 for knowledge kinds', () => {
+    for (const k of ['insight', 'decision', 'pattern', 'reference', 'note']) {
+      expect(kindBoost(k)).toBe(1.0);
+    }
+  });
+
+  it('returns 1.0 for entity kinds', () => {
+    for (const k of ['bucket', 'project', 'tool', 'contact']) {
+      expect(kindBoost(k)).toBe(1.0);
+    }
+  });
+
+  it('returns 1.0 for null/undefined', () => {
+    expect(kindBoost(null)).toBe(1.0);
+    expect(kindBoost(undefined)).toBe(1.0);
+  });
+});
+
+describe('task-runs ranking parity', () => {
+  it('task entry outscores equivalent insight entry despite event-category decay', () => {
+    // Both entries are 14 days old — insight gets no recency decay, task gets event decay.
+    // kindBoost must more than compensate.
+    const createdAt = new Date(Date.now() - 14 * 86400000).toISOString();
+    const baseRrf = 1 / (60 + 1); // rank 0 in a single RRF list
+
+    const insightDecay = recencyBoost(createdAt, 'knowledge', 30);
+    const taskDecay = recencyBoost(createdAt, 'event', 30);
+
+    const insightScore = baseRrf * insightDecay * 1.0; // no kindBoost
+    const taskScore = baseRrf * taskDecay * kindBoost('task');
+
+    expect(taskScore).toBeGreaterThan(insightScore);
+  });
+
+  it('kindBoost reduces the event-decay penalty by ≥ 40% at 30-day age', () => {
+    // At 30 days event decay is 0.5 (half-life). Without kindBoost a task entry would
+    // score half of an equivalent knowledge entry. The 1.5x boost brings it to 0.75x —
+    // a 50% reduction in the penalty gap (from 0.5 to 0.75 relative to 1.0 baseline).
+    const createdAt = new Date(Date.now() - 30 * 86400000).toISOString();
+    const baseRrf = 1 / (60 + 1);
+
+    const unboostedTaskScore = baseRrf * recencyBoost(createdAt, 'event', 30) * 1.0;
+    const boostedTaskScore = baseRrf * recencyBoost(createdAt, 'event', 30) * kindBoost('task');
+    const knowledgeScore = baseRrf * recencyBoost(createdAt, 'knowledge', 30) * 1.0;
+
+    // Boost must meaningfully close the gap: boosted gap ≤ 60% of unboosted gap
+    const unboostedGap = knowledgeScore - unboostedTaskScore;
+    const boostedGap = knowledgeScore - boostedTaskScore;
+    expect(boostedGap).toBeLessThan(unboostedGap * 0.6);
+  });
+});

package/node_modules/@context-vault/core/src/search.ts

@@ -3,6 +3,33 @@ import { embedBatch } from './embed.js';
 
 const NEAR_DUP_THRESHOLD = 0.92;
 const RRF_K = 60;
+
+/**
+ * Build text used for embedding generation. Includes tags and kind so the
+ * vector representation captures metadata, not just title/body content.
+ */
+export function buildEmbeddingText(
+  title: string | null | undefined,
+  body: string | null | undefined,
+  tags: string | null | undefined,
+  kind: string | null | undefined
+): string {
+  const parts: string[] = [];
+  if (title) parts.push(title);
+  if (body) parts.push(body);
+  if (tags) {
+    try {
+      const parsed = JSON.parse(tags);
+      if (Array.isArray(parsed)) {
+        parts.push(`[tags: ${parsed.join(', ')}]`);
+      }
+    } catch {
+      parts.push(`[tags: ${tags}]`);
+    }
+  }
+  if (kind) parts.push(`[kind: ${kind}]`);
+  return parts.join(' ');
+}
 const RECALL_BOOST_CAP = 2.0;
 const RECALL_HALF_LIFE_DAYS = 30;
 const DISCOVERY_SLOTS = 2;
@@ -29,7 +56,8 @@ export function buildFtsQuery(query: string): string | null {
   const phrase = `"${words.join(' ')}"`;
   const near = `NEAR(${words.map((w) => `"${w}"`).join(' ')}, 10)`;
   const and = words.map((w) => `"${w}"`).join(' AND ');
-  return `${phrase} OR ${near} OR ${and}`;
+  const or = words.map((w) => `"${w}"`).join(' OR ');
+  return `${phrase} OR ${near} OR ${and} OR ${or}`;
 }
 
 export function recencyBoost(createdAt: string, category: string, decayDays = 30): number {
@@ -50,6 +78,15 @@ export function recallBoost(recallCount: number, lastRecalledAt: string | null):
   return Math.min(boost, RECALL_BOOST_CAP);
 }
 
+// task entries (kind='task') are event-category and would otherwise be penalised by
+// recencyBoost's event decay; this boost restores their ranking parity for free-text queries.
+const TASK_KIND_BOOST = 1.5;
+
+export function kindBoost(kind: string | null | undefined): number {
+  if (kind === 'task') return TASK_KIND_BOOST;
+  return 1.0;
+}
+
 export function buildFilterClauses({
   categoryFilter,
   excludeEvents = false,
@@ -202,7 +239,7 @@ export async function hybridSearch(
           if (missing.length > 0) {
             const entries = missing.map((r) => {
               const entry = rowMap.get(r.id);
-              return { rowid: r.rowid, text: [entry?.title, entry?.body].filter(Boolean).join(' ') };
+              return { rowid: r.rowid, text: buildEmbeddingText(entry?.title, entry?.body, entry?.tags, entry?.kind) };
             });
             const embeddings = await embedBatch(entries.map((e) => e.text));
             for (let i = 0; i < entries.length; i++) {
@@ -232,7 +269,7 @@ export async function hybridSearch(
     if (vecCount > 0) {
       queryVec = await ctx.embed(query);
       if (queryVec) {
-        const vecLimit = kindFilter ? 30 : 15;
+        const vecLimit = kindFilter ? 60 : 40;
         const vecRows = ctx.db
           .prepare(
             `SELECT v.rowid, v.distance FROM vault_vec v WHERE embedding MATCH ? ORDER BY distance LIMIT ?`
@@ -327,10 +364,73 @@ export async function hybridSearch(
     }
   }
 
+  // Tag-based search lane: match query keywords against tags JSON and kind
+  const tagRankedIds: string[] = [];
+  try {
+    const tagWords = query
+      .split(/[\s-]+/)
+      .map((w) => w.replace(/[*"():^~{}]/g, '').toLowerCase())
+      .filter((w) => w.length > 1);
+    if (tagWords.length > 0) {
+      const tagWhereParts = ['indexed = 1', "(expires_at IS NULL OR expires_at > datetime('now'))", 'superseded_by IS NULL'];
+      const tagParams: (string | number | null)[] = [];
+
+      const likeClauses = tagWords.map((w) => {
+        tagParams.push(`%${w}%`);
+        return `tags LIKE ?`;
+      });
+      const kindClauses = tagWords.map((w) => {
+        tagParams.push(`%${w}%`);
+        return `kind LIKE ?`;
+      });
+      tagWhereParts.push(`(${[...likeClauses, ...kindClauses].join(' OR ')})`);
+
+      if (kindFilter) {
+        tagWhereParts.push('kind = ?');
+        tagParams.push(kindFilter);
+      }
+      if (categoryFilter) {
+        tagWhereParts.push('category = ?');
+        tagParams.push(categoryFilter);
+      }
+      if (excludeEvents && !categoryFilter) {
+        tagWhereParts.push("category != 'event'");
+      }
+      if (since) {
+        tagWhereParts.push('created_at >= ?');
+        tagParams.push(since);
+      }
+      if (until) {
+        tagWhereParts.push('created_at <= ?');
+        tagParams.push(until);
+      }
+      if (!includeSuperseeded) {
+        // already have superseded_by IS NULL above
+      }
+      if (!includeEphemeral) {
+        tagWhereParts.push("tier != 'ephemeral'");
+      }
+
+      const tagSQL = `SELECT id FROM vault WHERE ${tagWhereParts.join(' AND ')} ORDER BY recall_count DESC LIMIT 20`;
+      const tagRows = ctx.db.prepare(tagSQL).all(...tagParams) as { id: string }[];
+
+      for (const row of tagRows) {
+        tagRankedIds.push(row.id);
+        if (!rowMap.has(row.id)) {
+          const full = ctx.db.prepare('SELECT * FROM vault WHERE id = ?').get(row.id) as VaultEntry | undefined;
+          if (full) rowMap.set(full.id, full);
+        }
+      }
+    }
+  } catch (err) {
+    console.error(`[retrieve] Tag search error: ${(err as Error).message}`);
+  }
+
   if (rowMap.size === 0) return [];
 
-  // Build ranked lists for RRF: content FTS + content vec + optional context vec
+  // Build ranked lists for RRF: content FTS + content vec + tags + optional context vec
   const rankedLists = [ftsRankedIds, vecRankedIds];
+  if (tagRankedIds.length > 0) rankedLists.push(tagRankedIds);
   if (ctxRankedIds.length > 0) rankedLists.push(ctxRankedIds);
   const rrfScores = reciprocalRankFusion(rankedLists);
 
@@ -341,7 +441,14 @@ export async function hybridSearch(
       entry.last_recalled_at ?? null
     );
     const durable = entry.tier === 'durable' ? 1.3 : 1.0;
-    rrfScores.set(id, (rrfScores.get(id) ?? 0) * boost * recall * durable);
+    // Heat-tier ranking boost: frequently recalled entries rank higher
+    const heatMultiplier =
+      entry.heat_tier === 'hot' ? 1.4 :
+      entry.heat_tier === 'warm' ? 1.1 :
+      entry.heat_tier === 'cold' ? 0.9 :
+      entry.heat_tier === 'frozen' ? 0.7 : 1.0;
+    const kBoost = kindBoost(entry.kind);
+    rrfScores.set(id, (rrfScores.get(id) ?? 0) * boost * recall * durable * heatMultiplier * kBoost);
   }
 
   const candidates: SearchResult[] = [...rowMap.values()].map((entry) => ({