@context-vault/core 3.1.6 → 3.1.7

package/src/ingest-url.ts

@@ -1,121 +1,96 @@
 export function htmlToMarkdown(html: string): string {
   let md = html;
-  md = md.replace(/<script[\s\S]*?<\/script>/gi, "");
-  md = md.replace(/<style[\s\S]*?<\/style>/gi, "");
-  md = md.replace(/<nav[\s\S]*?<\/nav>/gi, "");
-  md = md.replace(/<header[\s\S]*?<\/header>/gi, "");
-  md = md.replace(/<footer[\s\S]*?<\/footer>/gi, "");
-  md = md.replace(/<aside[\s\S]*?<\/aside>/gi, "");
-  md = md.replace(
-    /<h1[^>]*>([\s\S]*?)<\/h1>/gi,
-    (_, c: string) => `\n# ${stripTags(c).trim()}\n`,
-  );
-  md = md.replace(
-    /<h2[^>]*>([\s\S]*?)<\/h2>/gi,
-    (_, c: string) => `\n## ${stripTags(c).trim()}\n`,
-  );
+  md = md.replace(/<script[\s\S]*?<\/script>/gi, '');
+  md = md.replace(/<style[\s\S]*?<\/style>/gi, '');
+  md = md.replace(/<nav[\s\S]*?<\/nav>/gi, '');
+  md = md.replace(/<header[\s\S]*?<\/header>/gi, '');
+  md = md.replace(/<footer[\s\S]*?<\/footer>/gi, '');
+  md = md.replace(/<aside[\s\S]*?<\/aside>/gi, '');
+  md = md.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, (_, c: string) => `\n# ${stripTags(c).trim()}\n`);
+  md = md.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, (_, c: string) => `\n## ${stripTags(c).trim()}\n`);
   md = md.replace(
     /<h3[^>]*>([\s\S]*?)<\/h3>/gi,
-    (_, c: string) => `\n### ${stripTags(c).trim()}\n`,
+    (_, c: string) => `\n### ${stripTags(c).trim()}\n`
   );
   md = md.replace(
     /<h4[^>]*>([\s\S]*?)<\/h4>/gi,
-    (_, c: string) => `\n#### ${stripTags(c).trim()}\n`,
+    (_, c: string) => `\n#### ${stripTags(c).trim()}\n`
   );
   md = md.replace(
     /<h5[^>]*>([\s\S]*?)<\/h5>/gi,
-    (_, c: string) => `\n##### ${stripTags(c).trim()}\n`,
+    (_, c: string) => `\n##### ${stripTags(c).trim()}\n`
   );
   md = md.replace(
     /<h6[^>]*>([\s\S]*?)<\/h6>/gi,
-    (_, c: string) => `\n###### ${stripTags(c).trim()}\n`,
+    (_, c: string) => `\n###### ${stripTags(c).trim()}\n`
   );
   md = md.replace(
     /<a[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi,
     (_, href: string, text: string) => {
       const cleanText = stripTags(text).trim();
-      return cleanText ? `[${cleanText}](${href})` : "";
-    },
+      return cleanText ? `[${cleanText}](${href})` : '';
+    }
   );
   md = md.replace(
     /<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi,
-    (_, c: string) => `\n\`\`\`\n${decodeEntities(c).trim()}\n\`\`\`\n`,
+    (_, c: string) => `\n\`\`\`\n${decodeEntities(c).trim()}\n\`\`\`\n`
   );
   md = md.replace(
     /<pre[^>]*>([\s\S]*?)<\/pre>/gi,
-    (_, c: string) =>
-      `\n\`\`\`\n${decodeEntities(stripTags(c)).trim()}\n\`\`\`\n`,
+    (_, c: string) => `\n\`\`\`\n${decodeEntities(stripTags(c)).trim()}\n\`\`\`\n`
   );
   md = md.replace(
     /<code[^>]*>([\s\S]*?)<\/code>/gi,
-    (_, c: string) => `\`${decodeEntities(c).trim()}\``,
+    (_, c: string) => `\`${decodeEntities(c).trim()}\``
   );
   md = md.replace(
     /<(strong|b)[^>]*>([\s\S]*?)<\/\1>/gi,
-    (_, __: string, c: string) => `**${stripTags(c).trim()}**`,
+    (_, __: string, c: string) => `**${stripTags(c).trim()}**`
   );
   md = md.replace(
     /<(em|i)[^>]*>([\s\S]*?)<\/\1>/gi,
-    (_, __: string, c: string) => `*${stripTags(c).trim()}*`,
-  );
-  md = md.replace(
-    /<li[^>]*>([\s\S]*?)<\/li>/gi,
-    (_, c: string) => `- ${stripTags(c).trim()}\n`,
-  );
-  md = md.replace(/<br\s*\/?>/gi, "\n");
-  md = md.replace(
-    /<p[^>]*>([\s\S]*?)<\/p>/gi,
-    (_, c: string) => `\n${stripTags(c).trim()}\n`,
-  );
-  md = md.replace(
-    /<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi,
-    (_, c: string) => {
-      return (
-        "\n" +
-        stripTags(c)
-          .trim()
-          .split("\n")
-          .map((l: string) => `> ${l}`)
-          .join("\n") +
-        "\n"
-      );
-    },
+    (_, __: string, c: string) => `*${stripTags(c).trim()}*`
   );
+  md = md.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_, c: string) => `- ${stripTags(c).trim()}\n`);
+  md = md.replace(/<br\s*\/?>/gi, '\n');
+  md = md.replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, (_, c: string) => `\n${stripTags(c).trim()}\n`);
+  md = md.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, (_, c: string) => {
+    return (
+      '\n' +
+      stripTags(c)
+        .trim()
+        .split('\n')
+        .map((l: string) => `> ${l}`)
+        .join('\n') +
+      '\n'
+    );
+  });
   md = stripTags(md);
   md = decodeEntities(md);
-  md = md.replace(/\n{3,}/g, "\n\n").trim();
+  md = md.replace(/\n{3,}/g, '\n\n').trim();
   return md;
 }
 
 function stripTags(html: string): string {
-  return html.replace(/<[^>]+>/g, "");
+  return html.replace(/<[^>]+>/g, '');
 }
 
 function decodeEntities(text: string): string {
   return text
-    .replace(/&amp;/g, "&")
-    .replace(/&lt;/g, "<")
-    .replace(/&gt;/g, ">")
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
     .replace(/&quot;/g, '"')
     .replace(/&#39;/g, "'")
-    .replace(/&nbsp;/g, " ")
-    .replace(/&#(\d+);/g, (_, n: string) =>
-      String.fromCharCode(parseInt(n, 10)),
-    )
-    .replace(/&#x([0-9a-f]+);/gi, (_, n: string) =>
-      String.fromCharCode(parseInt(n, 16)),
-    );
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&#(\d+);/g, (_, n: string) => String.fromCharCode(parseInt(n, 10)))
+    .replace(/&#x([0-9a-f]+);/gi, (_, n: string) => String.fromCharCode(parseInt(n, 16)));
 }
 
-export function extractHtmlContent(
-  html: string,
-  _url: string,
-): { title: string; body: string } {
+export function extractHtmlContent(html: string, _url: string): { title: string; body: string } {
   const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
-  const title = titleMatch
-    ? stripTags(decodeEntities(titleMatch[1])).trim()
-    : "";
-  let contentHtml = "";
+  const title = titleMatch ? stripTags(decodeEntities(titleMatch[1])).trim() : '';
+  let contentHtml = '';
   const articleMatch = html.match(/<article[^>]*>([\s\S]*?)<\/article>/i);
   const mainMatch = html.match(/<main[^>]*>([\s\S]*?)<\/main>/i);
   if (articleMatch) contentHtml = articleMatch[1];
@@ -136,7 +111,7 @@ export async function ingestUrl(
     source?: string;
     maxBodyLength?: number;
     timeoutMs?: number;
-  } = {},
+  } = {}
 ): Promise<{
   kind: string;
   title: string;
@@ -145,13 +120,7 @@ export async function ingestUrl(
   meta: Record<string, unknown>;
   source: string;
 }> {
-  const {
-    kind = "reference",
-    tags = [],
-    source,
-    maxBodyLength = 50000,
-    timeoutMs = 15000,
-  } = opts;
+  const { kind = 'reference', tags = [], source, maxBodyLength = 50000, timeoutMs = 15000 } = opts;
   let domain: string;
   try {
     domain = new URL(url).hostname;
@@ -165,27 +134,22 @@ export async function ingestUrl(
     response = await fetch(url, {
       signal: controller.signal,
       headers: {
-        "User-Agent":
-          "ContextVault/1.0 (+https://github.com/fellanH/context-vault)",
-        Accept: "text/html,application/xhtml+xml,text/plain,*/*",
+        'User-Agent': 'ContextVault/1.0 (+https://github.com/fellanH/context-vault)',
+        Accept: 'text/html,application/xhtml+xml,text/plain,*/*',
       },
     });
   } catch (err) {
-    if ((err as Error).name === "AbortError")
+    if ((err as Error).name === 'AbortError')
       throw new Error(`Request timed out after ${timeoutMs}ms`);
     throw new Error(`Fetch failed: ${(err as Error).message}`);
   } finally {
     clearTimeout(timeout);
   }
-  if (!response.ok)
-    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-  const contentType = response.headers.get("content-type") || "";
+  if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+  const contentType = response.headers.get('content-type') || '';
   const html = await response.text();
   let title: string, body: string;
-  if (
-    contentType.includes("text/html") ||
-    contentType.includes("application/xhtml")
-  ) {
+  if (contentType.includes('text/html') || contentType.includes('application/xhtml')) {
     const extracted = extractHtmlContent(html, url);
     title = extracted.title;
     body = extracted.body;
@@ -193,19 +157,18 @@ export async function ingestUrl(
     title = domain;
     body = html;
   }
-  if (body.length > maxBodyLength)
-    body = body.slice(0, maxBodyLength) + "\n\n[Content truncated]";
-  if (!body.trim()) throw new Error("No readable content extracted from URL");
+  if (body.length > maxBodyLength) body = body.slice(0, maxBodyLength) + '\n\n[Content truncated]';
+  if (!body.trim()) throw new Error('No readable content extracted from URL');
   return {
     kind,
     title: title || domain,
     body,
-    tags: [...tags, "web-import"],
+    tags: [...tags, 'web-import'],
     meta: {
       url,
       domain,
       fetched_at: new Date().toISOString(),
-      content_type: contentType.split(";")[0].trim() || "text/html",
+      content_type: contentType.split(';')[0].trim() || 'text/html',
     },
     source: source || domain,
   };

package/src/main.ts

@@ -13,7 +13,7 @@ export type {
   ReindexStats,
   BaseCtx,
   SearchOptions,
-} from "./types.js";
+} from './types.js';
 
 // Constants
 export {
@@ -31,7 +31,7 @@ export {
   MAX_IDENTITY_KEY_LENGTH,
   DEFAULT_GROWTH_THRESHOLDS,
   DEFAULT_LIFECYCLE,
-} from "./constants.js";
+} from './constants.js';
 
 // Categories
 export {
@@ -40,10 +40,10 @@ export {
   defaultTierFor,
   CATEGORY_DIRS,
   KIND_STALENESS_DAYS,
-} from "./categories.js";
+} from './categories.js';
 
 // Config
-export { parseArgs, resolveConfig } from "./config.js";
+export { parseArgs, resolveConfig } from './config.js';
 
 // Files
 export {
@@ -55,7 +55,7 @@ export {
   kindToPath,
   safeJoin,
   walkDir,
-} from "./files.js";
+} from './files.js';
 
 // Frontmatter
 export {
@@ -63,10 +63,10 @@ export {
   parseFrontmatter,
   extractCustomMeta,
   parseEntryFromMarkdown,
-} from "./frontmatter.js";
+} from './frontmatter.js';
 
 // Formatters
-export { formatBody } from "./formatters.js";
+export { formatBody } from './formatters.js';
 
 // Database
 export {
@@ -77,18 +77,13 @@ export {
   insertVec,
   deleteVec,
   testConnection,
-} from "./db.js";
+} from './db.js';
 
 // Embeddings
-export {
-  embed,
-  embedBatch,
-  resetEmbedPipeline,
-  isEmbedAvailable,
-} from "./embed.js";
+export { embed, embedBatch, resetEmbedPipeline, isEmbedAvailable } from './embed.js';
 
 // Index (reindex + indexEntry)
-export { indexEntry, reindex, pruneExpired } from "./index.js";
+export { indexEntry, reindex, pruneExpired } from './index.js';
 
 // Search (retrieve)
 export {
@@ -99,10 +94,10 @@ export {
   recencyDecayScore,
   dotProduct,
   reciprocalRankFusion,
-} from "./search.js";
+} from './search.js';
 
 // Capture
-export { writeEntry, updateEntryFile, captureAndIndex } from "./capture.js";
+export { writeEntry, updateEntryFile, captureAndIndex } from './capture.js';
 
 // Ingest URL
-export { htmlToMarkdown, extractHtmlContent, ingestUrl } from "./ingest-url.js";
+export { htmlToMarkdown, extractHtmlContent, ingestUrl } from './ingest-url.js';

package/src/search.ts

@@ -1,17 +1,9 @@
-import type {
-  BaseCtx,
-  SearchResult,
-  SearchOptions,
-  VaultEntry,
-} from "./types.js";
+import type { BaseCtx, SearchResult, SearchOptions, VaultEntry } from './types.js';
 
 const NEAR_DUP_THRESHOLD = 0.92;
 const RRF_K = 60;
 
-export function recencyDecayScore(
-  updatedAt: string | null | undefined,
-  decayRate = 0.05,
-): number {
+export function recencyDecayScore(updatedAt: string | null | undefined, decayRate = 0.05): number {
   if (updatedAt == null) return 0.5;
   const ageDays = (Date.now() - new Date(updatedAt).getTime()) / 86400000;
   return Math.exp(-decayRate * ageDays);
@@ -26,22 +18,18 @@ export function dotProduct(a: Float32Array, b: Float32Array): number {
 export function buildFtsQuery(query: string): string | null {
   const words = query
     .split(/[\s-]+/)
-    .map((w) => w.replace(/[*"():^~{}]/g, ""))
+    .map((w) => w.replace(/[*"():^~{}]/g, ''))
     .filter((w) => w.length > 0);
   if (!words.length) return null;
   if (words.length === 1) return `"${words[0]}"`;
-  const phrase = `"${words.join(" ")}"`;
-  const near = `NEAR(${words.map((w) => `"${w}"`).join(" ")}, 10)`;
-  const and = words.map((w) => `"${w}"`).join(" AND ");
+  const phrase = `"${words.join(' ')}"`;
+  const near = `NEAR(${words.map((w) => `"${w}"`).join(' ')}, 10)`;
+  const and = words.map((w) => `"${w}"`).join(' AND ');
   return `${phrase} OR ${near} OR ${and}`;
 }
 
-export function recencyBoost(
-  createdAt: string,
-  category: string,
-  decayDays = 30,
-): number {
-  if (category !== "event") return 1.0;
+export function recencyBoost(createdAt: string, category: string, decayDays = 30): number {
+  if (category !== 'event') return 1.0;
   const ageDays = (Date.now() - new Date(createdAt).getTime()) / 86400000;
   return 1 / (1 + ageDays / decayDays);
 }
@@ -62,30 +50,30 @@ export function buildFilterClauses({
   const clauses: string[] = [];
   const params: (string | number | null)[] = [];
   if (categoryFilter) {
-    clauses.push("e.category = ?");
+    clauses.push('e.category = ?');
     params.push(categoryFilter);
   }
   if (excludeEvents && !categoryFilter) {
     clauses.push("e.category != 'event'");
   }
   if (since) {
-    clauses.push("e.created_at >= ?");
+    clauses.push('e.created_at >= ?');
     params.push(since);
   }
   if (until) {
-    clauses.push("e.created_at <= ?");
+    clauses.push('e.created_at <= ?');
     params.push(until);
   }
   clauses.push("(e.expires_at IS NULL OR e.expires_at > datetime('now'))");
   if (!includeSuperseeded) {
-    clauses.push("e.superseded_by IS NULL");
+    clauses.push('e.superseded_by IS NULL');
   }
   return { clauses, params };
 }
 
 export function reciprocalRankFusion(
   rankedLists: string[][],
-  k: number = RRF_K,
+  k: number = RRF_K
 ): Map<string, number> {
   const scores = new Map<string, number>();
   for (const list of rankedLists) {
@@ -100,7 +88,7 @@ export function reciprocalRankFusion(
 export async function hybridSearch(
   ctx: BaseCtx,
   query: string,
-  opts: SearchOptions = {},
+  opts: SearchOptions = {}
 ): Promise<SearchResult[]> {
   const {
     kindFilter = null,
@@ -131,27 +119,27 @@ export async function hybridSearch(
   const ftsQuery = buildFtsQuery(query);
   if (ftsQuery) {
     try {
-      const whereParts = ["vault_fts MATCH ?"];
+      const whereParts = ['vault_fts MATCH ?'];
       const ftsParams: (string | number | null)[] = [ftsQuery];
 
       if (kindFilter) {
-        whereParts.push("e.kind = ?");
+        whereParts.push('e.kind = ?');
         ftsParams.push(kindFilter);
       }
       whereParts.push(...extraFilters.clauses);
       ftsParams.push(...extraFilters.params);
 
-      const ftsSQL = `SELECT e.*, rank FROM vault_fts f JOIN vault e ON f.rowid = e.rowid WHERE ${whereParts.join(" AND ")} ORDER BY rank LIMIT 15`;
-      const rows = ctx.db
-        .prepare(ftsSQL)
-        .all(...ftsParams) as unknown as (VaultEntry & { rank: number })[];
+      const ftsSQL = `SELECT e.*, rank FROM vault_fts f JOIN vault e ON f.rowid = e.rowid WHERE ${whereParts.join(' AND ')} ORDER BY rank LIMIT 15`;
+      const rows = ctx.db.prepare(ftsSQL).all(...ftsParams) as unknown as (VaultEntry & {
+        rank: number;
+      })[];
 
       for (const { rank: _rank, ...row } of rows) {
         ftsRankedIds.push(row.id);
         if (!rowMap.has(row.id)) rowMap.set(row.id, row);
       }
     } catch (err) {
-      if (!(err as Error).message?.includes("fts5: syntax error")) {
+      if (!(err as Error).message?.includes('fts5: syntax error')) {
         console.error(`[retrieve] FTS search error: ${(err as Error).message}`);
       }
     }
@@ -162,7 +150,7 @@ export async function hybridSearch(
 
   try {
     const vecCount = (
-      ctx.db.prepare("SELECT COUNT(*) as c FROM vault_vec").get() as {
+      ctx.db.prepare('SELECT COUNT(*) as c FROM vault_vec').get() as {
         c: number;
       }
     ).c;
@@ -172,17 +160,15 @@ export async function hybridSearch(
         const vecLimit = kindFilter ? 30 : 15;
         const vecRows = ctx.db
           .prepare(
-            `SELECT v.rowid, v.distance FROM vault_vec v WHERE embedding MATCH ? ORDER BY distance LIMIT ?`,
+            `SELECT v.rowid, v.distance FROM vault_vec v WHERE embedding MATCH ? ORDER BY distance LIMIT ?`
           )
           .all(queryVec, vecLimit) as { rowid: number; distance: number }[];
 
         if (vecRows.length) {
           const rowids = vecRows.map((vr) => vr.rowid);
-          const placeholders = rowids.map(() => "?").join(",");
+          const placeholders = rowids.map(() => '?').join(',');
           const hydrated = ctx.db
-            .prepare(
-              `SELECT rowid, * FROM vault WHERE rowid IN (${placeholders})`,
-            )
+            .prepare(`SELECT rowid, * FROM vault WHERE rowid IN (${placeholders})`)
             .all(...rowids) as unknown as (VaultEntry & { rowid: number })[];
 
           const byRowid = new Map<number, VaultEntry & { rowid: number }>();
@@ -193,11 +179,10 @@ export async function hybridSearch(
             if (!row) continue;
             if (kindFilter && row.kind !== kindFilter) continue;
             if (categoryFilter && row.category !== categoryFilter) continue;
-            if (excludeEvents && row.category === "event") continue;
+            if (excludeEvents && row.category === 'event') continue;
             if (since && row.created_at < since) continue;
             if (until && row.created_at > until) continue;
-            if (row.expires_at && new Date(row.expires_at) <= new Date())
-              continue;
+            if (row.expires_at && new Date(row.expires_at) <= new Date()) continue;
 
             const { rowid: _rowid, ...cleanRow } = row;
             idToRowid.set(cleanRow.id, Number(row.rowid));
@@ -212,10 +197,8 @@ export async function hybridSearch(
       }
     }
   } catch (err) {
-    if (!(err as Error).message?.includes("no such table")) {
-      console.error(
-        `[retrieve] Vector search error: ${(err as Error).message}`,
-      );
+    if (!(err as Error).message?.includes('no such table')) {
+      console.error(`[retrieve] Vector search error: ${(err as Error).message}`);
     }
   }
 
@@ -241,20 +224,15 @@ export async function hybridSearch(
 
     const rowidsToFetch = [...idToRowid.values()];
     try {
-      const placeholders = rowidsToFetch.map(() => "?").join(",");
+      const placeholders = rowidsToFetch.map(() => '?').join(',');
       const vecData = ctx.db
-        .prepare(
-          `SELECT rowid, embedding FROM vault_vec WHERE rowid IN (${placeholders})`,
-        )
+        .prepare(`SELECT rowid, embedding FROM vault_vec WHERE rowid IN (${placeholders})`)
         .all(...rowidsToFetch) as { rowid: number; embedding: Buffer }[];
       for (const row of vecData) {
         const id = rowidToId.get(Number(row.rowid));
         const buf = row.embedding;
         if (id && buf) {
-          embeddingMap.set(
-            id,
-            new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4),
-          );
+          embeddingMap.set(id, new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4));
         }
       }
     } catch {
@@ -292,10 +270,10 @@ export async function hybridSearch(
 function trackAccess(ctx: BaseCtx, entries: SearchResult[]): void {
   if (!entries.length) return;
   try {
-    const placeholders = entries.map(() => "?").join(",");
+    const placeholders = entries.map(() => '?').join(',');
     ctx.db
       .prepare(
-        `UPDATE vault SET hit_count = hit_count + 1, last_accessed_at = datetime('now') WHERE id IN (${placeholders})`,
+        `UPDATE vault SET hit_count = hit_count + 1, last_accessed_at = datetime('now') WHERE id IN (${placeholders})`
       )
       .run(...entries.map((e) => e.id));
   } catch {

package/src/types.ts

@@ -1,4 +1,4 @@
-import type { DatabaseSync, StatementSync } from "node:sqlite";
+import type { DatabaseSync, StatementSync } from 'node:sqlite';
 
 export interface VaultConfig {
   vaultDir: string;