@context-vault/core 2.17.0 → 3.0.0

package/src/index.ts

@@ -0,0 +1,351 @@
+import { readFileSync, readdirSync, existsSync, unlinkSync } from "node:fs";
+import { join, basename } from "node:path";
+import { dirToKind, walkDir, ulid } from "./files.js";
+import { categoryFor, defaultTierFor, CATEGORY_DIRS } from "./categories.js";
+import { parseFrontmatter, parseEntryFromMarkdown } from "./frontmatter.js";
+import { embedBatch } from "./embed.js";
+import type { BaseCtx, IndexEntryInput, ReindexStats } from "./types.js";
+
+const EXCLUDED_DIRS = new Set(["projects", "_archive"]);
+const EXCLUDED_FILES = new Set(["context.md", "memory.md", "README.md"]);
+const EMBED_BATCH_SIZE = 32;
+
+export async function indexEntry(
+  ctx: BaseCtx,
+  entry: IndexEntryInput & { supersedes?: string[] | null; related_to?: string[] | null },
+): Promise<void> {
+  const {
+    id, kind, category, title, body, meta, tags, source,
+    filePath, createdAt, identity_key, expires_at, source_files, tier,
+  } = entry;
+
+  if (expires_at && new Date(expires_at) <= new Date()) return;
+
+  const tagsJson = tags ? JSON.stringify(tags) : null;
+  const metaJson = meta ? JSON.stringify(meta) : null;
+  const sourceFilesJson = source_files ? JSON.stringify(source_files) : null;
+  const cat = category || categoryFor(kind);
+  const effectiveTier = tier || defaultTierFor(kind);
+
+  let wasUpdate = false;
+
+  if (cat === "entity" && identity_key) {
+    const existing = ctx.stmts.getByIdentityKey.get(kind, identity_key) as Record<string, unknown> | undefined;
+    if (existing) {
+      ctx.stmts.upsertByIdentityKey.run(
+        title || null, body, metaJson, tagsJson,
+        source || "claude-code", cat, filePath,
+        expires_at || null, sourceFilesJson,
+        kind, identity_key,
+      );
+      wasUpdate = true;
+    }
+  }
+
+  if (!wasUpdate) {
+    try {
+      ctx.stmts.insertEntry.run(
+        id, kind, cat, title || null, body, metaJson, tagsJson,
+        source || "claude-code", filePath,
+        identity_key || null, expires_at || null,
+        createdAt, createdAt, sourceFilesJson, effectiveTier,
+      );
+    } catch (e) {
+      if ((e as Error).message.includes("UNIQUE constraint")) {
+        ctx.stmts.updateEntry.run(
+          title || null, body, metaJson, tagsJson,
+          source || "claude-code", cat,
+          identity_key || null, expires_at || null, filePath,
+        );
+        if (sourceFilesJson !== null && ctx.stmts.updateSourceFiles) {
+          const entryRow = ctx.stmts.getRowidByPath.get(filePath) as { rowid: number } | undefined;
+          if (entryRow) {
+            const idRow = ctx.db
+              .prepare("SELECT id FROM vault WHERE file_path = ?")
+              .get(filePath) as { id: string } | undefined;
+            if (idRow)
+              ctx.stmts.updateSourceFiles.run(sourceFilesJson, idRow.id);
+          }
+        }
+        wasUpdate = true;
+      } else {
+        throw e;
+      }
+    }
+  }
+
+  const rowidResult = wasUpdate
+    ? ctx.stmts.getRowidByPath.get(filePath) as { rowid: number } | undefined
+    : ctx.stmts.getRowid.get(id) as { rowid: number } | undefined;
+
+  if (!rowidResult || rowidResult.rowid == null) {
+    throw new Error(
+      `Could not find rowid for entry: ${wasUpdate ? `file_path=${filePath}` : `id=${id}`}`,
+    );
+  }
+
+  const rowid = Number(rowidResult.rowid);
+  if (!Number.isFinite(rowid) || rowid < 1) {
+    throw new Error(
+      `Invalid rowid retrieved: ${rowidResult.rowid} (type: ${typeof rowidResult.rowid})`,
+    );
+  }
+
+  if (cat !== "event") {
+    const embeddingText = [title, body].filter(Boolean).join(" ");
+    const embedding = await ctx.embed(embeddingText);
+
+    if (embedding) {
+      try { ctx.deleteVec(rowid); } catch { /* no-op */ }
+      ctx.insertVec(rowid, embedding);
+    }
+  }
+}
+
+export async function pruneExpired(ctx: BaseCtx): Promise<number> {
+  const expired = ctx.db
+    .prepare(
+      "SELECT id, file_path FROM vault WHERE expires_at IS NOT NULL AND expires_at <= datetime('now')",
+    )
+    .all() as { id: string; file_path: string | null }[];
+
+  for (const row of expired) {
+    if (row.file_path) {
+      try { unlinkSync(row.file_path); } catch {}
+    }
+    const vRowid = (ctx.stmts.getRowid.get(row.id) as { rowid: number } | undefined)?.rowid;
+    if (vRowid) {
+      try { ctx.deleteVec(Number(vRowid)); } catch {}
+    }
+    ctx.stmts.deleteEntry.run(row.id);
+  }
+
+  return expired.length;
+}
+
+export async function reindex(
+  ctx: BaseCtx,
+  opts: { fullSync?: boolean } = {},
+): Promise<ReindexStats> {
+  const { fullSync = true } = opts;
+  const stats: ReindexStats = { added: 0, updated: 0, removed: 0, unchanged: 0 };
+
+  if (!existsSync(ctx.config.vaultDir)) return stats;
+
+  const upsertEntry = ctx.db.prepare(
+    `INSERT OR IGNORE INTO vault (id, kind, category, title, body, meta, tags, source, file_path, identity_key, expires_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+  );
+
+  const kindEntries: { kind: string; dir: string }[] = [];
+  const topDirs = readdirSync(ctx.config.vaultDir, {
+    withFileTypes: true,
+  }).filter(
+    (d) =>
+      d.isDirectory() && !EXCLUDED_DIRS.has(d.name) && !d.name.startsWith("_"),
+  );
+
+  for (const d of topDirs) {
+    if (CATEGORY_DIRS.has(d.name)) {
+      const catDir = join(ctx.config.vaultDir, d.name);
+      const subDirs = readdirSync(catDir, { withFileTypes: true }).filter(
+        (sd) => sd.isDirectory() && !sd.name.startsWith("_"),
+      );
+      for (const sd of subDirs) {
+        kindEntries.push({
+          kind: dirToKind(sd.name),
+          dir: join(catDir, sd.name),
+        });
+      }
+    } else {
+      kindEntries.push({
+        kind: dirToKind(d.name),
+        dir: join(ctx.config.vaultDir, d.name),
+      });
+    }
+  }
+
+  const pendingEmbeds: { rowid: number; text: string }[] = [];
+
+  ctx.db.exec("BEGIN");
+  try {
+    for (const { kind, dir } of kindEntries) {
+      const category = categoryFor(kind);
+      const mdFiles = walkDir(dir).filter(
+        (f) => !EXCLUDED_FILES.has(basename(f.filePath)),
+      );
+
+      const dbRows = ctx.db
+        .prepare(
+          "SELECT id, file_path, body, title, tags, meta, related_to FROM vault WHERE kind = ?",
+        )
+        .all(kind) as Record<string, unknown>[];
+      const dbByPath = new Map(dbRows.map((r) => [r.file_path as string, r]));
+      const diskPaths = new Set(mdFiles.map((e) => e.filePath));
+
+      for (const { filePath, relDir } of mdFiles) {
+        const existing = dbByPath.get(filePath);
+
+        if (!fullSync && existing) {
+          stats.unchanged++;
+          continue;
+        }
+
+        const raw = readFileSync(filePath, "utf-8");
+        if (!raw.startsWith("---\n")) {
+          console.error(`[reindex] skipping (no frontmatter): ${filePath}`);
+          continue;
+        }
+        const { meta: fmMeta, body: rawBody } = parseFrontmatter(raw);
+        const parsed = parseEntryFromMarkdown(kind, rawBody, fmMeta);
+
+        const identity_key = (fmMeta.identity_key as string) || null;
+        const expires_at = (fmMeta.expires_at as string) || null;
+        const related_to = Array.isArray(fmMeta.related_to)
+          ? (fmMeta.related_to as string[])
+          : null;
+        const relatedToJson = related_to?.length
+          ? JSON.stringify(related_to)
+          : null;
+
+        const meta: Record<string, unknown> = { ...(parsed.meta || {}) };
+        if (relDir) meta.folder = relDir;
+        else delete meta.folder;
+        const metaJson = Object.keys(meta).length ? JSON.stringify(meta) : null;
+
+        if (!existing) {
+          const id = (fmMeta.id as string) || ulid();
+          const tagsJson = fmMeta.tags ? JSON.stringify(fmMeta.tags) : null;
+          const created = (fmMeta.created as string) || new Date().toISOString();
+
+          const result = upsertEntry.run(
+            id, kind, category, parsed.title || null, parsed.body,
+            metaJson, tagsJson, (fmMeta.source as string) || "file",
+            filePath, identity_key, expires_at,
+            created, (fmMeta.updated as string) || created,
+          );
+          if ((result as { changes: number }).changes > 0) {
+            if (relatedToJson && ctx.stmts.updateRelatedTo) {
+              ctx.stmts.updateRelatedTo.run(relatedToJson, id);
+            }
+            if (category !== "event") {
+              const rowidResult = ctx.stmts.getRowid.get(id) as { rowid: number } | undefined;
+              if (rowidResult?.rowid) {
+                const embeddingText = [parsed.title, parsed.body]
+                  .filter(Boolean)
+                  .join(" ");
+                pendingEmbeds.push({
+                  rowid: rowidResult.rowid,
+                  text: embeddingText,
+                });
+              }
+            }
+            stats.added++;
+          } else {
+            stats.unchanged++;
+          }
+        } else if (fullSync) {
+          const tagsJson = fmMeta.tags ? JSON.stringify(fmMeta.tags) : null;
+          const titleChanged = (parsed.title || null) !== ((existing.title as string) || null);
+          const bodyChanged = (existing.body as string) !== parsed.body;
+          const tagsChanged = tagsJson !== ((existing.tags as string) || null);
+          const metaChanged = metaJson !== ((existing.meta as string) || null);
+          const relatedToChanged = relatedToJson !== ((existing.related_to as string) || null);
+
+          if (bodyChanged || titleChanged || tagsChanged || metaChanged || relatedToChanged) {
+            ctx.stmts.updateEntry.run(
+              parsed.title || null, parsed.body, metaJson, tagsJson,
+              (fmMeta.source as string) || "file", category,
+              identity_key, expires_at, filePath,
+            );
+            if (relatedToChanged && ctx.stmts.updateRelatedTo) {
+              ctx.stmts.updateRelatedTo.run(relatedToJson, existing.id as string);
+            }
+
+            if ((bodyChanged || titleChanged) && category !== "event") {
+              const rowid = (ctx.stmts.getRowid.get(existing.id as string) as { rowid: number } | undefined)?.rowid;
+              if (rowid) {
+                const embeddingText = [parsed.title, parsed.body]
+                  .filter(Boolean)
+                  .join(" ");
+                pendingEmbeds.push({ rowid, text: embeddingText });
+              }
+            }
+            stats.updated++;
+          } else {
+            stats.unchanged++;
+          }
+        } else {
+          stats.unchanged++;
+        }
+      }
+
+      if (fullSync) {
+        for (const [dbPath, row] of dbByPath) {
+          if (!diskPaths.has(dbPath)) {
+            const vRowid = (ctx.stmts.getRowid.get(row.id as string) as { rowid: number } | undefined)?.rowid;
+            if (vRowid) {
+              try { ctx.deleteVec(vRowid); } catch {}
+            }
+            ctx.stmts.deleteEntry.run(row.id as string);
+            stats.removed++;
+          }
+        }
+      }
+    }
+
+    if (fullSync) {
+      const indexedKinds = new Set(kindEntries.map((ke) => ke.kind));
+      const allDbKinds = ctx.db
+        .prepare("SELECT DISTINCT kind FROM vault")
+        .all() as { kind: string }[];
+      for (const { kind } of allDbKinds) {
+        if (!indexedKinds.has(kind)) {
+          const orphaned = ctx.db
+            .prepare("SELECT id, rowid FROM vault WHERE kind = ?")
+            .all(kind) as { id: string; rowid: number }[];
+          for (const row of orphaned) {
+            try { ctx.deleteVec(row.rowid); } catch {}
+            ctx.stmts.deleteEntry.run(row.id);
+            stats.removed++;
+          }
+        }
+      }
+    }
+
+    const expired = ctx.db
+      .prepare(
+        "SELECT id, file_path FROM vault WHERE expires_at IS NOT NULL AND expires_at <= datetime('now')",
+      )
+      .all() as { id: string; file_path: string | null }[];
+
+    for (const row of expired) {
+      if (row.file_path) {
+        try { unlinkSync(row.file_path); } catch {}
+      }
+      const vRowid = (ctx.stmts.getRowid.get(row.id) as { rowid: number } | undefined)?.rowid;
+      if (vRowid) {
+        try { ctx.deleteVec(Number(vRowid)); } catch {}
+      }
+      ctx.stmts.deleteEntry.run(row.id);
+      stats.removed++;
+    }
+
+    ctx.db.exec("COMMIT");
+  } catch (e) {
+    ctx.db.exec("ROLLBACK");
+    throw e;
+  }
+
+  for (let i = 0; i < pendingEmbeds.length; i += EMBED_BATCH_SIZE) {
+    const batch = pendingEmbeds.slice(i, i + EMBED_BATCH_SIZE);
+    const embeddings = await embedBatch(batch.map((e) => e.text));
+    for (let j = 0; j < batch.length; j++) {
+      if (embeddings[j]) {
+        try { ctx.deleteVec(batch[j].rowid); } catch {}
+        ctx.insertVec(batch[j].rowid, embeddings[j]!);
+      }
+    }
+  }
+
+  return stats;
+}

package/src/ingest-url.ts

@@ -0,0 +1,99 @@
+export function htmlToMarkdown(html: string): string {
+  let md = html;
+  md = md.replace(/<script[\s\S]*?<\/script>/gi, "");
+  md = md.replace(/<style[\s\S]*?<\/style>/gi, "");
+  md = md.replace(/<nav[\s\S]*?<\/nav>/gi, "");
+  md = md.replace(/<header[\s\S]*?<\/header>/gi, "");
+  md = md.replace(/<footer[\s\S]*?<\/footer>/gi, "");
+  md = md.replace(/<aside[\s\S]*?<\/aside>/gi, "");
+  md = md.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, (_, c: string) => `\n# ${stripTags(c).trim()}\n`);
+  md = md.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, (_, c: string) => `\n## ${stripTags(c).trim()}\n`);
+  md = md.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, (_, c: string) => `\n### ${stripTags(c).trim()}\n`);
+  md = md.replace(/<h4[^>]*>([\s\S]*?)<\/h4>/gi, (_, c: string) => `\n#### ${stripTags(c).trim()}\n`);
+  md = md.replace(/<h5[^>]*>([\s\S]*?)<\/h5>/gi, (_, c: string) => `\n##### ${stripTags(c).trim()}\n`);
+  md = md.replace(/<h6[^>]*>([\s\S]*?)<\/h6>/gi, (_, c: string) => `\n###### ${stripTags(c).trim()}\n`);
+  md = md.replace(/<a[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi, (_, href: string, text: string) => {
+    const cleanText = stripTags(text).trim();
+    return cleanText ? `[${cleanText}](${href})` : "";
+  });
+  md = md.replace(/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (_, c: string) => `\n\`\`\`\n${decodeEntities(c).trim()}\n\`\`\`\n`);
+  md = md.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, (_, c: string) => `\n\`\`\`\n${decodeEntities(stripTags(c)).trim()}\n\`\`\`\n`);
+  md = md.replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, (_, c: string) => `\`${decodeEntities(c).trim()}\``);
+  md = md.replace(/<(strong|b)[^>]*>([\s\S]*?)<\/\1>/gi, (_, __: string, c: string) => `**${stripTags(c).trim()}**`);
+  md = md.replace(/<(em|i)[^>]*>([\s\S]*?)<\/\1>/gi, (_, __: string, c: string) => `*${stripTags(c).trim()}*`);
+  md = md.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_, c: string) => `- ${stripTags(c).trim()}\n`);
+  md = md.replace(/<br\s*\/?>/gi, "\n");
+  md = md.replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, (_, c: string) => `\n${stripTags(c).trim()}\n`);
+  md = md.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, (_, c: string) => {
+    return "\n" + stripTags(c).trim().split("\n").map((l: string) => `> ${l}`).join("\n") + "\n";
+  });
+  md = stripTags(md);
+  md = decodeEntities(md);
+  md = md.replace(/\n{3,}/g, "\n\n").trim();
+  return md;
+}
+
+function stripTags(html: string): string {
+  return html.replace(/<[^>]+>/g, "");
+}
+
+function decodeEntities(text: string): string {
+  return text
+    .replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&nbsp;/g, " ")
+    .replace(/&#(\d+);/g, (_, n: string) => String.fromCharCode(parseInt(n, 10)))
+    .replace(/&#x([0-9a-f]+);/gi, (_, n: string) => String.fromCharCode(parseInt(n, 16)));
+}
+
+export function extractHtmlContent(html: string, _url: string): { title: string; body: string } {
+  const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+  const title = titleMatch ? stripTags(decodeEntities(titleMatch[1])).trim() : "";
+  let contentHtml = "";
+  const articleMatch = html.match(/<article[^>]*>([\s\S]*?)<\/article>/i);
+  const mainMatch = html.match(/<main[^>]*>([\s\S]*?)<\/main>/i);
+  if (articleMatch) contentHtml = articleMatch[1];
+  else if (mainMatch) contentHtml = mainMatch[1];
+  else {
+    const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
+    contentHtml = bodyMatch ? bodyMatch[1] : html;
+  }
+  const body = htmlToMarkdown(contentHtml);
+  return { title, body };
+}
+
+export async function ingestUrl(
+  url: string,
+  opts: { kind?: string; tags?: string[]; source?: string; maxBodyLength?: number; timeoutMs?: number } = {},
+): Promise<{ kind: string; title: string; body: string; tags: string[]; meta: Record<string, unknown>; source: string }> {
+  const { kind = "reference", tags = [], source, maxBodyLength = 50000, timeoutMs = 15000 } = opts;
+  let domain: string;
+  try { domain = new URL(url).hostname; } catch { throw new Error(`Invalid URL: ${url}`); }
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+  let response: Response;
+  try {
+    response = await fetch(url, {
+      signal: controller.signal,
+      headers: { "User-Agent": "ContextVault/1.0 (+https://github.com/fellanH/context-vault)", Accept: "text/html,application/xhtml+xml,text/plain,*/*" },
+    });
+  } catch (err) {
+    if ((err as Error).name === "AbortError") throw new Error(`Request timed out after ${timeoutMs}ms`);
+    throw new Error(`Fetch failed: ${(err as Error).message}`);
+  } finally { clearTimeout(timeout); }
+  if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+  const contentType = response.headers.get("content-type") || "";
+  const html = await response.text();
+  let title: string, body: string;
+  if (contentType.includes("text/html") || contentType.includes("application/xhtml")) {
+    const extracted = extractHtmlContent(html, url);
+    title = extracted.title; body = extracted.body;
+  } else { title = domain; body = html; }
+  if (body.length > maxBodyLength) body = body.slice(0, maxBodyLength) + "\n\n[Content truncated]";
+  if (!body.trim()) throw new Error("No readable content extracted from URL");
+  return {
+    kind, title: title || domain, body,
+    tags: [...tags, "web-import"],
+    meta: { url, domain, fetched_at: new Date().toISOString(), content_type: contentType.split(";")[0].trim() || "text/html" },
+    source: source || domain,
+  };
+}

package/src/main.ts

@@ -0,0 +1,111 @@
+// Types
+export type {
+  VaultConfig,
+  RecallConfig,
+  ConsolidationConfig,
+  GrowthThresholds,
+  PreparedStatements,
+  VaultEntry,
+  SearchResult,
+  CaptureInput,
+  CaptureResult,
+  IndexEntryInput,
+  ReindexStats,
+  BaseCtx,
+  SearchOptions,
+} from "./types.js";
+
+// Constants
+export {
+  APP_URL,
+  API_URL,
+  MARKETING_URL,
+  GITHUB_ISSUES_URL,
+  MAX_BODY_LENGTH,
+  MAX_TITLE_LENGTH,
+  MAX_KIND_LENGTH,
+  MAX_TAG_LENGTH,
+  MAX_TAGS_COUNT,
+  MAX_META_LENGTH,
+  MAX_SOURCE_LENGTH,
+  MAX_IDENTITY_KEY_LENGTH,
+  DEFAULT_GROWTH_THRESHOLDS,
+  DEFAULT_LIFECYCLE,
+} from "./constants.js";
+
+// Categories
+export {
+  categoryFor,
+  categoryDirFor,
+  defaultTierFor,
+  CATEGORY_DIRS,
+  KIND_STALENESS_DAYS,
+} from "./categories.js";
+
+// Config
+export { parseArgs, resolveConfig } from "./config.js";
+
+// Files
+export {
+  ulid,
+  slugify,
+  kindToDir,
+  dirToKind,
+  normalizeKind,
+  kindToPath,
+  safeJoin,
+  walkDir,
+} from "./files.js";
+
+// Frontmatter
+export {
+  formatFrontmatter,
+  parseFrontmatter,
+  extractCustomMeta,
+  parseEntryFromMarkdown,
+} from "./frontmatter.js";
+
+// Formatters
+export { formatBody } from "./formatters.js";
+
+// Database
+export {
+  SCHEMA_DDL,
+  NativeModuleError,
+  initDatabase,
+  prepareStatements,
+  insertVec,
+  deleteVec,
+  testConnection,
+} from "./db.js";
+
+// Embeddings
+export { embed, embedBatch, resetEmbedPipeline, isEmbedAvailable } from "./embed.js";
+
+// Index (reindex + indexEntry)
+export { indexEntry, reindex, pruneExpired } from "./index.js";
+
+// Search (retrieve)
+export {
+  hybridSearch,
+  buildFtsQuery,
+  buildFilterClauses,
+  recencyBoost,
+  recencyDecayScore,
+  dotProduct,
+  reciprocalRankFusion,
+} from "./search.js";
+
+// Capture
+export {
+  writeEntry,
+  updateEntryFile,
+  captureAndIndex,
+} from "./capture.js";
+
+// Ingest URL
+export {
+  htmlToMarkdown,
+  extractHtmlContent,
+  ingestUrl,
+} from "./ingest-url.js";