@voidwire/lore 2.0.1 → 2.1.0

package/lib/indexers/commits.ts

@@ -13,9 +13,9 @@
 import { readdirSync, existsSync } from "fs";
 import { join } from "path";
 import { spawnSync } from "child_process";
-import { checkPath, type IndexerContext } from "../indexer";
+import { checkPath, type IndexerContext, type IndexerPlugin } from "../indexer";
 
-export async function indexCommits(ctx: IndexerContext): Promise<void> {
+async function indexCommits(ctx: IndexerContext): Promise<void> {
   const projectsDir = ctx.config.paths.projects;
   if (!checkPath("commits", "paths.projects", projectsDir)) return;
 
@@ -84,3 +84,15 @@ export async function indexCommits(ctx: IndexerContext): Promise<void> {
     }
   }
 }
+
+export const commitsPlugin: IndexerPlugin = {
+  manifest: {
+    name: "commits",
+    description: "Indexes git commit history from project directories",
+    requiredConfig: ["paths.projects"],
+    optionalConfig: [],
+    rebuildExcluded: false,
+  },
+  canRun: (config) => !!config.paths?.projects,
+  run: indexCommits,
+};

package/lib/indexers/development.ts

@@ -10,9 +10,9 @@
 
 import { readdirSync, readFileSync, statSync, existsSync } from "fs";
 import { join } from "path";
-import { checkPath, type IndexerContext } from "../indexer";
+import { checkPath, type IndexerContext, type IndexerPlugin } from "../indexer";
 
-export async function indexDevelopment(ctx: IndexerContext): Promise<void> {
+async function indexDevelopment(ctx: IndexerContext): Promise<void> {
   const projectsDir = ctx.config.paths.projects;
   if (!checkPath("development", "paths.projects", projectsDir)) return;
 
@@ -62,3 +62,15 @@ export async function indexDevelopment(ctx: IndexerContext): Promise<void> {
     }
   }
 }
+
+export const developmentPlugin: IndexerPlugin = {
+  manifest: {
+    name: "development",
+    description: "Indexes project summary files from development directories",
+    requiredConfig: ["paths.projects"],
+    optionalConfig: [],
+    rebuildExcluded: false,
+  },
+  canRun: (config) => !!config.paths?.projects,
+  run: indexDevelopment,
+};

package/lib/indexers/explorations.ts

@@ -12,7 +12,7 @@
 
 import { readdirSync, readFileSync, statSync } from "fs";
 import { join, basename, dirname } from "path";
-import { checkPath, type IndexerContext } from "../indexer";
+import { checkPath, type IndexerContext, type IndexerPlugin } from "../indexer";
 
 function walkMarkdownFiles(dir: string, files: string[] = []): string[] {
   const entries = readdirSync(dir, { withFileTypes: true });
@@ -30,7 +30,7 @@ function walkMarkdownFiles(dir: string, files: string[] = []): string[] {
   return files;
 }
 
-export async function indexExplorations(ctx: IndexerContext): Promise<void> {
+async function indexExplorations(ctx: IndexerContext): Promise<void> {
   const explorationsDir = ctx.config.paths.explorations;
 
   if (!checkPath("explorations", "paths.explorations", explorationsDir)) return;
@@ -84,3 +84,15 @@ export async function indexExplorations(ctx: IndexerContext): Promise<void> {
     }
   }
 }
+
+export const explorationsPlugin: IndexerPlugin = {
+  manifest: {
+    name: "explorations",
+    description: "Indexes markdown files from the explorations directory",
+    requiredConfig: ["paths.explorations"],
+    optionalConfig: [],
+    rebuildExcluded: false,
+  },
+  canRun: (config) => !!config.paths?.explorations,
+  run: indexExplorations,
+};

package/lib/indexers/flux.ts

@@ -14,9 +14,9 @@
 
 import { readdirSync, readFileSync, existsSync, statSync } from "fs";
 import { join, basename } from "path";
-import { checkPath, type IndexerContext } from "../indexer";
+import { checkPath, type IndexerContext, type IndexerPlugin } from "../indexer";
 
-export async function indexFlux(ctx: IndexerContext): Promise<void> {
+async function indexFlux(ctx: IndexerContext): Promise<void> {
   const fluxDir = ctx.config.paths.flux;
   const fluxProjectsDir = ctx.config.paths.flux_projects;
   let found = false;
@@ -141,3 +141,15 @@ function parseFluxFile(
     });
   }
 }
+
+export const fluxPlugin: IndexerPlugin = {
+  manifest: {
+    name: "flux",
+    description: "Indexes todo and idea entries from flux markdown files",
+    requiredConfig: ["paths.flux"],
+    optionalConfig: [],
+    rebuildExcluded: false,
+  },
+  canRun: (config) => !!config.paths?.flux,
+  run: indexFlux,
+};

package/lib/indexers/index.ts

@@ -1,35 +1,35 @@
 /**
  * lib/indexers/index.ts - Indexer registry
  *
- * Maps source names to indexer functions.
- * Populated by tasks 3.1, 3.2, 3.3.
+ * Maps source names to indexer plugins.
+ * Each plugin provides manifest metadata, a canRun gate, and the run function.
  */
 
-import type { IndexerFunction } from "../indexer";
-import { indexReadmes } from "./readmes";
-import { indexDevelopment } from "./development";
-import { indexCaptures } from "./captures";
-import { indexTeachings } from "./teachings";
-import { indexObservations } from "./observations";
-import { indexExplorations } from "./explorations";
-import { indexSessions } from "./sessions";
-import { indexFlux } from "./flux";
-import { indexObsidian } from "./obsidian";
-import { indexCommits } from "./commits";
-import { indexBlogs } from "./blogs";
-import { indexPersonal } from "./personal";
+import type { IndexerPlugin } from "../indexer";
+import { readmesPlugin } from "./readmes";
+import { developmentPlugin } from "./development";
+import { capturesPlugin } from "./captures";
+import { teachingsPlugin } from "./teachings";
+import { observationsPlugin } from "./observations";
+import { explorationsPlugin } from "./explorations";
+import { sessionsPlugin } from "./sessions";
+import { fluxPlugin } from "./flux";
+import { obsidianPlugin } from "./obsidian";
+import { commitsPlugin } from "./commits";
+import { blogsPlugin } from "./blogs";
+import { personalPlugin } from "./personal";
 
-export const indexers: Record<string, IndexerFunction> = {
-  readmes: indexReadmes,
-  development: indexDevelopment,
-  captures: indexCaptures,
-  teachings: indexTeachings,
-  observations: indexObservations,
-  explorations: indexExplorations,
-  sessions: indexSessions,
-  flux: indexFlux,
-  obsidian: indexObsidian,
-  commits: indexCommits,
-  blogs: indexBlogs,
-  personal: indexPersonal,
+export const indexers: Record<string, IndexerPlugin> = {
+  readmes: readmesPlugin,
+  development: developmentPlugin,
+  captures: capturesPlugin,
+  teachings: teachingsPlugin,
+  observations: observationsPlugin,
+  explorations: explorationsPlugin,
+  sessions: sessionsPlugin,
+  flux: fluxPlugin,
+  obsidian: obsidianPlugin,
+  commits: commitsPlugin,
+  blogs: blogsPlugin,
+  personal: personalPlugin,
 };

package/lib/indexers/observations.ts

@@ -11,9 +11,9 @@
  */
 
 import { readFileSync } from "fs";
-import { checkPath, type IndexerContext } from "../indexer";
+import { checkPath, type IndexerContext, type IndexerPlugin } from "../indexer";
 
-export async function indexObservations(ctx: IndexerContext): Promise<void> {
+async function indexObservations(ctx: IndexerContext): Promise<void> {
   const logPath = `${ctx.config.paths.data}/log.jsonl`;
   if (
     !checkPath(
@@ -56,3 +56,15 @@ export async function indexObservations(ctx: IndexerContext): Promise<void> {
     }
   }
 }
+
+export const observationsPlugin: IndexerPlugin = {
+  manifest: {
+    name: "observations",
+    description: "Indexes observation captures from log.jsonl",
+    requiredConfig: ["paths.data"],
+    optionalConfig: [],
+    rebuildExcluded: true,
+  },
+  canRun: (config) => !!config.paths?.data,
+  run: indexObservations,
+};

package/lib/indexers/obsidian.ts

@@ -13,7 +13,7 @@
 
 import { readdirSync, readFileSync, statSync } from "fs";
 import { join, basename, dirname } from "path";
-import { checkPath, type IndexerContext } from "../indexer";
+import { checkPath, type IndexerContext, type IndexerPlugin } from "../indexer";
 
 function walkMarkdownFiles(
   dir: string,
@@ -38,7 +38,7 @@ function walkMarkdownFiles(
   return files;
 }
 
-export async function indexObsidian(ctx: IndexerContext): Promise<void> {
+async function indexObsidian(ctx: IndexerContext): Promise<void> {
   const obsidianDir = ctx.config.paths.obsidian;
 
   if (!checkPath("obsidian", "paths.obsidian", obsidianDir)) return;
@@ -148,3 +148,15 @@ export async function indexObsidian(ctx: IndexerContext): Promise<void> {
     }
   }
 }
+
+export const obsidianPlugin: IndexerPlugin = {
+  manifest: {
+    name: "obsidian",
+    description: "Indexes markdown files from the Obsidian vault",
+    requiredConfig: ["paths.obsidian"],
+    optionalConfig: [],
+    rebuildExcluded: false,
+  },
+  canRun: (config) => !!config.paths?.obsidian,
+  run: indexObsidian,
+};

package/lib/indexers/personal.ts

@@ -12,7 +12,7 @@
 
 import { readFileSync, statSync, existsSync } from "fs";
 import { join } from "path";
-import { checkPath, type IndexerContext } from "../indexer";
+import { checkPath, type IndexerContext, type IndexerPlugin } from "../indexer";
 import { complete } from "@voidwire/llm-core";
 
 function fileMtime(path: string): string {
@@ -91,7 +91,7 @@ async function enrich(type: string, input: string): Promise<string | null> {
   }
 }
 
-export async function indexPersonal(ctx: IndexerContext): Promise<void> {
+async function indexPersonal(ctx: IndexerContext): Promise<void> {
   enrichmentFailures = 0;
   const personalDir = ctx.config.paths.personal;
 
@@ -354,3 +354,15 @@ export async function indexPersonal(ctx: IndexerContext): Promise<void> {
     }
   }
 }
+
+export const personalPlugin: IndexerPlugin = {
+  manifest: {
+    name: "personal",
+    description: "Indexes personal data JSON files (books, contacts, movies, etc.)",
+    requiredConfig: ["paths.personal"],
+    optionalConfig: [],
+    rebuildExcluded: false,
+  },
+  canRun: (config) => !!config.paths?.personal,
+  run: indexPersonal,
+};

package/lib/indexers/readmes.ts

@@ -10,9 +10,9 @@
 
 import { readdirSync, readFileSync, statSync, existsSync } from "fs";
 import { join } from "path";
-import { checkPath, type IndexerContext } from "../indexer";
+import { checkPath, type IndexerContext, type IndexerPlugin } from "../indexer";
 
-export async function indexReadmes(ctx: IndexerContext): Promise<void> {
+async function indexReadmes(ctx: IndexerContext): Promise<void> {
   const projectsDir = ctx.config.paths.projects;
   if (!checkPath("readmes", "paths.projects", projectsDir)) return;
 
@@ -43,3 +43,15 @@ export async function indexReadmes(ctx: IndexerContext): Promise<void> {
     }
   }
 }
+
+export const readmesPlugin: IndexerPlugin = {
+  manifest: {
+    name: "readmes",
+    description: "Indexes README.md files from project directories",
+    requiredConfig: ["paths.projects"],
+    optionalConfig: [],
+    rebuildExcluded: false,
+  },
+  canRun: (config) => !!config.paths?.projects,
+  run: indexReadmes,
+};

package/lib/indexers/sessions.ts

@@ -12,7 +12,7 @@
 
 import { readdirSync, readFileSync } from "fs";
 import { join } from "path";
-import { checkPath, type IndexerContext } from "../indexer";
+import { checkPath, type IndexerContext, type IndexerPlugin } from "../indexer";
 
 interface SessionData {
   project: string;
@@ -26,7 +26,7 @@ interface SessionData {
   lastTs: string;
 }
 
-export async function indexSessions(ctx: IndexerContext): Promise<void> {
+async function indexSessions(ctx: IndexerContext): Promise<void> {
   // Check each event directory individually for clear diagnostics
   const eventDirs: string[] = [];
   for (const { name, path } of [
@@ -136,3 +136,15 @@ export async function indexSessions(ctx: IndexerContext): Promise<void> {
     });
   }
 }
+
+export const sessionsPlugin: IndexerPlugin = {
+  manifest: {
+    name: "sessions",
+    description: "Indexes session event telemetry aggregated by session ID",
+    requiredConfig: ["paths.session_events", "paths.sable_events"],
+    optionalConfig: [],
+    rebuildExcluded: false,
+  },
+  canRun: (config) => !!config.paths?.session_events && !!config.paths?.sable_events,
+  run: indexSessions,
+};

package/lib/indexers/teachings.ts

@@ -11,9 +11,9 @@
  */
 
 import { readFileSync } from "fs";
-import { checkPath, type IndexerContext } from "../indexer";
+import { checkPath, type IndexerContext, type IndexerPlugin } from "../indexer";
 
-export async function indexTeachings(ctx: IndexerContext): Promise<void> {
+async function indexTeachings(ctx: IndexerContext): Promise<void> {
   const logPath = `${ctx.config.paths.data}/log.jsonl`;
   if (
     !checkPath(
@@ -55,3 +55,15 @@ export async function indexTeachings(ctx: IndexerContext): Promise<void> {
     }
   }
 }
+
+export const teachingsPlugin: IndexerPlugin = {
+  manifest: {
+    name: "teachings",
+    description: "Indexes teaching captures from log.jsonl",
+    requiredConfig: ["paths.data"],
+    optionalConfig: [],
+    rebuildExcluded: true,
+  },
+  canRun: (config) => !!config.paths?.data,
+  run: indexTeachings,
+};

package/lib/info.ts

@@ -5,9 +5,8 @@
  * Uses Bun's built-in SQLite for zero external dependencies.
  */
 
-import { Database } from "bun:sqlite";
 import { existsSync } from "fs";
-import { getDatabasePath } from "./db.js";
+import { getDatabasePath, openDatabaseBasic } from "./db.js";
 import { projects as getProjects } from "./projects.js";
 
 export interface SourceInfo {
@@ -41,7 +40,7 @@ export function info(): InfoOutput {
     };
   }
 
-  const db = new Database(dbPath, { readonly: true });
+  const db = openDatabaseBasic(true);
 
   try {
     // Get distinct sources with counts

package/lib/init.ts

@@ -12,6 +12,7 @@
 import { existsSync, mkdirSync, writeFileSync } from "fs";
 import { homedir } from "os";
 import { Database } from "bun:sqlite";
+import { getLoadablePath, load } from "sqlite-vec";
 
 interface DetectedPaths {
   obsidian?: string;
@@ -76,7 +77,7 @@ export async function runInit(homeOverride?: string): Promise<void> {
   }
 
   // 5. Initialize or verify database
-  initDatabase(dbPath, detected.customSqlite, detected.sqliteVec);
+  initDatabase(dbPath, detected.customSqlite);
 }
 
 async function detectPaths(home: string): Promise<DetectedPaths> {
@@ -122,7 +123,15 @@ async function detectPaths(home: string): Promise<DetectedPaths> {
 function detectSqliteVec(): string | undefined {
   const ext = process.platform === "darwin" ? "dylib" : "so";
 
-  // Strategy 1: brew --prefix sqlite-vec
+  // Strategy 1: npm package (preferred — self-resolves platform binary)
+  try {
+    const npmPath = getLoadablePath();
+    if (existsSync(npmPath)) return npmPath;
+  } catch {
+    // npm package not installed or getLoadablePath() failed — try fallbacks
+  }
+
+  // Strategy 2: brew --prefix sqlite-vec
   const result = Bun.spawnSync(["brew", "--prefix", "sqlite-vec"]);
   if (result.exitCode === 0) {
     const prefix = new TextDecoder().decode(result.stdout).trim();
@@ -130,11 +139,6 @@ function detectSqliteVec(): string | undefined {
     if (existsSync(candidate)) return candidate;
   }
 
-  // Strategy 2: npm-installed sqlite-vec (node_modules)
-  const platformPkg = `sqlite-vec-${process.platform}-${process.arch}`;
-  const nmCandidate = `${import.meta.dir}/../node_modules/${platformPkg}/vec0.${ext}`;
-  if (existsSync(nmCandidate)) return nmCandidate;
-
   // Strategy 3: common macOS/Linux system paths
   for (const p of [
     "/opt/homebrew/lib/sqlite-vec/vec0.dylib",
@@ -146,7 +150,7 @@ function detectSqliteVec(): string | undefined {
     if (existsSync(p)) return p;
   }
 
-  // Strategy 4: pip-installed sqlite-vec (Python site-packages)
+  // Strategy 4: pip-installed sqlite-vec (Python site-packages — last resort)
   const pipGlob =
     process.platform === "darwin"
       ? "/opt/homebrew/lib/python3.*/site-packages/sqlite_vec/vec0.dylib"
@@ -189,7 +193,7 @@ ${pathLine("sable_events", detected.sableEvents ? "~/.local/share/sable/events"
 [database]
 sqlite = "${dbPath}"
 ${pathLine("custom_sqlite", detected.customSqlite)}
-${pathLine("sqlite_vec", detected.sqliteVec)}
+# sqlite_vec — no longer needed; loaded from npm package via load(db)
 
 [embedding]
 model = "nomic-ai/nomic-embed-text-v1.5"
@@ -197,11 +201,7 @@ dimensions = 768
 `;
 }
 
-function initDatabase(
-  dbPath: string,
-  customSqlite: string | undefined,
-  vecPath: string | undefined,
-): void {
+function initDatabase(dbPath: string, customSqlite: string | undefined): void {
   if (customSqlite && existsSync(customSqlite)) {
     try {
       Database.setCustomSQLite(customSqlite);
@@ -215,21 +215,17 @@ function initDatabase(
   const db = new Database(dbPath);
   db.exec("PRAGMA journal_mode=WAL");
 
-  // Load sqlite-vec for vec0 table creation
+  // Load sqlite-vec from npm package for vec0 table creation
   let vecLoaded = false;
-  if (vecPath && existsSync(vecPath)) {
-    try {
-      db.loadExtension(vecPath);
-      vecLoaded = true;
-    } catch (e) {
-      // Extension loading fails if custom_sqlite wasn't set (Bun's built-in
-      // sqlite doesn't support extensions). Warn instead of crashing.
-      console.warn(
-        `sqlite-vec found at ${vecPath} but extension loading not supported — need custom_sqlite`,
-      );
-    }
-  } else {
-    console.warn("sqlite-vec not found — embeddings table not created");
+  try {
+    load(db);
+    vecLoaded = true;
+  } catch (e) {
+    // Extension loading fails if custom_sqlite wasn't set (Bun's built-in
+    // sqlite doesn't support extensions). Warn instead of crashing.
+    console.warn(
+      `sqlite-vec load failed — extension loading may not be supported without custom_sqlite`,
+    );
   }
 
   // Create tables (IF NOT EXISTS = idempotent)

package/lib/list.ts

@@ -5,9 +5,8 @@
  * Uses Bun's built-in SQLite for zero external dependencies.
  */
 
-import { Database } from "bun:sqlite";
-import { existsSync } from "fs";
-import { getDatabasePath } from "./db.js";
+import type { Database } from "bun:sqlite";
+import { openDatabaseBasic } from "./db.js";
 
 // Source types - data sources that can be listed
 export type Source =
@@ -194,13 +193,7 @@ export function list(source: Source, options: ListOptions = {}): ListResult {
     );
   }
 
-  const dbPath = getDatabasePath();
-
-  if (!existsSync(dbPath)) {
-    throw new Error(`Database not found: ${dbPath}. Run lore-db-init first.`);
-  }
-
-  const db = new Database(dbPath, { readonly: true });
+  const db = openDatabaseBasic(true);
 
   try {
     let entries: ListEntry[];

package/lib/projects.ts

@@ -5,9 +5,8 @@
  * project-scoped sources.
  */
 
-import { Database } from "bun:sqlite";
 import { existsSync } from "fs";
-import { getDatabasePath } from "./db.js";
+import { getDatabasePath, openDatabaseBasic } from "./db.js";
 
 const PROJECT_SOURCES = [
   "commits",
@@ -30,7 +29,7 @@ export function projects(): string[] {
     return [];
   }
 
-  const db = new Database(dbPath, { readonly: true });
+  const db = openDatabaseBasic(true);
 
   try {
     const placeholders = PROJECT_SOURCES.map(() => "?").join(", ");

package/lib/realtime.ts

@@ -15,12 +15,7 @@
  */
 
 import { Database } from "bun:sqlite";
-import {
-  embedDocuments,
-  MODEL_NAME,
-  EMBEDDING_DIM,
-  serializeEmbedding,
-} from "./semantic.js";
+import { embedDocuments, MODEL_NAME, serializeEmbedding } from "./semantic.js";
 import { openDatabase } from "./db.js";
 import { hashContent, getCachedEmbedding, cacheEmbedding } from "./cache.js";
 import type { CaptureEvent } from "./capture.js";
@@ -262,7 +257,7 @@ async function embedWithCache(
       const embedding = embeddings[i];
 
       results[idx] = embedding;
-      cacheEmbedding(db, hashContent(content), embedding, MODEL_NAME);
+      cacheEmbedding(db, hashContent(content), embedding, MODEL_NAME());
     }
   }
 

package/lib/search.ts

@@ -5,9 +5,7 @@
  * Uses Bun's built-in SQLite for zero external dependencies.
  */
 
-import { Database } from "bun:sqlite";
-import { existsSync } from "fs";
-import { getDatabasePath } from "./db.js";
+import { openDatabaseBasic } from "./db.js";
 
 export interface SearchResult {
   rowid: number;
@@ -53,13 +51,7 @@ export function search(
   query: string,
   options: SearchOptions = {},
 ): SearchResult[] {
-  const dbPath = getDatabasePath();
-
-  if (!existsSync(dbPath)) {
-    throw new Error(`Database not found: ${dbPath}. Run lore-db-init first.`);
-  }
-
-  const db = new Database(dbPath, { readonly: true });
+  const db = openDatabaseBasic(true);
 
   try {
     const limit = options.limit ?? 20;
@@ -117,18 +109,73 @@ export function search(
 }
 
 /**
- * List all available sources in the database
+ * Raw content result from FTS5 query (no snippet formatting).
+ * Used by contradiction detection where clean content is needed for LLM comparison.
+ */
+export interface RawSearchResult {
+  rowid: number;
+  source: string;
+  title: string;
+  content: string;
+  metadata: string;
+  topic: string;
+  type: string;
+  timestamp: string;
+}
+
+/**
+ * Search the FTS5 database returning raw content (no snippet markers).
  *
- * @returns Array of source names with entry counts
+ * Unlike search(), this returns the raw content column without FTS5 snippet()
+ * formatting. Used by contradiction detection where snippet markers (arrows)
+ * would corrupt LLM text comparison.
+ *
+ * @param query - Text to match against FTS5 index
+ * @param options - Optional source filter and result limit
+ * @returns Array of raw search results ranked by relevance
+ * @throws Error if database doesn't exist or query fails
  */
-export function listSources(): { source: string; count: number }[] {
-  const dbPath = getDatabasePath();
+export function rawSearch(
+  query: string,
+  options: { source?: string; limit?: number } = {},
+): RawSearchResult[] {
+  const db = openDatabaseBasic(true);
+
+  try {
+    const limit = options.limit ?? 5;
+
+    const conditions: string[] = ["search MATCH ?"];
+    const params: (string | number)[] = [escapeFts5Query(query)];
+
+    if (options.source) {
+      conditions.push("source = ?");
+      params.push(options.source);
+    }
+
+    params.push(limit);
+
+    const sql = `
+      SELECT rowid, source, title, content, metadata, topic, type, timestamp
+      FROM search
+      WHERE ${conditions.join(" AND ")}
+      ORDER BY rank
+      LIMIT ?
+    `;
 
-  if (!existsSync(dbPath)) {
-    throw new Error(`Database not found: ${dbPath}. Run lore-db-init first.`);
+    const stmt = db.prepare(sql);
+    return stmt.all(...params) as RawSearchResult[];
+  } finally {
+    db.close();
   }
+}
 
-  const db = new Database(dbPath, { readonly: true });
+/**
+ * List all available sources in the database
+ *
+ * @returns Array of source names with entry counts
+ */
+export function listSources(): { source: string; count: number }[] {
+  const db = openDatabaseBasic(true);
 
   try {
     const stmt = db.prepare(`