@voidwire/lore 0.6.0 → 0.6.2

package/cli.ts

@@ -36,6 +36,7 @@ import {
   captureNote,
   captureTeaching,
   captureObservation,
+  indexAndEmbed,
   semanticSearch,
   formatBriefSearch,
   hasEmbeddings,
@@ -522,7 +523,7 @@ Examples:
 // Capture Command
 // ============================================================================
 
-function handleCaptureTask(args: string[]): void {
+async function handleCaptureTask(args: string[]): Promise<void> {
   const parsed = parseArgs(args);
 
   const required = ["topic", "name", "problem", "solution"];
@@ -546,18 +547,26 @@ function handleCaptureTask(args: string[]): void {
   };
 
   const result = captureTask(input);
-  output(result);
 
-  if (result.success) {
-    console.error("✅ Task logged");
-    process.exit(0);
+  if (result.success && result.event) {
+    try {
+      await indexAndEmbed([result.event]);
+      output(result);
+      console.error("✅ Task logged and indexed");
+      process.exit(0);
+    } catch (error) {
+      output(result);
+      console.error(`✅ Task logged (indexing failed: ${error})`);
+      process.exit(0);
+    }
   } else {
+    output(result);
     console.error(`❌ ${result.error}`);
     process.exit(2);
   }
 }
 
-function handleCaptureKnowledge(args: string[]): void {
+async function handleCaptureKnowledge(args: string[]): Promise<void> {
   const parsed = parseArgs(args);
 
   const required = ["topic", "text", "subtype"];
@@ -573,18 +582,26 @@ function handleCaptureKnowledge(args: string[]): void {
   };
 
   const result = captureKnowledge(input);
-  output(result);
 
-  if (result.success) {
-    console.error("✅ Knowledge logged");
-    process.exit(0);
+  if (result.success && result.event) {
+    try {
+      await indexAndEmbed([result.event]);
+      output(result);
+      console.error("✅ Knowledge logged and indexed");
+      process.exit(0);
+    } catch (error) {
+      output(result);
+      console.error(`✅ Knowledge logged (indexing failed: ${error})`);
+      process.exit(0);
+    }
   } else {
+    output(result);
     console.error(`❌ ${result.error}`);
     process.exit(1);
   }
 }
 
-function handleCaptureNote(args: string[]): void {
+async function handleCaptureNote(args: string[]): Promise<void> {
   const parsed = parseArgs(args);
 
   if (!parsed.has("text")) {
@@ -598,18 +615,26 @@ function handleCaptureNote(args: string[]): void {
   };
 
   const result = captureNote(input);
-  output(result);
 
-  if (result.success) {
-    console.error("✅ Note logged");
-    process.exit(0);
+  if (result.success && result.event) {
+    try {
+      await indexAndEmbed([result.event]);
+      output(result);
+      console.error("✅ Note logged and indexed");
+      process.exit(0);
+    } catch (error) {
+      output(result);
+      console.error(`✅ Note logged (indexing failed: ${error})`);
+      process.exit(0);
+    }
   } else {
+    output(result);
     console.error(`❌ ${result.error}`);
     process.exit(2);
   }
 }
 
-function handleCaptureTeaching(args: string[]): void {
+async function handleCaptureTeaching(args: string[]): Promise<void> {
   const parsed = parseArgs(args);
 
   const required = ["topic", "confidence", "text"];
@@ -626,18 +651,26 @@ function handleCaptureTeaching(args: string[]): void {
   };
 
   const result = captureTeaching(input);
-  output(result);
 
-  if (result.success) {
-    console.error("✅ Teaching logged");
-    process.exit(0);
+  if (result.success && result.event) {
+    try {
+      await indexAndEmbed([result.event]);
+      output(result);
+      console.error("✅ Teaching logged and indexed");
+      process.exit(0);
+    } catch (error) {
+      output(result);
+      console.error(`✅ Teaching logged (indexing failed: ${error})`);
+      process.exit(0);
+    }
   } else {
+    output(result);
     console.error(`❌ ${result.error}`);
     process.exit(2);
   }
 }
 
-function handleCaptureObservation(args: string[]): void {
+async function handleCaptureObservation(args: string[]): Promise<void> {
   const parsed = parseArgs(args);
 
   const required = ["topic", "subtype", "confidence", "text"];
@@ -655,18 +688,26 @@ function handleCaptureObservation(args: string[]): void {
   };
 
   const result = captureObservation(input);
-  output(result);
 
-  if (result.success) {
-    console.error("✅ Observation logged");
-    process.exit(0);
+  if (result.success && result.event) {
+    try {
+      await indexAndEmbed([result.event]);
+      output(result);
+      console.error("✅ Observation logged and indexed");
+      process.exit(0);
+    } catch (error) {
+      output(result);
+      console.error(`✅ Observation logged (indexing failed: ${error})`);
+      process.exit(0);
+    }
   } else {
+    output(result);
     console.error(`❌ ${result.error}`);
     process.exit(2);
   }
 }
 
-function handleCapture(args: string[]): void {
+async function handleCapture(args: string[]): Promise<void> {
   if (hasFlag(args, "help")) {
     showCaptureHelp();
   }
@@ -682,19 +723,19 @@ function handleCapture(args: string[]): void {
 
   switch (captureType) {
     case "task":
-      handleCaptureTask(captureArgs);
+      await handleCaptureTask(captureArgs);
       break;
     case "knowledge":
-      handleCaptureKnowledge(captureArgs);
+      await handleCaptureKnowledge(captureArgs);
       break;
     case "note":
-      handleCaptureNote(captureArgs);
+      await handleCaptureNote(captureArgs);
       break;
     case "teaching":
-      handleCaptureTeaching(captureArgs);
+      await handleCaptureTeaching(captureArgs);
       break;
     case "observation":
-      handleCaptureObservation(captureArgs);
+      await handleCaptureObservation(captureArgs);
       break;
     default:
       fail(
@@ -1052,7 +1093,7 @@ Examples:
   process.exit(0);
 }
 
-function main(): void {
+async function main(): Promise<void> {
   const args = process.argv.slice(2);
 
   // Show global help only when no args or help is first arg
@@ -1065,7 +1106,7 @@ function main(): void {
 
   switch (command) {
     case "search":
-      handleSearch(commandArgs);
+      await handleSearch(commandArgs);
       break;
     case "list":
       handleList(commandArgs);
@@ -1083,7 +1124,7 @@ function main(): void {
       handleAbout(commandArgs);
       break;
     case "capture":
-      handleCapture(commandArgs);
+      await handleCapture(commandArgs);
       break;
     default:
       fail(

package/index.ts

@@ -93,3 +93,6 @@ export {
   type SemanticResult,
   type SemanticSearchOptions,
 } from "./lib/semantic";
+
+// Real-time indexing
+export { indexAndEmbed } from "./lib/realtime";

package/lib/cache.ts

@@ -0,0 +1,86 @@
+/**
+ * lib/cache.ts - Embedding cache utilities
+ *
+ * Hash-based caching to avoid re-embedding unchanged content.
+ * Used by real-time indexing and batch lore-embed-all.
+ */
+
+import { createHash } from "crypto";
+import type { Database } from "bun:sqlite";
+
+/**
+ * Generate SHA256 hash of content for cache lookup
+ */
+export function hashContent(content: string): string {
+  return createHash("sha256").update(content).digest("hex");
+}
+
+/**
+ * Get cached embedding by content hash
+ * @returns embedding array or null if not cached
+ */
+export function getCachedEmbedding(
+  db: Database,
+  hash: string,
+): number[] | null {
+  const stmt = db.prepare(
+    "SELECT embedding FROM embedding_cache WHERE hash = ?",
+  );
+  const row = stmt.get(hash) as { embedding: Uint8Array } | null;
+
+  if (!row) {
+    return null;
+  }
+
+  // Convert blob back to number array
+  const float32 = new Float32Array(row.embedding.buffer);
+  return Array.from(float32);
+}
+
+/**
+ * Store embedding in cache
+ */
+export function cacheEmbedding(
+  db: Database,
+  hash: string,
+  embedding: number[],
+  model: string,
+): void {
+  // Serialize embedding to blob
+  const buffer = new Float32Array(embedding);
+  const blob = new Uint8Array(buffer.buffer);
+
+  const stmt = db.prepare(`
+    INSERT OR REPLACE INTO embedding_cache (hash, embedding, model, dims, created_at)
+    VALUES (?, ?, ?, ?, ?)
+  `);
+
+  stmt.run(hash, blob, model, embedding.length, Date.now());
+}
+
+/**
+ * Check if embedding exists in cache (without retrieving it)
+ */
+export function hasEmbeddingCached(db: Database, hash: string): boolean {
+  const stmt = db.prepare(
+    "SELECT 1 FROM embedding_cache WHERE hash = ? LIMIT 1",
+  );
+  return stmt.get(hash) !== null;
+}
+
+/**
+ * Batch check which hashes are missing from cache
+ * @returns array of hashes that need embedding
+ */
+export function getMissingHashes(db: Database, hashes: string[]): string[] {
+  if (hashes.length === 0) return [];
+
+  const placeholders = hashes.map(() => "?").join(",");
+  const stmt = db.prepare(
+    `SELECT hash FROM embedding_cache WHERE hash IN (${placeholders})`,
+  );
+  const rows = stmt.all(...hashes) as { hash: string }[];
+
+  const cachedSet = new Set(rows.map((r) => r.hash));
+  return hashes.filter((h) => !cachedSet.has(h));
+}

package/lib/capture.ts

@@ -12,6 +12,7 @@ import { homedir } from "os";
 export interface CaptureResult {
   success: boolean;
   error?: string;
+  event?: CaptureEvent;
   [key: string]: unknown;
 }
 
@@ -220,12 +221,15 @@ function writeEvent(event: CaptureEvent): CaptureResult {
   ensureLogDirectory();
 
   const logPath = getLogPath();
-  const eventWithTimestamp = { ...event, timestamp: getTimestamp() };
+  const eventWithTimestamp = {
+    ...event,
+    timestamp: getTimestamp(),
+  } as CaptureEvent;
   const jsonLine = JSON.stringify(eventWithTimestamp) + "\n";
 
   try {
     appendFileSync(logPath, jsonLine, "utf8");
-    return { success: true };
+    return { success: true, event: eventWithTimestamp };
   } catch (error) {
     return {
       success: false,

package/lib/db.ts

@@ -0,0 +1,64 @@
+/**
+ * lib/db.ts - Shared database utilities
+ *
+ * Centralizes SQLite setup and database access for both
+ * semantic search and real-time indexing.
+ */
+
+import { Database } from "bun:sqlite";
+import { existsSync } from "fs";
+import { homedir } from "os";
+
+// Use Homebrew SQLite on macOS to enable extension loading
+// Must be called before any Database instances are created
+const HOMEBREW_SQLITE = "/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib";
+if (existsSync(HOMEBREW_SQLITE)) {
+  Database.setCustomSQLite(HOMEBREW_SQLITE);
+}
+
+/**
+ * Get the path to the lore database
+ */
+export function getDatabasePath(): string {
+  return `${homedir()}/.local/share/lore/lore.db`;
+}
+
+/**
+ * Open the lore database with sqlite-vec extension loaded
+ * @param readonly - Open in readonly mode (default: false)
+ */
+export function openDatabase(readonly = false): Database {
+  const dbPath = getDatabasePath();
+
+  if (!existsSync(dbPath)) {
+    throw new Error(`Database not found: ${dbPath}. Run lore-db-init first.`);
+  }
+
+  const db = new Database(dbPath, { readonly });
+
+  // Load sqlite-vec extension
+  const vecPath = process.env.SQLITE_VEC_PATH;
+  if (!vecPath) {
+    throw new Error(
+      'SQLITE_VEC_PATH not set. Get path with: python3 -c "import sqlite_vec; print(sqlite_vec.loadable_path())"',
+    );
+  }
+
+  db.loadExtension(vecPath);
+
+  return db;
+}
+
+/**
+ * Open the lore database without sqlite-vec (for FTS5-only operations)
+ * @param readonly - Open in readonly mode (default: false)
+ */
+export function openDatabaseBasic(readonly = false): Database {
+  const dbPath = getDatabasePath();
+
+  if (!existsSync(dbPath)) {
+    throw new Error(`Database not found: ${dbPath}. Run lore-db-init first.`);
+  }
+
+  return new Database(dbPath, { readonly });
+}

package/lib/list.ts

@@ -71,7 +71,7 @@ const PROJECT_FIELD: Record<string, string> = {
   commits: "project",
   sessions: "project",
   tasks: "project",
-  insights: "project",
+  insights: "topic",
   captures: "topic",
   teachings: "topic",
   learnings: "topic",
@@ -161,23 +161,28 @@ function queryPersonalType(
   type: string,
   limit?: number,
 ): ListEntry[] {
-  // Query personal source, then filter by type in metadata
-  const sql = limit
-    ? `SELECT title, content, metadata FROM search WHERE source = 'personal' LIMIT ?`
-    : `SELECT title, content, metadata FROM search WHERE source = 'personal'`;
+  // Filter by type in SQL, not JS - avoids LIMIT truncation bug
+  let sql = `
+    SELECT title, content, metadata FROM search
+    WHERE source = 'personal'
+      AND json_extract(metadata, '$.type') = ?
+    ORDER BY json_extract(metadata, '$.timestamp') DESC
+  `;
+  const params: (string | number)[] = [type];
 
-  const stmt = db.prepare(sql);
-  const rows = (limit ? stmt.all(limit * 10) : stmt.all()) as RawRow[]; // Over-fetch for filtering
+  if (limit) {
+    sql += " LIMIT ?";
+    params.push(limit);
+  }
 
-  const filtered = rows
-    .map((row) => ({
-      title: row.title,
-      content: row.content,
-      metadata: JSON.parse(row.metadata || "{}"),
-    }))
-    .filter((entry) => entry.metadata.type === type);
+  const stmt = db.prepare(sql);
+  const rows = stmt.all(...params) as RawRow[];
 
-  return limit ? filtered.slice(0, limit) : filtered;
+  return rows.map((row) => ({
+    title: row.title,
+    content: row.content,
+    metadata: JSON.parse(row.metadata || "{}"),
+  }));
 }
 
 /**

package/lib/projects.ts

@@ -14,7 +14,7 @@ const PROJECT_FIELD: Record<string, string> = {
   commits: "project",
   sessions: "project",
   tasks: "project",
-  insights: "project",
+  insights: "topic",
   captures: "topic",
   teachings: "topic",
   learnings: "topic",

package/lib/realtime.ts

@@ -0,0 +1,265 @@
+/**
+ * lib/realtime.ts - Real-time indexing for captures
+ *
+ * Makes captures immediately searchable (keyword + semantic) without
+ * waiting for batch indexers.
+ *
+ * Usage:
+ *   // CLI - single capture
+ *   const event = captureKnowledge(input);
+ *   await indexAndEmbed([event]);
+ *
+ *   // Hook - batch captures
+ *   const events = captures.map(cap => captureKnowledge(cap));
+ *   await indexAndEmbed(events);
+ */
+
+import { Database } from "bun:sqlite";
+import { existsSync } from "fs";
+import {
+  embedDocuments,
+  getDatabasePath,
+  MODEL_NAME,
+  EMBEDDING_DIM,
+  serializeEmbedding,
+} from "./semantic.js";
+import {
+  hashContent,
+  getCachedEmbedding,
+  cacheEmbedding,
+  getMissingHashes,
+} from "./cache.js";
+import type { CaptureEvent } from "./capture.js";
+
+/**
+ * Index and embed capture events for immediate searchability
+ *
+ * 1. Insert into FTS5 search table (instant keyword search)
+ * 2. Generate embeddings with cache (instant semantic search)
+ * 3. Insert into embeddings table
+ */
+export async function indexAndEmbed(events: CaptureEvent[]): Promise<void> {
+  if (events.length === 0) return;
+
+  const dbPath = getDatabasePath();
+  if (!existsSync(dbPath)) {
+    throw new Error(`Database not found: ${dbPath}. Run lore-db-init first.`);
+  }
+
+  const db = new Database(dbPath);
+
+  try {
+    // Load sqlite-vec extension for embeddings table
+    const vecPath = process.env.SQLITE_VEC_PATH;
+    if (!vecPath) {
+      throw new Error(
+        'SQLITE_VEC_PATH not set. Get path with: python3 -c "import sqlite_vec; print(sqlite_vec.loadable_path())"',
+      );
+    }
+    db.loadExtension(vecPath);
+
+    // 1. Insert into FTS5 and collect doc IDs
+    const docIds: number[] = [];
+    for (const event of events) {
+      const docId = insertSearchEntry(db, event);
+      docIds.push(docId);
+    }
+
+    // 2. Generate embeddings with cache
+    const contents = events.map((e) => getContentForEmbedding(e));
+    const embeddings = await embedWithCache(db, contents);
+
+    // 3. Insert embeddings
+    for (let i = 0; i < events.length; i++) {
+      insertEmbedding(db, docIds[i], embeddings[i], events[i]);
+    }
+  } finally {
+    db.close();
+  }
+}
+
+/**
+ * Insert event into FTS5 search table
+ * @returns rowid of inserted entry (used as doc_id for embeddings)
+ */
+function insertSearchEntry(db: Database, event: CaptureEvent): number {
+  const source = getSourceForEvent(event);
+  const title = buildTitle(event);
+  const content = getContentForEmbedding(event);
+  const metadata = buildMetadata(event);
+
+  const stmt = db.prepare(`
+    INSERT INTO search (source, title, content, metadata)
+    VALUES (?, ?, ?, ?)
+  `);
+
+  const result = stmt.run(source, title, content, metadata);
+  return Number(result.lastInsertRowid);
+}
+
+/**
+ * Map event type to source name used in search table
+ */
+function getSourceForEvent(event: CaptureEvent): string {
+  switch (event.type) {
+    case "knowledge":
+      return "captures";
+    case "teaching":
+      return "teachings";
+    case "observation":
+      return "observations";
+    case "insight":
+      return "insights";
+    case "learning":
+      return "learnings";
+    case "task":
+      return "tasks";
+    case "note":
+      return "captures";
+    default:
+      return "captures";
+  }
+}
+
+/**
+ * Build title for FTS5 entry, matching existing indexer format
+ */
+function buildTitle(event: CaptureEvent): string {
+  const data = event.data as Record<string, unknown>;
+
+  switch (event.type) {
+    case "knowledge":
+      return `[${data.subtype || "knowledge"}] ${data.topic || "general"}`;
+    case "teaching":
+      return `[${data.topic || "general"}] (${data.confidence || "medium"})`;
+    case "observation":
+      return `[${data.subtype || "pattern"}] ${data.topic || "general"}`;
+    case "insight":
+      return `[${data.subtype || "insight"}] ${data.topic || "general"}`;
+    case "learning":
+      return `[learning] ${data.topic || "general"}`;
+    case "task":
+      return `[task] ${data.topic || "general"}: ${data.name || "untitled"}`;
+    case "note":
+      return `[note] ${data.topic || "general"}`;
+  }
+}
+
+/**
+ * Extract content for embedding from event
+ */
+function getContentForEmbedding(event: CaptureEvent): string {
+  const data = event.data as Record<string, unknown>;
+  return String(data.content || data.text || "");
+}
+
+/**
+ * Build metadata JSON matching existing indexer format
+ */
+function buildMetadata(event: CaptureEvent): string {
+  const data = event.data as Record<string, unknown>;
+  const timestamp = event.timestamp;
+  const date = timestamp ? timestamp.substring(0, 10) : "";
+
+  const metadata: Record<string, unknown> = {
+    topic: data.topic || "general",
+    timestamp,
+    date,
+    content: getContentForEmbedding(event),
+  };
+
+  // Add type-specific fields
+  switch (event.type) {
+    case "knowledge":
+      metadata.subtype = data.subtype;
+      break;
+    case "teaching":
+      metadata.confidence = data.confidence;
+      metadata.capture_source = data.source || "manual";
+      break;
+    case "observation":
+      metadata.subtype = data.subtype;
+      metadata.confidence = data.confidence;
+      metadata.capture_source = data.source || "auto";
+      break;
+    case "insight":
+      metadata.subtype = data.subtype;
+      metadata.session_id = data.session_id;
+      break;
+    case "learning":
+      metadata.persona = data.persona;
+      break;
+    case "task":
+      metadata.name = data.name;
+      metadata.problem = data.problem;
+      metadata.solution = data.solution;
+      break;
+    case "note":
+      metadata.tags = data.tags;
+      break;
+  }
+
+  return JSON.stringify(metadata);
+}
+
+/**
+ * Embed contents with cache lookup
+ * Only generates embeddings for cache misses
+ */
+async function embedWithCache(
+  db: Database,
+  contents: string[],
+): Promise<number[][]> {
+  const results: (number[] | null)[] = new Array(contents.length).fill(null);
+  const toEmbed: { idx: number; content: string }[] = [];
+
+  // Check cache for each content
+  const hashes = contents.map((c) => hashContent(c));
+
+  for (let i = 0; i < contents.length; i++) {
+    const cached = getCachedEmbedding(db, hashes[i]);
+    if (cached) {
+      results[i] = cached;
+    } else {
+      toEmbed.push({ idx: i, content: contents[i] });
+    }
+  }
+
+  // Embed cache misses
+  if (toEmbed.length > 0) {
+    const embeddings = await embedDocuments(toEmbed.map((t) => t.content));
+
+    for (let i = 0; i < toEmbed.length; i++) {
+      const { idx, content } = toEmbed[i];
+      const embedding = embeddings[i];
+
+      results[idx] = embedding;
+      cacheEmbedding(db, hashContent(content), embedding, MODEL_NAME);
+    }
+  }
+
+  return results as number[][];
+}
+
+/**
+ * Insert embedding into vec0 table
+ */
+function insertEmbedding(
+  db: Database,
+  docId: number,
+  embedding: number[],
+  event: CaptureEvent,
+): void {
+  const source = getSourceForEvent(event);
+  const data = event.data as Record<string, unknown>;
+  const topic = String(data.topic || "");
+
+  const embeddingBlob = serializeEmbedding(embedding);
+
+  const stmt = db.prepare(`
+    INSERT INTO embeddings (doc_id, chunk_idx, source, topic, embedding)
+    VALUES (?, 0, ?, ?, ?)
+  `);
+
+  stmt.run(docId, source, topic, embeddingBlob);
+}

package/lib/search.ts

@@ -27,6 +27,20 @@ function getDatabasePath(): string {
   return `${homedir()}/.local/share/lore/lore.db`;
 }
 
+/**
+ * Escape a query for safe FTS5 MATCH
+ * Wraps terms in double quotes to prevent FTS5 syntax interpretation
+ * (e.g., "real-time" being parsed as column:term)
+ */
+function escapeFts5Query(query: string): string {
+  // Split on whitespace, wrap each term in quotes, rejoin
+  return query
+    .split(/\s+/)
+    .filter(Boolean)
+    .map((term) => `"${term.replace(/"/g, '""')}"`)
+    .join(" ");
+}
+
 /**
  * Search the Lore FTS5 database
  *
@@ -51,7 +65,7 @@ export function search(
     const limit = options.limit ?? 20;
 
     const conditions: string[] = ["search MATCH ?"];
-    const params: (string | number)[] = [query];
+    const params: (string | number)[] = [escapeFts5Query(query)];
 
     if (options.source) {
       conditions.push("source = ?");

package/lib/semantic.ts

@@ -4,22 +4,12 @@
  * Query embedding using @huggingface/transformers with nomic-embed-text-v1.5.
  * KNN search against sqlite-vec virtual table.
  * Uses Bun's built-in SQLite with sqlite-vec extension.
- *
- * Note: macOS ships Apple's SQLite which disables extension loading.
- * We use Homebrew's SQLite via setCustomSQLite() to enable sqlite-vec.
  */
 
 import { Database } from "bun:sqlite";
-import { homedir } from "os";
 import { existsSync } from "fs";
 import { pipeline } from "@huggingface/transformers";
-
-// Use Homebrew SQLite on macOS to enable extension loading
-// Must be called before any Database instances are created
-const HOMEBREW_SQLITE = "/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib";
-if (existsSync(HOMEBREW_SQLITE)) {
-  Database.setCustomSQLite(HOMEBREW_SQLITE);
-}
+import { getDatabasePath, openDatabase } from "./db.js";
 
 export interface SemanticResult {
   source: string;
@@ -43,7 +33,7 @@ const PROJECT_FIELD: Record<string, string> = {
   commits: "project",
   sessions: "project",
   tasks: "project",
-  insights: "project",
+  insights: "topic",
   captures: "topic",
   teachings: "topic",
   learnings: "topic",
@@ -51,6 +41,7 @@ const PROJECT_FIELD: Record<string, string> = {
 };
 
 const MODEL_NAME = "nomic-ai/nomic-embed-text-v1.5";
+const EMBEDDING_DIM = 768;
 
 interface EmbeddingPipeline {
   (
@@ -64,10 +55,6 @@ interface EmbeddingPipeline {
 // Cache the pipeline to avoid reloading on every query
 let cachedPipeline: EmbeddingPipeline | null = null;
 
-function getDatabasePath(): string {
-  return `${homedir()}/.local/share/lore/lore.db`;
-}
-
 /**
  * Get or create the embedding pipeline
  * Pipeline is cached after first load for performance
@@ -111,9 +98,9 @@ export async function embedQuery(query: string): Promise<number[]> {
   // Output is a Tensor, convert to array
   const embedding = Array.from(output.data as Float32Array);
 
-  if (embedding.length !== 768) {
+  if (embedding.length !== EMBEDDING_DIM) {
     throw new Error(
-      `Invalid embedding: expected 768 dims, got ${embedding.length}`,
+      `Invalid embedding: expected ${EMBEDDING_DIM} dims, got ${embedding.length}`,
     );
   }
 
@@ -121,33 +108,79 @@ export async function embedQuery(query: string): Promise<number[]> {
 }
 
 /**
- * Check if embeddings table has any data
+ * Embed a document string using local transformers.js model
+ * Uses "search_document: " prefix as required by nomic-embed-text
+ * @returns 768-dimensional embedding vector
  */
-export function hasEmbeddings(): boolean {
-  const dbPath = getDatabasePath();
+export async function embedDocument(text: string): Promise<number[]> {
+  const embedder = await getEmbeddingPipeline();
 
-  if (!existsSync(dbPath)) {
-    return false;
+  const prefixedText = `search_document: ${text}`;
+  const output = await embedder(prefixedText, {
+    pooling: "mean",
+    normalize: true,
+  });
+
+  const embedding = Array.from(output.data as Float32Array);
+
+  if (embedding.length !== EMBEDDING_DIM) {
+    throw new Error(
+      `Invalid embedding: expected ${EMBEDDING_DIM} dims, got ${embedding.length}`,
+    );
   }
 
-  const db = new Database(dbPath, { readonly: true });
+  return embedding;
+}
 
-  try {
-    // Load sqlite-vec extension
-    const vecPath = process.env.SQLITE_VEC_PATH;
-    if (!vecPath) {
-      return false;
+/**
+ * Batch embed multiple documents
+ * More efficient than individual calls when embedding several documents
+ * @returns array of 768-dimensional embedding vectors
+ */
+export async function embedDocuments(texts: string[]): Promise<number[][]> {
+  if (texts.length === 0) return [];
+
+  const embedder = await getEmbeddingPipeline();
+  const results: number[][] = [];
+
+  // Process one at a time (transformers.js doesn't batch well)
+  // But we benefit from cached pipeline
+  for (const text of texts) {
+    const prefixedText = `search_document: ${text}`;
+    const output = await embedder(prefixedText, {
+      pooling: "mean",
+      normalize: true,
+    });
+
+    const embedding = Array.from(output.data as Float32Array);
+
+    if (embedding.length !== EMBEDDING_DIM) {
+      throw new Error(
+        `Invalid embedding: expected ${EMBEDDING_DIM} dims, got ${embedding.length}`,
+      );
     }
 
-    db.loadExtension(vecPath);
+    results.push(embedding);
+  }
+
+  return results;
+}
 
-    const stmt = db.prepare("SELECT COUNT(*) as count FROM embeddings");
-    const result = stmt.get() as { count: number };
-    return result.count > 0;
+/**
+ * Check if embeddings table has any data
+ */
+export function hasEmbeddings(): boolean {
+  try {
+    const db = openDatabase(true);
+    try {
+      const stmt = db.prepare("SELECT COUNT(*) as count FROM embeddings");
+      const result = stmt.get() as { count: number };
+      return result.count > 0;
+    } finally {
+      db.close();
+    }
   } catch {
     return false;
-  } finally {
-    db.close();
   }
 }
 
@@ -166,97 +199,53 @@ export async function semanticSearch(
   query: string,
   options: SemanticSearchOptions = {},
 ): Promise<SemanticResult[]> {
-  const dbPath = getDatabasePath();
-
-  if (!existsSync(dbPath)) {
-    throw new Error(`Database not found: ${dbPath}. Run lore-db-init first.`);
-  }
-
   // Get query embedding
   const queryEmbedding = await embedQuery(query);
   const queryBlob = serializeEmbedding(queryEmbedding);
 
-  const db = new Database(dbPath, { readonly: true });
+  const db = openDatabase(true);
 
   try {
-    // Load sqlite-vec extension
-    const vecPath = process.env.SQLITE_VEC_PATH;
-    if (!vecPath) {
-      throw new Error(
-        'SQLITE_VEC_PATH not set. Get path with: python3 -c "import sqlite_vec; print(sqlite_vec.loadable_path())"',
-      );
-    }
-
-    db.loadExtension(vecPath);
-
     const limit = options.limit ?? 20;
 
     // KNN query - 1:1 mapping between search rows and embeddings
     // Content is pre-chunked at ingest time
+    // source/topic partition columns enable filtered KNN (filter BEFORE search)
     let sql: string;
     const params: (Uint8Array | string | number)[] = [queryBlob];
 
+    // Build KNN query with optional partition filters
+    const conditions = ["e.embedding MATCH ?", "k = ?"];
+    params.push(limit);
+
     if (options.source) {
-      // Filter by e.source (partition column) for KNN pre-filtering
-      // This filters BEFORE KNN, not after — critical for domain-specific search
-      sql = `
-        SELECT
-          s.source,
-          s.title,
-          s.content,
-          s.metadata,
-          e.distance
-        FROM embeddings e
-        JOIN search s ON e.doc_id = s.rowid
-        WHERE e.embedding MATCH ?
-          AND k = ?
-          AND e.source = ?
-        ORDER BY e.distance
-        LIMIT ?
-      `;
-      params.push(limit);
+      conditions.push("e.source = ?");
       params.push(options.source);
-      params.push(limit);
-    } else {
-      sql = `
-        SELECT
-          s.source,
-          s.title,
-          s.content,
-          s.metadata,
-          e.distance
-        FROM embeddings e
-        JOIN search s ON e.doc_id = s.rowid
-        WHERE e.embedding MATCH ?
-          AND k = ?
-        ORDER BY e.distance
-        LIMIT ?
-      `;
-      params.push(limit);
-      params.push(limit);
     }
 
-    const stmt = db.prepare(sql);
-    const results = stmt.all(...params) as SemanticResult[];
-
-    // Post-filter by project if specified
-    // KNN WHERE clause doesn't support json_extract on joined metadata,
-    // so we filter after the query returns
     if (options.project) {
-      return results.filter((result) => {
-        const field = PROJECT_FIELD[result.source];
-        if (!field) return false;
-
-        try {
-          const metadata = JSON.parse(result.metadata);
-          return metadata[field] === options.project;
-        } catch {
-          // Skip results with malformed metadata
-          return false;
-        }
-      });
+      conditions.push("e.topic = ?");
+      params.push(options.project);
     }
 
+    sql = `
+      SELECT
+        s.source,
+        s.title,
+        s.content,
+        s.metadata,
+        e.distance
+      FROM embeddings e
+      JOIN search s ON e.doc_id = s.rowid
+      WHERE ${conditions.join("\n        AND ")}
+      ORDER BY e.distance
+      LIMIT ?
+    `;
+    params.push(limit);
+
+    const stmt = db.prepare(sql);
+    const results = stmt.all(...params) as SemanticResult[];
+
     return results;
   } finally {
     db.close();
@@ -349,3 +338,6 @@ export function formatBriefSearch(results: SemanticResult[]): string {
 
   return sections.join("\n\n");
 }
+
+// Export constants and helpers for realtime.ts
+export { MODEL_NAME, EMBEDDING_DIM, serializeEmbedding, getDatabasePath };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voidwire/lore",
-  "version": "0.6.0",
+  "version": "0.6.2",
   "description": "Unified knowledge CLI - Search, list, and capture your indexed knowledge",
   "type": "module",
   "main": "./index.ts",